Package dbf :: Module tables
[hide private]
[frames] | no frames]

Source Code for Module dbf.tables

   1  "table definitions" 
   2  import os 
   3  import sys 
   4  import csv 
   5  from array import array 
   6  from decimal import Decimal 
   7  from dbf import _io as io 
   8  from dbf.dates import Date, DateTime, Time 
   9  from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing 
  10   
  11   
  12  version_map = { 
  13          '\x02' : 'FoxBASE', 
  14          '\x03' : 'dBase III Plus', 
  15          '\x04' : 'dBase IV', 
  16          '\x05' : 'dBase V', 
  17          '\x30' : 'Visual FoxPro', 
  18          '\x31' : 'Visual FoxPro (auto increment field)', 
  19          '\x43' : 'dBase IV SQL', 
  20          '\x7b' : 'dBase IV w/memos', 
  21          '\x83' : 'dBase III Plus w/memos', 
  22          '\x8b' : 'dBase IV w/memos', 
  23          '\x8e' : 'dBase IV w/SQL table' } 
  24   
  25  code_pages = { 
  26          '\x01' : ('cp437', 'U.S. MS-DOS'), 
  27          '\x02' : ('cp850', 'International MS-DOS'), 
  28          '\x03' : ('cp1252', 'Windows ANSI'), 
  29          '\x04' : ('mac_roman', 'Standard Macintosh'), 
  30          '\x64' : ('cp852', 'Eastern European MS-DOS'), 
  31          '\x65' : ('cp866', 'Russian MS-DOS'), 
  32          '\x66' : ('cp865', 'Nordic MS-DOS'), 
  33          '\x67' : ('cp861', 'Icelandic MS-DOS'), 
  34   
  35          '\x68' : (None, 'Kamenicky (Czech) MS-DOS'), 
  36          '\x69' : (None, 'Mazovia (Polish) MS-DOS'), 
  37   
  38          '\x6a' : ('cp737', 'Greek MS-DOS (437G)'), 
  39          '\x6b' : ('cp857', 'Turkish MS-DOS'), 
  40          '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'), 
  41          '\x79' : ('cp949', 'Korean Windows'), 
  42          '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'), 
  43          '\x7b' : ('cp932', 'Japanese Windows'), 
  44          '\x7c' : ('cp874', 'Thai Windows'), 
  45          '\x7d' : ('cp1255', 'Hebrew Windows'), 
  46          '\x7e' : ('cp1256', 'Arabic Windows'), 
  47          '\xc8' : ('cp1250', 'Eastern European Windows'), 
  48          '\xc9' : ('cp1251', 'Russian Windows'), 
  49          '\xca' : ('cp1254', 'Turkish Windows'), 
  50          '\xcb' : ('cp1253', 'Greek Windows'), 
  51          '\x96' : ('mac_cyrillic', 'Russian Macintosh'), 
  52          '\x97' : ('mac_latin2', 'Macintosh EE'), 
  53          '\x98' : ('mac_greek', 'Greek Macintosh') } 
  54   
55 -class _MetaData(dict):
56 blankrecord = None 57 fields = None 58 filename = None 59 dfd = None 60 memoname = None 61 newmemofile = False 62 memo = None 63 mfd = None 64 ignorememos = False 65 memofields = None 66 index = [] # never mutated 67 index_reversed = False 68 orderresults = None 69 current = -1
70 -class _TableHeader(object):
71 - def __init__(yo, data):
72 if len(data) != 32: 73 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data)) 74 yo._data = array('c', data + '\x0d')
75 - def codepage(yo, cp=None):
76 "get/set code page of table" 77 if cp is None: 78 return yo._data[29] 79 else: 80 yo._data[29] =cp
81 - def data(yo, bytes=None):
82 "get/set entire structure" 83 if bytes is None: 84 date = io.packDate(Date.today()) 85 yo._data[1:4] = array('c', date) 86 return yo._data.tostring() 87 else: 88 if len(bytes) < 32: 89 raise DbfError("length for data of %d is less than 32" % len(bytes)) 90 yo._data[:] = array('c', bytes)
91 - def extra(yo, data=None):
92 "get/set any extra dbf info (located after headers, before data records)" 93 fieldblock = yo._data[32:] 94 for i in range(len(fieldblock)//32+1): 95 cr = i * 32 96 if fieldblock[cr] == '\x0d': 97 break 98 else: 99 raise DbfError("corrupt field structure") 100 cr += 33 # skip past CR 101 if data is None: 102 return yo._data[cr:].tostring() 103 else: 104 yo._data[cr:] = array('c', data) # extra 105 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start
106 - def fieldcount(yo):
107 "number of fields (read-only)" 108 fieldblock = yo._data[32:] 109 for i in range(len(fieldblock)//32+1): 110 cr = i * 32 111 if fieldblock[cr] == '\x0d': 112 break 113 else: 114 raise DbfError("corrupt field structure") 115 return len(fieldblock[:cr]) // 32
116 - def fields(yo, block=None):
117 "get/set field block structure" 118 fieldblock = yo._data[32:] 119 for i in range(len(fieldblock)//32+1): 120 cr = i * 32 121 if fieldblock[cr] == '\x0d': 122 break 123 else: 124 raise DbfError("corrupt field structure") 125 if block is None: 126 return fieldblock[:cr].tostring() 127 else: 128 cr += 32 # convert to indexing main structure 129 fieldlen = len(block) 130 if fieldlen % 32 != 0: 131 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 132 yo._data[32:cr] = array('c', block) # fields 133 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start 134 fieldlen = fieldlen // 32 135 recordlen = 1 # deleted flag 136 for i in range(fieldlen): 137 recordlen += ord(block[i*32+16]) 138 yo._data[10:12] = array('c', io.packShortInt(recordlen))
139 - def recordcount(yo, count=None):
140 "get/set number of records (maximum 16,777,215)" 141 if count is None: 142 return io.unpackLongInt(yo._data[4:8].tostring()) 143 else: 144 yo._data[4:8] = array('c', io.packLongInt(count))
145 - def recordlength(yo):
146 "length of a record (read_only) (max of 65,535)" 147 return io.unpackShortInt(yo._data[10:12].tostring())
148 - def start(yo, pos=None):
149 "starting position of first record in file (must be within first 64K)" 150 if pos is None: 151 return io.unpackShortInt(yo._data[8:10].tostring()) 152 else: 153 yo._data[8:10] = array('c', io.packShortInt(pos))
154 - def update(yo):
155 "date of last table modification (read-only)" 156 return io.unpackDate(yo._data[1:4].tostring())
157 - def version(yo, ver=None):
158 "dbf version" 159 if ver is None: 160 return yo._data[0] 161 else: 162 yo._data[0] = ver
163 -class _DbfRecord(object):
164 """Provides routines to extract and save data within the fields of a dbf record.""" 165 __slots__ = ['_recnum', '_layout', '_data']
166 - def _retrieveFieldValue(yo, record_data, fielddef):
167 "calls appropriate routine to fetch value stored in field from array" 168 return yo._layout.fieldtypes[fielddef['type']]['Retrieve'](record_data, fielddef, yo._layout.memo)
169 - def _updateFieldValue(yo, fielddef, value):
170 "calls appropriate routine to convert value to ascii bytes, and save it in record" 171 type = fielddef['type'] 172 fieldtype = yo._layout.fieldtypes[type] 173 callable = fieldtype['Update'] 174 update = list(yo._layout.fieldtypes[fielddef['type']]['Update'](value, fielddef, yo._layout.memo)) 175 size = fielddef['length'] 176 if len(update) > size: 177 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(update), size)) 178 blank = array('c', ' ' * size) 179 start = fielddef['start'] 180 end = start + size 181 new_data = array('c', update) 182 blank[:len(update)] = new_data[:] 183 yo._data[start:end] = blank[:] 184 yo._updateDisk(yo._recnum * yo._layout.header.recordlength() + yo._layout.header.start(), yo._data.tostring())
185 - def _updateDisk(yo, location='', data=None):
186 if not yo._layout.inmemory: 187 if yo._recnum < 0: 188 raise DbfError("Attempted to update record that has been packed") 189 if location == '': 190 location = yo._recnum * yo._layout.header.recordlength() + yo._layout.header.start() 191 if data is None: 192 data = yo._data 193 yo._layout.dfd.seek(location) 194 yo._layout.dfd.write(data)
195 - def __call__(yo, *specs):
196 results = [] 197 if not specs: 198 specs = yo._layout.index 199 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop]) 200 for field, func in specs: 201 results.append(func(yo[field])) 202 return tuple(results)
203
204 - def __contains__(yo, key):
205 return key in yo._layout.fields
206 - def __iter__(yo):
207 return (yo[field] for field in yo._layout.fields)
208 - def __getattr__(yo, name):
209 if name[0:2] == '__' and name[-2:] == '__': 210 raise AttributeError, 'Method %s is not implemented.' % name 211 elif not name in yo._layout.fields: 212 raise FieldMissing(name) 213 try: 214 fielddef = yo._layout[name] 215 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef) 216 return value 217 except DbfError, error: 218 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 219 raise
220 - def __getitem__(yo, item):
221 if type(item) == int: 222 if not -yo._layout.header.fieldcount() <= item < yo._layout.header.fieldcount(): 223 raise IndexError("Field offset %d is not in record" % item) 224 return yo[yo._layout.fields[item]] 225 elif type(item) == slice: 226 sequence = [] 227 for index in yo._layout.fields[item]: 228 sequence.append(yo[index]) 229 return sequence 230 elif type(item) == str: 231 return yo.__getattr__(item) 232 else: 233 raise TypeError("%s is not a field name" % item)
234 - def __len__(yo):
235 return yo._layout.header.fieldcount()
236 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
237 """record = ascii array of entire record; layout=record specification; memo = memo object for table""" 238 record = object.__new__(cls) 239 record._recnum = recnum 240 record._layout = layout 241 if layout.blankrecord is None and not _fromdisk: 242 record._createBlankRecord() 243 record._data = layout.blankrecord 244 if recnum == -1: # not a disk-backed record 245 return record 246 elif type(kamikaze) == array: 247 record._data = kamikaze[:] 248 elif type(kamikaze) == str: 249 record._data = array('c', kamikaze) 250 else: 251 record._data = kamikaze._data[:] 252 datalen = len(record._data) 253 if datalen < layout.header.recordlength(): 254 record._data.extend(layout.blankrecord[datalen:]) 255 elif datalen > layout.header.recordlength(): 256 record._data = record._data[:layout.header.recordlength()] 257 if not _fromdisk and not layout.inmemory: 258 record._updateDisk() 259 return record
260 - def __setattr__(yo, name, value):
261 if name in yo.__slots__: 262 object.__setattr__(yo, name, value) 263 return 264 elif not name in yo._layout.fields: 265 raise FieldMissing(name) 266 fielddef = yo._layout[name] 267 try: 268 yo._updateFieldValue(fielddef, value) 269 except DbfError, error: 270 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 271 error.data = name 272 raise 273 raise DbfError(message)
274 - def __setitem__(yo, name, value):
275 if type(name) == str: 276 yo.__setattr__(name, value) 277 elif type(name) in (int, long): 278 yo.__setattr__(yo._layout.fields[name], value) 279 else: 280 raise TypeError("%s is not a field name" % name)
281 - def __str__(yo):
282 result = [] 283 for field in yo.field_names(): 284 result.append("%-10s: %s" % (field, yo[field])) 285 return '\n'.join(result)
286 - def __repr__(yo):
287 return yo._data.tostring()
288 - def _createBlankRecord(yo):
289 "creates a blank record data chunk" 290 layout = yo._layout 291 ondisk = layout.ondisk 292 layout.ondisk = False 293 yo._data = array('c', ' ' * layout.header.recordlength()) 294 layout.memofields = [] 295 for field in layout.fields: 296 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']()) 297 if layout[field]['type'] in layout.memotypes: 298 layout.memofields.append(field) 299 layout.blankrecord = yo._data[:] 300 layout.ondisk = ondisk
301 - def record_number(yo):
302 "physical record number" 303 return yo._recnum
304 - def has_been_deleted(yo):
305 "marked for deletion?" 306 return yo._data[0] == '*'
307 - def field_names(yo):
308 "fields in table/record" 309 return yo._layout.fields[:]
310 - def delete_record(yo):
311 "marks record as deleted" 312 yo._data[0] = '*' 313 yo._updateDisk(data='*')
314 - def gather_fields(yo, dict, drop=False):
315 "saves a dictionary into a records fields\nkeys with no matching field will raise a FieldMissing exception unless drop = True" 316 for key in dict: 317 if not key in yo.field_names(): 318 if drop: 319 continue 320 raise FieldMissing(key) 321 yo.__setattr__(key, dict[key])
322 - def reset_record(yo, keep_fields=None):
323 "blanks record" 324 if keep_fields is None: 325 keep_fields = [] 326 keep = {} 327 for field in keep_fields: 328 keep[field] = yo[field] 329 if yo._layout.blankrecord == None: 330 yo._createBlankRecord() 331 yo._data[:] = yo._layout.blankrecord[:] 332 for field in keep_fields: 333 yo[field] = keep[field]
334 - def scatter_fields(yo, blank=False):
335 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty." 336 keys = yo._layout.fields 337 if blank: 338 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys] 339 else: 340 values = [yo[field] for field in keys] 341 return dict(zip(keys, values))
342 - def undelete_record(yo):
343 "marks record as active" 344 yo._data[0] = ' ' 345 yo._updateDisk(data=' ')
346 -class _DbfMemo(object):
347 """Provides access to memo fields as dictionaries 348 must override _init, _get_memo, and _put_memo to 349 store memo contents to disk"""
350 - def _init(yo):
351 "initialize disk file usage"
352 - def _get_memo(yo, block):
353 "retrieve memo contents from disk"
354 - def _put_memo(yo, data):
355 "store memo contents to disk"
356 - def __init__(yo, meta):
357 "" 358 yo.meta = meta 359 yo.memory = {} 360 yo.nextmemo = 1 361 yo._init() 362 yo.meta.newmemofile = False
363 - def get_memo(yo, block, field):
364 "gets the memo in block" 365 if yo.meta.ignorememos or not block: 366 return '' 367 if yo.meta.ondisk: 368 return yo._get_memo(block) 369 else: 370 return yo.memory[block]
371 - def put_memo(yo, data):
372 "stores data in memo file, returns block number" 373 if yo.meta.ignorememos or data == '': 374 return 0 375 if yo.meta.inmemory: 376 thismemo = yo.nextmemo 377 yo.nextmemo += 1 378 yo.memory[thismemo] = data 379 else: 380 thismemo = yo._put_memo(data) 381 return thismemo
382 -class _Db3Memo(_DbfMemo):
383 - def _init(yo):
384 "dBase III specific" 385 yo.meta.memo_size= 512 386 yo.record_header_length = 2 387 if yo.meta.ondisk and not yo.meta.ignorememos: 388 if yo.meta.newmemofile: 389 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 390 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508) 391 else: 392 try: 393 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 394 yo.meta.mfd.seek(0) 395 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4)) 396 except: 397 raise DbfError("memo file appears to be corrupt")
398 - def _get_memo(yo, block):
399 block = int(block) 400 yo.meta.mfd.seek(block * yo.meta.memo_size) 401 eom = -1 402 data = '' 403 while eom == -1: 404 newdata = yo.meta.mfd.read(yo.meta.memo_size) 405 if not newdata: 406 return data 407 data += newdata 408 eom = data.find('\x1a\x1a') 409 return data[:eom].rstrip()
410 - def _put_memo(yo, data):
411 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 412 blocks = length // yo.meta.memo_size 413 if length % yo.meta.memo_size: 414 blocks += 1 415 thismemo = yo.nextmemo 416 yo.nextmemo = thismemo + blocks 417 yo.meta.mfd.seek(0) 418 yo.meta.mfd.write(io.packLongInt(yo.nextmemo)) 419 yo.meta.mfd.seek(thismemo * yo.meta.memo_size) 420 yo.meta.mfd.write(data) 421 yo.meta.mfd.write('\x1a\x1a') 422 if len(yo._get_memo(thismemo)) != len(data): 423 raise DbfError("unknown error: memo not saved") 424 return thismemo
425 -class _VfpMemo(_DbfMemo):
426 - def _init(yo):
427 "Visual Foxpro 6 specific" 428 if yo.meta.ondisk and not yo.meta.ignorememos: 429 yo.record_header_length = 8 430 if yo.meta.newmemofile: 431 if yo.meta.memo_size == 0: 432 yo.meta.memo_size = 1 433 elif 1 < yo.meta.memo_size < 33: 434 yo.meta.memo_size *= 512 435 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 436 nextmemo = 512 // yo.meta.memo_size 437 if nextmemo * yo.meta.memo_size < 512: 438 nextmemo += 1 439 yo.nextmemo = nextmemo 440 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \ 441 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504) 442 else: 443 try: 444 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 445 yo.meta.mfd.seek(0) 446 header = yo.meta.mfd.read(512) 447 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True) 448 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True) 449 except: 450 raise DbfError("memo file appears to be corrupt")
451 - def _get_memo(yo, block):
452 yo.meta.mfd.seek(block * yo.meta.memo_size) 453 header = yo.meta.mfd.read(8) 454 length = io.unpackLongInt(header[4:], bigendian=True) 455 return yo.meta.mfd.read(length)
456 - def _put_memo(yo, data):
457 yo.meta.mfd.seek(0) 458 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True) 459 yo.meta.mfd.seek(0) 460 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 461 blocks = length // yo.meta.memo_size 462 if length % yo.meta.memo_size: 463 blocks += 1 464 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True)) 465 yo.meta.mfd.seek(thismemo*yo.meta.memo_size) 466 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data) 467 return thismemo
468 -class DbfTable(object):
469 """Provides a framework for dbf style tables.""" 470 _version = 'basic memory table' 471 _versionabbv = 'dbf' 472 _fieldtypes = { 473 'D' : { 'Type':'Date', 474 'Init':io.addDate, 475 'Blank':Date.today, 476 'Retrieve':io.retrieveDate, 477 'Update':io.updateDate, 478 }, 479 'L' : { 480 'Type':'Logical', 481 'Init':io.addLogical, 482 'Blank':bool, 483 'Retrieve':io.retrieveLogical, 484 'Update':io.updateLogical, 485 }, 486 'M' : { 487 'Type':'Memo', 488 'Init':io.addMemo, 489 'Blank':str, 490 'Retrieve':io.retrieveMemo, 491 'Update':io.updateMemo, 492 } } 493 _memoext = '' 494 _memotypes = tuple() 495 _memoClass = _DbfMemo 496 _yesMemoMask = '' 497 _noMemoMask = '' 498 _fixedFields = tuple() 499 _decimalFields = tuple() 500 _variableFields = tuple() 501 _numericFields = tuple() 502 _dbfTableHeader = array('c', '\x00' * 32) 503 _dbfTableHeader[0] = '\x00' # table type - none 504 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 505 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 506 _dbfTableHeader[29] = '\x00' # code page -- none, using plain ascii 507 _dbfTableHeader = _dbfTableHeader.tostring() 508 _dbfTableHeaderExtra = '' 509 _supported_tables = [] 510 _read_only = False 511 _meta_only = False 512 _use_deleted = True
513 - class DbfIterator(object):
514 "returns records using current index"
515 - def __init__(yo, table):
516 yo._table = table 517 yo._index = -1 518 yo._more_records = True
519 - def __iter__(yo):
520 return yo
521 - def next(yo):
522 while yo._more_records: 523 yo._index += 1 524 if yo._index >= len(yo._table): 525 yo._more_records = False 526 continue 527 record = yo._table[yo._index] 528 if not yo._table.use_deleted() and record.has_been_deleted(): 529 continue 530 return record 531 else: 532 raise StopIteration
533 - def _buildHeaderFields(yo):
534 "constructs fieldblock for disk table" 535 fieldblock = array('c', '') 536 memo = False 537 yo._meta.header.version(chr(ord(yo._meta.header.version()) & ord(yo._noMemoMask))) 538 for field in yo._meta.fields: 539 if yo._meta.fields.count(field) > 1: 540 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)") 541 fielddef = array('c', '\x00' * 32) 542 fielddef[:11] = array('c', io.packStr(field)) 543 fielddef[11] = yo._meta[field]['type'] 544 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start'])) 545 fielddef[16] = chr(yo._meta[field]['length']) 546 fielddef[17] = chr(yo._meta[field]['decimals']) 547 fielddef[18] = chr(yo._meta[field]['flags']) 548 fieldblock.extend(fielddef) 549 if yo._meta[field]['type'] in yo._meta.memotypes: 550 memo = True 551 yo._meta.header.fields(fieldblock.tostring()) 552 if memo: 553 yo._meta.header.version(chr(ord(yo._meta.header.version()) | ord(yo._yesMemoMask))) 554 if yo._meta.memo is None: 555 yo._meta.memo = yo._memoClass(yo._meta)
556 - def _checkMemoIntegrity(yo):
557 "dBase III specific" 558 if yo._meta.header.version() == '\x83': 559 try: 560 yo._meta.memo = yo._memoClass(yo._meta) 561 except: 562 yo._meta.dfd.close() 563 yo._meta.dfd = None 564 raise 565 if not yo._meta.ignorememos: 566 for field in yo._meta.fields: 567 if yo._meta[field]['type'] in yo._memotypes: 568 if yo._meta.header.version() != '\x83': 569 yo._meta.dfd.close() 570 yo._meta.dfd = None 571 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 572 elif not os.path.exists(yo._meta.memoname): 573 yo._meta.dfd.close() 574 yo._meta.dfd = None 575 raise DbfError("Table structure corrupt: memo fields exist without memo file") 576 break
577 - def _initializeFields(yo):
578 "builds the FieldList of names, types, and descriptions from the disk file" 579 offset = 1 580 fieldsdef = yo._meta.header.fields() 581 if len(fieldsdef) % 32 != 0: 582 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 583 if len(fieldsdef) // 32 != yo.field_count(): 584 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count(), len(fieldsdef)//32)) 585 for i in range(yo.field_count()): 586 fieldblock = fieldsdef[i*32:(i+1)*32] 587 name = io.unpackStr(fieldblock[:11]) 588 type = fieldblock[11] 589 if not type in yo._meta.fieldtypes: 590 raise DbfError("Unknown field type: %s" % type) 591 start = offset 592 length = ord(fieldblock[16]) 593 offset += length 594 end = start + length 595 decimals = ord(fieldblock[17]) 596 flags = ord(fieldblock[18]) 597 yo._meta.fields.append(name) 598 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
599 - def _fieldLayout(yo, i):
600 "Returns field information Name Type(Length[,Decimals])" 601 name = yo._meta.fields[i] 602 type = yo._meta[name]['type'] 603 length = yo._meta[name]['length'] 604 decimals = yo._meta[name]['decimals'] 605 if type in yo._decimalFields: 606 description = "%s %s(%d,%d)" % (name, type, length, decimals) 607 elif type in yo._fixedFields: 608 description = "%s %s" % (name, type) 609 else: 610 description = "%s %s(%d)" % (name, type, length) 611 return description
612 - def _loadtable(yo):
613 "loads the records from disk to memory" 614 if yo._meta_only: 615 raise DbfError("%s has been closed, records are unavailable" % yo.filename()) 616 dfd = yo._meta.dfd 617 header = yo._meta.header 618 dfd.seek(header.start()) 619 allrecords = dfd.read() # kludge to get around mysterious errno 0 problems 620 dfd.seek(0) 621 length = header.recordlength() 622 for i in range(header.recordcount()): 623 try: 624 record_data = allrecords[length*i:length*i+length] 625 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True)) 626 except: 627 print 628 print yo.filename(), yo.field_names() 629 print "record length: %d" % length 630 print "data length: %d" % len(record_data) 631 print "data <%s>" % record_data 632 print 633 raise 634 yo._index.append(i) 635 dfd.seek(0)
636 - def _updateDisk(yo, headeronly=False):
637 "synchronizes the disk file with current data" 638 if yo._meta.inmemory: 639 return 640 fd = yo._meta.dfd 641 fd.seek(0) 642 fd.write(yo._meta.header.data()) 643 if not headeronly: 644 for record in yo._table: 645 record._updateDisk() 646 fd.flush() 647 fd.truncate(yo._meta.header.start() + yo._meta.header.recordcount() * yo._meta.header.recordlength())
648 - def __contains__(yo, key):
649 return key in yo.field_names()
650 - def __getattr__(yo, name):
651 if name in ('_index','_table'): 652 yo._index = [] 653 yo._table = [] 654 yo._loadtable() 655 return object.__getattribute__(yo, name)
656 - def __getitem__(yo, value):
657 if type(value) == int: 658 if not -yo._meta.header.recordcount() <= value < yo._meta.header.recordcount(): 659 raise IndexError("Record %d is not in table." % value) 660 return yo._table[yo._index[value]] 661 elif type(value) == slice: 662 sequence = [] 663 for index in yo._index[value]: 664 record = yo._table[index] 665 if yo.use_deleted() is True or not record.has_been_deleted(): 666 sequence.append(record) 667 return DbfList(sequence, desc='%s --> %s' % (yo.filename(), value)) 668 else: 669 raise TypeError('type <%s> not valid for indexing' % type(value))
670 - def __init__(yo, filename=':memory:', field_spec=None, memo_size=128, ignore_memos=False, 671 read_only=False, keep_memos=False, meta_only=False, codepage='cp1252'):
672 """open/create dbf file 673 filename should include path if needed 674 field_spec can be either a ;-delimited string or a list of strings 675 memo_size is always 512 for db3 memos 676 ignore_memos is useful if the memo file is missing or corrupt 677 read_only will load records into memory, then close the disk file 678 keep_memos will also load any memo fields into memory 679 meta_only will ignore all records, keeping only basic table information 680 codepage will override whatever is set in the table itself""" 681 if filename == ':memory:': 682 if field_spec is None: 683 raise DbfError("field list must be specified for in-memory tables") 684 elif type(yo) is DbfTable: 685 raise DbfError("only memory tables supported") 686 yo._meta = _MetaData() 687 yo._meta.filename = filename 688 yo._meta.fields = [] 689 yo._meta.fieldtypes = yo._fieldtypes 690 yo._meta.memotypes = yo._memotypes 691 yo._meta.ignorememos = ignore_memos 692 yo._meta.memo_size = memo_size 693 header = _TableHeader(yo._dbfTableHeader) 694 header.extra(yo._dbfTableHeaderExtra) 695 header.data() #force update of date 696 yo._meta.header = header 697 if filename == ':memory:': 698 yo._index = [] 699 yo._table = [] 700 yo._meta.ondisk = False 701 yo._meta.inmemory = True 702 yo._meta.memoname = ':memory:' 703 else: 704 base, ext = os.path.splitext(filename) 705 if ext == '': 706 yo._meta.filename = base + '.dbf' 707 yo._meta.memoname = base + yo._memoext 708 yo._meta.ondisk = True 709 yo._meta.inmemory = False 710 if field_spec: 711 if yo._meta.ondisk: 712 yo._meta.dfd = open(yo._meta.filename, 'w+b') 713 yo._meta.newmemofile = True 714 yo.add_fields(field_spec) 715 return 716 dfd = yo._meta.dfd = open(yo._meta.filename, 'r+b') 717 dfd.seek(0) 718 yo._meta.header = header = _TableHeader(dfd.read(32)) 719 if not header.version() in yo._supported_tables: 720 dfd.close() 721 dfd = None 722 raise TypeError("Unsupported dbf type: %s [%x]" % (version_map.get(yo._meta.header.version, 'Unknown: %s' % yo._meta.header.version), ord(yo._meta.header.version))) 723 fieldblock = dfd.read(header.start() - 32) 724 for i in range(len(fieldblock)//32+1): 725 fieldend = i * 32 726 if fieldblock[fieldend] == '\x0d': 727 break 728 else: 729 raise DbfError("corrupt field structure in header") 730 if len(fieldblock[:fieldend]) % 32 != 0: 731 raise DbfError("corrupt field structure in header") 732 header.fields(fieldblock[:fieldend]) 733 header.extra(fieldblock[fieldend+1:]) # skip trailing \r 734 yo._initializeFields() 735 yo._checkMemoIntegrity() 736 yo._meta.current = -1 737 dfd.seek(0) 738 if meta_only: 739 yo.close(keep_table=False, keep_memos=False) 740 elif read_only: 741 yo.close(keep_table=True, keep_memos=keep_memos)
742 - def __iter__(yo):
743 return yo.DbfIterator(yo)
744 - def __len__(yo):
745 return yo._meta.header.recordcount()
746 - def __repr__(yo):
747 if yo._read_only: 748 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename 749 elif yo._meta_only: 750 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename 751 else: 752 return __name__ + ".Table('%s')" % yo._meta.filename
753 - def __str__(yo):
754 if yo._read_only: 755 status = "read-only" 756 elif yo._meta_only: 757 status = "meta-only" 758 else: 759 status = "read/write" 760 str = """ 761 Table: %s 762 Type: %s 763 Status: %s 764 Last updated: %s 765 Record count: %d 766 Field count: %d 767 Record length: %d 768 """ % (yo.filename, version_map.get(yo._meta.header.version(), 'unknown - ' + hex(ord(yo._meta.header.version()))), 769 status, yo.last_update(), len(yo), yo.field_count(), yo.record_length()) 770 str += "\n --Fields--\n" 771 for i in range(len(yo._meta.fields)): 772 str += " " + yo._fieldLayout(i) + "\n" 773 return str
774 - def field_count(yo):
775 "the number of fields in the table" 776 return yo._meta.header.fieldcount()
777 - def field_names(yo):
778 "a list of the fields in the table" 779 return yo._meta.fields[:]
780 - def filename(yo):
781 "table's file name, including path (if specified on open)" 782 return yo._meta.filename
783 - def last_update(yo):
784 "date of last update" 785 return yo._meta.header.update()
786 - def memoname(yo):
787 "table's memo name (if path included in filename on open)" 788 return yo._meta.memoname
789 - def record_length(yo):
790 "number of bytes in a record" 791 return yo._meta.header.recordlength()
792 - def record_number(yo):
793 "index number of the current record" 794 return yo._meta.current
795 - def supported_tables(yo):
796 "allowable table types" 797 return yo._supported_tables
798 - def use_deleted(yo, new_setting=None):
799 if new_setting is None: 800 return yo._use_deleted 801 else: 802 yo._use_deleted = new_setting
803 - def version(yo):
804 "returns the dbf type of the table" 805 return yo._version
806 - def add_fields(yo, field_spec):
807 "adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]" 808 yo._meta.blankrecord = None 809 meta = yo._meta 810 offset = meta.header.recordlength() 811 if isinstance(field_spec, str): 812 fields = field_spec.replace('; ',';').split(';') 813 else: 814 fields = list(field_spec) 815 for field in fields: 816 try: 817 name, format = field.split() 818 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum(): 819 raise DbfError("Field names cannot start with _ or digits, and can only contain the _, letters, and digits") 820 name = name.lower() 821 if name in meta.fields: 822 raise DbfError("Field '%s' already exists" % name) 823 field_type = format[0].upper() 824 if len(name) > 10: 825 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name))) 826 if not field_type in meta.fieldtypes.keys(): 827 raise DbfError("Unknown field type: %s" % field_type) 828 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format) 829 except ValueError: 830 raise DbfError("invalid field specifier: %s" % field) 831 start = offset 832 end = offset + length 833 offset = end 834 meta.fields.append(name) 835 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0} 836 if meta[name]['type'] in yo._memotypes and meta.memo is None: 837 meta.memo = yo._memoClass(meta) 838 for record in yo: 839 record[name] = meta.fieldtypes[field_type]['Blank']() 840 yo._buildHeaderFields() 841 yo._updateDisk()
842 - def append(yo, kamikaze='', drop=False, multiple=1):
843 "adds <multiple> blank records, and fills fields with dict/tuple values if present" 844 if not yo.field_count(): 845 raise DbfError("No fields defined, cannot append") 846 dictdata = False 847 tupledata = False 848 if not isinstance(kamikaze, _DbfRecord): 849 if isinstance(kamikaze, dict): 850 dictdata = kamikaze 851 kamikaze = '' 852 elif isinstance(kamikaze, tuple): 853 tupledata = kamikaze 854 kamikaze = '' 855 yo._table.append(_DbfRecord(recnum=yo._meta.header.recordcount(), layout=yo._meta, kamikaze=kamikaze)) 856 yo._index.append(yo._meta.header.recordcount()) 857 yo._meta.header.recordcount(yo._meta.header.recordcount() + 1) 858 #yo._meta.current = yo._meta.header.recordcount - 1 859 newrecord = yo._table[-1] 860 if dictdata: 861 newrecord.gather_fields(dictdata, drop) 862 elif tupledata: 863 for index, item in enumerate(tupledata): 864 newrecord[index] = item 865 elif kamikaze == str: 866 for field in yo._meta.memofields: 867 newrecord[field] = '' 868 elif kamikaze: 869 for field in yo._meta.memofields: 870 newrecord[field] = kamikaze[field] 871 multiple -= 1 872 if multiple: 873 data = newrecord._data 874 single = yo._meta.header.recordcount() 875 total = single + multiple 876 while single < total: 877 yo._table.append(_DbfRecord(single, yo._meta, kamikaze=data)) 878 yo._index.append(single) 879 for field in yo._meta.memofields: 880 lastrecord = yo._table[-1] 881 lastrecord[field] = newrecord[field] 882 single += 1 883 yo._meta.header.recordcount(total) # += multiple 884 yo._meta.current = yo._meta.header.recordcount() - 1 885 yo._updateDisk(headeronly=True) 886 return newrecord
887 - def bof(yo):
888 "moves record pointer to previous usable record; returns True if no more usable records" 889 while yo._meta.current > 0: 890 yo._meta.current -= 1 891 if yo.use_deleted() or not yo.current().has_been_deleted(): 892 break 893 else: 894 yo._meta.current = -1 895 return True 896 return False
897 - def bottom(yo, get_record=False):
898 """sets record pointer to bottom of table 899 if get_record, seeks to and returns last (non-deleted) record 900 DbfError if table is empty 901 Bof if all records deleted and use_deleted() is False""" 902 yo._meta.current = yo._meta.header.recordcount() 903 if get_record: 904 try: 905 return yo.prev() 906 except Bof: 907 yo._meta.current = yo._meta.header.recordcount() 908 raise Eof()
909 - def close(yo, keep_table=False, keep_memos=False):
910 """closes disk files 911 ensures table data is available if keep_table 912 ensures memo data is available if keep_memos""" 913 if keep_table: 914 yo._table # force read of table if not already in memory 915 else: 916 if '_index' in dir(yo): 917 del yo._table 918 del yo._index 919 yo._meta.inmemory = True 920 if yo._meta.ondisk: 921 yo._meta.dfd.close() 922 yo._meta.dfd = None 923 if '_index' in dir(yo): 924 yo._read_only = True 925 else: 926 yo._meta_only = True 927 if yo._meta.mfd is not None: 928 if not keep_memos: 929 yo._meta.ignorememos = True 930 else: 931 memo_fields = [] 932 for field in yo.field_names(): 933 if yo.is_memotype(field): 934 memo_fields.append(field) 935 for record in yo: 936 for field in memo_fields: 937 record[field] = record[field] 938 yo._meta.mfd.close() 939 yo._meta.mfd = None 940 yo._meta.ondisk = False
941 - def current(yo, index=False):
942 "returns current logical record, or its index" 943 if yo._meta.current < 0: 944 raise Bof() 945 elif yo._meta.current >= yo._meta.header.recordcount(): 946 raise Eof() 947 if index: 948 return yo._meta.current 949 return yo._table[yo._index[yo._meta.current]]
950 - def delete_fields(yo, fields):
951 "removes field(s) from the table" 952 doomedfields = fields.lower().replace('; ',';').split(';') 953 for victim in doomedfields: 954 if victim not in yo._meta.fields: 955 raise DbfError("field %s not in table -- delete aborted" % victim) 956 for victim in doomedfields: 957 yo._meta.fields.pop(yo._meta.fields.index(victim)) 958 start = yo._meta[victim]['start'] 959 end = yo._meta[victim]['end'] 960 for record in yo: 961 record._data = record._data[:start] + record._data[end:] 962 for field in yo._meta.fields: 963 if yo._meta[field]['start'] == end: 964 end = yo._meta[field]['end'] 965 yo._meta[field]['start'] = start 966 yo._meta[field]['end'] = start + yo._meta[field]['length'] 967 start = yo._meta[field]['end'] 968 yo._buildHeaderFields() 969 yo._updateDisk()
970 - def eof(yo):
971 "moves record pointer to next usable record; returns True if no more usable records" 972 while yo._meta.current < yo._meta.header.recordcount() - 1: 973 yo._meta.current += 1 974 if yo.use_deleted() or not yo.current().has_been_deleted(): 975 break 976 else: 977 yo._meta.current = yo._meta.header.recordcount() 978 return True 979 return False
980 - def export(yo, records=None, filename=None, field_spec=None, format='csv', header=True):
981 """writes the table using CSV or tab-delimited format, using the filename 982 given if specified, otherwise the table name""" 983 if filename is None: 984 filename = yo.filename() 985 if field_spec is None: 986 field_spec = yo.field_names() 987 else: 988 field_spec = field_spec.replace(', ',',').split(',') 989 if records is None: 990 records = yo 991 format = format.lower() 992 if format not in ('csv', 'tab'): 993 raise DbfError("export format: csv or tab, not %s" % format) 994 base, ext = os.path.splitext(filename) 995 if ext.lower() in ('', '.dbf'): 996 filename = base + "." + format 997 fd = open(filename, 'wb') 998 try: 999 if format == 'csv': 1000 csvfile = csv.writer(fd, dialect='dbf') 1001 if header: 1002 csvfile.writerow(field_spec) 1003 for record in records: 1004 fields = [] 1005 for fieldname in field_spec: 1006 fields.append(record[fieldname]) 1007 csvfile.writerow(fields) 1008 else: 1009 if header: 1010 fd.write('\t'.join(field_spec) + '\n') 1011 for record in records: 1012 fields = [] 1013 for fieldname in field_spec: 1014 fields.append(str(record[fieldname])) 1015 fd.write('\t'.join(fields) + '\n') 1016 finally: 1017 fd.close() 1018 fd = None 1019 return len(records)
1020 - def goto(yo, criteria):
1021 """changes the record pointer to the first matching (non-deleted) record 1022 criteria should be either a tuple of tuple(value, field, func) triples, 1023 or an integer to go to""" 1024 if isinstance(criteria, int): 1025 if not -yo._meta.header.recordcount() <= criteria < yo._meta.header.recordcount(): 1026 raise IndexError("Record %d does not exist" % criteria) 1027 if criteria < 0: 1028 criteria += yo._meta.header.recordcount() 1029 yo._meta.current = criteria 1030 return yo.current() 1031 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop]) 1032 specs = tuple([(field, func) for value, field, func in criteria]) 1033 match = tuple([value for value, field, func in criteria]) 1034 current = yo.current(index=True) 1035 matchlen = len(match) 1036 while not yo.Eof(): 1037 record = yo.current() 1038 results = record(*specs) 1039 if results == match: 1040 return record 1041 return yo.goto(current)
1042 - def index(yo, sort=None, reverse=False):
1043 "orders the table using the sort provided; removes index if no sort provided" 1044 if sort is None: 1045 results = [] 1046 for field, func in yo._meta.index: 1047 results.append("%s(%s)" % (func.__name__, field)) 1048 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed]) 1049 yo._meta.index_reversed = reverse 1050 if sort == 'ORIGINAL': 1051 yo._index = range(yo._meta.header.recordcount()) 1052 yo._meta.index = [] 1053 if reverse: 1054 yo._index.reverse() 1055 return 1056 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop]) 1057 yo._meta.index = tuple(new_sort) 1058 yo._meta.orderresults = [''] * len(yo) 1059 for record in yo: 1060 yo._meta.orderresults[record.record_number()] = record() 1061 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
1062 - def is_memotype(yo, name):
1063 "returns True if name is a memo type field" 1064 return yo._meta[name]['type'] in yo._memotypes
1065 - def new(yo, filename, _field_spec=None):
1066 "returns a new table of the same type" 1067 if _field_spec is None: 1068 _field_spec = yo.structure() 1069 if filename != ':memory:': 1070 path, name = os.path.split(filename) 1071 if path == "": 1072 filename = os.path.join(os.path.split(yo.filename)[0], filename) 1073 elif name == "": 1074 filename = os.path.join(path, os.path.split(yo.filename)[1]) 1075 return yo.__class__(filename, _field_spec)
1076 - def next(yo):
1077 "set record pointer to next (non-deleted) record, and return it" 1078 if yo.eof(): 1079 raise Eof() 1080 return yo.current()
1081 - def pack(yo, _pack=True):
1082 "physically removes all deleted records" 1083 newtable = [] 1084 newindex = [] 1085 i = 0 1086 for record in yo._table: 1087 if record.has_been_deleted() and _pack: 1088 record._recnum = -1 1089 else: 1090 record._recnum = i 1091 newtable.append(record) 1092 newindex.append(i) 1093 i += 1 1094 yo._table = newtable 1095 yo._index = newindex 1096 yo._meta.header.recordcount(i) 1097 yo._current = -1 1098 yo._meta.index = '' 1099 yo._updateDisk()
1100 - def prev(yo):
1101 "set record pointer to previous (non-deleted) record, and return it" 1102 if yo.bof(): 1103 raise Bof 1104 return yo.current()
1105 - def query(yo, sql=None, python=None):
1106 "uses exec to perform python queries on the table" 1107 if python is None: 1108 raise DbfError("query: python parameter must be specified") 1109 possible = DbfList(desc="%s --> %s" % (yo.filename(), python)) 1110 query_result = {} 1111 select = 'query_result["keep"] = %s' % python 1112 g = {} 1113 for record in yo: 1114 query_result['keep'] = False 1115 g['query_result'] = query_result 1116 exec select in g, record 1117 if query_result['keep']: 1118 possible.append(record) 1119 return possible
1120 - def rename_field(yo, oldname, newname):
1121 "renames an existing field" 1122 if not oldname in yo._meta.fields: 1123 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname) 1124 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum(): 1125 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits") 1126 newname = newname.lower() 1127 if newname in yo._meta.fields: 1128 raise DbfError("field --%s-- already exists" % newname) 1129 if len(newname) > 10: 1130 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname))) 1131 yo._meta[newname] = yo._meta[oldname] 1132 yo._meta.fields[yo._meta.fields.index(oldname)] = newname 1133 yo._buildHeaderFields() 1134 yo._updateDisk(headeronly=True)
1135 - def search(yo, match, fuzzy=None, indices=False):
1136 """searches using a binary algorythm 1137 looking for records that match the criteria in match, which is a tuple 1138 with a data item per ordered field. table must be sorted. if index, 1139 returns a list of records' indices from the current sort order. 1140 """ 1141 if yo._meta.index is None: 1142 raise DbfError('table must be indexed to use Search') 1143 matchlen = len(match) 1144 if fuzzy: 1145 matchlen -= 1 1146 fuzzy_match = match[-1] 1147 fuzzy_field = yo._meta.index[matchlen][0] 1148 match = match[:-1] 1149 records = DbfList(desc="%s --> search: index=%s, match=%s, fuzzy=%s(%s))" % (yo.filename(), yo.index(), match, fuzzy.__name__, fuzzy_match)) 1150 else: 1151 records = DbfList(desc="%s --> search: index=%s, match=%s)" % (yo.filename(), yo.index(), match)) 1152 if indices: 1153 records = [] 1154 if not isinstance(match, tuple): 1155 match = tuple(match) 1156 segment = len(yo) 1157 current = 0 1158 toosoon = True 1159 notFound = True 1160 while notFound: 1161 segment = segment // 2 1162 if toosoon: 1163 current += segment 1164 else: 1165 current -= segment 1166 if current % 2: 1167 segment += 1 1168 if current == len(yo) or segment == 0: 1169 break 1170 value = yo._meta.orderresults[yo[current].record_number()][:matchlen] 1171 if value < match: 1172 toosoon = True 1173 elif value > match: 1174 toosoon = False 1175 else: 1176 notFound = False 1177 break 1178 if current == 0: 1179 break 1180 if notFound: 1181 return records 1182 while current > 0: 1183 current -= 1 1184 value = yo._meta.orderresults[yo[current].record_number()][:matchlen] 1185 if value != match: 1186 current += 1 1187 break 1188 while True: 1189 value = yo._meta.orderresults[yo[current].record_number()][:matchlen] 1190 if value != match: 1191 break 1192 if yo.use_deleted() or not yo[current].has_been_deleted(): 1193 if indices: 1194 records.append(current) 1195 else: 1196 records.append(yo[current]) 1197 current += 1 1198 if current == len(yo): 1199 break 1200 if fuzzy: 1201 if indices: 1202 records = [rec for rec in records if fuzzy(yo[rec][fuzzy_field]) == fuzzy_match] 1203 else: 1204 records[:] = [rec for rec in records if fuzzy(rec[fuzzy_field]) == fuzzy_match] 1205 return records
1206 - def size(yo, field):
1207 "returns size of field as a tuple of (length, decimals)" 1208 if field in yo: 1209 return (yo._meta[field]['length'], yo._meta[field]['decimals']) 1210 raise DbfError("%s is not a field in %s" % (field, yo.filename()))
1211 - def structure(yo, fields=None):
1212 "return list of fields suitable for creating same table layout" 1213 field_spec = [] 1214 try: 1215 if fields is None: 1216 for i in range(len(yo._meta.fields)): 1217 field_spec.append(yo._fieldLayout(i)) 1218 else: 1219 for name in fields.replace(' ','').split(';'): 1220 field_spec.append(yo._fieldLayout(yo.field_names().index(name))) 1221 except ValueError: 1222 raise DbfError("field --%s-- does not exist" % name) 1223 return field_spec
1224 - def top(yo, get_record=False):
1225 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record 1226 DbfError if table is empty 1227 Eof if all records are deleted and use_deleted() is False""" 1228 yo._meta.current = -1 1229 if get_record: 1230 try: 1231 return yo.next() 1232 except Eof: 1233 yo._meta.current = -1 1234 raise Bof()
1235 - def type(yo, field):
1236 "returns type of field" 1237 if field in yo: 1238 return yo._meta[field]['type'] 1239 raise DbfError("%s is not a field in %s" % (field, yo.filename()))
1240 - def zap(yo, areyousure=False):
1241 """removes all records from table -- this cannot be undone! 1242 areyousure must be True, else error is raised""" 1243 if areyousure: 1244 yo._table = [] 1245 yo._index = [] 1246 yo._meta.header.recordcount(0) 1247 yo._current = -1 1248 yo._meta.index = '' 1249 yo._updateDisk() 1250 else: 1251 raise DbfError("You must say you are sure to wipe the table")
1252 # these asignments are for backward compatibility, and will go away
1253 -class Db3Table(DbfTable):
1254 """Provides an interface for working with dBase III tables.""" 1255 _version = 'dBase III Plus' 1256 _versionabbv = 'db3' 1257 _fieldtypes = { 1258 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1259 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1260 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1261 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1262 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric} } 1263 _memoext = '.dbt' 1264 _memotypes = ('M',) 1265 _memoClass = _Db3Memo 1266 _yesMemoMask = '\x80' 1267 _noMemoMask = '\x7f' 1268 _fixedFields = ('D','L','M') 1269 _decimalFields = ('N',) 1270 _variableFields = ('C',) 1271 _numericFields = ('N',) 1272 _dbfTableHeader = array('c', '\x00' * 32) 1273 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 1274 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 1275 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1276 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1277 _dbfTableHeader = _dbfTableHeader.tostring() 1278 _dbfTableHeaderExtra = '' 1279 _supported_tables = ['\x03', '\x83'] 1280 _read_only = False 1281 _meta_only = False 1282 _use_deleted = True
1283 - def _checkMemoIntegrity(yo):
1284 "dBase III specific" 1285 if yo._meta.header.version() == '\x83': 1286 try: 1287 yo._meta.memo = yo._memoClass(yo._meta) 1288 except: 1289 yo._meta.dfd.close() 1290 yo._meta.dfd = None 1291 raise 1292 if not yo._meta.ignorememos: 1293 for field in yo._meta.fields: 1294 if yo._meta[field]['type'] in yo._memotypes: 1295 if yo._meta.header.version() != '\x83': 1296 yo._meta.dfd.close() 1297 yo._meta.dfd = None 1298 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 1299 elif not os.path.exists(yo._meta.memoname): 1300 yo._meta.dfd.close() 1301 yo._meta.dfd = None 1302 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1303 break
1304 - def _initializeFields(yo):
1305 "builds the FieldList of names, types, and descriptions" 1306 offset = 1 1307 fieldsdef = yo._meta.header.fields() 1308 if len(fieldsdef) % 32 != 0: 1309 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1310 if len(fieldsdef) // 32 != yo.field_count(): 1311 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count(), len(fieldsdef)//32)) 1312 for i in range(yo.field_count()): 1313 fieldblock = fieldsdef[i*32:(i+1)*32] 1314 name = io.unpackStr(fieldblock[:11]) 1315 type = fieldblock[11] 1316 if not type in yo._meta.fieldtypes: 1317 raise DbfError("Unknown field type: %s" % type) 1318 start = offset 1319 length = ord(fieldblock[16]) 1320 offset += length 1321 end = start + length 1322 decimals = ord(fieldblock[17]) 1323 flags = ord(fieldblock[18]) 1324 yo._meta.fields.append(name) 1325 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1326 -class VfpTable(DbfTable):
1327 version = 'Provides an interface for working with Visual FoxPro 6 tables' 1328 _versionabbv = 'vfp' 1329 _fieldtypes = { 1330 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1331 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 1332 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 1333 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 1334 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 1335 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 1336 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1337 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1338 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 1339 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1340 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1341 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1342 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 1343 _memoext = '.fpt' 1344 _memotypes = ('G','M','P') 1345 _memoClass = _VfpMemo 1346 _yesMemoMask = '\x30' # 0011 0000 1347 _noMemoMask = '\x30' # 0011 0000 1348 _fixedFields = ('B','D','G','I','L','M','P','T','Y') 1349 _decimalFields = ('F','N') 1350 _variableFields = ('C',) 1351 _numericFields = ('B','F','I','N','Y') 1352 _supported_tables = ('\x30',) 1353 _dbfTableHeader = array('c', '\x00' * 32) 1354 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1355 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1356 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1357 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1358 _dbfTableHeader = _dbfTableHeader.tostring() 1359 _dbfTableHeaderExtra = '\x00' * 263 1360 _use_deleted = True
1361 - def _checkMemoIntegrity(yo):
1362 if os.path.exists(yo._meta.memoname): 1363 try: 1364 yo._meta.memo = yo._memoClass(yo._meta) 1365 except: 1366 yo._meta.dfd.close() 1367 yo._meta.dfd = None 1368 raise 1369 if not yo._meta.ignorememos: 1370 for field in yo._meta.fields: 1371 if yo._meta[field]['type'] in yo._memotypes: 1372 if not os.path.exists(yo._meta.memoname): 1373 yo._meta.dfd.close() 1374 yo._meta.dfd = None 1375 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1376 break
1377 - def _initializeFields(yo):
1378 "builds the FieldList of names, types, and descriptions" 1379 offset = 1 1380 fieldsdef = yo._meta.header.fields() 1381 for i in range(yo.field_count()): 1382 fieldblock = fieldsdef[i*32:(i+1)*32] 1383 name = io.unpackStr(fieldblock[:11]) 1384 type = fieldblock[11] 1385 if not type in yo._meta.fieldtypes: 1386 raise DbfError("Unknown field type: %s" % type) 1387 elif type == '0': 1388 return # ignore nullflags 1389 start = io.unpackLongInt(fieldblock[12:16]) 1390 length = ord(fieldblock[16]) 1391 offset += length 1392 end = start + length 1393 decimals = ord(fieldblock[17]) 1394 flags = ord(fieldblock[18]) 1395 yo._meta.fields.append(name) 1396 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1397 -class DbfList(object):
1398 "list of Dbf records" 1399 _desc = ''
1400 - def __init__(yo, new_records=None, desc=None):
1401 if new_records is not None: 1402 yo._list_of_records = list(new_records) 1403 yo._current_record = 0 1404 else: 1405 yo._list_of_records = [] 1406 yo._current_record = -1 1407 if desc is not None: 1408 yo._desc = desc
1409 - def __delitem__(yo, key):
1410 if inlist(type(key), int, slice): 1411 return yo._list_of_records.__delitem__[key] 1412 else: 1413 raise TypeError
1414 - def __getitem__(yo, key):
1415 if type(key) == int: 1416 count = len(yo._list_of_records) 1417 if not -count <= key < count: 1418 raise IndexError("Record %d is not in list." % key) 1419 return yo._list_of_records[key] 1420 elif type(key) == slice: 1421 return DbfList(yo._list_of_records[key]) 1422 else: 1423 raise TypeError
1424 - def __iter__(yo):
1425 return (record for record in yo._list_of_records)
1426 - def __len__(yo):
1427 return len(yo._list_of_records)
1428 - def __nonzero__(yo):
1429 return len(yo) > 0
1430 - def __repr__(yo):
1431 if yo._desc: 1432 return "DbfList(%s - %d records)" % (yo._desc, len(yo._list_of_records)) 1433 else: 1434 return "DbfList(%d records)" % len(yo._list_of_records)
1435 - def __setitem__(yo, key, value):
1436 if isinstance(key, (int, slice)): 1437 return yo._list_of_records.__setitem__(key, value) 1438 else: 1439 raise TypeError
1440 - def append(yo, new_record):
1441 yo._list_of_records.append(new_record) 1442 yo._current_record = len(yo._list_of_records) - 1
1443 - def bottom(yo):
1444 if yo._list_of_records: 1445 yo._current_record = len(yo._list_of_records) - 1 1446 return yo._list_of_records[-1] 1447 raise DbfError("DbfList is empty")
1448 - def current(yo):
1449 if yo._current_record < 0: 1450 raise Bof() 1451 elif yo._current_record == len(yo._list_of_records): 1452 raise Eof() 1453 return yo._list_of_records[yo._current_record]
1454 - def count(yo, record):
1455 return yo._list_of_records.count(record)
1456 - def extend(yo, new_records):
1457 yo._list_of_records.extend(list(new_records)) 1458 yo._current_record = len(yo._list_of_records) - 1
1459 - def goto(yo, index_number):
1460 if yo._list_of_records: 1461 if 0 <= index_number <= len(yo._list_of_records): 1462 record = yo[index_number] 1463 yo._current_record = index_number 1464 return yo._list_of_records[yo._current_record] 1465 raise DbfError("index %d not in DbfList of %d records" % (index_number, len(yo._list_of_records))) 1466 raise DbfError("DbfList is empty")
1467 - def index(yo, record, i=None, j=None):
1468 if i is None: 1469 i = 0 1470 if j is None: 1471 j = len(yo) 1472 return yo._list_of_records.index(record, i, j)
1473 - def insert(yo, i, record):
1474 return yo._list_of_records.insert(i, record)
1475 - def next(yo):
1476 if yo._current_record < len(yo._list_of_records): 1477 yo._current_record += 1 1478 if yo._current_record < yo._list_of_records: 1479 return yo._list_of_records[yo._current_record] 1480 raise Eof()
1481 - def pop(yo, index=None):
1482 if index is None: 1483 return yo._list_of_records.pop() 1484 else: 1485 return yo._list_of_records.pop(index)
1486 - def prev(yo):
1487 if yo._current_record >= 0: 1488 yo._current_record -= 1 1489 if yo._current_record > -1: 1490 return yo._list_of_records[yo._current_record] 1491 raise Bof()
1492 - def remove(yo, record):
1493 return yo._list_of_records.remove(record)
1494 - def reverse(yo):
1495 return yo._list_of_records.reverse()
1496 - def top(yo):
1497 if yo._list_of_records: 1498 yo._current_record = 0 1499 return yo._list_of_records[0] 1500 raise DbfError("DbfList is empty")
1501 - def sort(yo, cmp=None, key=None, reverse=None):
1502 if reverse is not None: 1503 return yo._list_of_records.sort(cmp, key, reverse) 1504 elif key is not None: 1505 return yo._list_of_records.sort(cmp, key) 1506 elif cmp is not None: 1507 return yo._list_of_records.sort(cmp) 1508 else: 1509 return yo._list_of_records.sort()
1510 -class DbfCsv(csv.Dialect):
1511 "csv format for exporting tables" 1512 delimiter = ',' 1513 doublequote = True 1514 escapechar = None 1515 lineterminator = '\r\n' 1516 quotechar = '"' 1517 skipinitialspace = True 1518 quoting = csv.QUOTE_NONNUMERIC
1519 csv.register_dialect('dbf', DbfCsv) 1520
1521 -def _nop(value):
1522 "returns parameter unchanged" 1523 return value
1524 -def _normalize_tuples(tuples, length, filler):
1525 "ensures each tuple is the same length, using filler[-missing] for the gaps" 1526 final = [] 1527 for t in tuples: 1528 if len(t) < length: 1529 final.append( tuple([item for item in t] + filler[len(t)-length:]) ) 1530 else: 1531 final.append(t) 1532 return tuple(final)
1533 -class _Db4Table(DbfTable):
1534 version = 'dBase IV w/memos (non-functional)' 1535 _versionabbv = 'db4' 1536 _fieldtypes = { 1537 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1538 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 1539 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 1540 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 1541 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 1542 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 1543 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1544 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1545 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 1546 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1547 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1548 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1549 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 1550 _memoext = '.dbt' 1551 _memotypes = ('G','M','P') 1552 _memoClass = _VfpMemo 1553 _yesMemoMask = '\x8b' # 0011 0000 1554 _noMemoMask = '\x04' # 0011 0000 1555 _fixedFields = ('B','D','G','I','L','M','P','T','Y') 1556 _decimalFields = ('F','N') 1557 _variableFields = ('C',) 1558 _numericFields = ('B','F','I','N','Y') 1559 _supported_tables = ('\x04', '\x8b') 1560 _dbfTableHeader = ['\x00'] * 32 1561 _dbfTableHeader[0] = '\x8b' # version - Foxpro 6 0011 0000 1562 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1563 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1564 _dbfTableHeader = ''.join(_dbfTableHeader) 1565 _dbfTableHeaderExtra = '' 1566 _use_deleted = True
1567 - def _checkMemoIntegrity(yo):
1568 "dBase III specific" 1569 if yo._meta.header.version == '\x8b': 1570 try: 1571 yo._meta.memo = yo._memoClass(yo._meta) 1572 except: 1573 yo._meta.dfd.close() 1574 yo._meta.dfd = None 1575 raise 1576 if not yo._meta.ignorememos: 1577 for field in yo._meta.fields: 1578 if yo._meta[field]['type'] in yo._memotypes: 1579 if yo._meta.header.version != '\x8b': 1580 yo._meta.dfd.close() 1581 yo._meta.dfd = None 1582 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 1583 elif not os.path.exists(yo._meta.memoname): 1584 yo._meta.dfd.close() 1585 yo._meta.dfd = None 1586 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1587 break
1588