1 "table definitions"
2 import os
3 import sys
4 import csv
5 import codecs
6 import locale
7 import unicodedata
8 import weakref
9 from array import array
10 from bisect import bisect_left, bisect_right
11 from decimal import Decimal
12 from shutil import copyfileobj
13 import dbf
14 from dbf import _io as io
15 from dbf.dates import Date, DateTime, Time
16 from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex
17
18 input_decoding = locale.getdefaultlocale()[1]
19 default_codepage = 'cp1252'
20 return_ascii = False
21 temp_dir = os.environ.get("DBF_TEMP") or os.environ.get("TEMP") or ""
22
23 version_map = {
24 '\x02' : 'FoxBASE',
25 '\x03' : 'dBase III Plus',
26 '\x04' : 'dBase IV',
27 '\x05' : 'dBase V',
28 '\x30' : 'Visual FoxPro',
29 '\x31' : 'Visual FoxPro (auto increment field)',
30 '\x43' : 'dBase IV SQL',
31 '\x7b' : 'dBase IV w/memos',
32 '\x83' : 'dBase III Plus w/memos',
33 '\x8b' : 'dBase IV w/memos',
34 '\x8e' : 'dBase IV w/SQL table',
35 '\xf5' : 'FoxPro w/memos'}
36
37 code_pages = {
38 '\x00' : ('ascii', "plain ol' ascii"),
39 '\x01' : ('cp437', 'U.S. MS-DOS'),
40 '\x02' : ('cp850', 'International MS-DOS'),
41 '\x03' : ('cp1252', 'Windows ANSI'),
42 '\x04' : ('mac_roman', 'Standard Macintosh'),
43 '\x08' : ('cp865', 'Danish OEM'),
44 '\x09' : ('cp437', 'Dutch OEM'),
45 '\x0A' : ('cp850', 'Dutch OEM (secondary)'),
46 '\x0B' : ('cp437', 'Finnish OEM'),
47 '\x0D' : ('cp437', 'French OEM'),
48 '\x0E' : ('cp850', 'French OEM (secondary)'),
49 '\x0F' : ('cp437', 'German OEM'),
50 '\x10' : ('cp850', 'German OEM (secondary)'),
51 '\x11' : ('cp437', 'Italian OEM'),
52 '\x12' : ('cp850', 'Italian OEM (secondary)'),
53 '\x13' : ('cp932', 'Japanese Shift-JIS'),
54 '\x14' : ('cp850', 'Spanish OEM (secondary)'),
55 '\x15' : ('cp437', 'Swedish OEM'),
56 '\x16' : ('cp850', 'Swedish OEM (secondary)'),
57 '\x17' : ('cp865', 'Norwegian OEM'),
58 '\x18' : ('cp437', 'Spanish OEM'),
59 '\x19' : ('cp437', 'English OEM (Britain)'),
60 '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'),
61 '\x1B' : ('cp437', 'English OEM (U.S.)'),
62 '\x1C' : ('cp863', 'French OEM (Canada)'),
63 '\x1D' : ('cp850', 'French OEM (secondary)'),
64 '\x1F' : ('cp852', 'Czech OEM'),
65 '\x22' : ('cp852', 'Hungarian OEM'),
66 '\x23' : ('cp852', 'Polish OEM'),
67 '\x24' : ('cp860', 'Portugese OEM'),
68 '\x25' : ('cp850', 'Potugese OEM (secondary)'),
69 '\x26' : ('cp866', 'Russian OEM'),
70 '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'),
71 '\x40' : ('cp852', 'Romanian OEM'),
72 '\x4D' : ('cp936', 'Chinese GBK (PRC)'),
73 '\x4E' : ('cp949', 'Korean (ANSI/OEM)'),
74 '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'),
75 '\x50' : ('cp874', 'Thai (ANSI/OEM)'),
76 '\x57' : ('cp1252', 'ANSI'),
77 '\x58' : ('cp1252', 'Western European ANSI'),
78 '\x59' : ('cp1252', 'Spanish ANSI'),
79 '\x64' : ('cp852', 'Eastern European MS-DOS'),
80 '\x65' : ('cp866', 'Russian MS-DOS'),
81 '\x66' : ('cp865', 'Nordic MS-DOS'),
82 '\x67' : ('cp861', 'Icelandic MS-DOS'),
83 '\x68' : (None, 'Kamenicky (Czech) MS-DOS'),
84 '\x69' : (None, 'Mazovia (Polish) MS-DOS'),
85 '\x6a' : ('cp737', 'Greek MS-DOS (437G)'),
86 '\x6b' : ('cp857', 'Turkish MS-DOS'),
87 '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'),
88 '\x79' : ('cp949', 'Korean Windows'),
89 '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'),
90 '\x7b' : ('cp932', 'Japanese Windows'),
91 '\x7c' : ('cp874', 'Thai Windows'),
92 '\x7d' : ('cp1255', 'Hebrew Windows'),
93 '\x7e' : ('cp1256', 'Arabic Windows'),
94 '\xc8' : ('cp1250', 'Eastern European Windows'),
95 '\xc9' : ('cp1251', 'Russian Windows'),
96 '\xca' : ('cp1254', 'Turkish Windows'),
97 '\xcb' : ('cp1253', 'Greek Windows'),
98 '\x96' : ('mac_cyrillic', 'Russian Macintosh'),
99 '\x97' : ('mac_latin2', 'Macintosh EE'),
100 '\x98' : ('mac_greek', 'Greek Macintosh') }
101
102 if sys.version_info[:2] < (2, 6):
105 "Emulate PyProperty_Type() in Objects/descrobject.c"
106
107 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
108 self.fget = fget
109 self.fset = fset
110 self.fdel = fdel
111 self.__doc__ = doc or fget.__doc__
113 self.fget = func
114 if not self.__doc__:
115 self.__doc__ = fget.__doc__
116 - def __get__(self, obj, objtype=None):
117 if obj is None:
118 return self
119 if self.fget is None:
120 raise AttributeError, "unreadable attribute"
121 return self.fget(obj)
123 if self.fset is None:
124 raise AttributeError, "can't set attribute"
125 self.fset(obj, value)
127 if self.fdel is None:
128 raise AttributeError, "can't delete attribute"
129 self.fdel(obj)
131 self.fset = func
132 return self
134 self.fdel = func
135 return self
136
138 """Provides routines to extract and save data within the fields of a dbf record."""
139 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
141 """calls appropriate routine to fetch value stored in field from array
142 @param record_data: the data portion of the record
143 @type record_data: array of characters
144 @param fielddef: description of the field definition
145 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags'
146 @returns: python data stored in field"""
147
148 field_type = fielddef['type']
149 classtype = yo._layout.fieldtypes[field_type]['Class']
150 retrieve = yo._layout.fieldtypes[field_type]['Retrieve']
151 if classtype is not None:
152 datum = retrieve(record_data, fielddef, yo._layout.memo, classtype)
153 else:
154 datum = retrieve(record_data, fielddef, yo._layout.memo)
155 if field_type in yo._layout.character_fields:
156 datum = yo._layout.decoder(datum)[0]
157 if yo._layout.return_ascii:
158 try:
159 datum = yo._layout.output_encoder(datum)[0]
160 except UnicodeEncodeError:
161 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore')
162 return datum
164 "calls appropriate routine to convert value to ascii bytes, and save it in record"
165 field_type = fielddef['type']
166 update = yo._layout.fieldtypes[field_type]['Update']
167 if field_type in yo._layout.character_fields:
168 if not isinstance(value, unicode):
169 if yo._layout.input_decoder is None:
170 raise NonUnicode("String not in unicode format, no default encoding specified")
171 value = yo._layout.input_decoder(value)[0]
172 value = yo._layout.encoder(value)[0]
173 bytes = array('c', update(value, fielddef, yo._layout.memo))
174 size = fielddef['length']
175 if len(bytes) > size:
176 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size))
177 blank = array('c', ' ' * size)
178 start = fielddef['start']
179 end = start + size
180 blank[:len(bytes)] = bytes[:]
181 yo._data[start:end] = blank[:]
182 yo._dirty = True
201 if name[0:2] == '__' and name[-2:] == '__':
202 raise AttributeError, 'Method %s is not implemented.' % name
203 elif name == 'record_number':
204 return yo._recnum
205 elif name == 'delete_flag':
206 return yo._data[0] != ' '
207 elif not name in yo._layout.fields:
208 raise FieldMissing(name)
209 try:
210 fielddef = yo._layout[name]
211 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef)
212 return value
213 except DbfError, error:
214 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
215 raise
232 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
271 if type(name) == str:
272 yo.__setattr__(name, value)
273 elif type(name) in (int, long):
274 yo.__setattr__(yo._layout.fields[name], value)
275 elif type(name) == slice:
276 sequence = []
277 for field in yo._layout.fields[name]:
278 sequence.append(field)
279 if len(sequence) != len(value):
280 raise DbfError("length of slices not equal")
281 for field, val in zip(sequence, value):
282 yo[field] = val
283 else:
284 raise TypeError("%s is not a field name" % name)
286 result = []
287 for seq, field in enumerate(yo.field_names):
288 result.append("%3d - %-10s: %s" % (seq, field, yo[field]))
289 return '\n'.join(result)
291 return yo._data.tostring()
293 "creates a blank record data chunk"
294 layout = yo._layout
295 ondisk = layout.ondisk
296 layout.ondisk = False
297 yo._data = array('c', ' ' * layout.header.record_length)
298 layout.memofields = []
299 for field in layout.fields:
300 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']())
301 if layout[field]['type'] in layout.memotypes:
302 layout.memofields.append(field)
303 layout.blankrecord = yo._data[:]
304 layout.ondisk = ondisk
306 "marks record as deleted"
307 yo._data[0] = '*'
308 yo._dirty = True
309 return yo
310 @property
315 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True"
316 old_data = yo._data[:]
317 try:
318 for key in dictionary:
319 if not key in yo.field_names:
320 if drop:
321 continue
322 raise FieldMissing(key)
323 yo.__setattr__(key, dictionary[key])
324 except:
325 yo._data[:] = old_data
326 raise
327 return yo
328 @property
330 "marked for deletion?"
331 return yo._data[0] == '*'
340 @property
342 "physical record number"
343 return yo._recnum
344 @property
346 table = yo._layout.table()
347 if table is None:
348 raise DbfError("table is no longer available")
349 return table
351 for dbfindex in yo._layout.table()._indexen:
352 dbfindex(yo)
354 "blanks record"
355 if keep_fields is None:
356 keep_fields = []
357 keep = {}
358 for field in keep_fields:
359 keep[field] = yo[field]
360 if yo._layout.blankrecord == None:
361 yo._createBlankRecord()
362 yo._data[:] = yo._layout.blankrecord[:]
363 for field in keep_fields:
364 yo[field] = keep[field]
365 yo._dirty = True
366 return yo
368 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty."
369 keys = yo._layout.fields
370 if blank:
371 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys]
372 else:
373 values = [yo[field] for field in keys]
374 return dict(zip(keys, values))
376 "marks record as active"
377 yo._data[0] = ' '
378 yo._dirty = True
379 return yo
389 """Provides access to memo fields as dictionaries
390 must override _init, _get_memo, and _put_memo to
391 store memo contents to disk"""
393 "initialize disk file usage"
395 "retrieve memo contents from disk"
397 "store memo contents to disk"
399 ""
400 yo.meta = meta
401 yo.memory = {}
402 yo.nextmemo = 1
403 yo._init()
404 yo.meta.newmemofile = False
406 "gets the memo in block"
407 if yo.meta.ignorememos or not block:
408 return ''
409 if yo.meta.ondisk:
410 return yo._get_memo(block)
411 else:
412 return yo.memory[block]
414 "stores data in memo file, returns block number"
415 if yo.meta.ignorememos or data == '':
416 return 0
417 if yo.meta.inmemory:
418 thismemo = yo.nextmemo
419 yo.nextmemo += 1
420 yo.memory[thismemo] = data
421 else:
422 thismemo = yo._put_memo(data)
423 return thismemo
426 "dBase III specific"
427 yo.meta.memo_size= 512
428 yo.record_header_length = 2
429 if yo.meta.ondisk and not yo.meta.ignorememos:
430 if yo.meta.newmemofile:
431 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
432 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508)
433 else:
434 try:
435 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
436 yo.meta.mfd.seek(0)
437 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4))
438 except:
439 raise DbfError("memo file appears to be corrupt")
441 block = int(block)
442 yo.meta.mfd.seek(block * yo.meta.memo_size)
443 eom = -1
444 data = ''
445 while eom == -1:
446 newdata = yo.meta.mfd.read(yo.meta.memo_size)
447 if not newdata:
448 return data
449 data += newdata
450 eom = data.find('\x1a\x1a')
451 return data[:eom].rstrip()
453 data = data.rstrip()
454 length = len(data) + yo.record_header_length
455 blocks = length // yo.meta.memo_size
456 if length % yo.meta.memo_size:
457 blocks += 1
458 thismemo = yo.nextmemo
459 yo.nextmemo = thismemo + blocks
460 yo.meta.mfd.seek(0)
461 yo.meta.mfd.write(io.packLongInt(yo.nextmemo))
462 yo.meta.mfd.seek(thismemo * yo.meta.memo_size)
463 yo.meta.mfd.write(data)
464 yo.meta.mfd.write('\x1a\x1a')
465 double_check = yo._get_memo(thismemo)
466 if len(double_check) != len(data):
467 uhoh = open('dbf_memo_dump.err','wb')
468 uhoh.write('thismemo: %d' % thismemo)
469 uhoh.write('nextmemo: %d' % yo.nextmemo)
470 uhoh.write('saved: %d bytes' % len(data))
471 uhoh.write(data)
472 uhoh.write('retrieved: %d bytes' % len(double_check))
473 uhoh.write(double_check)
474 uhoh.close()
475 raise DbfError("unknown error: memo not saved")
476 return thismemo
479 "Visual Foxpro 6 specific"
480 if yo.meta.ondisk and not yo.meta.ignorememos:
481 yo.record_header_length = 8
482 if yo.meta.newmemofile:
483 if yo.meta.memo_size == 0:
484 yo.meta.memo_size = 1
485 elif 1 < yo.meta.memo_size < 33:
486 yo.meta.memo_size *= 512
487 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
488 nextmemo = 512 // yo.meta.memo_size
489 if nextmemo * yo.meta.memo_size < 512:
490 nextmemo += 1
491 yo.nextmemo = nextmemo
492 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \
493 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504)
494 else:
495 try:
496 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
497 yo.meta.mfd.seek(0)
498 header = yo.meta.mfd.read(512)
499 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True)
500 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True)
501 except:
502 raise DbfError("memo file appears to be corrupt")
504 yo.meta.mfd.seek(block * yo.meta.memo_size)
505 header = yo.meta.mfd.read(8)
506 length = io.unpackLongInt(header[4:], bigendian=True)
507 return yo.meta.mfd.read(length)
509 data = data.rstrip()
510 yo.meta.mfd.seek(0)
511 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True)
512 yo.meta.mfd.seek(0)
513 length = len(data) + yo.record_header_length
514 blocks = length // yo.meta.memo_size
515 if length % yo.meta.memo_size:
516 blocks += 1
517 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True))
518 yo.meta.mfd.seek(thismemo*yo.meta.memo_size)
519 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data)
520 return thismemo
521
523 """Provides a framework for dbf style tables."""
524 _version = 'basic memory table'
525 _versionabbv = 'dbf'
526 _fieldtypes = {
527 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Class':None},
528 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Class':None},
529 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Class':None} }
530 _memoext = ''
531 _memotypes = tuple('M', )
532 _memoClass = _DbfMemo
533 _yesMemoMask = ''
534 _noMemoMask = ''
535 _fixed_fields = ('M','D','L')
536 _variable_fields = tuple()
537 _character_fields = tuple('M', )
538 _decimal_fields = tuple()
539 _numeric_fields = tuple()
540 _currency_fields = tuple()
541 _dbfTableHeader = array('c', '\x00' * 32)
542 _dbfTableHeader[0] = '\x00'
543 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
544 _dbfTableHeader[10] = '\x01'
545 _dbfTableHeader[29] = '\x00'
546 _dbfTableHeader = _dbfTableHeader.tostring()
547 _dbfTableHeaderExtra = ''
548 _supported_tables = []
549 _read_only = False
550 _meta_only = False
551 _use_deleted = True
552 backup = False
554 "implements the weakref structure for DbfLists"
558 yo._lists = set([s for s in yo._lists if s() is not None])
559 return (s() for s in yo._lists if s() is not None)
561 yo._lists = set([s for s in yo._lists if s() is not None])
562 return len(yo._lists)
563 - def add(yo, new_list):
564 yo._lists.add(weakref.ref(new_list))
565 yo._lists = set([s for s in yo._lists if s() is not None])
567 "implements the weakref structure for seperate indexes"
571 yo._indexen = set([s for s in yo._indexen if s() is not None])
572 return (s() for s in yo._indexen if s() is not None)
574 yo._indexen = set([s for s in yo._indexen if s() is not None])
575 return len(yo._indexen)
576 - def add(yo, new_list):
577 yo._indexen.add(weakref.ref(new_list))
578 yo._indexen = set([s for s in yo._indexen if s() is not None])
593 if len(data) != 32:
594 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data))
595 yo._data = array('c', data + '\x0d')
597 "get/set code page of table"
598 if cp is None:
599 return yo._data[29]
600 else:
601 cp, sd, ld = _codepage_lookup(cp)
602 yo._data[29] = cp
603 return cp
604 @property
610 @data.setter
612 if len(bytes) < 32:
613 raise DbfError("length for data of %d is less than 32" % len(bytes))
614 yo._data[:] = array('c', bytes)
615 @property
617 "extra dbf info (located after headers, before data records)"
618 fieldblock = yo._data[32:]
619 for i in range(len(fieldblock)//32+1):
620 cr = i * 32
621 if fieldblock[cr] == '\x0d':
622 break
623 else:
624 raise DbfError("corrupt field structure")
625 cr += 33
626 return yo._data[cr:].tostring()
627 @extra.setter
629 fieldblock = yo._data[32:]
630 for i in range(len(fieldblock)//32+1):
631 cr = i * 32
632 if fieldblock[cr] == '\x0d':
633 break
634 else:
635 raise DbfError("corrupt field structure")
636 cr += 33
637 yo._data[cr:] = array('c', data)
638 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
639 @property
641 "number of fields (read-only)"
642 fieldblock = yo._data[32:]
643 for i in range(len(fieldblock)//32+1):
644 cr = i * 32
645 if fieldblock[cr] == '\x0d':
646 break
647 else:
648 raise DbfError("corrupt field structure")
649 return len(fieldblock[:cr]) // 32
650 @property
652 "field block structure"
653 fieldblock = yo._data[32:]
654 for i in range(len(fieldblock)//32+1):
655 cr = i * 32
656 if fieldblock[cr] == '\x0d':
657 break
658 else:
659 raise DbfError("corrupt field structure")
660 return fieldblock[:cr].tostring()
661 @fields.setter
663 fieldblock = yo._data[32:]
664 for i in range(len(fieldblock)//32+1):
665 cr = i * 32
666 if fieldblock[cr] == '\x0d':
667 break
668 else:
669 raise DbfError("corrupt field structure")
670 cr += 32
671 fieldlen = len(block)
672 if fieldlen % 32 != 0:
673 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen)
674 yo._data[32:cr] = array('c', block)
675 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
676 fieldlen = fieldlen // 32
677 recordlen = 1
678 for i in range(fieldlen):
679 recordlen += ord(block[i*32+16])
680 yo._data[10:12] = array('c', io.packShortInt(recordlen))
681 @property
683 "number of records (maximum 16,777,215)"
684 return io.unpackLongInt(yo._data[4:8].tostring())
685 @record_count.setter
688 @property
690 "length of a record (read_only) (max of 65,535)"
691 return io.unpackShortInt(yo._data[10:12].tostring())
692 @property
694 "starting position of first record in file (must be within first 64K)"
695 return io.unpackShortInt(yo._data[8:10].tostring())
696 @start.setter
699 @property
701 "date of last table modification (read-only)"
702 return io.unpackDate(yo._data[1:4].tostring())
703 @property
705 "dbf version"
706 return yo._data[0]
707 @version.setter
711 "implements the weakref table for records"
713 yo._meta = meta
714 yo._weakref_list = [weakref.ref(lambda x: None)] * count
716 maybe = yo._weakref_list[index]()
717 if maybe is None:
718 if index < 0:
719 index += yo._meta.header.record_count
720 size = yo._meta.header.record_length
721 location = index * size + yo._meta.header.start
722 yo._meta.dfd.seek(location)
723 if yo._meta.dfd.tell() != location:
724 raise ValueError("unable to seek to offset %d in file" % location)
725 bytes = yo._meta.dfd.read(size)
726 if not bytes:
727 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location))
728 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True)
729 yo._weakref_list[index] = weakref.ref(maybe)
730 return maybe
732 yo._weakref_list.append(weakref.ref(record))
734 yo._weakref_list[:] = []
736 return yo._weakref_list.pop()
738 "returns records using current index"
740 yo._table = table
741 yo._index = -1
742 yo._more_records = True
746 while yo._more_records:
747 yo._index += 1
748 if yo._index >= len(yo._table):
749 yo._more_records = False
750 continue
751 record = yo._table[yo._index]
752 if not yo._table.use_deleted and record.has_been_deleted:
753 continue
754 return record
755 else:
756 raise StopIteration
758 "constructs fieldblock for disk table"
759 fieldblock = array('c', '')
760 memo = False
761 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask))
762 for field in yo._meta.fields:
763 if yo._meta.fields.count(field) > 1:
764 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)")
765 fielddef = array('c', '\x00' * 32)
766 fielddef[:11] = array('c', io.packStr(field))
767 fielddef[11] = yo._meta[field]['type']
768 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start']))
769 fielddef[16] = chr(yo._meta[field]['length'])
770 fielddef[17] = chr(yo._meta[field]['decimals'])
771 fielddef[18] = chr(yo._meta[field]['flags'])
772 fieldblock.extend(fielddef)
773 if yo._meta[field]['type'] in yo._meta.memotypes:
774 memo = True
775 yo._meta.header.fields = fieldblock.tostring()
776 if memo:
777 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask))
778 if yo._meta.memo is None:
779 yo._meta.memo = yo._memoClass(yo._meta)
781 "dBase III specific"
782 if yo._meta.header.version == '\x83':
783 try:
784 yo._meta.memo = yo._memoClass(yo._meta)
785 except:
786 yo._meta.dfd.close()
787 yo._meta.dfd = None
788 raise
789 if not yo._meta.ignorememos:
790 for field in yo._meta.fields:
791 if yo._meta[field]['type'] in yo._memotypes:
792 if yo._meta.header.version != '\x83':
793 yo._meta.dfd.close()
794 yo._meta.dfd = None
795 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
796 elif not os.path.exists(yo._meta.memoname):
797 yo._meta.dfd.close()
798 yo._meta.dfd = None
799 raise DbfError("Table structure corrupt: memo fields exist without memo file")
800 break
802 "builds the FieldList of names, types, and descriptions from the disk file"
803 yo._meta.fields[:] = []
804 offset = 1
805 fieldsdef = yo._meta.header.fields
806 if len(fieldsdef) % 32 != 0:
807 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
808 if len(fieldsdef) // 32 != yo.field_count:
809 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
810 for i in range(yo.field_count):
811 fieldblock = fieldsdef[i*32:(i+1)*32]
812 name = io.unpackStr(fieldblock[:11])
813 type = fieldblock[11]
814 if not type in yo._meta.fieldtypes:
815 raise DbfError("Unknown field type: %s" % type)
816 start = offset
817 length = ord(fieldblock[16])
818 offset += length
819 end = start + length
820 decimals = ord(fieldblock[17])
821 flags = ord(fieldblock[18])
822 if name in yo._meta.fields:
823 raise DbfError('Duplicate field name found: %s' % name)
824 yo._meta.fields.append(name)
825 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
827 "Returns field information Name Type(Length[,Decimals])"
828 name = yo._meta.fields[i]
829 type = yo._meta[name]['type']
830 length = yo._meta[name]['length']
831 decimals = yo._meta[name]['decimals']
832 if type in yo._decimal_fields:
833 description = "%s %s(%d,%d)" % (name, type, length, decimals)
834 elif type in yo._fixed_fields:
835 description = "%s %s" % (name, type)
836 else:
837 description = "%s %s(%d)" % (name, type, length)
838 return description
840 "loads the records from disk to memory"
841 if yo._meta_only:
842 raise DbfError("%s has been closed, records are unavailable" % yo.filename)
843 dfd = yo._meta.dfd
844 header = yo._meta.header
845 dfd.seek(header.start)
846 allrecords = dfd.read()
847 dfd.seek(0)
848 length = header.record_length
849 for i in range(header.record_count):
850 record_data = allrecords[length*i:length*i+length]
851 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True))
852 dfd.seek(0)
854 if specs is None:
855 specs = yo.field_names
856 elif isinstance(specs, str):
857 specs = specs.split(sep)
858 else:
859 specs = list(specs)
860 specs = [s.strip() for s in specs]
861 return specs
863 "synchronizes the disk file with current data"
864 if yo._meta.inmemory:
865 return
866 fd = yo._meta.dfd
867 fd.seek(0)
868 fd.write(yo._meta.header.data)
869 if not headeronly:
870 for record in yo._table:
871 record._update_disk()
872 fd.flush()
873 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length)
874 if 'db3' in yo._versionabbv:
875 fd.seek(0, os.SEEK_END)
876 fd.write('\x1a')
877 fd.flush()
878 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
879
887 if name in ('_table'):
888 if yo._meta.ondisk:
889 yo._table = yo._Table(len(yo), yo._meta)
890 else:
891 yo._table = []
892 yo._loadtable()
893 return object.__getattribute__(yo, name)
895 if type(value) == int:
896 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count:
897 raise IndexError("Record %d is not in table." % value)
898 return yo._table[value]
899 elif type(value) == slice:
900 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names)
901 yo._dbflists.add(sequence)
902 for index in range(len(yo))[value]:
903 record = yo._table[index]
904 if yo.use_deleted is True or not record.has_been_deleted:
905 sequence.append(record)
906 return sequence
907 else:
908 raise TypeError('type <%s> not valid for indexing' % type(value))
909 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False,
910 read_only=False, keep_memos=False, meta_only=False, codepage=None,
911 numbers='default', strings=str, currency=Decimal):
912 """open/create dbf file
913 filename should include path if needed
914 field_specs can be either a ;-delimited string or a list of strings
915 memo_size is always 512 for db3 memos
916 ignore_memos is useful if the memo file is missing or corrupt
917 read_only will load records into memory, then close the disk file
918 keep_memos will also load any memo fields into memory
919 meta_only will ignore all records, keeping only basic table information
920 codepage will override whatever is set in the table itself"""
921 if filename[0] == filename[-1] == ':':
922 if field_specs is None:
923 raise DbfError("field list must be specified for memory tables")
924 elif type(yo) is DbfTable:
925 raise DbfError("only memory tables supported")
926 yo._dbflists = yo._DbfLists()
927 yo._indexen = yo._Indexen()
928 yo._meta = meta = yo._MetaData()
929 for datatypes, classtype in (
930 (yo._character_fields, strings),
931 (yo._numeric_fields, numbers),
932 (yo._currency_fields, currency),
933 ):
934 for datatype in datatypes:
935 yo._fieldtypes[datatype]['Class'] = classtype
936 meta.numbers = numbers
937 meta.strings = strings
938 meta.currency = currency
939 meta.table = weakref.ref(yo)
940 meta.filename = filename
941 meta.fields = []
942 meta.fieldtypes = yo._fieldtypes
943 meta.fixed_fields = yo._fixed_fields
944 meta.variable_fields = yo._variable_fields
945 meta.character_fields = yo._character_fields
946 meta.decimal_fields = yo._decimal_fields
947 meta.numeric_fields = yo._numeric_fields
948 meta.memotypes = yo._memotypes
949 meta.ignorememos = ignore_memos
950 meta.memo_size = memo_size
951 meta.input_decoder = codecs.getdecoder(input_decoding)
952 meta.output_encoder = codecs.getencoder(input_decoding)
953 meta.return_ascii = return_ascii
954 meta.header = header = yo._TableHeader(yo._dbfTableHeader)
955 header.extra = yo._dbfTableHeaderExtra
956 header.data
957 if filename[0] == filename[-1] == ':':
958 yo._table = []
959 meta.ondisk = False
960 meta.inmemory = True
961 meta.memoname = filename
962 else:
963 base, ext = os.path.splitext(filename)
964 if ext == '':
965 meta.filename = base + '.dbf'
966 meta.memoname = base + yo._memoext
967 meta.ondisk = True
968 meta.inmemory = False
969 if field_specs:
970 if meta.ondisk:
971 meta.dfd = open(meta.filename, 'w+b')
972 meta.newmemofile = True
973 yo.add_fields(field_specs)
974 header.codepage(codepage or default_codepage)
975 cp, sd, ld = _codepage_lookup(meta.header.codepage())
976 meta.decoder = codecs.getdecoder(sd)
977 meta.encoder = codecs.getencoder(sd)
978 return
979 try:
980 dfd = meta.dfd = open(meta.filename, 'r+b')
981 except IOError, e:
982 raise DbfError(str(e))
983 dfd.seek(0)
984 meta.header = header = yo._TableHeader(dfd.read(32))
985 if not header.version in yo._supported_tables:
986 dfd.close()
987 dfd = None
988 raise DbfError(
989 "%s does not support %s [%x]" %
990 (yo._version,
991 version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version),
992 ord(meta.header.version)))
993 cp, sd, ld = _codepage_lookup(meta.header.codepage())
994 yo._meta.decoder = codecs.getdecoder(sd)
995 yo._meta.encoder = codecs.getencoder(sd)
996 fieldblock = dfd.read(header.start - 32)
997 for i in range(len(fieldblock)//32+1):
998 fieldend = i * 32
999 if fieldblock[fieldend] == '\x0d':
1000 break
1001 else:
1002 raise DbfError("corrupt field structure in header")
1003 if len(fieldblock[:fieldend]) % 32 != 0:
1004 raise DbfError("corrupt field structure in header")
1005 header.fields = fieldblock[:fieldend]
1006 header.extra = fieldblock[fieldend+1:]
1007 yo._initializeFields()
1008 yo._checkMemoIntegrity()
1009 meta.current = -1
1010 if len(yo) > 0:
1011 meta.current = 0
1012 dfd.seek(0)
1013 if meta_only:
1014 yo.close(keep_table=False, keep_memos=False)
1015 elif read_only:
1016 yo.close(keep_table=True, keep_memos=keep_memos)
1017 if codepage is not None:
1018 cp, sd, ld = _codepage_lookup(codepage)
1019 yo._meta.decoder = codecs.getdecoder(sd)
1020 yo._meta.encoder = codecs.getencoder(sd)
1021
1029 if yo._read_only:
1030 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename
1031 elif yo._meta_only:
1032 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename
1033 else:
1034 return __name__ + ".Table('%s')" % yo._meta.filename
1036 if yo._read_only:
1037 status = "read-only"
1038 elif yo._meta_only:
1039 status = "meta-only"
1040 else:
1041 status = "read/write"
1042 str = """
1043 Table: %s
1044 Type: %s
1045 Codepage: %s
1046 Status: %s
1047 Last updated: %s
1048 Record count: %d
1049 Field count: %d
1050 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version,
1051 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status,
1052 yo.last_update, len(yo), yo.field_count, yo.record_length)
1053 str += "\n --Fields--\n"
1054 for i in range(len(yo._meta.fields)):
1055 str += "%11d) %s\n" % (i, yo._fieldLayout(i))
1056 return str
1057 @property
1059 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1060 @codepage.setter
1061 - def codepage(yo, cp):
1062 cp = code_pages[yo._meta.header.codepage(cp)][0]
1063 yo._meta.decoder = codecs.getdecoder(cp)
1064 yo._meta.encoder = codecs.getencoder(cp)
1065 yo._update_disk(headeronly=True)
1066 @property
1068 "the number of fields in the table"
1069 return yo._meta.header.field_count
1070 @property
1072 "a list of the fields in the table"
1073 return yo._meta.fields[:]
1074 @property
1076 "table's file name, including path (if specified on open)"
1077 return yo._meta.filename
1078 @property
1080 "date of last update"
1081 return yo._meta.header.update
1082 @property
1084 "table's memo name (if path included in filename on open)"
1085 return yo._meta.memoname
1086 @property
1088 "number of bytes in a record"
1089 return yo._meta.header.record_length
1090 @property
1092 "index number of the current record"
1093 return yo._meta.current
1094 @property
1098 @property
1100 "process or ignore deleted records"
1101 return yo._use_deleted
1102 @use_deleted.setter
1105 @property
1107 "returns the dbf type of the table"
1108 return yo._version
1110 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]
1111 backup table is created with _backup appended to name
1112 then modifies current structure"""
1113 all_records = [record for record in yo]
1114 if yo:
1115 yo.create_backup()
1116 yo._meta.blankrecord = None
1117 meta = yo._meta
1118 offset = meta.header.record_length
1119 fields = yo._list_fields(field_specs, sep=';')
1120 for field in fields:
1121 try:
1122 name, format = field.split()
1123 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum():
1124 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name)
1125 name = name.lower()
1126 if name in meta.fields:
1127 raise DbfError("Field '%s' already exists" % name)
1128 field_type = format[0].upper()
1129 if len(name) > 10:
1130 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name)))
1131 if not field_type in meta.fieldtypes.keys():
1132 raise DbfError("Unknown field type: %s" % field_type)
1133 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format)
1134 except ValueError:
1135 raise DbfError("invalid field specifier: %s (multiple fields should be separated with ';'" % field)
1136 start = offset
1137 end = offset + length
1138 offset = end
1139 meta.fields.append(name)
1140 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0}
1141 if meta[name]['type'] in yo._memotypes and meta.memo is None:
1142 meta.memo = yo._memoClass(meta)
1143 for record in yo:
1144 record[name] = meta.fieldtypes[field_type]['Blank']()
1145 yo._buildHeaderFields()
1146 yo._update_disk()
1147 - def append(yo, kamikaze='', drop=False, multiple=1):
1148 "adds <multiple> blank records, and fills fields with dict/tuple values if present"
1149 if not yo.field_count:
1150 raise DbfError("No fields defined, cannot append")
1151 empty_table = len(yo) == 0
1152 dictdata = False
1153 tupledata = False
1154 if not isinstance(kamikaze, _DbfRecord):
1155 if isinstance(kamikaze, dict):
1156 dictdata = kamikaze
1157 kamikaze = ''
1158 elif isinstance(kamikaze, tuple):
1159 tupledata = kamikaze
1160 kamikaze = ''
1161 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze)
1162 yo._table.append(newrecord)
1163 yo._meta.header.record_count += 1
1164 try:
1165 if dictdata:
1166 newrecord.gather_fields(dictdata, drop=drop)
1167 elif tupledata:
1168 for index, item in enumerate(tupledata):
1169 newrecord[index] = item
1170 elif kamikaze == str:
1171 for field in yo._meta.memofields:
1172 newrecord[field] = ''
1173 elif kamikaze:
1174 for field in yo._meta.memofields:
1175 newrecord[field] = kamikaze[field]
1176 newrecord.write_record()
1177 except Exception:
1178 yo._table.pop()
1179 yo._meta.header.record_count = yo._meta.header.record_count - 1
1180 yo._update_disk()
1181 raise
1182 multiple -= 1
1183 if multiple:
1184 data = newrecord._data
1185 single = yo._meta.header.record_count
1186 total = single + multiple
1187 while single < total:
1188 multi_record = _DbfRecord(single, yo._meta, kamikaze=data)
1189 yo._table.append(multi_record)
1190 for field in yo._meta.memofields:
1191 multi_record[field] = newrecord[field]
1192 single += 1
1193 multi_record.write_record()
1194 yo._meta.header.record_count = total
1195 yo._meta.current = yo._meta.header.record_count - 1
1196 newrecord = multi_record
1197 yo._update_disk(headeronly=True)
1198 if empty_table:
1199 yo._meta.current = 0
1200 return newrecord
1201 - def bof(yo, _move=False):
1216 - def bottom(yo, get_record=False):
1217 """sets record pointer to bottom of table
1218 if get_record, seeks to and returns last (non-deleted) record
1219 DbfError if table is empty
1220 Bof if all records deleted and use_deleted is False"""
1221 yo._meta.current = yo._meta.header.record_count
1222 if get_record:
1223 try:
1224 return yo.prev()
1225 except Bof:
1226 yo._meta.current = yo._meta.header.record_count
1227 raise Eof()
1228 - def close(yo, keep_table=False, keep_memos=False):
1229 """closes disk files
1230 ensures table data is available if keep_table
1231 ensures memo data is available if keep_memos"""
1232 yo._meta.inmemory = True
1233 if keep_table:
1234 replacement_table = []
1235 for record in yo._table:
1236 replacement_table.append(record)
1237 yo._table = replacement_table
1238 else:
1239 if yo._meta.ondisk:
1240 yo._meta_only = True
1241 if yo._meta.mfd is not None:
1242 if not keep_memos:
1243 yo._meta.ignorememos = True
1244 else:
1245 memo_fields = []
1246 for field in yo.field_names:
1247 if yo.is_memotype(field):
1248 memo_fields.append(field)
1249 for record in yo:
1250 for field in memo_fields:
1251 record[field] = record[field]
1252 yo._meta.mfd.close()
1253 yo._meta.mfd = None
1254 if yo._meta.ondisk:
1255 yo._meta.dfd.close()
1256 yo._meta.dfd = None
1257 if keep_table:
1258 yo._read_only = True
1259 yo._meta.ondisk = False
1261 "creates a backup table -- ignored if memory table"
1262 if yo.filename[0] == yo.filename[-1] == ':':
1263 return
1264 if new_name is None:
1265 upper = yo.filename.isupper()
1266 name, ext = os.path.splitext(os.path.split(yo.filename)[1])
1267 extra = '_BACKUP' if upper else '_backup'
1268 new_name = os.path.join(temp_dir, name + extra + ext)
1269 else:
1270 overwrite = True
1271 if overwrite or not yo.backup:
1272 bkup = open(new_name, 'wb')
1273 try:
1274 yo._meta.dfd.seek(0)
1275 copyfileobj(yo._meta.dfd, bkup)
1276 yo.backup = new_name
1277 finally:
1278 bkup.close()
1282 "returns current logical record, or its index"
1283 if yo._meta.current < 0:
1284 raise Bof()
1285 elif yo._meta.current >= yo._meta.header.record_count:
1286 raise Eof()
1287 if index:
1288 return yo._meta.current
1289 return yo._table[yo._meta.current]
1291 """removes field(s) from the table
1292 creates backup files with _backup appended to the file name,
1293 then modifies current structure"""
1294 doomed = yo._list_fields(doomed)
1295 for victim in doomed:
1296 if victim not in yo._meta.fields:
1297 raise DbfError("field %s not in table -- delete aborted" % victim)
1298 all_records = [record for record in yo]
1299 yo.create_backup()
1300 for victim in doomed:
1301 yo._meta.fields.pop(yo._meta.fields.index(victim))
1302 start = yo._meta[victim]['start']
1303 end = yo._meta[victim]['end']
1304 for record in yo:
1305 record._data = record._data[:start] + record._data[end:]
1306 for field in yo._meta.fields:
1307 if yo._meta[field]['start'] == end:
1308 end = yo._meta[field]['end']
1309 yo._meta[field]['start'] = start
1310 yo._meta[field]['end'] = start + yo._meta[field]['length']
1311 start = yo._meta[field]['end']
1312 yo._buildHeaderFields()
1313 yo._update_disk()
1314 - def eof(yo, _move=False):
1329 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1330 """writes the table using CSV or tab-delimited format, using the filename
1331 given if specified, otherwise the table name"""
1332 if filename is not None:
1333 path, filename = os.path.split(filename)
1334 else:
1335 path, filename = os.path.split(yo.filename)
1336 filename = os.path.join(path, filename)
1337 field_specs = yo._list_fields(field_specs)
1338 if records is None:
1339 records = yo
1340 format = format.lower()
1341 if format not in ('csv', 'tab', 'fixed'):
1342 raise DbfError("export format: csv, tab, or fixed -- not %s" % format)
1343 if format == 'fixed':
1344 format = 'txt'
1345 base, ext = os.path.splitext(filename)
1346 if ext.lower() in ('', '.dbf'):
1347 filename = base + "." + format[:3]
1348 fd = open(filename, 'w')
1349 try:
1350 if format == 'csv':
1351 csvfile = csv.writer(fd, dialect='dbf')
1352 if header:
1353 csvfile.writerow(field_specs)
1354 for record in records:
1355 fields = []
1356 for fieldname in field_specs:
1357 fields.append(record[fieldname])
1358 csvfile.writerow(fields)
1359 elif format == 'tab':
1360 if header:
1361 fd.write('\t'.join(field_specs) + '\n')
1362 for record in records:
1363 fields = []
1364 for fieldname in field_specs:
1365 fields.append(str(record[fieldname]))
1366 fd.write('\t'.join(fields) + '\n')
1367 else:
1368 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w')
1369 header.write("%-15s Size\n" % "Field Name")
1370 header.write("%-15s ----\n" % ("-" * 15))
1371 sizes = []
1372 for field in field_specs:
1373 size = yo.size(field)[0]
1374 sizes.append(size)
1375 header.write("%-15s %3d\n" % (field, size))
1376 header.write('\nTotal Records in file: %d\n' % len(records))
1377 header.close()
1378 for record in records:
1379 fields = []
1380 for i, field_name in enumerate(field_specs):
1381 fields.append("%-*s" % (sizes[i], record[field_name]))
1382 fd.write(''.join(fields) + '\n')
1383 finally:
1384 fd.close()
1385 fd = None
1386 return len(records)
1387 - def find(yo, command):
1388 "uses exec to perform queries on the table"
1389 possible = List(desc="%s --> %s" % (yo.filename, command), field_names=yo.field_names)
1390 yo._dbflists.add(possible)
1391 result = {}
1392 select = 'result["keep"] = %s' % command
1393 g = {}
1394 use_deleted = yo.use_deleted
1395 for record in yo:
1396 result['keep'] = False
1397 g['result'] = result
1398 exec select in g, record
1399 if result['keep']:
1400 possible.append(record)
1401 record.write_record()
1402 return possible
1404 "returns record at physical_index[recno]"
1405 return yo._table[recno]
1406 - def goto(yo, criteria):
1407 """changes the record pointer to the first matching (non-deleted) record
1408 criteria should be either a tuple of tuple(value, field, func) triples,
1409 or an integer to go to"""
1410 if isinstance(criteria, int):
1411 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count:
1412 raise IndexError("Record %d does not exist" % criteria)
1413 if criteria < 0:
1414 criteria += yo._meta.header.record_count
1415 yo._meta.current = criteria
1416 return yo.current()
1417 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop])
1418 specs = tuple([(field, func) for value, field, func in criteria])
1419 match = tuple([value for value, field, func in criteria])
1420 current = yo.current(index=True)
1421 matchlen = len(match)
1422 while not yo.Eof():
1423 record = yo.current()
1424 results = record(*specs)
1425 if results == match:
1426 return record
1427 return yo.goto(current)
1429 "returns True if name is a variable-length field type"
1430 return yo._meta[name]['type'] in yo._decimal_fields
1432 "returns True if name is a memo type field"
1433 return yo._meta[name]['type'] in yo._memotypes
1434 - def new(yo, filename, field_specs=None, codepage=None):
1448 "set record pointer to next (non-deleted) record, and return it"
1449 if yo.eof(_move=True):
1450 raise Eof()
1451 return yo.current()
1453 meta = yo._meta
1454 meta.inmemory = False
1455 meta.ondisk = True
1456 yo._read_only = False
1457 yo._meta_only = False
1458 if '_table' in dir(yo):
1459 del yo._table
1460 dfd = meta.dfd = open(meta.filename, 'r+b')
1461 dfd.seek(0)
1462 meta.header = header = yo._TableHeader(dfd.read(32))
1463 if not header.version in yo._supported_tables:
1464 dfd.close()
1465 dfd = None
1466 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
1467 cp, sd, ld = _codepage_lookup(meta.header.codepage())
1468 meta.decoder = codecs.getdecoder(sd)
1469 meta.encoder = codecs.getencoder(sd)
1470 fieldblock = dfd.read(header.start - 32)
1471 for i in range(len(fieldblock)//32+1):
1472 fieldend = i * 32
1473 if fieldblock[fieldend] == '\x0d':
1474 break
1475 else:
1476 raise DbfError("corrupt field structure in header")
1477 if len(fieldblock[:fieldend]) % 32 != 0:
1478 raise DbfError("corrupt field structure in header")
1479 header.fields = fieldblock[:fieldend]
1480 header.extra = fieldblock[fieldend+1:]
1481 yo._initializeFields()
1482 yo._checkMemoIntegrity()
1483 meta.current = -1
1484 if len(yo) > 0:
1485 meta.current = 0
1486 dfd.seek(0)
1487
1488 - def pack(yo, _pack=True):
1489 "physically removes all deleted records"
1490 for dbfindex in yo._indexen:
1491 dbfindex.clear()
1492 newtable = []
1493 index = 0
1494 offset = 0
1495 for record in yo._table:
1496 found = False
1497 if record.has_been_deleted and _pack:
1498 for dbflist in yo._dbflists:
1499 if dbflist._purge(record, record.record_number - offset, 1):
1500 found = True
1501 record._recnum = -1
1502 else:
1503 record._recnum = index
1504 newtable.append(record)
1505 index += 1
1506 if found:
1507 offset += 1
1508 found = False
1509 yo._table.clear()
1510 for record in newtable:
1511 yo._table.append(record)
1512 yo._meta.header.record_count = index
1513 yo._current = -1
1514 yo._update_disk()
1515 yo.reindex()
1517 "set record pointer to previous (non-deleted) record, and return it"
1518 if yo.bof(_move=True):
1519 raise Bof
1520 return yo.current()
1521 - def query(yo, sql_command=None, python=None):
1522 "deprecated: use .find or .sql"
1523 if sql_command:
1524 return yo.sql(sql_command)
1525 elif python:
1526 return yo.find(python)
1527 raise DbfError("query: python parameter must be specified")
1529 for dbfindex in yo._indexen:
1530 dbfindex.reindex()
1532 "renames an existing field"
1533 if yo:
1534 yo.create_backup()
1535 if not oldname in yo._meta.fields:
1536 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname)
1537 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum():
1538 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits")
1539 newname = newname.lower()
1540 if newname in yo._meta.fields:
1541 raise DbfError("field --%s-- already exists" % newname)
1542 if len(newname) > 10:
1543 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname)))
1544 yo._meta[newname] = yo._meta[oldname]
1545 yo._meta.fields[yo._meta.fields.index(oldname)] = newname
1546 yo._buildHeaderFields()
1547 yo._update_disk(headeronly=True)
1549 """resizes field (C only at this time)
1550 creates backup file, then modifies current structure"""
1551 if not 0 < new_size < 256:
1552 raise DbfError("new_size must be between 1 and 255 (use delete_fields to remove a field)")
1553 doomed = yo._list_fields(doomed)
1554 for victim in doomed:
1555 if victim not in yo._meta.fields:
1556 raise DbfError("field %s not in table -- resize aborted" % victim)
1557 all_records = [record for record in yo]
1558 yo.create_backup()
1559 for victim in doomed:
1560 start = yo._meta[victim]['start']
1561 end = yo._meta[victim]['end']
1562 eff_end = min(yo._meta[victim]['length'], new_size)
1563 yo._meta[victim]['length'] = new_size
1564 yo._meta[victim]['end'] = start + new_size
1565 blank = array('c', ' ' * new_size)
1566 for record in yo:
1567 new_data = blank[:]
1568 new_data[:eff_end] = record._data[start:start+eff_end]
1569 record._data = record._data[:start] + new_data + record._data[end:]
1570 for field in yo._meta.fields:
1571 if yo._meta[field]['start'] == end:
1572 end = yo._meta[field]['end']
1573 yo._meta[field]['start'] = start + new_size
1574 yo._meta[field]['end'] = start + new_size + yo._meta[field]['length']
1575 start = yo._meta[field]['end']
1576 yo._buildHeaderFields()
1577 yo._update_disk()
1578 - def size(yo, field):
1579 "returns size of field as a tuple of (length, decimals)"
1580 if field in yo:
1581 return (yo._meta[field]['length'], yo._meta[field]['decimals'])
1582 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1583 - def sql(yo, command):
1584 "passes calls through to module level sql function"
1585 return sql(yo, command)
1587 """return list of fields suitable for creating same table layout
1588 @param fields: list of fields or None for all fields"""
1589 field_specs = []
1590 fields = yo._list_fields(fields)
1591 try:
1592 for name in fields:
1593 field_specs.append(yo._fieldLayout(yo.field_names.index(name)))
1594 except ValueError:
1595 raise DbfError("field --%s-- does not exist" % name)
1596 return field_specs
1597 - def top(yo, get_record=False):
1598 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record
1599 DbfError if table is empty
1600 Eof if all records are deleted and use_deleted is False"""
1601 yo._meta.current = -1
1602 if get_record:
1603 try:
1604 return yo.next()
1605 except Eof:
1606 yo._meta.current = -1
1607 raise Bof()
1608 - def type(yo, field):
1609 "returns type of field"
1610 if field in yo:
1611 return yo._meta[field]['type']
1612 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1613 - def zap(yo, areyousure=False):
1614 """removes all records from table -- this cannot be undone!
1615 areyousure must be True, else error is raised"""
1616 if areyousure:
1617 if yo._meta.inmemory:
1618 yo._table = []
1619 else:
1620 yo._table.clear()
1621 yo._meta.header.record_count = 0
1622 yo._current = -1
1623 yo._update_disk()
1624 else:
1625 raise DbfError("You must say you are sure to wipe the table")
1627 """Provides an interface for working with dBase III tables."""
1628 _version = 'dBase III Plus'
1629 _versionabbv = 'db3'
1630 _fieldtypes = {
1631 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1632 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1633 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1634 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1635 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric, 'Class':None} }
1636 _memoext = '.dbt'
1637 _memotypes = ('M',)
1638 _memoClass = _Db3Memo
1639 _yesMemoMask = '\x80'
1640 _noMemoMask = '\x7f'
1641 _fixed_fields = ('D','L','M')
1642 _variable_fields = ('C','N')
1643 _character_fields = ('C','M')
1644 _decimal_fields = ('N',)
1645 _numeric_fields = ('N',)
1646 _currency_fields = tuple()
1647 _dbfTableHeader = array('c', '\x00' * 32)
1648 _dbfTableHeader[0] = '\x03'
1649 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
1650 _dbfTableHeader[10] = '\x01'
1651 _dbfTableHeader[29] = '\x03'
1652 _dbfTableHeader = _dbfTableHeader.tostring()
1653 _dbfTableHeaderExtra = ''
1654 _supported_tables = ['\x03', '\x83']
1655 _read_only = False
1656 _meta_only = False
1657 _use_deleted = True
1659 "dBase III specific"
1660 if yo._meta.header.version == '\x83':
1661 try:
1662 yo._meta.memo = yo._memoClass(yo._meta)
1663 except:
1664 yo._meta.dfd.close()
1665 yo._meta.dfd = None
1666 raise
1667 if not yo._meta.ignorememos:
1668 for field in yo._meta.fields:
1669 if yo._meta[field]['type'] in yo._memotypes:
1670 if yo._meta.header.version != '\x83':
1671 yo._meta.dfd.close()
1672 yo._meta.dfd = None
1673 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
1674 elif not os.path.exists(yo._meta.memoname):
1675 yo._meta.dfd.close()
1676 yo._meta.dfd = None
1677 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1678 break
1680 "builds the FieldList of names, types, and descriptions"
1681 yo._meta.fields[:] = []
1682 offset = 1
1683 fieldsdef = yo._meta.header.fields
1684 if len(fieldsdef) % 32 != 0:
1685 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1686 if len(fieldsdef) // 32 != yo.field_count:
1687 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1688 for i in range(yo.field_count):
1689 fieldblock = fieldsdef[i*32:(i+1)*32]
1690 name = io.unpackStr(fieldblock[:11])
1691 type = fieldblock[11]
1692 if not type in yo._meta.fieldtypes:
1693 raise DbfError("Unknown field type: %s" % type)
1694 start = offset
1695 length = ord(fieldblock[16])
1696 offset += length
1697 end = start + length
1698 decimals = ord(fieldblock[17])
1699 flags = ord(fieldblock[18])
1700 yo._meta.fields.append(name)
1701 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1703 'Provides an interface for working with FoxPro 2 tables'
1704 _version = 'Foxpro'
1705 _versionabbv = 'fp'
1706 _fieldtypes = {
1707 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1708 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None},
1709 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None},
1710 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1711 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1712 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1713 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1714 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1715 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} }
1716 _memoext = '.fpt'
1717 _memotypes = ('G','M','P')
1718 _memoClass = _VfpMemo
1719 _yesMemoMask = '\xf5'
1720 _noMemoMask = '\x03'
1721 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1722 _variable_fields = ('C','F','N')
1723 _character_fields = ('C','M')
1724 _decimal_fields = ('F','N')
1725 _numeric_fields = ('F','N')
1726 _currency_fields = tuple()
1727 _supported_tables = ('\x03', '\xf5')
1728 _dbfTableHeader = array('c', '\x00' * 32)
1729 _dbfTableHeader[0] = '\x30'
1730 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1731 _dbfTableHeader[10] = '\x01'
1732 _dbfTableHeader[29] = '\x03'
1733 _dbfTableHeader = _dbfTableHeader.tostring()
1734 _dbfTableHeaderExtra = '\x00' * 263
1735 _use_deleted = True
1737 if os.path.exists(yo._meta.memoname):
1738 try:
1739 yo._meta.memo = yo._memoClass(yo._meta)
1740 except:
1741 yo._meta.dfd.close()
1742 yo._meta.dfd = None
1743 raise
1744 if not yo._meta.ignorememos:
1745 for field in yo._meta.fields:
1746 if yo._meta[field]['type'] in yo._memotypes:
1747 if not os.path.exists(yo._meta.memoname):
1748 yo._meta.dfd.close()
1749 yo._meta.dfd = None
1750 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1751 break
1753 "builds the FieldList of names, types, and descriptions"
1754 yo._meta.fields[:] = []
1755 offset = 1
1756 fieldsdef = yo._meta.header.fields
1757 if len(fieldsdef) % 32 != 0:
1758 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1759 if len(fieldsdef) // 32 != yo.field_count:
1760 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1761 for i in range(yo.field_count):
1762 fieldblock = fieldsdef[i*32:(i+1)*32]
1763 name = io.unpackStr(fieldblock[:11])
1764 type = fieldblock[11]
1765 if not type in yo._meta.fieldtypes:
1766 raise DbfError("Unknown field type: %s" % type)
1767 elif type == '0':
1768 return
1769 start = offset
1770 length = ord(fieldblock[16])
1771 offset += length
1772 end = start + length
1773 decimals = ord(fieldblock[17])
1774 flags = ord(fieldblock[18])
1775 yo._meta.fields.append(name)
1776 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1777
1779 'Provides an interface for working with Visual FoxPro 6 tables'
1780 _version = 'Visual Foxpro v6'
1781 _versionabbv = 'vfp'
1782 _fieldtypes = {
1783 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1784 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency, 'Class':None},
1785 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble, 'Class':None},
1786 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None},
1787 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None},
1788 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger, 'Class':None},
1789 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1790 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1791 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime, 'Class':None},
1792 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1793 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1794 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1795 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} }
1796 _memoext = '.fpt'
1797 _memotypes = ('G','M','P')
1798 _memoClass = _VfpMemo
1799 _yesMemoMask = '\x30'
1800 _noMemoMask = '\x30'
1801 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1802 _variable_fields = ('C','F','N')
1803 _character_fields = ('C','M')
1804 _decimal_fields = ('F','N')
1805 _numeric_fields = ('B','F','I','N','Y')
1806 _currency_fields = ('Y',)
1807 _supported_tables = ('\x30',)
1808 _dbfTableHeader = array('c', '\x00' * 32)
1809 _dbfTableHeader[0] = '\x30'
1810 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1811 _dbfTableHeader[10] = '\x01'
1812 _dbfTableHeader[29] = '\x03'
1813 _dbfTableHeader = _dbfTableHeader.tostring()
1814 _dbfTableHeaderExtra = '\x00' * 263
1815 _use_deleted = True
1817 if os.path.exists(yo._meta.memoname):
1818 try:
1819 yo._meta.memo = yo._memoClass(yo._meta)
1820 except:
1821 yo._meta.dfd.close()
1822 yo._meta.dfd = None
1823 raise
1824 if not yo._meta.ignorememos:
1825 for field in yo._meta.fields:
1826 if yo._meta[field]['type'] in yo._memotypes:
1827 if not os.path.exists(yo._meta.memoname):
1828 yo._meta.dfd.close()
1829 yo._meta.dfd = None
1830 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1831 break
1833 "builds the FieldList of names, types, and descriptions"
1834 yo._meta.fields[:] = []
1835 offset = 1
1836 fieldsdef = yo._meta.header.fields
1837 for i in range(yo.field_count):
1838 fieldblock = fieldsdef[i*32:(i+1)*32]
1839 name = io.unpackStr(fieldblock[:11])
1840 type = fieldblock[11]
1841 if not type in yo._meta.fieldtypes:
1842 raise DbfError("Unknown field type: %s" % type)
1843 elif type == '0':
1844 return
1845 start = io.unpackLongInt(fieldblock[12:16])
1846 length = ord(fieldblock[16])
1847 offset += length
1848 end = start + length
1849 decimals = ord(fieldblock[17])
1850 flags = ord(fieldblock[18])
1851 yo._meta.fields.append(name)
1852 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1853 -class List(object):
1854 "list of Dbf records, with set-like behavior"
1855 _desc = ''
1856 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1857 yo.field_names = field_names
1858 yo._list = []
1859 yo._set = set()
1860 if key is not None:
1861 yo.key = key
1862 if key.__doc__ is None:
1863 key.__doc__ = 'unknown'
1864 key = yo.key
1865 yo._current = -1
1866 if isinstance(new_records, yo.__class__) and key is new_records.key:
1867 yo._list = new_records._list[:]
1868 yo._set = new_records._set.copy()
1869 yo._current = 0
1870 elif new_records is not None:
1871 for record in new_records:
1872 value = key(record)
1873 item = (record.record_table, record.record_number, value)
1874 if value not in yo._set:
1875 yo._set.add(value)
1876 yo._list.append(item)
1877 yo._current = 0
1878 if desc is not None:
1879 yo._desc = desc
1881 key = yo.key
1882 if isinstance(other, (DbfTable, list)):
1883 other = yo.__class__(other, key=key)
1884 if isinstance(other, yo.__class__):
1885 result = yo.__class__()
1886 result._set = yo._set.copy()
1887 result._list[:] = yo._list[:]
1888 result.key = yo.key
1889 if key is other.key:
1890 for item in other._list:
1891 if item[2] not in result._set:
1892 result._set.add(item[2])
1893 result._list.append(item)
1894 else:
1895 for rec in other:
1896 value = key(rec)
1897 if value not in result._set:
1898 result._set.add(value)
1899 result._list.append((rec.record_table, rec.record_number, value))
1900 result._current = 0 if result else -1
1901 return result
1902 return NotImplemented
1904 if isinstance(record, tuple):
1905 item = record
1906 else:
1907 item = yo.key(record)
1908 return item in yo._set
1910 if isinstance(key, int):
1911 item = yo._list.pop[key]
1912 yo._set.remove(item[2])
1913 elif isinstance(key, slice):
1914 yo._set.difference_update([item[2] for item in yo._list[key]])
1915 yo._list.__delitem__(key)
1916 elif isinstance(key, _DbfRecord):
1917 index = yo.index(key)
1918 item = yo._list.pop[index]
1919 yo._set.remove(item[2])
1920 else:
1921 raise TypeError
1923 if isinstance(key, int):
1924 count = len(yo._list)
1925 if not -count <= key < count:
1926 raise IndexError("Record %d is not in list." % key)
1927 return yo._get_record(*yo._list[key])
1928 elif isinstance(key, slice):
1929 result = yo.__class__()
1930 result._list[:] = yo._list[key]
1931 result._set = set(result._list)
1932 result.key = yo.key
1933 result._current = 0 if result else -1
1934 return result
1935 elif isinstance(key, _DbfRecord):
1936 index = yo.index(key)
1937 return yo._get_record(*yo._list[index])
1938 else:
1939 raise TypeError('indices must be integers')
1941 return (table.get_record(recno) for table, recno, value in yo._list)
1943 return len(yo._list)
1949 if yo._desc:
1950 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list))
1951 else:
1952 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1954 key = yo.key
1955 if isinstance(other, (DbfTable, list)):
1956 other = yo.__class__(other, key=key)
1957 if isinstance(other, yo.__class__):
1958 result = yo.__class__()
1959 result._list[:] = other._list[:]
1960 result._set = other._set.copy()
1961 result.key = key
1962 lost = set()
1963 if key is other.key:
1964 for item in yo._list:
1965 if item[2] in result._list:
1966 result._set.remove(item[2])
1967 lost.add(item)
1968 else:
1969 for rec in other:
1970 value = key(rec)
1971 if value in result._set:
1972 result._set.remove(value)
1973 lost.add((rec.record_table, rec.record_number, value))
1974 result._list = [item for item in result._list if item not in lost]
1975 result._current = 0 if result else -1
1976 return result
1977 return NotImplemented
1979 key = yo.key
1980 if isinstance(other, (DbfTable, list)):
1981 other = yo.__class__(other, key=key)
1982 if isinstance(other, yo.__class__):
1983 result = yo.__class__()
1984 result._list[:] = yo._list[:]
1985 result._set = yo._set.copy()
1986 result.key = key
1987 lost = set()
1988 if key is other.key:
1989 for item in other._list:
1990 if item[2] in result._set:
1991 result._set.remove(item[2])
1992 lost.add(item[2])
1993 else:
1994 for rec in other:
1995 value = key(rec)
1996 if value in result._set:
1997 result._set.remove(value)
1998 lost.add(value)
1999 result._list = [item for item in result._list if item[2] not in lost]
2000 result._current = 0 if result else -1
2001 return result
2002 return NotImplemented
2004 if item[2] not in yo._set:
2005 yo._set.add(item[2])
2006 yo._list.append(item)
2007 - def _get_record(yo, table=None, rec_no=None, value=None):
2008 if table is rec_no is None:
2009 table, rec_no, value = yo._list[yo._current]
2010 return table.get_record(rec_no)
2011 - def _purge(yo, record, old_record_number, offset):
2012 partial = record.record_table, old_record_number
2013 records = sorted(yo._list, key=lambda item: (item[0], item[1]))
2014 for item in records:
2015 if partial == item[:2]:
2016 found = True
2017 break
2018 elif partial[0] is item[0] and partial[1] < item[1]:
2019 found = False
2020 break
2021 else:
2022 found = False
2023 if found:
2024 yo._list.pop(yo._list.index(item))
2025 yo._set.remove(item[2])
2026 start = records.index(item) + found
2027 for item in records[start:]:
2028 if item[0] is not partial[0]:
2029 break
2030 i = yo._list.index(item)
2031 yo._set.remove(item[2])
2032 item = item[0], (item[1] - offset), item[2]
2033 yo._list[i] = item
2034 yo._set.add(item[2])
2035 return found
2040
2042 if yo._list:
2043 yo._current = len(yo._list) - 1
2044 return yo._get_record()
2045 raise DbfError("dbf.List is empty")
2047 yo._list = []
2048 yo._set = set()
2049 yo._current = -1
2051 if yo._current < 0:
2052 raise Bof()
2053 elif yo._current == len(yo._list):
2054 raise Eof()
2055 return yo._get_record()
2056 - def extend(yo, new_records):
2072 - def goto(yo, index_number):
2073 if yo._list:
2074 if 0 <= index_number <= len(yo._list):
2075 yo._current = index_number
2076 return yo._get_record()
2077 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list)))
2078 raise DbfError("dbf.List is empty")
2079 - def index(yo, sort=None, reverse=False):
2080 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'"
2081 if sort is None:
2082 results = []
2083 for field, func in yo._meta.index:
2084 results.append("%s(%s)" % (func.__name__, field))
2085 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed])
2086 yo._meta.index_reversed = reverse
2087 if sort == 'ORIGINAL':
2088 yo._index = range(yo._meta.header.record_count)
2089 yo._meta.index = 'ORIGINAL'
2090 if reverse:
2091 yo._index.reverse()
2092 return
2093 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop])
2094 yo._meta.index = tuple(new_sort)
2095 yo._meta.orderresults = [''] * len(yo)
2096 for record in yo:
2097 yo._meta.orderresults[record.record_number] = record()
2098 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2099 - def index(yo, record, start=None, stop=None):
2100 item = record.record_table, record.record_number, yo.key(record)
2101 key = yo.key(record)
2102 if start is None:
2103 start = 0
2104 if stop is None:
2105 stop = len(yo._list)
2106 for i in range(start, stop):
2107 if yo._list[i][2] == key:
2108 return i
2109 else:
2110 raise ValueError("dbf.List.index(x): <x=%r> not in list" % (key,))
2116 - def key(yo, record):
2120 if yo._current < len(yo._list):
2121 yo._current += 1
2122 if yo._current < len(yo._list):
2123 return yo._get_record()
2124 raise Eof()
2125 - def pop(yo, index=None):
2126 if index is None:
2127 table, recno, value = yo._list.pop()
2128 else:
2129 table, recno, value = yo._list.pop(index)
2130 yo._set.remove(value)
2131 return yo._get_record(table, recno, value)
2133 if yo._current >= 0:
2134 yo._current -= 1
2135 if yo._current > -1:
2136 return yo._get_record()
2137 raise Bof()
2145 if yo._list:
2146 yo._current = 0
2147 return yo._get_record()
2148 raise DbfError("dbf.List is empty")
2149 - def sort(yo, key=None, reverse=False):
2153
2165 "returns records using this index"
2167 yo.table = table
2168 yo.records = records
2169 yo.index = 0
2181 - def __init__(yo, table, key, field_names=None):
2182 yo._table = table
2183 yo._values = []
2184 yo._rec_by_val = []
2185 yo._records = {}
2186 yo.__doc__ = key.__doc__ or 'unknown'
2187 yo.key = key
2188 yo.field_names = field_names or table.field_names
2189 for record in table:
2190 value = key(record)
2191 if value is DoNotIndex:
2192 continue
2193 rec_num = record.record_number
2194 if not isinstance(value, tuple):
2195 value = (value, )
2196 vindex = bisect_right(yo._values, value)
2197 yo._values.insert(vindex, value)
2198 yo._rec_by_val.insert(vindex, rec_num)
2199 yo._records[rec_num] = value
2200 table._indexen.add(yo)
2202 rec_num = record.record_number
2203 if rec_num in yo._records:
2204 value = yo._records[rec_num]
2205 vindex = bisect_left(yo._values, value)
2206 yo._values.pop(vindex)
2207 yo._rec_by_val.pop(vindex)
2208 value = yo.key(record)
2209 if value is DoNotIndex:
2210 return
2211 if not isinstance(value, tuple):
2212 value = (value, )
2213 vindex = bisect_right(yo._values, value)
2214 yo._values.insert(vindex, value)
2215 yo._rec_by_val.insert(vindex, rec_num)
2216 yo._records[rec_num] = value
2218 if isinstance(match, _DbfRecord):
2219 if match.record_table is yo._table:
2220 return match.record_number in yo._records
2221 match = yo.key(match)
2222 elif not isinstance(match, tuple):
2223 match = (match, )
2224 return yo.find(match) != -1
2226 if isinstance(key, int):
2227 count = len(yo._values)
2228 if not -count <= key < count:
2229 raise IndexError("Record %d is not in list." % key)
2230 rec_num = yo._rec_by_val[key]
2231 return yo._table.get_record(rec_num)
2232 elif isinstance(key, slice):
2233 result = List(field_names=yo._table.field_names)
2234 yo._table._dbflists.add(result)
2235 start, stop, step = key.start, key.stop, key.step
2236 if start is None: start = 0
2237 if stop is None: stop = len(yo._rec_by_val)
2238 if step is None: step = 1
2239 for loc in range(start, stop, step):
2240 record = yo._table.get_record(yo._rec_by_val[loc])
2241 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2242 result._current = 0 if result else -1
2243 return result
2244 elif isinstance (key, (str, unicode, tuple, _DbfRecord)):
2245 if isinstance(key, _DbfRecord):
2246 key = yo.key(key)
2247 elif not isinstance(key, tuple):
2248 key = (key, )
2249 loc = yo.find(key)
2250 if loc == -1:
2251 raise KeyError(key)
2252 return yo._table.get_record(yo._rec_by_val[loc])
2253 else:
2254 raise TypeError('indices must be integers, match objects must by strings or tuples')
2258 yo._table.close()
2259 yo._values[:] = []
2260 yo._rec_by_val[:] = []
2261 yo._records.clear()
2262 return False
2266 return len(yo._records)
2268 target = target[:len(match)]
2269 if isinstance(match[-1], (str, unicode)):
2270 target = list(target)
2271 target[-1] = target[-1][:len(match[-1])]
2272 target = tuple(target)
2273 return target == match
2275 value = yo._records.get(rec_num)
2276 if value is not None:
2277 vindex = bisect_left(yo._values, value)
2278 del yo._records[rec_num]
2279 yo._values.pop(vindex)
2280 yo._rec_by_val.pop(vindex)
2281 - def _search(yo, match, lo=0, hi=None):
2282 if hi is None:
2283 hi = len(yo._values)
2284 return bisect_left(yo._values, match, lo, hi)
2286 "removes all entries from index"
2287 yo._values[:] = []
2288 yo._rec_by_val[:] = []
2289 yo._records.clear()
2292 - def find(yo, match, partial=False):
2293 "returns numeric index of (partial) match, or -1"
2294 if isinstance(match, _DbfRecord):
2295 if match.record_number in yo._records:
2296 return yo._values.index(yo._records[match.record_number])
2297 else:
2298 return -1
2299 if not isinstance(match, tuple):
2300 match = (match, )
2301 loc = yo._search(match)
2302 while loc < len(yo._values) and yo._values[loc] == match:
2303 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2304 loc += 1
2305 continue
2306 return loc
2307 if partial:
2308 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2309 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2310 loc += 1
2311 continue
2312 return loc
2313 return -1
2315 "returns numeric index of either (partial) match, or position of where match would be"
2316 if isinstance(match, _DbfRecord):
2317 if match.record_number in yo._records:
2318 return yo._values.index(yo._records[match.record_number])
2319 else:
2320 match = yo.key(match)
2321 if not isinstance(match, tuple):
2322 match = (match, )
2323 loc = yo._search(match)
2324 return loc
2325 - def index(yo, match, partial=False):
2326 "returns numeric index of (partial) match, or raises ValueError"
2327 loc = yo.find(match, partial)
2328 if loc == -1:
2329 if isinstance(match, _DbfRecord):
2330 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__))
2331 else:
2332 raise ValueError("match criteria <%s> not in index" % (match, ))
2333 return loc
2335 "reindexes all records"
2336 for record in yo._table:
2337 yo(record)
2338 - def query(yo, sql_command=None, python=None):
2339 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL"""
2340 if sql_command:
2341 return sql(yo, sql_command)
2342 elif python is None:
2343 raise DbfError("query: python parameter must be specified")
2344 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names)
2345 yo._table._dbflists.add(possible)
2346 query_result = {}
2347 select = 'query_result["keep"] = %s' % python
2348 g = {}
2349 for record in yo:
2350 query_result['keep'] = False
2351 g['query_result'] = query_result
2352 exec select in g, record
2353 if query_result['keep']:
2354 possible.append(record)
2355 record.write_record()
2356 return possible
2357 - def search(yo, match, partial=False):
2358 "returns dbf.List of all (partially) matching records"
2359 result = List(field_names=yo._table.field_names)
2360 yo._table._dbflists.add(result)
2361 if not isinstance(match, tuple):
2362 match = (match, )
2363 loc = yo._search(match)
2364 if loc == len(yo._values):
2365 return result
2366 while loc < len(yo._values) and yo._values[loc] == match:
2367 record = yo._table.get_record(yo._rec_by_val[loc])
2368 if not yo._table.use_deleted and record.has_been_deleted:
2369 loc += 1
2370 continue
2371 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2372 loc += 1
2373 if partial:
2374 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2375 record = yo._table.get_record(yo._rec_by_val[loc])
2376 if not yo._table.use_deleted and record.has_been_deleted:
2377 loc += 1
2378 continue
2379 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2380 loc += 1
2381 return result
2382
2383 csv.register_dialect('dbf', DbfCsv)
2384
2385 -def sql_select(records, chosen_fields, condition, field_names):
2386 if chosen_fields != '*':
2387 field_names = chosen_fields.replace(' ','').split(',')
2388 result = condition(records)
2389 result.modified = 0, 'record' + ('','s')[len(result)>1]
2390 result.field_names = field_names
2391 return result
2392
2393 -def sql_update(records, command, condition, field_names):
2394 possible = condition(records)
2395 modified = sql_cmd(command, field_names)(possible)
2396 possible.modified = modified, 'record' + ('','s')[modified>1]
2397 return possible
2398
2399 -def sql_delete(records, dead_fields, condition, field_names):
2400 deleted = condition(records)
2401 deleted.modified = len(deleted), 'record' + ('','s')[len(deleted)>1]
2402 deleted.field_names = field_names
2403 if dead_fields == '*':
2404 for record in deleted:
2405 record.delete_record()
2406 record.write_record()
2407 else:
2408 keep = [f for f in field_names if f not in dead_fields.replace(' ','').split(',')]
2409 for record in deleted:
2410 record.reset_record(keep_fields=keep)
2411 record.write_record()
2412 return deleted
2413
2414 -def sql_recall(records, all_fields, condition, field_names):
2435
2436 -def sql_add(records, new_fields, condition, field_names):
2446
2447 -def sql_drop(records, dead_fields, condition, field_names):
2457
2458 -def sql_pack(records, command, condition, field_names):
2468
2469 -def sql_resize(records, fieldname_newsize, condition, field_names):
2470 tables = set()
2471 possible = condition(records)
2472 for record in possible:
2473 tables.add(record.record_table)
2474 fieldname, newsize = fieldname_newsize.split()
2475 newsize = int(newsize)
2476 for table in tables:
2477 table.resize_field(fieldname, newsize)
2478 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1]
2479 possible.field_names = field_names
2480 return possible
2481
2482 sql_functions = {
2483 'select' : sql_select,
2484 'update' : sql_update,
2485 'replace': sql_update,
2486 'insert' : None,
2487 'delete' : sql_delete,
2488 'recall' : sql_recall,
2489 'add' : sql_add,
2490 'drop' : sql_drop,
2491 'count' : None,
2492 'pack' : sql_pack,
2493 'resize' : sql_resize,
2494 }
2497 "creates a function matching the sql criteria"
2498 function = """def func(records):
2499 \"\"\"%s\"\"\"
2500 matched = List(field_names=records[0].field_names)
2501 for rec in records:
2502 %s
2503
2504 if %s:
2505 matched.append(rec)
2506 return matched"""
2507 fields = []
2508 for field in records[0].field_names:
2509 if field in criteria:
2510 fields.append(field)
2511 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2512 g = dbf.sql_user_functions.copy()
2513 g['List'] = List
2514 function %= (criteria, fields, criteria)
2515
2516 exec function in g
2517 return g['func']
2518
2519 -def sql_cmd(command, field_names):
2520 "creates a function matching to apply command to each record in records"
2521 function = """def func(records):
2522 \"\"\"%s\"\"\"
2523 changed = 0
2524 for rec in records:
2525 %s
2526
2527 %s
2528
2529 %s
2530 changed += rec.write_record()
2531 return changed"""
2532 fields = []
2533 for field in field_names:
2534 if field in command:
2535 fields.append(field)
2536 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2537 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields])
2538 g = dbf.sql_user_functions.copy()
2539 if '=' not in command and ' with ' in command.lower():
2540 offset = command.lower().index(' with ')
2541 command = command[:offset] + ' = ' + command[offset+6:]
2542 function %= (command, pre_fields, command, post_fields)
2543
2544 exec function in g
2545 return g['func']
2546
2547 -def sql(records, command):
2548 """recognized sql commands are SELECT, UPDATE | REPLACE, DELETE, RECALL, ADD, DROP"""
2549 sql_command = command
2550 if ' where ' in command:
2551 command, condition = command.split(' where ', 1)
2552 condition = sql_criteria(records, condition)
2553 else:
2554 def condition(records):
2555 return records[:]
2556 name, command = command.split(' ', 1)
2557 command = command.strip()
2558 name = name.lower()
2559 field_names = records[0].field_names
2560 if sql_functions.get(name) is None:
2561 raise DbfError('unknown SQL command: %s' % name.upper())
2562 result = sql_functions[name](records, command, condition, field_names)
2563 tables = set()
2564 for record in result:
2565 tables.add(record.record_table)
2566 for list_table in tables:
2567 list_table._dbflists.add(result)
2568 return result
2570 "returns parameter unchanged"
2571 return value
2573 "ensures each tuple is the same length, using filler[-missing] for the gaps"
2574 final = []
2575 for t in tuples:
2576 if len(t) < length:
2577 final.append( tuple([item for item in t] + filler[len(t)-length:]) )
2578 else:
2579 final.append(t)
2580 return tuple(final)
2582 if cp not in code_pages:
2583 for code_page in sorted(code_pages.keys()):
2584 sd, ld = code_pages[code_page]
2585 if cp == sd or cp == ld:
2586 if sd is None:
2587 raise DbfError("Unsupported codepage: %s" % ld)
2588 cp = code_page
2589 break
2590 else:
2591 raise DbfError("Unsupported codepage: %s" % cp)
2592 sd, ld = code_pages[cp]
2593 return cp, sd, ld
2594 -def ascii(new_setting=None):
2601 -def codepage(cp=None):
2602 "get/set default codepage for any new tables"
2603 global default_codepage
2604 cp, sd, ld = _codepage_lookup(cp or default_codepage)
2605 default_codepage = sd
2606 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2614 version = 'dBase IV w/memos (non-functional)'
2615 _versionabbv = 'db4'
2616 _fieldtypes = {
2617 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
2618 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
2619 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
2620 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
2621 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
2622 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
2623 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
2624 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
2625 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
2626 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2627 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2628 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2629 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
2630 _memoext = '.dbt'
2631 _memotypes = ('G','M','P')
2632 _memoClass = _VfpMemo
2633 _yesMemoMask = '\x8b'
2634 _noMemoMask = '\x04'
2635 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
2636 _variable_fields = ('C','F','N')
2637 _character_fields = ('C','M')
2638 _decimal_fields = ('F','N')
2639 _numeric_fields = ('B','F','I','N','Y')
2640 _currency_fields = ('Y',)
2641 _supported_tables = ('\x04', '\x8b')
2642 _dbfTableHeader = ['\x00'] * 32
2643 _dbfTableHeader[0] = '\x8b'
2644 _dbfTableHeader[10] = '\x01'
2645 _dbfTableHeader[29] = '\x03'
2646 _dbfTableHeader = ''.join(_dbfTableHeader)
2647 _dbfTableHeaderExtra = ''
2648 _use_deleted = True
2650 "dBase III specific"
2651 if yo._meta.header.version == '\x8b':
2652 try:
2653 yo._meta.memo = yo._memoClass(yo._meta)
2654 except:
2655 yo._meta.dfd.close()
2656 yo._meta.dfd = None
2657 raise
2658 if not yo._meta.ignorememos:
2659 for field in yo._meta.fields:
2660 if yo._meta[field]['type'] in yo._memotypes:
2661 if yo._meta.header.version != '\x8b':
2662 yo._meta.dfd.close()
2663 yo._meta.dfd = None
2664 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
2665 elif not os.path.exists(yo._meta.memoname):
2666 yo._meta.dfd.close()
2667 yo._meta.dfd = None
2668 raise DbfError("Table structure corrupt: memo fields exist without memo file")
2669 break
2670