Coverage for /usr/lib/python3/dist-packages/fontTools/ttLib/tables/otBase.py: 20%
918 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-14 15:55 +0200
1from fontTools.config import OPTIONS
2from fontTools.misc.textTools import Tag, bytesjoin
3from .DefaultTable import DefaultTable
4from enum import IntEnum
5import sys
6import array
7import struct
8import logging
9from functools import lru_cache
10from typing import Iterator, NamedTuple, Optional, Tuple
12log = logging.getLogger(__name__)
14have_uharfbuzz = False
15try:
16 import uharfbuzz as hb
18 # repack method added in uharfbuzz >= 0.23; if uharfbuzz *can* be
19 # imported but repack method is missing, behave as if uharfbuzz
20 # is not available (fallback to the slower Python implementation)
21 have_uharfbuzz = callable(getattr(hb, "repack", None))
22except ImportError:
23 pass
25USE_HARFBUZZ_REPACKER = OPTIONS[f"{__name__}:USE_HARFBUZZ_REPACKER"]
28class OverflowErrorRecord(object):
29 def __init__(self, overflowTuple):
30 self.tableType = overflowTuple[0]
31 self.LookupListIndex = overflowTuple[1]
32 self.SubTableIndex = overflowTuple[2]
33 self.itemName = overflowTuple[3]
34 self.itemIndex = overflowTuple[4]
36 def __repr__(self):
37 return str(
38 (
39 self.tableType,
40 "LookupIndex:",
41 self.LookupListIndex,
42 "SubTableIndex:",
43 self.SubTableIndex,
44 "ItemName:",
45 self.itemName,
46 "ItemIndex:",
47 self.itemIndex,
48 )
49 )
52class OTLOffsetOverflowError(Exception):
53 def __init__(self, overflowErrorRecord):
54 self.value = overflowErrorRecord
56 def __str__(self):
57 return repr(self.value)
60class RepackerState(IntEnum):
61 # Repacking control flow is implemnted using a state machine. The state machine table:
62 #
63 # State | Packing Success | Packing Failed | Exception Raised |
64 # ------------+-----------------+----------------+------------------+
65 # PURE_FT | Return result | PURE_FT | Return failure |
66 # HB_FT | Return result | HB_FT | FT_FALLBACK |
67 # FT_FALLBACK | HB_FT | FT_FALLBACK | Return failure |
69 # Pack only with fontTools, don't allow sharing between extensions.
70 PURE_FT = 1
72 # Attempt to pack with harfbuzz (allowing sharing between extensions)
73 # use fontTools to attempt overflow resolution.
74 HB_FT = 2
76 # Fallback if HB/FT packing gets stuck. Pack only with fontTools, don't allow sharing between
77 # extensions.
78 FT_FALLBACK = 3
81class BaseTTXConverter(DefaultTable):
83 """Generic base class for TTX table converters. It functions as an
84 adapter between the TTX (ttLib actually) table model and the model
85 we use for OpenType tables, which is necessarily subtly different.
86 """
88 def decompile(self, data, font):
89 """Create an object from the binary data. Called automatically on access."""
90 from . import otTables
92 reader = OTTableReader(data, tableTag=self.tableTag)
93 tableClass = getattr(otTables, self.tableTag)
94 self.table = tableClass()
95 self.table.decompile(reader, font)
97 def compile(self, font):
98 """Compiles the table into binary. Called automatically on save."""
100 # General outline:
101 # Create a top-level OTTableWriter for the GPOS/GSUB table.
102 # Call the compile method for the the table
103 # for each 'converter' record in the table converter list
104 # call converter's write method for each item in the value.
105 # - For simple items, the write method adds a string to the
106 # writer's self.items list.
107 # - For Struct/Table/Subtable items, it add first adds new writer to the
108 # to the writer's self.items, then calls the item's compile method.
109 # This creates a tree of writers, rooted at the GUSB/GPOS writer, with
110 # each writer representing a table, and the writer.items list containing
111 # the child data strings and writers.
112 # call the getAllData method
113 # call _doneWriting, which removes duplicates
114 # call _gatherTables. This traverses the tables, adding unique occurences to a flat list of tables
115 # Traverse the flat list of tables, calling getDataLength on each to update their position
116 # Traverse the flat list of tables again, calling getData each get the data in the table, now that
117 # pos's and offset are known.
119 # If a lookup subtable overflows an offset, we have to start all over.
120 overflowRecord = None
121 # this is 3-state option: default (None) means automatically use hb.repack or
122 # silently fall back if it fails; True, use it and raise error if not possible
123 # or it errors out; False, don't use it, even if you can.
124 use_hb_repack = font.cfg[USE_HARFBUZZ_REPACKER]
125 if self.tableTag in ("GSUB", "GPOS"):
126 if use_hb_repack is False:
127 log.debug(
128 "hb.repack disabled, compiling '%s' with pure-python serializer",
129 self.tableTag,
130 )
131 elif not have_uharfbuzz:
132 if use_hb_repack is True:
133 raise ImportError("No module named 'uharfbuzz'")
134 else:
135 assert use_hb_repack is None
136 log.debug(
137 "uharfbuzz not found, compiling '%s' with pure-python serializer",
138 self.tableTag,
139 )
141 if (
142 use_hb_repack in (None, True)
143 and have_uharfbuzz
144 and self.tableTag in ("GSUB", "GPOS")
145 ):
146 state = RepackerState.HB_FT
147 else:
148 state = RepackerState.PURE_FT
150 hb_first_error_logged = False
151 lastOverflowRecord = None
152 while True:
153 try:
154 writer = OTTableWriter(tableTag=self.tableTag)
155 self.table.compile(writer, font)
156 if state == RepackerState.HB_FT:
157 return self.tryPackingHarfbuzz(writer, hb_first_error_logged)
158 elif state == RepackerState.PURE_FT:
159 return self.tryPackingFontTools(writer)
160 elif state == RepackerState.FT_FALLBACK:
161 # Run packing with FontTools only, but don't return the result as it will
162 # not be optimally packed. Once a successful packing has been found, state is
163 # changed back to harfbuzz packing to produce the final, optimal, packing.
164 self.tryPackingFontTools(writer)
165 log.debug(
166 "Re-enabling sharing between extensions and switching back to "
167 "harfbuzz+fontTools packing."
168 )
169 state = RepackerState.HB_FT
171 except OTLOffsetOverflowError as e:
172 hb_first_error_logged = True
173 ok = self.tryResolveOverflow(font, e, lastOverflowRecord)
174 lastOverflowRecord = e.value
176 if ok:
177 continue
179 if state is RepackerState.HB_FT:
180 log.debug(
181 "Harfbuzz packing out of resolutions, disabling sharing between extensions and "
182 "switching to fontTools only packing."
183 )
184 state = RepackerState.FT_FALLBACK
185 else:
186 raise
188 def tryPackingHarfbuzz(self, writer, hb_first_error_logged):
189 try:
190 log.debug("serializing '%s' with hb.repack", self.tableTag)
191 return writer.getAllDataUsingHarfbuzz(self.tableTag)
192 except (ValueError, MemoryError, hb.RepackerError) as e:
193 # Only log hb repacker errors the first time they occur in
194 # the offset-overflow resolution loop, they are just noisy.
195 # Maybe we can revisit this if/when uharfbuzz actually gives
196 # us more info as to why hb.repack failed...
197 if not hb_first_error_logged:
198 error_msg = f"{type(e).__name__}"
199 if str(e) != "":
200 error_msg += f": {e}"
201 log.warning(
202 "hb.repack failed to serialize '%s', attempting fonttools resolutions "
203 "; the error message was: %s",
204 self.tableTag,
205 error_msg,
206 )
207 hb_first_error_logged = True
208 return writer.getAllData(remove_duplicate=False)
210 def tryPackingFontTools(self, writer):
211 return writer.getAllData()
213 def tryResolveOverflow(self, font, e, lastOverflowRecord):
214 ok = 0
215 if lastOverflowRecord == e.value:
216 # Oh well...
217 return ok
219 overflowRecord = e.value
220 log.info("Attempting to fix OTLOffsetOverflowError %s", e)
222 if overflowRecord.itemName is None:
223 from .otTables import fixLookupOverFlows
225 ok = fixLookupOverFlows(font, overflowRecord)
226 else:
227 from .otTables import fixSubTableOverFlows
229 ok = fixSubTableOverFlows(font, overflowRecord)
231 if ok:
232 return ok
234 # Try upgrading lookup to Extension and hope
235 # that cross-lookup sharing not happening would
236 # fix overflow...
237 from .otTables import fixLookupOverFlows
239 return fixLookupOverFlows(font, overflowRecord)
241 def toXML(self, writer, font):
242 self.table.toXML2(writer, font)
244 def fromXML(self, name, attrs, content, font):
245 from . import otTables
247 if not hasattr(self, "table"):
248 tableClass = getattr(otTables, self.tableTag)
249 self.table = tableClass()
250 self.table.fromXML(name, attrs, content, font)
251 self.table.populateDefaults()
253 def ensureDecompiled(self, recurse=True):
254 self.table.ensureDecompiled(recurse=recurse)
257# https://github.com/fonttools/fonttools/pull/2285#issuecomment-834652928
258assert len(struct.pack("i", 0)) == 4
259assert array.array("i").itemsize == 4, "Oops, file a bug against fonttools."
262class OTTableReader(object):
264 """Helper class to retrieve data from an OpenType table."""
266 __slots__ = ("data", "offset", "pos", "localState", "tableTag")
268 def __init__(self, data, localState=None, offset=0, tableTag=None):
269 self.data = data
270 self.offset = offset
271 self.pos = offset
272 self.localState = localState
273 self.tableTag = tableTag
275 def advance(self, count):
276 self.pos += count
278 def seek(self, pos):
279 self.pos = pos
281 def copy(self):
282 other = self.__class__(self.data, self.localState, self.offset, self.tableTag)
283 other.pos = self.pos
284 return other
286 def getSubReader(self, offset):
287 offset = self.offset + offset
288 return self.__class__(self.data, self.localState, offset, self.tableTag)
290 def readValue(self, typecode, staticSize):
291 pos = self.pos
292 newpos = pos + staticSize
293 (value,) = struct.unpack(f">{typecode}", self.data[pos:newpos])
294 self.pos = newpos
295 return value
297 def readArray(self, typecode, staticSize, count):
298 pos = self.pos
299 newpos = pos + count * staticSize
300 value = array.array(typecode, self.data[pos:newpos])
301 if sys.byteorder != "big":
302 value.byteswap()
303 self.pos = newpos
304 return value.tolist()
306 def readInt8(self):
307 return self.readValue("b", staticSize=1)
309 def readInt8Array(self, count):
310 return self.readArray("b", staticSize=1, count=count)
312 def readShort(self):
313 return self.readValue("h", staticSize=2)
315 def readShortArray(self, count):
316 return self.readArray("h", staticSize=2, count=count)
318 def readLong(self):
319 return self.readValue("i", staticSize=4)
321 def readLongArray(self, count):
322 return self.readArray("i", staticSize=4, count=count)
324 def readUInt8(self):
325 return self.readValue("B", staticSize=1)
327 def readUInt8Array(self, count):
328 return self.readArray("B", staticSize=1, count=count)
330 def readUShort(self):
331 return self.readValue("H", staticSize=2)
333 def readUShortArray(self, count):
334 return self.readArray("H", staticSize=2, count=count)
336 def readULong(self):
337 return self.readValue("I", staticSize=4)
339 def readULongArray(self, count):
340 return self.readArray("I", staticSize=4, count=count)
342 def readUInt24(self):
343 pos = self.pos
344 newpos = pos + 3
345 (value,) = struct.unpack(">l", b"\0" + self.data[pos:newpos])
346 self.pos = newpos
347 return value
349 def readUInt24Array(self, count):
350 return [self.readUInt24() for _ in range(count)]
352 def readTag(self):
353 pos = self.pos
354 newpos = pos + 4
355 value = Tag(self.data[pos:newpos])
356 assert len(value) == 4, value
357 self.pos = newpos
358 return value
360 def readData(self, count):
361 pos = self.pos
362 newpos = pos + count
363 value = self.data[pos:newpos]
364 self.pos = newpos
365 return value
367 def __setitem__(self, name, value):
368 state = self.localState.copy() if self.localState else dict()
369 state[name] = value
370 self.localState = state
372 def __getitem__(self, name):
373 return self.localState and self.localState[name]
375 def __contains__(self, name):
376 return self.localState and name in self.localState
379class OffsetToWriter(object):
380 def __init__(self, subWriter, offsetSize):
381 self.subWriter = subWriter
382 self.offsetSize = offsetSize
384 def __eq__(self, other):
385 if type(self) != type(other):
386 return NotImplemented
387 return self.subWriter == other.subWriter and self.offsetSize == other.offsetSize
389 def __hash__(self):
390 # only works after self._doneWriting() has been called
391 return hash((self.subWriter, self.offsetSize))
394class OTTableWriter(object):
396 """Helper class to gather and assemble data for OpenType tables."""
398 def __init__(self, localState=None, tableTag=None):
399 self.items = []
400 self.pos = None
401 self.localState = localState
402 self.tableTag = tableTag
403 self.parent = None
405 def __setitem__(self, name, value):
406 state = self.localState.copy() if self.localState else dict()
407 state[name] = value
408 self.localState = state
410 def __getitem__(self, name):
411 return self.localState[name]
413 def __delitem__(self, name):
414 del self.localState[name]
416 # assembler interface
418 def getDataLength(self):
419 """Return the length of this table in bytes, without subtables."""
420 l = 0
421 for item in self.items:
422 if hasattr(item, "getCountData"):
423 l += item.size
424 elif hasattr(item, "subWriter"):
425 l += item.offsetSize
426 else:
427 l = l + len(item)
428 return l
430 def getData(self):
431 """Assemble the data for this writer/table, without subtables."""
432 items = list(self.items) # make a shallow copy
433 pos = self.pos
434 numItems = len(items)
435 for i in range(numItems):
436 item = items[i]
438 if hasattr(item, "subWriter"):
439 if item.offsetSize == 4:
440 items[i] = packULong(item.subWriter.pos - pos)
441 elif item.offsetSize == 2:
442 try:
443 items[i] = packUShort(item.subWriter.pos - pos)
444 except struct.error:
445 # provide data to fix overflow problem.
446 overflowErrorRecord = self.getOverflowErrorRecord(
447 item.subWriter
448 )
450 raise OTLOffsetOverflowError(overflowErrorRecord)
451 elif item.offsetSize == 3:
452 items[i] = packUInt24(item.subWriter.pos - pos)
453 else:
454 raise ValueError(item.offsetSize)
456 return bytesjoin(items)
458 def getDataForHarfbuzz(self):
459 """Assemble the data for this writer/table with all offset field set to 0"""
460 items = list(self.items)
461 packFuncs = {2: packUShort, 3: packUInt24, 4: packULong}
462 for i, item in enumerate(items):
463 if hasattr(item, "subWriter"):
464 # Offset value is not needed in harfbuzz repacker, so setting offset to 0 to avoid overflow here
465 if item.offsetSize in packFuncs:
466 items[i] = packFuncs[item.offsetSize](0)
467 else:
468 raise ValueError(item.offsetSize)
470 return bytesjoin(items)
472 def __hash__(self):
473 # only works after self._doneWriting() has been called
474 return hash(self.items)
476 def __ne__(self, other):
477 result = self.__eq__(other)
478 return result if result is NotImplemented else not result
480 def __eq__(self, other):
481 if type(self) != type(other):
482 return NotImplemented
483 return self.items == other.items
485 def _doneWriting(self, internedTables, shareExtension=False):
486 # Convert CountData references to data string items
487 # collapse duplicate table references to a unique entry
488 # "tables" are OTTableWriter objects.
490 # For Extension Lookup types, we can
491 # eliminate duplicates only within the tree under the Extension Lookup,
492 # as offsets may exceed 64K even between Extension LookupTable subtables.
493 isExtension = hasattr(self, "Extension")
495 # Certain versions of Uniscribe reject the font if the GSUB/GPOS top-level
496 # arrays (ScriptList, FeatureList, LookupList) point to the same, possibly
497 # empty, array. So, we don't share those.
498 # See: https://github.com/fonttools/fonttools/issues/518
499 dontShare = hasattr(self, "DontShare")
501 if isExtension and not shareExtension:
502 internedTables = {}
504 items = self.items
505 for i in range(len(items)):
506 item = items[i]
507 if hasattr(item, "getCountData"):
508 items[i] = item.getCountData()
509 elif hasattr(item, "subWriter"):
510 item.subWriter._doneWriting(
511 internedTables, shareExtension=shareExtension
512 )
513 # At this point, all subwriters are hashable based on their items.
514 # (See hash and comparison magic methods above.) So the ``setdefault``
515 # call here will return the first writer object we've seen with
516 # equal content, or store it in the dictionary if it's not been
517 # seen yet. We therefore replace the subwriter object with an equivalent
518 # object, which deduplicates the tree.
519 if not dontShare:
520 items[i].subWriter = internedTables.setdefault(
521 item.subWriter, item.subWriter
522 )
523 self.items = tuple(items)
525 def _gatherTables(self, tables, extTables, done):
526 # Convert table references in self.items tree to a flat
527 # list of tables in depth-first traversal order.
528 # "tables" are OTTableWriter objects.
529 # We do the traversal in reverse order at each level, in order to
530 # resolve duplicate references to be the last reference in the list of tables.
531 # For extension lookups, duplicate references can be merged only within the
532 # writer tree under the extension lookup.
534 done[id(self)] = True
536 numItems = len(self.items)
537 iRange = list(range(numItems))
538 iRange.reverse()
540 isExtension = hasattr(self, "Extension")
542 selfTables = tables
544 if isExtension:
545 assert (
546 extTables is not None
547 ), "Program or XML editing error. Extension subtables cannot contain extensions subtables"
548 tables, extTables, done = extTables, None, {}
550 # add Coverage table if it is sorted last.
551 sortCoverageLast = False
552 if hasattr(self, "sortCoverageLast"):
553 # Find coverage table
554 for i in range(numItems):
555 item = self.items[i]
556 if (
557 hasattr(item, "subWriter")
558 and getattr(item.subWriter, "name", None) == "Coverage"
559 ):
560 sortCoverageLast = True
561 break
562 if id(item.subWriter) not in done:
563 item.subWriter._gatherTables(tables, extTables, done)
564 else:
565 # We're a new parent of item
566 pass
568 for i in iRange:
569 item = self.items[i]
570 if not hasattr(item, "subWriter"):
571 continue
573 if (
574 sortCoverageLast
575 and (i == 1)
576 and getattr(item.subWriter, "name", None) == "Coverage"
577 ):
578 # we've already 'gathered' it above
579 continue
581 if id(item.subWriter) not in done:
582 item.subWriter._gatherTables(tables, extTables, done)
583 else:
584 # Item is already written out by other parent
585 pass
587 selfTables.append(self)
589 def _gatherGraphForHarfbuzz(self, tables, obj_list, done, objidx, virtual_edges):
590 real_links = []
591 virtual_links = []
592 item_idx = objidx
594 # Merge virtual_links from parent
595 for idx in virtual_edges:
596 virtual_links.append((0, 0, idx))
598 sortCoverageLast = False
599 coverage_idx = 0
600 if hasattr(self, "sortCoverageLast"):
601 # Find coverage table
602 for i, item in enumerate(self.items):
603 if getattr(item, "name", None) == "Coverage":
604 sortCoverageLast = True
605 if id(item) not in done:
606 coverage_idx = item_idx = item._gatherGraphForHarfbuzz(
607 tables, obj_list, done, item_idx, virtual_edges
608 )
609 else:
610 coverage_idx = done[id(item)]
611 virtual_edges.append(coverage_idx)
612 break
614 child_idx = 0
615 offset_pos = 0
616 for i, item in enumerate(self.items):
617 if hasattr(item, "subWriter"):
618 pos = offset_pos
619 elif hasattr(item, "getCountData"):
620 offset_pos += item.size
621 continue
622 else:
623 offset_pos = offset_pos + len(item)
624 continue
626 if id(item.subWriter) not in done:
627 child_idx = item_idx = item.subWriter._gatherGraphForHarfbuzz(
628 tables, obj_list, done, item_idx, virtual_edges
629 )
630 else:
631 child_idx = done[id(item.subWriter)]
633 real_edge = (pos, item.offsetSize, child_idx)
634 real_links.append(real_edge)
635 offset_pos += item.offsetSize
637 tables.append(self)
638 obj_list.append((real_links, virtual_links))
639 item_idx += 1
640 done[id(self)] = item_idx
641 if sortCoverageLast:
642 virtual_edges.pop()
644 return item_idx
646 def getAllDataUsingHarfbuzz(self, tableTag):
647 """The Whole table is represented as a Graph.
648 Assemble graph data and call Harfbuzz repacker to pack the table.
649 Harfbuzz repacker is faster and retain as much sub-table sharing as possible, see also:
650 https://github.com/harfbuzz/harfbuzz/blob/main/docs/repacker.md
651 The input format for hb.repack() method is explained here:
652 https://github.com/harfbuzz/uharfbuzz/blob/main/src/uharfbuzz/_harfbuzz.pyx#L1149
653 """
654 internedTables = {}
655 self._doneWriting(internedTables, shareExtension=True)
656 tables = []
657 obj_list = []
658 done = {}
659 objidx = 0
660 virtual_edges = []
661 self._gatherGraphForHarfbuzz(tables, obj_list, done, objidx, virtual_edges)
662 # Gather all data in two passes: the absolute positions of all
663 # subtable are needed before the actual data can be assembled.
664 pos = 0
665 for table in tables:
666 table.pos = pos
667 pos = pos + table.getDataLength()
669 data = []
670 for table in tables:
671 tableData = table.getDataForHarfbuzz()
672 data.append(tableData)
674 if hasattr(hb, "repack_with_tag"):
675 return hb.repack_with_tag(str(tableTag), data, obj_list)
676 else:
677 return hb.repack(data, obj_list)
679 def getAllData(self, remove_duplicate=True):
680 """Assemble all data, including all subtables."""
681 if remove_duplicate:
682 internedTables = {}
683 self._doneWriting(internedTables)
684 tables = []
685 extTables = []
686 done = {}
687 self._gatherTables(tables, extTables, done)
688 tables.reverse()
689 extTables.reverse()
690 # Gather all data in two passes: the absolute positions of all
691 # subtable are needed before the actual data can be assembled.
692 pos = 0
693 for table in tables:
694 table.pos = pos
695 pos = pos + table.getDataLength()
697 for table in extTables:
698 table.pos = pos
699 pos = pos + table.getDataLength()
701 data = []
702 for table in tables:
703 tableData = table.getData()
704 data.append(tableData)
706 for table in extTables:
707 tableData = table.getData()
708 data.append(tableData)
710 return bytesjoin(data)
712 # interface for gathering data, as used by table.compile()
714 def getSubWriter(self):
715 subwriter = self.__class__(self.localState, self.tableTag)
716 subwriter.parent = (
717 self # because some subtables have idential values, we discard
718 )
719 # the duplicates under the getAllData method. Hence some
720 # subtable writers can have more than one parent writer.
721 # But we just care about first one right now.
722 return subwriter
724 def writeValue(self, typecode, value):
725 self.items.append(struct.pack(f">{typecode}", value))
727 def writeArray(self, typecode, values):
728 a = array.array(typecode, values)
729 if sys.byteorder != "big":
730 a.byteswap()
731 self.items.append(a.tobytes())
733 def writeInt8(self, value):
734 assert -128 <= value < 128, value
735 self.items.append(struct.pack(">b", value))
737 def writeInt8Array(self, values):
738 self.writeArray("b", values)
740 def writeShort(self, value):
741 assert -32768 <= value < 32768, value
742 self.items.append(struct.pack(">h", value))
744 def writeShortArray(self, values):
745 self.writeArray("h", values)
747 def writeLong(self, value):
748 self.items.append(struct.pack(">i", value))
750 def writeLongArray(self, values):
751 self.writeArray("i", values)
753 def writeUInt8(self, value):
754 assert 0 <= value < 256, value
755 self.items.append(struct.pack(">B", value))
757 def writeUInt8Array(self, values):
758 self.writeArray("B", values)
760 def writeUShort(self, value):
761 assert 0 <= value < 0x10000, value
762 self.items.append(struct.pack(">H", value))
764 def writeUShortArray(self, values):
765 self.writeArray("H", values)
767 def writeULong(self, value):
768 self.items.append(struct.pack(">I", value))
770 def writeULongArray(self, values):
771 self.writeArray("I", values)
773 def writeUInt24(self, value):
774 assert 0 <= value < 0x1000000, value
775 b = struct.pack(">L", value)
776 self.items.append(b[1:])
778 def writeUInt24Array(self, values):
779 for value in values:
780 self.writeUInt24(value)
782 def writeTag(self, tag):
783 tag = Tag(tag).tobytes()
784 assert len(tag) == 4, tag
785 self.items.append(tag)
787 def writeSubTable(self, subWriter, offsetSize):
788 self.items.append(OffsetToWriter(subWriter, offsetSize))
790 def writeCountReference(self, table, name, size=2, value=None):
791 ref = CountReference(table, name, size=size, value=value)
792 self.items.append(ref)
793 return ref
795 def writeStruct(self, format, values):
796 data = struct.pack(*(format,) + values)
797 self.items.append(data)
799 def writeData(self, data):
800 self.items.append(data)
802 def getOverflowErrorRecord(self, item):
803 LookupListIndex = SubTableIndex = itemName = itemIndex = None
804 if self.name == "LookupList":
805 LookupListIndex = item.repeatIndex
806 elif self.name == "Lookup":
807 LookupListIndex = self.repeatIndex
808 SubTableIndex = item.repeatIndex
809 else:
810 itemName = getattr(item, "name", "<none>")
811 if hasattr(item, "repeatIndex"):
812 itemIndex = item.repeatIndex
813 if self.name == "SubTable":
814 LookupListIndex = self.parent.repeatIndex
815 SubTableIndex = self.repeatIndex
816 elif self.name == "ExtSubTable":
817 LookupListIndex = self.parent.parent.repeatIndex
818 SubTableIndex = self.parent.repeatIndex
819 else: # who knows how far below the SubTable level we are! Climb back up to the nearest subtable.
820 itemName = ".".join([self.name, itemName])
821 p1 = self.parent
822 while p1 and p1.name not in ["ExtSubTable", "SubTable"]:
823 itemName = ".".join([p1.name, itemName])
824 p1 = p1.parent
825 if p1:
826 if p1.name == "ExtSubTable":
827 LookupListIndex = p1.parent.parent.repeatIndex
828 SubTableIndex = p1.parent.repeatIndex
829 else:
830 LookupListIndex = p1.parent.repeatIndex
831 SubTableIndex = p1.repeatIndex
833 return OverflowErrorRecord(
834 (self.tableTag, LookupListIndex, SubTableIndex, itemName, itemIndex)
835 )
838class CountReference(object):
839 """A reference to a Count value, not a count of references."""
841 def __init__(self, table, name, size=None, value=None):
842 self.table = table
843 self.name = name
844 self.size = size
845 if value is not None:
846 self.setValue(value)
848 def setValue(self, value):
849 table = self.table
850 name = self.name
851 if table[name] is None:
852 table[name] = value
853 else:
854 assert table[name] == value, (name, table[name], value)
856 def getValue(self):
857 return self.table[self.name]
859 def getCountData(self):
860 v = self.table[self.name]
861 if v is None:
862 v = 0
863 return {1: packUInt8, 2: packUShort, 4: packULong}[self.size](v)
866def packUInt8(value):
867 return struct.pack(">B", value)
870def packUShort(value):
871 return struct.pack(">H", value)
874def packULong(value):
875 assert 0 <= value < 0x100000000, value
876 return struct.pack(">I", value)
879def packUInt24(value):
880 assert 0 <= value < 0x1000000, value
881 return struct.pack(">I", value)[1:]
884class BaseTable(object):
886 """Generic base class for all OpenType (sub)tables."""
888 def __getattr__(self, attr):
889 reader = self.__dict__.get("reader")
890 if reader:
891 del self.reader
892 font = self.font
893 del self.font
894 self.decompile(reader, font)
895 return getattr(self, attr)
897 raise AttributeError(attr)
899 def ensureDecompiled(self, recurse=False):
900 reader = self.__dict__.get("reader")
901 if reader:
902 del self.reader
903 font = self.font
904 del self.font
905 self.decompile(reader, font)
906 if recurse:
907 for subtable in self.iterSubTables():
908 subtable.value.ensureDecompiled(recurse)
910 def __getstate__(self):
911 # before copying/pickling 'lazy' objects, make a shallow copy of OTTableReader
912 # https://github.com/fonttools/fonttools/issues/2965
913 if "reader" in self.__dict__:
914 state = self.__dict__.copy()
915 state["reader"] = self.__dict__["reader"].copy()
916 return state
917 return self.__dict__
919 @classmethod
920 def getRecordSize(cls, reader):
921 totalSize = 0
922 for conv in cls.converters:
923 size = conv.getRecordSize(reader)
924 if size is NotImplemented:
925 return NotImplemented
926 countValue = 1
927 if conv.repeat:
928 if conv.repeat in reader:
929 countValue = reader[conv.repeat] + conv.aux
930 else:
931 return NotImplemented
932 totalSize += size * countValue
933 return totalSize
935 def getConverters(self):
936 return self.converters
938 def getConverterByName(self, name):
939 return self.convertersByName[name]
941 def populateDefaults(self, propagator=None):
942 for conv in self.getConverters():
943 if conv.repeat:
944 if not hasattr(self, conv.name):
945 setattr(self, conv.name, [])
946 countValue = len(getattr(self, conv.name)) - conv.aux
947 try:
948 count_conv = self.getConverterByName(conv.repeat)
949 setattr(self, conv.repeat, countValue)
950 except KeyError:
951 # conv.repeat is a propagated count
952 if propagator and conv.repeat in propagator:
953 propagator[conv.repeat].setValue(countValue)
954 else:
955 if conv.aux and not eval(conv.aux, None, self.__dict__):
956 continue
957 if hasattr(self, conv.name):
958 continue # Warn if it should NOT be present?!
959 if hasattr(conv, "writeNullOffset"):
960 setattr(self, conv.name, None) # Warn?
961 # elif not conv.isCount:
962 # # Warn?
963 # pass
964 if hasattr(conv, "DEFAULT"):
965 # OptionalValue converters (e.g. VarIndex)
966 setattr(self, conv.name, conv.DEFAULT)
968 def decompile(self, reader, font):
969 self.readFormat(reader)
970 table = {}
971 self.__rawTable = table # for debugging
972 for conv in self.getConverters():
973 if conv.name == "SubTable":
974 conv = conv.getConverter(reader.tableTag, table["LookupType"])
975 if conv.name == "ExtSubTable":
976 conv = conv.getConverter(reader.tableTag, table["ExtensionLookupType"])
977 if conv.name == "FeatureParams":
978 conv = conv.getConverter(reader["FeatureTag"])
979 if conv.name == "SubStruct":
980 conv = conv.getConverter(reader.tableTag, table["MorphType"])
981 try:
982 if conv.repeat:
983 if isinstance(conv.repeat, int):
984 countValue = conv.repeat
985 elif conv.repeat in table:
986 countValue = table[conv.repeat]
987 else:
988 # conv.repeat is a propagated count
989 countValue = reader[conv.repeat]
990 countValue += conv.aux
991 table[conv.name] = conv.readArray(reader, font, table, countValue)
992 else:
993 if conv.aux and not eval(conv.aux, None, table):
994 continue
995 table[conv.name] = conv.read(reader, font, table)
996 if conv.isPropagated:
997 reader[conv.name] = table[conv.name]
998 except Exception as e:
999 name = conv.name
1000 e.args = e.args + (name,)
1001 raise
1003 if hasattr(self, "postRead"):
1004 self.postRead(table, font)
1005 else:
1006 self.__dict__.update(table)
1008 del self.__rawTable # succeeded, get rid of debugging info
1010 def compile(self, writer, font):
1011 self.ensureDecompiled()
1012 # TODO Following hack to be removed by rewriting how FormatSwitching tables
1013 # are handled.
1014 # https://github.com/fonttools/fonttools/pull/2238#issuecomment-805192631
1015 if hasattr(self, "preWrite"):
1016 deleteFormat = not hasattr(self, "Format")
1017 table = self.preWrite(font)
1018 deleteFormat = deleteFormat and hasattr(self, "Format")
1019 else:
1020 deleteFormat = False
1021 table = self.__dict__.copy()
1023 # some count references may have been initialized in a custom preWrite; we set
1024 # these in the writer's state beforehand (instead of sequentially) so they will
1025 # be propagated to all nested subtables even if the count appears in the current
1026 # table only *after* the offset to the subtable that it is counting.
1027 for conv in self.getConverters():
1028 if conv.isCount and conv.isPropagated:
1029 value = table.get(conv.name)
1030 if isinstance(value, CountReference):
1031 writer[conv.name] = value
1033 if hasattr(self, "sortCoverageLast"):
1034 writer.sortCoverageLast = 1
1036 if hasattr(self, "DontShare"):
1037 writer.DontShare = True
1039 if hasattr(self.__class__, "LookupType"):
1040 writer["LookupType"].setValue(self.__class__.LookupType)
1042 self.writeFormat(writer)
1043 for conv in self.getConverters():
1044 value = table.get(
1045 conv.name
1046 ) # TODO Handle defaults instead of defaulting to None!
1047 if conv.repeat:
1048 if value is None:
1049 value = []
1050 countValue = len(value) - conv.aux
1051 if isinstance(conv.repeat, int):
1052 assert len(value) == conv.repeat, "expected %d values, got %d" % (
1053 conv.repeat,
1054 len(value),
1055 )
1056 elif conv.repeat in table:
1057 CountReference(table, conv.repeat, value=countValue)
1058 else:
1059 # conv.repeat is a propagated count
1060 writer[conv.repeat].setValue(countValue)
1061 try:
1062 conv.writeArray(writer, font, table, value)
1063 except Exception as e:
1064 e.args = e.args + (conv.name + "[]",)
1065 raise
1066 elif conv.isCount:
1067 # Special-case Count values.
1068 # Assumption: a Count field will *always* precede
1069 # the actual array(s).
1070 # We need a default value, as it may be set later by a nested
1071 # table. We will later store it here.
1072 # We add a reference: by the time the data is assembled
1073 # the Count value will be filled in.
1074 # We ignore the current count value since it will be recomputed,
1075 # unless it's a CountReference that was already initialized in a custom preWrite.
1076 if isinstance(value, CountReference):
1077 ref = value
1078 ref.size = conv.staticSize
1079 writer.writeData(ref)
1080 table[conv.name] = ref.getValue()
1081 else:
1082 ref = writer.writeCountReference(table, conv.name, conv.staticSize)
1083 table[conv.name] = None
1084 if conv.isPropagated:
1085 writer[conv.name] = ref
1086 elif conv.isLookupType:
1087 # We make sure that subtables have the same lookup type,
1088 # and that the type is the same as the one set on the
1089 # Lookup object, if any is set.
1090 if conv.name not in table:
1091 table[conv.name] = None
1092 ref = writer.writeCountReference(
1093 table, conv.name, conv.staticSize, table[conv.name]
1094 )
1095 writer["LookupType"] = ref
1096 else:
1097 if conv.aux and not eval(conv.aux, None, table):
1098 continue
1099 try:
1100 conv.write(writer, font, table, value)
1101 except Exception as e:
1102 name = value.__class__.__name__ if value is not None else conv.name
1103 e.args = e.args + (name,)
1104 raise
1105 if conv.isPropagated:
1106 writer[conv.name] = value
1108 if deleteFormat:
1109 del self.Format
1111 def readFormat(self, reader):
1112 pass
1114 def writeFormat(self, writer):
1115 pass
1117 def toXML(self, xmlWriter, font, attrs=None, name=None):
1118 tableName = name if name else self.__class__.__name__
1119 if attrs is None:
1120 attrs = []
1121 if hasattr(self, "Format"):
1122 attrs = attrs + [("Format", self.Format)]
1123 xmlWriter.begintag(tableName, attrs)
1124 xmlWriter.newline()
1125 self.toXML2(xmlWriter, font)
1126 xmlWriter.endtag(tableName)
1127 xmlWriter.newline()
1129 def toXML2(self, xmlWriter, font):
1130 # Simpler variant of toXML, *only* for the top level tables (like GPOS, GSUB).
1131 # This is because in TTX our parent writes our main tag, and in otBase.py we
1132 # do it ourselves. I think I'm getting schizophrenic...
1133 for conv in self.getConverters():
1134 if conv.repeat:
1135 value = getattr(self, conv.name, [])
1136 for i in range(len(value)):
1137 item = value[i]
1138 conv.xmlWrite(xmlWriter, font, item, conv.name, [("index", i)])
1139 else:
1140 if conv.aux and not eval(conv.aux, None, vars(self)):
1141 continue
1142 value = getattr(
1143 self, conv.name, None
1144 ) # TODO Handle defaults instead of defaulting to None!
1145 conv.xmlWrite(xmlWriter, font, value, conv.name, [])
1147 def fromXML(self, name, attrs, content, font):
1148 try:
1149 conv = self.getConverterByName(name)
1150 except KeyError:
1151 raise # XXX on KeyError, raise nice error
1152 value = conv.xmlRead(attrs, content, font)
1153 if conv.repeat:
1154 seq = getattr(self, conv.name, None)
1155 if seq is None:
1156 seq = []
1157 setattr(self, conv.name, seq)
1158 seq.append(value)
1159 else:
1160 setattr(self, conv.name, value)
1162 def __ne__(self, other):
1163 result = self.__eq__(other)
1164 return result if result is NotImplemented else not result
1166 def __eq__(self, other):
1167 if type(self) != type(other):
1168 return NotImplemented
1170 self.ensureDecompiled()
1171 other.ensureDecompiled()
1173 return self.__dict__ == other.__dict__
1175 class SubTableEntry(NamedTuple):
1176 """See BaseTable.iterSubTables()"""
1178 name: str
1179 value: "BaseTable"
1180 index: Optional[int] = None # index into given array, None for single values
1182 def iterSubTables(self) -> Iterator[SubTableEntry]:
1183 """Yield (name, value, index) namedtuples for all subtables of current table.
1185 A sub-table is an instance of BaseTable (or subclass thereof) that is a child
1186 of self, the current parent table.
1187 The tuples also contain the attribute name (str) of the of parent table to get
1188 a subtable, and optionally, for lists of subtables (i.e. attributes associated
1189 with a converter that has a 'repeat'), an index into the list containing the
1190 given subtable value.
1191 This method can be useful to traverse trees of otTables.
1192 """
1193 for conv in self.getConverters():
1194 name = conv.name
1195 value = getattr(self, name, None)
1196 if value is None:
1197 continue
1198 if isinstance(value, BaseTable):
1199 yield self.SubTableEntry(name, value)
1200 elif isinstance(value, list):
1201 yield from (
1202 self.SubTableEntry(name, v, index=i)
1203 for i, v in enumerate(value)
1204 if isinstance(v, BaseTable)
1205 )
1207 # instance (not @class)method for consistency with FormatSwitchingBaseTable
1208 def getVariableAttrs(self):
1209 return getVariableAttrs(self.__class__)
1212class FormatSwitchingBaseTable(BaseTable):
1214 """Minor specialization of BaseTable, for tables that have multiple
1215 formats, eg. CoverageFormat1 vs. CoverageFormat2."""
1217 @classmethod
1218 def getRecordSize(cls, reader):
1219 return NotImplemented
1221 def getConverters(self):
1222 try:
1223 fmt = self.Format
1224 except AttributeError:
1225 # some FormatSwitchingBaseTables (e.g. Coverage) no longer have 'Format'
1226 # attribute after fully decompiled, only gain one in preWrite before being
1227 # recompiled. In the decompiled state, these hand-coded classes defined in
1228 # otTables.py lose their format-specific nature and gain more high-level
1229 # attributes that are not tied to converters.
1230 return []
1231 return self.converters.get(self.Format, [])
1233 def getConverterByName(self, name):
1234 return self.convertersByName[self.Format][name]
1236 def readFormat(self, reader):
1237 self.Format = reader.readUShort()
1239 def writeFormat(self, writer):
1240 writer.writeUShort(self.Format)
1242 def toXML(self, xmlWriter, font, attrs=None, name=None):
1243 BaseTable.toXML(self, xmlWriter, font, attrs, name)
1245 def getVariableAttrs(self):
1246 return getVariableAttrs(self.__class__, self.Format)
1249class UInt8FormatSwitchingBaseTable(FormatSwitchingBaseTable):
1250 def readFormat(self, reader):
1251 self.Format = reader.readUInt8()
1253 def writeFormat(self, writer):
1254 writer.writeUInt8(self.Format)
1257formatSwitchingBaseTables = {
1258 "uint16": FormatSwitchingBaseTable,
1259 "uint8": UInt8FormatSwitchingBaseTable,
1260}
1263def getFormatSwitchingBaseTableClass(formatType):
1264 try:
1265 return formatSwitchingBaseTables[formatType]
1266 except KeyError:
1267 raise TypeError(f"Unsupported format type: {formatType!r}")
1270# memoize since these are parsed from otData.py, thus stay constant
1271@lru_cache()
1272def getVariableAttrs(cls: BaseTable, fmt: Optional[int] = None) -> Tuple[str]:
1273 """Return sequence of variable table field names (can be empty).
1275 Attributes are deemed "variable" when their otData.py's description contain
1276 'VarIndexBase + {offset}', e.g. COLRv1 PaintVar* tables.
1277 """
1278 if not issubclass(cls, BaseTable):
1279 raise TypeError(cls)
1280 if issubclass(cls, FormatSwitchingBaseTable):
1281 if fmt is None:
1282 raise TypeError(f"'fmt' is required for format-switching {cls.__name__}")
1283 converters = cls.convertersByName[fmt]
1284 else:
1285 converters = cls.convertersByName
1286 # assume if no 'VarIndexBase' field is present, table has no variable fields
1287 if "VarIndexBase" not in converters:
1288 return ()
1289 varAttrs = {}
1290 for name, conv in converters.items():
1291 offset = conv.getVarIndexOffset()
1292 if offset is not None:
1293 varAttrs[name] = offset
1294 return tuple(sorted(varAttrs, key=varAttrs.__getitem__))
1297#
1298# Support for ValueRecords
1299#
1300# This data type is so different from all other OpenType data types that
1301# it requires quite a bit of code for itself. It even has special support
1302# in OTTableReader and OTTableWriter...
1303#
1305valueRecordFormat = [
1306 # Mask Name isDevice signed
1307 (0x0001, "XPlacement", 0, 1),
1308 (0x0002, "YPlacement", 0, 1),
1309 (0x0004, "XAdvance", 0, 1),
1310 (0x0008, "YAdvance", 0, 1),
1311 (0x0010, "XPlaDevice", 1, 0),
1312 (0x0020, "YPlaDevice", 1, 0),
1313 (0x0040, "XAdvDevice", 1, 0),
1314 (0x0080, "YAdvDevice", 1, 0),
1315 # reserved:
1316 (0x0100, "Reserved1", 0, 0),
1317 (0x0200, "Reserved2", 0, 0),
1318 (0x0400, "Reserved3", 0, 0),
1319 (0x0800, "Reserved4", 0, 0),
1320 (0x1000, "Reserved5", 0, 0),
1321 (0x2000, "Reserved6", 0, 0),
1322 (0x4000, "Reserved7", 0, 0),
1323 (0x8000, "Reserved8", 0, 0),
1324]
1327def _buildDict():
1328 d = {}
1329 for mask, name, isDevice, signed in valueRecordFormat:
1330 d[name] = mask, isDevice, signed
1331 return d
1334valueRecordFormatDict = _buildDict()
1337class ValueRecordFactory(object):
1339 """Given a format code, this object convert ValueRecords."""
1341 def __init__(self, valueFormat):
1342 format = []
1343 for mask, name, isDevice, signed in valueRecordFormat:
1344 if valueFormat & mask:
1345 format.append((name, isDevice, signed))
1346 self.format = format
1348 def __len__(self):
1349 return len(self.format)
1351 def readValueRecord(self, reader, font):
1352 format = self.format
1353 if not format:
1354 return None
1355 valueRecord = ValueRecord()
1356 for name, isDevice, signed in format:
1357 if signed:
1358 value = reader.readShort()
1359 else:
1360 value = reader.readUShort()
1361 if isDevice:
1362 if value:
1363 from . import otTables
1365 subReader = reader.getSubReader(value)
1366 value = getattr(otTables, name)()
1367 value.decompile(subReader, font)
1368 else:
1369 value = None
1370 setattr(valueRecord, name, value)
1371 return valueRecord
1373 def writeValueRecord(self, writer, font, valueRecord):
1374 for name, isDevice, signed in self.format:
1375 value = getattr(valueRecord, name, 0)
1376 if isDevice:
1377 if value:
1378 subWriter = writer.getSubWriter()
1379 writer.writeSubTable(subWriter, offsetSize=2)
1380 value.compile(subWriter, font)
1381 else:
1382 writer.writeUShort(0)
1383 elif signed:
1384 writer.writeShort(value)
1385 else:
1386 writer.writeUShort(value)
1389class ValueRecord(object):
1390 # see ValueRecordFactory
1392 def __init__(self, valueFormat=None, src=None):
1393 if valueFormat is not None:
1394 for mask, name, isDevice, signed in valueRecordFormat:
1395 if valueFormat & mask:
1396 setattr(self, name, None if isDevice else 0)
1397 if src is not None:
1398 for key, val in src.__dict__.items():
1399 if not hasattr(self, key):
1400 continue
1401 setattr(self, key, val)
1402 elif src is not None:
1403 self.__dict__ = src.__dict__.copy()
1405 def getFormat(self):
1406 format = 0
1407 for name in self.__dict__.keys():
1408 format = format | valueRecordFormatDict[name][0]
1409 return format
1411 def getEffectiveFormat(self):
1412 format = 0
1413 for name, value in self.__dict__.items():
1414 if value:
1415 format = format | valueRecordFormatDict[name][0]
1416 return format
1418 def toXML(self, xmlWriter, font, valueName, attrs=None):
1419 if attrs is None:
1420 simpleItems = []
1421 else:
1422 simpleItems = list(attrs)
1423 for mask, name, isDevice, format in valueRecordFormat[:4]: # "simple" values
1424 if hasattr(self, name):
1425 simpleItems.append((name, getattr(self, name)))
1426 deviceItems = []
1427 for mask, name, isDevice, format in valueRecordFormat[4:8]: # device records
1428 if hasattr(self, name):
1429 device = getattr(self, name)
1430 if device is not None:
1431 deviceItems.append((name, device))
1432 if deviceItems:
1433 xmlWriter.begintag(valueName, simpleItems)
1434 xmlWriter.newline()
1435 for name, deviceRecord in deviceItems:
1436 if deviceRecord is not None:
1437 deviceRecord.toXML(xmlWriter, font, name=name)
1438 xmlWriter.endtag(valueName)
1439 xmlWriter.newline()
1440 else:
1441 xmlWriter.simpletag(valueName, simpleItems)
1442 xmlWriter.newline()
1444 def fromXML(self, name, attrs, content, font):
1445 from . import otTables
1447 for k, v in attrs.items():
1448 setattr(self, k, int(v))
1449 for element in content:
1450 if not isinstance(element, tuple):
1451 continue
1452 name, attrs, content = element
1453 value = getattr(otTables, name)()
1454 for elem2 in content:
1455 if not isinstance(elem2, tuple):
1456 continue
1457 name2, attrs2, content2 = elem2
1458 value.fromXML(name2, attrs2, content2, font)
1459 setattr(self, name, value)
1461 def __ne__(self, other):
1462 result = self.__eq__(other)
1463 return result if result is NotImplemented else not result
1465 def __eq__(self, other):
1466 if type(self) != type(other):
1467 return NotImplemented
1468 return self.__dict__ == other.__dict__