REPO: D:\mintdim
PURPOSE: inspect current segment encoding patch so it can be reverted to segment-local tokenization.
GENERATED: 2026-05-29 15:52:47

====================================================================================================
FILE: src\mintdim\apis\unit_build\runtime.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import hashlib
    4: import sys
    5: from bisect import insort
    6: from dataclasses import dataclass, field
    7: from pathlib import Path
    8: from typing import Iterator, Sequence
    9: 
   10: from mintdim.apis.unit_build.ui.logger import log_event
   11: 
   12: from mintdim.apis.unit_build.errors import OutputValidationError, UnitOverflowError
   13: from mintdim.apis.unit_build.config import TokenizerEntry, UnitBuildConfig
   14: from mintdim.apis.unit_build.contracts import TokenizerHandle, TokenizerMetadata
   15: from mintdim.apis.unit_build.adapters.output.directory import prepare_output_dir
   16: from mintdim.apis.unit_build.adapters.output.indices.duplicate import HashCollector, write_duplicate_index
   17: from mintdim.apis.unit_build.adapters.output.histogram import UnitHistogram, write_histogram
   18: from mintdim.apis.unit_build.adapters.output.length_field import write_length_field
   19: from mintdim.apis.unit_build.adapters.output.manifest import write_manifest
   20: from mintdim.apis.unit_build.adapters.output.indices.overflow import OverflowIndexWriter
   21: from mintdim.apis.unit_build.adapters.output.indices.sample import SampleIndexWriter
   22: from mintdim.apis.unit_build.adapters.output.stats import StatsAccumulator, write_stats
   23: from mintdim.apis.unit_build.adapters.output.indices.unknown import UnkIndexWriter
   24: from mintdim.apis.unit_build.adapters.sources.jsonl import iter_jsonl_samples
   25: from mintdim.apis.unit_build.adapters.sources.template import (
   26:     CompiledTemplate,
   27:     LiteralUnkInfo,
   28:     Segment,
   29:     compile_template,
   30:     preflight_template,
   31:     render_segments,
   32: )
   33: from mintdim.apis.unit_build.adapters.tokenizers import load_tokenizer
   34: from mintdim.apis.unit_build.domain.overflow import OverflowAction, OverflowContext, OverflowResolution
   35: from mintdim.apis.unit_build.domain.planner import find_unit
   36: from mintdim.apis.unit_build.adapters.output.shard_writer import (
   37:     ShardRecord,
   38:     ShardWriter,
   39:     assert_no_magic_collision,
   40:     bytes_per_token,
   41:     magic_for_dtype,
   42:     select_token_dtype,
   43: )
   44: 
   45: 
   46: @dataclass(frozen=True)
   47: class _Session:
   48:     files: list[str]
   49:     fields: list[str]
   50:     template: str
   51:     tokenizer_type: str
   52:     tokenizer_entry: TokenizerEntry
   53:     sizes: list[int]
   54:     build_batch: int
   55:     output_dir: str
   56:     samples_per_shard: int
   57:     file_index_for_template: int
   58: 
   59: 
   60: @dataclass(frozen=True)
   61: class _PreparedSession:
   62:     session: _Session
   63:     tokenizer: TokenizerHandle
   64:     compiled: CompiledTemplate
   65:     token_dtype: str
   66:     bytes_per_token: int
   67:     template_unk_findings: list[LiteralUnkInfo]
   68: 
   69: 
   70: @dataclass
   71: class _Policies:
   72:     """Behavior controls plumbed from `.run(on_template_unk=..., on_overflow=...)`."""
   73: 
   74:     template_unk: str  # "prompt" | "abort" | "continue"
   75:     overflow: str  # "prompt" | "abort" | "truncate" | "skip"
   76:     latched_overflow: OverflowResolution | None = None
   77:     latched_extend: bool = False
   78: 
   79: 
   80: @dataclass
   81: class _BatchRecord:
   82:     sample_id: int
   83:     source_file: str
   84:     source_line: int
   85:     segment_tokens: list[list[int]]
   86:     segment_lengths: list[int]
   87:     token_count: int
   88:     unit_size: int
   89:     content_hash: str
   90:     unk_positions: list[int]
   91:     unk_chars: list[str]
   92:     unk_field_indices: list[int]
   93:     empty_fields: list[str]
   94:     shard_path: str = ""
   95: 
   96: 
   97: @dataclass
   98: class _OverflowSkip:
   99:     sample_id: int
  100:     source_file: str
  101:     source_line: int
  102:     token_count: int
  103:     max_unit_size: int
  104:     action: str
  105:     new_unit_size: int | None = None
  106: 
  107: 
  108: def run_unit_build_pipeline(
  109:     raw_or_config,
  110:     *,
  111:     on_template_unk: str = "prompt",
  112:     on_overflow: str = "prompt",
  113: ) -> dict:
  114:     if isinstance(raw_or_config, UnitBuildConfig):
  115:         config = raw_or_config
  116:     else:
  117:         from .validation import validate_unit_build_pipeline
  118: 
  119:         config = validate_unit_build_pipeline(raw_or_config)
  120: 
  121:     sessions = _plan_sessions(config)
  122:     _preflight_output_dirs(sessions)
  123:     prepared = [_prepare_session(session) for session in sessions]
  124:     outputs = []
  125:     for prep in prepared:
  126:         policies = _Policies(template_unk=on_template_unk, overflow=on_overflow)
  127:         if prep.template_unk_findings:
  128:             _resolve_template_unk_policy(prep=prep, policies=policies)
  129:         outputs.append(_run_session(prep, policies=policies))
  130:     return {"outputs": outputs}
  131: 
  132: 
  133: def _plan_sessions(config: UnitBuildConfig) -> list[_Session]:
  134:     if config.is_shared:
  135:         return [
  136:             _Session(
  137:                 files=list(config.source.files),
  138:                 fields=list(config.source.fields[0]),
  139:                 template=config.source.templates[0],
  140:                 tokenizer_type=config.tokenizer.type,
  141:                 tokenizer_entry=config.tokenizer.configs[0],
  142:                 sizes=list(config.units.sizes[0]),
  143:                 build_batch=config.units.build_batch[0],
  144:                 output_dir=config.output.paths[0],
  145:                 samples_per_shard=config.output.samples_per_shard[0],
  146:                 file_index_for_template=0,
  147:             )
  148:         ]
  149: 
  150:     return [
  151:         _Session(
  152:             files=[config.source.files[i]],
  153:             fields=list(_for_file(config.source.fields, i)),
  154:             template=_for_file(config.source.templates, i),
  155:             tokenizer_type=config.tokenizer.type,
  156:             tokenizer_entry=_for_file(config.tokenizer.configs, i),
  157:             sizes=list(_for_file(config.units.sizes, i)),
  158:             build_batch=_for_file(config.units.build_batch, i),
  159:             output_dir=_for_file(config.output.paths, i),
  160:             samples_per_shard=_for_file(config.output.samples_per_shard, i),
  161:             file_index_for_template=i,
  162:         )
  163:         for i in range(config.file_count)
  164:     ]
  165: 
  166: 
  167: def _for_file(values: Sequence, index: int):
  168:     return values[0] if len(values) == 1 else values[index]
  169: 
  170: 
  171: def _preflight_output_dirs(sessions: list[_Session]) -> None:
  172:     for session in sessions:
  173:         path = Path(session.output_dir)
  174:         if path.exists() and any(path.iterdir()):
  175:             raise OutputValidationError(directory=session.output_dir)
  176: 
  177: 
  178: def _prepare_session(session: _Session) -> _PreparedSession:
  179:     tokenizer = load_tokenizer(
  180:         tokenizer_type=session.tokenizer_type,
  181:         entry=session.tokenizer_entry,
  182:     )
  183:     compiled = compile_template(
  184:         file_index=session.file_index_for_template,
  185:         template=session.template,
  186:         declared_fields=session.fields,
  187:         source_files=session.files,
  188:     )
  189:     template_unk_findings = preflight_template(
  190:         compiled=compiled, tokenizer=tokenizer
  191:     )
  192:     token_dtype = select_token_dtype(tokenizer.vocab_size)
  193:     assert_no_magic_collision(
  194:         vocab_size=tokenizer.vocab_size, token_dtype=token_dtype
  195:     )
  196:     return _PreparedSession(
  197:         session=session,
  198:         tokenizer=tokenizer,
  199:         compiled=compiled,
  200:         token_dtype=token_dtype,
  201:         bytes_per_token=bytes_per_token(token_dtype),
  202:         template_unk_findings=template_unk_findings,
  203:     )
  204: 
  205: 
  206: def _resolve_template_unk_policy(*, prep: _PreparedSession, policies: _Policies) -> None:
  207:     """Run P1 decision for this session before the build starts."""
  208:     from mintdim.apis.unit_build.errors import TemplateTokenizerError
  209: 
  210:     file_index = prep.session.file_index_for_template
  211:     if policies.template_unk == "continue":
  212:         return
  213:     if policies.template_unk == "abort":
  214:         raise TemplateTokenizerError(
  215:             file_index=file_index, template=prep.compiled.raw
  216:         )
  217:     if policies.template_unk == "prompt":
  218:         from mintdim.apis.unit_build.ui.prompts import prompt_template_unk, stdin_is_tty
  219: 
  220:         if not stdin_is_tty():
  221:             raise TemplateTokenizerError(
  222:                 file_index=file_index, template=prep.compiled.raw
  223:             )
  224:         proceed = prompt_template_unk(
  225:             file_index=file_index,
  226:             n_findings=len(prep.template_unk_findings),
  227:         )
  228:         if not proceed:
  229:             raise TemplateTokenizerError(
  230:                 file_index=file_index, template=prep.compiled.raw
  231:             )
  232:         return
  233:     raise ValueError(f"unknown on_template_unk value: {policies.template_unk!r}")
  234: 
  235: 
  236: def _resolve_overflow_policy(
  237:     *,
  238:     ctx: OverflowContext,
  239:     policies: _Policies,
  240: ) -> OverflowResolution:
  241:     """Apply the user's overflow policy and return what action to take."""
  242:     if policies.latched_overflow is not None:
  243:         return policies.latched_overflow
  244:     if policies.latched_extend:
  245:         return OverflowResolution(
  246:             action=OverflowAction.EXTEND, new_unit_size=ctx.token_count
  247:         )
  248: 
  249:     if policies.overflow == "abort":
  250:         raise UnitOverflowError(
  251:             sample=f"{ctx.source_file}:{ctx.source_line}",
  252:             token_count=ctx.token_count,
  253:             max_unit_size=ctx.max_unit_size,
  254:         )
  255:     if policies.overflow == "truncate":
  256:         return OverflowResolution(action=OverflowAction.TRUNCATE)
  257:     if policies.overflow == "skip":
  258:         return OverflowResolution(action=OverflowAction.SKIP)
  259: 
  260:     if policies.overflow != "prompt":
  261:         raise ValueError(f"unknown on_overflow value: {policies.overflow!r}")
  262: 
  263:     from mintdim.apis.unit_build.ui.prompts import resolve_overflow_via_prompt, stdin_is_tty
  264: 
  265:     if not stdin_is_tty():
  266:         raise UnitOverflowError(
  267:             sample=f"{ctx.source_file}:{ctx.source_line}",
  268:             token_count=ctx.token_count,
  269:             max_unit_size=ctx.max_unit_size,
  270:         )
  271: 
  272:     resolution, scope = resolve_overflow_via_prompt(ctx=ctx)
  273:     if scope == "all":
  274:         if resolution.action == OverflowAction.EXTEND:
  275:             policies.latched_extend = True
  276:         else:
  277:             policies.latched_overflow = resolution
  278:     return resolution
  279: 
  280: 
  281: def _run_session(prepared: _PreparedSession, *, policies: _Policies) -> dict:
  282:     session = prepared.session
  283:     sizes = list(session.sizes)
  284:     out_dir = prepare_output_dir(session.output_dir)
  285:     n_segments = len(prepared.compiled.segments)
  286: 
  287:     log_event(
  288:         "start",
  289:         output=out_dir,
  290:         files=len(session.files),
  291:         units=",".join(str(size) for size in sizes),
  292:         build_batch=session.build_batch,
  293:     )
  294: 
  295:     shard_writers: dict[int, ShardWriter] = {}
  296:     for unit_size in sizes:
  297:         shard_writers[unit_size] = _make_shard_writer(
  298:             out_dir=out_dir,
  299:             unit_size=unit_size,
  300:             prepared=prepared,
  301:             samples_per_shard=session.samples_per_shard,
  302:             n_segments=n_segments,
  303:         )
  304: 
  305:     sample_index = SampleIndexWriter(out_dir)
  306:     unk_index = UnkIndexWriter(out_dir)
  307:     overflow_index = OverflowIndexWriter(out_dir)
  308:     sample_index.open()
  309:     unk_index.open()
  310:     overflow_index.open()
  311: 
  312:     hashes = HashCollector()
  313:     histogram = UnitHistogram()
  314:     stats = StatsAccumulator(
  315:         token_dtype=prepared.token_dtype,
  316:         bytes_per_token=prepared.bytes_per_token,
  317:     )
  318: 
  319:     sample_id = 0
  320:     batch_index = 0
  321:     success = False
  322:     failure_reason: str | None = None
  323:     try:
  324:         for source_file in session.files:
  325:             for batch in _iter_batches(
  326:                 file_path=source_file,
  327:                 fields=session.fields,
  328:                 batch_size=session.build_batch,
  329:             ):
  330:                 records, skips = _build_batch_records(
  331:                     source_file=source_file,
  332:                     batch=batch,
  333:                     compiled=prepared.compiled,
  334:                     tokenizer=prepared.tokenizer,
  335:                     sample_id_start=sample_id,
  336:                     sizes=sizes,
  337:                     policies=policies,
  338:                     fields=session.fields,
  339:                 )
  340: 
  341:                 _ensure_writers_for_sizes(
  342:                     shard_writers=shard_writers,
  343:                     sizes=sizes,
  344:                     out_dir=out_dir,
  345:                     prepared=prepared,
  346:                     samples_per_shard=session.samples_per_shard,
  347:                     n_segments=n_segments,
  348:                 )
  349:                 _write_records_by_unit(records, shard_writers)
  350:                 sample_index.write_many(_sample_index_rows(records))
  351:                 unk_index.write_many(_unk_index_rows(records))
  352:                 overflow_index.write_many(_overflow_index_rows(skips))
  353:                 hashes.add_many(_hash_rows(records))
  354:                 histogram.add_many([rec.unit_size for rec in records])
  355:                 stats.record_many(
  356:                     token_counts=[rec.token_count for rec in records],
  357:                     unit_sizes=[rec.unit_size for rec in records],
  358:                     unk_counts=[len(rec.unk_positions) for rec in records],
  359:                 )
  360: 
  361:                 sample_id += len(batch)
  362:                 batch_index += 1
  363:                 _log_batch_progress(
  364:                     out_dir=out_dir,
  365:                     batch_index=batch_index,
  366:                     total_samples=sample_id,
  367:                     total_tokens=stats.total_tokens,
  368:                     records=records,
  369:                 )
  370:         success = True
  371:     except BaseException as exc:
  372:         failure_reason = f"{type(exc).__name__}: {exc}"
  373:         log_event(
  374:             "failed",
  375:             output=out_dir,
  376:             batches=batch_index,
  377:             samples=sample_id,
  378:             error=failure_reason,
  379:         )
  380:         raise
  381:     finally:
  382:         for writer in shard_writers.values():
  383:             writer.close(success=success, failure_reason=failure_reason)
  384:         sample_index.close()
  385:         unk_index.close()
  386:         overflow_index.close()
  387: 
  388:     groups = hashes.groups()
  389:     duplicate_groups = sum(1 for group in groups.values() if len(group) > 1)
  390:     duplicate_samples = sum(len(group) for group in groups.values() if len(group) > 1)
  391:     stats.record_duplicates(groups=duplicate_groups, samples=duplicate_samples)
  392: 
  393:     write_duplicate_index(out_dir, groups)
  394:     write_histogram(out_dir, histogram)
  395:     write_stats(out_dir, stats)
  396: 
  397:     magic = magic_for_dtype(prepared.token_dtype)
  398:     write_length_field(
  399:         out_dir=out_dir,
  400:         magic=magic,
  401:         sequence_template=prepared.compiled.sequence_template,
  402:         token_dtype=prepared.token_dtype,
  403:         pad_token_id=prepared.tokenizer.pad_id,
  404:     )
  405: 
  406:     import mintdim
  407: 
  408:     tokenizer_meta = TokenizerMetadata(
  409:         type=session.tokenizer_type,
  410:         path=session.tokenizer_entry.path,
  411:         vocab_size=prepared.tokenizer.vocab_size,
  412:         unk_id=prepared.tokenizer.unk_id,
  413:         pad_id=prepared.tokenizer.pad_id,
  414:     )
  415:     write_manifest(
  416:         out_dir=out_dir,
  417:         mintdim_version=mintdim.__version__,
  418:         tokenizer=tokenizer_meta,
  419:         token_dtype=prepared.token_dtype,
  420:         bytes_per_token=prepared.bytes_per_token,
  421:         magic=magic,
  422:         sequence_template=prepared.compiled.sequence_template,
  423:         sizes=sizes,
  424:         build_batch=session.build_batch,
  425:         samples_per_shard=session.samples_per_shard,
  426:         files=session.files,
  427:         fields=[session.fields],
  428:         templates=[session.template],
  429:     )
  430:     log_event(
  431:         "done",
  432:         output=out_dir,
  433:         batches=batch_index,
  434:         samples=stats.total_samples,
  435:         tokens=stats.total_tokens,
  436:     )
  437: 
  438:     return {
  439:         "output_dir": str(out_dir),
  440:         "total_samples": stats.total_samples,
  441:         "total_tokens": stats.total_tokens,
  442:         "samples_with_unk": stats.samples_with_unk,
  443:         "duplicate_groups": duplicate_groups,
  444:         "sizes": list(sizes),
  445:     }
  446: 
  447: 
  448: def _make_shard_writer(
  449:     *,
  450:     out_dir: Path,
  451:     unit_size: int,
  452:     prepared: _PreparedSession,
  453:     samples_per_shard: int,
  454:     n_segments: int,
  455: ) -> ShardWriter:
  456:     return ShardWriter(
  457:         root=out_dir / f"unit_{unit_size}",
  458:         unit_size=unit_size,
  459:         dtype=prepared.token_dtype,
  460:         pad_id=prepared.tokenizer.pad_id,
  461:         samples_per_shard=samples_per_shard,
  462:         n_segments=n_segments,
  463:     )
  464: 
  465: 
  466: def _ensure_writers_for_sizes(
  467:     *,
  468:     shard_writers: dict[int, ShardWriter],
  469:     sizes: list[int],
  470:     out_dir: Path,
  471:     prepared: _PreparedSession,
  472:     samples_per_shard: int,
  473:     n_segments: int,
  474: ) -> None:
  475:     for unit_size in sizes:
  476:         if unit_size in shard_writers:
  477:             continue
  478:         shard_writers[unit_size] = _make_shard_writer(
  479:             out_dir=out_dir,
  480:             unit_size=unit_size,
  481:             prepared=prepared,
  482:             samples_per_shard=samples_per_shard,
  483:             n_segments=n_segments,
  484:         )
  485: 
  486: 
  487: def _iter_batches(
  488:     *,
  489:     file_path: str,
  490:     fields: list[str],
  491:     batch_size: int,
  492: ) -> Iterator[list[tuple[int, dict]]]:
  493:     buffer: list[tuple[int, dict]] = []
  494:     for line_number, sample in iter_jsonl_samples(file_path, fields):
  495:         buffer.append((line_number, sample))
  496:         if len(buffer) >= batch_size:
  497:             yield buffer
  498:             buffer = []
  499:     if buffer:
  500:         yield buffer
  501: 
  502: 
  503: def _encode_segments(
  504:     *,
  505:     compiled: CompiledTemplate,
  506:     tokenizer: TokenizerHandle,
  507:     sample: dict,
  508: ) -> tuple[list[list[int]], list[str], list[list[str]]]:
  509:     """Encode the full rendered sample once, then attribute tokens to template segments.
  510: 
  511:     This preserves tokenizer context across field/literal boundaries. Literal
  512:     segments are first-class segments because their tokens enter the training
  513:     payload exactly like field tokens.
  514:     """
  515:     rendered_segments = render_segments(compiled, sample)
  516:     rendered_text = "".join(rendered_segments)
  517: 
  518:     token_ids, token_surfaces = tokenizer.encode_with_offsets(rendered_text)
  519: 
  520:     if len(token_ids) != len(token_surfaces):
  521:         raise ValueError(
  522:             "tokenizer.encode_with_offsets returned mismatched token/surface counts"
  523:         )
  524: 
  525:     segment_spans = _segment_char_spans(rendered_segments)
  526:     token_spans = _token_char_spans(rendered_text, token_surfaces)
  527: 
  528:     segment_tokens: list[list[int]] = [[] for _ in rendered_segments]
  529:     segment_surfaces: list[list[str]] = [[] for _ in rendered_segments]
  530: 
  531:     for token_id, surface, token_span in zip(token_ids, token_surfaces, token_spans):
  532:         segment_index = _segment_index_for_token(
  533:             segment_spans=segment_spans,
  534:             token_start=token_span[0],
  535:             token_end=token_span[1],
  536:         )
  537:         segment_tokens[segment_index].append(token_id)
  538:         segment_surfaces[segment_index].append(surface)
  539: 
  540:     return segment_tokens, rendered_segments, segment_surfaces
  541: 
  542: 
  543: def _segment_char_spans(rendered_segments: list[str]) -> list[tuple[int, int]]:
  544:     spans: list[tuple[int, int]] = []
  545:     cursor = 0
  546: 
  547:     for text in rendered_segments:
  548:         start = cursor
  549:         cursor += len(text)
  550:         spans.append((start, cursor))
  551: 
  552:     return spans
  553: 
  554: 
  555: def _token_char_spans(text: str, surfaces: list[str]) -> list[tuple[int, int]]:
  556:     spans: list[tuple[int, int]] = []
  557:     cursor = 0
  558: 
  559:     for surface in surfaces:
  560:         if surface == "":
  561:             spans.append((cursor, cursor))
  562:             continue
  563: 
  564:         matched = surface
  565: 
  566:         if text.startswith(matched, cursor):
  567:             start = cursor
  568:         else:
  569:             normalized = surface.replace("▁", " ")
  570: 
  571:             if normalized != surface and text.startswith(normalized, cursor):
  572:                 matched = normalized
  573:                 start = cursor
  574:             else:
  575:                 found = text.find(matched, cursor)
  576: 
  577:                 if found < 0 and normalized != surface:
  578:                     found = text.find(normalized, cursor)
  579:                     if found >= 0:
  580:                         matched = normalized
  581: 
  582:                 if found < 0:
  583:                     # Last-resort monotonic fallback. This should be rare, but it
  584:                     # keeps the shard writer deterministic for tokenizers whose
  585:                     # surface strings are not byte-for-byte slices.
  586:                     start = cursor
  587:                     matched = text[cursor : cursor + len(surface)]
  588:                 else:
  589:                     start = found
  590: 
  591:         end = start + len(matched)
  592:         spans.append((start, end))
  593:         cursor = max(cursor, end)
  594: 
  595:     return spans
  596: 
  597: 
  598: def _segment_index_for_token(
  599:     *,
  600:     segment_spans: list[tuple[int, int]],
  601:     token_start: int,
  602:     token_end: int,
  603: ) -> int:
  604:     # Primary rule: token belongs to the segment containing its first character.
  605:     # This gives literals control when a token starts inside a literal span.
  606:     for index, (start, end) in enumerate(segment_spans):
  607:         if start <= token_start < end:
  608:             return index
  609: 
  610:     # Boundary/zero-width fallback: assign to first overlapping segment.
  611:     for index, (start, end) in enumerate(segment_spans):
  612:         if token_start < end and token_end > start:
  613:             return index
  614: 
  615:     # If token starts exactly after the last non-empty segment, attach it to the
  616:     # closest previous segment.
  617:     for index in range(len(segment_spans) - 1, -1, -1):
  618:         start, end = segment_spans[index]
  619:         if token_start >= end:
  620:             return index
  621: 
  622:     return 0
  623: 
  624: def _build_batch_records(
  625:     *,
  626:     source_file: str,
  627:     batch: list[tuple[int, dict]],
  628:     compiled: CompiledTemplate,
  629:     tokenizer: TokenizerHandle,
  630:     sample_id_start: int,
  631:     sizes: list[int],
  632:     policies: _Policies,
  633:     fields: list[str],
  634: ) -> tuple[list[_BatchRecord], list[_OverflowSkip]]:
  635:     records: list[_BatchRecord] = []
  636:     skips: list[_OverflowSkip] = []
  637: 
  638:     for offset, (line_number, sample) in enumerate(batch):
  639:         sample_id = sample_id_start + offset
  640: 
  641:         segment_tokens, segment_texts, segment_surfaces = _encode_segments(
  642:             compiled=compiled, tokenizer=tokenizer, sample=sample
  643:         )
  644:         segment_lengths = [len(toks) for toks in segment_tokens]
  645:         token_count = sum(segment_lengths)
  646: 
  647:         empty_fields = [
  648:             seg.name
  649:             for seg, text in zip(compiled.segments, segment_texts)
  650:             if seg.kind == "field" and text == ""
  651:         ]
  652:         content_hash = _content_hash("".join(segment_texts))
  653: 
  654:         fit = find_unit(token_count=token_count, sizes=sizes)
  655:         skip_action: str | None = None
  656:         new_unit_size: int | None = None
  657: 
  658:         if fit is None:
  659:             ctx = OverflowContext(
  660:                 source_file=source_file,
  661:                 source_line=line_number,
  662:                 sample_id=sample_id,
  663:                 token_count=token_count,
  664:                 max_unit_size=sizes[-1],
  665:             )
  666:             resolution = _resolve_overflow_policy(ctx=ctx, policies=policies)
  667: 
  668:             if resolution.action == OverflowAction.SKIP:
  669:                 skip_action = "skip"
  670:                 skips.append(
  671:                     _OverflowSkip(
  672:                         sample_id=sample_id,
  673:                         source_file=source_file,
  674:                         source_line=line_number,
  675:                         token_count=token_count,
  676:                         max_unit_size=sizes[-1],
  677:                         action="skip",
  678:                     )
  679:                 )
  680:                 continue
  681: 
  682:             if resolution.action == OverflowAction.TRUNCATE:
  683:                 skip_action = "truncate"
  684:                 max_size = sizes[-1]
  685:                 segment_tokens, segment_lengths = _truncate_segments(
  686:                     segment_tokens=segment_tokens, max_total=max_size
  687:                 )
  688:                 truncated_token_count = sum(segment_lengths)
  689:                 skips.append(
  690:                     _OverflowSkip(
  691:                         sample_id=sample_id,
  692:                         source_file=source_file,
  693:                         source_line=line_number,
  694:                         token_count=token_count,
  695:                         max_unit_size=max_size,
  696:                         action="truncate",
  697:                     )
  698:                 )
  699:                 token_count = truncated_token_count
  700:                 fit = max_size
  701:             elif resolution.action == OverflowAction.EXTEND:
  702:                 new_unit_size = resolution.new_unit_size
  703:                 assert new_unit_size is not None
  704:                 if new_unit_size not in sizes:
  705:                     insort(sizes, new_unit_size)
  706:                 skips.append(
  707:                     _OverflowSkip(
  708:                         sample_id=sample_id,
  709:                         source_file=source_file,
  710:                         source_line=line_number,
  711:                         token_count=token_count,
  712:                         max_unit_size=ctx.max_unit_size,
  713:                         action="extend",
  714:                         new_unit_size=new_unit_size,
  715:                     )
  716:                 )
  717:                 fit = new_unit_size
  718:             else:  # pragma: no cover â€” exhaustive enum
  719:                 raise ValueError(f"unhandled overflow action: {resolution.action}")
  720: 
  721:         unit_size = fit
  722:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
  723: 
  724:         unk_positions: list[int] = []
  725:         unk_field_indices: list[int] = []
  726:         unk_chars: list[str] = []
  727:         unk_id = tokenizer.unk_id
  728:         cursor = 0
  729:         for seg_index, seg_toks in enumerate(segment_tokens):
  730:             seg_surfaces = segment_surfaces[seg_index]
  731:             for local, token_id in enumerate(seg_toks):
  732:                 if token_id != unk_id:
  733:                     continue
  734:                 unk_positions.append(cursor + local)
  735:                 unk_field_indices.append(seg_index)
  736:                 if local < len(seg_surfaces):
  737:                     unk_chars.append(seg_surfaces[local])
  738:                 else:
  739:                     unk_chars.append("")
  740:             cursor += len(seg_toks)
  741: 
  742:         records.append(
  743:             _BatchRecord(
  744:                 sample_id=sample_id,
  745:                 source_file=source_file,
  746:                 source_line=line_number,
  747:                 segment_tokens=segment_tokens,
  748:                 segment_lengths=segment_lengths,
  749:                 token_count=token_count,
  750:                 unit_size=unit_size,
  751:                 content_hash=content_hash,
  752:                 unk_positions=unk_positions,
  753:                 unk_chars=unk_chars,
  754:                 unk_field_indices=unk_field_indices,
  755:                 empty_fields=empty_fields,
  756:             )
  757:         )
  758: 
  759:     return records, skips
  760: 
  761: 
  762: def _truncate_segments(
  763:     *, segment_tokens: list[list[int]], max_total: int
  764: ) -> tuple[list[list[int]], list[int]]:
  765:     """Trim trailing segments so that sum(lengths) == max_total.
  766: 
  767:     Walks segments in order, keeping each full segment until the remaining
  768:     budget can't absorb it; the segment that hits the limit is sliced.
  769:     """
  770:     out: list[list[int]] = []
  771:     budget = max_total
  772:     for toks in segment_tokens:
  773:         if budget <= 0:
  774:             out.append([])
  775:             continue
  776:         if len(toks) <= budget:
  777:             out.append(list(toks))
  778:             budget -= len(toks)
  779:         else:
  780:             out.append(list(toks[:budget]))
  781:             budget = 0
  782:     lengths = [len(toks) for toks in out]
  783:     return out, lengths
  784: 
  785: 
  786: def _write_records_by_unit(
  787:     records: list[_BatchRecord],
  788:     shard_writers: dict[int, ShardWriter],
  789: ) -> None:
  790:     grouped: dict[int, list[_BatchRecord]] = {}
  791:     for record in records:
  792:         grouped.setdefault(record.unit_size, []).append(record)
  793: 
  794:     for unit_size, unit_records in grouped.items():
  795:         shard_records = [
  796:             ShardRecord(
  797:                 segment_lengths=list(rec.segment_lengths),
  798:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
  799:             )
  800:             for rec in unit_records
  801:         ]
  802:         locations = shard_writers[unit_size].write_batch(shard_records)
  803:         for record, (shard_index, _position) in zip(unit_records, locations):
  804:             record.shard_path = f"unit_{unit_size}/shard_{shard_index:06d}.bin"
  805: 
  806: 
  807: def _sample_index_rows(records: list[_BatchRecord]) -> list[dict]:
  808:     return [
  809:         {
  810:             "sample_id": record.sample_id,
  811:             "hash": record.content_hash,
  812:             "token_count": record.token_count,
  813:             "unit_size": record.unit_size,
  814:             "source_file": record.source_file,
  815:             "source_line": record.source_line,
  816:             "shard_path": record.shard_path,
  817:             "empty_fields": record.empty_fields,
  818:         }
  819:         for record in records
  820:     ]
  821: 
  822: 
  823: def _unk_index_rows(records: list[_BatchRecord]) -> list[dict]:
  824:     return [
  825:         {
  826:             "sample_id": record.sample_id,
  827:             "hash": record.content_hash,
  828:             "source_file": record.source_file,
  829:             "source_line": record.source_line,
  830:             "unk_count": len(record.unk_positions),
  831:             "unk_positions": record.unk_positions,
  832:             "unk_field_indices": record.unk_field_indices,
  833:             "unk_chars": record.unk_chars,
  834:         }
  835:         for record in records
  836:         if record.unk_positions
  837:     ]
  838: 
  839: 
  840: def _overflow_index_rows(skips: list[_OverflowSkip]) -> list[dict]:
  841:     rows: list[dict] = []
  842:     for skip in skips:
  843:         row: dict = {
  844:             "sample_id": skip.sample_id,
  845:             "source_file": skip.source_file,
  846:             "source_line": skip.source_line,
  847:             "token_count": skip.token_count,
  848:             "max_unit_size": skip.max_unit_size,
  849:             "action": skip.action,
  850:         }
  851:         if skip.new_unit_size is not None:
  852:             row["new_unit_size"] = skip.new_unit_size
  853:         rows.append(row)
  854:     return rows
  855: 
  856: 
  857: def _hash_rows(records: list[_BatchRecord]) -> list[dict]:
  858:     return [
  859:         {
  860:             "sample_id": record.sample_id,
  861:             "hash": record.content_hash,
  862:             "source_file": record.source_file,
  863:             "source_line": record.source_line,
  864:         }
  865:         for record in records
  866:     ]
  867: 
  868: 
  869: def _content_hash(rendered: str) -> str:
  870:     return hashlib.sha256(rendered.encode("utf-8")).hexdigest()
  871: 
  872: 
  873: def _log_batch_progress(
  874:     *,
  875:     out_dir: Path,
  876:     batch_index: int,
  877:     total_samples: int,
  878:     total_tokens: int,
  879:     records: list[_BatchRecord],
  880: ) -> None:
  881:     log_event(
  882:         "progress",
  883:         batch=batch_index,
  884:         batch_samples=len(records),
  885:         total_samples=total_samples,
  886:         batch_tokens=sum(record.token_count for record in records),
  887:         total_tokens=total_tokens,
  888:     )

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\sources\template.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import re
    4: from dataclasses import dataclass
    5: from typing import Literal
    6: 
    7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
    8: from mintdim.apis.unit_build.errors import TemplateFieldMismatchError
    9: 
   10: 
   11: _PLACEHOLDER_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
   12: 
   13: 
   14: @dataclass(frozen=True)
   15: class Segment:
   16:     kind: Literal["literal", "field"]
   17:     name: str
   18:     content: str | None
   19: 
   20: 
   21: @dataclass(frozen=True)
   22: class CompiledTemplate:
   23:     raw: str
   24:     segments: list[Segment]
   25:     sequence_template: list[str]
   26:     placeholders: list[str]
   27: 
   28: 
   29: @dataclass(frozen=True)
   30: class LiteralUnkInfo:
   31:     segment_index: int
   32:     literal: str
   33:     unk_count: int
   34: 
   35: 
   36: def compile_template(
   37:     *,
   38:     file_index: int,
   39:     template: str,
   40:     declared_fields: list[str],
   41:     source_files: list[str] | None = None,
   42: ) -> CompiledTemplate:
   43:     source_files = source_files or []
   44:     placeholders: list[str] = []
   45:     raw_static_parts: list[str] = []
   46:     cursor = 0
   47: 
   48:     for match in _PLACEHOLDER_RE.finditer(template):
   49:         raw_static_parts.append(template[cursor : match.start()])
   50:         name = match.group(1)
   51:         if name not in declared_fields:
   52:             raise TemplateFieldMismatchError(
   53:                 file_index=file_index,
   54:                 template=template,
   55:                 unknown_placeholder=name,
   56:                 declared_fields=declared_fields,
   57:                 source_files=source_files,
   58:                 template_fields=[*placeholders, name],
   59:             )
   60:         placeholders.append(name)
   61:         cursor = match.end()
   62:     raw_static_parts.append(template[cursor:])
   63: 
   64:     unused_fields = sorted(set(declared_fields) - set(placeholders))
   65:     if unused_fields:
   66:         raise TemplateFieldMismatchError(
   67:             file_index=file_index,
   68:             template=template,
   69:             declared_fields=declared_fields,
   70:             source_files=source_files,
   71:             template_fields=placeholders,
   72:             unused_fields=unused_fields,
   73:         )
   74: 
   75:     segments: list[Segment] = []
   76:     for i, static in enumerate(raw_static_parts):
   77:         if static:
   78:             segments.append(
   79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
   80:             )
   81:         if i < len(placeholders):
   82:             segments.append(
   83:                 Segment(kind="field", name=placeholders[i], content=None)
   84:             )
   85: 
   86:     return CompiledTemplate(
   87:         raw=template,
   88:         segments=segments,
   89:         sequence_template=[seg.name for seg in segments],
   90:         placeholders=placeholders,
   91:     )
   92: 
   93: 
   94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
   95:     """Return one rendered text per segment, parallel to compiled.segments."""
   96:     out: list[str] = []
   97:     for seg in compiled.segments:
   98:         if seg.kind == "literal":
   99:             out.append(seg.content or "")
  100:         else:
  101:             out.append(str(sample[seg.name]))
  102:     return out
  103: 
  104: 
  105: def render(compiled: CompiledTemplate, sample: dict) -> str:
  106:     """Concatenated rendering â€” used for content hashing and offset recovery."""
  107:     return "".join(render_segments(compiled, sample))
  108: 
  109: 
  110: def preflight_template(
  111:     *,
  112:     compiled: CompiledTemplate,
  113:     tokenizer,
  114: ) -> list[LiteralUnkInfo]:
  115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
  116: 
  117:     Returns a (possibly empty) list of LiteralUnkInfo. The caller decides
  118:     whether to abort, prompt, or continue.
  119:     """
  120:     findings: list[LiteralUnkInfo] = []
  121:     for index, seg in enumerate(compiled.segments):
  122:         if seg.kind != "literal":
  123:             continue
  124:         literal = seg.content or ""
  125:         if not literal:
  126:             continue
  127:         ids = tokenizer.encode_batch([literal])[0]
  128:         unk_count = sum(1 for tid in ids if tid == tokenizer.unk_id)
  129:         if unk_count:
  130:             findings.append(
  131:                 LiteralUnkInfo(
  132:                     segment_index=index,
  133:                     literal=literal,
  134:                     unk_count=unk_count,
  135:                 )
  136:             )
  137:     return findings
  138: 
  139: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\contracts.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from dataclasses import dataclass
    4: from typing import Protocol, Literal
    5: 
    6: TemplateUnkPolicy = Literal["prompt", "abort", "continue"]
    7: OverflowPolicy = Literal["prompt", "abort", "truncate", "skip"]
    8: OverflowScope = Literal["only", "all"]
    9: 
   10: 
   11: class TokenizerHandle(Protocol):
   12:     vocab_size: int
   13:     unk_id: int
   14:     pad_id: int
   15: 
   16:     def encode_batch(self, texts: list[str]) -> list[list[int]]: ...
   17: 
   18:     def encode_with_offsets(
   19:         self, text: str
   20:     ) -> tuple[list[int], list[str]]:
   21:         """Return (token_ids, surfaces) where surfaces[i] is the slice of the
   22:         input that produced token_ids[i]. Used to recover the original
   23:         characters behind an UNK token."""
   24:         ...
   25: 
   26: 
   27: @dataclass(frozen=True)
   28: class TokenizerMetadata:
   29:     type: str
   30:     path: str
   31:     vocab_size: int
   32:     unk_id: int
   33:     pad_id: int

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\tokenizers\hf_json.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from mintdim.apis.unit_build.errors import TokenizerSpecialTokenError
    4: from mintdim.apis.unit_build.config import TokenizerEntry
    5: 
    6: 
    7: class HFTokenizerHandle:
    8:     """Adapter exposing the TokenizerHandle protocol over a Hugging Face tokenizer."""
    9: 
   10:     __slots__ = ("_tk", "vocab_size", "unk_id", "pad_id")
   11: 
   12:     def __init__(self, tk, *, unk_id: int, pad_id: int) -> None:
   13:         self._tk = tk
   14:         self.vocab_size = int(tk.get_vocab_size())
   15:         self.unk_id = unk_id
   16:         self.pad_id = pad_id
   17: 
   18:     def encode_batch(self, texts: list[str]) -> list[list[int]]:
   19:         encodings = self._tk.encode_batch(texts, add_special_tokens=False)
   20:         return [list(e.ids) for e in encodings]
   21: 
   22:     def encode_with_offsets(
   23:         self, text: str
   24:     ) -> tuple[list[int], list[str]]:
   25:         encoding = self._tk.encode(text, add_special_tokens=False)
   26:         ids = [int(t) for t in encoding.ids]
   27:         surfaces = [text[start:end] for (start, end) in encoding.offsets]
   28:         return ids, surfaces
   29: 
   30: 
   31: def load_hf_json(entry: TokenizerEntry) -> HFTokenizerHandle:
   32:     from tokenizers import Tokenizer
   33: 
   34:     tk = Tokenizer.from_file(entry.path)
   35:     handle = HFTokenizerHandle(
   36:         tk,
   37:         unk_id=_hf_unk_id(tk, path=entry.path),
   38:         pad_id=_hf_pad_id(tk, path=entry.path),
   39:     )
   40:     return handle
   41: 
   42: 
   43: def _hf_unk_id(tk, *, path: str) -> int:
   44:     model = getattr(tk, "model", None)
   45:     unk_token = getattr(model, "unk_token", None)
   46:     if isinstance(unk_token, str):
   47:         token_id = tk.token_to_id(unk_token)
   48:         if token_id is not None:
   49:             return int(token_id)
   50: 
   51:     raise TokenizerSpecialTokenError(
   52:         tokenizer_path=path,
   53:         token_name="unk_id",
   54:         problem="Hugging Face tokenizer model did not expose a resolvable unk_token",
   55:     )
   56: 
   57: 
   58: def _hf_pad_id(tk, *, path: str) -> int:
   59:     padding = tk.padding
   60:     if isinstance(padding, dict):
   61:         pad_id = padding.get("pad_id")
   62:         if pad_id is not None:
   63:             return int(pad_id)
   64: 
   65:         pad_token = padding.get("pad_token")
   66:         if isinstance(pad_token, str):
   67:             token_id = tk.token_to_id(pad_token)
   68:             if token_id is not None:
   69:                 return int(token_id)
   70: 
   71:     raise TokenizerSpecialTokenError(
   72:         tokenizer_path=path,
   73:         token_name="pad_id",
   74:         problem="Hugging Face tokenizer JSON did not expose a resolvable padding config",
   75:     )
   76: 
   77: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\tokenizers\sentencepiece.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from mintdim.apis.unit_build.errors import TokenizerSpecialTokenError
    4: from mintdim.apis.unit_build.config import TokenizerEntry
    5: 
    6: 
    7: class SentencePieceHandle:
    8:     """Adapter exposing the TokenizerHandle protocol over a SentencePieceProcessor."""
    9: 
   10:     __slots__ = ("_sp", "vocab_size", "unk_id", "pad_id")
   11: 
   12:     def __init__(self, sp, *, unk_id: int, pad_id: int) -> None:
   13:         self._sp = sp
   14:         self.vocab_size = int(sp.get_piece_size())
   15:         self.unk_id = unk_id
   16:         self.pad_id = pad_id
   17: 
   18:     def encode_batch(self, texts: list[str]) -> list[list[int]]:
   19:         encoded = self._sp.encode(texts, out_type=int)
   20:         return [list(ids) for ids in encoded]
   21: 
   22:     def encode_with_offsets(
   23:         self, text: str
   24:     ) -> tuple[list[int], list[str]]:
   25:         proto = self._sp.encode(text, out_type="immutable_proto")
   26:         ids: list[int] = []
   27:         surfaces: list[str] = []
   28:         for piece in proto.pieces:
   29:             ids.append(int(piece.id))
   30:             surfaces.append(piece.surface)
   31:         return ids, surfaces
   32: 
   33: 
   34: def load_sentencepiece(entry: TokenizerEntry) -> SentencePieceHandle:
   35:     import sentencepiece as spm
   36: 
   37:     sp = spm.SentencePieceProcessor()
   38:     sp.Load(entry.path)
   39:     handle = SentencePieceHandle(
   40:         sp,
   41:         unk_id=_special_id(sp.unk_id(), token_name="unk_id", path=entry.path),
   42:         pad_id=_special_id(sp.pad_id(), token_name="pad_id", path=entry.path),
   43:     )
   44:     return handle
   45: 
   46: 
   47: def _special_id(value: int, *, token_name: str, path: str) -> int:
   48:     token_id = int(value)
   49:     if token_id < 0:
   50:         raise TokenizerSpecialTokenError(
   51:             tokenizer_path=path,
   52:             token_name=token_name,
   53:             problem=f"SentencePiece returned {token_name}={token_id}",
   54:         )
   55:     return token_id
   56: 
   57: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\tokenizers\__init__.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from mintdim.apis.unit_build.adapters.tokenizers.sentencepiece import SentencePieceHandle, load_sentencepiece
    4: from mintdim.apis.unit_build.adapters.tokenizers.hf_json import HFTokenizerHandle, load_hf_json
    5: from mintdim.apis.unit_build.config import TokenizerEntry
    6: from mintdim.apis.unit_build.contracts import TokenizerHandle
    7: from mintdim.apis.unit_build.errors import UnsupportedTokenizerFormatError
    8: 
    9: 
   10: def load_tokenizer(tokenizer_type: str, entry: TokenizerEntry) -> TokenizerHandle:
   11:     if tokenizer_type == "sentencepiece":
   12:         return load_sentencepiece(entry)
   13:     if tokenizer_type == "hf_json":
   14:         return load_hf_json(entry)
   15:     raise UnsupportedTokenizerFormatError()

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\output\shard_writer.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from dataclasses import dataclass
    4: from pathlib import Path
    5: 
    6: import numpy as np
    7: 
    8: from mintdim.apis.unit_build.ui.logger import log_event
    9: 
   10: from mintdim.apis.unit_build.domain.rules import (
   11:     BYTES_PER_TOKEN_BY_DTYPE as _BYTES_PER_TOKEN,
   12:     EMPTY_UNIT_NOTICE as _EMPTY_UNIT_NOTICE,
   13:     FAILED_UNIT_NOTICE as _FAILED_UNIT_NOTICE,
   14:     MAGIC_VALUE_BY_DTYPE as _MAGIC_VALUE,
   15:     RECOMMENDED_SHARD_MIB as _RECOMMENDED_SHARD_MIB,
   16:     TOKEN_DTYPE_CAPACITY as _MAX_VOCAB_FOR_DTYPE,
   17: )
   18: from mintdim.apis.unit_build.errors import TokenDTypeCapacityError
   19: 
   20: 
   21: _DTYPE_MAP = {
   22:     "uint16": np.uint16,
   23:     "uint32": np.uint32,
   24: }
   25: 
   26: 
   27: # Per-dtype capacity = the largest vocab_size that fits without colliding
   28: # with the reserved record-magic value. The magic value is also the max
   29: # representable integer for the dtype, so a vocab equal to the capacity
   30: # means token ids [0, capacity-1], which never produces an id that equals
   31: # the magic.
   32: 
   33: 
   34: def select_token_dtype(vocab_size: int) -> str:
   35:     for dtype, capacity in _MAX_VOCAB_FOR_DTYPE.items():
   36:         if vocab_size <= capacity:
   37:             return dtype
   38:     raise TokenDTypeCapacityError(
   39:         vocab_size=vocab_size,
   40:         dtype_capacities=_MAX_VOCAB_FOR_DTYPE,
   41:     )
   42: 
   43: 
   44: def bytes_per_token(dtype: str) -> int:
   45:     return _BYTES_PER_TOKEN[dtype]
   46: 
   47: 
   48: def magic_for_dtype(dtype: str) -> int:
   49:     return _MAGIC_VALUE[dtype]
   50: 
   51: 
   52: def assert_no_magic_collision(*, vocab_size: int, token_dtype: str) -> None:
   53:     capacity = _MAX_VOCAB_FOR_DTYPE[token_dtype]
   54:     if vocab_size > capacity:
   55:         raise TokenDTypeCapacityError(
   56:             vocab_size=vocab_size,
   57:             dtype_capacities={token_dtype: capacity},
   58:         )
   59: 
   60: 
   61: @dataclass(frozen=True)
   62: class ShardRecord:
   63:     """One sample's payload as it will be written to a shard.
   64: 
   65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
   66:     sum must be <= unit_size; the remainder of the payload area is padded.
   67:     `payload_tokens` is the concatenation of every segment's tokens, in
   68:     sequence_template order.
   69:     """
   70: 
   71:     segment_lengths: list[int]
   72:     payload_tokens: list[int]
   73: 
   74: 
   75: class ShardWriter:
   76:     """Append fixed-width records to `unit_<size>/shard_*.bin`.
   77: 
   78:     Each record on disk is laid out as:
   79:         [magic] [len_seg_0] ... [len_seg_{n-1}] [payload_0..unit_size-1]
   80:     All slots are `dtype` (uint16 or uint32). Total record size in slots
   81:     is `1 + n_segments + unit_size`.
   82:     """
   83: 
   84:     def __init__(
   85:         self,
   86:         *,
   87:         root: Path,
   88:         unit_size: int,
   89:         dtype: str,
   90:         pad_id: int,
   91:         samples_per_shard: int,
   92:         n_segments: int,
   93:     ) -> None:
   94:         self.root = Path(root)
   95:         self.unit_size = unit_size
   96:         self.dtype = dtype
   97:         self.np_dtype = _DTYPE_MAP[dtype]
   98:         self.pad_id = pad_id
   99:         self.samples_per_shard = samples_per_shard
  100:         self.n_segments = n_segments
  101:         self.magic = _MAGIC_VALUE[dtype]
  102:         self.record_slots = 1 + n_segments + unit_size
  103: 
  104:         self._shard_index = -1
  105:         self._samples_in_current = 0
  106:         self._total_samples = 0
  107:         self._bytes_in_current = 0
  108:         self._handle = None
  109:         self._current_shard_path: Path | None = None
  110: 
  111:         self.root.mkdir(parents=True, exist_ok=True)
  112: 
  113:     def write_batch(self, records: list[ShardRecord]) -> list[tuple[int, int]]:
  114:         """Append a batch and return (shard_index, position_in_shard) per record."""
  115:         locations: list[tuple[int, int]] = []
  116:         cursor = 0
  117:         while cursor < len(records):
  118:             if self._handle is None or self._samples_in_current >= self.samples_per_shard:
  119:                 self._roll()
  120: 
  121:             capacity = self.samples_per_shard - self._samples_in_current
  122:             chunk = records[cursor : cursor + capacity]
  123:             shard_index = self._shard_index
  124:             start_position = self._samples_in_current
  125: 
  126:             arr = np.full(
  127:                 (len(chunk), self.record_slots),
  128:                 self.pad_id,
  129:                 dtype=self.np_dtype,
  130:             )
  131:             payload_offset = 1 + self.n_segments
  132:             for row, rec in enumerate(chunk):
  133:                 arr[row, 0] = self.magic
  134:                 if len(rec.segment_lengths) != self.n_segments:
  135:                     raise ValueError(
  136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
  137:                         f"got {len(rec.segment_lengths)}"
  138:                     )
  139:                 for j, length in enumerate(rec.segment_lengths):
  140:                     arr[row, 1 + j] = length
  141:                 payload = rec.payload_tokens
  142:                 if len(payload) > self.unit_size:
  143:                     raise ValueError(
  144:                         f"payload_tokens length {len(payload)} exceeds unit_size {self.unit_size}"
  145:                     )
  146:                 arr[row, payload_offset : payload_offset + len(payload)] = payload
  147:             arr.tofile(self._handle)
  148: 
  149:             locations.extend(
  150:                 (shard_index, start_position + offset) for offset in range(len(chunk))
  151:             )
  152:             self._samples_in_current += len(chunk)
  153:             self._total_samples += len(chunk)
  154:             self._bytes_in_current += (
  155:                 len(chunk) * self.record_slots * _BYTES_PER_TOKEN[self.dtype]
  156:             )
  157:             cursor += len(chunk)
  158: 
  159:         return locations
  160: 
  161:     def close(self, *, success: bool = True, failure_reason: str | None = None) -> None:
  162:         if self._handle is not None:
  163:             self._handle.close()
  164:             self._handle = None
  165:             self._emit_shard_status()
  166: 
  167:         if success and self._total_samples == 0:
  168:             self._write_empty_unit_notice()
  169:         elif not success:
  170:             self._write_failed_unit_notice(failure_reason)
  171: 
  172:     def _roll(self) -> None:
  173:         if self._handle is not None:
  174:             self._handle.close()
  175:             self._emit_shard_status()
  176:         self._shard_index += 1
  177:         path = self.root / f"shard_{self._shard_index:06d}.bin"
  178:         self._handle = open(path, "wb")
  179:         self._current_shard_path = path
  180:         self._samples_in_current = 0
  181:         self._bytes_in_current = 0
  182: 
  183:     def _emit_shard_status(self) -> None:
  184:         mib = self._bytes_in_current / (1024 * 1024)
  185: 
  186:         fields: dict[str, object] = {
  187:             "shard": f"{self._shard_index:06d}",
  188:             "samples": self._samples_in_current,
  189:             "bytes": f"{mib:.1f} MiB",
  190:             "unit": self.unit_size,
  191:         }
  192:         if self._current_shard_path is not None:
  193:             fields["path"] = str(self._current_shard_path)
  194: 
  195:         log_event("shard", **fields)
  196: 
  197:         if mib > _RECOMMENDED_SHARD_MIB:
  198:             log_event(
  199:                 "warning",
  200:                 shard=f"{self._shard_index:06d}",
  201:                 size=f"{mib:.1f} MiB",
  202:                 recommended=f"{_RECOMMENDED_SHARD_MIB:.1f} MiB",
  203:             )
  204: 
  205:     def _write_empty_unit_notice(self) -> None:
  206:         path = self.root / _EMPTY_UNIT_NOTICE
  207:         path.write_text(
  208:             "\n".join(
  209:                 [
  210:                     "# EmptyUnitNotice",
  211:                     "",
  212:                     "Status:",
  213:                     "No samples were assigned to this unit.",
  214:                     "",
  215:                     "Unit:",
  216:                     str(self.unit_size),
  217:                     "",
  218:                     "Reason:",
  219:                     (
  220:                         "No rendered JSONL sample produced a token_count that "
  221:                         "the unit planner assigned to this unit."
  222:                     ),
  223:                     "",
  224:                     "This is not an error:",
  225:                     (
  226:                         "The build completed normally. The unit directory is kept "
  227:                         "so the declared output layout remains visible."
  228:                     ),
  229:                     "",
  230:                 ]
  231:             ),
  232:             encoding="utf-8",
  233:         )
  234: 
  235:     def _write_failed_unit_notice(self, failure_reason: str | None) -> None:
  236:         path = self.root / _FAILED_UNIT_NOTICE
  237:         path.write_text(
  238:             "\n".join(
  239:                 [
  240:                     "# UnitBuildFailedNotice",
  241:                     "",
  242:                     "Status:",
  243:                     "The unit-build run failed before this output was finalized.",
  244:                     "",
  245:                     "Unit:",
  246:                     str(self.unit_size),
  247:                     "",
  248:                     "Samples written before failure:",
  249:                     str(self._total_samples),
  250:                     "",
  251:                     "Failure:",
  252:                     failure_reason or "Unknown failure",
  253:                     "",
  254:                     "This is not an empty unit:",
  255:                     (
  256:                         "EMPTY_UNIT.md is written only after a successful build "
  257:                         "where no samples were assigned to this unit."
  258:                     ),
  259:                     "",
  260:                 ]
  261:             ),
  262:             encoding="utf-8",
  263:         )

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\output\length_field.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import json
    4: from pathlib import Path
    5: 
    6: 
    7: def write_length_field(
    8:     *,
    9:     out_dir: Path,
   10:     magic: int,
   11:     sequence_template: list[str],
   12:     token_dtype: str,
   13:     pad_token_id: int,
   14: ) -> Path:
   15:     """Write `length_field.json` describing the per-record header layout.
   16: 
   17:     Readers use this file to:
   18:       - verify the magic marker at offset 0 of every record
   19:       - decode the n length headers that follow
   20:       - know which segment (field or literal) each header refers to
   21:     """
   22:     payload = {
   23:         "magic": magic,
   24:         "sequence_template": list(sequence_template),
   25:         "token_dtype": token_dtype,
   26:         "pad_token_id": pad_token_id,
   27:     }
   28:     path = out_dir / "length_field.json"
   29:     with open(path, "w", encoding="utf-8") as fp:
   30:         json.dump(payload, fp, ensure_ascii=False, indent=2)
   31:     return path
   32: 
   33: 

====================================================================================================
FILE: tests\unit\apis\unit_build\test_template.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import pytest
    4: 
    5: from mintdim.apis.unit_build.adapters.sources.template import (
    6:     LITERAL_SEGMENT_NAME,
    7:     LiteralUnkInfo,
    8:     compile_template,
    9:     preflight_template,
   10:     render,
   11:     render_segments,
   12: )
   13: from mintdim.apis.unit_build.errors import TemplateFieldMismatchError
   14: 
   15: 
   16: def test_compile_simple_template():
   17:     compiled = compile_template(
   18:         file_index=0,
   19:         template="Q: {q} A: {a}",
   20:         declared_fields=["q", "a"],
   21:     )
   22:     assert compiled.placeholders == ["q", "a"]
   23:     assert compiled.sequence_template == [
   24:         LITERAL_SEGMENT_NAME,
   25:         "q",
   26:         LITERAL_SEGMENT_NAME,
   27:         "a",
   28:     ]
   29:     assert [seg.kind for seg in compiled.segments] == [
   30:         "literal",
   31:         "field",
   32:         "literal",
   33:         "field",
   34:     ]
   35:     assert compiled.segments[0].content == "Q: "
   36:     assert compiled.segments[2].content == " A: "
   37: 
   38: 
   39: def test_compile_skips_empty_static_parts():
   40:     compiled = compile_template(
   41:         file_index=0,
   42:         template="{a}{b}",
   43:         declared_fields=["a", "b"],
   44:     )
   45:     assert compiled.sequence_template == ["a", "b"]
   46:     assert all(seg.kind == "field" for seg in compiled.segments)
   47: 
   48: 
   49: def test_compile_template_with_no_placeholders():
   50:     compiled = compile_template(
   51:         file_index=0,
   52:         template="static only",
   53:         declared_fields=[],
   54:     )
   55:     assert compiled.placeholders == []
   56:     assert compiled.sequence_template == [LITERAL_SEGMENT_NAME]
   57:     assert compiled.segments[0].content == "static only"
   58: 
   59: 
   60: def test_compile_rejects_unknown_placeholder():
   61:     with pytest.raises(TemplateFieldMismatchError) as exc:
   62:         compile_template(
   63:             file_index=2,
   64:             template="{q} -> {missing}",
   65:             declared_fields=["q"],
   66:         )
   67:     assert exc.value.file_index == 2
   68:     assert exc.value.unknown_placeholder == "missing"
   69: 
   70: 
   71: def test_compile_rejects_unused_declared_field():
   72:     with pytest.raises(TemplateFieldMismatchError) as exc:
   73:         compile_template(
   74:             file_index=1,
   75:             template="{prompt}{separator}{answer}",
   76:             declared_fields=["prompt", "answer", "separator", "source"],
   77:         )
   78: 
   79:     assert exc.value.file_index == 1
   80:     assert exc.value.unused_fields == ["source"]
   81: 
   82: 
   83: def test_render_interpolates_placeholders():
   84:     compiled = compile_template(
   85:         file_index=0,
   86:         template="{a}-{b}",
   87:         declared_fields=["a", "b"],
   88:     )
   89:     assert render(compiled, {"a": "hello", "b": "world"}) == "hello-world"
   90: 
   91: 
   92: def test_render_segments_returns_parallel_list():
   93:     compiled = compile_template(
   94:         file_index=0,
   95:         template="Q: {q} A: {a}",
   96:         declared_fields=["q", "a"],
   97:     )
   98:     parts = render_segments(compiled, {"q": "what", "a": "this"})
   99:     assert parts == ["Q: ", "what", " A: ", "this"]
  100:     assert len(parts) == len(compiled.segments)
  101: 
  102: 
  103: def test_render_stringifies_non_strings():
  104:     compiled = compile_template(
  105:         file_index=0,
  106:         template="{n}",
  107:         declared_fields=["n"],
  108:     )
  109:     assert render(compiled, {"n": 42}) == "42"
  110: 
  111: 
  112: def test_preflight_returns_empty_when_clean(fake_tokenizer):
  113:     compiled = compile_template(
  114:         file_index=0,
  115:         template="Q: {q} A: {a}",
  116:         declared_fields=["q", "a"],
  117:     )
  118:     findings = preflight_template(compiled=compiled, tokenizer=fake_tokenizer)
  119:     assert findings == []
  120: 
  121: 
  122: def test_preflight_reports_unk_findings():
  123:     from tests.conftest import FakeTokenizerHandle
  124: 
  125:     handle = FakeTokenizerHandle(unk_chars={"Z"})
  126:     compiled = compile_template(
  127:         file_index=4,
  128:         template="Z {x}",
  129:         declared_fields=["x"],
  130:     )
  131:     findings = preflight_template(compiled=compiled, tokenizer=handle)
  132:     assert len(findings) == 1
  133:     assert isinstance(findings[0], LiteralUnkInfo)
  134:     assert findings[0].segment_index == 0
  135:     assert "Z" in findings[0].literal
  136:     assert findings[0].unk_count >= 1

====================================================================================================
FILE: tests\unit\apis\unit_build\test_shard_writer.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from pathlib import Path
    4: 
    5: import numpy as np
    6: import pytest
    7: 
    8: from mintdim.apis.unit_build.adapters.output.shard_writer import (
    9:     ShardRecord,
   10:     ShardWriter,
   11:     assert_no_magic_collision,
   12:     bytes_per_token,
   13:     magic_for_dtype,
   14:     select_token_dtype,
   15: )
   16: from mintdim.apis.unit_build.errors import TokenDTypeCapacityError
   17: 
   18: 
   19: def test_select_token_dtype_uint16_boundary():
   20:     assert select_token_dtype(1) == "uint16"
   21:     assert select_token_dtype(0xFFFF) == "uint16"
   22: 
   23: 
   24: def test_select_token_dtype_promotes_at_magic_boundary():
   25:     assert select_token_dtype(0xFFFF + 1) == "uint32"
   26: 
   27: 
   28: def test_select_token_dtype_uint32_boundary():
   29:     assert select_token_dtype(0xFFFFFFFF) == "uint32"
   30: 
   31: 
   32: def test_select_token_dtype_overflow():
   33:     with pytest.raises(TokenDTypeCapacityError):
   34:         select_token_dtype(0xFFFFFFFF + 1)
   35: 
   36: 
   37: def test_bytes_per_token():
   38:     assert bytes_per_token("uint16") == 2
   39:     assert bytes_per_token("uint32") == 4
   40: 
   41: 
   42: def test_magic_for_dtype():
   43:     assert magic_for_dtype("uint16") == 0xFFFF
   44:     assert magic_for_dtype("uint32") == 0xFFFFFFFF
   45: 
   46: 
   47: def test_assert_no_magic_collision_passes():
   48:     assert_no_magic_collision(vocab_size=0xFFFF, token_dtype="uint16")
   49: 
   50: 
   51: def test_assert_no_magic_collision_raises():
   52:     with pytest.raises(TokenDTypeCapacityError) as exc:
   53:         assert_no_magic_collision(vocab_size=0xFFFF + 1, token_dtype="uint16")
   54:     assert exc.value.vocab_size == 0xFFFF + 1
   55:     assert exc.value.dtype_capacities == {"uint16": 0xFFFF}
   56: 
   57: 
   58: def _read_records(path: Path, dtype, record_slots: int) -> np.ndarray:
   59:     flat = np.fromfile(path, dtype=dtype)
   60:     assert flat.size % record_slots == 0
   61:     return flat.reshape(-1, record_slots)
   62: 
   63: 
   64: def test_shard_writer_emits_magic_lengths_payload(tmp_path: Path):
   65:     sw = ShardWriter(
   66:         root=tmp_path / "unit_8",
   67:         unit_size=8,
   68:         dtype="uint16",
   69:         pad_id=0,
   70:         samples_per_shard=10,
   71:         n_segments=3,
   72:     )
   73:     rec = ShardRecord(
   74:         segment_lengths=[2, 1, 2],
   75:         payload_tokens=[10, 20, 30, 40, 50],
   76:     )
   77:     locations = sw.write_batch([rec])
   78:     sw.close()
   79: 
   80:     assert locations == [(0, 0)]
   81:     arr = _read_records(
   82:         tmp_path / "unit_8" / "shard_000000.bin",
   83:         np.uint16,
   84:         record_slots=1 + 3 + 8,
   85:     )
   86:     assert arr.shape == (1, 12)
   87:     assert arr[0, 0] == 0xFFFF
   88:     assert list(arr[0, 1:4]) == [2, 1, 2]
   89:     assert list(arr[0, 4:9]) == [10, 20, 30, 40, 50]
   90:     assert list(arr[0, 9:]) == [0, 0, 0]
   91: 
   92: 
   93: def test_shard_writer_uses_uint32_magic(tmp_path: Path):
   94:     sw = ShardWriter(
   95:         root=tmp_path / "unit_4",
   96:         unit_size=4,
   97:         dtype="uint32",
   98:         pad_id=0,
   99:         samples_per_shard=10,
  100:         n_segments=1,
  101:     )
  102:     sw.write_batch([ShardRecord(segment_lengths=[2], payload_tokens=[100, 200])])
  103:     sw.close()
  104: 
  105:     arr = _read_records(
  106:         tmp_path / "unit_4" / "shard_000000.bin",
  107:         np.uint32,
  108:         record_slots=1 + 1 + 4,
  109:     )
  110:     assert arr[0, 0] == 0xFFFFFFFF
  111:     assert arr[0, 1] == 2
  112:     assert list(arr[0, 2:4]) == [100, 200]
  113: 
  114: 
  115: def test_shard_writer_pads_payload_with_pad_id(tmp_path: Path):
  116:     sw = ShardWriter(
  117:         root=tmp_path / "unit_8",
  118:         unit_size=8,
  119:         dtype="uint16",
  120:         pad_id=7,
  121:         samples_per_shard=10,
  122:         n_segments=1,
  123:     )
  124:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
  125:     sw.close()
  126: 
  127:     arr = _read_records(
  128:         tmp_path / "unit_8" / "shard_000000.bin",
  129:         np.uint16,
  130:         record_slots=1 + 1 + 8,
  131:     )
  132:     assert list(arr[0, 2:5]) == [1, 2, 3]
  133:     assert list(arr[0, 5:]) == [7, 7, 7, 7, 7]
  134: 
  135: 
  136: def test_shard_writer_rolls_over_at_samples_per_shard(tmp_path: Path):
  137:     sw = ShardWriter(
  138:         root=tmp_path / "unit_4",
  139:         unit_size=4,
  140:         dtype="uint16",
  141:         pad_id=0,
  142:         samples_per_shard=2,
  143:         n_segments=1,
  144:     )
  145:     records = [
  146:         ShardRecord(segment_lengths=[2], payload_tokens=[i, i + 1])
  147:         for i in range(5)
  148:     ]
  149:     locations = sw.write_batch(records)
  150:     sw.close()
  151: 
  152:     assert locations == [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0)]
  153:     shards = sorted((tmp_path / "unit_4").glob("shard_*.bin"))
  154:     assert len(shards) == 3
  155:     record_bytes = (1 + 1 + 4) * 2
  156:     assert shards[0].stat().st_size == 2 * record_bytes
  157:     assert shards[1].stat().st_size == 2 * record_bytes
  158:     assert shards[2].stat().st_size == 1 * record_bytes
  159: 
  160: 
  161: def test_shard_writer_rejects_segment_length_arity(tmp_path: Path):
  162:     sw = ShardWriter(
  163:         root=tmp_path / "unit_4",
  164:         unit_size=4,
  165:         dtype="uint16",
  166:         pad_id=0,
  167:         samples_per_shard=10,
  168:         n_segments=2,
  169:     )
  170:     with pytest.raises(ValueError, match="segment_lengths"):
  171:         sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
  172:     sw.close()
  173: 
  174: 
  175: def test_shard_writer_rejects_payload_overflow(tmp_path: Path):
  176:     sw = ShardWriter(
  177:         root=tmp_path / "unit_4",
  178:         unit_size=4,
  179:         dtype="uint16",
  180:         pad_id=0,
  181:         samples_per_shard=10,
  182:         n_segments=1,
  183:     )
  184:     with pytest.raises(ValueError, match="exceeds unit_size"):
  185:         sw.write_batch(
  186:             [ShardRecord(segment_lengths=[5], payload_tokens=[1, 2, 3, 4, 5])]
  187:         )
  188:     sw.close()
  189: 
  190: 
  191: def test_shard_writer_writes_empty_unit_notice_when_no_samples(tmp_path: Path):
  192:     sw = ShardWriter(
  193:         root=tmp_path / "unit_16",
  194:         unit_size=16,
  195:         dtype="uint16",
  196:         pad_id=0,
  197:         samples_per_shard=10,
  198:         n_segments=2,
  199:     )
  200:     sw.close()
  201: 
  202:     notice = tmp_path / "unit_16" / "EMPTY_UNIT.md"
  203:     assert notice.exists()
  204:     assert not list((tmp_path / "unit_16").glob("shard_*.bin"))
  205:     text = notice.read_text(encoding="utf-8")
  206:     assert "EmptyUnitNotice" in text
  207:     assert "No samples were assigned to this unit." in text
  208:     assert "Unit:\n16" in text
  209: 
  210: 
  211: def test_shard_writer_does_not_write_empty_unit_notice_when_samples_exist(tmp_path: Path):
  212:     sw = ShardWriter(
  213:         root=tmp_path / "unit_16",
  214:         unit_size=16,
  215:         dtype="uint16",
  216:         pad_id=0,
  217:         samples_per_shard=10,
  218:         n_segments=1,
  219:     )
  220:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
  221:     sw.close()
  222: 
  223:     assert not (tmp_path / "unit_16" / "EMPTY_UNIT.md").exists()
  224: 
  225: 
  226: def test_shard_writer_writes_failed_notice_on_failed_close(tmp_path: Path):
  227:     sw = ShardWriter(
  228:         root=tmp_path / "unit_16",
  229:         unit_size=16,
  230:         dtype="uint16",
  231:         pad_id=0,
  232:         samples_per_shard=10,
  233:         n_segments=1,
  234:     )
  235:     sw.close(success=False, failure_reason="UnitOverflowError: sample too long")
  236: 
  237:     failed = tmp_path / "unit_16" / "UNIT_BUILD_FAILED.md"
  238:     assert failed.exists()
  239:     text = failed.read_text(encoding="utf-8")
  240:     assert "UnitBuildFailedNotice" in text
  241:     assert "UnitOverflowError: sample too long" in text
  242: 
  243: 
  244: def test_shard_writer_zero_padded_six_digit_filenames(tmp_path: Path):
  245:     sw = ShardWriter(
  246:         root=tmp_path / "unit_4",
  247:         unit_size=4,
  248:         dtype="uint16",
  249:         pad_id=0,
  250:         samples_per_shard=1,
  251:         n_segments=1,
  252:     )
  253:     sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
  254:     sw.close()
  255:     assert (tmp_path / "unit_4" / "shard_000000.bin").exists()

====================================================================================================
FILE: tests\unit\apis\unit_build\test_units.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import pytest
    4: 
    5: from mintdim.apis.unit_build.domain.planner import assign_unit
    6: from mintdim.apis.unit_build.validation import validate_units
    7: from mintdim.apis.unit_build.errors import UnitOverflowError, UnitValidationError
    8: 
    9: 
   10: def test_assign_unit_picks_smallest_fit():
   11:     assert assign_unit(token_count=10, sizes=[64, 128, 256], sample_locator="x:1") == 64
   12:     assert assign_unit(token_count=64, sizes=[64, 128, 256], sample_locator="x:1") == 64
   13:     assert assign_unit(token_count=65, sizes=[64, 128, 256], sample_locator="x:1") == 128
   14: 
   15: 
   16: def test_assign_unit_overflow_raises():
   17:     with pytest.raises(UnitOverflowError) as exc:
   18:         assign_unit(token_count=999, sizes=[64, 128], sample_locator="train.jsonl:42")
   19:     assert exc.value.token_count == 999
   20:     assert exc.value.max_unit_size == 128
   21:     assert exc.value.sample == "train.jsonl:42"
   22: 
   23: 
   24: def test_validate_units_happy():
   25:     cfg = validate_units({"sizes": [[64, 128, 256]], "build_batch": [4096]})
   26:     assert cfg.sizes == [[64, 128, 256]]
   27:     assert cfg.build_batch == [4096]
   28: 
   29: 
   30: def test_validate_units_rejects_unsorted():
   31:     with pytest.raises(UnitValidationError):
   32:         validate_units({"sizes": [[128, 64]], "build_batch": [4096]})
   33: 
   34: 
   35: def test_validate_units_rejects_duplicates():
   36:     with pytest.raises(UnitValidationError):
   37:         validate_units({"sizes": [[64, 64]], "build_batch": [4096]})
   38: 
   39: 
   40: def test_validate_units_rejects_non_positive_sizes():
   41:     with pytest.raises(UnitValidationError):
   42:         validate_units({"sizes": [[0, 64]], "build_batch": [4096]})
   43: 
   44: 
   45: def test_validate_units_rejects_non_int_sizes():
   46:     with pytest.raises(UnitValidationError):
   47:         validate_units({"sizes": [[64.0, 128.0]], "build_batch": [4096]})
   48: 
   49: 
   50: def test_validate_units_rejects_non_positive_batch():
   51:     with pytest.raises(UnitValidationError):
   52:         validate_units({"sizes": [[64]], "build_batch": [0]})
   53: 
   54: 
   55: def test_validate_units_empty_sizes():
   56:     with pytest.raises(UnitValidationError):
   57:         validate_units({"sizes": [], "build_batch": [4096]})
   58: 
   59: 
   60: def test_validate_units_empty_group():
   61:     with pytest.raises(UnitValidationError):
   62:         validate_units({"sizes": [[]], "build_batch": [4096]})

====================================================================================================
FILE: tests\apis\unit_build\test_template.py
====================================================================================================
[MISSING]

====================================================================================================
FILE: tests\apis\unit_build\test_shard_writer.py
====================================================================================================
[MISSING]

====================================================================================================
FILE: tests\apis\unit_build\test_units.py
====================================================================================================
[MISSING]

====================================================================================================
SEARCH: segment encoding / offset references
====================================================================================================
README.md:77: → tokenizer.encode_batch(texts)
README.md:155: - any other character is treated as a literal (\n, \t, custom separators, etc.)
README.md:270: - treat every character outside {placeholders} in a template as a literal
README.md:271: - accept any template literal whose tokenization does not produce UNK
README.md:291:   as long as your tokenizer encodes the literal characters without UNK
README.md:292: - literal characters between placeholders are tokenized by the same
README.md:301: Hugging Face : Tokenizer.encode_batch(texts, add_special_tokens=False)
README.md:305: directly in your template. MintDim will pass it to the tokenizer as literal
README.md:312: runs on every text MintDim hands to the tokenizer, including template literals
README.md:424: Before any data is processed, every literal segment of every template is tokenized once. If any literal produces UNK, MintDim **prompts the user before building**:
README.md:429:   literal segments with UNK <n>
README.md:447: When the build continues with template UNK, the literal UNK positions are recorded in `unk_index.jsonl` like any other UNK — each entry carries `unk_field_indices` so readers can tell which segment the UNK came from.
README.md:453: - `unk_field_indices` — which segment of `sequence_template` each UNK came from (0-base)
README.md:1073: Template literal segments produce UNK when encoded by the tokenizer.
README.md:1082: template literal characters are not fully covered by tokenizer vocabulary
README.md:1085: adjust template literal characters, use a tokenizer that covers them, or run with an explicit template-UNK policy.
README.md:1089: Các đoạn literal của template tạo UNK khi được tokenizer encode.
README.md:1098: ký tự literal trong template không được tokenizer vocabulary bao phủ đầy đủ
README.md:1101: chỉnh ký tự literal trong template, dùng tokenizer bao phủ các ký tự đó, hoặc chạy với template-UNK policy rõ ràng.
README.md:1465: → pipeline metadata, build configuration, tokenizer metadata, token dtype, magic, sequence_template
README.md:1468: → per-record header schema readers use to decode shards (magic, sequence_template, dtype, pad_token_id)
README.md:1519:   "sequence_template": [
README.md:1552:   "sequence_template": [
README.md:1567: - decode the N length headers that follow (`N = len(sequence_template)`)
README.md:1568: - know which segment (field or literal) each header refers to
README.md:1570: `"token_template"` is the reserved name for a literal segment of the template. Field segments use their declared field name.
README.md:1633: `empty_fields` lists the field names whose values rendered to an empty string for this sample (the field stays in `sequence_template` with `len_field = 0`). It's a fast filter for finding samples missing actual content in a slot.
README.md:1691: Each line describes one sample whose tokenization produced UNK. This includes both UNK from field values (Tier 2) and, when the build was allowed to continue past Tier 1, UNK from template literals.
README.md:1715: → segment index in sequence_template (0-base) where the UNK originated;
README.md:1773: - `n = len(sequence_template)`. Segment names live in `length_field.json`.
README.md:1775: - The payload concatenates each segment's tokens in `sequence_template` order, then pads with `pad_id` to `unit_size`.
README.md:1870: sequence_template preserves field/literal boundaries on disk
_code_context_segments.txt:2: PURPOSE: inspect how template fields/literals become segment_lengths in .bin
_code_context_segments.txt:11:     5: from typing import Literal
_code_context_segments.txt:13:     7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
_code_context_segments.txt:22:    16:     kind: Literal["literal", "field"]
_code_context_segments.txt:31:    25:     sequence_template: list[str]
_code_context_segments.txt:36:    30: class LiteralUnkInfo:
_code_context_segments.txt:38:    32:     literal: str
_code_context_segments.txt:85:    79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
_code_context_segments.txt:95:    89:         sequence_template=[seg.name for seg in segments],
_code_context_segments.txt:100:    94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
_code_context_segments.txt:104:    98:         if seg.kind == "literal":
_code_context_segments.txt:113:   107:     return "".join(render_segments(compiled, sample))
_code_context_segments.txt:120:   114: ) -> list[LiteralUnkInfo]:
_code_context_segments.txt:121:   115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
_code_context_segments.txt:123:   117:     Returns a (possibly empty) list of LiteralUnkInfo. The caller decides
_code_context_segments.txt:126:   120:     findings: list[LiteralUnkInfo] = []
_code_context_segments.txt:128:   122:         if seg.kind != "literal":
_code_context_segments.txt:130:   124:         literal = seg.content or ""
_code_context_segments.txt:131:   125:         if not literal:
_code_context_segments.txt:133:   127:         ids = tokenizer.encode_batch([literal])[0]
_code_context_segments.txt:137:   131:                 LiteralUnkInfo(
_code_context_segments.txt:139:   133:                     literal=literal,
_code_context_segments.txt:176:    27:     LiteralUnkInfo,
_code_context_segments.txt:180:    31:     render_segments,
_code_context_segments.txt:216:    67:     template_unk_findings: list[LiteralUnkInfo]
_code_context_segments.txt:234:    85:     segment_tokens: list[list[int]]
_code_context_segments.txt:235:    86:     segment_lengths: list[int]
_code_context_segments.txt:550:   401:         sequence_template=prepared.compiled.sequence_template,
_code_context_segments.txt:571:   422:         sequence_template=prepared.compiled.sequence_template,
_code_context_segments.txt:652:   503: def _encode_segments(
_code_context_segments.txt:663:   514:     rendered = render_segments(compiled, sample)
_code_context_segments.txt:664:   515:     encoded_per_segment = tokenizer.encode_batch(rendered)
_code_context_segments.txt:685:   536:         segment_tokens, segment_texts = _encode_segments(
_code_context_segments.txt:688:   539:         segment_lengths = [len(toks) for toks in segment_tokens]
_code_context_segments.txt:689:   540:         token_count = sum(segment_lengths)
_code_context_segments.txt:729:   580:                 segment_tokens, segment_lengths = _truncate_segments(
_code_context_segments.txt:730:   581:                     segment_tokens=segment_tokens, max_total=max_size
_code_context_segments.txt:732:   583:                 truncated_token_count = sum(segment_lengths)
_code_context_segments.txt:766:   617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
_code_context_segments.txt:773:   624:         for seg_index, seg_toks in enumerate(segment_tokens):
_code_context_segments.txt:779:   630:                 _ids, surfaces = tokenizer.encode_with_offsets(seg_text)
_code_context_segments.txt:794:   645:                 segment_tokens=segment_tokens,
_code_context_segments.txt:795:   646:                 segment_lengths=segment_lengths,
_code_context_segments.txt:810:   661:     *, segment_tokens: list[list[int]], max_total: int
_code_context_segments.txt:819:   670:     for toks in segment_tokens:
_code_context_segments.txt:844:   695:                 segment_lengths=list(rec.segment_lengths),
_code_context_segments.txt:845:   696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
_code_context_segments.txt:1006:    65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
_code_context_segments.txt:1008:    67:     `payload_tokens` is the concatenation of every segment's tokens, in
_code_context_segments.txt:1009:    68:     sequence_template order.
_code_context_segments.txt:1012:    71:     segment_lengths: list[int]
_code_context_segments.txt:1013:    72:     payload_tokens: list[int]
_code_context_segments.txt:1075:   134:                 if len(rec.segment_lengths) != self.n_segments:
_code_context_segments.txt:1077:   136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
_code_context_segments.txt:1078:   137:                         f"got {len(rec.segment_lengths)}"
_code_context_segments.txt:1080:   139:                 for j, length in enumerate(rec.segment_lengths):
_code_context_segments.txt:1082:   141:                 payload = rec.payload_tokens
_code_context_segments.txt:1085:   144:                         f"payload_tokens length {len(payload)} exceeds unit_size {self.unit_size}"
_code_context_segments.txt:1219:    11:     sequence_template: list[str],
_code_context_segments.txt:1228:    20:       - know which segment (field or literal) each header refers to
_code_context_segments.txt:1232:    24:         "sequence_template": list(sequence_template),
_code_context_segments.txt:1262:    17:     sequence_template: list[str],
_code_context_segments.txt:1285:    40:         "sequence_template": list(sequence_template),
_code_context_segments.txt:1479: README.md:424: Before any data is processed, every literal segment of every template is tokenized once. If any literal produces UNK, MintDim **prompts the user before building**:
_code_context_segments.txt:1480: README.md:429:   literal segments with UNK <n>
_code_context_segments.txt:1481: README.md:447: When the build continues with template UNK, the literal UNK positions are recorded in `unk_index.jsonl` like any other UNK — each entry carries `unk_field_indices` so readers can tell which segment the UNK came from.
_code_context_segments.txt:1482: README.md:453: - `unk_field_indices` — which segment of `sequence_template` each UNK came from (0-base)
_code_context_segments.txt:1486: README.md:1073: Template literal segments produce UNK when encoded by the tokenizer.
_code_context_segments.txt:1495: README.md:1568: - know which segment (field or literal) each header refers to
_code_context_segments.txt:1496: README.md:1570: `"token_template"` is the reserved name for a literal segment of the template. Field segments use their declared field name.
_code_context_segments.txt:1498: README.md:1715: → segment index in sequence_template (0-base) where the UNK originated;
_code_context_segments.txt:1502: README.md:1773: - `n = len(sequence_template)`. Segment names live in `length_field.json`.
_code_context_segments.txt:1503: README.md:1775: - The payload concatenates each segment's tokens in `sequence_template` order, then pads with `pad_id` to `unit_size`.
_code_context_segments.txt:1509: _code_context_cli_logger.txt:942:    61:     """Ask whether to continue when literal template segments produce UNK tokens.
_code_context_segments.txt:1510: _code_context_cli_logger.txt:956:    75:             f"  {_label('literal segments with UNK')} {_value(n_findings)}",
_code_context_segments.txt:1520: _code_context_cli_logger.txt:1259:    31:     render_segments,
_code_context_segments.txt:1523: _code_context_cli_logger.txt:1313:    85:     segment_tokens: list[list[int]]
_code_context_segments.txt:1524: _code_context_cli_logger.txt:1314:    86:     segment_lengths: list[int]
_code_context_segments.txt:1539: _code_context_cli_logger.txt:1731:   503: def _encode_segments(
_code_context_segments.txt:1544: _code_context_cli_logger.txt:1742:   514:     rendered = render_segments(compiled, sample)
_code_context_segments.txt:1545: _code_context_cli_logger.txt:1743:   515:     encoded_per_segment = tokenizer.encode_batch(rendered)
_code_context_segments.txt:1548: _code_context_cli_logger.txt:1764:   536:         segment_tokens, segment_texts = _encode_segments(
_code_context_segments.txt:1549: _code_context_cli_logger.txt:1767:   539:         segment_lengths = [len(toks) for toks in segment_tokens]
_code_context_segments.txt:1550: _code_context_cli_logger.txt:1768:   540:         token_count = sum(segment_lengths)
_code_context_segments.txt:1556: _code_context_cli_logger.txt:1808:   580:                 segment_tokens, segment_lengths = _truncate_segments(
_code_context_segments.txt:1557: _code_context_cli_logger.txt:1809:   581:                     segment_tokens=segment_tokens, max_total=max_size
_code_context_segments.txt:1558: _code_context_cli_logger.txt:1811:   583:                 truncated_token_count = sum(segment_lengths)
_code_context_segments.txt:1562: _code_context_cli_logger.txt:1845:   617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
_code_context_segments.txt:1563: _code_context_cli_logger.txt:1852:   624:         for seg_index, seg_toks in enumerate(segment_tokens):
_code_context_segments.txt:1565: _code_context_cli_logger.txt:1873:   645:                 segment_tokens=segment_tokens,
_code_context_segments.txt:1566: _code_context_cli_logger.txt:1874:   646:                 segment_lengths=segment_lengths,
_code_context_segments.txt:1569: _code_context_cli_logger.txt:1889:   661:     *, segment_tokens: list[list[int]], max_total: int
_code_context_segments.txt:1573: _code_context_cli_logger.txt:1898:   670:     for toks in segment_tokens:
_code_context_segments.txt:1575: _code_context_cli_logger.txt:1923:   695:                 segment_lengths=list(rec.segment_lengths),
_code_context_segments.txt:1576: _code_context_cli_logger.txt:1924:   696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
_code_context_segments.txt:1582: src\mintdim\apis\unit_build\errors.py:330:             "Template literal segments produce UNK when encoded by the tokenizer.\n\n"
_code_context_segments.txt:1590: src\mintdim\apis\unit_build\runtime.py:31:     render_segments,
_code_context_segments.txt:1593: src\mintdim\apis\unit_build\runtime.py:85:     segment_tokens: list[list[int]]
_code_context_segments.txt:1594: src\mintdim\apis\unit_build\runtime.py:86:     segment_lengths: list[int]
_code_context_segments.txt:1609: src\mintdim\apis\unit_build\runtime.py:503: def _encode_segments(
_code_context_segments.txt:1614: src\mintdim\apis\unit_build\runtime.py:514:     rendered = render_segments(compiled, sample)
_code_context_segments.txt:1615: src\mintdim\apis\unit_build\runtime.py:515:     encoded_per_segment = tokenizer.encode_batch(rendered)
_code_context_segments.txt:1618: src\mintdim\apis\unit_build\runtime.py:536:         segment_tokens, segment_texts = _encode_segments(
_code_context_segments.txt:1619: src\mintdim\apis\unit_build\runtime.py:539:         segment_lengths = [len(toks) for toks in segment_tokens]
_code_context_segments.txt:1620: src\mintdim\apis\unit_build\runtime.py:540:         token_count = sum(segment_lengths)
_code_context_segments.txt:1626: src\mintdim\apis\unit_build\runtime.py:580:                 segment_tokens, segment_lengths = _truncate_segments(
_code_context_segments.txt:1627: src\mintdim\apis\unit_build\runtime.py:581:                     segment_tokens=segment_tokens, max_total=max_size
_code_context_segments.txt:1628: src\mintdim\apis\unit_build\runtime.py:583:                 truncated_token_count = sum(segment_lengths)
_code_context_segments.txt:1632: src\mintdim\apis\unit_build\runtime.py:617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
_code_context_segments.txt:1633: src\mintdim\apis\unit_build\runtime.py:624:         for seg_index, seg_toks in enumerate(segment_tokens):
_code_context_segments.txt:1635: src\mintdim\apis\unit_build\runtime.py:645:                 segment_tokens=segment_tokens,
_code_context_segments.txt:1636: src\mintdim\apis\unit_build\runtime.py:646:                 segment_lengths=segment_lengths,
_code_context_segments.txt:1639: src\mintdim\apis\unit_build\runtime.py:661:     *, segment_tokens: list[list[int]], max_total: int
_code_context_segments.txt:1643: src\mintdim\apis\unit_build\runtime.py:670:     for toks in segment_tokens:
_code_context_segments.txt:1645: src\mintdim\apis\unit_build\runtime.py:695:                 segment_lengths=list(rec.segment_lengths),
_code_context_segments.txt:1646: src\mintdim\apis\unit_build\runtime.py:696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
_code_context_segments.txt:1652: src\mintdim\apis\unit_build\adapters\output\length_field.py:20:       - know which segment (field or literal) each header refers to
_code_context_segments.txt:1656: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
_code_context_segments.txt:1657: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:67:     `payload_tokens` is the concatenation of every segment's tokens, in
_code_context_segments.txt:1658: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:71:     segment_lengths: list[int]
_code_context_segments.txt:1665: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:134:                 if len(rec.segment_lengths) != self.n_segments:
_code_context_segments.txt:1666: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
_code_context_segments.txt:1667: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:137:                         f"got {len(rec.segment_lengths)}"
_code_context_segments.txt:1668: src\mintdim\apis\unit_build\adapters\output\shard_writer.py:139:                 for j, length in enumerate(rec.segment_lengths):
_code_context_segments.txt:1673: src\mintdim\apis\unit_build\adapters\sources\template.py:7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
_code_context_segments.txt:1681: src\mintdim\apis\unit_build\adapters\sources\template.py:79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
_code_context_segments.txt:1686: src\mintdim\apis\unit_build\adapters\sources\template.py:89:         sequence_template=[seg.name for seg in segments],
_code_context_segments.txt:1687: src\mintdim\apis\unit_build\adapters\sources\template.py:94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
_code_context_segments.txt:1691: src\mintdim\apis\unit_build\adapters\sources\template.py:107:     return "".join(render_segments(compiled, sample))
_code_context_segments.txt:1693: src\mintdim\apis\unit_build\adapters\sources\template.py:115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
_code_context_segments.txt:1704: src\mintdim\apis\unit_build\domain\rules.py:23: LITERAL_SEGMENT_NAME = "token_template"
_code_context_segments.txt:1705: src\mintdim\apis\unit_build\ui\prompts.py:61:     """Ask whether to continue when literal template segments produce UNK tokens.
_code_context_segments.txt:1706: src\mintdim\apis\unit_build\ui\prompts.py:75:             f"  {_label('literal segments with UNK')} {_value(n_findings)}",
_code_context_segments.txt:1722: tests\integration\apis\unit_build\test_smoke.py:471:     # Literal "Z " sits in segment 0 (token_template). Field "x" is segment 1.
_code_context_segments.txt:1735: tests\unit\apis\unit_build\test_shard_writer.py:74:         segment_lengths=[2, 1, 2],
_code_context_segments.txt:1737: tests\unit\apis\unit_build\test_shard_writer.py:102:     sw.write_batch([ShardRecord(segment_lengths=[2], payload_tokens=[100, 200])])
_code_context_segments.txt:1739: tests\unit\apis\unit_build\test_shard_writer.py:124:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
_code_context_segments.txt:1741: tests\unit\apis\unit_build\test_shard_writer.py:146:         ShardRecord(segment_lengths=[2], payload_tokens=[i, i + 1])
_code_context_segments.txt:1744: tests\unit\apis\unit_build\test_shard_writer.py:170:     with pytest.raises(ValueError, match="segment_lengths"):
_code_context_segments.txt:1745: tests\unit\apis\unit_build\test_shard_writer.py:171:         sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
_code_context_segments.txt:1747: tests\unit\apis\unit_build\test_shard_writer.py:186:             [ShardRecord(segment_lengths=[5], payload_tokens=[1, 2, 3, 4, 5])]
_code_context_segments.txt:1750: tests\unit\apis\unit_build\test_shard_writer.py:220:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
_code_context_segments.txt:1753: tests\unit\apis\unit_build\test_shard_writer.py:253:     sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
_code_context_segments.txt:1754: tests\unit\apis\unit_build\test_template.py:6:     LITERAL_SEGMENT_NAME,
_code_context_segments.txt:1755: tests\unit\apis\unit_build\test_template.py:11:     render_segments,
_code_context_segments.txt:1756: tests\unit\apis\unit_build\test_template.py:24:         LITERAL_SEGMENT_NAME,
_code_context_segments.txt:1757: tests\unit\apis\unit_build\test_template.py:26:         LITERAL_SEGMENT_NAME,
_code_context_segments.txt:1762: tests\unit\apis\unit_build\test_template.py:56:     assert compiled.sequence_template == [LITERAL_SEGMENT_NAME]
_code_context_segments.txt:1764: tests\unit\apis\unit_build\test_template.py:92: def test_render_segments_returns_parallel_list():
_code_context_segments.txt:1765: tests\unit\apis\unit_build\test_template.py:98:     parts = render_segments(compiled, {"q": "what", "a": "this"})
src\mintdim\apis\unit_build\config.py:4: from typing import Literal
src\mintdim\apis\unit_build\config.py:21:     type: Literal["sentencepiece", "hf_json"]
src\mintdim\apis\unit_build\contracts.py:4: from typing import Protocol, Literal
src\mintdim\apis\unit_build\contracts.py:6: TemplateUnkPolicy = Literal["prompt", "abort", "continue"]
src\mintdim\apis\unit_build\contracts.py:7: OverflowPolicy = Literal["prompt", "abort", "truncate", "skip"]
src\mintdim\apis\unit_build\contracts.py:8: OverflowScope = Literal["only", "all"]
src\mintdim\apis\unit_build\contracts.py:16:     def encode_batch(self, texts: list[str]) -> list[list[int]]: ...
src\mintdim\apis\unit_build\contracts.py:18:     def encode_with_offsets(
src\mintdim\apis\unit_build\errors.py:330:             "Template literal segments produce UNK when encoded by the tokenizer.\n\n"
src\mintdim\apis\unit_build\errors.py:336:             "template literal characters are not fully covered by tokenizer vocabulary\n\n"
src\mintdim\apis\unit_build\errors.py:338:             "adjust template literal characters, use a tokenizer that covers them, "
src\mintdim\apis\unit_build\runtime.py:27:     LiteralUnkInfo,
src\mintdim\apis\unit_build\runtime.py:31:     render_segments,
src\mintdim\apis\unit_build\runtime.py:67:     template_unk_findings: list[LiteralUnkInfo]
src\mintdim\apis\unit_build\runtime.py:85:     segment_tokens: list[list[int]]
src\mintdim\apis\unit_build\runtime.py:86:     segment_lengths: list[int]
src\mintdim\apis\unit_build\runtime.py:401:         sequence_template=prepared.compiled.sequence_template,
src\mintdim\apis\unit_build\runtime.py:422:         sequence_template=prepared.compiled.sequence_template,
src\mintdim\apis\unit_build\runtime.py:503: def _encode_segments(
src\mintdim\apis\unit_build\runtime.py:511:     This preserves tokenizer context across field/literal boundaries. Literal
src\mintdim\apis\unit_build\runtime.py:515:     rendered_segments = render_segments(compiled, sample)
src\mintdim\apis\unit_build\runtime.py:518:     token_ids, token_surfaces = tokenizer.encode_with_offsets(rendered_text)
src\mintdim\apis\unit_build\runtime.py:522:             "tokenizer.encode_with_offsets returned mismatched token/surface counts"
src\mintdim\apis\unit_build\runtime.py:525:     segment_spans = _segment_char_spans(rendered_segments)
src\mintdim\apis\unit_build\runtime.py:526:     token_spans = _token_char_spans(rendered_text, token_surfaces)
src\mintdim\apis\unit_build\runtime.py:528:     segment_tokens: list[list[int]] = [[] for _ in rendered_segments]
src\mintdim\apis\unit_build\runtime.py:529:     segment_surfaces: list[list[str]] = [[] for _ in rendered_segments]
src\mintdim\apis\unit_build\runtime.py:532:         segment_index = _segment_index_for_token(
src\mintdim\apis\unit_build\runtime.py:537:         segment_tokens[segment_index].append(token_id)
src\mintdim\apis\unit_build\runtime.py:538:         segment_surfaces[segment_index].append(surface)
src\mintdim\apis\unit_build\runtime.py:540:     return segment_tokens, rendered_segments, segment_surfaces
src\mintdim\apis\unit_build\runtime.py:543: def _segment_char_spans(rendered_segments: list[str]) -> list[tuple[int, int]]:
src\mintdim\apis\unit_build\runtime.py:555: def _token_char_spans(text: str, surfaces: list[str]) -> list[tuple[int, int]]:
src\mintdim\apis\unit_build\runtime.py:598: def _segment_index_for_token(
src\mintdim\apis\unit_build\runtime.py:605:     # This gives literals control when a token starts inside a literal span.
src\mintdim\apis\unit_build\runtime.py:641:         segment_tokens, segment_texts, segment_surfaces = _encode_segments(
src\mintdim\apis\unit_build\runtime.py:644:         segment_lengths = [len(toks) for toks in segment_tokens]
src\mintdim\apis\unit_build\runtime.py:645:         token_count = sum(segment_lengths)
src\mintdim\apis\unit_build\runtime.py:685:                 segment_tokens, segment_lengths = _truncate_segments(
src\mintdim\apis\unit_build\runtime.py:686:                     segment_tokens=segment_tokens, max_total=max_size
src\mintdim\apis\unit_build\runtime.py:688:                 truncated_token_count = sum(segment_lengths)
src\mintdim\apis\unit_build\runtime.py:722:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
src\mintdim\apis\unit_build\runtime.py:729:         for seg_index, seg_toks in enumerate(segment_tokens):
src\mintdim\apis\unit_build\runtime.py:730:             seg_surfaces = segment_surfaces[seg_index]
src\mintdim\apis\unit_build\runtime.py:747:                 segment_tokens=segment_tokens,
src\mintdim\apis\unit_build\runtime.py:748:                 segment_lengths=segment_lengths,
src\mintdim\apis\unit_build\runtime.py:763:     *, segment_tokens: list[list[int]], max_total: int
src\mintdim\apis\unit_build\runtime.py:772:     for toks in segment_tokens:
src\mintdim\apis\unit_build\runtime.py:797:                 segment_lengths=list(rec.segment_lengths),
src\mintdim\apis\unit_build\runtime.py:798:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
src\mintdim\apis\unit_build\adapters\output\length_field.py:11:     sequence_template: list[str],
src\mintdim\apis\unit_build\adapters\output\length_field.py:20:       - know which segment (field or literal) each header refers to
src\mintdim\apis\unit_build\adapters\output\length_field.py:24:         "sequence_template": list(sequence_template),
src\mintdim\apis\unit_build\adapters\output\manifest.py:17:     sequence_template: list[str],
src\mintdim\apis\unit_build\adapters\output\manifest.py:40:         "sequence_template": list(sequence_template),
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:67:     `payload_tokens` is the concatenation of every segment's tokens, in
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:68:     sequence_template order.
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:71:     segment_lengths: list[int]
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:72:     payload_tokens: list[int]
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:134:                 if len(rec.segment_lengths) != self.n_segments:
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:137:                         f"got {len(rec.segment_lengths)}"
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:139:                 for j, length in enumerate(rec.segment_lengths):
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:141:                 payload = rec.payload_tokens
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:144:                         f"payload_tokens length {len(payload)} exceeds unit_size {self.unit_size}"
src\mintdim\apis\unit_build\adapters\sources\template.py:5: from typing import Literal
src\mintdim\apis\unit_build\adapters\sources\template.py:7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
src\mintdim\apis\unit_build\adapters\sources\template.py:16:     kind: Literal["literal", "field"]
src\mintdim\apis\unit_build\adapters\sources\template.py:25:     sequence_template: list[str]
src\mintdim\apis\unit_build\adapters\sources\template.py:30: class LiteralUnkInfo:
src\mintdim\apis\unit_build\adapters\sources\template.py:32:     literal: str
src\mintdim\apis\unit_build\adapters\sources\template.py:79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
src\mintdim\apis\unit_build\adapters\sources\template.py:89:         sequence_template=[seg.name for seg in segments],
src\mintdim\apis\unit_build\adapters\sources\template.py:94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
src\mintdim\apis\unit_build\adapters\sources\template.py:98:         if seg.kind == "literal":
src\mintdim\apis\unit_build\adapters\sources\template.py:107:     return "".join(render_segments(compiled, sample))
src\mintdim\apis\unit_build\adapters\sources\template.py:114: ) -> list[LiteralUnkInfo]:
src\mintdim\apis\unit_build\adapters\sources\template.py:115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
src\mintdim\apis\unit_build\adapters\sources\template.py:117:     Returns a (possibly empty) list of LiteralUnkInfo. The caller decides
src\mintdim\apis\unit_build\adapters\sources\template.py:120:     findings: list[LiteralUnkInfo] = []
src\mintdim\apis\unit_build\adapters\sources\template.py:122:         if seg.kind != "literal":
src\mintdim\apis\unit_build\adapters\sources\template.py:124:         literal = seg.content or ""
src\mintdim\apis\unit_build\adapters\sources\template.py:125:         if not literal:
src\mintdim\apis\unit_build\adapters\sources\template.py:127:         ids = tokenizer.encode_batch([literal])[0]
src\mintdim\apis\unit_build\adapters\sources\template.py:131:                 LiteralUnkInfo(
src\mintdim\apis\unit_build\adapters\sources\template.py:133:                     literal=literal,
src\mintdim\apis\unit_build\adapters\tokenizers\hf_json.py:18:     def encode_batch(self, texts: list[str]) -> list[list[int]]:
src\mintdim\apis\unit_build\adapters\tokenizers\hf_json.py:19:         encodings = self._tk.encode_batch(texts, add_special_tokens=False)
src\mintdim\apis\unit_build\adapters\tokenizers\hf_json.py:22:     def encode_with_offsets(
src\mintdim\apis\unit_build\adapters\tokenizers\sentencepiece.py:18:     def encode_batch(self, texts: list[str]) -> list[list[int]]:
src\mintdim\apis\unit_build\adapters\tokenizers\sentencepiece.py:22:     def encode_with_offsets(
src\mintdim\apis\unit_build\domain\models.py:3: from typing import Literal
src\mintdim\apis\unit_build\domain\models.py:5: TokenDType = Literal["uint16", "uint32"]
src\mintdim\apis\unit_build\domain\models.py:6: TokenizerType = Literal["sentencepiece", "hf_json"]
src\mintdim\apis\unit_build\domain\rules.py:23: LITERAL_SEGMENT_NAME = "token_template"
src\mintdim\apis\unit_build\ui\prompts.py:4: from typing import Literal, TextIO
src\mintdim\apis\unit_build\ui\prompts.py:61:     """Ask whether to continue when literal template segments produce UNK tokens.
src\mintdim\apis\unit_build\ui\prompts.py:75:             f"  {_label('literal segments with UNK')} {_value(n_findings)}",
src\mintdim\apis\unit_build\ui\prompts.py:180: ) -> Literal["only", "all"]:
src\mintdim\apis\unit_build\ui\prompts.py:203: ) -> tuple[OverflowResolution, Literal["only", "all"]]:
tests\conftest.py:18:     def encode_batch(self, texts: list[str]) -> list[list[int]]:
tests\conftest.py:28:     def encode_with_offsets(self, text: str) -> tuple[list[int], list[str]]:
tests\integration\apis\unit_build\test_smoke.py:62:     assert manifest["sequence_template"] == [
tests\integration\apis\unit_build\test_smoke.py:74:         "sequence_template": ["token_template", "q", "token_template", "a"],
tests\integration\apis\unit_build\test_smoke.py:471:     # Literal "Z " sits in segment 0 (token_template). Field "x" is segment 1.
tests\unit\apis\unit_build\test_indices.py:151:         sequence_template=["token_template", "prompt", "answer"],
tests\unit\apis\unit_build\test_indices.py:158:         "sequence_template": ["token_template", "prompt", "answer"],
tests\unit\apis\unit_build\test_loaders.py:10: - encode_batch delegates faithfully (no special-token injection)
tests\unit\apis\unit_build\test_loaders.py:76: def test_load_sentencepiece_encode_batch_delegates(patched_sp: Path):
tests\unit\apis\unit_build\test_loaders.py:82:     assert handle.encode_batch(["a", "bc"]) == [
tests\unit\apis\unit_build\test_loaders.py:145:     def encode_batch(
tests\unit\apis\unit_build\test_loaders.py:171: def test_load_hf_json_encode_batch_disables_special_tokens(patched_hf: Path):
tests\unit\apis\unit_build\test_loaders.py:180:     assert handle.encode_batch(["x", "yz"]) == [
tests\unit\apis\unit_build\test_shard_writer.py:74:         segment_lengths=[2, 1, 2],
tests\unit\apis\unit_build\test_shard_writer.py:75:         payload_tokens=[10, 20, 30, 40, 50],
tests\unit\apis\unit_build\test_shard_writer.py:102:     sw.write_batch([ShardRecord(segment_lengths=[2], payload_tokens=[100, 200])])
tests\unit\apis\unit_build\test_shard_writer.py:124:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
tests\unit\apis\unit_build\test_shard_writer.py:146:         ShardRecord(segment_lengths=[2], payload_tokens=[i, i + 1])
tests\unit\apis\unit_build\test_shard_writer.py:170:     with pytest.raises(ValueError, match="segment_lengths"):
tests\unit\apis\unit_build\test_shard_writer.py:171:         sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
tests\unit\apis\unit_build\test_shard_writer.py:186:             [ShardRecord(segment_lengths=[5], payload_tokens=[1, 2, 3, 4, 5])]
tests\unit\apis\unit_build\test_shard_writer.py:220:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
tests\unit\apis\unit_build\test_shard_writer.py:253:     sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
tests\unit\apis\unit_build\test_template.py:6:     LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:7:     LiteralUnkInfo,
tests\unit\apis\unit_build\test_template.py:11:     render_segments,
tests\unit\apis\unit_build\test_template.py:23:     assert compiled.sequence_template == [
tests\unit\apis\unit_build\test_template.py:24:         LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:26:         LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:30:         "literal",
tests\unit\apis\unit_build\test_template.py:32:         "literal",
tests\unit\apis\unit_build\test_template.py:45:     assert compiled.sequence_template == ["a", "b"]
tests\unit\apis\unit_build\test_template.py:56:     assert compiled.sequence_template == [LITERAL_SEGMENT_NAME]
tests\unit\apis\unit_build\test_template.py:92: def test_render_segments_returns_parallel_list():
tests\unit\apis\unit_build\test_template.py:98:     parts = render_segments(compiled, {"q": "what", "a": "this"})
tests\unit\apis\unit_build\test_template.py:133:     assert isinstance(findings[0], LiteralUnkInfo)
tests\unit\apis\unit_build\test_template.py:135:     assert "Z" in findings[0].literal
