REPO: D:\mintdim
PURPOSE: inspect how template fields/literals become segment_lengths in .bin

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\sources\template.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import re
    4: from dataclasses import dataclass
    5: from typing import Literal
    6: 
    7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
    8: from mintdim.apis.unit_build.errors import TemplateFieldMismatchError
    9: 
   10: 
   11: _PLACEHOLDER_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
   12: 
   13: 
   14: @dataclass(frozen=True)
   15: class Segment:
   16:     kind: Literal["literal", "field"]
   17:     name: str
   18:     content: str | None
   19: 
   20: 
   21: @dataclass(frozen=True)
   22: class CompiledTemplate:
   23:     raw: str
   24:     segments: list[Segment]
   25:     sequence_template: list[str]
   26:     placeholders: list[str]
   27: 
   28: 
   29: @dataclass(frozen=True)
   30: class LiteralUnkInfo:
   31:     segment_index: int
   32:     literal: str
   33:     unk_count: int
   34: 
   35: 
   36: def compile_template(
   37:     *,
   38:     file_index: int,
   39:     template: str,
   40:     declared_fields: list[str],
   41:     source_files: list[str] | None = None,
   42: ) -> CompiledTemplate:
   43:     source_files = source_files or []
   44:     placeholders: list[str] = []
   45:     raw_static_parts: list[str] = []
   46:     cursor = 0
   47: 
   48:     for match in _PLACEHOLDER_RE.finditer(template):
   49:         raw_static_parts.append(template[cursor : match.start()])
   50:         name = match.group(1)
   51:         if name not in declared_fields:
   52:             raise TemplateFieldMismatchError(
   53:                 file_index=file_index,
   54:                 template=template,
   55:                 unknown_placeholder=name,
   56:                 declared_fields=declared_fields,
   57:                 source_files=source_files,
   58:                 template_fields=[*placeholders, name],
   59:             )
   60:         placeholders.append(name)
   61:         cursor = match.end()
   62:     raw_static_parts.append(template[cursor:])
   63: 
   64:     unused_fields = sorted(set(declared_fields) - set(placeholders))
   65:     if unused_fields:
   66:         raise TemplateFieldMismatchError(
   67:             file_index=file_index,
   68:             template=template,
   69:             declared_fields=declared_fields,
   70:             source_files=source_files,
   71:             template_fields=placeholders,
   72:             unused_fields=unused_fields,
   73:         )
   74: 
   75:     segments: list[Segment] = []
   76:     for i, static in enumerate(raw_static_parts):
   77:         if static:
   78:             segments.append(
   79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
   80:             )
   81:         if i < len(placeholders):
   82:             segments.append(
   83:                 Segment(kind="field", name=placeholders[i], content=None)
   84:             )
   85: 
   86:     return CompiledTemplate(
   87:         raw=template,
   88:         segments=segments,
   89:         sequence_template=[seg.name for seg in segments],
   90:         placeholders=placeholders,
   91:     )
   92: 
   93: 
   94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
   95:     """Return one rendered text per segment, parallel to compiled.segments."""
   96:     out: list[str] = []
   97:     for seg in compiled.segments:
   98:         if seg.kind == "literal":
   99:             out.append(seg.content or "")
  100:         else:
  101:             out.append(str(sample[seg.name]))
  102:     return out
  103: 
  104: 
  105: def render(compiled: CompiledTemplate, sample: dict) -> str:
  106:     """Concatenated rendering â€” used for content hashing and offset recovery."""
  107:     return "".join(render_segments(compiled, sample))
  108: 
  109: 
  110: def preflight_template(
  111:     *,
  112:     compiled: CompiledTemplate,
  113:     tokenizer,
  114: ) -> list[LiteralUnkInfo]:
  115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
  116: 
  117:     Returns a (possibly empty) list of LiteralUnkInfo. The caller decides
  118:     whether to abort, prompt, or continue.
  119:     """
  120:     findings: list[LiteralUnkInfo] = []
  121:     for index, seg in enumerate(compiled.segments):
  122:         if seg.kind != "literal":
  123:             continue
  124:         literal = seg.content or ""
  125:         if not literal:
  126:             continue
  127:         ids = tokenizer.encode_batch([literal])[0]
  128:         unk_count = sum(1 for tid in ids if tid == tokenizer.unk_id)
  129:         if unk_count:
  130:             findings.append(
  131:                 LiteralUnkInfo(
  132:                     segment_index=index,
  133:                     literal=literal,
  134:                     unk_count=unk_count,
  135:                 )
  136:             )
  137:     return findings
  138: 
  139: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\runtime.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import hashlib
    4: import sys
    5: from bisect import insort
    6: from dataclasses import dataclass, field
    7: from pathlib import Path
    8: from typing import Iterator, Sequence
    9: 
   10: from mintdim.apis.unit_build.ui.logger import log_event
   11: 
   12: from mintdim.apis.unit_build.errors import OutputValidationError, UnitOverflowError
   13: from mintdim.apis.unit_build.config import TokenizerEntry, UnitBuildConfig
   14: from mintdim.apis.unit_build.contracts import TokenizerHandle, TokenizerMetadata
   15: from mintdim.apis.unit_build.adapters.output.directory import prepare_output_dir
   16: from mintdim.apis.unit_build.adapters.output.indices.duplicate import HashCollector, write_duplicate_index
   17: from mintdim.apis.unit_build.adapters.output.histogram import UnitHistogram, write_histogram
   18: from mintdim.apis.unit_build.adapters.output.length_field import write_length_field
   19: from mintdim.apis.unit_build.adapters.output.manifest import write_manifest
   20: from mintdim.apis.unit_build.adapters.output.indices.overflow import OverflowIndexWriter
   21: from mintdim.apis.unit_build.adapters.output.indices.sample import SampleIndexWriter
   22: from mintdim.apis.unit_build.adapters.output.stats import StatsAccumulator, write_stats
   23: from mintdim.apis.unit_build.adapters.output.indices.unknown import UnkIndexWriter
   24: from mintdim.apis.unit_build.adapters.sources.jsonl import iter_jsonl_samples
   25: from mintdim.apis.unit_build.adapters.sources.template import (
   26:     CompiledTemplate,
   27:     LiteralUnkInfo,
   28:     Segment,
   29:     compile_template,
   30:     preflight_template,
   31:     render_segments,
   32: )
   33: from mintdim.apis.unit_build.adapters.tokenizers import load_tokenizer
   34: from mintdim.apis.unit_build.domain.overflow import OverflowAction, OverflowContext, OverflowResolution
   35: from mintdim.apis.unit_build.domain.planner import find_unit
   36: from mintdim.apis.unit_build.adapters.output.shard_writer import (
   37:     ShardRecord,
   38:     ShardWriter,
   39:     assert_no_magic_collision,
   40:     bytes_per_token,
   41:     magic_for_dtype,
   42:     select_token_dtype,
   43: )
   44: 
   45: 
   46: @dataclass(frozen=True)
   47: class _Session:
   48:     files: list[str]
   49:     fields: list[str]
   50:     template: str
   51:     tokenizer_type: str
   52:     tokenizer_entry: TokenizerEntry
   53:     sizes: list[int]
   54:     build_batch: int
   55:     output_dir: str
   56:     samples_per_shard: int
   57:     file_index_for_template: int
   58: 
   59: 
   60: @dataclass(frozen=True)
   61: class _PreparedSession:
   62:     session: _Session
   63:     tokenizer: TokenizerHandle
   64:     compiled: CompiledTemplate
   65:     token_dtype: str
   66:     bytes_per_token: int
   67:     template_unk_findings: list[LiteralUnkInfo]
   68: 
   69: 
   70: @dataclass
   71: class _Policies:
   72:     """Behavior controls plumbed from `.run(on_template_unk=..., on_overflow=...)`."""
   73: 
   74:     template_unk: str  # "prompt" | "abort" | "continue"
   75:     overflow: str  # "prompt" | "abort" | "truncate" | "skip"
   76:     latched_overflow: OverflowResolution | None = None
   77:     latched_extend: bool = False
   78: 
   79: 
   80: @dataclass
   81: class _BatchRecord:
   82:     sample_id: int
   83:     source_file: str
   84:     source_line: int
   85:     segment_tokens: list[list[int]]
   86:     segment_lengths: list[int]
   87:     token_count: int
   88:     unit_size: int
   89:     content_hash: str
   90:     unk_positions: list[int]
   91:     unk_chars: list[str]
   92:     unk_field_indices: list[int]
   93:     empty_fields: list[str]
   94:     shard_path: str = ""
   95: 
   96: 
   97: @dataclass
   98: class _OverflowSkip:
   99:     sample_id: int
  100:     source_file: str
  101:     source_line: int
  102:     token_count: int
  103:     max_unit_size: int
  104:     action: str
  105:     new_unit_size: int | None = None
  106: 
  107: 
  108: def run_unit_build_pipeline(
  109:     raw_or_config,
  110:     *,
  111:     on_template_unk: str = "prompt",
  112:     on_overflow: str = "prompt",
  113: ) -> dict:
  114:     if isinstance(raw_or_config, UnitBuildConfig):
  115:         config = raw_or_config
  116:     else:
  117:         from .validation import validate_unit_build_pipeline
  118: 
  119:         config = validate_unit_build_pipeline(raw_or_config)
  120: 
  121:     sessions = _plan_sessions(config)
  122:     _preflight_output_dirs(sessions)
  123:     prepared = [_prepare_session(session) for session in sessions]
  124:     outputs = []
  125:     for prep in prepared:
  126:         policies = _Policies(template_unk=on_template_unk, overflow=on_overflow)
  127:         if prep.template_unk_findings:
  128:             _resolve_template_unk_policy(prep=prep, policies=policies)
  129:         outputs.append(_run_session(prep, policies=policies))
  130:     return {"outputs": outputs}
  131: 
  132: 
  133: def _plan_sessions(config: UnitBuildConfig) -> list[_Session]:
  134:     if config.is_shared:
  135:         return [
  136:             _Session(
  137:                 files=list(config.source.files),
  138:                 fields=list(config.source.fields[0]),
  139:                 template=config.source.templates[0],
  140:                 tokenizer_type=config.tokenizer.type,
  141:                 tokenizer_entry=config.tokenizer.configs[0],
  142:                 sizes=list(config.units.sizes[0]),
  143:                 build_batch=config.units.build_batch[0],
  144:                 output_dir=config.output.paths[0],
  145:                 samples_per_shard=config.output.samples_per_shard[0],
  146:                 file_index_for_template=0,
  147:             )
  148:         ]
  149: 
  150:     return [
  151:         _Session(
  152:             files=[config.source.files[i]],
  153:             fields=list(_for_file(config.source.fields, i)),
  154:             template=_for_file(config.source.templates, i),
  155:             tokenizer_type=config.tokenizer.type,
  156:             tokenizer_entry=_for_file(config.tokenizer.configs, i),
  157:             sizes=list(_for_file(config.units.sizes, i)),
  158:             build_batch=_for_file(config.units.build_batch, i),
  159:             output_dir=_for_file(config.output.paths, i),
  160:             samples_per_shard=_for_file(config.output.samples_per_shard, i),
  161:             file_index_for_template=i,
  162:         )
  163:         for i in range(config.file_count)
  164:     ]
  165: 
  166: 
  167: def _for_file(values: Sequence, index: int):
  168:     return values[0] if len(values) == 1 else values[index]
  169: 
  170: 
  171: def _preflight_output_dirs(sessions: list[_Session]) -> None:
  172:     for session in sessions:
  173:         path = Path(session.output_dir)
  174:         if path.exists() and any(path.iterdir()):
  175:             raise OutputValidationError(directory=session.output_dir)
  176: 
  177: 
  178: def _prepare_session(session: _Session) -> _PreparedSession:
  179:     tokenizer = load_tokenizer(
  180:         tokenizer_type=session.tokenizer_type,
  181:         entry=session.tokenizer_entry,
  182:     )
  183:     compiled = compile_template(
  184:         file_index=session.file_index_for_template,
  185:         template=session.template,
  186:         declared_fields=session.fields,
  187:         source_files=session.files,
  188:     )
  189:     template_unk_findings = preflight_template(
  190:         compiled=compiled, tokenizer=tokenizer
  191:     )
  192:     token_dtype = select_token_dtype(tokenizer.vocab_size)
  193:     assert_no_magic_collision(
  194:         vocab_size=tokenizer.vocab_size, token_dtype=token_dtype
  195:     )
  196:     return _PreparedSession(
  197:         session=session,
  198:         tokenizer=tokenizer,
  199:         compiled=compiled,
  200:         token_dtype=token_dtype,
  201:         bytes_per_token=bytes_per_token(token_dtype),
  202:         template_unk_findings=template_unk_findings,
  203:     )
  204: 
  205: 
  206: def _resolve_template_unk_policy(*, prep: _PreparedSession, policies: _Policies) -> None:
  207:     """Run P1 decision for this session before the build starts."""
  208:     from mintdim.apis.unit_build.errors import TemplateTokenizerError
  209: 
  210:     file_index = prep.session.file_index_for_template
  211:     if policies.template_unk == "continue":
  212:         return
  213:     if policies.template_unk == "abort":
  214:         raise TemplateTokenizerError(
  215:             file_index=file_index, template=prep.compiled.raw
  216:         )
  217:     if policies.template_unk == "prompt":
  218:         from mintdim.apis.unit_build.ui.prompts import prompt_template_unk, stdin_is_tty
  219: 
  220:         if not stdin_is_tty():
  221:             raise TemplateTokenizerError(
  222:                 file_index=file_index, template=prep.compiled.raw
  223:             )
  224:         proceed = prompt_template_unk(
  225:             file_index=file_index,
  226:             n_findings=len(prep.template_unk_findings),
  227:         )
  228:         if not proceed:
  229:             raise TemplateTokenizerError(
  230:                 file_index=file_index, template=prep.compiled.raw
  231:             )
  232:         return
  233:     raise ValueError(f"unknown on_template_unk value: {policies.template_unk!r}")
  234: 
  235: 
  236: def _resolve_overflow_policy(
  237:     *,
  238:     ctx: OverflowContext,
  239:     policies: _Policies,
  240: ) -> OverflowResolution:
  241:     """Apply the user's overflow policy and return what action to take."""
  242:     if policies.latched_overflow is not None:
  243:         return policies.latched_overflow
  244:     if policies.latched_extend:
  245:         return OverflowResolution(
  246:             action=OverflowAction.EXTEND, new_unit_size=ctx.token_count
  247:         )
  248: 
  249:     if policies.overflow == "abort":
  250:         raise UnitOverflowError(
  251:             sample=f"{ctx.source_file}:{ctx.source_line}",
  252:             token_count=ctx.token_count,
  253:             max_unit_size=ctx.max_unit_size,
  254:         )
  255:     if policies.overflow == "truncate":
  256:         return OverflowResolution(action=OverflowAction.TRUNCATE)
  257:     if policies.overflow == "skip":
  258:         return OverflowResolution(action=OverflowAction.SKIP)
  259: 
  260:     if policies.overflow != "prompt":
  261:         raise ValueError(f"unknown on_overflow value: {policies.overflow!r}")
  262: 
  263:     from mintdim.apis.unit_build.ui.prompts import resolve_overflow_via_prompt, stdin_is_tty
  264: 
  265:     if not stdin_is_tty():
  266:         raise UnitOverflowError(
  267:             sample=f"{ctx.source_file}:{ctx.source_line}",
  268:             token_count=ctx.token_count,
  269:             max_unit_size=ctx.max_unit_size,
  270:         )
  271: 
  272:     resolution, scope = resolve_overflow_via_prompt(ctx=ctx)
  273:     if scope == "all":
  274:         if resolution.action == OverflowAction.EXTEND:
  275:             policies.latched_extend = True
  276:         else:
  277:             policies.latched_overflow = resolution
  278:     return resolution
  279: 
  280: 
  281: def _run_session(prepared: _PreparedSession, *, policies: _Policies) -> dict:
  282:     session = prepared.session
  283:     sizes = list(session.sizes)
  284:     out_dir = prepare_output_dir(session.output_dir)
  285:     n_segments = len(prepared.compiled.segments)
  286: 
  287:     log_event(
  288:         "start",
  289:         output=out_dir,
  290:         files=len(session.files),
  291:         units=",".join(str(size) for size in sizes),
  292:         build_batch=session.build_batch,
  293:     )
  294: 
  295:     shard_writers: dict[int, ShardWriter] = {}
  296:     for unit_size in sizes:
  297:         shard_writers[unit_size] = _make_shard_writer(
  298:             out_dir=out_dir,
  299:             unit_size=unit_size,
  300:             prepared=prepared,
  301:             samples_per_shard=session.samples_per_shard,
  302:             n_segments=n_segments,
  303:         )
  304: 
  305:     sample_index = SampleIndexWriter(out_dir)
  306:     unk_index = UnkIndexWriter(out_dir)
  307:     overflow_index = OverflowIndexWriter(out_dir)
  308:     sample_index.open()
  309:     unk_index.open()
  310:     overflow_index.open()
  311: 
  312:     hashes = HashCollector()
  313:     histogram = UnitHistogram()
  314:     stats = StatsAccumulator(
  315:         token_dtype=prepared.token_dtype,
  316:         bytes_per_token=prepared.bytes_per_token,
  317:     )
  318: 
  319:     sample_id = 0
  320:     batch_index = 0
  321:     success = False
  322:     failure_reason: str | None = None
  323:     try:
  324:         for source_file in session.files:
  325:             for batch in _iter_batches(
  326:                 file_path=source_file,
  327:                 fields=session.fields,
  328:                 batch_size=session.build_batch,
  329:             ):
  330:                 records, skips = _build_batch_records(
  331:                     source_file=source_file,
  332:                     batch=batch,
  333:                     compiled=prepared.compiled,
  334:                     tokenizer=prepared.tokenizer,
  335:                     sample_id_start=sample_id,
  336:                     sizes=sizes,
  337:                     policies=policies,
  338:                     fields=session.fields,
  339:                 )
  340: 
  341:                 _ensure_writers_for_sizes(
  342:                     shard_writers=shard_writers,
  343:                     sizes=sizes,
  344:                     out_dir=out_dir,
  345:                     prepared=prepared,
  346:                     samples_per_shard=session.samples_per_shard,
  347:                     n_segments=n_segments,
  348:                 )
  349:                 _write_records_by_unit(records, shard_writers)
  350:                 sample_index.write_many(_sample_index_rows(records))
  351:                 unk_index.write_many(_unk_index_rows(records))
  352:                 overflow_index.write_many(_overflow_index_rows(skips))
  353:                 hashes.add_many(_hash_rows(records))
  354:                 histogram.add_many([rec.unit_size for rec in records])
  355:                 stats.record_many(
  356:                     token_counts=[rec.token_count for rec in records],
  357:                     unit_sizes=[rec.unit_size for rec in records],
  358:                     unk_counts=[len(rec.unk_positions) for rec in records],
  359:                 )
  360: 
  361:                 sample_id += len(batch)
  362:                 batch_index += 1
  363:                 _log_batch_progress(
  364:                     out_dir=out_dir,
  365:                     batch_index=batch_index,
  366:                     total_samples=sample_id,
  367:                     total_tokens=stats.total_tokens,
  368:                     records=records,
  369:                 )
  370:         success = True
  371:     except BaseException as exc:
  372:         failure_reason = f"{type(exc).__name__}: {exc}"
  373:         log_event(
  374:             "failed",
  375:             output=out_dir,
  376:             batches=batch_index,
  377:             samples=sample_id,
  378:             error=failure_reason,
  379:         )
  380:         raise
  381:     finally:
  382:         for writer in shard_writers.values():
  383:             writer.close(success=success, failure_reason=failure_reason)
  384:         sample_index.close()
  385:         unk_index.close()
  386:         overflow_index.close()
  387: 
  388:     groups = hashes.groups()
  389:     duplicate_groups = sum(1 for group in groups.values() if len(group) > 1)
  390:     duplicate_samples = sum(len(group) for group in groups.values() if len(group) > 1)
  391:     stats.record_duplicates(groups=duplicate_groups, samples=duplicate_samples)
  392: 
  393:     write_duplicate_index(out_dir, groups)
  394:     write_histogram(out_dir, histogram)
  395:     write_stats(out_dir, stats)
  396: 
  397:     magic = magic_for_dtype(prepared.token_dtype)
  398:     write_length_field(
  399:         out_dir=out_dir,
  400:         magic=magic,
  401:         sequence_template=prepared.compiled.sequence_template,
  402:         token_dtype=prepared.token_dtype,
  403:         pad_token_id=prepared.tokenizer.pad_id,
  404:     )
  405: 
  406:     import mintdim
  407: 
  408:     tokenizer_meta = TokenizerMetadata(
  409:         type=session.tokenizer_type,
  410:         path=session.tokenizer_entry.path,
  411:         vocab_size=prepared.tokenizer.vocab_size,
  412:         unk_id=prepared.tokenizer.unk_id,
  413:         pad_id=prepared.tokenizer.pad_id,
  414:     )
  415:     write_manifest(
  416:         out_dir=out_dir,
  417:         mintdim_version=mintdim.__version__,
  418:         tokenizer=tokenizer_meta,
  419:         token_dtype=prepared.token_dtype,
  420:         bytes_per_token=prepared.bytes_per_token,
  421:         magic=magic,
  422:         sequence_template=prepared.compiled.sequence_template,
  423:         sizes=sizes,
  424:         build_batch=session.build_batch,
  425:         samples_per_shard=session.samples_per_shard,
  426:         files=session.files,
  427:         fields=[session.fields],
  428:         templates=[session.template],
  429:     )
  430:     log_event(
  431:         "done",
  432:         output=out_dir,
  433:         batches=batch_index,
  434:         samples=stats.total_samples,
  435:         tokens=stats.total_tokens,
  436:     )
  437: 
  438:     return {
  439:         "output_dir": str(out_dir),
  440:         "total_samples": stats.total_samples,
  441:         "total_tokens": stats.total_tokens,
  442:         "samples_with_unk": stats.samples_with_unk,
  443:         "duplicate_groups": duplicate_groups,
  444:         "sizes": list(sizes),
  445:     }
  446: 
  447: 
  448: def _make_shard_writer(
  449:     *,
  450:     out_dir: Path,
  451:     unit_size: int,
  452:     prepared: _PreparedSession,
  453:     samples_per_shard: int,
  454:     n_segments: int,
  455: ) -> ShardWriter:
  456:     return ShardWriter(
  457:         root=out_dir / f"unit_{unit_size}",
  458:         unit_size=unit_size,
  459:         dtype=prepared.token_dtype,
  460:         pad_id=prepared.tokenizer.pad_id,
  461:         samples_per_shard=samples_per_shard,
  462:         n_segments=n_segments,
  463:     )
  464: 
  465: 
  466: def _ensure_writers_for_sizes(
  467:     *,
  468:     shard_writers: dict[int, ShardWriter],
  469:     sizes: list[int],
  470:     out_dir: Path,
  471:     prepared: _PreparedSession,
  472:     samples_per_shard: int,
  473:     n_segments: int,
  474: ) -> None:
  475:     for unit_size in sizes:
  476:         if unit_size in shard_writers:
  477:             continue
  478:         shard_writers[unit_size] = _make_shard_writer(
  479:             out_dir=out_dir,
  480:             unit_size=unit_size,
  481:             prepared=prepared,
  482:             samples_per_shard=samples_per_shard,
  483:             n_segments=n_segments,
  484:         )
  485: 
  486: 
  487: def _iter_batches(
  488:     *,
  489:     file_path: str,
  490:     fields: list[str],
  491:     batch_size: int,
  492: ) -> Iterator[list[tuple[int, dict]]]:
  493:     buffer: list[tuple[int, dict]] = []
  494:     for line_number, sample in iter_jsonl_samples(file_path, fields):
  495:         buffer.append((line_number, sample))
  496:         if len(buffer) >= batch_size:
  497:             yield buffer
  498:             buffer = []
  499:     if buffer:
  500:         yield buffer
  501: 
  502: 
  503: def _encode_segments(
  504:     *,
  505:     compiled: CompiledTemplate,
  506:     tokenizer: TokenizerHandle,
  507:     sample: dict,
  508: ) -> tuple[list[list[int]], list[str]]:
  509:     """Tokenize each segment of one sample independently.
  510: 
  511:     Returns (segment_token_ids_per_segment, rendered_segment_texts).
  512:     Both lists are parallel to compiled.segments.
  513:     """
  514:     rendered = render_segments(compiled, sample)
  515:     encoded_per_segment = tokenizer.encode_batch(rendered)
  516:     return encoded_per_segment, rendered
  517: 
  518: 
  519: def _build_batch_records(
  520:     *,
  521:     source_file: str,
  522:     batch: list[tuple[int, dict]],
  523:     compiled: CompiledTemplate,
  524:     tokenizer: TokenizerHandle,
  525:     sample_id_start: int,
  526:     sizes: list[int],
  527:     policies: _Policies,
  528:     fields: list[str],
  529: ) -> tuple[list[_BatchRecord], list[_OverflowSkip]]:
  530:     records: list[_BatchRecord] = []
  531:     skips: list[_OverflowSkip] = []
  532: 
  533:     for offset, (line_number, sample) in enumerate(batch):
  534:         sample_id = sample_id_start + offset
  535: 
  536:         segment_tokens, segment_texts = _encode_segments(
  537:             compiled=compiled, tokenizer=tokenizer, sample=sample
  538:         )
  539:         segment_lengths = [len(toks) for toks in segment_tokens]
  540:         token_count = sum(segment_lengths)
  541: 
  542:         empty_fields = [
  543:             seg.name
  544:             for seg, text in zip(compiled.segments, segment_texts)
  545:             if seg.kind == "field" and text == ""
  546:         ]
  547:         content_hash = _content_hash("".join(segment_texts))
  548: 
  549:         fit = find_unit(token_count=token_count, sizes=sizes)
  550:         skip_action: str | None = None
  551:         new_unit_size: int | None = None
  552: 
  553:         if fit is None:
  554:             ctx = OverflowContext(
  555:                 source_file=source_file,
  556:                 source_line=line_number,
  557:                 sample_id=sample_id,
  558:                 token_count=token_count,
  559:                 max_unit_size=sizes[-1],
  560:             )
  561:             resolution = _resolve_overflow_policy(ctx=ctx, policies=policies)
  562: 
  563:             if resolution.action == OverflowAction.SKIP:
  564:                 skip_action = "skip"
  565:                 skips.append(
  566:                     _OverflowSkip(
  567:                         sample_id=sample_id,
  568:                         source_file=source_file,
  569:                         source_line=line_number,
  570:                         token_count=token_count,
  571:                         max_unit_size=sizes[-1],
  572:                         action="skip",
  573:                     )
  574:                 )
  575:                 continue
  576: 
  577:             if resolution.action == OverflowAction.TRUNCATE:
  578:                 skip_action = "truncate"
  579:                 max_size = sizes[-1]
  580:                 segment_tokens, segment_lengths = _truncate_segments(
  581:                     segment_tokens=segment_tokens, max_total=max_size
  582:                 )
  583:                 truncated_token_count = sum(segment_lengths)
  584:                 skips.append(
  585:                     _OverflowSkip(
  586:                         sample_id=sample_id,
  587:                         source_file=source_file,
  588:                         source_line=line_number,
  589:                         token_count=token_count,
  590:                         max_unit_size=max_size,
  591:                         action="truncate",
  592:                     )
  593:                 )
  594:                 token_count = truncated_token_count
  595:                 fit = max_size
  596:             elif resolution.action == OverflowAction.EXTEND:
  597:                 new_unit_size = resolution.new_unit_size
  598:                 assert new_unit_size is not None
  599:                 if new_unit_size not in sizes:
  600:                     insort(sizes, new_unit_size)
  601:                 skips.append(
  602:                     _OverflowSkip(
  603:                         sample_id=sample_id,
  604:                         source_file=source_file,
  605:                         source_line=line_number,
  606:                         token_count=token_count,
  607:                         max_unit_size=ctx.max_unit_size,
  608:                         action="extend",
  609:                         new_unit_size=new_unit_size,
  610:                     )
  611:                 )
  612:                 fit = new_unit_size
  613:             else:  # pragma: no cover â€” exhaustive enum
  614:                 raise ValueError(f"unhandled overflow action: {resolution.action}")
  615: 
  616:         unit_size = fit
  617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
  618: 
  619:         unk_positions: list[int] = []
  620:         unk_field_indices: list[int] = []
  621:         unk_chars: list[str] = []
  622:         unk_id = tokenizer.unk_id
  623:         cursor = 0
  624:         for seg_index, seg_toks in enumerate(segment_tokens):
  625:             seg_text = segment_texts[seg_index]
  626:             seg_unk_local_positions = [
  627:                 i for i, t in enumerate(seg_toks) if t == unk_id
  628:             ]
  629:             if seg_unk_local_positions:
  630:                 _ids, surfaces = tokenizer.encode_with_offsets(seg_text)
  631:                 for local in seg_unk_local_positions:
  632:                     unk_positions.append(cursor + local)
  633:                     unk_field_indices.append(seg_index)
  634:                     if local < len(surfaces):
  635:                         unk_chars.append(surfaces[local])
  636:                     else:
  637:                         unk_chars.append("")
  638:             cursor += len(seg_toks)
  639: 
  640:         records.append(
  641:             _BatchRecord(
  642:                 sample_id=sample_id,
  643:                 source_file=source_file,
  644:                 source_line=line_number,
  645:                 segment_tokens=segment_tokens,
  646:                 segment_lengths=segment_lengths,
  647:                 token_count=token_count,
  648:                 unit_size=unit_size,
  649:                 content_hash=content_hash,
  650:                 unk_positions=unk_positions,
  651:                 unk_chars=unk_chars,
  652:                 unk_field_indices=unk_field_indices,
  653:                 empty_fields=empty_fields,
  654:             )
  655:         )
  656: 
  657:     return records, skips
  658: 
  659: 
  660: def _truncate_segments(
  661:     *, segment_tokens: list[list[int]], max_total: int
  662: ) -> tuple[list[list[int]], list[int]]:
  663:     """Trim trailing segments so that sum(lengths) == max_total.
  664: 
  665:     Walks segments in order, keeping each full segment until the remaining
  666:     budget can't absorb it; the segment that hits the limit is sliced.
  667:     """
  668:     out: list[list[int]] = []
  669:     budget = max_total
  670:     for toks in segment_tokens:
  671:         if budget <= 0:
  672:             out.append([])
  673:             continue
  674:         if len(toks) <= budget:
  675:             out.append(list(toks))
  676:             budget -= len(toks)
  677:         else:
  678:             out.append(list(toks[:budget]))
  679:             budget = 0
  680:     lengths = [len(toks) for toks in out]
  681:     return out, lengths
  682: 
  683: 
  684: def _write_records_by_unit(
  685:     records: list[_BatchRecord],
  686:     shard_writers: dict[int, ShardWriter],
  687: ) -> None:
  688:     grouped: dict[int, list[_BatchRecord]] = {}
  689:     for record in records:
  690:         grouped.setdefault(record.unit_size, []).append(record)
  691: 
  692:     for unit_size, unit_records in grouped.items():
  693:         shard_records = [
  694:             ShardRecord(
  695:                 segment_lengths=list(rec.segment_lengths),
  696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
  697:             )
  698:             for rec in unit_records
  699:         ]
  700:         locations = shard_writers[unit_size].write_batch(shard_records)
  701:         for record, (shard_index, _position) in zip(unit_records, locations):
  702:             record.shard_path = f"unit_{unit_size}/shard_{shard_index:06d}.bin"
  703: 
  704: 
  705: def _sample_index_rows(records: list[_BatchRecord]) -> list[dict]:
  706:     return [
  707:         {
  708:             "sample_id": record.sample_id,
  709:             "hash": record.content_hash,
  710:             "token_count": record.token_count,
  711:             "unit_size": record.unit_size,
  712:             "source_file": record.source_file,
  713:             "source_line": record.source_line,
  714:             "shard_path": record.shard_path,
  715:             "empty_fields": record.empty_fields,
  716:         }
  717:         for record in records
  718:     ]
  719: 
  720: 
  721: def _unk_index_rows(records: list[_BatchRecord]) -> list[dict]:
  722:     return [
  723:         {
  724:             "sample_id": record.sample_id,
  725:             "hash": record.content_hash,
  726:             "source_file": record.source_file,
  727:             "source_line": record.source_line,
  728:             "unk_count": len(record.unk_positions),
  729:             "unk_positions": record.unk_positions,
  730:             "unk_field_indices": record.unk_field_indices,
  731:             "unk_chars": record.unk_chars,
  732:         }
  733:         for record in records
  734:         if record.unk_positions
  735:     ]
  736: 
  737: 
  738: def _overflow_index_rows(skips: list[_OverflowSkip]) -> list[dict]:
  739:     rows: list[dict] = []
  740:     for skip in skips:
  741:         row: dict = {
  742:             "sample_id": skip.sample_id,
  743:             "source_file": skip.source_file,
  744:             "source_line": skip.source_line,
  745:             "token_count": skip.token_count,
  746:             "max_unit_size": skip.max_unit_size,
  747:             "action": skip.action,
  748:         }
  749:         if skip.new_unit_size is not None:
  750:             row["new_unit_size"] = skip.new_unit_size
  751:         rows.append(row)
  752:     return rows
  753: 
  754: 
  755: def _hash_rows(records: list[_BatchRecord]) -> list[dict]:
  756:     return [
  757:         {
  758:             "sample_id": record.sample_id,
  759:             "hash": record.content_hash,
  760:             "source_file": record.source_file,
  761:             "source_line": record.source_line,
  762:         }
  763:         for record in records
  764:     ]
  765: 
  766: 
  767: def _content_hash(rendered: str) -> str:
  768:     return hashlib.sha256(rendered.encode("utf-8")).hexdigest()
  769: 
  770: 
  771: def _log_batch_progress(
  772:     *,
  773:     out_dir: Path,
  774:     batch_index: int,
  775:     total_samples: int,
  776:     total_tokens: int,
  777:     records: list[_BatchRecord],
  778: ) -> None:
  779:     log_event(
  780:         "progress",
  781:         batch=batch_index,
  782:         batch_samples=len(records),
  783:         total_samples=total_samples,
  784:         batch_tokens=sum(record.token_count for record in records),
  785:         total_tokens=total_tokens,
  786:     )
  787: 
  788: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\output\shard_writer.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from dataclasses import dataclass
    4: from pathlib import Path
    5: 
    6: import numpy as np
    7: 
    8: from mintdim.apis.unit_build.ui.logger import log_event
    9: 
   10: from mintdim.apis.unit_build.domain.rules import (
   11:     BYTES_PER_TOKEN_BY_DTYPE as _BYTES_PER_TOKEN,
   12:     EMPTY_UNIT_NOTICE as _EMPTY_UNIT_NOTICE,
   13:     FAILED_UNIT_NOTICE as _FAILED_UNIT_NOTICE,
   14:     MAGIC_VALUE_BY_DTYPE as _MAGIC_VALUE,
   15:     RECOMMENDED_SHARD_MIB as _RECOMMENDED_SHARD_MIB,
   16:     TOKEN_DTYPE_CAPACITY as _MAX_VOCAB_FOR_DTYPE,
   17: )
   18: from mintdim.apis.unit_build.errors import TokenDTypeCapacityError
   19: 
   20: 
   21: _DTYPE_MAP = {
   22:     "uint16": np.uint16,
   23:     "uint32": np.uint32,
   24: }
   25: 
   26: 
   27: # Per-dtype capacity = the largest vocab_size that fits without colliding
   28: # with the reserved record-magic value. The magic value is also the max
   29: # representable integer for the dtype, so a vocab equal to the capacity
   30: # means token ids [0, capacity-1], which never produces an id that equals
   31: # the magic.
   32: 
   33: 
   34: def select_token_dtype(vocab_size: int) -> str:
   35:     for dtype, capacity in _MAX_VOCAB_FOR_DTYPE.items():
   36:         if vocab_size <= capacity:
   37:             return dtype
   38:     raise TokenDTypeCapacityError(
   39:         vocab_size=vocab_size,
   40:         dtype_capacities=_MAX_VOCAB_FOR_DTYPE,
   41:     )
   42: 
   43: 
   44: def bytes_per_token(dtype: str) -> int:
   45:     return _BYTES_PER_TOKEN[dtype]
   46: 
   47: 
   48: def magic_for_dtype(dtype: str) -> int:
   49:     return _MAGIC_VALUE[dtype]
   50: 
   51: 
   52: def assert_no_magic_collision(*, vocab_size: int, token_dtype: str) -> None:
   53:     capacity = _MAX_VOCAB_FOR_DTYPE[token_dtype]
   54:     if vocab_size > capacity:
   55:         raise TokenDTypeCapacityError(
   56:             vocab_size=vocab_size,
   57:             dtype_capacities={token_dtype: capacity},
   58:         )
   59: 
   60: 
   61: @dataclass(frozen=True)
   62: class ShardRecord:
   63:     """One sample's payload as it will be written to a shard.
   64: 
   65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
   66:     sum must be <= unit_size; the remainder of the payload area is padded.
   67:     `payload_tokens` is the concatenation of every segment's tokens, in
   68:     sequence_template order.
   69:     """
   70: 
   71:     segment_lengths: list[int]
   72:     payload_tokens: list[int]
   73: 
   74: 
   75: class ShardWriter:
   76:     """Append fixed-width records to `unit_<size>/shard_*.bin`.
   77: 
   78:     Each record on disk is laid out as:
   79:         [magic] [len_seg_0] ... [len_seg_{n-1}] [payload_0..unit_size-1]
   80:     All slots are `dtype` (uint16 or uint32). Total record size in slots
   81:     is `1 + n_segments + unit_size`.
   82:     """
   83: 
   84:     def __init__(
   85:         self,
   86:         *,
   87:         root: Path,
   88:         unit_size: int,
   89:         dtype: str,
   90:         pad_id: int,
   91:         samples_per_shard: int,
   92:         n_segments: int,
   93:     ) -> None:
   94:         self.root = Path(root)
   95:         self.unit_size = unit_size
   96:         self.dtype = dtype
   97:         self.np_dtype = _DTYPE_MAP[dtype]
   98:         self.pad_id = pad_id
   99:         self.samples_per_shard = samples_per_shard
  100:         self.n_segments = n_segments
  101:         self.magic = _MAGIC_VALUE[dtype]
  102:         self.record_slots = 1 + n_segments + unit_size
  103: 
  104:         self._shard_index = -1
  105:         self._samples_in_current = 0
  106:         self._total_samples = 0
  107:         self._bytes_in_current = 0
  108:         self._handle = None
  109:         self._current_shard_path: Path | None = None
  110: 
  111:         self.root.mkdir(parents=True, exist_ok=True)
  112: 
  113:     def write_batch(self, records: list[ShardRecord]) -> list[tuple[int, int]]:
  114:         """Append a batch and return (shard_index, position_in_shard) per record."""
  115:         locations: list[tuple[int, int]] = []
  116:         cursor = 0
  117:         while cursor < len(records):
  118:             if self._handle is None or self._samples_in_current >= self.samples_per_shard:
  119:                 self._roll()
  120: 
  121:             capacity = self.samples_per_shard - self._samples_in_current
  122:             chunk = records[cursor : cursor + capacity]
  123:             shard_index = self._shard_index
  124:             start_position = self._samples_in_current
  125: 
  126:             arr = np.full(
  127:                 (len(chunk), self.record_slots),
  128:                 self.pad_id,
  129:                 dtype=self.np_dtype,
  130:             )
  131:             payload_offset = 1 + self.n_segments
  132:             for row, rec in enumerate(chunk):
  133:                 arr[row, 0] = self.magic
  134:                 if len(rec.segment_lengths) != self.n_segments:
  135:                     raise ValueError(
  136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
  137:                         f"got {len(rec.segment_lengths)}"
  138:                     )
  139:                 for j, length in enumerate(rec.segment_lengths):
  140:                     arr[row, 1 + j] = length
  141:                 payload = rec.payload_tokens
  142:                 if len(payload) > self.unit_size:
  143:                     raise ValueError(
  144:                         f"payload_tokens length {len(payload)} exceeds unit_size {self.unit_size}"
  145:                     )
  146:                 arr[row, payload_offset : payload_offset + len(payload)] = payload
  147:             arr.tofile(self._handle)
  148: 
  149:             locations.extend(
  150:                 (shard_index, start_position + offset) for offset in range(len(chunk))
  151:             )
  152:             self._samples_in_current += len(chunk)
  153:             self._total_samples += len(chunk)
  154:             self._bytes_in_current += (
  155:                 len(chunk) * self.record_slots * _BYTES_PER_TOKEN[self.dtype]
  156:             )
  157:             cursor += len(chunk)
  158: 
  159:         return locations
  160: 
  161:     def close(self, *, success: bool = True, failure_reason: str | None = None) -> None:
  162:         if self._handle is not None:
  163:             self._handle.close()
  164:             self._handle = None
  165:             self._emit_shard_status()
  166: 
  167:         if success and self._total_samples == 0:
  168:             self._write_empty_unit_notice()
  169:         elif not success:
  170:             self._write_failed_unit_notice(failure_reason)
  171: 
  172:     def _roll(self) -> None:
  173:         if self._handle is not None:
  174:             self._handle.close()
  175:             self._emit_shard_status()
  176:         self._shard_index += 1
  177:         path = self.root / f"shard_{self._shard_index:06d}.bin"
  178:         self._handle = open(path, "wb")
  179:         self._current_shard_path = path
  180:         self._samples_in_current = 0
  181:         self._bytes_in_current = 0
  182: 
  183:     def _emit_shard_status(self) -> None:
  184:         mib = self._bytes_in_current / (1024 * 1024)
  185: 
  186:         fields: dict[str, object] = {
  187:             "shard": f"{self._shard_index:06d}",
  188:             "samples": self._samples_in_current,
  189:             "bytes": f"{mib:.1f} MiB",
  190:             "unit": self.unit_size,
  191:         }
  192:         if self._current_shard_path is not None:
  193:             fields["path"] = str(self._current_shard_path)
  194: 
  195:         log_event("shard", **fields)
  196: 
  197:         if mib > _RECOMMENDED_SHARD_MIB:
  198:             log_event(
  199:                 "warning",
  200:                 shard=f"{self._shard_index:06d}",
  201:                 size=f"{mib:.1f} MiB",
  202:                 recommended=f"{_RECOMMENDED_SHARD_MIB:.1f} MiB",
  203:             )
  204: 
  205:     def _write_empty_unit_notice(self) -> None:
  206:         path = self.root / _EMPTY_UNIT_NOTICE
  207:         path.write_text(
  208:             "\n".join(
  209:                 [
  210:                     "# EmptyUnitNotice",
  211:                     "",
  212:                     "Status:",
  213:                     "No samples were assigned to this unit.",
  214:                     "",
  215:                     "Unit:",
  216:                     str(self.unit_size),
  217:                     "",
  218:                     "Reason:",
  219:                     (
  220:                         "No rendered JSONL sample produced a token_count that "
  221:                         "the unit planner assigned to this unit."
  222:                     ),
  223:                     "",
  224:                     "This is not an error:",
  225:                     (
  226:                         "The build completed normally. The unit directory is kept "
  227:                         "so the declared output layout remains visible."
  228:                     ),
  229:                     "",
  230:                 ]
  231:             ),
  232:             encoding="utf-8",
  233:         )
  234: 
  235:     def _write_failed_unit_notice(self, failure_reason: str | None) -> None:
  236:         path = self.root / _FAILED_UNIT_NOTICE
  237:         path.write_text(
  238:             "\n".join(
  239:                 [
  240:                     "# UnitBuildFailedNotice",
  241:                     "",
  242:                     "Status:",
  243:                     "The unit-build run failed before this output was finalized.",
  244:                     "",
  245:                     "Unit:",
  246:                     str(self.unit_size),
  247:                     "",
  248:                     "Samples written before failure:",
  249:                     str(self._total_samples),
  250:                     "",
  251:                     "Failure:",
  252:                     failure_reason or "Unknown failure",
  253:                     "",
  254:                     "This is not an empty unit:",
  255:                     (
  256:                         "EMPTY_UNIT.md is written only after a successful build "
  257:                         "where no samples were assigned to this unit."
  258:                     ),
  259:                     "",
  260:                 ]
  261:             ),
  262:             encoding="utf-8",
  263:         )

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\output\length_field.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import json
    4: from pathlib import Path
    5: 
    6: 
    7: def write_length_field(
    8:     *,
    9:     out_dir: Path,
   10:     magic: int,
   11:     sequence_template: list[str],
   12:     token_dtype: str,
   13:     pad_token_id: int,
   14: ) -> Path:
   15:     """Write `length_field.json` describing the per-record header layout.
   16: 
   17:     Readers use this file to:
   18:       - verify the magic marker at offset 0 of every record
   19:       - decode the n length headers that follow
   20:       - know which segment (field or literal) each header refers to
   21:     """
   22:     payload = {
   23:         "magic": magic,
   24:         "sequence_template": list(sequence_template),
   25:         "token_dtype": token_dtype,
   26:         "pad_token_id": pad_token_id,
   27:     }
   28:     path = out_dir / "length_field.json"
   29:     with open(path, "w", encoding="utf-8") as fp:
   30:         json.dump(payload, fp, ensure_ascii=False, indent=2)
   31:     return path
   32: 
   33: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\adapters\output\manifest.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: import json
    4: from pathlib import Path
    5: 
    6: from mintdim.apis.unit_build.contracts import TokenizerMetadata
    7: 
    8: 
    9: def write_manifest(
   10:     *,
   11:     out_dir: Path,
   12:     mintdim_version: str,
   13:     tokenizer: TokenizerMetadata,
   14:     token_dtype: str,
   15:     bytes_per_token: int,
   16:     magic: int,
   17:     sequence_template: list[str],
   18:     sizes: list[int],
   19:     build_batch: int,
   20:     samples_per_shard: int,
   21:     files: list[str],
   22:     fields: list[list[str]],
   23:     templates: list[str],
   24: ) -> Path:
   25:     manifest = {
   26:         "pipeline": "unit-build",
   27:         "mintdim_version": mintdim_version,
   28:         "tokenizer": {
   29:             "type": tokenizer.type,
   30:             "path": tokenizer.path,
   31:             "vocab_size": tokenizer.vocab_size,
   32:             "unk_id": tokenizer.unk_id,
   33:             "pad_id": tokenizer.pad_id,
   34:         },
   35:         "token_storage": {
   36:             "dtype": token_dtype,
   37:             "bytes_per_token": bytes_per_token,
   38:             "magic": magic,
   39:         },
   40:         "sequence_template": list(sequence_template),
   41:         "length_field_metadata": "length_field.json",
   42:         "units": {
   43:             "sizes": list(sizes),
   44:             "build_batch": build_batch,
   45:         },
   46:         "source": {
   47:             "files": files,
   48:             "fields": fields,
   49:             "templates": templates,
   50:         },
   51:         "output": {
   52:             "samples_per_shard": samples_per_shard,
   53:         },
   54:     }
   55:     path = out_dir / "manifest.json"
   56:     with open(path, "w", encoding="utf-8") as fp:
   57:         json.dump(manifest, fp, ensure_ascii=False, indent=2)
   58:     return path
   59: 
   60: 

====================================================================================================
FILE: src\mintdim\apis\unit_build\validation.py
====================================================================================================
    1: from __future__ import annotations
    2: 
    3: from mintdim.apis.unit_build.config import (
    4:     OutputConfig,
    5:     SourceConfig,
    6:     TokenizerConfig,
    7:     TokenizerEntry,
    8:     UnitBuildConfig,
    9:     UnitsConfig,
   10: )
   11: from mintdim.apis.unit_build.errors import (
   12:     FieldMappingError,
   13:     OutputMappingError,
   14:     PipelineValidationError,
   15:     SamplesPerShardError,
   16:     TemplateMappingError,
   17:     TokenizerMappingError,
   18:     UnitMappingError,
   19:     UnitValidationError,
   20: )
   21: 
   22: 
   23: def validate_unit_build_pipeline(raw: dict) -> UnitBuildConfig:
   24:     """Validate raw pipeline config dict and return a typed UnitBuildConfig.
   25: 
   26:     Validation order:
   27:     1. PipelineValidationError if any top-level block is missing
   28:     2. Per-block structural validation (delegated to subpackage validators)
   29:     3. Per-block mapping arity check (each list is shared or per-file)
   30: 
   31:     File-specific template/tokenizer preflight happens before output is created.
   32:     """
   33:     _check_top_level(raw)
   34: 
   35:     source = _build_source(raw["source"])
   36:     tokenizer = _build_tokenizer(raw["tokenizer"])
   37:     units = _build_units(raw["units"])
   38:     output = _build_output(raw["output"])
   39: 
   40:     config = UnitBuildConfig(
   41:         source=source,
   42:         tokenizer=tokenizer,
   43:         units=units,
   44:         output=output,
   45:     )
   46: 
   47:     _check_mapping_arity(config)
   48:     return config
   49: 
   50: 
   51: def _check_top_level(raw: dict) -> None:
   52:     missing: list[str] = []
   53:     if not raw.get("source"):
   54:         missing.extend(["source.files", "source.fields", "source.templates"])
   55:     else:
   56:         src = raw["source"]
   57:         if not src.get("files"):
   58:             missing.append("source.files")
   59:         if not src.get("fields"):
   60:             missing.append("source.fields")
   61:         if not src.get("templates"):
   62:             missing.append("source.templates")
   63:     if not raw.get("tokenizer"):
   64:         missing.append("tokenizer")
   65:     if not raw.get("units"):
   66:         missing.extend(["units.sizes", "units.build_batch"])
   67:     else:
   68:         u = raw["units"]
   69:         if not u.get("sizes"):
   70:             missing.append("units.sizes")
   71:         if not u.get("build_batch"):
   72:             missing.append("units.build_batch")
   73:     if not raw.get("output"):
   74:         missing.extend(["output.dir", "output.samples_per_shard"])
   75:     else:
   76:         out = raw["output"]
   77:         if not out.get("paths"):
   78:             missing.append("output.dir")
   79:         if not out.get("samples_per_shard"):
   80:             missing.append("output.samples_per_shard")
   81:     if missing:
   82:         raise PipelineValidationError(missing=missing)
   83: 
   84: 
   85: def _build_source(raw: dict) -> SourceConfig:
   86:     from mintdim.apis.unit_build.adapters.sources.validation import validate_source
   87: 
   88:     return validate_source(raw)
   89: 
   90: 
   91: def _build_tokenizer(raw: dict) -> TokenizerConfig:
   92:     from mintdim.apis.unit_build.adapters.tokenizers.validation import validate_tokenizer_block
   93: 
   94:     return validate_tokenizer_block(raw)
   95: 
   96: 
   97: def _build_units(raw: dict) -> UnitsConfig:
   98:     return validate_units(raw)
   99: 
  100: 
  101: def _build_output(raw: dict) -> OutputConfig:
  102:     from mintdim.apis.unit_build.adapters.output.validation import validate_output_block
  103: 
  104:     return validate_output_block(raw)
  105: 
  106: 
  107: def validate_units(raw: dict) -> UnitsConfig:
  108:     sizes = raw.get("sizes") or []
  109:     build_batch = raw.get("build_batch") or []
  110: 
  111:     if not sizes or not build_batch:
  112:         raise UnitValidationError()
  113: 
  114:     normalized_sizes: list[list[int]] = []
  115:     for group in sizes:
  116:         glist = list(group)
  117:         if not glist:
  118:             raise UnitValidationError()
  119:         if any(type(s) is not int or s <= 0 for s in glist):
  120:             raise UnitValidationError()
  121:         if glist != sorted(glist):
  122:             raise UnitValidationError()
  123:         if len(set(glist)) != len(glist):
  124:             raise UnitValidationError()
  125:         normalized_sizes.append(glist)
  126: 
  127:     normalized_batch: list[int] = []
  128:     for b in build_batch:
  129:         if type(b) is not int or b <= 0:
  130:             raise UnitValidationError()
  131:         normalized_batch.append(b)
  132: 
  133:     return UnitsConfig(sizes=normalized_sizes, build_batch=normalized_batch)
  134: 
  135: 
  136: def _check_mapping_arity(config: UnitBuildConfig) -> None:
  137:     n = config.file_count
  138: 
  139:     def arity_ok(k: int) -> bool:
  140:         return k == 1 or k == n
  141: 
  142:     if not arity_ok(len(config.source.fields)):
  143:         raise FieldMappingError(input_files=n, field_configs=len(config.source.fields))
  144:     if not arity_ok(len(config.source.templates)):
  145:         raise TemplateMappingError(input_files=n, templates=len(config.source.templates))
  146:     if not arity_ok(len(config.tokenizer.configs)):
  147:         raise TokenizerMappingError(input_files=n, tokenizers=len(config.tokenizer.configs))
  148:     if not arity_ok(len(config.units.sizes)):
  149:         raise UnitMappingError(input_files=n, unit_configs=len(config.units.sizes))
  150:     if not arity_ok(len(config.units.build_batch)):
  151:         raise UnitMappingError(input_files=n, unit_configs=len(config.units.build_batch))
  152:     if not arity_ok(len(config.output.paths)):
  153:         raise OutputMappingError(input_files=n, output_dirs=len(config.output.paths))
  154:     if not arity_ok(len(config.output.samples_per_shard)):
  155:         raise SamplesPerShardError.arity_mismatch(
  156:             input_files=n, entries=len(config.output.samples_per_shard)
  157:         )
  158: 
  159: 

====================================================================================================
SEARCH: segment/length render references
====================================================================================================
README.md:78: → assign unit by exact token_count inside the batch
README.md:357: token_count = 143
README.md:362: If a sample's `sum(segment token lengths)` exceeds `max(sizes)`, MintDim pauses the build at that sample and asks the user how to recover:
README.md:368:   token_count  <n>
README.md:383: Choosing `extend` triggers a follow-up asking for an integer `new_unit_size >= token_count`. After picking an action, MintDim asks the scope:
README.md:399: extend / all   → every subsequent overflow auto-adds a new unit_size = token_count
README.md:424: Before any data is processed, every literal segment of every template is tokenized once. If any literal produces UNK, MintDim **prompts the user before building**:
README.md:429:   literal segments with UNK <n>
README.md:447: When the build continues with template UNK, the literal UNK positions are recorded in `unk_index.jsonl` like any other UNK — each entry carries `unk_field_indices` so readers can tell which segment the UNK came from.
README.md:453: - `unk_field_indices` — which segment of `sequence_template` each UNK came from (0-base)
README.md:465: → record in unk_index.jsonl with segment/field index
README.md:470: MintDim writes tokenized shards as binary token arrays. Every slot on disk — magic marker, segment lengths, and token payload — uses the same dtype.
README.md:491: Selected dtype and magic are written to `manifest.json` and `length_field.json`.
README.md:1073: Template literal segments produce UNK when encoded by the tokenizer.
README.md:1168: token_count:
README.md:1185: token_count:
README.md:1368: │        │     ├─ length_field.py
README.md:1432: ├─ length_field.json
README.md:1467: length_field.json
README.md:1486: → samples whose token_count exceeded max(sizes) and how the policy resolved them
README.md:1527:   "length_field_metadata": "length_field.json",
README.md:1545: ### length_field.json
README.md:1568: - know which segment (field or literal) each header refers to
README.md:1570: `"token_template"` is the reserved name for a literal segment of the template. Field segments use their declared field name.
README.md:1624:   "token_count": 143,
README.md:1715: → segment index in sequence_template (0-base) where the UNK originated;
README.md:1728: - unknown token analysis (by source segment)
README.md:1736: Each line records one sample whose `sum(segment token lengths)` exceeded `max(sizes)` and how the policy resolved it.
README.md:1745:   "token_count": 412,
README.md:1773: - `n = len(sequence_template)`. Segment names live in `length_field.json`.
README.md:1775: - The payload concatenates each segment's tokens in `sequence_template` order, then pads with `pad_id` to `unit_size`.
README.md:1779: To slice a segment back out:
README.md:1783: segment_i_start   = payload_offset + sum(len_seg_0..i-1)
README.md:1784: segment_i_end     = segment_i_start + len_seg_i
README.md:1866: exact per-segment tokenization
README.md:1869: length_field.json describes per-record header schema for readers
_code_context_cli_logger.txt:942:    61:     """Ask whether to continue when literal template segments produce UNK tokens.
_code_context_cli_logger.txt:956:    75:             f"  {_label('literal segments with UNK')} {_value(n_findings)}",
_code_context_cli_logger.txt:1000:   119:             f"  {_label('token_count')} {_value(ctx.token_count)}",
_code_context_cli_logger.txt:1022:   141:     token_count: int,
_code_context_cli_logger.txt:1032:   151:             f"  {_label('new unit_size')} must be an integer >= {token_count}",
_code_context_cli_logger.txt:1051:   170:         if value < token_count:
_code_context_cli_logger.txt:1052:   171:             sys.stderr.write(f"  invalid, must be >= {token_count}\n")
_code_context_cli_logger.txt:1092:   211:             token_count=ctx.token_count,
_code_context_cli_logger.txt:1246:    18: from mintdim.apis.unit_build.adapters.output.length_field import write_length_field
_code_context_cli_logger.txt:1254:    26:     CompiledTemplate,
_code_context_cli_logger.txt:1256:    28:     Segment,
_code_context_cli_logger.txt:1259:    31:     render_segments,
_code_context_cli_logger.txt:1265:    37:     ShardRecord,
_code_context_cli_logger.txt:1292:    64:     compiled: CompiledTemplate
_code_context_cli_logger.txt:1313:    85:     segment_tokens: list[list[int]]
_code_context_cli_logger.txt:1314:    86:     segment_lengths: list[int]
_code_context_cli_logger.txt:1315:    87:     token_count: int
_code_context_cli_logger.txt:1330:   102:     token_count: int
_code_context_cli_logger.txt:1474:   246:             action=OverflowAction.EXTEND, new_unit_size=ctx.token_count
_code_context_cli_logger.txt:1480:   252:             token_count=ctx.token_count,
_code_context_cli_logger.txt:1496:   268:             token_count=ctx.token_count,
_code_context_cli_logger.txt:1513:   285:     n_segments = len(prepared.compiled.segments)
_code_context_cli_logger.txt:1530:   302:             n_segments=n_segments,
_code_context_cli_logger.txt:1575:   347:                     n_segments=n_segments,
_code_context_cli_logger.txt:1584:   356:                     token_counts=[rec.token_count for rec in records],
_code_context_cli_logger.txt:1626:   398:     write_length_field(
_code_context_cli_logger.txt:1682:   454:     n_segments: int,
_code_context_cli_logger.txt:1690:   462:         n_segments=n_segments,
_code_context_cli_logger.txt:1701:   473:     n_segments: int,
_code_context_cli_logger.txt:1711:   483:             n_segments=n_segments,
_code_context_cli_logger.txt:1731:   503: def _encode_segments(
_code_context_cli_logger.txt:1733:   505:     compiled: CompiledTemplate,
_code_context_cli_logger.txt:1737:   509:     """Tokenize each segment of one sample independently.
_code_context_cli_logger.txt:1739:   511:     Returns (segment_token_ids_per_segment, rendered_segment_texts).
_code_context_cli_logger.txt:1740:   512:     Both lists are parallel to compiled.segments.
_code_context_cli_logger.txt:1742:   514:     rendered = render_segments(compiled, sample)
_code_context_cli_logger.txt:1743:   515:     encoded_per_segment = tokenizer.encode_batch(rendered)
_code_context_cli_logger.txt:1744:   516:     return encoded_per_segment, rendered
_code_context_cli_logger.txt:1751:   523:     compiled: CompiledTemplate,
_code_context_cli_logger.txt:1764:   536:         segment_tokens, segment_texts = _encode_segments(
_code_context_cli_logger.txt:1767:   539:         segment_lengths = [len(toks) for toks in segment_tokens]
_code_context_cli_logger.txt:1768:   540:         token_count = sum(segment_lengths)
_code_context_cli_logger.txt:1772:   544:             for seg, text in zip(compiled.segments, segment_texts)
_code_context_cli_logger.txt:1775:   547:         content_hash = _content_hash("".join(segment_texts))
_code_context_cli_logger.txt:1777:   549:         fit = find_unit(token_count=token_count, sizes=sizes)
_code_context_cli_logger.txt:1786:   558:                 token_count=token_count,
_code_context_cli_logger.txt:1798:   570:                         token_count=token_count,
_code_context_cli_logger.txt:1808:   580:                 segment_tokens, segment_lengths = _truncate_segments(
_code_context_cli_logger.txt:1809:   581:                     segment_tokens=segment_tokens, max_total=max_size
_code_context_cli_logger.txt:1811:   583:                 truncated_token_count = sum(segment_lengths)
_code_context_cli_logger.txt:1817:   589:                         token_count=token_count,
_code_context_cli_logger.txt:1822:   594:                 token_count = truncated_token_count
_code_context_cli_logger.txt:1834:   606:                         token_count=token_count,
_code_context_cli_logger.txt:1845:   617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
_code_context_cli_logger.txt:1852:   624:         for seg_index, seg_toks in enumerate(segment_tokens):
_code_context_cli_logger.txt:1853:   625:             seg_text = segment_texts[seg_index]
_code_context_cli_logger.txt:1873:   645:                 segment_tokens=segment_tokens,
_code_context_cli_logger.txt:1874:   646:                 segment_lengths=segment_lengths,
_code_context_cli_logger.txt:1875:   647:                 token_count=token_count,
_code_context_cli_logger.txt:1888:   660: def _truncate_segments(
_code_context_cli_logger.txt:1889:   661:     *, segment_tokens: list[list[int]], max_total: int
_code_context_cli_logger.txt:1891:   663:     """Trim trailing segments so that sum(lengths) == max_total.
_code_context_cli_logger.txt:1893:   665:     Walks segments in order, keeping each full segment until the remaining
_code_context_cli_logger.txt:1894:   666:     budget can't absorb it; the segment that hits the limit is sliced.
_code_context_cli_logger.txt:1898:   670:     for toks in segment_tokens:
_code_context_cli_logger.txt:1922:   694:             ShardRecord(
_code_context_cli_logger.txt:1923:   695:                 segment_lengths=list(rec.segment_lengths),
_code_context_cli_logger.txt:1924:   696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
_code_context_cli_logger.txt:1938:   710:             "token_count": record.token_count,
_code_context_cli_logger.txt:1973:   745:             "token_count": skip.token_count,
_code_context_cli_logger.txt:2012:   784:         batch_tokens=sum(record.token_count for record in records),
dist\mintdim-0.1.14-py3-none-any.whl:121: "�*&�9�"N�j�8�{�1�0 ��Ζ���[)%nǄ�t��V���bj�hk� ��x����4��Z8���q+�-�#rqr����cQ�]���"���?�����q*W�%��?��3�!�f 'YY���f�sV⫮ԕ�PK      BP}_�  b  7   mintdim/apis/unit_build/adapters/output/length_field.pyeS���0����⤥�bH�=/{���z���%Wz^J��Xr�.9=�̛�F6)�:3���1&�
dist\mintdim-0.1.14-py3-none-any.whl:437: f�  �             ��u  mintdim/apis/unit_build/api.pyPK       BP`���  .  !           ���  mintdim/apis/unit_build/config.pyPK       BP�6�Р  V  $           ���  mintdim/apis/unit_build/contracts.pyPK       BP�P��  �=  !           ���	  mintdim/apis/unit_build/errors.pyPK       BP��G��   �   !           ���  mintdim/apis/unit_build/events.pyPK       BPW#��>  �f  "           ��v  mintdim/apis/unit_build/runtime.pyPK       BPF��&  f  %           ���-  mintdim/apis/unit_build/validation.pyPK       BP           ,           ��]3  mintdim/apis/unit_build/adapters/__init__.pyPK       BP           3           ���3  mintdim/apis/unit_build/adapters/output/__init__.pyPK       BPAJ�   �   4           ���3  mintdim/apis/unit_build/adapters/output/directory.pyPK       BP���\  �  4           ���4  mintdim/apis/unit_build/adapters/output/histogram.pyPK       BP}_�  b  7           ���6  mintdim/apis/unit_build/adapters/output/length_field.pyPK       BPk��>  (  3           ���8  mintdim/apis/unit_build/adapters/output/manifest.pyPK       BP���d0
src\mintdim\apis\unit_build\errors.py:330:             "Template literal segments produce UNK when encoded by the tokenizer.\n\n"
src\mintdim\apis\unit_build\errors.py:396:         token_count: int,
src\mintdim\apis\unit_build\errors.py:400:         self.token_count = token_count
src\mintdim\apis\unit_build\errors.py:408:             "token_count:\n"
src\mintdim\apis\unit_build\errors.py:409:             f"{token_count}\n\n"
src\mintdim\apis\unit_build\runtime.py:18: from mintdim.apis.unit_build.adapters.output.length_field import write_length_field
src\mintdim\apis\unit_build\runtime.py:26:     CompiledTemplate,
src\mintdim\apis\unit_build\runtime.py:28:     Segment,
src\mintdim\apis\unit_build\runtime.py:31:     render_segments,
src\mintdim\apis\unit_build\runtime.py:37:     ShardRecord,
src\mintdim\apis\unit_build\runtime.py:64:     compiled: CompiledTemplate
src\mintdim\apis\unit_build\runtime.py:85:     segment_tokens: list[list[int]]
src\mintdim\apis\unit_build\runtime.py:86:     segment_lengths: list[int]
src\mintdim\apis\unit_build\runtime.py:87:     token_count: int
src\mintdim\apis\unit_build\runtime.py:102:     token_count: int
src\mintdim\apis\unit_build\runtime.py:246:             action=OverflowAction.EXTEND, new_unit_size=ctx.token_count
src\mintdim\apis\unit_build\runtime.py:252:             token_count=ctx.token_count,
src\mintdim\apis\unit_build\runtime.py:268:             token_count=ctx.token_count,
src\mintdim\apis\unit_build\runtime.py:285:     n_segments = len(prepared.compiled.segments)
src\mintdim\apis\unit_build\runtime.py:302:             n_segments=n_segments,
src\mintdim\apis\unit_build\runtime.py:347:                     n_segments=n_segments,
src\mintdim\apis\unit_build\runtime.py:356:                     token_counts=[rec.token_count for rec in records],
src\mintdim\apis\unit_build\runtime.py:398:     write_length_field(
src\mintdim\apis\unit_build\runtime.py:454:     n_segments: int,
src\mintdim\apis\unit_build\runtime.py:462:         n_segments=n_segments,
src\mintdim\apis\unit_build\runtime.py:473:     n_segments: int,
src\mintdim\apis\unit_build\runtime.py:483:             n_segments=n_segments,
src\mintdim\apis\unit_build\runtime.py:503: def _encode_segments(
src\mintdim\apis\unit_build\runtime.py:505:     compiled: CompiledTemplate,
src\mintdim\apis\unit_build\runtime.py:509:     """Tokenize each segment of one sample independently.
src\mintdim\apis\unit_build\runtime.py:511:     Returns (segment_token_ids_per_segment, rendered_segment_texts).
src\mintdim\apis\unit_build\runtime.py:512:     Both lists are parallel to compiled.segments.
src\mintdim\apis\unit_build\runtime.py:514:     rendered = render_segments(compiled, sample)
src\mintdim\apis\unit_build\runtime.py:515:     encoded_per_segment = tokenizer.encode_batch(rendered)
src\mintdim\apis\unit_build\runtime.py:516:     return encoded_per_segment, rendered
src\mintdim\apis\unit_build\runtime.py:523:     compiled: CompiledTemplate,
src\mintdim\apis\unit_build\runtime.py:536:         segment_tokens, segment_texts = _encode_segments(
src\mintdim\apis\unit_build\runtime.py:539:         segment_lengths = [len(toks) for toks in segment_tokens]
src\mintdim\apis\unit_build\runtime.py:540:         token_count = sum(segment_lengths)
src\mintdim\apis\unit_build\runtime.py:544:             for seg, text in zip(compiled.segments, segment_texts)
src\mintdim\apis\unit_build\runtime.py:547:         content_hash = _content_hash("".join(segment_texts))
src\mintdim\apis\unit_build\runtime.py:549:         fit = find_unit(token_count=token_count, sizes=sizes)
src\mintdim\apis\unit_build\runtime.py:558:                 token_count=token_count,
src\mintdim\apis\unit_build\runtime.py:570:                         token_count=token_count,
src\mintdim\apis\unit_build\runtime.py:580:                 segment_tokens, segment_lengths = _truncate_segments(
src\mintdim\apis\unit_build\runtime.py:581:                     segment_tokens=segment_tokens, max_total=max_size
src\mintdim\apis\unit_build\runtime.py:583:                 truncated_token_count = sum(segment_lengths)
src\mintdim\apis\unit_build\runtime.py:589:                         token_count=token_count,
src\mintdim\apis\unit_build\runtime.py:594:                 token_count = truncated_token_count
src\mintdim\apis\unit_build\runtime.py:606:                         token_count=token_count,
src\mintdim\apis\unit_build\runtime.py:617:         payload_tokens = [tid for seg in segment_tokens for tid in seg]
src\mintdim\apis\unit_build\runtime.py:624:         for seg_index, seg_toks in enumerate(segment_tokens):
src\mintdim\apis\unit_build\runtime.py:625:             seg_text = segment_texts[seg_index]
src\mintdim\apis\unit_build\runtime.py:645:                 segment_tokens=segment_tokens,
src\mintdim\apis\unit_build\runtime.py:646:                 segment_lengths=segment_lengths,
src\mintdim\apis\unit_build\runtime.py:647:                 token_count=token_count,
src\mintdim\apis\unit_build\runtime.py:660: def _truncate_segments(
src\mintdim\apis\unit_build\runtime.py:661:     *, segment_tokens: list[list[int]], max_total: int
src\mintdim\apis\unit_build\runtime.py:663:     """Trim trailing segments so that sum(lengths) == max_total.
src\mintdim\apis\unit_build\runtime.py:665:     Walks segments in order, keeping each full segment until the remaining
src\mintdim\apis\unit_build\runtime.py:666:     budget can't absorb it; the segment that hits the limit is sliced.
src\mintdim\apis\unit_build\runtime.py:670:     for toks in segment_tokens:
src\mintdim\apis\unit_build\runtime.py:694:             ShardRecord(
src\mintdim\apis\unit_build\runtime.py:695:                 segment_lengths=list(rec.segment_lengths),
src\mintdim\apis\unit_build\runtime.py:696:                 payload_tokens=[t for seg in rec.segment_tokens for t in seg],
src\mintdim\apis\unit_build\runtime.py:710:             "token_count": record.token_count,
src\mintdim\apis\unit_build\runtime.py:745:             "token_count": skip.token_count,
src\mintdim\apis\unit_build\runtime.py:784:         batch_tokens=sum(record.token_count for record in records),
src\mintdim\apis\unit_build\adapters\output\length_field.py:7: def write_length_field(
src\mintdim\apis\unit_build\adapters\output\length_field.py:15:     """Write `length_field.json` describing the per-record header layout.
src\mintdim\apis\unit_build\adapters\output\length_field.py:20:       - know which segment (field or literal) each header refers to
src\mintdim\apis\unit_build\adapters\output\length_field.py:28:     path = out_dir / "length_field.json"
src\mintdim\apis\unit_build\adapters\output\manifest.py:41:         "length_field_metadata": "length_field.json",
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:62: class ShardRecord:
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:65:     `segment_lengths` is parallel to the session's `sequence_template`. Its
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:67:     `payload_tokens` is the concatenation of every segment's tokens, in
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:71:     segment_lengths: list[int]
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:81:     is `1 + n_segments + unit_size`.
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:92:         n_segments: int,
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:100:         self.n_segments = n_segments
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:102:         self.record_slots = 1 + n_segments + unit_size
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:113:     def write_batch(self, records: list[ShardRecord]) -> list[tuple[int, int]]:
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:131:             payload_offset = 1 + self.n_segments
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:134:                 if len(rec.segment_lengths) != self.n_segments:
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:136:                         f"ShardRecord.segment_lengths must have length {self.n_segments}, "
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:137:                         f"got {len(rec.segment_lengths)}"
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:139:                 for j, length in enumerate(rec.segment_lengths):
src\mintdim\apis\unit_build\adapters\output\shard_writer.py:220:                         "No rendered JSONL sample produced a token_count that "
src\mintdim\apis\unit_build\adapters\output\stats.py:23:         token_counts: list[int],
src\mintdim\apis\unit_build\adapters\output\stats.py:27:         self.total_samples += len(token_counts)
src\mintdim\apis\unit_build\adapters\output\stats.py:28:         self.total_tokens += sum(token_counts)
src\mintdim\apis\unit_build\adapters\sources\template.py:7: from mintdim.apis.unit_build.domain.rules import LITERAL_SEGMENT_NAME
src\mintdim\apis\unit_build\adapters\sources\template.py:15: class Segment:
src\mintdim\apis\unit_build\adapters\sources\template.py:22: class CompiledTemplate:
src\mintdim\apis\unit_build\adapters\sources\template.py:24:     segments: list[Segment]
src\mintdim\apis\unit_build\adapters\sources\template.py:31:     segment_index: int
src\mintdim\apis\unit_build\adapters\sources\template.py:42: ) -> CompiledTemplate:
src\mintdim\apis\unit_build\adapters\sources\template.py:75:     segments: list[Segment] = []
src\mintdim\apis\unit_build\adapters\sources\template.py:78:             segments.append(
src\mintdim\apis\unit_build\adapters\sources\template.py:79:                 Segment(kind="literal", name=LITERAL_SEGMENT_NAME, content=static)
src\mintdim\apis\unit_build\adapters\sources\template.py:82:             segments.append(
src\mintdim\apis\unit_build\adapters\sources\template.py:83:                 Segment(kind="field", name=placeholders[i], content=None)
src\mintdim\apis\unit_build\adapters\sources\template.py:86:     return CompiledTemplate(
src\mintdim\apis\unit_build\adapters\sources\template.py:88:         segments=segments,
src\mintdim\apis\unit_build\adapters\sources\template.py:89:         sequence_template=[seg.name for seg in segments],
src\mintdim\apis\unit_build\adapters\sources\template.py:94: def render_segments(compiled: CompiledTemplate, sample: dict) -> list[str]:
src\mintdim\apis\unit_build\adapters\sources\template.py:95:     """Return one rendered text per segment, parallel to compiled.segments."""
src\mintdim\apis\unit_build\adapters\sources\template.py:97:     for seg in compiled.segments:
src\mintdim\apis\unit_build\adapters\sources\template.py:105: def render(compiled: CompiledTemplate, sample: dict) -> str:
src\mintdim\apis\unit_build\adapters\sources\template.py:107:     return "".join(render_segments(compiled, sample))
src\mintdim\apis\unit_build\adapters\sources\template.py:112:     compiled: CompiledTemplate,
src\mintdim\apis\unit_build\adapters\sources\template.py:115:     """Tier-1 UNK survey â€” encode every literal segment and report any UNKs.
src\mintdim\apis\unit_build\adapters\sources\template.py:121:     for index, seg in enumerate(compiled.segments):
src\mintdim\apis\unit_build\adapters\sources\template.py:132:                     segment_index=index,
src\mintdim\apis\unit_build\domain\overflow.py:25:     token_count: int
src\mintdim\apis\unit_build\domain\planner.py:8: def find_unit(*, token_count: int, sizes: list[int]) -> int | None:
src\mintdim\apis\unit_build\domain\planner.py:9:     """Return the smallest unit_size in `sizes` that fits `token_count`.
src\mintdim\apis\unit_build\domain\planner.py:11:     Returns None on overflow (token_count > max(sizes)). Caller decides the
src\mintdim\apis\unit_build\domain\planner.py:14:     idx = bisect_left(sizes, token_count)
src\mintdim\apis\unit_build\domain\planner.py:22:     token_count: int,
src\mintdim\apis\unit_build\domain\planner.py:30:     fit = find_unit(token_count=token_count, sizes=sizes)
src\mintdim\apis\unit_build\domain\planner.py:34:             token_count=token_count,
src\mintdim\apis\unit_build\domain\rules.py:23: LITERAL_SEGMENT_NAME = "token_template"
src\mintdim\apis\unit_build\ui\prompts.py:61:     """Ask whether to continue when literal template segments produce UNK tokens.
src\mintdim\apis\unit_build\ui\prompts.py:75:             f"  {_label('literal segments with UNK')} {_value(n_findings)}",
src\mintdim\apis\unit_build\ui\prompts.py:119:             f"  {_label('token_count')} {_value(ctx.token_count)}",
src\mintdim\apis\unit_build\ui\prompts.py:141:     token_count: int,
src\mintdim\apis\unit_build\ui\prompts.py:151:             f"  {_label('new unit_size')} must be an integer >= {token_count}",
src\mintdim\apis\unit_build\ui\prompts.py:170:         if value < token_count:
src\mintdim\apis\unit_build\ui\prompts.py:171:             sys.stderr.write(f"  invalid, must be >= {token_count}\n")
src\mintdim\apis\unit_build\ui\prompts.py:211:             token_count=ctx.token_count,
tests\integration\apis\unit_build\test_smoke.py:69:     length_field = json.loads(
tests\integration\apis\unit_build\test_smoke.py:70:         (out_dir / "length_field.json").read_text(encoding="utf-8")
tests\integration\apis\unit_build\test_smoke.py:72:     assert length_field == {
tests\integration\apis\unit_build\test_smoke.py:103:     # n_segments = 4 (token_template, q, token_template, a)
tests\integration\apis\unit_build\test_smoke.py:104:     n_segments = 4
tests\integration\apis\unit_build\test_smoke.py:105:     record_slots = 1 + n_segments + 64
tests\integration\apis\unit_build\test_smoke.py:113:     # Sample 0 â€” magic and per-segment lengths for "Q: alpha A: beta"
tests\integration\apis\unit_build\test_smoke.py:116:     payload_start = 1 + n_segments
tests\integration\apis\unit_build\test_smoke.py:373:     assert overflow[0]["token_count"] > 4
tests\integration\apis\unit_build\test_smoke.py:471:     # Literal "Z " sits in segment 0 (token_template). Field "x" is segment 1.
tests\unit\apis\unit_build\test_indices.py:14: from mintdim.apis.unit_build.adapters.output.length_field import write_length_field
tests\unit\apis\unit_build\test_indices.py:37:                 "token_count": 10,
tests\unit\apis\unit_build\test_indices.py:51:             "token_count": 10,
tests\unit\apis\unit_build\test_indices.py:130:         token_counts=[10, 8, 20],
tests\unit\apis\unit_build\test_indices.py:147: def test_length_field_writes_expected_schema(tmp_path: Path):
tests\unit\apis\unit_build\test_indices.py:148:     write_length_field(
tests\unit\apis\unit_build\test_indices.py:155:     payload = json.loads((tmp_path / "length_field.json").read_text(encoding="utf-8"))
tests\unit\apis\unit_build\test_indices.py:173:                 "token_count": 500,
tests\unit\apis\unit_build\test_indices.py:181:                 "token_count": 800,
tests\unit\apis\unit_build\test_shard_writer.py:9:     ShardRecord,
tests\unit\apis\unit_build\test_shard_writer.py:71:         n_segments=3,
tests\unit\apis\unit_build\test_shard_writer.py:73:     rec = ShardRecord(
tests\unit\apis\unit_build\test_shard_writer.py:74:         segment_lengths=[2, 1, 2],
tests\unit\apis\unit_build\test_shard_writer.py:100:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:102:     sw.write_batch([ShardRecord(segment_lengths=[2], payload_tokens=[100, 200])])
tests\unit\apis\unit_build\test_shard_writer.py:122:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:124:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
tests\unit\apis\unit_build\test_shard_writer.py:143:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:146:         ShardRecord(segment_lengths=[2], payload_tokens=[i, i + 1])
tests\unit\apis\unit_build\test_shard_writer.py:161: def test_shard_writer_rejects_segment_length_arity(tmp_path: Path):
tests\unit\apis\unit_build\test_shard_writer.py:168:         n_segments=2,
tests\unit\apis\unit_build\test_shard_writer.py:170:     with pytest.raises(ValueError, match="segment_lengths"):
tests\unit\apis\unit_build\test_shard_writer.py:171:         sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
tests\unit\apis\unit_build\test_shard_writer.py:182:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:186:             [ShardRecord(segment_lengths=[5], payload_tokens=[1, 2, 3, 4, 5])]
tests\unit\apis\unit_build\test_shard_writer.py:198:         n_segments=2,
tests\unit\apis\unit_build\test_shard_writer.py:218:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:220:     sw.write_batch([ShardRecord(segment_lengths=[3], payload_tokens=[1, 2, 3])])
tests\unit\apis\unit_build\test_shard_writer.py:233:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:251:         n_segments=1,
tests\unit\apis\unit_build\test_shard_writer.py:253:     sw.write_batch([ShardRecord(segment_lengths=[1], payload_tokens=[1])])
tests\unit\apis\unit_build\test_template.py:6:     LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:11:     render_segments,
tests\unit\apis\unit_build\test_template.py:24:         LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:26:         LITERAL_SEGMENT_NAME,
tests\unit\apis\unit_build\test_template.py:29:     assert [seg.kind for seg in compiled.segments] == [
tests\unit\apis\unit_build\test_template.py:35:     assert compiled.segments[0].content == "Q: "
tests\unit\apis\unit_build\test_template.py:36:     assert compiled.segments[2].content == " A: "
tests\unit\apis\unit_build\test_template.py:46:     assert all(seg.kind == "field" for seg in compiled.segments)
tests\unit\apis\unit_build\test_template.py:56:     assert compiled.sequence_template == [LITERAL_SEGMENT_NAME]
tests\unit\apis\unit_build\test_template.py:57:     assert compiled.segments[0].content == "static only"
tests\unit\apis\unit_build\test_template.py:92: def test_render_segments_returns_parallel_list():
tests\unit\apis\unit_build\test_template.py:98:     parts = render_segments(compiled, {"q": "what", "a": "this"})
tests\unit\apis\unit_build\test_template.py:100:     assert len(parts) == len(compiled.segments)
tests\unit\apis\unit_build\test_template.py:134:     assert findings[0].segment_index == 0
tests\unit\apis\unit_build\test_units.py:11:     assert assign_unit(token_count=10, sizes=[64, 128, 256], sample_locator="x:1") == 64
tests\unit\apis\unit_build\test_units.py:12:     assert assign_unit(token_count=64, sizes=[64, 128, 256], sample_locator="x:1") == 64
tests\unit\apis\unit_build\test_units.py:13:     assert assign_unit(token_count=65, sizes=[64, 128, 256], sample_locator="x:1") == 128
tests\unit\apis\unit_build\test_units.py:18:         assign_unit(token_count=999, sizes=[64, 128], sample_locator="train.jsonl:42")
tests\unit\apis\unit_build\test_units.py:19:     assert exc.value.token_count == 999
