.python-version
MANIFEST.in
Makefile
README.md
pyproject.toml
uv.lock
docs/architecture.md
docs/curation.md
docs/development.md
docs/enrichment.md
docs/interactive.md
docs/persistence.md
docs/training.md
examples/01_tiny_shakespeare_snapshots.py
examples/02_c4_filtering.py
examples/03_c4_deduplication.py
examples/04_s2orc_distributions.py
examples/05_tiny_shakespeare_packing.py
examples/06_tiny_shakespeare_decontamination.py
examples/07_c4_quality_scores.py
examples/08_pile_source_mixture.py
examples/09_regmix_candidates.py
examples/10_tiny_shakespeare_resume.py
examples/README.md
examples/_tutorial.py
examples/data/tiny_shakespeare_excerpt.txt
proto/README.md
proto/premixdb/internal/derivation.proto
proto/premixdb/v1/corpus.proto
proto/premixdb/v1/dataset.proto
proto/premixdb/v1/field.proto
proto/premixdb/v1/index.proto
proto/premixdb/v1/intrinsic.proto
proto/premixdb/v1/profile.proto
proto/premixdb/v1/query.proto
proto/premixdb/v1/snapshot.proto
proto/premixdb/v1/status.proto
proto/premixdb/v1/storage.proto
scripts/export_enrichment.py
scripts/generate_protos.py
scripts/prepare_c4.py
scripts/prepare_s2orc.py
src/_premixdb_build.py
src/premixdb/__init__.py
src/premixdb/__main__.py
src/premixdb/_api.py
src/premixdb/_catalog.py
src/premixdb/_cli.py
src/premixdb/_curation.py
src/premixdb/_dataset_preview.py
src/premixdb/_default_tokenizer.py
src/premixdb/_display.py
src/premixdb/_enums.py
src/premixdb/_field_expr.py
src/premixdb/_field_ids.py
src/premixdb/_hub.py
src/premixdb/_identity.py
src/premixdb/_ids.py
src/premixdb/_inputs.py
src/premixdb/_mixing.py
src/premixdb/_policies.py
src/premixdb/_profiles.py
src/premixdb/_progress.py
src/premixdb/_protobuf.py
src/premixdb/_reader.py
src/premixdb/_requests.py
src/premixdb/_resources.py
src/premixdb/_runtime.py
src/premixdb/_shell.py
src/premixdb/_storage.py
src/premixdb/_torch.py
src/premixdb/_types.py
src/premixdb/_typing.py
src/premixdb/_unions.py
src/premixdb/_version.py
src/premixdb/_wire.py
src/premixdb/fields.py
src/premixdb/local.py
src/premixdb/py.typed
src/premixdb.egg-info/PKG-INFO
src/premixdb.egg-info/SOURCES.txt
src/premixdb.egg-info/dependency_links.txt
src/premixdb.egg-info/entry_points.txt
src/premixdb.egg-info/requires.txt
src/premixdb.egg-info/top_level.txt
src/premixdb/data/gpt2-LICENSE.txt
src/premixdb/data/gpt2-tokenizer.json
src/premixdb/data/gpt2-tokenizer.md
src/premixdb/data/tiny_shakespeare.txt
src/premixdb/data/tiny_shakespeare_excerpt.txt
src/premixdb/engine/__init__.py
src/premixdb/engine/contracts.py
src/premixdb/engine/curation.py
src/premixdb/engine/dataset_plan.py
src/premixdb/engine/datasets.py
src/premixdb/engine/execution.py
src/premixdb/engine/identity.py
src/premixdb/engine/mixtures.py
src/premixdb/engine/plans.py
src/premixdb/engine/queries.py
src/premixdb/engine/snapshots.py
src/premixdb/engine/spill.py
src/premixdb/engine/token_cache.py
src/premixdb/engine/value_cache.py
src/premixdb/enrichment/__init__.py
src/premixdb/enrichment/classification.py
src/premixdb/enrichment/datatrove.py
src/premixdb/enrichment/dupekit.py
src/premixdb/enrichment/interfaces.py
src/premixdb/enrichment/language.py
src/premixdb/enrichment/models.py
src/premixdb/enrichment/types.py
src/premixdb/execution/__init__.py
src/premixdb/execution/assets.py
src/premixdb/execution/cache.py
src/premixdb/execution/catalog.py
src/premixdb/execution/catalog_reader.py
src/premixdb/execution/coordinator.py
src/premixdb/execution/encodings.py
src/premixdb/execution/enrichment.py
src/premixdb/execution/inspection.py
src/premixdb/execution/materialization.py
src/premixdb/execution/metadata.py
src/premixdb/execution/mixing.py
src/premixdb/execution/partition_types.py
src/premixdb/execution/partitions.py
src/premixdb/execution/pipeline.py
src/premixdb/execution/planner.py
src/premixdb/execution/previewing.py
src/premixdb/execution/profiles.py
src/premixdb/execution/selections.py
src/premixdb/execution/sources.py
src/premixdb/execution/storage.py
src/premixdb/execution/tokens.py
src/premixdb/internal/__init__.py
src/premixdb/v1/__init__.py
tests/_type_support.py
tests/test_catalog_listings.py
tests/test_cli.py
tests/test_curation.py
tests/test_dataset_preview.py
tests/test_design.py
tests/test_embedding_service.py
tests/test_engine.py
tests/test_enrichment.py
tests/test_enrichment_service.py
tests/test_extended_curation.py
tests/test_extended_resources.py
tests/test_field_expressions.py
tests/test_hub.py
tests/test_jsonl_sources.py
tests/test_metadata.py
tests/test_mix.py
tests/test_packaging.py
tests/test_partitions.py
tests/test_premixdb.py
tests/test_profile_planning.py
tests/test_profiles.py
tests/test_progress.py
tests/test_properties.py
tests/test_public_ids.py
tests/test_readme_workflows.py
tests/test_requests.py
tests/test_resource_display.py
tests/test_reuse.py
tests/test_runtime.py
tests/test_s2orc_demo.py
tests/test_service.py
tests/test_shell_api.py
tests/test_snapshot_engine.py
tests/test_storage.py
tests/test_torch.py
tests/test_training_options.py
tests/test_training_reads.py
tests/test_tutorials.py
tests/benchmarks/test_benchmarks.py
tests/fixtures/bpe.json
tests/fixtures/mixture-v1.json
tests/fixtures/snapshot-v2.json
tests/fixtures/wordpiece.json