.python-version
MANIFEST.in
Makefile
README.md
pyproject.toml
uv.lock
docs/architecture.md
docs/curation.md
docs/development.md
docs/enrichment.md
docs/interactive.md
docs/persistence.md
docs/query-performance.md
docs/serialization.md
docs/training.md
examples/01_tiny_shakespeare_snapshots.py
examples/02_c4_filtering.py
examples/03_c4_deduplication.py
examples/04_s2orc_distributions.py
examples/05_tiny_shakespeare_packing.py
examples/06_tiny_shakespeare_decontamination.py
examples/07_c4_quality_scores.py
examples/08_pile_source_mixture.py
examples/09_regmix_candidates.py
examples/10_tiny_shakespeare_resume.py
examples/11_c4_pretraining_ablation.py
examples/README.md
examples/_tutorial.py
examples/data/README.md
examples/data/c4.jsonl
examples/data/code.jsonl
examples/data/literature.jsonl
examples/data/questions.jsonl
examples/data/reference.jsonl
examples/data/s2orc-train.jsonl
examples/data/s2orc-validation.jsonl
examples/data/sources.json
examples/recipes/README.md
examples/recipes/__init__.py
examples/recipes/_common.py
examples/recipes/apply.py
examples/recipes/falcon.py
examples/recipes/gopher.py
examples/recipes/gpt3.py
examples/recipes/llama.py
examples/recipes/mix.py
examples/recipes/t5.py
proto/README.md
proto/premixdb/internal/analytics.proto
proto/premixdb/internal/derivation.proto
proto/premixdb/internal/transport.proto
proto/premixdb/v1/corpus.proto
proto/premixdb/v1/data_mixture.proto
proto/premixdb/v1/field.proto
proto/premixdb/v1/index.proto
proto/premixdb/v1/intrinsic.proto
proto/premixdb/v1/profile.proto
proto/premixdb/v1/query.proto
proto/premixdb/v1/snapshot.proto
proto/premixdb/v1/status.proto
proto/premixdb/v1/storage.proto
scripts/benchmark_query_profiles.py
scripts/benchmark_serialization.py
scripts/export_enrichment.py
scripts/generate_protos.py
scripts/prepare_c4.py
scripts/prepare_s2orc.py
src/_premixdb_build.py
src/premixdb/__init__.py
src/premixdb/__main__.py
src/premixdb/contracts.py
src/premixdb/py.typed
src/premixdb/version.py
src/premixdb.egg-info/PKG-INFO
src/premixdb.egg-info/SOURCES.txt
src/premixdb.egg-info/dependency_links.txt
src/premixdb.egg-info/entry_points.txt
src/premixdb.egg-info/requires.txt
src/premixdb.egg-info/top_level.txt
src/premixdb/api/__init__.py
src/premixdb/api/base.py
src/premixdb/api/collections.py
src/premixdb/api/corpus.py
src/premixdb/api/curation.py
src/premixdb/api/database.py
src/premixdb/api/dataset.py
src/premixdb/api/dataset_preview.py
src/premixdb/api/display.py
src/premixdb/api/mixture.py
src/premixdb/api/profiles.py
src/premixdb/api/progress.py
src/premixdb/api/query.py
src/premixdb/api/snapshot.py
src/premixdb/api/unions.py
src/premixdb/cli/__init__.py
src/premixdb/cli/main.py
src/premixdb/cli/shell.py
src/premixdb/data/gpt2-LICENSE.txt
src/premixdb/data/gpt2-tokenizer.json
src/premixdb/data/gpt2-tokenizer.md
src/premixdb/data/tiny_shakespeare.txt
src/premixdb/data/tiny_shakespeare_excerpt.txt
src/premixdb/engine/__init__.py
src/premixdb/engine/analytics.py
src/premixdb/engine/concatenated.py
src/premixdb/engine/contracts.py
src/premixdb/engine/curation.py
src/premixdb/engine/dataset_plan.py
src/premixdb/engine/datasets.py
src/premixdb/engine/execution.py
src/premixdb/engine/identity.py
src/premixdb/engine/indexed.py
src/premixdb/engine/mixing.py
src/premixdb/engine/mixtures.py
src/premixdb/engine/names.py
src/premixdb/engine/plans.py
src/premixdb/engine/policies.py
src/premixdb/engine/queries.py
src/premixdb/engine/records.py
src/premixdb/engine/snapshots.py
src/premixdb/engine/sources.py
src/premixdb/engine/spill.py
src/premixdb/engine/token_cache.py
src/premixdb/engine/token_codec.py
src/premixdb/engine/tokenizer_assets.py
src/premixdb/engine/value_cache.py
src/premixdb/enrichment/__init__.py
src/premixdb/enrichment/classification.py
src/premixdb/enrichment/datatrove.py
src/premixdb/enrichment/dupekit.py
src/premixdb/enrichment/interfaces.py
src/premixdb/enrichment/language.py
src/premixdb/enrichment/models.py
src/premixdb/enrichment/types.py
src/premixdb/fields/__init__.py
src/premixdb/fields/catalog.py
src/premixdb/fields/expressions.py
src/premixdb/fields/ids.py
src/premixdb/fields/selectors.py
src/premixdb/internal/__init__.py
src/premixdb/runtime/__init__.py
src/premixdb/runtime/analytical_preparation.py
src/premixdb/runtime/analytics.py
src/premixdb/runtime/assets.py
src/premixdb/runtime/catalog.py
src/premixdb/runtime/coordinator.py
src/premixdb/runtime/datasets.py
src/premixdb/runtime/encodings.py
src/premixdb/runtime/enrichment.py
src/premixdb/runtime/environment.py
src/premixdb/runtime/hub_capture.py
src/premixdb/runtime/materialization.py
src/premixdb/runtime/mixing.py
src/premixdb/runtime/mixtures.py
src/premixdb/runtime/partition_types.py
src/premixdb/runtime/partitions.py
src/premixdb/runtime/pipeline.py
src/premixdb/runtime/planner.py
src/premixdb/runtime/preview_execution.py
src/premixdb/runtime/profiles.py
src/premixdb/runtime/sources.py
src/premixdb/runtime/split_datasets.py
src/premixdb/schemas/__init__.py
src/premixdb/schemas/binary.py
src/premixdb/schemas/enums.py
src/premixdb/schemas/ids.py
src/premixdb/schemas/messages.py
src/premixdb/schemas/protobuf.py
src/premixdb/schemas/requests.py
src/premixdb/schemas/splits.py
src/premixdb/schemas/validation.py
src/premixdb/storage/__init__.py
src/premixdb/storage/analytics.py
src/premixdb/storage/cache.py
src/premixdb/storage/catalog.py
src/premixdb/storage/lineage.py
src/premixdb/storage/metadata.py
src/premixdb/storage/objects.py
src/premixdb/storage/preview.py
src/premixdb/storage/profiles.py
src/premixdb/storage/publication.py
src/premixdb/storage/ranges.py
src/premixdb/storage/selections.py
src/premixdb/storage/tokens.py
src/premixdb/training/__init__.py
src/premixdb/training/reader.py
src/premixdb/training/sequences.py
src/premixdb/training/torch.py
src/premixdb/v1/__init__.py
tests/_reference.py
tests/_type_support.py
tests/conftest.py
tests/test_analytics.py
tests/test_architecture.py
tests/test_binary_codecs.py
tests/test_catalog_listings.py
tests/test_cli.py
tests/test_curation.py
tests/test_dataset_planning.py
tests/test_dataset_preview.py
tests/test_dataset_splits.py
tests/test_design.py
tests/test_document_pages.py
tests/test_document_preview.py
tests/test_embedding_service.py
tests/test_encodings.py
tests/test_engine.py
tests/test_enrichment.py
tests/test_enrichment_service.py
tests/test_ephemeral_caches.py
tests/test_evidence_groups.py
tests/test_extended_curation.py
tests/test_extended_resources.py
tests/test_field_expressions.py
tests/test_field_values.py
tests/test_file_publication.py
tests/test_hub_capture.py
tests/test_initialization.py
tests/test_intrinsic_fields.py
tests/test_jsonl_sources.py
tests/test_materialization.py
tests/test_metadata.py
tests/test_mix.py
tests/test_mixture_pool.py
tests/test_model_recipes.py
tests/test_packaging.py
tests/test_partition_planning.py
tests/test_partitions.py
tests/test_premixdb.py
tests/test_pretraining_ablation.py
tests/test_preview_execution.py
tests/test_profile_planning.py
tests/test_profiles.py
tests/test_progress.py
tests/test_properties.py
tests/test_public_ids.py
tests/test_readme_workflows.py
tests/test_requests.py
tests/test_resource_display.py
tests/test_reuse.py
tests/test_runtime.py
tests/test_s2orc_demo.py
tests/test_sequence_pages.py
tests/test_sequence_prefix.py
tests/test_service.py
tests/test_shell_api.py
tests/test_snapshot_engine.py
tests/test_snapshot_inventory.py
tests/test_snapshot_loading.py
tests/test_storage.py
tests/test_token_codec.py
tests/test_token_publication.py
tests/test_token_ranges.py
tests/test_token_serialization.py
tests/test_tokenizer_loading.py
tests/test_torch.py
tests/test_training_options.py
tests/test_training_reads.py
tests/test_tutorials.py
tests/test_typed_records.py
tests/benchmarks/test_benchmarks.py
tests/fixtures/bpe.json
tests/fixtures/mixture-v1.json
tests/fixtures/snapshot-v2.json
tests/fixtures/wordpiece.json