CHANGELOG.md
CONTRIBUTING.md
LICENSE
MANIFEST.in
README.md
pyproject.toml
shog_ai/__init__.py
shog_ai.egg-info/PKG-INFO
shog_ai.egg-info/SOURCES.txt
shog_ai.egg-info/dependency_links.txt
shog_ai.egg-info/top_level.txt
shog_ai/core/corpus.py
shog_ai/core/deduplicator.py
shog_ai/core/id_generator.py
shog_ai/core/parser.py
shog_ai/core/record.py
shog_ai/core/text_cleaner.py
shog_ai/core/text_normalizer.py
shog_ai/core/validator.py
shog_ai/exporters/csv_exporter.py
shog_ai/exporters/jsonl_exporter.py
shog_ai/exporters/parquet_exporter.py
shog_ai/parsers/__init__.py
shog_ai/parsers/opus.py
shog_ai/parsers/wikipedia.py
shog_ai/pipeline/config.py
shog_ai/pipeline/metadata.py
shog_ai/pipeline/package_writer.py
shog_ai/pipeline/pipeline.py
shog_ai/pipeline/statistics.py
shog_ai/publishers/__init__.py
shog_ai/publishers/huggingface.py
shog_ai/publishers/local.py
shog_ai/utils/types.py