Metadata-Version: 2.3
Name: gyokusai
Version: 0.1.7
Summary: Gyokusai: A Web-Scale Text Processing Library
Requires-Dist: cugraph-cu13>=26.8.0
Requires-Dist: daft[lance,openai,ray]>=0.7.25
Requires-Dist: fastertext>=0.1.3
Requires-Dist: flash-kmeans>=0.3.1
Requires-Dist: ftfy>=6.3.1
Requires-Dist: gigatoken>=0.10.0
Requires-Dist: google-re2>=1.1.20251105
Requires-Dist: kernels<0.15
Requires-Dist: lark>=1.2.2
Requires-Dist: lxml>=6.1.3
Requires-Dist: magic-html>=0.1.8
Requires-Dist: presidio-analyzer>=2.2.364
Requires-Dist: presidio-anonymizer>=2.2.364
Requires-Dist: py-asciimath>=0.3.0
Requires-Dist: py7zr>=1.1.3
Requires-Dist: pyfastbloom>=0.1.0
Requires-Dist: pylance>=12.0.0
Requires-Dist: pylatexenc>=2.11
Requires-Dist: pywhichlang>=0.1.0
Requires-Dist: resiliparse>=1.0.9
Requires-Dist: sentence-transformers>=6.1.0
Requires-Dist: text-dedup>=0.4.1
Requires-Dist: tokenizers>=0.23.2
Requires-Dist: torch>=2.13.0
Requires-Dist: trafilatura>=2.2.0
Requires-Dist: transformers>=5.17.0
Requires-Dist: vllm>=0.30.0
Requires-Python: >=3.12
Description-Content-Type: text/markdown

# Installation
```bash
uv add gyokusai
```
```bash
ruff check --select I --fix . && ruff format .
```
```bibtex
@software{shippole2026gyokusai,
  title        = {Gyokusai: A Web-Scale Text Processing Library},
  author       = {Shippole, Enrico and Hu, Andrew and Li, Owen and Huang, Thomas and Jitsev, Jenia},
  year         = {2026},
  url          = {https://github.com/teraflop-ai/gyokusai},
  version      = {0.1.1},
  note         = {Python package available on PyPI at \url{https://pypi.org/project/gyokusai/}}
}
```

```bibtex
@misc{paster2023openwebmath,
    title={OpenWebMath: An Open Dataset of High-Quality Mathematical Web Text},
    author={Keiran Paster and Marco Dos Santos and Zhangir Azerbayev and Jimmy Ba},
    year={2023},
    eprint={2310.06786},
    archivePrefix={arXiv},
    primaryClass={cs.AI}
}
```
```bibtex
@misc{nemo_curator,
  title = {NeMo Curator: GPU-Accelerated Data Curation for Training AI Models},
  author = {NVIDIA},
  year = {2024},
  url = {https://github.com/NVIDIA-NeMo/Curator}
}
```
```bibtex
@article{parmar2024nemotron4,
  title = {Nemotron-4 15B Technical Report},
  author = {Parmar, Jupinder and Satheesh, Shrimai and others},
  journal = {arXiv preprint arXiv:2402.16819},
  year = {2024}
}
```