LICENSE
MANIFEST.in
README.md
pyproject.toml
setup.py
data/lookup_tables/o200k_base_32768_metadata.json
data/lookup_tables/o200k_base_32768_tail_256_tokens.txt
data/lookup_tables/o200k_base_32768_tokens.txt
data/lookup_tables/tokenizer_gemma4_32768_metadata.json
data/lookup_tables/tokenizer_gemma4_32768_tail_256_tokens.txt
data/lookup_tables/tokenizer_gemma4_32768_tokens.txt
src/utf_token/__init__.py
src/utf_token/_api.py
src/utf_token/_reversible.py
src/utf_token/_tables.py
src/utf_token/py.typed
src/utf_token.egg-info/PKG-INFO
src/utf_token.egg-info/SOURCES.txt
src/utf_token.egg-info/dependency_links.txt
src/utf_token.egg-info/requires.txt
src/utf_token.egg-info/top_level.txt
src/utf_token/data/__init__.py
tests/test_download_token_vocab.py
tests/test_niah_identifier_benchmark.py
tests/test_process_token_vocab.py
tests/test_utf_token.py