LICENSE
README.md
pyproject.toml
setup.py
filipino_tokenizer/__init__.py
filipino_tokenizer/base.py
filipino_tokenizer.egg-info/PKG-INFO
filipino_tokenizer.egg-info/SOURCES.txt
filipino_tokenizer.egg-info/dependency_links.txt
filipino_tokenizer.egg-info/not-zip-safe
filipino_tokenizer.egg-info/requires.txt
filipino_tokenizer.egg-info/top_level.txt
filipino_tokenizer/data/bisaya_roots.json
filipino_tokenizer/data/circumfix_table.json
filipino_tokenizer/data/infix_table.json
filipino_tokenizer/data/prefix_table.json
filipino_tokenizer/data/suffix_table.json
filipino_tokenizer/data/tagalog_roots.json
filipino_tokenizer/data/pretrained/merges.txt
filipino_tokenizer/data/pretrained/vocab.json
filipino_tokenizer/tagalog/__init__.py
filipino_tokenizer/tagalog/affixes.py
filipino_tokenizer/tagalog/bpe.py
filipino_tokenizer/tagalog/hf_tokenizer.py
filipino_tokenizer/tagalog/phonology.py
filipino_tokenizer/tagalog/roots.py
filipino_tokenizer/tagalog/segmenter.py
filipino_tokenizer/tagalog/tokenizer.py
tests/test_affixes.py
tests/test_rust_backend.py
tests/test_segmenter.py
tests/test_tokenizer.py