Metadata-Version: 2.4
Name: ion-tokenizer
Version: 1.0.0
Summary: Phrase-first semantic tokenizer compiler for neural language models
Requires-Python: >=3.9
Requires-Dist: click>=8.0
Requires-Dist: datasets>=2.14
Requires-Dist: spacy>=3.5
Requires-Dist: gensim>=4.3
Requires-Dist: tqdm>=4.65
Requires-Dist: orjson>=3.9
