MANIFEST.in
README.md
build_bundled.sh
exports.txt
exports_aix.txt
exports_mac.txt
pyproject.toml
setup.py
tokenizer_comparison_cheat_sheet.md
sentencepiece/CMakeLists.txt
sentencepiece/LICENSE
sentencepiece/README.md
sentencepiece/VERSION.txt
sentencepiece/config.h.in
sentencepiece/sentencepiece.pc.in
sentencepiece/cmake/ios.toolchain.cmake
sentencepiece/cmake/sentencepieceConfig.cmake.in
sentencepiece/data/nfkc.bin
sentencepiece/data/nfkc_cf.bin
sentencepiece/data/nmt_nfkc.bin
sentencepiece/data/nmt_nfkc_cf.bin
sentencepiece/lite/BUILD.bazel
sentencepiece/lite/CMakeLists.txt
sentencepiece/lite/README.md
sentencepiece/lite/amalgamate.py
sentencepiece/lite/cached_sentencepiece_lite.cc
sentencepiece/lite/cached_sentencepiece_lite.h
sentencepiece/lite/cached_sentencepiece_lite_test.cc
sentencepiece/lite/sentencepiece_lite.cc
sentencepiece/lite/sentencepiece_lite.fbs
sentencepiece/lite/sentencepiece_lite.h
sentencepiece/lite/sentencepiece_lite_benchmark.cc
sentencepiece/lite/sentencepiece_lite_canonical_test.cc
sentencepiece/lite/sentencepiece_lite_test.cc
sentencepiece/lite/sentencepiece_model_converters.cc
sentencepiece/lite/sentencepiece_model_converters.h
sentencepiece/lite/spm_lite_main.cc
sentencepiece/lite/spm_to_fb_main.cc
sentencepiece/src/BUILD.bazel
sentencepiece/src/CMakeLists.txt
sentencepiece/src/bpe_model.cc
sentencepiece/src/bpe_model.h
sentencepiece/src/bpe_model_test.cc
sentencepiece/src/bpe_model_trainer.cc
sentencepiece/src/bpe_model_trainer.h
sentencepiece/src/bpe_model_trainer_test.cc
sentencepiece/src/builder.cc
sentencepiece/src/builder.h
sentencepiece/src/builder_test.cc
sentencepiece/src/char_model.cc
sentencepiece/src/char_model.h
sentencepiece/src/char_model_test.cc
sentencepiece/src/char_model_trainer.cc
sentencepiece/src/char_model_trainer.h
sentencepiece/src/char_model_trainer_test.cc
sentencepiece/src/compile_charsmap_main.cc
sentencepiece/src/filesystem.cc
sentencepiece/src/filesystem.h
sentencepiece/src/filesystem_test.cc
sentencepiece/src/freelist.h
sentencepiece/src/gtest_prod.h
sentencepiece/src/init.cc
sentencepiece/src/init.h
sentencepiece/src/invalid_model_test.cc
sentencepiece/src/model_factory.cc
sentencepiece/src/model_factory.h
sentencepiece/src/model_factory_test.cc
sentencepiece/src/model_interface.cc
sentencepiece/src/model_interface.h
sentencepiece/src/model_interface_test.cc
sentencepiece/src/normalization_rule.h
sentencepiece/src/normalizer.cc
sentencepiece/src/normalizer.h
sentencepiece/src/normalizer_test.cc
sentencepiece/src/ret_check.h
sentencepiece/src/sentencepiece.proto
sentencepiece/src/sentencepiece_model.proto
sentencepiece/src/sentencepiece_processor.cc
sentencepiece/src/sentencepiece_processor.h
sentencepiece/src/sentencepiece_processor_benchmarks_test.cc
sentencepiece/src/sentencepiece_processor_parallel_test.cc
sentencepiece/src/sentencepiece_processor_test.cc
sentencepiece/src/sentencepiece_trainer.cc
sentencepiece/src/sentencepiece_trainer.h
sentencepiece/src/sentencepiece_trainer_test.cc
sentencepiece/src/spec_parser.h
sentencepiece/src/spm_decode_main.cc
sentencepiece/src/spm_encode_main.cc
sentencepiece/src/spm_eval_main.cc
sentencepiece/src/spm_export_vocab_main.cc
sentencepiece/src/spm_normalize_main.cc
sentencepiece/src/spm_train_main.cc
sentencepiece/src/test_main.cc
sentencepiece/src/trainer_factory.cc
sentencepiece/src/trainer_factory.h
sentencepiece/src/trainer_factory_test.cc
sentencepiece/src/trainer_interface.cc
sentencepiece/src/trainer_interface.h
sentencepiece/src/trainer_interface_test.cc
sentencepiece/src/unicode_script.cc
sentencepiece/src/unicode_script.h
sentencepiece/src/unicode_script_map.h
sentencepiece/src/unicode_script_test.cc
sentencepiece/src/unigram_model.cc
sentencepiece/src/unigram_model.h
sentencepiece/src/unigram_model_test.cc
sentencepiece/src/unigram_model_trainer.cc
sentencepiece/src/unigram_model_trainer.h
sentencepiece/src/unigram_model_trainer_test.cc
sentencepiece/src/util.cc
sentencepiece/src/util.h
sentencepiece/src/util_test.cc
sentencepiece/src/word_model.cc
sentencepiece/src/word_model.h
sentencepiece/src/word_model_test.cc
sentencepiece/src/word_model_trainer.cc
sentencepiece/src/word_model_trainer.h
sentencepiece/src/word_model_trainer_test.cc
sentencepiece/third_party/BUILD.bazel
sentencepiece/third_party/darts_clone/LICENSE
sentencepiece/third_party/darts_clone/darts.h
sentencepiece/third_party/libsais/LICENSE
sentencepiece/third_party/libsais/libsais.c
sentencepiece/third_party/libsais/libsais.h
sentencepiece/third_party/rapidhash/rapidhash.h
src/sentencepiece/__init__.py
src/sentencepiece/__init__.pyi
src/sentencepiece/_version.py
src/sentencepiece/py.typed
src/sentencepiece/sentencepiece_model_pb2.py
src/sentencepiece/sentencepiece_pb2.py
src/sentencepiece/sentencepiece_pybind.cc
src/sentencepiece.egg-info/PKG-INFO
src/sentencepiece.egg-info/SOURCES.txt
src/sentencepiece.egg-info/dependency_links.txt
src/sentencepiece.egg-info/requires.txt
src/sentencepiece.egg-info/top_level.txt
src/sentencepiece/package_data/nfkc.bin
src/sentencepiece/package_data/nfkc_cf.bin
src/sentencepiece/package_data/nmt_nfkc.bin
src/sentencepiece/package_data/nmt_nfkc_cf.bin
test/__init__.py
test/botchan.txt
test/botchan_en_bpe_1000.model
test/botchan_en_unigram_1000.model
test/gen_stubs_test.py
test/gil_release_test.py
test/kyoto_ja_unigram_8000.model
test/numpy_test.py
test/sentencepiece_test.py
tools/gen_stubs.py