numpy>=1.24.0
pyyaml>=6.0
tqdm>=4.66.0
orjson>=3.9.0
cachetools>=5.3.0
tenacity>=8.2.0
trafilatura>=2.0.0

[all]
indw[ann,compress,dedup,dev,distributed,embedding,language,monitor,token,train]

[ann]
faiss-cpu>=1.7.4

[compress]
zstandard>=0.22.0

[dedup]
datasketch>=1.6.0

[dev]
pytest>=8.0
pytest-cov>=5.0
pytest-xdist>=3.5

[distributed]
dask>=2024.1.0
distributed>=2024.1.0

[embedding]
sentence-transformers>=2.2.0

[language]
langid>=1.1.6

[monitor]
prometheus_client>=0.20.0
opentelemetry-api>=1.24.0
opentelemetry-sdk>=1.24.0

[token]
tokenizers>=0.15.0

[train]
torch
tokenizers>=0.15.0
