datafusion != 43.1.0, != 44.0.0  # https://github.com/apache/datafusion/issues/14055
luigi >= 3.8.1
pyarrow < 19.0.0  # https://github.com/apache/arrow/issues/45283
python-magic
pyzstd
tqdm
scancode-toolkit==32.2.1  # output may vary across versions; so we pin it in order to have reproducible datasets
commoncode==32.4.2 # typing may vary across versions; pinned to work well with scancode-toolkit