dpk_doc_id/__init__.py,sha256=x-q6Vv8zqM58pO0cWX3IMj1cPhRF66d0NY36tD9iWZA,106
dpk_doc_id/local.py,sha256=rQU6iCA8tn4vjF_veIxtI7tiWjxfj_ycO5dzmKNkj9E,2188
dpk_doc_id/local_python.py,sha256=3KtwOXRi6qJtxzRqYRKWEoUK2jFmSb9llgOgtHQRto8,2209
dpk_doc_id/transform.py,sha256=5PqGPqEjNP8bsxDwg8POsf8oITWK0TerMHi5EgUyJ48,6679
dpk_doc_id/transform_python.py,sha256=kSpdim9Et9tUck0ICzx5f8Quu4R5IOQNGYA9ukgid9I,5694
dpk_doc_id/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_doc_id/ray/local.py,sha256=HLt7JSSnuC6whFku47V5ikPInWbDQRHQhFwHYijYrwY,2261
dpk_doc_id/ray/s3.py,sha256=CT4H_bbLiVfzWlIvM_phFMgwVF6r_1z5Pby0K6-rekg,2251
dpk_doc_id/ray/transform.py,sha256=_uZKBeVfeUbu7Ws5p06YUZCG9J58QGpbS2oZz_jU_M8,5513
dpk_doc_id/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_doc_id/spark/local.py,sha256=O2zhDeglaE-jgSIiNnq4zTFwY1ZrDJM3gAxK4RhuTWs,2118
dpk_doc_id/spark/transform.py,sha256=85hWlWR2feDAdDxGL-BrRP5uA1z4PBJzUhiNCv2Kjnk,7896
dpk_doc_quality/__init__.py,sha256=x-q6Vv8zqM58pO0cWX3IMj1cPhRF66d0NY36tD9iWZA,106
dpk_doc_quality/cc_net_prepro.py,sha256=B9lt_2ZFVsORqXHdepNQ3B-wd3CT_9wNrC41Qcedtg0,4929
dpk_doc_quality/doc_Gopher_statistics.py,sha256=KxZpWjXKX5l9FkQnyLzwDb3fmdG-8awoGbKWnleFtPI,6530
dpk_doc_quality/doc_c4_statistics.py,sha256=r1GPFg0mmQIqtA9Pdt3F3hi3FpwugOP3Cey-aXAjFpg,6068
dpk_doc_quality/local.py,sha256=GYVZIsQqq4ICaJUIh7G37CbDDSUXZCZUIBxuGG2O83M,1731
dpk_doc_quality/local_python.py,sha256=UVeXbnWd3cORbGs3uhVRO5s7zUvrzGUvYZDQAyZ9VE4,2646
dpk_doc_quality/transform.py,sha256=qVPjaWssgfQl9A1bWNHzc5GgGGvfDUcX0Y8QoRCc-Ok,10904
dpk_doc_quality/transform_python.py,sha256=hM7dOOVE0OWYcoIyyhDamTD63Zbzr78O7t0c5G26sSU,3315
dpk_doc_quality/utils.py,sha256=n8vHmiLLAtO45_7S8lnseH4sUmyPrhBNIPuUEOhCqfI,2264
dpk_doc_quality/ldnoobw/de,sha256=6-RNd3KJc5Dif3vsQS_c3w78KU5NsVf4SHxM2NT6vVU,534
dpk_doc_quality/ldnoobw/en,sha256=r4UezvHV8hLKuhczmxKsOcwv7314x0h29nI3ZE_O6L0,3777
dpk_doc_quality/ldnoobw/es,sha256=BzM0JhzC58CDOfrv0vGfliPoJAoKwY_iomw2i4l0ls4,617
dpk_doc_quality/ldnoobw/fr,sha256=eXY6cF4IJWs5OVKFK7Y6qQP0SF4O8C9wc19eUO__ITw,793
dpk_doc_quality/ldnoobw/ja,sha256=-DxNCEPqZHTBe_Fx7dcrYWs69AP4YT8s3hvYQfr28xY,2339
dpk_doc_quality/ldnoobw/pt,sha256=Be9SSwu3-D-HJX9uxNq_cOFri7aTD3afuT3q9gj_GW0,629
dpk_doc_quality/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_doc_quality/ray/local.py,sha256=nN32AfJN41wE3Trenwz6g1LcoJbzSZJ7-GvUIy6zig8,2435
dpk_doc_quality/ray/s3.py,sha256=s41OEMKU6tyDXftnLpQFIrkHm_VPoXA9antgbHS7pBQ,2706
dpk_doc_quality/ray/transform.py,sha256=DNHM5ikMNAxAPSzSbKKuxe8VIh9K7-Ywoho-IblDZ3U,3314
dpk_ededup/__init__.py,sha256=YimSNPReHMSVGPHFqppIKcrtml8tnwyrCdnOAJCpWNs,30
dpk_ededup/local.py,sha256=mu1vj-p9xtWy4GqgBoGMKpurc7nBzZbIHiv-vh-Z0ik,2019
dpk_ededup/local_python.py,sha256=zmyAdk4KpE4cq8fLK2sVmwOkabTsnJzuQsvWq1Nt8LM,1936
dpk_ededup/local_python_incremental.py,sha256=Yo7suBDkhNsPHRHwJ9iVSF7MsC1AobgUfCL7L0_9gb0,2094
dpk_ededup/transform_base.py,sha256=zaEBFFqjE2FZko5J_EAZ5dU7hXwmyzRnHumCzFpdogI,9144
dpk_ededup/transform_python.py,sha256=nWC7Mmvs9y5wgPPPV6BzLstDDrhbA0czgHsb-oGN23Y,7084
dpk_ededup/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_ededup/ray/cluster_estimator.py,sha256=dBF6oC4-syYleZmWrJH5Hvku38jVi1lLmws6fOcgQoA,2227
dpk_ededup/ray/local.py,sha256=otnbZQ8-EBkedpq2QudgItAxiJoqASx3yrgmmNdVaMo,2254
dpk_ededup/ray/local_incremental.py,sha256=ubdtdm5c45XRvseeLftzyuDUcRBv9B8fEkDKqu3yq6I,2412
dpk_ededup/ray/s3.py,sha256=WBkvyZdVv0livBK9EJj9Ik5qnu4yVv4AonJwEFsxZeA,2254
dpk_ededup/ray/transform.py,sha256=PvUItjVvD9UGnhCxbglcixYaIZLJWLVOP8OCoAg2l1c,10919
dpk_extreme_tokenized/__init__.py,sha256=dENM5wNH4Xils3v7SAh-xc5BVR5awJltaqjI-SEJhQU,25
dpk_extreme_tokenized/common.py,sha256=BjjkhsdkihIi-MeRf7jlef6C7UOLKbeuClOo_W-8Je8,1763
dpk_extreme_tokenized/runtime.py,sha256=EBh_Miy4Nn-Jn1Cdd9AvGrS9IhMqUNt0EDNwVQtGzUw,4941
dpk_extreme_tokenized/transform.py,sha256=Gby0TWVgIomkoFx4X7wPsQWquzAWj9HXQXZSsmmUDXY,6701
dpk_extreme_tokenized/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_extreme_tokenized/ray/runtime.py,sha256=lhTUG8sMQNtwEqviRU830Yqrr6bYmnbwhlMyRSWQX8I,2313
dpk_filter/__init__.py,sha256=dENM5wNH4Xils3v7SAh-xc5BVR5awJltaqjI-SEJhQU,25
dpk_filter/local.py,sha256=ja2kHbTOmUf6n8xgIFzMBepdO8mfx0uZsS5HFG_JEhs,2193
dpk_filter/local_python.py,sha256=IUOlYX4MLnXFCMKZ66gQInTuI3nNiLeA0pWTp3GZnE8,2424
dpk_filter/test_support.py,sha256=8ms5N1jwOIzxsOpTxwKtzZYs2h_KSEu0_DWisXBVVI8,5692
dpk_filter/transform.py,sha256=TFzfNnWceUhxsRp89oQHH-qKz97wh_Y1xUAks8xZEUk,8260
dpk_filter/transform_python.py,sha256=v4oJ0cT9HUhL6dBWrmr8VO5iIazxnPcb6ipaFtIMWNA,2191
dpk_filter/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_filter/ray/local.py,sha256=Fh1z_d53lCZeUaXdaChP9II1_1EppksLaKfiIAtxuKA,2705
dpk_filter/ray/s3.py,sha256=J-MruQLEj06hwAM5w3UbB0tsCRxsyjoZKBDB4WUqm5w,2709
dpk_filter/ray/transform.py,sha256=NRKwofFuMTfE2vqYYjm-vPizhUGnvwCoTAa4V02i9pY,2495
dpk_filter/spark/local.py,sha256=qK5wu8xSpwu7HKCxCWMi8WihgG6bzwrhEWD9VXIlXt0,2406
dpk_filter/spark/transform.py,sha256=dPa4-ZTT1Wr3RcqZwAvF_pBbLeORKFbsV1XuQ7lPokw,1661
dpk_lang_id/lang_models.py,sha256=ZniqCjUwVKiL5DSY--Z7NfFcMEf0cWHUGyR745i9Hl8,1891
dpk_lang_id/local.py,sha256=HwF_JAJzvvzJImC0W9Hm977EPRUuV53JJG7PVCqEqD4,1973
dpk_lang_id/local_python.py,sha256=P4sqj9CY6IOakCfOlpqFkB1wDtmZxmclvMA62joDF24,2374
dpk_lang_id/nlp.py,sha256=hdqabpC9PCBPJn0OnGP1waMf3Idf2oJukFFcBFl1tXo,1863
dpk_lang_id/transform.py,sha256=O267yn6bmARxOldLn9bG-xy6FT6ObMjeSEfZGlPZsds,7279
dpk_lang_id/transform_python.py,sha256=gftOqpfT7KeXqT0xArKrIjTjFuT1hAhiP0hdwkwOteY,2634
dpk_lang_id/ray/local.py,sha256=zLlNg5CXO7xaKZAd8Lgfo6_lkOvnJTxxc6gGZIrr6P8,2731
dpk_lang_id/ray/s3.py,sha256=hVblcazG-YjldzJztQsuHCidELJoLFMmoAV6g8hB438,2674
dpk_lang_id/ray/transform.py,sha256=_-J586atSbfVHJXgNknG_1YVt31kD08qtGSqbwRe_8E,2973
dpk_readability/__init__.py,sha256=dENM5wNH4Xils3v7SAh-xc5BVR5awJltaqjI-SEJhQU,25
dpk_readability/common.py,sha256=bReVJhs82T7Jo7_wZenb7Q07Szf-G3hIGwYLZbPn6nM,4374
dpk_readability/runtime.py,sha256=JCair7iTrGedq3KV9rPiW-8VEHAd7rUOE6481qM-rDA,6498
dpk_readability/transform.py,sha256=5qmogXbnJnW6co6kjXevUQ4hRL7DSYyZV3s4h-_cV1I,9233
dpk_readability/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_readability/ray/runtime.py,sha256=BSoekJsdj_1sYWwV2TkEDwA9H5FhwKCeJQUxwJzNoKc,2287
dpk_rep_removal/__init__.py,sha256=2TA7YdzHwJJm1_bLx59kTHPoG0KfcmIKVx4m9RwqkCI,24
dpk_rep_removal/dedup_Rust_scripts.py,sha256=7i7xvF0BDn7ly7TPlds_JpOSOWAf1X1sbJ72iQlCWbQ,4912
dpk_rep_removal/dedup_pq_level.py,sha256=pWl-Gr2W2xMzY22h4gXzecC8iUTAV5uGwkSEidDwPIw,6953
dpk_rep_removal/make_suffix_array.py,sha256=phDSrLx9UHgpw-Bt_Rl78f6dLWnliX3GahGU5n0TVFE,7144
dpk_rep_removal/runtime.py,sha256=jLJo7H9VrGYysp3le1sHueFY_Nc0tryU0JY0s-g4IRE,5260
dpk_rep_removal/transform.py,sha256=ehAcmqOgC4qF9FlVYeqDM546Lih09dPm58E7CUXkLFs,5172
dpk_rep_removal/utils.py,sha256=EW2YpkfWT3UCl9IXwhcJzeOEB_BM5ChRFRELTC0psgU,11123
dpk_rep_removal/gpt2/merges.txt,sha256=HOFmR3PFDz4MyIQmGak-3EYkUltyixiKngvjO3cmrcU,456318
dpk_rep_removal/gpt2/special_tokens_map.json,sha256=WoQr6CMoPiMhQc4oaZ5UbqFuNHECas2AowZfjd0aEfM,510
dpk_rep_removal/gpt2/tokenizer_config.json,sha256=N_1hECrBbWn2M5QPqkx52QSma9AuBRRmRChAYYYA28E,825
dpk_rep_removal/gpt2/vocab.json,sha256=N1uEyxpv6vPeFtiCMIFLY3TnBq5cjH4LCipVjFJS3p0,1099700
dpk_rep_removal/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_rep_removal/ray/runtime.py,sha256=bD5Phj4I6mDKDZ1l0a-nKpwL_HWLCsNsg-hFSmR3--c,2257
dpk_rep_removal/rust/Cargo.toml,sha256=UYMZyaxHnGUX7L7um58VCI4dEMu3w4nn0MjWNeS3a40,413
dpk_rep_removal/rust/src/main.rs,sha256=-8Ka6AmdlW7Q0FsYU7j6X_fYe1F5CFKkgDRwq5XXMOU,51032
dpk_rep_removal/rust/src/table.rs,sha256=d2GjYBH5CkbWh1JU0WtSf7DNtv69tM2ldTvNWt4lo08,29734
dpk_rep_removal/rust/target/release/dedup_dataset,sha256=zNTRFGWHYn29YmBiS71Nn8bew1XdLwx2HBTKOf-47bk,5644616
dpk_rep_removal/rust/target/release/dedup_dataset.d,sha256=WWzi-e5ost9Vo9RxwQEb2e0-2L9d2kYSgeFYD4FvmII,136
dpk_resize/__init__.py,sha256=REXlfK9wXjZ0qbP9Zg05S2FZ2RrbDRe2AzbtRKDwLC0,48
dpk_resize/local.py,sha256=SInGmOwOq5t0roT6m2t1d3bqouRBTjHKzdbyCI5DFVU,1692
dpk_resize/local_python.py,sha256=w4CqIuPk_XwgZ0JOpUUYg9Fsf33M3W07QW0Ht-ND_DE,1864
dpk_resize/runtime.py,sha256=W9bvN95Lj5qIMMcQGg_pfCSvfmwHRdrOg2j6sLDBXRw,2360
dpk_resize/transform.py,sha256=QUxt7DOfL-v4Lm_Qie05Ml_0TUcU8dgotIhmqTf6A2w,8946
dpk_resize/ray/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_resize/ray/local.py,sha256=3NSbCGC3_SPssCO-YpL4eWA44J49th0EeBzGXC8853M,2033
dpk_resize/ray/runtime.py,sha256=aUm6yrsj53tQKCrjxCDemvmjLov7x6q3jbcD6Iwdlrg,2693
dpk_resize/ray/s3.py,sha256=-XxyILjXwr_oFmFtNFtbN-I1EMDcRtERYqhXR5IPXh4,2061
dpk_resize/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
dpk_resize/spark/local.py,sha256=9Eqpm4_vWoWl8jzgkFWh95L-AYCN8tRJBWrTdrcmadQ,1993
dpk_resize/spark/runtime.py,sha256=SGGePYoquDxKJrOPnJROu72JsxLEWKJrf4amt0eTXM8,1615
data_prep_toolkit_idiud-1.1.0.dist-info/METADATA,sha256=IYBnA5wSVAn41q0e93R7gIlMOFDCsy55rDw_t0fdPdw,3925
data_prep_toolkit_idiud-1.1.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
data_prep_toolkit_idiud-1.1.0.dist-info/top_level.txt,sha256=9_2-ehnLfs5BL8GIvzjw-8YqqkD2-9qazMaq1KLaHq4,126
data_prep_toolkit_idiud-1.1.0.dist-info/RECORD,,
