include ../Makefile.python

GRAPH_NAME ?= 2025-10-08-vulnerabilities
GRAPH_PATH ?= /dev/shm/vlorentz/$(GRAPH_NAME)/graph
DIGESTMAP_PATH ?= /srv/softwareheritage/ssd/data/vlorentz/datasets/$(GRAPH_NAME)/digestmap/

all: output/stats/.stamp output/identifiers.md output/git_origins.md output/packages.md no-md
no-md: data/vuln2commit_using_cherrypicks/.stamp data/commit2vuln_without_cherrypicks/.stamp data/commit2vuln_using_cherrypicks/.stamp data/vulnerabilities/.stamp

all.zip:
	wget https://osv-vulnerabilities.storage.googleapis.com/all.zip

nvd_cve/.stamp:
	python3 -m swh.osv.download_nvd_osv
	touch $@

all.sqlite: all.zip nvd_cve/.stamp
	@echo "[vulnerability reports to sqlite]"
	python3 -m swh.osv.to_sqlite

data/vulnerabilities/.stamp: all.sqlite
	@echo "[exporting vulnerabilities from sqlite to parquet]"
	cargo run --release --bin export-vulnerabilities -- \
		--db all.sqlite \
		--vulnerabilities-out data/vulnerabilities/

output/stats/.stamp: swh/osv/stats.py all.sqlite
	@echo "[generic stats]"
	python3 -m swh.osv.stats
	touch $@

output/identifiers.md: swh/osv/identifiers.py all.sqlite
	@echo "[identifiers stats]"
	python3 -m swh.osv.identifiers | tee $@

output/git_origins.md: swh/osv/map_git_origins.py all.sqlite
	@echo "[git originstats]"
	python3 -m swh.osv.map_git_origins | tee $@

output/packages.md: swh/osv/map_packages.py all.sqlite
	@echo "[package originstats]"
	python3 -m swh.osv.map_packages | tee $@

data/packages_to_vcs/.stamp: all.sqlite
	@echo "[package to VCS]"
	mkdir -p data/
	cargo run --release --bin packages -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--out-dir data/packages_to_vcs/
	touch $@

data/connected_components.wccs: all.sqlite data/packages_to_vcs/.stamp
	@echo "[connected components]"
	mkdir -p data/
	cargo run --release --bin compute-connected-components -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--package2vcs data/packages_to_vcs/ \
		--include-vcs-matches \
		--subgraphwccs-out data/connected_components.wccs

data/origins.csv: all.sqlite data/packages_to_vcs/.stamp
	@echo "[connected components]"
	mkdir -p data/
	cargo run --release --bin list-origins -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--package2vcs data/packages_to_vcs/ \
		--include-vcs-matches \
		--origins-out data/origins.csv

data/cherrypicks2originals.graph data/originals2cherrypicks.graph: all.sqlite data/connected_components.wccs
	@echo "[cherrypicks]"
	cargo run --release --bin list-cherry-picks -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--cherrypicks2originals-out data/cherrypicks2originals \
		--originals2cherrypicks-out data/originals2cherrypicks

data/cherrypicks2originals_reducetransitive.graph data/originals2cherrypicks_reducetransitive.graph: all.sqlite data/connected_components.wccs
	@echo "[cherrypicks (reduced transitive cherry-picking)]"
	cargo run --release --bin list-cherrypicks -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--cherrypicks2originals-out data/cherrypicks2originals_reducetransitive \
		--originals2cherrypicks-out data/originals2cherrypicks_reducetransitive \
		--reduce-transitive-cherrypicks

data/cherrypicks2originals_subgraphnodeids.graph data/originals2cherrypicks_subgraphnodeids.graph: all.sqlite data/connected_components.wccs
	@echo "[cherrypicks (renumbered)]"
	cargo run --release --bin list-cherrypicks -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--cherrypicks2originals-out data/cherrypicks2originals_subgraphnodeids \
		--originals2cherrypicks-out data/originals2cherrypicks_subgraphnodeids \
		--subgraph-nodeids

data/cherrypicks2originals_reducetransitive_subgraphnodeids.graph data/originals2cherrypicks_reducetransitive_subgraphnodeids.graph: all.sqlite data/connected_components.wccs
	@echo "[cherry-picks (renumbered + reduced transitive cherry-picking)]"
	cargo run --release --bin list-cherrypicks -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--cherrypicks2originals-out data/cherrypicks2originals_reducetransitive_subgraphnodeids \
		--originals2cherrypicks-out data/originals2cherrypicks_reducetransitive_subgraphnodeids \
		--subgraph-nodeids \
		--reduce-transitive-cherrypicks

data/vuln2commit_without_cherrypicks/.stamp: all.sqlite data/connected_components.wccs
	@echo "[list vuln2commit (with renumbering + without cherry-picks)]"
	cargo run --release --bin build-vuln2commit-parquet -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--out-dir data/vuln2commit_without_cherrypicks/
	touch $@

data/vuln2commit_using_cherrypicks/.stamp: all.sqlite data/connected_components.wccs data/cherrypicks2originals_subgraphnodeids.graph data/cherrypicks2originals_subgraphnodeids.ef data/originals2cherrypicks_subgraphnodeids.graph data/originals2cherrypicks_subgraphnodeids.ef
	@echo "[list vuln2commit (with renumbering + cherry-picks)]"
	cargo run --release --bin build-vuln2commit-parquet -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--cherrypicks2originals data/cherrypicks2originals_subgraphnodeids \
		--originals2cherrypicks data/originals2cherrypicks_subgraphnodeids \
		--out-dir data/vuln2commit_using_cherrypicks/
	touch $@

data/commit2vuln_without_cherrypicks.graph data/commit2vuln_without_cherrypicks.offsets data/commit2vuln_without_cherrypicks.properties: all.sqlite data/connected_components.wccs data/vuln2commit_without_cherrypicks/.stamp
	@echo "[commit2vuln enumeration (with renumbering + without cherry-picks)]"
	cargo run --release --bin build-commit2vuln-bv -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--vuln2commit-dir data/vuln2commit_without_cherrypicks/ \
		--multimap-out data/commit2vuln_without_cherrypicks

data/commit2vuln_using_cherrypicks.graph data/commit2vuln_using_cherrypicks.offsets data/commit2vuln_using_cherrypicks.properties: all.sqlite data/connected_components.wccs data/vuln2commit_using_cherrypicks/.stamp
	@echo "[commit2vuln enumeration (with renumbering + cherry-picks)]"
	cargo run --release --bin build-commit2vuln-bv -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--subgraphwccs data/connected_components.wccs \
		--vuln2commit-dir data/vuln2commit_using_cherrypicks/ \
		--multimap-out data/commit2vuln_using_cherrypicks

data/commit2vuln_without_cherrypicks/.stamp: data/commit2vuln_without_cherrypicks.graph data/commit2vuln_without_cherrypicks.ef
	@echo "[commit2vuln .graph to .parquet (with SWHIDs + without cherry-picks)]"
	cargo run --release --bin build-commit2vuln-parquet -- \
		--graph $(GRAPH_PATH) \
		--subgraphwccs data/connected_components.wccs \
		--multimap data/commit2vuln_without_cherrypicks \
		--parquet-out data/commit2vuln_without_cherrypicks

data/commit2vuln_using_cherrypicks/.stamp: data/commit2vuln_using_cherrypicks.graph data/commit2vuln_using_cherrypicks.ef
	@echo "[commit2vuln .graph to .parquet (with SWHIDs + cherry-picks)]"
	cargo run --release --bin build-commit2vuln-parquet -- \
		--graph $(GRAPH_PATH) \
		--subgraphwccs data/connected_components.wccs \
		--multimap data/commit2vuln_using_cherrypicks \
		--parquet-out data/commit2vuln_using_cherrypicks


# .graph is not actually needed for the computation, but Make needs this to resolve the dependency because we don't have targets for .offsets and .properties files
data/%.ef: data/%.graph
	@echo "[index .graph using .ef]"
	cargo run --release --bin build-ef -- \
		--graph $(basename $@)

data/reachable_contents/.stamp: data/connected_components.wccs
	@echo "[list reachable contents]"
	cargo run --release --bin list-reachable-contents -- \
		--graph $(GRAPH_PATH) \
		--subgraphwccs data/connected_components.wccs \
		--dir-out data/reachable_contents/
	touch $@

data/contents.rocksdb-complete.stamp /srv/softwareheritage/ssd/data/vlorentz/osv/data/contents.rocksdb: data/reachable_contents/.stamp
	@echo "[pre-cache contents]"
	@# to get download-contents:
	@# cargo install --git https://gitlab.softwareheritage.org/swh/devel/swh-datasets.git swh-contents
	pv --wait /srv/softwareheritage/ssd/data/vlorentz/osv/data/reachable_contents/*.zst \
		| zstdcat \
		| grep -v 'SWHID' \
		| download-contents \
			--graph $(GRAPH_PATH) \
			--digestmap $(DIGESTMAP_PATH) \
			--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
			--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
			--db data/contents.rocksdb
	touch $@

data/szz.tar: all.sqlite # add this to pre-fill the cache: data/contents.rocksdb-complete.stamp
	@echo "[SZZ]"
	cargo run --release --bin szz -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--max-tree-diff-items 100 \
		--tokenizer whitespace-stripped-line \
		--line-details-out-dir data/szz-line-details/ \
		--vuln-out-dir data/szz/
	cd data; tar -c szz/ > szz.tar
	cd data; tar -c szz-line-details/ > szz-line-details.tar

data/szz-diffs.tar.zst: data/szz.tar
	@echo "[post-SZZ diffs]"
	cargo run --release --bin compute-diffs -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/szz/ \
		--out-dir data/szz-diffs/
	cd data/; tar -c szz-diffs/ | pv | zstdmt -19 > szz-diffs.tar.zst

data/szz-checkouts/.stamp: data/szz.tar
	@echo "[SZZ checkouts]"
	cargo run --release --bin compute-checkouts -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/szz/ \
		--out-dir data/szz-checkouts/
	touch $@

data/vszz90.tar data/vszz90-line-details.tar data/vszz90-middle-revisions.tar: all.sqlite # add this to pre-fill the cache: data/contents.rocksdb-complete.stamp
	@echo "[VSZZ (similarity >= 90%)]"
	cargo run --release --bin szz -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--max-tree-diff-items 100 \
		--tokenizer whitespace-stripped-line \
		--min-line-similarity-ratio 0.90 \
		--middle-revisions-out-dir data/vszz90-middle-revisions/ \
		--line-details-out-dir data/vszz90-line-details/ \
		--vuln-out-dir data/vszz90/
	cd data; tar -c vszz90/ > vszz90.tar
	cd data; tar -c vszz90-line-details/ > vszz90-line-details.tar
	cd data; tar -c vszz90-middle-revisions/ > vszz90-middle-revisions.tar

data/vszz90-diffs.tar.zst: data/vszz90.tar
	@echo "[post-VSZZ diffs (similarity >= 90%)]"
	cargo run --release --bin compute-diffs -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/vszz90/ \
		--middle-revisions data/vszz90-middle-revisions/ \
		--out-dir data/vszz90-diffs/
	cd data/; tar -c vszz90-diffs/ | pv | zstdmt -19 > vszz90-diffs.tar.zst

data/vszz90-checkouts/.stamp: data/vszz90.tar
	@echo "[VSZZ checkouts (similarity >= 90%)]"
	cargo run --release --bin compute-checkouts -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/szz/ \
		--middle-revisions data/vszz90-middle-revisions/ \
		--out-dir data/vszz90-checkouts/
	touch $@

data/vszz75.tar data/vszz75-line-details.tar data/vszz75-middle-revisions.tar: all.sqlite # add this to pre-fill the cache: data/contents.rocksdb-complete.stamp
	@echo "[VSZZ (similarity >= 75%)]"
	cargo run --release --bin szz -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--max-tree-diff-items 100 \
		--tokenizer whitespace-stripped-line \
		--min-line-similarity-ratio 0.75 \
		--middle-revisions-out-dir data/vszz75-middle-revisions/ \
		--line-details-out-dir data/vszz75-line-details/ \
		--vuln-out-dir data/vszz75/
	cd data; tar -c vszz75/ > vszz75.tar
	cd data; tar -c vszz75-line-details/ > vszz75-line-details.tar
	cd data; tar -c vszz75-middle-revisions/ > vszz75-middle-revisions.tar

data/vszz75-diffs.tar.zst: data/vszz75.tar
	@echo "[post-VSZZ diffs (similarity >= 75%)]"
	cargo run --release --bin compute-diffs -- \
		--graph $(GRAPH_PATH) \
		--digestmap $(DIGESTMAP_PATH) \
		--url gzip:https://softwareheritage.s3.amazonaws.com/content/{sha1} \
		--url plain:https://archive.softwareheritage.org/api/1/content/sha1:{sha1}/raw/ \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/vszz75/ \
		--middle-revisions data/vszz75-middle-revisions/ \
		--out-dir data/vszz75-diffs/
	cd data/; tar -c vszz75-diffs/ | pv | zstdmt -19 > vszz75-diffs.tar.zst

data/vszz75-checkouts/.stamp: data/vszz75.tar
	@echo "[VSZZ checkouts (similarity >= 75%)]"
	cargo run --release --bin compute-checkouts -- \
		--graph $(GRAPH_PATH) \
		--db all.sqlite \
		--content-cache data/contents.rocksdb/ \
		--szz-revs data/szz/ \
		--middle-revisions data/vszz75-middle-revisions/ \
		--out-dir data/vszz75-checkouts/
	touch $@

clean:
	rm -rf all.zip nvd_cve/ all.sqlite output/

.PHONY: clean
