DAY-EC activated.
DAY-EC activated.
## status
{
  "command": "bin/day_run produce_sent_align produce_dmd_dedup_cram produce_sentd_snv_vcf produce_alignstats produce_snv_concordances produce_relatedness produce_gatk_contam_estimate produce_site_mix_contam_estimate produce_global_contam_check produce_vep produce_expansionhunter produce_htd_calls produce_metagenomics produce_multiqc_all --config 'aligners=[\
  "completed_at": "2026-05-30T08:27:19Z",
  "exit_code": 1,
  "repo_path": "/fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis",
  "session_name": "ccv20260529r32_illumina_hg002_kitchensink_multiqc",
  "started_at": "2026-05-30T08:03:26Z"
}
## latest snakemake logs
1780129641.0000000000 29 /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/log/2026-05-30T082719.145028.snakemake.log
1780129637.0000000000 837498 /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/log/2026-05-30T080353.289444.snakemake.log
## latest snakemake grep
### /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/log/2026-05-30T082719.145028.snakemake.log
--- final 80
Unlocking working directory.
### /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/log/2026-05-30T080353.289444.snakemake.log
1516:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt || (printf 'ERROR: mosdepth global_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
1517:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt || (printf 'ERROR: mosdepth region_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
1519:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1522:Error executing rule mosdepth on cluster (jobid: 146, external: 2877, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.mosdepth.146.sh). For error details see the cluster log and the log files of the involved rule(s).
1544:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.summary.txt || (printf 'ERROR: mosdepth summary output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.summary.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
1545:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt || (printf 'ERROR: mosdepth global_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
1546:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt || (printf 'ERROR: mosdepth region_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
1549:Error in rule goleft:
1559:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1569:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1572:Error executing rule goleft on cluster (jobid: 148, external: 2879, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.goleft.148.sh). For error details see the cluster log and the log files of the involved rule(s).
1620:            test -s "$mqc" || (echo "ERROR: missing per-sample unmapped Ganon2 MQC TSV: $mqc" | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_ganon2_summary.log; exit 1)
1628:        test "$first" -eq 0 || (echo "ERROR: no unmapped Ganon2 MQC inputs were available." | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_ganon2_summary.log; exit 1)
1647:            test -s "$mqc" || (echo "ERROR: missing per-sample unmapped metagenomics MQC TSV: $mqc" | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_summary.log; exit 1)
1655:        test "$first" -eq 0 || (echo "ERROR: no unmapped metagenomics MQC inputs were available." | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_summary.log; exit 1)
1659:Error in rule goleft:
1669:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1679:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1682:Error executing rule goleft on cluster (jobid: 149, external: 2889, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.goleft.149.sh). For error details see the cluster log and the log files of the involved rule(s).
1742:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1800:        test -s results/day/hg38/other_reports/unmapped_metagenomics_mqc.tsv || (echo "ERROR: missing unmapped metagenomics summary: results/day/hg38/other_reports/unmapped_metagenomics_mqc.tsv" | tee -a results/day/hg38/reports/logs/unmapped_metagenomics_multiqc.log; exit 1)
1856:        test -s results/day/hg38/other_reports/unmapped_metagenomics_ganon2_mqc.tsv || (echo "ERROR: missing unmapped Ganon2 summary: results/day/hg38/other_reports/unmapped_metagenomics_ganon2_mqc.tsv" | tee -a results/day/hg38/reports/logs/unmapped_metagenomics_ganon2_multiqc.log; exit 1)
1910:            test -s "$mqc" || (echo "ERROR: missing per-sample unmapped sourmash MQC TSV: $mqc" | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_sourmash_summary.log; exit 1)
1918:        test "$first" -eq 0 || (echo "ERROR: no unmapped sourmash MQC inputs were available." | tee -a results/day/hg38/other_reports/logs/unmapped_metagenomics_sourmash_summary.log; exit 1)
1947:                printf 'ERROR: somalier relate did not create declared cohort output: %s\n' "$expected_output" >> results/day/hg38/other_reports/relatedness/sent/dmd/somalier/logs/cohort.log
1973:        test -s results/day/hg38/other_reports/unmapped_metagenomics_sourmash_mqc.tsv || (echo "ERROR: missing unmapped sourmash summary: results/day/hg38/other_reports/unmapped_metagenomics_sourmash_mqc.tsv" | tee -a results/day/hg38/reports/logs/unmapped_metagenomics_sourmash_multiqc.log; exit 1)
2066:Error in rule goleft:
2076:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
2086:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
2089:Error executing rule goleft on cluster (jobid: 148, external: 2894, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.goleft.148.sh). For error details see the cluster log and the log files of the involved rule(s).
2219:Error in rule mosdepth:
2232:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.summary.txt || (printf 'ERROR: mosdepth summary output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.summary.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
2233:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt || (printf 'ERROR: mosdepth global_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.global.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
2234:        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt || (printf 'ERROR: mosdepth region_dist output is missing or empty: %s\n' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.region.dist.txt | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/mosdepth/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.mosdepth.log >&2; exit 1)
2236:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
2239:Error executing rule mosdepth on cluster (jobid: 146, external: 2892, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.mosdepth.146.sh). For error details see the cluster log and the log files of the involved rule(s).
2298:            echo "ERROR: site_mix_contam production rule uses GATK pileup tables and does not support candidate_manifest donor attribution. Use the estimator CLI BAM/CRAM mode for donor attribution." >&2
2313:            if grep -Eq '^ERROR: Only [0-9]+ usable sites after depth filters; need at least [0-9]+' results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/contam/site_mix/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.site_mix.log; then
2463:            echo "ERROR: VEP chromosome config would drop input VCF contigs:" >&2
2496:            if grep -q 'peddy: no hets found for sample' "results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log" && grep -q 'IndexError: index 0 is out of bounds for axis 0 with size 0' "results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log"; then
2501:                printf 'ERROR: peddy exited with status %s
2510:                printf 'ERROR: peddy completed but expected output is missing or empty: %s
2668:            printf 'SNP_PAIRS ERROR_PAIRS DOUBLE_ERROR_PAIR_COUNT DOUBLE_ERROR_FRACTION REL_ERROR_FRACTION NONSENSE_FRACTION PASS_FAIL REASON
3649:Error in rule read_haps_contam_identity:
3672:            printf 'SNP_PAIRS ERROR_PAIRS DOUBLE_ERROR_PAIR_COUNT DOUBLE_ERROR_FRACTION REL_ERROR_FRACTION NONSENSE_FRACTION PASS_FAIL REASON
3682:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
3686:Error executing rule read_haps_contam_identity on cluster (jobid: 63, external: 2907, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.read_haps_contam_identity.63.sh). For error details see the cluster log and the log files of the involved rule(s).
3728:            printf 'SNP_PAIRS ERROR_PAIRS DOUBLE_ERROR_PAIR_COUNT DOUBLE_ERROR_FRACTION REL_ERROR_FRACTION NONSENSE_FRACTION PASS_FAIL REASON
3776:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
3815:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
3847:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
3883:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
3937:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
3969:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4001:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4033:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4065:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4097:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4129:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4198:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4230:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4262:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4294:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4326:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4358:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4390:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4422:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4454:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4486:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4518:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4550:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4589:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4621:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
4660:Error in rule read_haps_contam_identity:
4683:            printf 'SNP_PAIRS ERROR_PAIRS DOUBLE_ERROR_PAIR_COUNT DOUBLE_ERROR_FRACTION REL_ERROR_FRACTION NONSENSE_FRACTION PASS_FAIL REASON
4693:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
4696:Error executing rule read_haps_contam_identity on cluster (jobid: 63, external: 2942, jobscript: /fsx/analysis_results/ubuntu/ccv20260529r32_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/tmp.lnngim8j/snakejob.read_haps_contam_identity.63.sh). For error details see the cluster log and the log files of the involved rule(s).
4861:            echo "ERROR: VEP annotated variant count ($ann_count) does not match input chunk count ($input_count)" >&2
4869:            echo "ERROR: final VEP VCF count ($final_count) does not match input count ($input_count)" >&2
5466:Exiting because a job execution failed. Look above for error message
--- final 80
              "$allvar_mean_dp" \
              sent \
              dmd \
              sentd \
              > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHCv5q.parse_vcfeval_summary.log 2>&1

            # Hard check: the legacy script should have written the MQC output where we declared it.
            test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHCv5q_concordance.mqc.tsv

Submitted job 41 with external jobid '2979'.
[Sat May 30 08:26:57 2026]
Finished job 41.
141 of 161 steps (88%) done
Select jobs to execute...

[Sat May 30 08:26:57 2026]
rule prep_for_concordance_check:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.snv.sort.vcf.gz, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.snv.sort.vcf.gz.tbi, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/
    output: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fofn, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fin.cmds
    log: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.log
    jobid: 23
    benchmark: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/benchmarks/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.bench.tsv
    reason: Missing output files: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_ultima_concordance.mqc.tsv, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOV
    wildcards: sample=JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ, alnr=sent, ddup=dmd, snv=sentd
    priority: 48
    threads: 8
    resources: mem_mb=3000, mem_mib=2862, disk_mb=1000, disk_mib=954, tmpdir=<TBD>, threads=16, time=5440, partition=i192mem,i192bigmem, vcpu=16, distribution=block, exclusive=, constraint=, exclude=, include=


            set +euo pipefail;

            mkdir -p $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done);
            mkdir -p $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.log);

            utc_ts=$(date -u +%Y-%m-%dT%H:%M:%SZ);

            # Write informational fofn
            {
                echo "# Refactor: per-ROI jobs are scheduled by Snakemake; this file is informational.";
                echo "# generated_at_utc=$utc_ts";
                echo "# truth_dir=/fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/";
                echo "# footprints=altair-v1.1,clinvar_genes,giabHC,giabHC_x_clinvar_genes,giabHC_x_ultima,giabHC_x_ultima_x_clinvar,giabHCv5q,hg38,hg38_m_giabHC,ultima";
                for mqc in results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1.1/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_altair-v1.1_concordance.mqc.tsv results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_clinvar_genes/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_clinvar_genes_concordance.mqc.tsv results/day
                    echo "$mqc";
                done;
            } > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fofn;

            # Write informational fin.cmds
            {
                echo "# Refactor: see Snakemake DAG for exact commands.";
                echo "# generated_at_utc=$utc_ts";
            } > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fin.cmds;

            # SKIPPED sentinel if no mqcs
            if [ "10" -eq 0 ]; then
                echo "No truthset ROI directories found; concordance skipped." > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done.SKIPPED;
            fi;

            # Log
            echo "Concordance sentinel complete. footprints=altair-v1.1,clinvar_genes,giabHC,giabHC_x_clinvar_genes,giabHC_x_ultima,giabHC_x_ultima_x_clinvar,giabHCv5q,hg38,hg38_m_giabHC,ultima mqcs=10" >> results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.log;

Submitted job 23 with external jobid '2980'.
[Sat May 30 08:27:07 2026]
Finished job 23.
142 of 161 steps (88%) done
Select jobs to execute...

[Sat May 30 08:27:07 2026]
localrule produce_snv_concordances:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1.1/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_altair-v1.1_concordance.mqc.tsv, resul--output truncated--