DAY-EC activated.
DAY-EC activated.
## session
session=ccv20260529r26_illumina_hg002_kitchensink_multiqc
repo=/fsx/analysis_results/ubuntu/ccv20260529r26_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis
main_log=/fsx/analysis_results/ubuntu/ccv20260529r26_illumina_hg002_kitchensink_multiqc/daylily-omics-analysis/.snakemake/log/2026-05-30T065050.641034.snakemake.log
## failed sentinel
2026-05-30 07:40:59

## main snakemake failure markers
320:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.sort.log 2>&1;
383:Error in rule fastqc_subsampled:
430:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
490:Error in rule seqfu:
517:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
559:Error in rule fastqc_subsampled:
606:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
611:Error in rule seqfu:
638:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
679:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/logs/dedupe.dmd.JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.log 2>&1;
755:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
833:        rm $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/norm_cov_eveness/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.na.md)/*per-base* || echo 'rm perbase failed';
981:        rm $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/norm_cov_eveness/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.md)/*per-base* || echo 'rm perbase failed';
1015:	        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.sourmash.fastq.gz || (echo "ERROR: failed to write unmapped FASTQ: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.sourmash.fastq.gz" | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.unmapped_metagenomics_sourmash_gather.log; exit 1)
1062:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1194:	        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.ganon2.quick.fastq.gz || (echo "ERROR: failed to write unmapped FASTQ: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.ganon2.quick.fastq.gz" | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.unmapped_metagenomics_ganon2_quick.log; exit 1)
1282:	        test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.quick.fastq.gz || (echo "ERROR: failed to write unmapped FASTQ: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.human_unmapped.quick.fastq.gz" | tee -a results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/unmapped_metagenomics/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.unmapped_metagenomics_kraken2_quick.log; exit 1)
1340:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/log/vcfs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.1-24.snv.log 2>&1;
1376:Error in rule goleft:
1386:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1396:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1418:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1429:Error in rule mosdepth:
1446:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1451:Error in rule goleft:
1461:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1471:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1535:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1550:Error in rule goleft:
1560:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1570:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1575:Error in rule mosdepth:
1592:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1623:Error in rule mosdepth:
1640:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
1645:Error in rule goleft:
1655:        rm -rf $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
1665:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
2295:Error in rule mosdepth:
2312:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
2835:            if grep -q 'peddy: no hets found for sample' "results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log" && grep -q 'IndexError: index 0 is out of bounds for axis 0 with size 0' "results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log"; then
3653:Error in rule read_haps_contam_identity:
3671:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
3703:Error in rule read_haps_contam_identity:
3721:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
3739:Error in rule haplocheck_vcf_contam_identity:
3766:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
4703:Error in rule haplocheck_vcf_contam_identity:
4730:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
4868:Error in rule vep_concat_fofn:
4890:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
4922:Error in rule vep_concat_fofn:
4944:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
5507:Exiting because a job execution failed. Look above for error message
## main snakemake tail
    benchmark: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/benchmarks/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHC_x_ultima.parse_vcfeval_summary.bench.tsv
    reason: Missing output files: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHC_x_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHC_x_ultima_concordance.mqc.tsv; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concord
    wildcards: sample=JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ, alnr=sent, ddup=dmd, snv=sentd, cmpfootprint=giabHC_x_ultima
    threads: 16
    resources: mem_mb=16000, mem_mib=15259, disk_mb=1000, disk_mib=954, tmpdir=<TBD>, threads=16, time=5440, partition=i192mem,i192bigmem, vcpu=16, distribution=block, exclusive=, constraint=, exclude=, include=


            set -euo pipefail

            outdir="$(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHC_x_ultima/summary.txt)"
            # This file is legacy/debug; parse-vcfeval-summary.py uses its dirname to place the MQC TSV.
            legacy_parsed="$outdir/vcfeval_summary.parsed.tsv"

            # Keep mean depth behavior identical to the current rule (effectively NA/-1).
            allvar_mean_dp="na"

            # Prevent parse script from over-threading bcftools when many ROIs run concurrently.
            export DAYLILY_BCFTOOLS_THREADS="16"

            python workflow/scripts/parse-vcfeval-summary.py \
              results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHC_x_ultima/summary.txt \
              JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ \
              /fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/giabHC_x_ultima/HG002.bed \
              giabHC_x_ultima \
              HG002 \
              "$legacy_parsed" \
              "$allvar_mean_dp" \
              sent \
              dmd \
              sentd \
              > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHC_x_ultima.parse_vcfeval_summary.log 2>&1

            # Hard check: the legacy script should have written the MQC output where we declared it.
            test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHC_x_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHC_x_ultima_concordance.mqc.tsv

Submitted job 140 with external jobid '2770'.

[Sat May 30 07:25:07 2026]
rule parse_vcfeval_summary_roi:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/summary.txt, /fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/ultima/HG002.bed
    output: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_ultima_concordance.mqc.tsv
    log: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.ultima.parse_vcfeval_summary.log
    jobid: 150
    benchmark: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/benchmarks/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.ultima.parse_vcfeval_summary.bench.tsv
    reason: Missing output files: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_ultima_concordance.mqc.tsv; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/summa
    wildcards: sample=JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ, alnr=sent, ddup=dmd, snv=sentd, cmpfootprint=ultima
    threads: 16
    resources: mem_mb=16000, mem_mib=15259, disk_mb=1000, disk_mib=954, tmpdir=<TBD>, threads=16, time=5440, partition=i192mem,i192bigmem, vcpu=16, distribution=block, exclusive=, constraint=, exclude=, include=


            set -euo pipefail

            outdir="$(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/summary.txt)"
            # This file is legacy/debug; parse-vcfeval-summary.py uses its dirname to place the MQC TSV.
            legacy_parsed="$outdir/vcfeval_summary.parsed.tsv"

            # Keep mean depth behavior identical to the current rule (effectively NA/-1).
            allvar_mean_dp="na"

            # Prevent parse script from over-threading bcftools when many ROIs run concurrently.
            export DAYLILY_BCFTOOLS_THREADS="16"

            python workflow/scripts/parse-vcfeval-summary.py \
              results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/summary.txt \
              JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ \
              /fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/ultima/HG002.bed \
              ultima \
              HG002 \
              "$legacy_parsed" \
              "$allvar_mean_dp" \
              sent \
              dmd \
              sentd \
              > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.ultima.parse_vcfeval_summary.log 2>&1

            # Hard check: the legacy script should have written the MQC output where we declared it.
            test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_ultima/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_ultima_concordance.mqc.tsv

Submitted job 150 with external jobid '2771'.
[Sat May 30 07:25:17 2026]
Finished job 142.
127 of 161 steps (79%) done
[Sat May 30 07:25:17 2026]
Finished job 138.
128 of 161 steps (80%) done
[Sat May 30 07:25:17 2026]
Finished job 148.
129 of 161 steps (80%) done
[Sat May 30 07:25:17 2026]
Finished job 134.
130 of 161 steps (81%) done
[Sat May 30 07:25:47 2026]
Finished job 136.
131 of 161 steps (81%) done
[Sat May 30 07:25:47 2026]
Finished job 146.
132 of 161 steps (82%) done
[Sat May 30 07:25:47 2026]
Finished job 140.
133 of 161 steps (83%) done
[Sat May 30 07:25:57 2026]
Finished job 150.
134 of 161 steps (83%) done
[Sat May 30 07:39:59 2026]
Finished job 145.
135 of 161 steps (84%) done
Select jobs to execute...

[Sat May 30 07:39:59 2026]
rule parse_vcfeval_summary_roi:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/summary.txt, /fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/giabHCv5q/HG002.bed
    output: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHCv5q_concordance.mqc.tsv
    log: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHCv5q.parse_vcfeval_summary.log
    jobid: 144
    benchmark: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/benchmarks/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHCv5q.parse_vcfeval_summary.bench.tsv
    reason: Missing output files: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHCv5q_concordance.mqc.tsv; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHC
    wildcards: sample=JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ, alnr=sent, ddup=dmd, snv=sentd, cmpfootprint=giabHCv5q
    threads: 16
    resources: mem_mb=16000, mem_mib=15259, disk_mb=1000, disk_mib=954, tmpdir=<TBD>, threads=16, time=5440, partition=i192mem,i192bigmem, vcpu=16, distribution=block, exclusive=, constraint=, exclude=, include=


            set -euo pipefail

            outdir="$(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/summary.txt)"
            # This file is legacy/debug; parse-vcfeval-summary.py uses its dirname to place the MQC TSV.
            legacy_parsed="$outdir/vcfeval_summary.parsed.tsv"

            # Keep mean depth behavior identical to the current rule (effectively NA/-1).
            allvar_mean_dp="na"

            # Prevent parse script from over-threading bcftools when many ROIs run concurrently.
            export DAYLILY_BCFTOOLS_THREADS="16"

            python workflow/scripts/parse-vcfeval-summary.py \
              results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/summary.txt \
              JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ \
              /fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/giabHCv5q/HG002.bed \
              giabHCv5q \
              HG002 \
              "$legacy_parsed" \
              "$allvar_mean_dp" \
              sent \
              dmd \
              sentd \
              > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.giabHCv5q.parse_vcfeval_summary.log 2>&1

            # Hard check: the legacy script should have written the MQC output where we declared it.
            test -s results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_giabHCv5q/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_giabHCv5q_concordance.mqc.tsv

Submitted job 144 with external jobid '2775'.
[Sat May 30 07:40:39 2026]
Finished job 144.
136 of 161 steps (84%) done
Select jobs to execute...

[Sat May 30 07:40:39 2026]
rule prep_for_concordance_check:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.snv.sort.vcf.gz, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.snv.sort.vcf.gz.tbi, results/day/hg38/JEMILMN0
    output: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fofn, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fin.cmds
    log: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.log
    jobid: 131
    benchmark: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/benchmarks/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.bench.tsv
    reason: Missing output files: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.snv.sort.vcf.gz, results/day/hg3
    wildcards: sample=JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ, alnr=sent, ddup=dmd, snv=sentd
    priority: 48
    threads: 8
    resources: mem_mb=3000, mem_mib=2862, disk_mb=1000, disk_mib=954, tmpdir=<TBD>, threads=16, time=5440, partition=i192mem,i192bigmem, vcpu=16, distribution=block, exclusive=, constraint=, exclude=, include=


            set +euo pipefail;

            mkdir -p $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done);
            mkdir -p $(dirname results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.concordance.log);

            utc_ts=$(date -u +%Y-%m-%dT%H:%M:%SZ);

            # Write informational fofn
            {
                echo "# Refactor: per-ROI jobs are scheduled by Snakemake; this file is informational.";
                echo "# generated_at_utc=$utc_ts";
                echo "# truth_dir=/fsx/references/genomic_data/organism_annotations/H_sapiens/hg38/controls/giab/snv/v4.2.1/HG002/";
                echo "# footprints=altair-v1.1,clinvar_genes,giabHC,giabHC_x_clinvar_genes,giabHC_x_ultima,giabHC_x_ultima_x_clinvar,giabHCv5q,hg38,hg38_m_giabHC,ultima";
                for mqc in results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1.1/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_altair-v1.1_concordance.mqc.tsv results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_clinvar_genes/snv_JEMILMN0P1-HG002-0p1x-1-D0-P
                    echo "$mqc";
                done;
            } > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fofn;

            # Write informational fin.cmds
            {
                echo "# Refactor: see Snakemake DAG for exact commands.";
                echo "# generated_at_utc=$utc_ts";
            } > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.fin.cmds;

            # SKIPPED sentinel if no mqcs
            if [ "10" -eq 0 ]; then
                echo "No truthset ROI directories found; concordance skipped." > results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done.SKIPPED;
            fi;

            # Log
            echo "Concordance sentinel complete. footprints=altair-v1.1,clinvar_genes,giabHC,giabHC_x_clinvar_genes,giabHC_x_ultima,giabHC_x_ultima_x_clinvar,giabHCv5q,hg38,hg38_m_giabHC,ultima mqcs=10" >> results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.

Submitted job 131 with external jobid '2776'.
[Sat May 30 07:40:49 2026]
Finished job 131.
137 of 161 steps (85%) done
Select jobs to execute...

[Sat May 30 07:40:49 2026]
localrule produce_snv_concordances:
    input: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/concordance.done, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1.1/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_altair-v1.1_concordance.mqc.tsv, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-
    output: results/day/hg38/other_reports/giab_concordance_mqc.tsv
    jobid: 130
    reason: Missing output files: results/day/hg38/other_reports/giab_concordance_mqc.tsv; Input files updated by another job: results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1.1/snv_JEMILMN0P1-HG002-0p1x-1-D0-PF-ILMN-NOVASEQ_altair-v1.1_concordance.mqc.tsv, results/day/hg38/JEMILMN0P1-HG002-0p1x-1-D0
    priority: 48
    resources: mem_mb=3000, mem_mib=2862, disk_mb=1000, disk_mib=954, tmpdir=/dev/shm, threads=1, time=5440, partition=i192,i128,i192mem, vcpu=1, distribution=block, exclusive=, constraint=, exclude=, include=

[Sat May 30 07:40:52 2026]
Finished job 130.
138 of 161 steps (86%) done
Exiting because a job execution failed. Look above for error message
## sbatch--output truncated--