DAY-EC activated.
DAY-EC activated.
=== status ===
{
  "command": "bin/day_run produce_sent_align produce_dmd_dedup_cram produce_sentd_snv_vcf produce_alignstats produce_snv_concordances produce_relatedness produce_vep produce_multiqc_all --config 'multiqc_qc={\"enable_tools\":[\"vep\"]}' -p -j 100 -k",
  "completed_at": "2026-05-29T21:25:58Z",
  "exit_code": 1,
  "repo_path": "/fsx/analysis_results/ubuntu/ccv20260529r5_illumina_snv_alignstats_relatedness_vep_multiqc/daylily-omics-analysis",
  "session_name": "ccv20260529r5_illumina_snv_alignstats_relatedness_vep_multiqc",
  "started_at": "2026-05-29T20:38:10Z"
}
=== repo ===
/fsx/analysis_results/ubuntu/ccv20260529r5_illumina_snv_alignstats_relatedness_vep_multiqc/daylily-omics-analysis
=== latest snakemake logs ===
1780089957.0000000000 .snakemake/log/2026-05-29T203833.823076.snakemake.log
1780089961.0000000000 .snakemake/log/2026-05-29T212559.133087.snakemake.log
=== latest log path: .snakemake/log/2026-05-29T212559.133087.snakemake.log ===
=== latest log tail ===
Unlocking working directory.
=== rule logs with errors ===
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log:4:2026-05-29 21:17:47 i192-dy-all-3 matplotlib.font_manager[51077] INFO Failed to extract font properties from /usr/share/fonts/truetype/noto/NotoColorEmoji.ttf: Could not set the fontsize (invalid pixel size; error code 0x17)
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:11:[E::hts_open_format] Failed to open file "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/tp_vcf_gz_stripped.vcf.gz" : No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:12:[init_data] Error: cannot write to "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/tp_vcf_gz_stripped.vcf.gz": No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:13:tbx_index_build3 failed: results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/tp_vcf_gz_stripped.vcf.gz
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:15:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:28:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/tp_vcf_gz_stripped.vcf.gz'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:30:During handling of the above exception, another exception occurred:
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:32:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:36:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/tp_vcf_gz_stripped.vcf.gz.X_tv_snp.vcf'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:44:[E::hts_open_format] Failed to open file "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fp_vcf_gz_stripped.vcf.gz" : No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:45:[init_data] Error: cannot write to "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fp_vcf_gz_stripped.vcf.gz": No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:46:tbx_index_build3 failed: results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fp_vcf_gz_stripped.vcf.gz
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:48:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:61:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fp_vcf_gz_stripped.vcf.gz'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:63:During handling of the above exception, another exception occurred:
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:65:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:69:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fp_vcf_gz_stripped.vcf.gz.X_tv_snp.vcf'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:71:[E::hts_open_format] Failed to open file "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fn_vcf_gz_stripped.vcf.gz" : No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:72:[init_data] Error: cannot write to "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fn_vcf_gz_stripped.vcf.gz": No such file or directory
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:73:tbx_index_build3 failed: results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fn_vcf_gz_stripped.vcf.gz
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:75:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:88:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fn_vcf_gz_stripped.vcf.gz'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:90:During handling of the above exception, another exception occurred:
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:92:Traceback (most recent call last):
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.altair-v1.1.parse_vcfeval_summary.log:96:FileNotFoundError: [Errno 2] No such file or directory: 'results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/concordance/_altair-v1_1/fn_vcf_gz_stripped.vcf.gz.X_tv_snp.vcf'
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/logs/goleft.log:2:error: missing value for --sex
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/logs/goleft.log:2:error: missing value for --sex
results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/tmp/licsrvr.log:9:2026/05/29 20:55:57 Failed to bind to address 127.0.0.1: Address already in use
logs/slurm/relatedness_batch_somalier_extract/relatedness_batch_somalier_extract.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.116.err:57:            echo "ERROR: expected exactly one somalier extract output in $tmp_dir, found ${#somalier_outputs[@]}" >> results/day/hg38_broad/other_reports/relatedness/sent/dmd/somalier/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.extract.log
logs/slurm/peddy/peddy.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.123.err:54:            if grep -q 'peddy: no hets found for sample' "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log" && grep -q 'IndexError: index 0 is out of bounds for axis 0 with size 0' "results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/peddy/log/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.peddy.log"; then
logs/slurm/peddy/peddy.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.123.err:59:                printf 'ERROR: peddy exited with status %s
logs/slurm/peddy/peddy.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.123.err:68:                printf 'ERROR: peddy completed but expected output is missing or empty: %s
logs/slurm/doppelmark_dups/doppelmark_dups.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.6.err:50:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/logs/dedupe.dmd.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.log 2>&1;
logs/slurm/vep_concat_index_chunks/vep_concat_index_chunks.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.26.err:46:            echo "ERROR: VEP annotated variant count ($ann_count) does not match input chunk count ($input_count)" >&2
logs/slurm/vep_concat_index_chunks/vep_concat_index_chunks.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.26.err:54:            echo "ERROR: final VEP VCF count ($final_count) does not match input count ($input_count)" >&2
logs/slurm/sent_DNAscope/sent_DNAscope.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.5.err:51:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/snv/sentd/log/vcfs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.sentd.1-24.snv.log 2>&1;
logs/slurm/calc_coverage_evenness/calc_coverage_evenness.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.99.err:55:        rm $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/norm_cov_eveness/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.md)/*per-base* || echo 'rm perbase failed';
logs/slurm/calc_coverage_evenness/calc_coverage_evenness.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.107.err:55:        rm $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/norm_cov_eveness/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.md)/*per-base* || echo 'rm perbase failed';
logs/slurm/sentieon_bwa_sort/sentieon_bwa_sort.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.7.err:60:        ulimit -n 65536 || echo "ulimit mod failed" > results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.sort.log 2>&1;
logs/slurm/site_mix_contam/site_mix_contam.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.112.err:47:            echo "ERROR: site_mix_contam production rule uses GATK pileup tables and does not support candidate_manifest donor attribution. Use the estimator CLI BAM/CRAM mode for donor attribution." >&2
logs/slurm/site_mix_contam/site_mix_contam.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.112.err:62:            if grep -Eq '^ERROR: Only [0-9]+ usable sites after depth filters; need at least [0-9]+' results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/contam/site_mix/logs/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.dmd.site_mix.log; then
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:43:        rm -rf $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:55:Error in rule goleft:
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:59:    log: results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/logs/goleft.log (check log file(s) for error details)
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:65:        rm -rf $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/dmd/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:75:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.105.err:78:Exiting because a job execution failed. Look above for error message
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:43:        rm -rf $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:55:Error in rule goleft:
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:59:    log: results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/logs/goleft.log (check log file(s) for error details)
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:65:        rm -rf $(dirname results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/goleft/golefttwo.done ) || echo rmGOLfailed ;
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:75:        (one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)
logs/slurm/goleft/goleft.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.106.err:78:Exiting because a job execution failed. Look above for error message
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.77.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.45.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.39.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.37.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.28.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.53.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.61.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.31.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.43.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.71.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.69.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.67.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.55.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.63.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.51.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.49.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.35.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.47.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.75.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.59.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.57.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.65.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.73.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.41.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/vep_chromosome/vep_chromosome.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.33.err:44:            echo "ERROR: VEP cache not found: /fsx/references/runtime_assets/tool_specific_resources/vep//homo_sapiens/114_GRCh38" >&2
logs/slurm/relatedness_batch_somalier_relate/relatedness_batch_somalier_relate.relatedness_batch.115.err:53:                printf 'ERROR: somalier relate did not create declared cohort output: %s\n' "$expected_output" >> results/day/hg38_broad/other_reports/relatedness/sent/dmd/somalier/logs/cohort.log
logs/slurm/mosdepth/mosdepth.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.104.err:46:        test -s results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.summary.txt || (printf 'ERROR: mosdepth summary output is missing or empty: %s\n' results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.summary.txt | tee -a results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.log >&2; exit 1)
logs/slurm/mosdepth/mosdepth.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.104.err:47:        test -s results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.global.dist.txt || (printf 'ERROR: mosdepth global_dist output is missing or empty: %s\n' results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.global.dist.txt | tee -a results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.log >&2; exit 1)
logs/slurm/mosdepth/mosdepth.TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.104.err:48:        test -s results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ.sent.na.mosdepth.region.dist.txt || (printf 'ERROR: mosdepth region_dist output is missing or empty: %s\n' results/day/hg38_broad/TVBILMN5X-HG003-5x-1-D0-PF-ILMN-NOVASEQ/align/sent/na/alignqc/mosdepth/TVBILMN5X--output truncated--