DAY-EC activated.
COMMAND_ID=5493115a-f1f2-4976-874a-aaed8160b4b1
DAY-EC activated.
DAY-EC activated.
timestamp_utc	2026-06-03T07:03:21Z

== squeue ==
JOBID|PARTITION|NAME|STATE|TIME|NODES|CPUS|MIN_MEMORY|NODELIST(REASON)
47|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0003_tiles0263-0393|CONFIGURING|1:07|1|192|1200000M|i192bigmem-dy-all-1
48|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0005_tiles0525-0654|PENDING|0:00|1|192|1200000M|(Resources)
49|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0002_tiles0132-0262|PENDING|0:00|1|192|1200000M|(Priority)
50|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0004_tiles0394-0524|PENDING|0:00|1|192|1200000M|(Priority)
51|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0006_tiles0655-0784|PENDING|0:00|1|192|1200000M|(Priority)
52|i192bigmem|run_bclconvert_tile_shard-run_bclconvert_L003_0001_tiles0001-0131|PENDING|0:00|1|192|1200000M|(Priority)

== sinfo ==
PARTITION|AVAIL|TIMELIMIT|NODES|STATE|NODELIST|AVAIL_FEATURES|GRES
i192bigmem|up|infinite|1|alloc#|i192bigmem-dy-all-1|dynamic,all|(null)
bcl2fq|up|infinite|4|idle~|bcl2fq-dy-nvme48-[1-4]|dynamic,nvme48|(null)
bcl2fq|up|infinite|4|idle~|bcl2fq-dy-nvme64-[1-4]|dynamic,i4i.16xlarge,nvme64|(null)
bcl2fq|up|infinite|4|idle~|bcl2fq-dy-nvme96-[1-4]|dynamic,nvme96|(null)
bcl2fq|up|infinite|4|idle~|bcl2fq-dy-nvme128-[1-4]|dynamic,i4i.32xlarge,nvme128|(null)

== tmux tail ==
m_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/0006_tiles0655-0784/Reports/Dem
ultiplex_Stats.csv, /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_f
rom_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports
/L003/0006_tiles0655-0784/SampleSheet.csv
    log: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconv
ert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/run_bclconvert.L00
3.0006_tiles0655-0784.log
    jobid: 9
    benchmark: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_b
clconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/benchmarks/run_bc
lconvert.L003.0006_tiles0655-0784.bench.tsv
    reason: Missing output files: /fsx/run_dir_mounts/20260514_LH01106_0009_B23T
VLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700
Z/tile_reports/L003/0006_tiles0655-0784/bclconvert.done, /fsx/run_dir_mounts/202
60514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devsh
m_trapfix_20260603T0700Z/tile_fastqs/L003/0006_tiles0655-0784/Reports/fastq_list
.csv, /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert
_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/0006_tile
s0655-0784/Reports/Demultiplex_Stats.csv; Input files updated by another job: /f
sx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6s
hard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/validated.ok, /fsx/run_dir_mo
unts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigm
em_devshm_trapfix_20260603T0700Z/normalized.SampleSheet.csv
    wildcards: lane=L003, shard=0006_tiles0655-0784
    threads: 192
    resources: mem_mb=1200000, mem_mib=1144410, disk_mb=1000, disk_mib=954, tmpd
ir=<TBD>, threads=192, time=5440, partition=i192bigmem, vcpu=192, distribution=b
lock, exclusive=--exclusive, constraint=, exclude=, include=

TMPDIR=/dev/shm/dayoa_bclconvert_tmp bash workflow/scripts/run_bclconvert_lane.s
h docker://nfcore/bclconvert:4.0.3 /fsx/run_dir_mounts/20260514_LH01106_0009_B23
TVLGLT4 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconve
rt_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/0006_ti
les0655-0784 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bcl
convert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/normalized.SampleSh
eet.csv 3 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclcon
vert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports/L003/0006
_tiles0655-0784/SampleSheet.csv false false false 1 24 4 64 32 false true 1000 '
{"BarcodeMismatchesIndex1": "0", "BarcodeMismatchesIndex2": "0"}' '{}' __dayoa_n
o_force__ 192 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bc
lconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/run_bclconver
t.L003.0006_tiles0655-0784.log /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLG
LT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/t
ile_fastqs/L003/0006_tiles0655-0784/Reports/fastq_list.csv /fsx/run_dir_mounts/2
0260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_dev
shm_trapfix_20260603T0700Z/tile_fastqs/L003/0006_tiles0655-0784/Reports/Demultip
lex_Stats.csv /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bc
lconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports/L003/
0006_tiles0655-0784/bclconvert.done s_3_2367+s_3_2368+s_3_2369+s_3_2370+s_3_2371
+s_3_2372+s_3_2373+s_3_2374+s_3_2375+s_3_2376+s_3_2377+s_3_2378+s_3_2379+s_3_238
0+s_3_2381+s_3_2382+s_3_2383+s_3_2384+s_3_2385+s_3_2386+s_3_2387+s_3_2388+s_3_23
89+s_3_2390+s_3_2391+s_3_2392+s_3_2393+s_3_2394+s_3_2395+s_3_2396+s_3_2397+s_3_2
398+s_3_2401+s_3_2402+s_3_2403+s_3_2404+s_3_2405+s_3_2406+s_3_2407+s_3_2408+s_3_
2409+s_3_2410+s_3_2411+s_3_2412+s_3_2413+s_3_2414+s_3_2415+s_3_2416+s_3_2417+s_3
_2418+s_3_2419+s_3_2420+s_3_2421+s_3_2422+s_3_2423+s_3_2424+s_3_2425+s_3_2426+s_
3_2427+s_3_2428+s_3_2429+s_3_2430+s_3_2431+s_3_2432+s_3_2433+s_3_2434+s_3_2435+s
_3_2436+s_3_2437+s_3_2438+s_3_2439+s_3_2440+s_3_2441+s_3_2442+s_3_2443+s_3_2444+
s_3_2445+s_3_2446+s_3_2447+s_3_2448+s_3_2449+s_3_2450+s_3_2451+s_3_2452+s_3_2453
+s_3_2454+s_3_2455+s_3_2456+s_3_2457+s_3_2458+s_3_2459+s_3_2460+s_3_2461+s_3_246
2+s_3_2463+s_3_2464+s_3_2465+s_3_2466+s_3_2467+s_3_2468+s_3_2469+s_3_2470+s_3_24
71+s_3_2472+s_3_2473+s_3_2474+s_3_2475+s_3_2476+s_3_2477+s_3_2478+s_3_2479+s_3_2
480+s_3_2481+s_3_2482+s_3_2483+s_3_2484+s_3_2485+s_3_2486+s_3_2487+s_3_2488+s_3_
2489+s_3_2490+s_3_2491+s_3_2492+s_3_2493+s_3_2494+s_3_2495+s_3_2496+s_3_2497+s_3
_2498 /dev/shm/dayoa_bclconvert 650000000000
Submitted job 9 with external jobid '51'.

[Wed Jun  3 07:02:14 2026]
rule run_bclconvert_tile_shard:
    input: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclco
nvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/validated.ok, /f
sx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6s
hard_i192bigmem_devshm_trapfix_20260603T0700Z/normalized.SampleSheet.csv
    output: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclc
onvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports/L003/00
01_tiles0001-0131/bclconvert.done, /fsx/run_dir_mounts/20260514_LH01106_0009_B23
TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T070
0Z/tile_fastqs/L003/0001_tiles0001-0131/Reports/fastq_list.csv, /fsx/run_dir_mou
nts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigme
m_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/0001_tiles0001-0131/Reports/Dem
ultiplex_Stats.csv, /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_f
rom_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports
/L003/0001_tiles0001-0131/SampleSheet.csv
    log: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconv
ert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/run_bclconvert.L00
3.0001_tiles0001-0131.log
    jobid: 4
    benchmark: /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_b
clconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/benchmarks/run_bc
lconvert.L003.0001_tiles0001-0131.bench.tsv
    reason: Missing output files: /fsx/run_dir_mounts/20260514_LH01106_0009_B23T
VLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700
Z/tile_reports/L003/0001_tiles0001-0131/bclconvert.done, /fsx/run_dir_mounts/202
60514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devsh
m_trapfix_20260603T0700Z/tile_fastqs/L003/0001_tiles0001-0131/Reports/Demultiple
x_Stats.csv, /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bcl
convert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/00
01_tiles0001-0131/Reports/fastq_list.csv; Input files updated by another job: /f
sx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6s
hard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/validated.ok, /fsx/run_dir_mo
unts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigm
em_devshm_trapfix_20260603T0700Z/normalized.SampleSheet.csv
    wildcards: lane=L003, shard=0001_tiles0001-0131
    threads: 192
    resources: mem_mb=1200000, mem_mib=1144410, disk_mb=1000, disk_mib=954, tmpd
ir=<TBD>, threads=192, time=5440, partition=i192bigmem, vcpu=192, distribution=b
lock, exclusive=--exclusive, constraint=, exclude=, include=

TMPDIR=/dev/shm/dayoa_bclconvert_tmp bash workflow/scripts/run_bclconvert_lane.s
h docker://nfcore/bclconvert:4.0.3 /fsx/run_dir_mounts/20260514_LH01106_0009_B23
TVLGLT4 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconve
rt_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_fastqs/L003/0001_ti
les0001-0131 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bcl
convert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/normalized.SampleSh
eet.csv 3 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclcon
vert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports/L003/0001
_tiles0001-0131/SampleSheet.csv false false false 1 24 4 64 32 false true 1000 '
{"BarcodeMismatchesIndex1": "0", "BarcodeMismatchesIndex2": "0"}' '{}' __dayoa_n
o_force__ 192 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bc
lconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/run_bclconver
t.L003.0001_tiles0001-0131.log /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLG
LT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/t
ile_fastqs/L003/0001_tiles0001-0131/Reports/fastq_list.csv /fsx/run_dir_mounts/2
0260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_dev
shm_trapfix_20260603T0700Z/tile_fastqs/L003/0001_tiles0001-0131/Reports/Demultip
lex_Stats.csv /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bc
lconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/tile_reports/L003/
0001_tiles0001-0131/bclconvert.done s_3_1101+s_3_1102+s_3_1103+s_3_1104+s_3_1105
+s_3_1106+s_3_1107+s_3_1108+s_3_1109+s_3_1110+s_3_1111+s_3_1112+s_3_1113+s_3_111
4+s_3_1115+s_3_1116+s_3_1117+s_3_1118+s_3_1119+s_3_1120+s_3_1121+s_3_1122+s_3_11
23+s_3_1124+s_3_1125+s_3_1126+s_3_1127+s_3_1128+s_3_1129+s_3_1130+s_3_1131+s_3_1
132+s_3_1133+s_3_1134+s_3_1135+s_3_1136+s_3_1137+s_3_1138+s_3_1139+s_3_1140+s_3_
1141+s_3_1142+s_3_1143+s_3_1144+s_3_1145+s_3_1146+s_3_1147+s_3_1148+s_3_1149+s_3
_1150+s_3_1151+s_3_1152+s_3_1153+s_3_1154+s_3_1155+s_3_1156+s_3_1157+s_3_1158+s_
3_1159+s_3_1160+s_3_1161+s_3_1162+s_3_1163+s_3_1164+s_3_1165+s_3_1166+s_3_1167+s
_3_1168+s_3_1169+s_3_1170+s_3_1171+s_3_1172+s_3_1173+s_3_1174+s_3_1175+s_3_1176+
s_3_1177+s_3_1178+s_3_1179+s_3_1180+s_3_1181+s_3_1182+s_3_1183+s_3_1184+s_3_1185
+s_3_1186+s_3_1187+s_3_1188+s_3_1189+s_3_1190+s_3_1191+s_3_1192+s_3_1193+s_3_119
4+s_3_1195+s_3_1196+s_3_1197+s_3_1198+s_3_1201+s_3_1202+s_3_1203+s_3_1204+s_3_12
05+s_3_1206+s_3_1207+s_3_1208+s_3_1209+s_3_1210+s_3_1211+s_3_1212+s_3_1213+s_3_1
214+s_3_1215+s_3_1216+s_3_1217+s_3_1218+s_3_1219+s_3_1220+s_3_1221+s_3_1222+s_3_
1223+s_3_1224+s_3_1225+s_3_1226+s_3_1227+s_3_1228+s_3_1229+s_3_1230+s_3_1231+s_3
_1232+s_3_1233 /dev/shm/dayoa_bclconvert 650000000000
Submitted job 4 with external jobid '52'.


== controller processes ==
ubuntu    134744  133201  0 07:02 pts/5    00:00:00 bash bin/day_run produce_bclconvert_fastqs -p -k -j 10 --rerun-triggers mtime --configfile config/bclconvert_l003_6shard_i192bigmem_devshm_trapfix.yaml
ubuntu    134920  134744  0 07:02 pts/5    00:00:00 bash bin/day_run produce_bclconvert_fastqs -p -k -j 10 --rerun-triggers mtime --configfile config/bclconvert_l003_6shard_i192bigmem_devshm_trapfix.yaml
ubuntu    134921  134920  7 07:02 pts/5    00:00:05 /home/ubuntu/miniconda3/envs/DAYOA/bin/python /home/ubuntu/miniconda3/envs/DAYOA/bin/snakemake --profile=/fsx/analysis_results/dyec0602bcl/bcl_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/daylily-omics-analysis/config/day_profiles/slurm produce_bclconvert_fastqs -p -k -j 10 --rerun-triggers mtime --configfile config/bclconvert_l003_6shard_i192bigmem_devshm_trapfix.yaml
ubuntu    134922  134920  0 07:02 pts/5    00:00:00 bash bin/day_run produce_bclconvert_fastqs -p -k -j 10 --rerun-triggers mtime --configfile config/bclconvert_l003_6shard_i192bigmem_devshm_trapfix.yaml

== dest counts ==
fastq_count	0
done_count	0
fastq_list_count	0
bench_count	1

== bcl log signals ==
2026-06-03T07:02:14.0000000000 0 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/validated.ok
2026-06-03T07:02:14.0000000000 259 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/bclconvert_validate_inputs.log
2026-06-03T07:02:14.0000000000 259 /fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/fasts_from_bclconvert_l003_6shard_i192bigmem_devshm_trapfix_20260603T0700Z/logs/bclconvert_validate_inputs.warnings.log

== slurm logs ==

