DAY-EC activated.
DAY-EC activated.
2026-05-30T13:05:09Z
i192mem-dy-all-1.dyec-test.pcluster
=== ps top ===
    PID    PPID STAT %CPU %MEM     ELAPSED COMMAND
1858533   24241 Sl   1152  0.0       00:05 /snap/amazon-ssm-agent/13009/ssm-document-worker 8ec7f885-a9fe-4903-9bb9-82dea3a2c048.2026-05-30T13-05-04.063Z
  73008   72978 Sl    988 11.1    02:10:23 /usr/local/bin/bcl-convert --bcl-input-directory /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.bclconvert_scratch/3310.25321/run --output-directory /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.bclconvert_scratch/3310.25321/fastqs --sample-sheet results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/normalized.SampleSheet.csv --strict-mode false --first-tile-only false --bcl-sampleproject-subdirectories false --fastq-gzip-compression-level 1 --bcl-num-parallel-tiles 8 --bcl-num-conversion-threads 8 --bcl-num-compression-threads 12 --bcl-num-decompression-threads 4 --shared-thread-odirect-output false -f
   4377       2 S     5.0  0.0    03:00:49 [socknal_sd01_00]
   4378       2 S     4.9  0.0    03:00:49 [socknal_sd01_01]
   4379       2 S     4.9  0.0    03:00:49 [socknal_sd01_02]
   4380       2 S     4.9  0.0    03:00:49 [socknal_sd01_03]
   4381       2 S     4.9  0.0    03:00:49 [socknal_sd01_04]
   4382       2 S     4.9  0.0    03:00:49 [socknal_sd01_05]
  24422   24421 Sl    2.8  0.0    02:54:40 /home/ubuntu/miniconda3/envs/DAYOA/bin/python -m snakemake --snakefile /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/workflow/Snakefile --target-jobs run_bclconvert: --allowed-rules run_bclconvert --cores all --attempt 1 --force-use-threads --resources mem_mb=180000 mem_mib=171662 disk_mb=1000 disk_mib=954 threads=192 time=5440 vcpu=192 --wait-for-files /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.snakemake/tmp.tvper0zy results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/validated.ok results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/normalized.SampleSheet.csv --force --keep-target-files --keep-remote --max-inventory-time 0 --nocolor --notemp --no-hooks --nolock --ignore-incomplete --rerun-triggers mtime params software-env code input --skip-script-cleanup --use-conda --conda-frontend conda --conda-prefix /fsx/resources/environments/conda/ubuntu/ip-10-0-0-88 --conda-base-path /home/ubuntu/miniconda3 --use-singularity --singularity-prefix /fsx/resources/environments/containers/ubuntu/ip-10-0-0-88 --singularity-args   -B /fsx:/fsx -B /tmp:/tmp -B $PWD/:$PWD   -B /dev/fd:/dev/fd  -B /dev/shm:$PWD/resources/dev/shm -B /dev/shm:/dev/shm  --wrapper-prefix https://github.com/snakemake/snakemake-wrappers/raw/ --config run_context_file=config/runs.tsv bootstrap_bclconvert=true --printshellcmds --latency-wait 100 --scheduler greedy --scheduler-solver-path /home/ubuntu/miniconda3/envs/DAYOA/bin --default-resources mem_mb=3000 disk_mb=1000 tmpdir=system_tmpdir threads=1 time=5440 partition=i192,i128,i192mem vcpu=1 distribution=block exclusive='' constraint='' exclude='' include='' --mode 2
   4371       2 S     2.1  0.0    03:00:49 [socknal_sd00_00]
   4372       2 S     2.1  0.0    03:00:49 [socknal_sd00_01]
   4373       2 S     2.1  0.0    03:00:49 [socknal_sd00_02]
   4374       2 S     2.1  0.0    03:00:49 [socknal_sd00_03]
   4375       2 S     2.1  0.0    03:00:49 [socknal_sd00_04]
   4376       2 S     2.1  0.0    03:00:49 [socknal_sd00_05]
   3833       1 Ssl   0.5  0.0    03:01:03 /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent -config /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.toml -envconfig /opt/aws/amazon-cloudwatch-agent/etc/env-config.json -otelconfig /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.yaml -pidfile /opt/aws/amazon-cloudwatch-agent/var/amazon-cloudwatch-agent.pid
1857804   24241 Sl    0.5  0.0       00:08 /snap/amazon-ssm-agent/13009/ssm-document-worker 166847c6-d97f-48f1-b585-d6c2807eb90d
1857860       1 Ss    0.5  0.0       00:07 /lib/systemd/systemd --user
     15       2 S     0.3  0.0    03:02:12 [ksoftirqd/0]
   4387       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_00]
   4388       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_01]
   4389       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_02]
   4390       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_03]
   4391       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_04]
   4392       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_05]
   4393       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_06]
   4394       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_07]
   4395       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_08]
   4396       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_09]
   4397       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_10]
   4398       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_11]
   4399       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_12]
   4400       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_13]
   4401       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_14]
   4402       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_15]
   4403       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_16]
   4404       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_17]
   4405       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_18]
   4406       2 I     0.2  0.0    03:00:49 [ptlrpcd_00_19]
=== bcl/singularity ===
  24422   24421 Sl    2.8  0.0    02:54:40 /home/ubuntu/miniconda3/envs/DAYOA/bin/python -m snakemake --snakefile /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/workflow/Snakefile --target-jobs run_bclconvert: --allowed-rules run_bclconvert --cores all --attempt 1 --force-use-threads --resources mem_mb=180000 mem_mib=171662 disk_mb=1000 disk_mib=954 threads=192 time=5440 vcpu=192 --wait-for-files /fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.snakemake/tmp.tvper0zy results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/validated.ok results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/normalized.SampleSheet.csv --force --keep-target-files --keep-remote --max-inventory-time 0 --nocolor --notemp --no-hooks --nolock --ignore-incomplete --rerun-triggers mtime params software-env code input --skip-script-cleanup --use-conda --conda-frontend conda --conda-prefix /fsx/resources/environments/conda/ubuntu/ip-10-0-0-88 --conda-base-path /home/ubuntu/miniconda3 --use-singularity --singularity-prefix /fsx/resources/environments/containers/ubuntu/ip-10-0-0-88 --singularity-args   -B /fsx:/fsx -B /tmp:/tmp -B $PWD/:$PWD   -B /dev/fd:/dev/fd  -B /dev/shm:$PWD/resources/dev/shm -B /dev/shm:/dev/shm  --wrapper-prefix https://github.com/snakemake/snakemake-wrappers/raw/ --config run_context_file=config/runs.tsv bootstrap_bclconvert=true --printshellcmds --latency-wait 100 --scheduler greedy --scheduler-solver-path /home/ubuntu/miniconda3/envs/DAYOA/bin --default-resources mem_mb=3000 disk_mb=1000 tmpdir=system_tmpdir threads=1 time=5440 partition=i192,i128,i192mem vcpu=1 distribution=block exclusive='' constraint='' exclude='' include='' --mode 2
  25321   24422 S     0.0  0.0    02:54:35 /home/ubuntu/miniconda3/envs/DAY-EC/bin/bash -c set -euo pipefail;           set -euo pipefail         mkdir -p results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs         : > results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         export TMPDIR=/fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.bclconvert_scratch         mkdir -p "$TMPDIR"          staging_mode=mounted_dev_shm         scratch_root=/fsx/analysis_results/ubuntu/ccv20260529r39_illumina_bclconvert/daylily-omics-analysis/.bclconvert_scratch         retain_scratch=false         effective_run_dir=/fsx/run_dir_mounts/20260514_LH01106_0009_B23TVLGLT4/         effective_output_dir=results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs         scratch_dir=""          cleanup_bclconvert_scratch() {             if [ -n "$scratch_dir" ] && [ "$retain_scratch" != "true" ]; then                 rm -rf "$scratch_dir"             fi         }         trap cleanup_bclconvert_scratch EXIT          echo "run_bclconvert started: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "host: $(hostname)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "threads: 192" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "resources_mem_mb: 180000" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "TMPDIR: $TMPDIR" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "staging_mode: $staging_mode" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "scratch_root: $scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "bcl_input_directory: $effective_run_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "source_s3_uri: s3://lsmc-ssf-sequencing-data/basecalls/lsmc/ssf-hq/LH01106/2026/20260514_LH01106_0009_B23TVLGLT4/" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         echo "output_directory: $effective_output_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log         nproc >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true         df -h "$TMPDIR" results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true         command -v singularity >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1         singularity exec docker://nfcore/bclconvert:4.0.3 bcl-convert --version >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1          case "$staging_mode" in             direct)                 ;;             output_dev_shm)                 mkdir -p "$scratch_root"                 scratch_dir="$scratch_root/${SLURM_JOB_ID:-local}.$$"                 scratch_output_dir="$scratch_dir/fastqs"                 mkdir -p "$scratch_dir"                 input_disk_bytes="$(du -sB1 "$effective_run_dir" | awk '{print $1}')"                 input_apparent_bytes="$(du -sb "$effective_run_dir" | awk '{print $1}')"                 required_bytes="$((input_disk_bytes * 1 + 1073741824))"                 scratch_parent="$(dirname "$scratch_root")"                 available_bytes="$(df -PB1 "$scratch_parent" | awk 'NR == 2 {print $4}')"                 echo "scratch_dir: $scratch_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_disk_bytes: $input_disk_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_apparent_bytes: $input_apparent_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_required_bytes: $required_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_available_bytes: $available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if [ "$available_bytes" -lt "$required_bytes" ]; then                     echo "Insufficient scratch for bclconvert.staging_mode=output_dev_shm: required=$required_bytes available=$available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 effective_output_dir="$scratch_output_dir"                 df -h "$scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true                 ;;             dev_shm)                 mkdir -p "$scratch_root"                 scratch_dir="$scratch_root/${SLURM_JOB_ID:-local}.$$"                 scratch_run_dir="$scratch_dir/run"                 scratch_output_dir="$scratch_dir/fastqs"                 mkdir -p "$scratch_run_dir"                 input_disk_bytes="$(du -sB1 "$effective_run_dir" | awk '{print $1}')"                 input_apparent_bytes="$(du -sb "$effective_run_dir" | awk '{print $1}')"                 required_bytes="$((input_disk_bytes * 1 + 1073741824))"                 scratch_parent="$(dirname "$scratch_root")"                 available_bytes="$(df -PB1 "$scratch_parent" | awk 'NR == 2 {print $4}')"                 echo "scratch_dir: $scratch_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_disk_bytes: $input_disk_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_apparent_bytes: $input_apparent_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_required_bytes: $required_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_available_bytes: $available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if [ "$available_bytes" -lt "$required_bytes" ]; then                     echo "Insufficient scratch for bclconvert.staging_mode=dev_shm: required=$required_bytes available=$available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 echo "Copying BCL run directory to scratch: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 cp -aL --sparse=always "$effective_run_dir"/. "$scratch_run_dir"/                 effective_run_dir="$scratch_run_dir"                 effective_output_dir="$scratch_output_dir"                 df -h "$scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true                 ;;             mounted_dev_shm)                 mkdir -p "$scratch_root"                 scratch_dir="$scratch_root/${SLURM_JOB_ID:-local}.$$"                 scratch_run_dir="$scratch_dir/run"                 scratch_output_dir="$scratch_dir/fastqs"                 scratch_sync_log_dir="$scratch_dir/rsync_logs"                 stage_metadata_log="results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/mounted_metadata.log"                 stage_files_log="results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/mounted_files.log"                 mkdir -p "$scratch_run_dir" "$scratch_sync_log_dir"                 lane_root="$effective_run_dir/Data/Intensities/BaseCalls"                 if [ ! -d "$lane_root" ]; then                     echo "BCL run directory is missing lane root: $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 root_file_bytes="$(find "$effective_run_dir" -maxdepth 1 -type f -printf '%s\n' | awk '{s+=$1} END {print s+0}')"                 intensities_file_bytes="$(find "$effective_run_dir/Data/Intensities" -maxdepth 1 -type f -printf '%s\n' | awk '{s+=$1} END {print s+0}')"                 lane_root_disk_bytes="$(du -sB1 "$lane_root" | awk '{print $1}')"                 input_disk_bytes="$((root_file_bytes + intensities_file_bytes + lane_root_disk_bytes))"                 input_apparent_bytes="$input_disk_bytes"                 required_bytes="$((input_disk_bytes * 1 + 1073741824))"                 scratch_parent="$(dirname "$scratch_root")"                 available_bytes="$(df -PB1 "$scratch_parent" | awk 'NR == 2 {print $4}')"                 echo "scratch_dir: $scratch_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_basis: mounted_metadata_plus_lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_root_file_bytes: $root_file_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_intensities_file_bytes: $intensities_file_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_lane_root_disk_bytes: $lane_root_disk_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_disk_bytes: $input_disk_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_input_apparent_bytes: $input_apparent_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_required_bytes: $required_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "scratch_available_bytes: $available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if [ "$available_bytes" -lt "$required_bytes" ]; then                     echo "Insufficient scratch for bclconvert.staging_mode=mounted_dev_shm: required=$required_bytes available=$available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 lane_ids=$(find "$lane_root" -mindepth 1 -maxdepth 1 -type d -name 'L[0-9][0-9][0-9]' -printf '%f\n' | sort)                 if [ -z "$lane_ids" ]; then                     echo "BCL run directory has no L### lane directories under $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 echo "Staging mounted BCL run directory to scratch: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 echo "mounted_stage_lanes: $(printf "%s" "$lane_ids" | tr '\n' ' ')" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 mounted_stage_jobs=64                 if [ "$mounted_stage_jobs" -lt 1 ]; then                     echo "bclconvert.mounted_stage_jobs must be >= 1" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 echo "mounted_stage_jobs: $mounted_stage_jobs" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 mkdir -p "$scratch_run_dir/Data/Intensities/BaseCalls"                 if ! {                     echo "Copying root-level BCL metadata"                     find "$effective_run_dir" -maxdepth 1 -type f -print0 \                       | xargs -0 -r cp -L --sparse=always -t "$scratch_run_dir"                     find "$effective_run_dir/Data/Intensities" -maxdepth 1 -type f -print0 \                       | xargs -0 -r cp -L --sparse=always -t "$scratch_run_dir/Data/Intensities"                     echo "Skipping InterOp during BCLConvert scratch staging; run-QC rules own InterOp parsing"                 } > "$stage_metadata_log" 2>&1; then                     cat "$stage_metadata_log" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log || true                     echo "Mounted metadata staging failed" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 cycle_list="$scratch_sync_log_dir/cycle_dirs.txt"                 find "$lane_root" -mindepth 2 -maxdepth 2 -type d -name 'C*.1' -printf '%P\n' | sort > "$cycle_list"                 cycle_count="$(wc -l < "$cycle_list" | tr -d ' ')"                 echo "mounted_stage_cycle_dirs: $cycle_count" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if [ "$cycle_count" -lt 1 ]; then                     echo "BCL run directory has no L###/C*.1 cycle directories under $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 file_list="$scratch_sync_log_dir/basecall_files.nul"                 find "$lane_root" -mindepth 2 -type f -printf '%P\0' > "$file_list"                 file_count="$(tr -cd "\0" < "$file_list" | wc -c | tr -d " ")"                 echo "mounted_stage_regular_files: $file_count" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if [ "$file_count" -lt 1 ]; then                     echo "BCL run directory has no regular files under $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 echo "Copying mounted BCL files with sharded cp: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 if ! xargs -0 -r -P "$mounted_stage_jobs" -n 64 bash -c '                     set -euo pipefail                     lane_root="$1"                     scratch_run_dir="$2"                     shift 2                     for rel in "$@"; do                         src="$lane_root/$rel"                         dst="$scratch_run_dir/Data/Intensities/BaseCalls/$(dirname "$rel")"                         mkdir -p "$dst"                         cp -L --sparse=always "$src" "$dst/"                     done                 ' _ "$lane_root" "$scratch_run_dir" \                   < "$file_list" > "$stage_files_log" 2>&1; then                     cat "$stage_metadata_log" "$stage_files_log" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log || true                     echo "One or more mounted BCL file copy batches failed" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                     exit 2                 fi                 cat "$stage_metadata_log" "$stage_files_log" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log || true                 echo "Mounted scratch staging complete: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log                 du -sh "$scratch_run_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true                 effective_run_dir="$scratch_run_dir"                 effective_output_dir="$scratch_output_dir"                 df -h "$scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true                 ;;             s3_dev_shm)                 mkdir -p "$scratch_root"                 scratch_dir="$scratch_root/${SLURM_JOB_ID:-local}.$$"                 scratch_run_dir="$scratch_dir/run"                 scratch_output_dir="$scratch_dir/fastqs"                 scratch_sync_log_dir="$scratch_dir/aws_sync_logs"                 mkdir -p "$scratch_run_dir" "$scratch_sync_log_dir"                 source_s3_uri=s3://lsmc-ssf-sequencing-data/basecalls/lsmc/ssf-hq/LH01106/2026/20260514_LH01106_0009_B23TVLGLT4/                 run_region=us-west-2                 run_profile=lsmc                 if [ -z "$source_s3_uri" ]; then                     echo "bclconvert.staging_mode=s3_dev_shm requires SOURCE_S3_URI in config/runs.tsv" >> results/runs/20--output truncated--