DAY-EC activated.
DAY-EC activated.
                    echo "One or more mounted BCL file copy batches failed" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                cat "$stage_metadata_log" "$stage_files_log" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log || true
                echo "Mounted scratch staging complete: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                du -sh "$scratch_run_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true
                effective_run_dir="$scratch_run_dir"
                effective_output_dir="$scratch_output_dir"
                df -h "$scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true
                ;;
            s3_dev_shm)
                mkdir -p "$scratch_root"
                scratch_dir="$scratch_root/${SLURM_JOB_ID:-local}.$$"
                scratch_run_dir="$scratch_dir/run"
                scratch_output_dir="$scratch_dir/fastqs"
                scratch_sync_log_dir="$scratch_dir/aws_sync_logs"
                mkdir -p "$scratch_run_dir" "$scratch_sync_log_dir"
                source_s3_uri=s3://lsmc-ssf-sequencing-data/basecalls/lsmc/ssf-hq/LH01106/2026/20260514_LH01106_0009_B23TVLGLT4/
                run_region=us-west-2
                run_profile=lsmc
                if [ -z "$source_s3_uri" ]; then
                    echo "bclconvert.staging_mode=s3_dev_shm requires SOURCE_S3_URI in config/runs.tsv" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                if [ -z "$run_region" ]; then
                    echo "bclconvert.staging_mode=s3_dev_shm requires REGION in config/runs.tsv" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                command -v aws >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1
                echo "s3_stage_submit_profile: $run_profile" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "s3_stage_credential_mode: compute_instance_role" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                AWS_REGION="$run_region" AWS_DEFAULT_REGION="$run_region" AWS_MAX_ATTEMPTS=10 AWS_RETRY_MODE=adaptive \
                  aws sts get-caller-identity >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1
                input_disk_bytes="$(du -sB1 "$effective_run_dir" | awk '{print $1}')"
                input_apparent_bytes="$(du -sb "$effective_run_dir" | awk '{print $1}')"
                required_bytes="$((input_disk_bytes * 4 + 1073741824))"
                scratch_parent="$(dirname "$scratch_root")"
                available_bytes="$(df -PB1 "$scratch_parent" | awk 'NR == 2 {print $4}')"
                echo "scratch_dir: $scratch_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "scratch_input_disk_bytes: $input_disk_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "scratch_input_apparent_bytes: $input_apparent_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "scratch_required_bytes: $required_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "scratch_available_bytes: $available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                if [ "$available_bytes" -lt "$required_bytes" ]; then
                    echo "Insufficient scratch for bclconvert.staging_mode=s3_dev_shm: required=$required_bytes available=$available_bytes" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                run_uri=$(printf "%s" "$source_s3_uri" | sed 's:/*$::')
                lane_root="$effective_run_dir/Data/Intensities/BaseCalls"
                if [ ! -d "$lane_root" ]; then
                    echo "BCL run directory is missing lane root: $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                lane_ids=$(find "$lane_root" -mindepth 1 -maxdepth 1 -type d -name 'L[0-9][0-9][0-9]' -printf '%f\n' | sort)
                if [ -z "$lane_ids" ]; then
                    echo "BCL run directory has no L### lane directories under $lane_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                echo "Staging BCL run directory from S3 to scratch: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                echo "s3_stage_lanes: $(printf "%s" "$lane_ids" | tr '\n' ' ')" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                if ! AWS_REGION="$run_region" AWS_DEFAULT_REGION="$run_region" AWS_MAX_ATTEMPTS=10 AWS_RETRY_MODE=adaptive \
                  aws s3 sync "$run_uri/" "$scratch_run_dir/" \
                    --exclude "Analysis/*" \
                    --exclude "Data/Intensities/BaseCalls/L*/*" \
                    --only-show-errors \
                    > "$scratch_sync_log_dir/root.log" 2>&1; then
                    cat "$scratch_sync_log_dir/root.log" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    echo "Root-level aws s3 sync failed" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                pids=()
                for lane_id in $lane_ids; do
                    (
                        AWS_REGION="$run_region" AWS_DEFAULT_REGION="$run_region" AWS_MAX_ATTEMPTS=10 AWS_RETRY_MODE=adaptive \
                          aws s3 sync "$run_uri/Data/Intensities/BaseCalls/$lane_id/" "$scratch_run_dir/Data/Intensities/BaseCalls/$lane_id/" \
                            --only-show-errors
                    ) > "$scratch_sync_log_dir/$lane_id.log" 2>&1 &
                    pids+=("$!")
                done
                sync_rc=0
                for pid in "${pids[@]}"; do
                    if ! wait "$pid"; then
                        sync_rc=1
                    fi
                done
                cat "$scratch_sync_log_dir"/*.log >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                if [ "$sync_rc" -ne 0 ]; then
                    echo "One or more lane-level aws s3 sync processes failed" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                    exit 2
                fi
                echo "S3 scratch staging complete: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                du -sh "$scratch_run_dir" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true
                effective_run_dir="$scratch_run_dir"
                effective_output_dir="$scratch_output_dir"
                df -h "$scratch_root" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true
                ;;
            *)
                echo "Unsupported bclconvert.staging_mode: $staging_mode" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
                exit 2
                ;;
        esac

        parallel_tiles=8
        conversion_threads=8
        compression_threads=12
        decompression_threads=4
        per_tile_threads="$((conversion_threads + compression_threads + decompression_threads))"
        if [ "$per_tile_threads" -lt 1 ]; then
            echo "BCLConvert per-tile thread total must be >= 1" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
            exit 2
        fi
        max_parallel_tiles="$((192 / per_tile_threads))"
        if [ "$max_parallel_tiles" -lt 1 ]; then
            echo "BCLConvert thread allocation is too small: threads=192 per_tile_threads=$per_tile_threads" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
            exit 2
        fi
        if [ "$parallel_tiles" -gt "$max_parallel_tiles" ]; then
            echo "Reducing BCLConvert parallel tiles from $parallel_tiles to $max_parallel_tiles for threads=192" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
            parallel_tiles="$max_parallel_tiles"
        fi
        echo "bcl_num_parallel_tiles: $parallel_tiles" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        echo "bcl_num_conversion_threads: $conversion_threads" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        echo "bcl_num_compression_threads: $compression_threads" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        echo "bcl_num_decompression_threads: $decompression_threads" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log

        bcl_flags=(
          --bcl-input-directory "$effective_run_dir"
          --output-directory "$effective_output_dir"
          --sample-sheet results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/normalized.SampleSheet.csv
          --strict-mode false
          --first-tile-only false
          --bcl-sampleproject-subdirectories false
          --fastq-gzip-compression-level 1
          --bcl-num-parallel-tiles "$parallel_tiles"
          --bcl-num-conversion-threads "$conversion_threads"
          --bcl-num-compression-threads "$compression_threads"
          --bcl-num-decompression-threads "$decompression_threads"
          --shared-thread-odirect-output false
        )
        force_arg=
        if [ -n "$force_arg" ]; then
            bcl_flags+=("$force_arg")
        fi

        printf 'bcl-convert command:' >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        printf ' %q' singularity exec docker://nfcore/bclconvert:4.0.3 bcl-convert "${bcl_flags[@]}" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        printf '\n' >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        singularity exec docker://nfcore/bclconvert:4.0.3 bcl-convert "${bcl_flags[@]}" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1

        if [ "$staging_mode" = "dev_shm" ] || [ "$staging_mode" = "mounted_dev_shm" ] || [ "$staging_mode" = "s3_dev_shm" ] || [ "$staging_mode" = "output_dev_shm" ]; then
            echo "Copying BCLConvert outputs from scratch to result tree: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
            cp -a "$effective_output_dir"/. results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs/
            df -h "$scratch_root" results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log 2>&1 || true
        fi
        test -s results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs/Reports/fastq_list.csv
        test -s results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/fastqs/Reports/Demultiplex_Stats.csv
        echo "run_bclconvert finished: $(date -Is)" >> results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/run_bclconvert.log
        touch results/runs/20260514_LH01106_0009_B23TVLGLT4/bclconvert/logs/bclconvert.done
        
Submitted job 8 with external jobid '2650'.
