188:| 1125 | 2026-06-06T06:21:00Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN live status plus Slurm queue | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01125_live_status_jobs_poll_8.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01125_live_status_jobs_poll_8.stderr.txt | monitoring | no | none | monitoring only |
189:| 1126 | 2026-06-06T06:24:42Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN after ONT retry launch | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01126_live_status_jobs_poll_9.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01126_live_status_jobs_poll_9.stderr.txt | monitoring | no | none | monitoring only |
190:| 1127 | 2026-06-06T06:25:09Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN live markers after ONT retry launch | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 700 \| grep -E 'Building DAG\|Job stats\|Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|Creating conda\|Environment for\|Removing incomplete\|CreateConda' \| tail -n 120; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|Complete\|failed\|Failed\|Creating conda\|Environment for\|Removing incomplete\|CreateConda' \| tail -n 140 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01127_live_log_markers_6.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01127_live_log_markers_6.stderr.txt | monitoring | no | none | monitoring only |
198:| 1136 | 2026-06-06T06:35:34Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN after cache-prefix probe | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01136_live_status_jobs_poll_10.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01136_live_status_jobs_poll_10.stderr.txt | monitoring | no | none | monitoring only |
199:| 1137 | 2026-06-06T06:42:37Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN 5m after cache plan lock | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01137_live_status_jobs_poll_11.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01137_live_status_jobs_poll_11.stderr.txt | monitoring | no | none | monitoring only |
200:| 1138 | 2026-06-06T06:43:04Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN live markers 5m after cache plan lock | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 600 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats' \| tail -n 120; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 600 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats' \| tail -n 120 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01138_live_log_markers_7.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01138_live_log_markers_7.stderr.txt | monitoring | no | none | monitoring only |
201:| 1139 | 2026-06-06T06:49:07Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN status with cache sync gate enforced | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01139_live_status_jobs_poll_12.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01139_live_status_jobs_poll_12.stderr.txt | monitoring | no | none | monitoring only; cache sync blocked until both exit_code=0 |
202:| 1140 | 2026-06-06T06:49:34Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN live markers with cache sync gate enforced | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete' \| tail -n 120; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete' \| tail -n 120 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01140_live_log_markers_8.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01140_live_log_markers_8.stderr.txt | monitoring | no | none | monitoring only; cache sync blocked until both exit_code=0 |
203:| 1141 | 2026-06-06T06:55:34Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN status; cache sync remains gated | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01141_live_status_jobs_poll_13.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01141_live_status_jobs_poll_13.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
204:| 1142 | 2026-06-06T06:56:00Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN live markers; cache sync remains gated | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01142_live_log_markers_9.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01142_live_log_markers_9.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
205:| 1143 | 2026-06-06T07:01:59Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN status; cache sync gated by rc=0 | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01143_live_status_jobs_poll_14.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01143_live_status_jobs_poll_14.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
206:| 1144 | 2026-06-06T07:02:27Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN live markers; cache sync gated by rc=0 | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01144_live_log_markers_10.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01144_live_log_markers_10.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
207:| 1145 | 2026-06-06T07:10:22Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 and ILMN status; no cache sync before dual rc=0 | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01145_live_status_jobs_poll_15.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01145_live_status_jobs_poll_15.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
208:| 1146 | 2026-06-06T07:10:49Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Fetch ONT retry1 and ILMN markers; no cache sync before dual rc=0 | printf '## ONT retry1\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100; printf '## ILMN\n'; AWS_PROFILE=lsmc dyec workflow logs --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z --lines 700 \| grep -E 'Submitted job\|rule \|Error\|ERROR\|WorkflowError\|Finished job\|failed\|Failed\|CreateConda\|DAG of jobs\|Job stats\|Complete\|Nothing to be done' \| tail -n 100 | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01146_live_log_markers_11.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01146_live_log_markers_11.stderr.txt | monitoring | no | none | monitoring only; no cache sync until both exit_code=0 |
209:| 1147 | 2026-06-06T07:12:55Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 + ILMN live snakecontrollers and Slurm queue | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01147_live_status_jobs_poll_12.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01147_live_status_jobs_poll_12.stderr.txt | monitoring | no | none | monitoring only; cache sync blocked until both snakecontrollers rc=0 |
210:| 1149 | 2026-06-06T07:19:06Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | User-requested status and df -h /fsx | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode run-shell --profile lsmc --region us-west-2 --cluster hyb-only --command "df -h /fsx" | 127 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01149_live_status_fsx_poll_user_01.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01149_live_status_fsx_poll_user_01.stderr.txt | monitoring | no | none | status only; no cache sync until both snakecontrollers rc=0 |
212:| 1151 | 2026-06-06T07:19:26Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | User-requested status and df -h /fsx via DAY-EC | AWS_PROFILE=lsmc conda run -n DAY-EC dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc conda run -n DAY-EC dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc conda run -n DAY-EC dyec headnode run-shell --profile lsmc --region us-west-2 --cluster hyb-only --command "df -h /fsx" | 2 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01151_live_status_fsx_poll_user_02.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01151_live_status_fsx_poll_user_02.stderr.txt | monitoring | no | none | status only; no cache sync until both snakecontrollers rc=0 |
216:| 1154 | 2026-06-06T07:21:02Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 + ILMN live snakecontrollers and Slurm queue | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01154_live_status_jobs_poll_13.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01154_live_status_jobs_poll_13.stderr.txt | monitoring | no | none | monitoring only; cache sync blocked until both snakecontrollers rc=0 |
217:| 1156 | 2026-06-06T07:27:19Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Inspect latest Snakemake progress and DAG artifacts for ONT/ILMN controllers | cmd_id=$(AWS_PROFILE=lsmc aws ssm send-command --region us-west-2 --instance-ids i-05374380b57fad901 --document-name AWS-RunShellScript --parameters commands="runuser -u ubuntu -- bash -lc 'set -euo pipefail\nfor id in hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1 hybonly_ilmn_kitchensink_mounted_20260606T053415Z; do\n  repo=/fsx/analysis_results/ubuntu/$id/daylily-omics-analysis\n  echo \"## $id\"\n  echo \"repo=$repo\"\n  if [ ! -d \"$repo\" ]; then echo \"missing_repo=1\"; continue; fi\n  latest=$(ls -1t \"$repo\"/.snakemake/log/*.log 2>/dev/null \| head -1 \|\| true)\n  echo \"latest_log=$latest\"\n  if [ -n \"$latest\" ]; then\n    echo \"progress_lines_last5\"\n    grep -E \"[0-9]+ of [0-9]+ steps\|Finished job\|Submitted job\|Error in rule\|Failed jobs\|Exiting because\|Complete log\" \"$latest\" \| tail -n 80 \|\| true\n    echo \"summary_counts\"\n    python3 - \"$latest\" <<'PY'\nimport re, sys\np=sys.argv[1]\ntext=open(p, errors="replace").read().splitlines()\nprogress=[]\nfinished=0\nsubmitted=0\nerrors=[]\nfailed_jobs=[]\nfor line in text:\n    if "Finished job" in line:\n        finished += 1\n    if "Submitted job" in line:\n        submitted += 1\n    if "Error in rule" in line or "Failed jobs" in line or "Exiting because" in line:\n        errors.append(line)\n    m=re.search(r"(\\d+) of (\\d+) steps \\(([^)]+)\\) done", line)\n    if m:\n        progress.append((int(m.group(1)), int(m.group(2)), m.group(3), line))\nif progress:\n    done,total,pct,line=progress[-1]\nelse:\n    done=total=pct=None\nprint(f"finished_lines={finished}")\nprint(f"submitted_lines={submitted}")\nprint(f"latest_done={done}")\nprint(f"latest_total={total}")\nprint(f"latest_pct={pct}")\nif total is not None and done is not None:\n    print(f"remaining_by_latest_progress={total-done}")\nprint(f"error_marker_count={len(errors)}")\nfor e in errors[-10:]: print("error_marker="+e[:500])\nPY\n  fi\n  echo \"dag_files\"\n  find \"$repo\" -maxdepth 5 -type f \( -name \"*.dag\" -o -name \"*dag*.svg\" -o -name \"*dag*.png\" -o -name \"*dag*.pdf\" -o -name \"dag.svg\" -o -name \"dag.png\" -o -name \"dag.pdf\" \) -printf \"%p\\t%s\\t%TY-%Tm-%TdT%TH:%TM:%TS%TZ\\n\" 2>/dev/null \| sort \|\| true\n  echo\n done'" --query Command.CommandId --output text); echo COMMAND_ID=$cmd_id; AWS_PROFILE=lsmc aws ssm wait command-executed --region us-west-2 --command-id "$cmd_id" --instance-id i-05374380b57fad901; AWS_PROFILE=lsmc aws ssm get-command-invocation --region us-west-2 --command-id "$cmd_id" --instance-id i-05374380b57fad901 --query "{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}" --output json | 1 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01156_snakemake_progress_dag_probe_01.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01156_snakemake_progress_dag_probe_01.stderr.txt | monitoring | no | none | inspection only; no cache sync until both snakecontrollers rc=0 |
221:| 1159 | 2026-06-06T07:29:38Z | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster | local | Poll ONT retry1 + ILMN live snakecontrollers and Slurm queue | AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ont_kitchensink_mounted_20260606T053415Z_retry1; AWS_PROFILE=lsmc dyec --json workflow status --profile lsmc --region us-west-2 --cluster hyb-only --session hybonly_ilmn_kitchensink_mounted_20260606T053415Z; AWS_PROFILE=lsmc dyec headnode jobs --profile lsmc --region us-west-2 --cluster hyb-only | 0 | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01159_live_status_jobs_poll_14.stdout.txt | /Users/jmajor/projects/lsmc/daylily-ephemeral-cluster/docs/plans/20260605T000000Z_hyb_only_mounted_fastq_kitchensink_logs/01159_live_status_jobs_poll_14.stderr.txt | monitoring | no | none | monitoring only; cache sync blocked until both snakecontrollers rc=0 |
