DAY-EC activated.
DAY-EC activated.
COMPLETE_CONDA_TOPLEVEL
0b1aa0bf84b641b1fca3480164585362_	406986407	374	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/0b1aa0bf84b641b1fca3480164585362_
299e0fbc02639cada3d27463747d90ee_	352250021	131	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/299e0fbc02639cada3d27463747d90ee_
386784e83dca731ed0bee02ad1e29b2e_	419204461	152	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/386784e83dca731ed0bee02ad1e29b2e_
4e50312f9c958dfca057b2c605426602_	74385318	98	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/4e50312f9c958dfca057b2c605426602_
5054602efe27c1789764b4b6065a11d6_	95202643	102	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/5054602efe27c1789764b4b6065a11d6_
5b93cf1d179903b1bf3ce7b68514bf72_	241579779	188	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/5b93cf1d179903b1bf3ce7b68514bf72_
608d52898d6f0fe50a35015ee2955834_	480606467	139	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/608d52898d6f0fe50a35015ee2955834_
650f152e7cc096c99ec308ec652b338a_	1482574313	185	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/650f152e7cc096c99ec308ec652b338a_
6a98d5a34ca1b5ab4b0a831930b7bb10_	279209686	276	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/6a98d5a34ca1b5ab4b0a831930b7bb10_
6e599cfa2e57aebd626e37eb557a3324_	74385330	77	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/6e599cfa2e57aebd626e37eb557a3324_
70bd9d1b705d48d95c302435f63a874d_	241657550	205	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/70bd9d1b705d48d95c302435f63a874d_
788e729f1e0864dea51355773486d36a_	759895270	143	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/788e729f1e0864dea51355773486d36a_
7b3dbc6b9eae05a003fb9812033f0657_	105257785	158	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/7b3dbc6b9eae05a003fb9812033f0657_
93ebe1238530a7b88d9b0377f9f327b0_	240151874	136	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/93ebe1238530a7b88d9b0377f9f327b0_
953bd0b4f71e6bbda7ceac1c77ba9bcb_	528485571	137	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/953bd0b4f71e6bbda7ceac1c77ba9bcb_
ac1c3ad4a398b8d1d8ea053c637bc6fc_	1002305420	194	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/ac1c3ad4a398b8d1d8ea053c637bc6fc_
cbee441bab3740ee8e092ae8f9cbb3b9_	1205516133	141	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/cbee441bab3740ee8e092ae8f9cbb3b9_
d8a099cbe66aacd85d9653887f51ccba_	1426333317	147	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/d8a099cbe66aacd85d9653887f51ccba_
e41dc845345d49d1be556ec6298452d7_	675947463	133	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/e41dc845345d49d1be556ec6298452d7_
fc10530623cc428bfa51464b358f203e_	241486382	208	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/fc10530623cc428bfa51464b358f203e_
INCOMPLETE_CONDA_TOPLEVEL
YAML_WITHOUT_ENV
133	/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24/477cbca61a540fa9c86ae4110346556e_.yaml
REAL_CONTAINER_IMAGES
NEXTFLOW_TOPLEVEL
d	33280	/fsx/resources/environments/nextflow/24.10.5
d	33280	/fsx/resources/environments/nextflow/java-21
86K	/fsx/resources/environments/nextflow/24.10.5
665M	/fsx/resources/environments/nextflow/java-21
DOT_CONFIG_REFS
FILE=/home/ubuntu/.bashrc
FILE=/home/ubuntu/.bash_profile
FILE=/home/ubuntu/.profile
FILE=/etc/profile.d/01-locale-fix.sh
FILE=/etc/profile.d/Z97-byobu.sh
FILE=/etc/profile.d/Z99-cloud-locale-test.sh
FILE=/etc/profile.d/Z99-cloudinit-warnings.sh
FILE=/etc/profile.d/apps-bin-path.sh
FILE=/etc/profile.d/bash_completion.sh
FILE=/etc/profile.d/cedilla-portuguese.sh
FILE=/etc/profile.d/debuginfod.sh
FILE=/etc/profile.d/gawk.sh
FILE=/etc/profile.d/gnome-session_gnomerc.sh
FILE=/etc/profile.d/im-config_wayland.sh
FILE=/etc/profile.d/modules.sh
FILE=/etc/profile.d/path.sh
FILE=/etc/profile.d/pcluster_awsbatchcli.sh
FILE=/etc/profile.d/slurm.sh
FILE=/etc/profile.d/vte-2.91.sh
FILE=/etc/profile.d/xdg_dirs_desktop_session.sh
FILE=/etc/profile.d/zippy_efa.sh
FILE=/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/dyoainit
346:    export APPTAINER_HOME="/fsx/resources/environments/containers/$DAY_USER/$(hostname)/"
FILE=/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/config.yaml
42:conda-prefix: "/fsx/resources/environments/conda/ubuntu/ip-10-0-0-24"
49:singularity-prefix: "/fsx/resources/environments/containers/ubuntu/ip-10-0-0-24"
APPTAINER_TMP_DOT_REFS
/home/ubuntu/projects/daylily-ephemeral-cluster/daylily_ec/headnode.py:278:            'export APPTAINER_CACHEDIR="${APPTAINER_CACHEDIR:-/fsx/tmp/apptainer_cache/$USER}"',
/home/ubuntu/projects/daylily-ephemeral-cluster/daylily_ec/headnode.py:279:            'export SINGULARITY_CACHEDIR="${SINGULARITY_CACHEDIR:-$APPTAINER_CACHEDIR}"',
/home/ubuntu/projects/daylily-ephemeral-cluster/tests/test_headnode_init.py:200:        'export APPTAINER_CACHEDIR="${APPTAINER_CACHEDIR:-/fsx/tmp/apptainer_cache/$USER}"'
/home/ubuntu/projects/daylily-ephemeral-cluster/tests/test_headnode_init.py:204:        'export SINGULARITY_CACHEDIR="${SINGULARITY_CACHEDIR:-$APPTAINER_CACHEDIR}"' in shell_code
/home/ubuntu/projects/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_logs/gd3v206-rcbase5xr1_status_latest.log:18:  "stdout": "{\n  \"command\": \"mkdir -p /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1 /fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && export APPTAINER_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} SINGULARITY_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} APPTAINER_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 SINGULARITY_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && bin/day_run produce_alignstats produce_na_dedup_cram produce_rochehc_snv_vcf -p -j 5 -k --singularity-prefix /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1\",\n  \"completed_at\": \"2026-05-27T06:46:20Z\",\n  \"exit_code\": 1,\n  \"repo_path\": \"/fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis\",\n  \"session_name\": \"gd3v206-rcbase5xr1\",\n  \"started_at\": \"2026-05-27T06:45:57Z\"\n}\n"
/home/ubuntu/projects/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_logs/gd3v206-rcbase5xr1_launch.log:17:    "mkdir -p /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1 /fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && export APPTAINER_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} SINGULARITY_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} APPTAINER_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 SINGULARITY_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && bin/day_run produce_alignstats produce_na_dedup_cram produce_rochehc_snv_vcf -p -j 5 -k --singularity-prefix /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1",
/home/ubuntu/projects/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_logs/gd3v206-rcbase5xr1_tmux_tail.log:19:  "stdout": "DAY-EC activated.\nDAY-EC activated.\nCloning into '/fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis'...\nNote: switching to 'a073f31c6be04b88cc3f70dcfac77940f54e9282'.\n\nYou are in 'detached HEAD' state. You can look around, make experimental\nchanges and commit them, and you can discard any commits you make in this\nstate without impacting any branches by switching back to a branch.\n\nIf you want to create a new branch to retain commits you create, you may\ndo so (now or later) by using -c with the switch command. Example:\n\n  git switch -c <new-branch-name>\n\nOr undo this operation with:\n\n  git switch -\n\nTurn off this advice by setting config variable advice.detachedHead to false\n\nUpdating files:  53% (671/1248)\nUpdating files:  54% (674/1248)\nUpdating files:  55% (687/1248)\nUpdating files:  56% (699/1248)\nUpdating files:  57% (712/1248)\nUpdating files:  58% (724/1248)\nUpdating files:  59% (737/1248)\nUpdating files:  60% (749/1248)\nUpdating files:  61% (762/1248)\nUpdating files:  62% (774/1248)\nUpdating files:  63% (787/1248)\nUpdating files:  64% (799/1248)\nUpdating files:  65% (812/1248)\nUpdating files:  66% (824/1248)\nUpdating files:  67% (837/1248)\nUpdating files:  68% (849/1248)\nUpdating files:  69% (862/1248)\nUpdating files:  69% (872/1248)\nUpdating files:  70% (874/1248)\nUpdating files:  71% (887/1248)\nUpdating files:  72% (899/1248)\nUpdating files:  73% (912/1248)\nUpdating files:  74% (924/1248)\nUpdating files:  75% (936/1248)\nUpdating files:  76% (949/1248)\nUpdating files:  77% (961/1248)\nUpdating files:  78% (974/1248)\nUpdating files:  79% (986/1248)\nUpdating files:  80% (999/1248)\nUpdating files:  81% (1011/1248)\nUpdating files:  82% (1024/1248)\nUpdating files:  83% (1036/1248)\nUpdating files:  84% (1049/1248)\nUpdating files:  85% (1061/1248)\nUpdating files:  86% (1074/1248)\nUpdating files:  87% (1086/1248)\nUpdating files:  88% (1099/1248)\nUpdating files:  89% (1111/1248)\nUpdating files:  89% (1116/1248)\nUpdating files:  90% (1124/1248)\nUpdating files:  91% (1136/1248)\nUpdating files:  92% (1149/1248)\nUpdating files:  93% (1161/1248)\nUpdating files:  94% (1174/1248)\nUpdating files:  95% (1186/1248)\nUpdating files:  96% (1199/1248)\nUpdating files:  97% (1211/1248)\nUpdating files:  98% (1224/1248)\nUpdating files:  99% (1236/1248)\nUpdating files: 100% (1248/1248)\nUpdating files: 100% (1248/1248), done.\nCloning repository...\n\nGreat success! Daylily repository cloned.\nRepository: https://github.com/Daylily-Informatics/daylily-omics-analysis.git\nReference : 2.0.6\nLocation  : /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis\n\nTo get started:\n  cd /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis\n  # initialize and run the analysis repository per its documentation\nNotice: --project not set. Using default project name: goodole3\nProject: goodole3\nSkip Project Check: true\nSkipping project validation as --skip-project-check was passed.\n\n________________________________________________________\nAWS Budget lookup skipped for project 'goodole3' in region 'us-west-2'.\n  Total: NA\n  Used: NA\n  Percent Used: NA\n________________________________________________________\n\n# conda environments:\n#\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/030e82cada36dc8bacd2afa6880307e1_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/04b7f861fd27114cf3b78bfdefcc7942_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/0e6863af9e1cea1fa92326f2cbe6973d_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/164248b1ae003debe00695d602bcf28b_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/23d00e0a1780b8d3dfb954a2bffa779f_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/37a4bd2a7b34643d7ebfac6afb4aab2e_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/43265f5c58dd3b3e8b73327842668a4e_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/4c86ec1cc18311485ecc74ba9d3cfb03_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/4dfdd5548b48f7841a8bf41e63fdc8b5_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/60e21fed9dc67e8b629ccf9cf7f900ba_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/7b774ba5874429e50f97320a80515483_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/80c345590cfde8caa89fbebc92bcf19e_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/83c55c843541436ad00c6264bb8bea2b_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/94172e327f9dcc43d64f129e406196f0_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/95bfa9ae53e29ee5c7529651691f2ea6_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/c3c78c3dbac8709ca55a60cc4c9e0f7c_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/eaafa830003fa8245187b95e086b40fa_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/ee6a31f731be63cbdfefdb1f52b8d3bc_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/f363d6f385d4bd3350bf63e9f5c6a1cf_\n                       /fsx/resources/environments/conda/ubuntu/ip-10-0-0-144/f8cdc6f8a8452509ca4eb7207f69c23c_\nbase                   /home/ubuntu/miniconda3\nDAY-EC               * /home/ubuntu/miniconda3/envs/DAY-EC\nDAYOA                  /home/ubuntu/miniconda3/envs/DAYOA\n\nDay CLI initialized for project 'goodole3' in region 'us-west-2'.\nThe Daylily CLI is now available.\nAvailable commands (tab completion is enabled for all):\n \n\t(day-activate / dy-a) [slurm|local] ([hg38|hg38_broad|b37])- Activate a Slurm or local environment.\n\t                           \n\t{day-set-genome-build / dy-g) [b37|hg38|hg38_broad] - Set the genome build for the current environment.\n\t                           \n\t(day-run / dy-r)          - Run a command in the current environment.\n\t                           <tab> for exposed targets, -<tab> for all command line flags.\n\t                           \n\t(day-monitor / dy-m)      - Monitor workflow status (Snakemake, SLURM, logs).\n\t                           Use --block-and-poll to wait for completion.\n\t                           \n\t(day-deactivate / dy-d)   - Deactivate the current environment.\n\t                           Use 'dy-d reset' to hard reset the environment.\n\t                           \n\t                           \n\t                           \n\t                           \n\t. To Stage Sample Data, see daylily-ephemeral-cluster docs for running ~/projects/daylily-ephemeral-cluster/bin/daylily-stage-analysis-samples-headnode \n\t  ... which once run, you will copy the samples.tsv and units.tsv files to config/ in this directory.\n\t                           \n\t(example): ACTIVATE AN ANALYSIS ENV\n\n\n\n\n\n\t\tdy-a slurm hg38 # or hg38_broad or b37\n\n\t(example): RUN ANALYSES\n\n\t\tcp .test_data/data/0.01xwgs_HG002_hg38.samples.tsv config/samples.tsv\n\n\t\tcp .test_data/data/0.01xwgs_HG002_hg38.units.tsv config/units.tsv\n\n\n\n\t\t# Use target names directly (tab-complete available):\n\n\t\tdy-r produce_snv_concordances -p -k -j 20 -n   # Illumina short-read SNV concordance\n\n\t\tdy-r produce_alignstats -p -k -j 20 -n         # Alignment statistics\n\n\n\n\t\t# Platform-specific targets:\n\n\t\tdy-r produce_sentdont_vcf -p -k -j 20 -n       # ONT SNV calling\n\n\t\tdy-r produce_sentdpb_vcf -p -k -j 20 -n        # PacBio SNV calling\n\n\t\tdy-r produce_sentdug_vcf -p -k -j 20 -n        # Ultima SNV calling (use hg38_broad)\n\n\n\n\t\t# Hybrid workflow targets:\n\n\t\tdy-r produce_sentdhio_vcf -p -k -j 20 -n       # Hybrid Illumina+ONT CLI\n\n\t\tdy-r produce_sentdhuo_vcf -p -k -j 20 -n       # Hybrid Ultima+ONT CLI (use hg38_broad)\n\n\t\tdy-r produce_sentdhiom_vcf -p -k -j 20 -n      # Hybrid Illumina+ONT Modular\n\n\t\tdy-r produce_sentdhuom_vcf -p -k -j 20 -n      # Hybrid Ultima+ONT Modular (use hg38_broad)\n\n\n\n\t\t# Remove -n to execute (not dry-run)\n\nRequesting profile: slurm\nAttempting to deactivate existing environments & re-init with --project goodole3 .\nRemote call detected. Activating conda hack\n > >> >>> \nACTIVE CONFIG FILES NOT FOUND IN /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm ... copying\nCopying template yaml files to active config files /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm\nCopying template bash files to active config files /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm\nyq is already installed.\nWARNING: sentieon lic file /fsx/references/runtime_assets/cached_envs/x.lic not found.\nDetected sentieon license file: /fsx/references/runtime_assets/cached_envs/Life_Sciences_Manufacturing_Corporation_eval.lic will be used and overrides what is in ~/.config/daylily/daylily_cli_global.yaml\n\n!!!!\n!!!!\n    WARNING: gittag:null not found touched in  ~/.config/daylily/null \n     This might be fine if you have cloned a more recent release of daylily compared to the tagged version used to create this ephemeral cluster.\n     ... however, this is not expected. If you are running an ephemeral cluster headnode for days+, this is not the intended use of daylily, a fresh build might be called for.\n     This is checked by testing if ~/.config/daylily/null exists .\n!!!!\n > >> >>>   genome build set to ::: hg38_broad  \n       _______ GREAT SUCCESS _______       \nRemote call detected. Activating conda hack\nYour config files in /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm are newer than the templates. clear 2 go.\n...AUTO-CONFIG: dedupers=na (from target rules \u2192 env)\n...AUTO-CONFIG: snv_callers=rochehc (from target rules \u2192 env)\nExecuting: snakemake --profile=/fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm produce_alignstats produce_na_dedup_cram produce_rochehc_snv_vcf -p -j 5 -k --singularity-prefix /fsx/tmp/apptainer_cache/ubuntu/ip-10-0-0-144/containers/gd3v206-rcbase5xr1\nloading global: /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/global_AWSPC.yaml\nloading profile rule_config: /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/day_profiles/slurm/rule_config.yaml\nINFO::: The genome build hg38_broad is supported.  The genome build prefix is 'chr''.\n...WARNING: No aligners set in the config.\naligners (final): []\n...INFO: No dedupers set in config. Defaulting to na (no dedup).\n...INFO: Auto-detected dedupers. DDUP updated to: ['na']\ndeduper (final): [na]\n...WARNING: No snv_callers set in the config.\n...INFO: Auto-detected SNV callers from env: ['rochehc']\nSNV Callers (final): [rochehc]\nSomatic SNV Callers:[senttn]\n... WARNING: No sv_callers set in the config.\nSV Callers (final): []\nA    N   A   L  Y S I S    SAMPLE TABLE DETECTED ::: /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/samples.tsv\nA    N   A   L  Y S I S    UNIT TABLE DETECTED ::: /fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis/config/units.tsv\nBuilding DAG of jobs...\nPREP ROCHE BAM:: results/day/hg38_broad//TVBROCHE5X-HG003-5x-0-D0-PF-ROCHE-SBX-DUPLEX/align/roche/ ... \nPulling singularity image docker://roche/sbxd-small-variant-caller:latest.\nWorkflowError:\nFailed to pull singularity image from docker://roche/sbxd-small-variant-caller:latest:\nFATAL:   While making image from oci registry: error fetching image to cache: failed to get checksum for docker://roche/sbxd-small-variant-caller:latest: reading manifest latest in docker.io/roche/sbxd-small-variant-caller: requested access to the resource is denied\n\nRETURN CODE: 1\n[INFO] Workflow exited with status 1\nDAY-EC activated.\n(DAY-EC) ubuntu@ip-10-0-0-144:/fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis$\n"
/home/ubuntu/projects/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_runs.jsonl:211:{"analysis_id": "gd3v206-rcbase5xr1", "command_id": "roche_snv_alignstats_retry1", "exit_code": 1, "phase": "live", "session_name": "gd3v206-rcbase5xr1", "status": "completed", "status_log": "/Users/jmajor/.codex/worktrees/dyec-fsx-dra-mounts/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_logs/gd3v206-rcbase5xr1_status_latest.log", "status_payload": {"command": "mkdir -p /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1 /fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && export APPTAINER_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} SINGULARITY_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} APPTAINER_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 SINGULARITY_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && bin/day_run produce_alignstats produce_na_dedup_cram produce_rochehc_snv_vcf -p -j 5 -k --singularity-prefix /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1", "completed_at": "2026-05-27T06:46:20Z", "exit_code": 1, "repo_path": "/fsx/analysis_results/ubuntu/gd3v206-rcbase5xr1/daylily-omics-analysis", "session_name": "gd3v206-rcbase5xr1", "started_at": "2026-05-27T06:45:57Z"}, "tmux_tail_log": "/Users/jmajor/.codex/worktrees/dyec-fsx-dra-mounts/daylily-ephemeral-cluster/docs/plans/20260527T032100Z_goodole3_serial_kitchensink_v206_logs/gd3v206-rcbase5xr1_tmux_tail.log", "ts": "2026-05-27T06:48:07Z"}
/home/ubuntu/projects/daylily-ephemeral-cluster/docs/dayec-5.0.9-command-catalog-tests.md:557:| `bin/day_run` | `mkdir -p /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1 /fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && export APPTAINER_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} SINGULARITY_CACHEDIR=/fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME} APPTAINER_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 SINGULARITY_TMPDIR=/fsx/scratch/dayoa_apptainer_tmp/ubuntu/gd3v206-rcbase5xr1 && bin/day_run produce_alignstats produce_na_dedup_cram produce_rochehc_snv_vcf -p -j 5 -k --singularity-prefix /fsx/tmp/apptainer_cache/ubuntu/${HOSTNAME}/containers/gd3v206-rcbase5xr1` |
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/templates/profile_env.bash:8:export APPTAINER_CACHEDIR="/fsx/tmp/apptainer_cache/${dayoa_user}/${dayoa_host}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/templates/profile_env.bash:9:export SINGULARITY_CACHEDIR="${APPTAINER_CACHEDIR}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/templates/profile_env.bash:11:mkdir -p "${APPTAINER_TMPDIR}" "${APPTAINER_CACHEDIR}" || return 1
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/profile_env.bash:8:export APPTAINER_CACHEDIR="/fsx/tmp/apptainer_cache/${dayoa_user}/${dayoa_host}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/profile_env.bash:9:export SINGULARITY_CACHEDIR="${APPTAINER_CACHEDIR}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/slurm/profile_env.bash:11:mkdir -p "${APPTAINER_TMPDIR}" "${APPTAINER_CACHEDIR}" || return 1
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/local/templates/profile_env.bash:10:    export APPTAINER_CACHEDIR="${DAYOA_MAC_STATE_DIR}/apptainer_cache/${dayoa_user}/${dayoa_host}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/local/templates/profile_env.bash:13:    export APPTAINER_CACHEDIR="/fsx/tmp/apptainer_cache/${dayoa_user}/${dayoa_host}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/local/templates/profile_env.bash:16:export SINGULARITY_CACHEDIR="${APPTAINER_CACHEDIR}"
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/config/day_profiles/local/templates/profile_env.bash:18:mkdir -p "${APPTAINER_TMPDIR}" "${APPTAINER_CACHEDIR}" || return 1
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/tests/test_multiqc_sample_identifiers.py:1004:            'export APPTAINER_CACHEDIR="/fsx/tmp/apptainer_cache/${dayoa_user}/${dayoa_host}"'
/fsx/analysis_results/dyec5128/init/daylily-omics-analysis/tests/test_multiqc_sample_identifiers.py:1007:        assert 'export SINGULARITY_CACHEDIR="${APPTAINER_CACHEDIR}"' in profile_env
/fsx/analysis_results/dyec5128/init/daylily-omics-a--output truncated--
grep: /etc/profile.d/debuginfod.sh: Permission denied

