command_id=cce72f16-a3be-4879-8dc4-5e9fb1de705e
DAY-EC activated.
DAY-EC activated.
__DATE__=2026-06-03T02:55:37+00:00
__JOB_ID__=1
__SQUEUE__
             JOBID    PARTITION                     NAME     USER ST  NODES               NODELIST(REASON)
                 1       bcl2fq    bcl2fq_scratch_to_dra   ubuntu PD      1                    (BeginTime)
__JOB_SUMMARY__
JobId=1 JobName=bcl2fq_scratch_to_dra
   JobState=PENDING Reason=BeginTime Dependency=(null)
   RunTime=00:00:00 TimeLimit=365-00:00:00 TimeMin=N/A
   SubmitTime=2026-06-03T02:55:18 EligibleTime=2026-06-03T02:56:29
   StartTime=2026-06-03T02:56:29 EndTime=2027-06-03T02:56:29 Deadline=N/A
   Partition=bcl2fq AllocNode:Sid=ip-10-0-0-235:16061
   ReqNodeList=(null) ExcNodeList=(null)
   NodeList=
   NumNodes=1-1 NumCPUs=48 NumTasks=1 CPUs/Task=48 ReqB:S:C:T=0:0:*:*
   Features=nvme48 DelayBoot=00:00:00
   Comment=unknown-project-invalid-project 
   StdErr=/home/ubuntu/daylily-runs/bcl2fq_scratch_to_dra_20260603T011121Z/slurm-1.err
   StdOut=/home/ubuntu/daylily-runs/bcl2fq_scratch_to_dra_20260603T011121Z/slurm-1.out
__BCL2FQ_NODES__
NODELIST STATE CPUS MEMORY AVAIL_FEATURES REASON
bcl2fq-dy-nvme48-1 down# 48 373555 dynamic,nvme48 (Code:InsufficientInstanceCapacity)Failure when resuming nodes
bcl2fq-dy-nvme48-2 idle~ 48 373555 dynamic,nvme48 Enabling node since insufficient capacity timeout expired
bcl2fq-dy-nvme48-3 idle~ 48 373555 dynamic,nvme48 Enabling node since insufficient capacity timeout expired
bcl2fq-dy-nvme48-4 idle~ 48 373555 dynamic,nvme48 Enabling node since insufficient capacity timeout expired
bcl2fq-dy-nvme64-1 idle~ 64 498073 dynamic,i4i.16xlarge,nvme64 none
bcl2fq-dy-nvme64-2 idle~ 64 498073 dynamic,i4i.16xlarge,nvme64 none
bcl2fq-dy-nvme64-3 idle~ 64 498073 dynamic,i4i.16xlarge,nvme64 none
bcl2fq-dy-nvme64-4 idle~ 64 498073 dynamic,i4i.16xlarge,nvme64 none
bcl2fq-dy-nvme96-1 idle~ 96 747110 dynamic,nvme96 none
bcl2fq-dy-nvme96-2 idle~ 96 747110 dynamic,nvme96 none
bcl2fq-dy-nvme96-3 idle~ 96 747110 dynamic,nvme96 none
bcl2fq-dy-nvme96-4 idle~ 96 747110 dynamic,nvme96 none
bcl2fq-dy-nvme128-1 idle~ 128 996147 dynamic,i4i.32xlarge,nvme128 none
bcl2fq-dy-nvme128-2 idle~ 128 996147 dynamic,i4i.32xlarge,nvme128 none
bcl2fq-dy-nvme128-3 idle~ 128 996147 dynamic,i4i.32xlarge,nvme128 none
bcl2fq-dy-nvme128-4 idle~ 128 996147 dynamic,i4i.32xlarge,nvme128 none
__NODE_48__
NodeName=bcl2fq-dy-nvme48-1 CoresPerSocket=1 
   AvailableFeatures=dynamic,nvme48
   State=DOWN+CLOUD+NOT_RESPONDING+POWERING_UP ThreadsPerCore=1 TmpDisk=0 Weight=1000 Owner=N/A MCS_label=N/A
   BootTime=None SlurmdStartTime=None
   CfgTRES=cpu=48,mem=373555M,billing=48
   Reason=(Code:InsufficientInstanceCapacity)Failure when resuming nodes [root@2026-06-03T02:55:18]
__TMUX_CAPTURE__
da-us-west-2d-againokx  ubuntu,
da-us-west-2d-jem-07355 ubuntu,
da-us-west-2d-dayoa-07357       ubuntu,
da-us-west-2d-dayoa-07357x      ubuntu,ubuntu
da-us-west-2d-xxaabb    ubuntu,
da-us-west-2d-xxaabb2   ubuntu,ubuntu
da-us-west-2d-dyoa-usw2d-07375  ubuntu,
da-us-west-2d-dyoa-bigboy       ubuntu,
da-us-west-2d-dyoa-smn12-go     ubuntu,
da-us-west-2d-giab-test ubuntu,
da-us-west-2d-ifx-tests ubuntu,
da-us-west-2d-lfg-usw2d ubuntu,
da-us-west-2d-jem-split-test    ubuntu,
da-us-west-2d-jemtest2  ubuntu,
da-us-west-2d-jem-usw2-3        ubuntu,
da-us-west-2d-uswest2d-A        ubuntu,
da-us-west-2d-usw2d-B   ubuntu,
da-us-west-2d-cher-usw2 ubuntu,
da-us-west-2d-tearit-usw2       ubuntu,
da-us-west-2d-gogogo    ubuntu,
da-us-west-2d-josh-test-dev     ubuntu,
da-us-west-2d-agbt-usw2d-primary        ubuntu,
da-us-west-2d-agbt--wrkhrse-usw2        ubuntu,
da-us-west-2d-agbt--12-usw2d    ubuntu,
da-us-west-2d-agbt-12t-usw2d    ubuntu,
da-us-west-2d-day-test-uswest2d ubuntu,daylily-service
da-us-west-2d-day-forge-usw2d   ubuntu,root
da-us-west-2d-agbt-heavy        ubuntu,
da-us-west-2d-daylily-1773059211        ubuntu,root
da-us-west-2d-ifx-go    ubuntu,daylily-service
da-us-west-2d-and-again ubuntu,ubuntu
da-us-west-2d-jemx2     ubuntu,root
da-us-west-2d-y260609-frz       ubuntu
da-us-west-2d-lsbio-fork-260509-124701  ubuntu
da-us-west-2d-fk-260509-use     ubuntu
da-us-west-2d-pilot-mk  ubuntu
da-us-west-2d-pilot-largecluster        ubuntu
da-us-west-2d-may26-d   ubuntu
da-us-west-2d-dra-enabled       ubuntu
da-us-west-2d-XLpilot   ubuntu
da-us-west-2d-XL-pilot  ubuntu
da-us-west-2d-tstVer4-1-1       ubuntu
da-us-west-2d-fork-fixer        ubuntu
da-us-west-2d-tstClu-4-1-1c     ubuntu
da-us-west-2d-splitdra-ref-20260526     ubuntu
da-us-west-2d-jem-bucktst3      ubuntu
da-us-west-2d-blahab44  ubuntu
da-us-west-2d-goodole3  ubuntu
da-us-west-2d-dyec-test ubuntu
da-us-west-2d-dyec5115  ubuntu
da-us-west-2d-dyec5117  ubuntu
da-us-west-2d-dyec0602bcl       ubuntu

you may modify this script to fully block unknown projects, for now this will ta
g with the specified project and proceed.
a log of warnings/errors will be written to ./sbatch_errs.log
setting project=unknown-project
1
             JOBID    PARTITION                     NAME     USER ST       TIME
 NODES NODELIST(REASON)
                 1       bcl2fq    bcl2fq_scratch_to_dra   ubuntu PD       0:00
     1 (None)
(DAY-EC) ubuntu@ip-10-0-0-235:~/daylily-runs/bcl2fq_scratch_to_dra_20260603T0111
21Z$
__LOG_TAIL__
--- /home/ubuntu/daylily-runs/bcl2fq_scratch_to_dra_20260603T011121Z/sbatch_errs.log ---
da-us-west-2d-taketen	ubuntu, 
da-us-west-2d-takeeleven	ubuntu, 
da-us-west-2d-taketwelve	ubuntu, 
da-us-west-2d-take14	ubuntu, 
da-us-west-2d-take15	ubuntu, 
da-us-west-2d-take20	ubuntu, 
da-us-west-2d-jem-delme	ubuntu, 
da-us-west-2d-jem-delme	ubuntu, 
da-us-west-2d-jem-again	ubuntu, 
da-us-west-2d-jem-threee	ubuntu, 
da-us-west-2d-jem-fourx	ubuntu, 
da-us-west-2d-jem-andagain	ubuntu, 
da-us-west-2d-jem-againx	ubuntu, 
da-us-west-2d-jem-aaa	ubuntu, 
da-us-west-2d-jem-okgo	ubuntu, 
da-us-west-2d-bhfghdfgh	ubuntu, 
da-us-west-2d-jkj	ubuntu, 
da-us-west-2d-dfghfghfghfg	ubuntu, 
da-us-west-2d-blahfdfg	ubuntu, 
da-us-west-2d-yoyoaa	ubuntu, 
da-us-west-2d-kkkl	ubuntu, 
da-us-west-2d-majors-cluster	ubuntu, 
da-us-west-2d-ghjghjghj	ubuntu, 
da-us-west-2d-again	ubuntu, 
da-us-west-2d-branchtest	ubuntu, 
da-us-west-2d-jem-07348	ubuntu, 
da-us-west-2d-againokx	ubuntu, 
da-us-west-2d-jem-07355	ubuntu, 
da-us-west-2d-dayoa-07357	ubuntu, 
da-us-west-2d-dayoa-07357x	ubuntu,ubuntu 
da-us-west-2d-xxaabb	ubuntu, 
da-us-west-2d-xxaabb2	ubuntu,ubuntu 
da-us-west-2d-dyoa-usw2d-07375	ubuntu, 
da-us-west-2d-dyoa-bigboy	ubuntu, 
da-us-west-2d-dyoa-smn12-go	ubuntu, 
da-us-west-2d-giab-test	ubuntu, 
da-us-west-2d-ifx-tests	ubuntu, 
da-us-west-2d-lfg-usw2d	ubuntu, 
da-us-west-2d-jem-split-test	ubuntu, 
da-us-west-2d-jemtest2	ubuntu, 
da-us-west-2d-jem-usw2-3	ubuntu, 
da-us-west-2d-uswest2d-A	ubuntu, 
da-us-west-2d-usw2d-B	ubuntu, 
da-us-west-2d-cher-usw2	ubuntu, 
da-us-west-2d-tearit-usw2	ubuntu, 
da-us-west-2d-gogogo	ubuntu, 
da-us-west-2d-josh-test-dev	ubuntu, 
da-us-west-2d-agbt-usw2d-primary	ubuntu, 
da-us-west-2d-agbt--wrkhrse-usw2	ubuntu, 
da-us-west-2d-agbt--12-usw2d	ubuntu, 
da-us-west-2d-agbt-12t-usw2d	ubuntu, 
da-us-west-2d-day-test-uswest2d	ubuntu,daylily-service
da-us-west-2d-day-forge-usw2d	ubuntu,root
da-us-west-2d-agbt-heavy	ubuntu, 
da-us-west-2d-daylily-1773059211	ubuntu,root
da-us-west-2d-ifx-go	ubuntu,daylily-service
da-us-west-2d-and-again	ubuntu,ubuntu
da-us-west-2d-jemx2	ubuntu,root
da-us-west-2d-y260609-frz	ubuntu
da-us-west-2d-lsbio-fork-260509-124701	ubuntu
da-us-west-2d-fk-260509-use	ubuntu
da-us-west-2d-pilot-mk	ubuntu
da-us-west-2d-pilot-largecluster	ubuntu
da-us-west-2d-may26-d	ubuntu
da-us-west-2d-dra-enabled	ubuntu
da-us-west-2d-XLpilot	ubuntu
da-us-west-2d-XL-pilot	ubuntu
da-us-west-2d-tstVer4-1-1	ubuntu
da-us-west-2d-fork-fixer	ubuntu
da-us-west-2d-tstClu-4-1-1c	ubuntu
da-us-west-2d-splitdra-ref-20260526	ubuntu
da-us-west-2d-jem-bucktst3	ubuntu
da-us-west-2d-blahab44	ubuntu
da-us-west-2d-goodole3	ubuntu
da-us-west-2d-dyec-test	ubuntu
da-us-west-2d-dyec5115	ubuntu
da-us-west-2d-dyec5117	ubuntu
da-us-west-2d-dyec0602bcl	ubuntu

Proceeding with the tag: unknown-project-invalid-project  ...continuing in 3 seconds.
__DEST_COUNT__
0
