
 RUN  v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk

(node:108917) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M393KT82NQP31ZWBD3QYHA6A step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M393KT82NQP31ZWBD3QYHA6A/llm-1.
(Use `node --trace-warnings ...` to show where the warning was created)
(node:108917) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M393MYTHA4SAQGDXAAPE6GB3 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M393MYTHA4SAQGDXAAPE6GB3/llm-1.
 ❯ tests/authored-probe-cache.test.ts (5 tests | 5 failed) 87390ms
   × authored run CLI probe cache > nine calls have no expired attempts at capacity 1 37510ms
     → 01M393KT82NQP31ZWBD3QYHA6A.sqlite3: expected '[{"at_ms":1790233405701,"attempt":nul…' not to contain 'lease_expired'
   × authored run CLI probe cache > nine calls have no expired attempts at capacity 4 37883ms
     → 01M393MYTHA4SAQGDXAAPE6GB3.sqlite3: expected '[{"at_ms":1790233443157,"attempt":nul…' not to contain 'lease_expired'
   × authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6779ms
     → expected 3439.2573919999995 to be less than 1000
   × authored run CLI probe cache > caches probe failures while refusing all nine calls 3260ms
     → expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9
   × authored run CLI probe cache > shares probe results across agent calls too 1956ms
     → expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3

⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯

 FAIL  tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 1
AssertionError: 01M393KT82NQP31ZWBD3QYHA6A.sqlite3: expected '[{"at_ms":1790233405701,"attempt":nul…' not to contain 'lease_expired'

Expected: "lease_expired"
Received: "[{"at_ms":1790233405701,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-Z6S1j7/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"e6b476909fbe6ea959fb615e3667fe84c99cd7aa8d2840700827b83568adbc31"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790233405704,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790233405704,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","idempotency_key":"9dae83274af952459462b094bfe4bca1cd69287da66506eed1a45f7d9b42a85c","lease_deadline_ms":1790233435704,"lease_id":"01M393KT88MQF0V6DD1DSNCW8W","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790233435715,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790233435715,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30011},"step_spec_hash":"2ddf3aa02deec6eb05ab0ff95bcec6aa5f0ec5658b1554a47e119abaecbaf4d2","verification":null},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790233435715,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M393MQJ3SS1WEBDR0W44D1DS","wake_at_ms":1790233435715},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790233435724,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M393MQJ3SS1WEBDR0W44D1DS"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790233435726,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","idempotency_key":"9dae83274af952459462b094bfe4bca1cd69287da66506eed1a45f7d9b42a85c","lease_deadline_ms":1790233465726,"lease_id":"01M393MQJEHDMZF830TVHDR9Z4","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790233438844,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3118},"step_spec_hash":"2ddf3aa02deec6eb05ab0ff95bcec6aa5f0ec5658b1554a47e119abaecbaf4d2","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790233438851,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":9,"step_id":null}]"

 ❯ tests/authored-probe-cache.test.ts:82:49
     80|     for (const file of runs) {
     81|       const { entries } = await client.journalRead(file.slice(0, -8), …
     82|       expect(JSON.stringify(entries), file).not.toContain('lease_expir…
       |                                                 ^
     83|       expect(entries.filter(e => typeof e === 'object' && e !== null
     84|         && 'entry_type' in e && e.entry_type === 'step.attempt.started…

⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯

 FAIL  tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 4
AssertionError: 01M393MYTHA4SAQGDXAAPE6GB3.sqlite3: expected '[{"at_ms":1790233443157,"attempt":nul…' not to contain 'lease_expired'

Expected: "lease_expired"
Received: "[{"at_ms":1790233443157,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-L5o19j/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"825456894ebae30b76964e8f2be4e44800229df3a5fe9d97330e711667cd9942"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790233443160,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790233443160,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","idempotency_key":"30c5fbbfba3a60e60d4b2c2f1a4575e682bcc5338855708042d5e211522c65df","lease_deadline_ms":1790233473160,"lease_id":"01M393MYTRA9HDE33PGP3FVPR8","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790233473223,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790233473223,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30063},"step_spec_hash":"99f64b0a179f1cbbe2c9235c1f0e7130481a1ac4536fd9f607d735535a6d8772","verification":null},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790233473223,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M393NW67TMY3ETXH3V05BJ2X","wake_at_ms":1790233473223},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790233473234,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M393NW67TMY3ETXH3V05BJ2X"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790233473235,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","idempotency_key":"30c5fbbfba3a60e60d4b2c2f1a4575e682bcc5338855708042d5e211522c65df","lease_deadline_ms":1790233503235,"lease_id":"01M393NW6KT69VKBMWWEA8CBJ2","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790233476384,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3149},"step_spec_hash":"99f64b0a179f1cbbe2c9235c1f0e7130481a1ac4536fd9f607d735535a6d8772","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790233476387,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":9,"step_id":null}]"

 ❯ tests/authored-probe-cache.test.ts:82:49
     80|     for (const file of runs) {
     81|       const { entries } = await client.journalRead(file.slice(0, -8), …
     82|       expect(JSON.stringify(entries), file).not.toContain('lease_expir…
       |                                                 ^
     83|       expect(entries.filter(e => typeof e === 'object' && e !== null
     84|         && 'entry_type' in e && e.entry_type === 'step.attempt.started…

⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯

 FAIL  tests/authored-probe-cache.test.ts > authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run
AssertionError: expected 3439.2573919999995 to be less than 1000
 ❯ tests/authored-probe-cache.test.ts:104:41
    102|     // F1 permits one synchronous probe (~300ms), not nine (~3s).
    103|     // Leave process-startup headroom for loaded CI; exact counts belo…
    104|     expect(starts.at(-1)! - starts[0]!).toBeLessThan(1_000);
       |                                         ^
    105|     expectOneProbe(logs());
    106|     await executeAuthoredFlow(nine, client, undefined, options);

⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/5]⎯

 FAIL  tests/authored-probe-cache.test.ts > authored run CLI probe cache > caches probe failures while refusing all nine calls
AssertionError: expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9

- Expected
+ Received

- 1
+ 9

 ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47
     68| 
     69| function expectOneProbe(logs: ReturnType<Awaited<ReturnType<typeof slo…
     70|   expect(logs.filter(l => l.kind === 'auth')).toHaveLength(1);
       |                                               ^
     71|   expect(logs.filter(l => l.kind === 'identify').length - logs.filter(…
     72| }
 ❯ tests/authored-probe-cache.test.ts:123:5

⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/5]⎯

 FAIL  tests/authored-probe-cache.test.ts > authored run CLI probe cache > shares probe results across agent calls too
AssertionError: expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3

- Expected
+ Received

- 1
+ 3

 ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47
     68| 
     69| function expectOneProbe(logs: ReturnType<Awaited<ReturnType<typeof slo…
     70|   expect(logs.filter(l => l.kind === 'auth')).toHaveLength(1);
       |                                               ^
     71|   expect(logs.filter(l => l.kind === 'identify').length - logs.filter(…
     72| }
 ❯ tests/authored-probe-cache.test.ts:136:5

⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/5]⎯

 Test Files  1 failed (1)
      Tests  5 failed (5)
   Start at  07:03:19
   Duration  89.41s (transform 1.12s, setup 0ms, collect 1.84s, tests 87.39s, environment 0ms, prepare 42ms)

