diff --git a/packages/codework/test/__artifacts__/recovery.backoff.json b/packages/codework/test/__artifacts__/recovery.backoff.json index 724eb59e..f18e034a 100644 --- a/packages/codework/test/__artifacts__/recovery.backoff.json +++ b/packages/codework/test/__artifacts__/recovery.backoff.json @@ -37,10 +37,10 @@ { "seq": 11, "type": "session.retry.scheduled", - "attempt": 1, + "attempt": 2, "maxRetries": 3, "delayMs": 60000, - "message": "The host stopped while waiting to retry, before attempt 2" + "message": "The host stopped before attempt 2 was sent" }, { "seq": 12, @@ -74,7 +74,7 @@ "live": [ { "type": "session.retry.finished", - "attempt": 1, + "attempt": 2, "success": true } ], @@ -87,12 +87,12 @@ { "step": "resumed", "restarts": 0, - "attempt": 2 + "attempt": 3 }, { "step": "stopped", "restarts": 0, - "attempt": 2 + "attempt": 3 }, { "step": "finished", diff --git a/packages/codework/test/recovery.test.ts b/packages/codework/test/recovery.test.ts index 3e7b7c1d..d1f79cfe 100644 --- a/packages/codework/test/recovery.test.ts +++ b/packages/codework/test/recovery.test.ts @@ -1024,7 +1024,8 @@ describe("recovery after the host is killed", () => { record("backoff"); await first.kill(); - // The resume waits out the carried retry again: a kill in backoff counted no restart. + // The resume counts the attempt it slept for as failed and waits out the next: a kill in backoff + // counted no restart, the retry budget bounds it instead. const second = await start(); await until("the resumed run", async () => (await log(second.url, ids)).find( @@ -1047,8 +1048,8 @@ describe("recovery after the host is killed", () => { record("finished"); expect(steps.map((step) => step.restarts)).toEqual([0, 0, 0, 0]); - // The attempt the backoff waits for, past the one that failed: the shutdown keeps it, and it is sent next. - expect(steps.find((step) => step.step === "stopped")?.attempt).toBe(2); + // The attempt the backoff waits for, past the two that failed: the shutdown keeps it, and it is sent next. + expect(steps.find((step) => step.step === "stopped")?.attempt).toBe(3); await expect( artifact({ log: events diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index d878cb8b..ab09192e 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -616,8 +616,8 @@ export const layer = (options: Options = {}) => }) { // Only a host that stopped leaves an attempt marked: every run that ends otherwise clears it. It is // the request in flight, or the one a retry backoff waits for. A shutdown failed nothing: that - // request goes again as itself. A crash with the request streaming (its draft open) failed that - // attempt; one in backoff failed only the attempts before it. + // request goes again as itself. A crash failed that attempt, streaming or not: with no draft open + // the restart went uncounted, so the retry budget is what bounds a host that keeps dying there. const stored = Option.getOrUndefined( Option.flatMap(yield* threads.get(input.threadId), (row) => row.attempt), ); @@ -627,17 +627,12 @@ export const layer = (options: Options = {}) => ? undefined : input.shutdown === true ? { retries: stored - 1 } - : streaming - ? { - retries: stored, - failed: `The host stopped while the model was answering, on attempt ${stored}`, - } - : stored === 1 - ? { retries: 0 } - : { - retries: stored - 1, - failed: `The host stopped while waiting to retry, before attempt ${stored}`, - }; + : { + retries: stored, + failed: streaming + ? `The host stopped while the model was answering, on attempt ${stored}` + : `The host stopped before attempt ${stored} was sent`, + }; yield* healDraft(input.threadId, carried?.failed); const hasSteer = yield* inputs.hasPending(input.threadId, "steer"); diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index d25cb1e8..fcfc6fb4 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -963,9 +963,10 @@ export const layer = Layer.effect( /* * Over a `thread` row: its run sleeps in retry backoff, its attempt set and no draft open (the failed * attempt's draft was aborted before the sleep). A crash there is no restart: a receive count, as SQS - * keeps one per message, counts only deliveries being worked on, and a sleeping one is not; its - * carried attempt bounds it instead. The moment between setting the attempt and opening its draft - * reads as asleep too, bounded the same way. + * keeps one per message, counts only deliveries being worked on, and a sleeping one is not. The + * resume counts the attempt it slept for as failed, so the retry budget bounds a host that keeps dying + * there. The moment between setting the attempt and opening its draft reads as asleep too, bounded + * the same way. */ const asleepOf = sql`(thread.attempt IS NOT NULL AND NOT EXISTS ( SELECT 1 FROM entry WHERE entry.thread_id = thread.id AND entry.state = 'draft'