fix(harness): harden durable runs after the head/tail review

packages/codework/src/acp/agent.ts

@@ -5,10 +5,12 @@ import { Cause, Deferred, Effect, Exit, Option, Schema, type Scope, type Stdio,
 export class RequestError extends Schema.TaggedError<RequestError>()("AcpRequestError", {
 	code: Schema.Int,
 	message: Schema.String,
+	data: Schema.optionalKey(Schema.Json),
 }) {
 	static invalidRequest = (message: string) => new RequestError({ code: -32600, message });
 	static invalidParams = (message: string) => new RequestError({ code: -32602, message });
-	static internalError = (message: string) => new RequestError({ code: -32603, message });
+	static internalError = (message: string, data?: Schema.Json) =>
+		new RequestError({ code: -32603, message, ...(data === undefined ? {} : { data }) });
 	static resourceNotFound = (message: string) => new RequestError({ code: -32002, message });
 }
 
@@ -60,7 +62,7 @@ export const serve = Effect.fn("ACP.serve")(function* (
 	// A JSON-RPC answer for a handler that didn't succeed; the SDK sends what `handle` throws.
 	const toWire = (cause: Cause.Cause<RequestError>): Acp.RequestError => {
 		const error = Cause.findErrorOption(cause);
-		if (Option.isSome(error)) return new Acp.RequestError(error.value.code, error.value.message);
+		if (Option.isSome(error)) return new Acp.RequestError(error.value.code, error.value.message, error.value.data);
 		if (Cause.hasInterruptsOnly(cause)) return Acp.RequestError.requestCancelled();
 		return Acp.RequestError.internalError(undefined, String(Cause.squash(cause)));
 	};

packages/codework/src/acp/handlers.ts

@@ -2,7 +2,6 @@ import type * as Acp from "@agentclientprotocol/sdk";
 import {
 	Control,
 	EventList,
-	type EventSchema,
 	type Harness,
 	PromptSchema,
 	Session,
@@ -22,10 +21,9 @@ const isNotFound = Schema.is(SessionStore.SessionNotFoundError);
 const isConflict = Schema.is(Control.PromptConflictError);
 const isInvalidPrompt = Schema.is(PromptSchema.InvalidPromptError);
 const isLLMEnded = Schema.is(EventList.LLMEnded);
-const isSucceeded = Schema.is(EventList.ExecutionSucceeded);
 const isFailed = Schema.is(EventList.ExecutionFailed);
 const isInterrupted = Schema.is(EventList.ExecutionInterrupted);
-const isSettled = (event: EventSchema.Payload) => isSucceeded(event) || isFailed(event) || isInterrupted(event);
+const isRunFailed = Schema.is(EventList.RunFailed);
 
 /** Harness failures as JSON-RPC errors; handlers raise protocol-level ones directly. */
 const toRequestError = (error: { readonly message: string }): Agent.RequestError => {
@@ -271,7 +269,7 @@ export const make = Effect.gen(function* () {
 									onSome: (update) => client.sessionUpdate({ sessionId: id, update }).pipe(Effect.ignore),
 								});
 							}),
-							Stream.filter(isSettled),
+							Stream.filter(EventList.isRunEnded),
 							Stream.runHead,
 							Effect.forkScoped({ startImmediately: true }),
 						);
@@ -289,6 +287,10 @@ export const make = Effect.gen(function* () {
 
 						if (Option.isNone(end) || isInterrupted(end.value)) return { stopReason: "cancelled" as const };
 						if (isFailed(end.value)) return yield* Agent.RequestError.internalError(end.value.data.error.message);
+						if (isRunFailed(end.value)) {
+							const { reason, message } = end.value.data;
+							return yield* Agent.RequestError.internalError(message, { reason });
+						}
 						return { stopReason: reason === "length" ? ("max_tokens" as const) : ("end_turn" as const) };
 					}).pipe(Effect.scoped),
 				),

packages/codework/src/cli/error.ts

@@ -1,4 +1,12 @@
-import { ModelCatalog, Plugin, Runner, SandboxError, SessionStore, Settings } from "@codeworksh/harness/effect";
+import {
+	Database,
+	ModelCatalog,
+	Plugin,
+	Runner,
+	SandboxError,
+	SessionStore,
+	Settings,
+} from "@codeworksh/harness/effect";
 import { SandboxProvider } from "@codeworksh/harness/sandbox";
 import { Duration, Effect, Schema } from "effect";
 import { Client } from "../server/client.ts";
@@ -38,6 +46,7 @@ const isSettingsError = Schema.is(Settings.SettingsError);
 const isSandboxDriverNotRegisteredError = Schema.is(SandboxError.SandboxDriverNotRegisteredError);
 const isSandboxDriverRegistrationError = Schema.is(SandboxError.SandboxDriverRegistrationError);
 const isExecutionError = Schema.is(Client.ExecutionError);
+const isDatabaseOutdatedError = Schema.is(Database.OutdatedError);
 
 const credentialHint = (provider: string): string => {
 	switch (provider) {
@@ -196,6 +205,9 @@ export const renderError = (error: unknown): string => {
 	if (isInvalidInputError(error)) {
 		return `error: ${error.message}\n`;
 	}
+	if (isDatabaseOutdatedError(error)) {
+		return `error[database-outdated]: ${error.message}\n`;
+	}
 	if (isOAuthError(error)) {
 		return (
 			[

packages/codework/src/server/client.ts

@@ -57,7 +57,8 @@ export const run = Effect.fn("Client.run")(function* <E, R>(
 	// go idle, and only idleness proves our prompt is never running.
 	const stalled = yield* Deferred.make<void>();
 
-	const terminal = (event: EventSchema.Payload): ExecutionError | undefined => {
+	/** How a run ended, as this client reports it; `undefined` for a success. */
+	const failure = (event: EventList.RunEnded): ExecutionError | undefined => {
 		if (Schema.is(EventList.ExecutionInterrupted)(event))
 			return new ExecutionError({
 				type: "aborted",
@@ -93,18 +94,18 @@ export const run = Effect.fn("Client.run")(function* <E, R>(
 				}
 				if (Schema.is(EventList.Prompted)(event) && event.data.messageId === id) promoted = true;
 
-				const ended = terminal(event);
-				const succeeded = Schema.is(EventList.ExecutionSucceeded)(event);
+				const ended = EventList.isRunEnded(event);
+				const error = ended ? failure(event) : undefined;
 				if (!promoted) {
 					// Kept whatever arrives first: our admission comes from another fiber than the runs, so a
 					// run's end may be notified before it.
-					if ((ended !== undefined || succeeded) && seq !== undefined) ends.push({ at: seq, error: ended });
+					if (ended && seq !== undefined) ends.push({ at: seq, error });
 					if (admitted && ends.some(({ at }) => admittedAt === undefined || at > admittedAt))
 						yield* Deferred.succeed(stalled, undefined);
 					return;
 				}
-				if (ended !== undefined) return yield* ended;
-				if (succeeded) {
+				if (error !== undefined) return yield* error;
+				if (ended) {
 					settled = true;
 					return;
 				}

packages/codework/test/__artifacts__/acp.run-failed.json

@@ -0,0 +1,10 @@
+{
+	"answer": {
+		"code": -32603,
+		"message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended",
+		"data": {
+			"reason": "truncated"
+		}
+	},
+	"requests": 2
+}

packages/codework/test/__artifacts__/cli.old-database.json

@@ -0,0 +1,4 @@
+{
+	"status": 1,
+	"stderr": "error[database-outdated]: The database at <home>/data/codework.db was created by an older version and lacks thread.head, inbox.tail; delete it, with its -wal and -shm files, to recreate it\n"
+}

packages/codework/test/__artifacts__/recovery.after-failure.json

@@ -121,7 +121,7 @@
 		{
 			"seq": 21,
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 1",
 			"parts": []
 		},

packages/codework/test/__artifacts__/recovery.backoff.json

@@ -0,0 +1,111 @@
+{
+	"log": [
+		{
+			"seq": 3,
+			"type": "session.execution.started",
+			"resumed": false
+		},
+		{
+			"seq": 6,
+			"type": "session.llm.started"
+		},
+		{
+			"seq": 7,
+			"type": "session.llm.failed",
+			"reason": "error",
+			"errorMessage": "cannot connect to API: connect ECONNREFUSED <refused>",
+			"parts": []
+		},
+		{
+			"seq": 8,
+			"type": "session.retry.scheduled",
+			"attempt": 1,
+			"maxRetries": 3,
+			"delayMs": 60000,
+			"message": "mcp/mcp-model: cannot connect to API: connect ECONNREFUSED <refused>"
+		},
+		{
+			"seq": 9,
+			"type": "session.execution.interrupted",
+			"reason": "crash"
+		},
+		{
+			"seq": 10,
+			"type": "session.execution.started",
+			"resumed": true
+		},
+		{
+			"seq": 11,
+			"type": "session.retry.scheduled",
+			"attempt": 1,
+			"maxRetries": 3,
+			"delayMs": 60000,
+			"message": "The host stopped while waiting to retry, before attempt 2"
+		},
+		{
+			"seq": 12,
+			"type": "session.execution.interrupted",
+			"reason": "shutdown"
+		},
+		{
+			"seq": 13,
+			"type": "session.execution.started",
+			"resumed": true
+		},
+		{
+			"seq": 14,
+			"type": "session.llm.started"
+		},
+		{
+			"seq": 15,
+			"type": "session.llm.ended",
+			"reason": "stop",
+			"parts": [
+				{
+					"text": "Answered."
+				}
+			]
+		},
+		{
+			"seq": 17,
+			"type": "session.execution.succeeded"
+		}
+	],
+	"live": [
+		{
+			"type": "session.retry.finished",
+			"attempt": 1,
+			"success": true
+		}
+	],
+	"steps": [
+		{
+			"step": "backoff",
+			"restarts": 0,
+			"attempt": 2
+		},
+		{
+			"step": "resumed",
+			"restarts": 0,
+			"attempt": 2
+		},
+		{
+			"step": "stopped",
+			"restarts": 0,
+			"attempt": 2
+		},
+		{
+			"step": "finished",
+			"restarts": 0,
+			"attempt": null
+		}
+	],
+	"requests": [
+		[
+			{
+				"role": "user",
+				"text": "Wait it out."
+			}
+		]
+	]
+}

packages/codework/test/__artifacts__/recovery.crashloop.json

@@ -33,7 +33,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 1",
 			"parts": []
 		},
@@ -57,7 +57,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 2",
 			"parts": []
 		},
@@ -81,7 +81,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 3",
 			"parts": []
 		},
@@ -105,7 +105,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 4",
 			"parts": []
 		},
@@ -129,7 +129,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The run was interrupted 5 times in a row; it is not resumed again",
 			"parts": []
 		},

packages/codework/test/__artifacts__/recovery.interrupted.json

@@ -38,7 +38,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 1",
 			"parts": []
 		},
@@ -62,7 +62,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 2",
 			"parts": []
 		},

packages/codework/test/__artifacts__/recovery.isolation.json

@@ -0,0 +1,42 @@
+{
+	"broken": [
+		{
+			"type": "session.execution.started",
+			"resumed": false
+		},
+		{
+			"type": "session.execution.interrupted",
+			"reason": "crash"
+		},
+		{
+			"type": "session.execution.started",
+			"resumed": true
+		},
+		{
+			"type": "session.execution.failed"
+		}
+	],
+	"healthy": [
+		{
+			"type": "session.execution.started",
+			"resumed": false
+		},
+		{
+			"type": "session.execution.interrupted",
+			"reason": "crash"
+		},
+		{
+			"type": "session.execution.started",
+			"resumed": true
+		},
+		{
+			"type": "session.execution.succeeded"
+		}
+	],
+	"resent": [
+		{
+			"role": "user",
+			"text": "Healthy."
+		}
+	]
+}

packages/codework/test/__artifacts__/recovery.mid-stream.json

@@ -45,7 +45,7 @@
 		},
 		{
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 1",
 			"parts": [
 				{

packages/codework/test/__artifacts__/recovery.moved.json

@@ -125,7 +125,7 @@
 		{
 			"seq": 22,
 			"type": "session.llm.failed",
-			"reason": "error",
+			"reason": "aborted",
 			"errorMessage": "The host stopped while the model was answering, on attempt 1",
 			"parts": []
 		},

packages/codework/test/__artifacts__/recovery.shutdown.json

@@ -39,7 +39,8 @@
 				"tail": 2,
 				"promotedSeq": 4
 			}
-		]
+		],
+		"attempt": 1
 	},
 	"resumed": {
 		"head": 0,