From ec92fdfeb336886842ab378fd25904c81b84cfd3 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 18:38:48 +0530 Subject: [PATCH 01/56] feat(harness): carry an offset on live text and thinking deltas session.llm.text.delta and session.llm.thinking.delta gain `offset`, the part's text length (JS string length) before the delta, so a client can splice a delta onto its draft or notice a gap. Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/event/list.ts | 10 ++- packages/harness/src/runner/event.ts | 10 +++ .../test/__artifacts__/runner.llm.deltas.json | 74 +++++++++++++++++++ packages/harness/test/event.log.test.ts | 1 + packages/harness/test/event.subscribe.test.ts | 1 + packages/harness/test/runner.llm.test.ts | 73 +++++++++++++++++- 6 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 packages/harness/test/__artifacts__/runner.llm.deltas.json diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index 35ae9012..fc86aa7e 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -206,9 +206,15 @@ export const LLMTextStart = EventSchema.define({ }); export type LLMTextStart = typeof LLMTextStart.Type; +/** + * `offset` is the part's text length before this delta, in JS string length (UTF-16 code units), + * so a client compares it with its draft's `text.length`. A delta at 0 starts its part. + */ +const LLMDeltaFields = { ...LLMPartFields, offset: NonNegativeInt, delta: Schema.String }; + export const LLMTextDelta = EventSchema.define({ type: "session.llm.text.delta", - schema: { ...LLMPartFields, delta: Schema.String }, + schema: LLMDeltaFields, }); export type LLMTextDelta = typeof LLMTextDelta.Type; @@ -226,7 +232,7 @@ export type LLMThinkingStart = typeof LLMThinkingStart.Type; export const LLMThinkingDelta = EventSchema.define({ type: "session.llm.thinking.delta", - schema: { ...LLMPartFields, delta: Schema.String }, + schema: LLMDeltaFields, }); export type LLMThinkingDelta = typeof LLMThinkingDelta.Type; diff --git a/packages/harness/src/runner/event.ts b/packages/harness/src/runner/event.ts index 054c3079..b6c5bfca 100644 --- a/packages/harness/src/runner/event.ts +++ b/packages/harness/src/runner/event.ts @@ -111,6 +111,14 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { let stopped = false; let ticker: Fiber.Fiber | undefined; + // Each part's text length so far, counted from the deltas: `partial` may have moved on when read. + const lengths = new Map(); + const advance = (partIndex: number, delta: string) => { + const offset = lengths.get(partIndex) ?? 0; + lengths.set(partIndex, offset + delta.length); + return offset; + }; + const checkpoint = Effect.gen(function* () { dirty = false; if (latest === undefined || startedMessageId === undefined) return; @@ -250,6 +258,7 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { ...base, messageId: yield* identify(event.partial.messageId), partIndex: event.partIndex, + offset: advance(event.partIndex, event.delta), delta: event.delta, }); return; @@ -276,6 +285,7 @@ export const make = Effect.fn("LLMEventPublisher.make")(function* (input: { ...base, messageId: yield* identify(event.partial.messageId), partIndex: event.partIndex, + offset: advance(event.partIndex, event.delta), delta: event.delta, }); return; diff --git a/packages/harness/test/__artifacts__/runner.llm.deltas.json b/packages/harness/test/__artifacts__/runner.llm.deltas.json new file mode 100644 index 00000000..910b95a5 --- /dev/null +++ b/packages/harness/test/__artifacts__/runner.llm.deltas.json @@ -0,0 +1,74 @@ +[ + { + "type": "session.llm.thinking.delta", + "partIndex": 0, + "offset": 0, + "delta": "Let", + "accumulated": 0, + "continues": true + }, + { + "type": "session.llm.thinking.delta", + "partIndex": 0, + "offset": 3, + "delta": " me ", + "accumulated": 3, + "continues": true + }, + { + "type": "session.llm.thinking.delta", + "partIndex": 0, + "offset": 7, + "delta": "think โœจ", + "accumulated": 7, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 1, + "offset": 0, + "delta": "Hel", + "accumulated": 0, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 1, + "offset": 3, + "delta": "lo ๐ŸŒ", + "accumulated": 3, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 1, + "offset": 8, + "delta": " world", + "accumulated": 8, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 2, + "offset": 0, + "delta": "", + "accumulated": 0, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 2, + "offset": 0, + "delta": "Bye", + "accumulated": 0, + "continues": true + }, + { + "type": "session.llm.text.delta", + "partIndex": 2, + "offset": 3, + "delta": ".", + "accumulated": 3, + "continues": true + } +] diff --git a/packages/harness/test/event.log.test.ts b/packages/harness/test/event.log.test.ts index 92581048..f24c0d2e 100644 --- a/packages/harness/test/event.log.test.ts +++ b/packages/harness/test/event.log.test.ts @@ -160,6 +160,7 @@ describe("Event.log", () => { threadId, messageId: messageId("live"), partIndex: 0, + offset: 0, delta: "token", }); // Another aggregate's commit must not wake this one into emitting. diff --git a/packages/harness/test/event.subscribe.test.ts b/packages/harness/test/event.subscribe.test.ts index c1e80142..f2ec5570 100644 --- a/packages/harness/test/event.subscribe.test.ts +++ b/packages/harness/test/event.subscribe.test.ts @@ -40,6 +40,7 @@ describe("Event.subscribe", () => { threadId, messageId: SessionMessageSchema.ID.make("message_live"), partIndex: 0, + offset: 0, delta: "hello", }); const durable = yield* publish(events, 1); diff --git a/packages/harness/test/runner.llm.test.ts b/packages/harness/test/runner.llm.test.ts index 6c22ec84..c457fdbe 100644 --- a/packages/harness/test/runner.llm.test.ts +++ b/packages/harness/test/runner.llm.test.ts @@ -1,12 +1,14 @@ import "./utils/env.ts"; -import { Message } from "@codeworksh/aikit"; -import { Effect, Layer } from "effect"; +import { createAssistantMessageEventStream, Message } from "@codeworksh/aikit"; +import { Effect, Layer, Schema } from "effect"; import { describe, expect } from "vite-plus/test"; import { Database } from "../src/db/db.ts"; import { Event } from "../src/event/event.ts"; +import { EventList } from "../src/event/list.ts"; import { LLMEventPublisher } from "../src/runner/event.ts"; import { LLM } from "../src/runner/llm.ts"; +import { assistant } from "./fixtures/llm.ts"; import { seedSpace } from "./fixtures/space.ts"; import { SessionLive } from "../src/session/live.ts"; import { Session } from "../src/session/session.ts"; @@ -14,6 +16,8 @@ import { testEffect } from "./utils/effect.ts"; const layer = SessionLive.layer.pipe(Layer.provideMerge(Event.layer), Layer.provideMerge(Database.layer(":memory:"))); const { effect: it } = testEffect(layer); +const isTextDelta = Schema.is(EventList.LLMTextDelta); +const isThinkingDelta = Schema.is(EventList.LLMThinkingDelta); const setup = Effect.gen(function* () { const { spaceId, location } = yield* seedSpace(); @@ -25,6 +29,7 @@ const setup = Effect.gen(function* () { tag: "test", }); return { + events: yield* Event.Service, threadId: thread.id, publisher: yield* LLMEventPublisher.make({ sessionId: session.id, threadId: thread.id }), }; @@ -69,4 +74,68 @@ describe("runner LLM", () => { expect(failure.message).toBe("openai/gpt-4o-mini: iterator failed"); }), ); + + it( + "stamps each text and thinking delta with its part's length so far", + Effect.gen(function* () { + const { events, threadId, publisher } = yield* setup; + // Thinking at 0, text at 1 and 2; ๐ŸŒ counts two in JS string length. + const script = [ + { kind: "thinking", partIndex: 0, deltas: ["Let", " me ", "think โœจ"] }, + { kind: "text", partIndex: 1, deltas: ["Hel", "lo ๐ŸŒ", " world"] }, + { kind: "text", partIndex: 2, deltas: ["", "Bye", "."] }, + ] as const; + const open: LLM.Open = (input) => + Effect.sync(() => { + const message = assistant(input, 1, { + parts: script.map((part) => + part.kind === "thinking" + ? { type: "thinking", thinking: part.deltas.join("") } + : { type: "text", text: part.deltas.join("") }, + ), + }); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + for (const { kind, partIndex, deltas } of script) { + stream.push({ type: `${kind}.start`, partIndex, partial: message }); + for (const delta of deltas) + stream.push({ type: `${kind}.delta`, partIndex, delta, partial: message }); + stream.push({ type: `${kind}.end`, partIndex, content: deltas.join(""), partial: message }); + } + stream.push({ type: "done", reason: "stop", message }); + return stream; + }); + + const deltas: Array<{ type: string; partIndex: number; offset: number; delta: string }> = []; + yield* events.listen((event) => + Effect.sync(() => { + if (isTextDelta(event) || isThinkingDelta(event)) { + const { partIndex, offset, delta } = event.data; + deltas.push({ type: event.type, partIndex, offset, delta }); + } + }), + ); + yield* LLM.make(open)({ + threadId, + context, + provider: "openai", + model: "gpt-4o-mini", + resolvedModel: yield* LLM.resolve({ provider: "openai", model: "gpt-4o-mini" }), + publisher, + }); + + const accumulated = new Map(); + const artifact = deltas.map((item) => { + const before = accumulated.get(item.partIndex) ?? 0; + accumulated.set(item.partIndex, before + item.delta.length); + return { ...item, accumulated: before, continues: item.offset === before }; + }); + expect(artifact.every((item) => item.continues)).toBe(true); + yield* Effect.promise(() => + expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot( + "./__artifacts__/runner.llm.deltas.json", + ), + ); + }), + ); }); From 23f3ba75f282dc0106a414d5fa88376ea9e577e2 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 18:54:53 +0530 Subject: [PATCH 02/56] feat(harness): durable run lifecycle, stored run state and startup sweep The execution lifecycle events and retry.scheduled are durable, and the session projector writes the thread's run state (run_state, its times, outcome and error) from each event's own timestamp. At startup, before the server listens, the harness ends every run a dead process left open with execution.interrupted (crash, or shutdown when the run was closed); a run recovery resumes says so with execution.started { resumed: true }. Co-Authored-By: Claude Opus 5.5 --- packages/codework/src/server/server.ts | 5 + .../__artifacts__/recovery.crashloop.json | 82 +++++++++ .../__artifacts__/recovery.interrupted.json | 30 ++++ .../__artifacts__/recovery.lifecycle.json | 132 ++++++++++++++ .../__artifacts__/recovery.mid-stream.json | 22 +++ .../test/__artifacts__/recovery.tool.json | 15 ++ .../__artifacts__/recovery.toolcrash.json | 47 +++++ .../test/__artifacts__/recovery.tools.json | 22 +++ .../__artifacts__/recovery.truncated.json | 7 + .../test/__artifacts__/server.threads.json | 8 + packages/codework/test/recovery.test.ts | 109 ++++++++++++ packages/harness/src/control.ts | 24 ++- packages/harness/src/db/migrations.ts | 7 + packages/harness/src/db/schema.sql.ts | 8 + packages/harness/src/effect/session.ts | 30 ++++ packages/harness/src/event/list.ts | 15 +- packages/harness/src/runner/execute.ts | 34 ++-- packages/harness/src/session/projector.ts | 30 ++++ packages/harness/src/session/schema.ts | 5 +- packages/harness/src/thread/schema.ts | 24 +++ packages/harness/src/thread/thread.ts | 55 ++++++ .../__artifacts__/thread.constraints.json | 2 +- .../test/__artifacts__/thread.fork.json | 66 +++---- .../test/__artifacts__/thread.lifecycle.json | 139 +++++++++++++++ .../test/__artifacts__/thread.recovery.json | 50 +++--- .../test/__artifacts__/thread.reset.json | 40 +++-- .../__artifacts__/thread.walkthrough.json | 80 +++++---- .../__artifacts__/tools.toolsearch.fork.json | 8 +- .../__artifacts__/tools.toolsearch.load.json | 48 ++--- .../tools.toolsearch.unavailable.json | 8 +- packages/harness/test/runner.loop.test.ts | 12 +- packages/harness/test/thread.test.ts | 168 +++++++++++++++++- 32 files changed, 1167 insertions(+), 165 deletions(-) create mode 100644 packages/codework/test/__artifacts__/recovery.lifecycle.json create mode 100644 packages/harness/test/__artifacts__/thread.lifecycle.json diff --git a/packages/codework/src/server/server.ts b/packages/codework/src/server/server.ts index 49ec35ac..282baeb4 100644 --- a/packages/codework/src/server/server.ts +++ b/packages/codework/src/server/server.ts @@ -53,6 +53,9 @@ const managedShutdown = Layer.unwrap( }), ); +/** Ends the stored runs the last server left open, before the listener opens: a client never sees them live. */ +const sweep = Layer.effectDiscard(Effect.flatMap(Control.Service, (control) => control.sweep)); + /** * Resumes the runs the last server left unfinished, whether it crashed or shut down: the session * table records them as they start, so this needs nothing from the previous process. @@ -101,6 +104,8 @@ export const layer = (options: { host: string; port: number; harness: Harness.Op Layer.provideMerge(NodeHttpServer.layer(createServer, { host: options.host, port: options.port })), Layer.provide(RpcSerialization.layerNdjson), Layer.provide(Identity.layer), + // Built after the harness and before everything above it, the listener included. + Layer.provide(sweep), Layer.provide(Harness.layer(options.harness)), ); }; diff --git a/packages/codework/test/__artifacts__/recovery.crashloop.json b/packages/codework/test/__artifacts__/recovery.crashloop.json index 1b909d09..340b6a0b 100644 --- a/packages/codework/test/__artifacts__/recovery.crashloop.json +++ b/packages/codework/test/__artifacts__/recovery.crashloop.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Keep crashing." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -19,42 +23,110 @@ { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 1", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 1, + "maxRetries": 10, + "delayMs": 10, + "message": "The host stopped while the model was answering, on attempt 1" + }, { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 2", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 2, + "maxRetries": 10, + "delayMs": 20, + "message": "The host stopped while the model was answering, on attempt 2" + }, { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 3", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 3, + "maxRetries": 10, + "delayMs": 40, + "message": "The host stopped while the model was answering, on attempt 3" + }, { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 4", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 4, + "maxRetries": 10, + "delayMs": 80, + "message": "The host stopped while the model was answering, on attempt 4" + }, { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", @@ -66,11 +138,18 @@ "reason": "crashloop", "message": "The run was interrupted 5 times in a row; it is not resumed again" }, + { + "type": "session.execution.succeeded" + }, { "type": "session.prompt.admitted", "delivery": "steer", "text": "Try again." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -90,6 +169,9 @@ }, { "type": "session.turn.ended" + }, + { + "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/recovery.interrupted.json b/packages/codework/test/__artifacts__/recovery.interrupted.json index 41fc3566..bf41c9f7 100644 --- a/packages/codework/test/__artifacts__/recovery.interrupted.json +++ b/packages/codework/test/__artifacts__/recovery.interrupted.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "First." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -24,15 +28,38 @@ "delivery": "followUp", "text": "Then this." }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 1", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 1, + "maxRetries": 1, + "delayMs": 10, + "message": "The host stopped while the model was answering, on attempt 1" + }, { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", @@ -43,6 +70,9 @@ "type": "session.run.failed", "reason": "interrupted", "message": "The host stopped while the model was answering, on attempt 2" + }, + { + "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/recovery.lifecycle.json b/packages/codework/test/__artifacts__/recovery.lifecycle.json new file mode 100644 index 00000000..1f1c8114 --- /dev/null +++ b/packages/codework/test/__artifacts__/recovery.lifecycle.json @@ -0,0 +1,132 @@ +{ + "log": [ + { + "seq": 3, + "type": "session.execution.started", + "resumed": false + }, + { + "seq": 6, + "type": "session.llm.started" + }, + { + "seq": 9, + "type": "session.execution.succeeded" + }, + { + "seq": 11, + "type": "session.execution.started", + "resumed": false + }, + { + "seq": 13, + "type": "session.llm.started" + }, + { + "seq": 14, + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "seq": 15, + "type": "session.execution.started", + "resumed": true + }, + { + "seq": 18, + "type": "session.llm.started" + }, + { + "seq": 21, + "type": "session.execution.succeeded" + } + ], + "rows": [ + { + "step": "succeeded", + "state": "idle", + "stateAt": "9 session.execution.succeeded", + "startedAt": "3 session.execution.started", + "endedAt": "9 session.execution.succeeded", + "outcome": "succeeded", + "error": null, + "execution": null + }, + { + "step": "in flight", + "state": "running", + "stateAt": "13 session.llm.started", + "startedAt": "11 session.execution.started", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "killed", + "state": "running", + "stateAt": "13 session.llm.started", + "startedAt": "11 session.execution.started", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "resumed", + "state": "running", + "stateAt": "18 session.llm.started", + "startedAt": "15 session.execution.started", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "finished", + "state": "idle", + "stateAt": "21 session.execution.succeeded", + "startedAt": "15 session.execution.started", + "endedAt": "21 session.execution.succeeded", + "outcome": "succeeded", + "error": null, + "execution": null + } + ], + "requests": [ + [ + { + "role": "user", + "text": "First." + } + ], + [ + { + "role": "user", + "text": "First." + }, + { + "role": "assistant", + "text": "First answer." + }, + { + "role": "user", + "text": "Second." + } + ], + [ + { + "role": "user", + "text": "First." + }, + { + "role": "assistant", + "text": "First answer." + }, + { + "role": "user", + "text": "Second." + } + ] + ] +} diff --git a/packages/codework/test/__artifacts__/recovery.mid-stream.json b/packages/codework/test/__artifacts__/recovery.mid-stream.json index f40aebc2..ef60a11c 100644 --- a/packages/codework/test/__artifacts__/recovery.mid-stream.json +++ b/packages/codework/test/__artifacts__/recovery.mid-stream.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Answer at length." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -31,6 +35,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", @@ -41,6 +53,13 @@ } ] }, + { + "type": "session.retry.scheduled", + "attempt": 1, + "maxRetries": 3, + "delayMs": 10, + "message": "The host stopped while the model was answering, on attempt 1" + }, { "type": "session.llm.started" }, @@ -55,6 +74,9 @@ }, { "type": "session.turn.ended" + }, + { + "type": "session.execution.succeeded" } ], "live": [ diff --git a/packages/codework/test/__artifacts__/recovery.tool.json b/packages/codework/test/__artifacts__/recovery.tool.json index 761ff475..aa42241d 100644 --- a/packages/codework/test/__artifacts__/recovery.tool.json +++ b/packages/codework/test/__artifacts__/recovery.tool.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Run the slow command." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -29,6 +33,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -54,6 +66,9 @@ }, { "type": "session.turn.ended" + }, + { + "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/recovery.toolcrash.json b/packages/codework/test/__artifacts__/recovery.toolcrash.json index b3a15fca..0a0dc50b 100644 --- a/packages/codework/test/__artifacts__/recovery.toolcrash.json +++ b/packages/codework/test/__artifacts__/recovery.toolcrash.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Run the slow command." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -29,6 +33,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -53,6 +65,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -77,6 +97,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -101,6 +129,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -125,6 +161,14 @@ } ] }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.tool.settled", "part": { @@ -140,6 +184,9 @@ "type": "session.run.failed", "reason": "crashloop", "message": "The run was interrupted 5 times in a row; it is not resumed again" + }, + { + "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/recovery.tools.json b/packages/codework/test/__artifacts__/recovery.tools.json index a46fb4ff..16e0c616 100644 --- a/packages/codework/test/__artifacts__/recovery.tools.json +++ b/packages/codework/test/__artifacts__/recovery.tools.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Find the echo tool." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -46,12 +50,27 @@ { "type": "session.llm.started" }, + { + "type": "session.execution.interrupted", + "reason": "crash" + }, + { + "type": "session.execution.started", + "resumed": true + }, { "type": "session.llm.failed", "reason": "error", "errorMessage": "The host stopped while the model was answering, on attempt 1", "parts": [] }, + { + "type": "session.retry.scheduled", + "attempt": 1, + "maxRetries": 3, + "delayMs": 10, + "message": "The host stopped while the model was answering, on attempt 1" + }, { "type": "session.llm.started" }, @@ -66,6 +85,9 @@ }, { "type": "session.turn.ended" + }, + { + "type": "session.execution.succeeded" } ], "tools": [ diff --git a/packages/codework/test/__artifacts__/recovery.truncated.json b/packages/codework/test/__artifacts__/recovery.truncated.json index a10d664e..7df4fbdd 100644 --- a/packages/codework/test/__artifacts__/recovery.truncated.json +++ b/packages/codework/test/__artifacts__/recovery.truncated.json @@ -8,6 +8,10 @@ "delivery": "steer", "text": "Run a long command." }, + { + "type": "session.execution.started", + "resumed": false + }, { "type": "session.prompt.promoted", "delivery": "steer", @@ -68,6 +72,9 @@ "type": "session.run.failed", "reason": "truncated", "message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended" + }, + { + "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/server.threads.json b/packages/codework/test/__artifacts__/server.threads.json index ef9dfa9c..b3c44dc8 100644 --- a/packages/codework/test/__artifacts__/server.threads.json +++ b/packages/codework/test/__artifacts__/server.threads.json @@ -74,6 +74,10 @@ "type": "session.prompt.admitted", "thread": "" }, + { + "type": "session.execution.started", + "thread": "" + }, { "type": "session.prompt.promoted", "thread": "" @@ -90,6 +94,10 @@ "type": "session.turn.ended", "thread": "" }, + { + "type": "session.execution.succeeded", + "thread": "" + }, { "type": "log.synced" } diff --git a/packages/codework/test/recovery.test.ts b/packages/codework/test/recovery.test.ts index 656c9fa3..3856a2fd 100644 --- a/packages/codework/test/recovery.test.ts +++ b/packages/codework/test/recovery.test.ts @@ -5,6 +5,7 @@ import { Control } from "@codeworksh/harness/effect"; import { Effect, Fiber, Layer, Stream } from "effect"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; +import { DatabaseSync } from "node:sqlite"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it, onTestFinished } from "vite-plus/test"; @@ -149,6 +150,10 @@ const summary = (event: EventEnvelope) => { return { type: event.type, part: part(data["part"]) }; case "session.run.failed": return { type: event.type, reason: data["reason"], message: data["message"] }; + case "session.execution.started": + return { type: event.type, resumed: data["resumed"] }; + case "session.execution.interrupted": + return { type: event.type, reason: data["reason"] }; case "session.retry.scheduled": case "session.retry.finished": { const { sessionId: _s, threadId: _t, timestamp: _ts, ...rest } = data; @@ -198,7 +203,111 @@ const hang = (response: Reply) => response.send({ role: "assistant" }); const retry = { retry: { maxRetries: 3, baseDelayMs: 10, maxDelayMs: 1_000 } }; +/** The thread's stored run, read from the database file; times named by the log event they came from. */ +const runRow = (home: string, threadId: string, events: ReadonlyArray) => { + const db = new DatabaseSync(join(home, "data", "codework.db")); + const row = db + .prepare( + "SELECT run_state, run_state_at, run_started_at, run_ended_at, run_outcome, run_error, execution FROM thread WHERE id = ?", + ) + .get(threadId)!; + db.close(); + const at = (column: string, types: ReadonlyArray) => { + const time = row[column]; + if (time === null) return null; + const from = events.findLast((event) => types.includes(event.type) && (event.data as Json)["timestamp"] === time); + return from === undefined ? `unmatched ${String(time)}` : `${from.durable?.seq} ${from.type}`; + }; + const ends = [ + "session.execution.succeeded", + "session.execution.failed", + "session.execution.interrupted", + "session.run.failed", + ]; + const state = String(row["run_state"]); + return { + state, + stateAt: at( + "run_state_at", + state === "starting" ? ["session.execution.started"] : state === "running" ? ["session.llm.started"] : ends, + ), + startedAt: at("run_started_at", ["session.execution.started"]), + endedAt: at("run_ended_at", ends), + outcome: row["run_outcome"], + error: row["run_error"] === null ? null : JSON.parse(String(row["run_error"])), + execution: row["execution"], + }; +}; + describe("recovery after the host is killed", () => { + it("the stored run of a killed host is swept as a crash before clients connect, then resumed", async () => { + const resend = gate(); + const fake = await streaming((_body, response, index) => { + if (index === 0) return answer(response, "First answer."); + hang(response); + if (index === 2) void resend.opened.then(() => answer(response, "Resumed answer.")); + }); + const { home, start } = setup(fake.url, retry); + + const first = await start(); + const created = await call(first.url, (rpc) => rpc["session.create"]({})); + const ids = { sessionId: created.session.id, threadId: created.id }; + const rows: Array<{ step: string } & ReturnType> = []; + const record = (step: string, events: ReadonlyArray) => + rows.push({ step, ...runRow(home, ids.threadId, events) }); + + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "First." })); + await logged(first.url, ids, "session.execution.succeeded"); + record("succeeded", await log(first.url, ids)); + await call(first.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Second." })); + const inFlight = await logged(first.url, ids, "session.llm.started", 2); + record("in flight", inFlight); + await first.kill(); + // What the dead host left: the run still open. + record("killed", inFlight); + + const second = await start(); + // The sweep committed before the listener opened, so the first read already holds it. + const reopened = await log(second.url, ids); + const swept = reopened.findIndex( + (event) => event.type === "session.execution.interrupted" && (event.data as Json)["reason"] === "crash", + ); + expect(swept).toBeGreaterThan(0); + record("resumed", await logged(second.url, ids, "session.llm.started", 3)); + resend.open(); + await logged(second.url, ids, "session.turn.ended", 2); + await call(second.url, (rpc) => rpc["thread.wait"](ids)); + const events = await log(second.url, ids); + record("finished", events); + + const resumed = events.findIndex( + (event) => event.type === "session.execution.started" && (event.data as Json)["resumed"] === true, + ); + // Swept first, resumed after, with nothing of the run in between. + expect(resumed).toBe(swept + 1); + expect(rows.find((row) => row.step === "resumed")).toMatchObject({ state: "running", execution: "running" }); + + const lifecycle = new Set([ + "session.execution.started", + "session.execution.succeeded", + "session.execution.failed", + "session.execution.interrupted", + "session.run.failed", + "session.llm.started", + ]); + await expect( + artifact({ + log: events + .filter((event) => lifecycle.has(event.type)) + .map((event) => ({ seq: event.durable?.seq, ...summary(event) })), + // Read from the database file: after the first run, mid-run, with the resumed request in + // flight, and once it answered. + rows, + requests: fake.requests.map(sent), + }), + ).toMatchFileSnapshot("./__artifacts__/recovery.lifecycle.json"); + }, 90_000); + it("a reply killed mid-stream keeps its partial text, and the request is resent without it", async () => { const resend = gate(); const fake = await streaming((_body, response, index) => { diff --git a/packages/harness/src/control.ts b/packages/harness/src/control.ts index eb292386..c10435aa 100644 --- a/packages/harness/src/control.ts +++ b/packages/harness/src/control.ts @@ -14,7 +14,9 @@ * execution and on storage, and neither depends back. */ -import { Context, Effect, Layer, Option, Schema } from "effect"; +import { Context, DateTime, Effect, Layer, Option, Schema } from "effect"; +import { Event } from "./event/event.ts"; +import { EventList } from "./event/list.ts"; import { Inbox } from "./inbox/inbox.ts"; import { Mcp } from "./mcp/mcp.ts"; import type { Admitted } from "./inbox/schema.ts"; @@ -105,6 +107,12 @@ export interface Interface { * run has settled. */ readonly resumeUnfinished: Effect.Effect; + /** + * Ends every stored run the last process left open with a durable `execution.interrupted`: + * `shutdown` where it closed the run, `crash` where it died in it. Runs as the server starts, + * before it accepts connections and before {@link resumeUnfinished}. + */ + readonly sweep: Effect.Effect; /** Prompts admitted but not yet in the conversation, steers and follow-ups, in admission order. */ readonly pending: (threadId: ThreadSchema.ID) => Effect.Effect>; /** @@ -131,6 +139,7 @@ export const layer = Layer.effect( const inputs = yield* Inbox.make; const execution = yield* RunnerExecution.Service; const mcp = yield* Mcp.Service; + const events = yield* Event.Service; const isLifecycleConflict = Schema.is(Inbox.LifecycleConflict); // Marked before the wake, so a prompt accepted here survives a crash before its run starts. @@ -293,6 +302,18 @@ export const layer = Layer.effect( ); }).pipe(Effect.withSpan("Control.resumeUnfinished")); + const sweep = Effect.gen(function* () { + for (const row of yield* threads.open) { + const execution = Option.getOrUndefined(row.execution); + yield* events.publish(EventList.ExecutionInterrupted, { + timestamp: yield* DateTime.now, + sessionId: SessionSchema.ID.make(row.sessionId), + threadId: ThreadSchema.ID.make(row.id), + reason: execution === "closed" ? "shutdown" : "crash", + }); + } + }).pipe(Effect.withSpan("Control.sweep")); + return Service.of({ resolve, prompt, @@ -304,6 +325,7 @@ export const layer = Layer.effect( deleteSession, active: execution.active, resumeUnfinished, + sweep, pending: inputs.pending, cancel: inputs.cancel, update, diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index e3c9f3a1..a467dffa 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -123,6 +123,13 @@ export const migrations = { origin_call_id TEXT, title TEXT, execution TEXT, + -- the run lifecycle, projected from the execution.* events; times are the events' + run_state TEXT NOT NULL DEFAULT 'idle', + run_state_at INTEGER, + run_started_at INTEGER, + run_ended_at INTEGER, + run_outcome TEXT, + run_error TEXT, restarts INTEGER NOT NULL DEFAULT 0, attempt INTEGER, cost REAL NOT NULL DEFAULT 0, diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index 6213f195..d97269bd 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -142,6 +142,14 @@ export class ThreadRow extends Model.Class("ThreadRow")({ // Absent until the thread's first own prompt names it. title: Model.FieldOption(Schema.String), execution: Model.FieldOption(ThreadSchema.Execution), + // The run lifecycle, written only by the session projector (`Thread.projectRun`). `runStateAt` is + // when `runState` last changed; the rest describe the last run, absent before the first. + runState: Model.GeneratedByDb(ThreadSchema.RunState), + runStateAt: Model.GeneratedByDb(Schema.OptionFromNullOr(Schema.DateTimeUtcFromMillis)), + runStartedAt: Model.GeneratedByDb(Schema.OptionFromNullOr(Schema.DateTimeUtcFromMillis)), + runEndedAt: Model.GeneratedByDb(Schema.OptionFromNullOr(Schema.DateTimeUtcFromMillis)), + runOutcome: Model.GeneratedByDb(Schema.OptionFromNullOr(ThreadSchema.RunOutcome)), + runError: Model.GeneratedByDb(Schema.OptionFromNullOr(Schema.fromJsonString(ThreadSchema.RunError))), restarts: Model.GeneratedByDb(Schema.Int), // The 1-based number of the provider request in flight, from its start until its response ends, the // turn commits or the run ends: still set on recovery, the host died with that request out. diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index 1f5ffc3f..6fdb59b2 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -130,6 +130,17 @@ export interface ThreadInfo { readonly placement?: DocumentSchema.Placement; /** Billed usage of its own entries; what it inherits through a fork point counts for its source. */ readonly usage: Usage; + /** Where its run is, as stored: live whenever the harness is up. */ + readonly runState: ThreadSchema.RunState; + /** When `runState` last changed, epoch milliseconds; absent before the first run. */ + readonly runStateAt?: number; + /** The last run, absent before the first: times in epoch milliseconds; `error` unless it succeeded. */ + readonly lastRun?: { + readonly startedAt: number; + readonly endedAt?: number; + readonly outcome?: ThreadSchema.RunOutcome; + readonly error?: ThreadSchema.RunError; + }; readonly session: Info; } @@ -279,6 +290,13 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS const row = found.value; const agent = yield* documents.agent(id); const title = Option.getOrUndefined(row.title); + const millis = (time: Option.Option) => + Option.getOrUndefined(Option.map(time, DateTime.toEpochMillis)); + const runStateAt = millis(row.runStateAt); + const startedAt = millis(row.runStartedAt); + const endedAt = millis(row.runEndedAt); + const outcome = Option.getOrUndefined(row.runOutcome); + const error = Option.getOrUndefined(row.runError); return { id, sessionId: SessionSchema.ID.make(row.sessionId), @@ -296,6 +314,18 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS cacheRead: row.tokensCacheRead, cacheWrite: row.tokensCacheWrite, }, + runState: row.runState, + ...(runStateAt === undefined ? {} : { runStateAt }), + ...(startedAt === undefined + ? {} + : { + lastRun: { + startedAt, + ...(endedAt === undefined ? {} : { endedAt }), + ...(outcome === undefined ? {} : { outcome }), + ...(error === undefined ? {} : { error }), + }, + }), session: yield* info(SessionSchema.ID.make(row.sessionId)), } satisfies ThreadInfo; }); diff --git a/packages/harness/src/event/list.ts b/packages/harness/src/event/list.ts index 35ae9012..a246f0ce 100644 --- a/packages/harness/src/event/list.ts +++ b/packages/harness/src/event/list.ts @@ -170,6 +170,7 @@ export type TurnAborted = typeof TurnAborted.Type; */ export const RetryScheduled = EventSchema.define({ type: "session.retry.scheduled", + ...durableOptions, schema: { ...baseOptions, attempt: NonNegativeInt, @@ -323,24 +324,29 @@ export type ToolSettled = typeof ToolSettled.Type; */ export const ExecutionStarted = EventSchema.define({ type: "session.execution.started", - schema: baseOptions, + ...durableOptions, + /** `resumed`: recovery started it after a restart, for a run the last process left unfinished. */ + schema: { ...baseOptions, resumed: Schema.Boolean }, }); export type ExecutionStarted = typeof ExecutionStarted.Type; export const ExecutionSucceeded = EventSchema.define({ type: "session.execution.succeeded", + ...durableOptions, schema: baseOptions, }); export type ExecutionSucceeded = typeof ExecutionSucceeded.Type; export const ExecutionFailed = EventSchema.define({ type: "session.execution.failed", + ...durableOptions, schema: { ...baseOptions, error: SessionFailure.Error }, }); export type ExecutionFailed = typeof ExecutionFailed.Type; export const ExecutionInterrupted = EventSchema.define({ type: "session.execution.interrupted", + ...durableOptions, schema: { ...baseOptions, reason: SessionSchema.InterruptReason }, }); export type ExecutionInterrupted = typeof ExecutionInterrupted.Type; @@ -355,7 +361,7 @@ export const RunFailed = EventSchema.define({ ...durableOptions, schema: { ...baseOptions, - reason: Schema.Literals(["crashloop", "interrupted", "truncated"]), + reason: ThreadSchema.RunFailedReason, message: Schema.String, }, }); @@ -474,6 +480,10 @@ export const Definitions = EventSchema.inventory( export const PublicDefinitions = Definitions; export const DurableDefinitions = EventSchema.inventory( + ExecutionStarted, + ExecutionSucceeded, + ExecutionFailed, + ExecutionInterrupted, PromptAdmitted, Prompted, PromptCancelled, @@ -483,6 +493,7 @@ export const DurableDefinitions = EventSchema.inventory( ThreadConfigured, SystemChanged, TurnEnded, + RetryScheduled, LLMStarted, LLMProgress, LLMEnded, diff --git a/packages/harness/src/runner/execute.ts b/packages/harness/src/runner/execute.ts index 7847dbc4..85de0441 100644 --- a/packages/harness/src/runner/execute.ts +++ b/packages/harness/src/runner/execute.ts @@ -51,19 +51,10 @@ export const layer = Layer.effect( Effect.ignore, ); - // The session a thread belongs to, for the lifecycle events its log carries. Threads never - // move between sessions, so one lookup per thread is enough. - const owners = new Map(); + // The session a thread belongs to, for the lifecycle events its log carries. Read every time, not + // cached: the events are durable, and one for a session deleted since would recreate its log. const sessionOf = (threadId: ThreadId) => - Effect.gen(function* () { - const known = owners.get(threadId); - if (known !== undefined) return Option.some(known); - const thread = yield* threads.get(threadId); - if (Option.isNone(thread)) return Option.none(); - const sessionId = SessionSchema.ID.make(thread.value.sessionId); - owners.set(threadId, sessionId); - return Option.some(sessionId); - }); + threads.get(threadId).pipe(Effect.map(Option.map((thread) => SessionSchema.ID.make(thread.sessionId)))); const publishLifecycle = ( threadId: ThreadId, publish: (base: { @@ -89,14 +80,19 @@ export const layer = Layer.effect( // The run is recorded unfinished before it starts, so a process dying anywhere in it // leaves the thread for the next server start to resume. A recovered one already is. started: (threadId) => - reportLifecycle( - threadId, - (recovering.has(threadId) ? Effect.void : threads.markRunning(threadId)).pipe( - Effect.andThen( - publishLifecycle(threadId, (base) => events.publish(EventList.ExecutionStarted, base)), + Effect.suspend(() => { + const resumed = recovering.has(threadId); + return reportLifecycle( + threadId, + (resumed ? Effect.void : threads.markRunning(threadId)).pipe( + Effect.andThen( + publishLifecycle(threadId, (base) => + events.publish(EventList.ExecutionStarted, { ...base, resumed }), + ), + ), ), - ), - ), + ); + }), // One terminal per busy period, covering every drain it coalesced. settled: (threadId, exit, reason) => reportLifecycle( diff --git a/packages/harness/src/session/projector.ts b/packages/harness/src/session/projector.ts index bdf2bbf3..e05cce11 100644 --- a/packages/harness/src/session/projector.ts +++ b/packages/harness/src/session/projector.ts @@ -158,8 +158,38 @@ export const layer = Layer.effectDiscard( }), ); + // The run lifecycle onto the thread row, at each event's own time. + yield* events.project(EventList.ExecutionStarted, ({ data }) => + threads.projectRun(data.threadId, data.timestamp, { _tag: "started" }), + ); + yield* events.project(EventList.ExecutionSucceeded, ({ data }) => + threads.projectRun(data.threadId, data.timestamp, { _tag: "ended", outcome: "succeeded" }), + ); + yield* events.project(EventList.ExecutionFailed, ({ data }) => + threads.projectRun(data.threadId, data.timestamp, { + _tag: "ended", + outcome: "failed", + error: { _tag: "failed", error: data.error }, + }), + ); + yield* events.project(EventList.ExecutionInterrupted, ({ data }) => + threads.projectRun(data.threadId, data.timestamp, { + _tag: "ended", + outcome: "interrupted", + error: { _tag: "interrupted", reason: data.reason }, + }), + ); + yield* events.project(EventList.RunFailed, ({ data }) => + threads.projectRun(data.threadId, data.timestamp, { + _tag: "ended", + outcome: "failed", + error: { _tag: "runFailed", reason: data.reason, message: data.message }, + }), + ); + yield* events.project(EventList.LLMStarted, (event) => Effect.gen(function* () { + yield* threads.projectRun(event.data.threadId, event.data.timestamp, { _tag: "running" }); const encoded = yield* encodeAssistant(event.data); yield* threads .append({ diff --git a/packages/harness/src/session/schema.ts b/packages/harness/src/session/schema.ts index b30955f2..827f3aec 100644 --- a/packages/harness/src/session/schema.ts +++ b/packages/harness/src/session/schema.ts @@ -10,9 +10,10 @@ export const IDFromDb = Schema.String.pipe(Schema.brand("Session.ID")); /** * Why an execution stopped. Supplied by whoever asked for the interruption -- * a stop the user asked for and a stop the process imposed on shutdown are - * different facts, and inspecting the cause cannot tell them apart. + * different facts, and inspecting the cause cannot tell them apart. `crash` is + * the startup sweep's: the process that ran the execution died without ending it. */ -export const InterruptReason = Schema.Literals(["user", "shutdown"]); +export const InterruptReason = Schema.Literals(["user", "shutdown", "crash"]); export type InterruptReason = typeof InterruptReason.Type; // Shared persistence-boundary codecs. Append validates before writing; fork diff --git a/packages/harness/src/thread/schema.ts b/packages/harness/src/thread/schema.ts index c2b9c461..34d807ad 100644 --- a/packages/harness/src/thread/schema.ts +++ b/packages/harness/src/thread/schema.ts @@ -1,4 +1,6 @@ import { Schema } from "effect"; +import { SessionFailure } from "../session/failure.ts"; +import { SessionSchema } from "../session/schema.ts"; // Thread identity lives in `@codeworksh/plugin`, next to the session's: it is on every plugin // context and every tool call. @@ -30,6 +32,28 @@ export type Origin = export const Execution = Schema.Literals(["running", "closed"]); export type Execution = typeof Execution.Type; +/** + * Where the thread's run is, projected from its lifecycle events: `starting` from + * `execution.started` until the first `llm.started`, `running` after it, `idle` once it ended. + */ +export const RunState = Schema.Literals(["idle", "starting", "running"]); +export type RunState = typeof RunState.Type; + +export const RunOutcome = Schema.Literals(["succeeded", "failed", "interrupted"]); +export type RunOutcome = typeof RunOutcome.Type; + +/** Why a run ended without its answer and is not resumed; see `session.run.failed`. */ +export const RunFailedReason = Schema.Literals(["crashloop", "interrupted", "truncated"]); +export type RunFailedReason = typeof RunFailedReason.Type; + +/** Why the last run did not succeed; stored as `thread.run_error` JSON. */ +export const RunError = Schema.Union([ + Schema.TaggedStruct("failed", { error: SessionFailure.Error }), + Schema.TaggedStruct("runFailed", { reason: RunFailedReason, message: Schema.String }), + Schema.TaggedStruct("interrupted", { reason: SessionSchema.InterruptReason }), +]); +export type RunError = typeof RunError.Type; + /** `main` names a session's main thread wherever a thread id is accepted from a person. */ export const MAIN = "main"; diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index 6c6ada30..b8445cd4 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -140,6 +140,12 @@ export interface InsertThread { readonly seq: number; } +/** One step of a thread's run lifecycle, from the event that projects it. */ +export type RunChange = + | { readonly _tag: "started" } + | { readonly _tag: "running" } + | { readonly _tag: "ended"; readonly outcome: ThreadSchema.RunOutcome; readonly error?: ThreadSchema.RunError }; + // Entry row + its part rows, zipped. Message decode happens above this layer. export interface HydratedEntry { readonly entry: EntryRow; @@ -213,6 +219,13 @@ export interface Interface { readonly idle: (threadId: ThreadSchema.ID) => Effect.Effect; /** Root threads with a run left unfinished, oldest first. */ readonly unfinished: Effect.Effect>; + /** + * Projector-only: the `run_*` columns, at the time `at` of the event that changed them. The one + * writer of those columns. + */ + readonly projectRun: (threadId: ThreadSchema.ID, at: DateTime.Utc, change: RunChange) => Effect.Effect; + /** Threads whose stored run is not idle: at startup, the runs a dead process left open. */ + readonly open: Effect.Effect>; /** Set or clear (null) the bookmark label on an entry of the thread. */ readonly setLabel: (input: { threadId: ThreadSchema.ID; @@ -232,6 +245,7 @@ const decodeMessageEnvelopeIdentity = Schema.decodeUnknownEffect(SessionSchema.M const decodeCompactionData = Schema.decodeUnknownEffect(SessionSchema.CompactionData); const decodeJsonObject = Schema.decodeUnknownEffect(SessionSchema.JsonObject); const encodeJsonObject = Schema.encodeEffect(SessionSchema.JsonObject); +const encodeRunError = Schema.encodeEffect(Schema.fromJsonString(ThreadSchema.RunError)); const decodeUserText = Schema.decodeUnknownOption( Schema.fromJsonString(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), ); @@ -890,6 +904,45 @@ export const layer = Layer.effect( }); const unfinished = findUnfinished(undefined).pipe(Effect.orDie, Effect.withSpan("Thread.unfinished")); + const projectRun = Effect.fn("Thread.projectRun")(function* ( + threadId: ThreadSchema.ID, + at: DateTime.Utc, + change: RunChange, + ) { + const time = DateTime.toEpochMillis(at); + const state: ThreadSchema.RunState = + change._tag === "started" ? "starting" : change._tag === "running" ? "running" : "idle"; + // SET reads the old row, so `run_state_at` moves only when the state does. + const stateAt = sql`CASE WHEN run_state = ${state} THEN run_state_at ELSE ${time} END`; + if (change._tag === "running") { + yield* sql`UPDATE thread SET run_state_at = ${stateAt}, run_state = ${state} WHERE id = ${threadId}`; + return; + } + if (change._tag === "started") { + yield* sql` + UPDATE thread SET + run_state_at = ${stateAt}, run_state = ${state}, run_started_at = ${time}, + run_ended_at = NULL, run_outcome = NULL, run_error = NULL + WHERE id = ${threadId} + `; + return; + } + const error = change.error === undefined ? null : yield* encodeRunError(change.error); + yield* sql` + UPDATE thread SET + run_state_at = ${stateAt}, run_state = ${state}, run_ended_at = ${time}, + run_outcome = ${change.outcome}, run_error = ${error} + WHERE id = ${threadId} + `; + }, Effect.orDie); + + const findOpen = SqlSchema.findAll({ + Request: Schema.Void, + Result: ThreadRow, + execute: () => sql`SELECT * FROM thread WHERE run_state <> 'idle' ORDER BY updated_at`, + }); + const open = findOpen(undefined).pipe(Effect.orDie, Effect.withSpan("Thread.open")); + const setLabel = Effect.fn("Thread.setLabel")(function* (input: { threadId: ThreadSchema.ID; entryId: string; @@ -931,6 +984,8 @@ export const layer = Layer.effect( setAttempt, idle, unfinished, + projectRun, + open, setLabel, }); }), diff --git a/packages/harness/test/__artifacts__/thread.constraints.json b/packages/harness/test/__artifacts__/thread.constraints.json index 6fc67088..77c8cbfb 100644 --- a/packages/harness/test/__artifacts__/thread.constraints.json +++ b/packages/harness/test/__artifacts__/thread.constraints.json @@ -4,5 +4,5 @@ "twoRevisionsOneEvent": "constraint failed: revision.document_id, revision.seq", "draftWithCommit": "constraint failed: (state = 'draft') = (committed_seq IS NULL)", "forkWithoutCut": "constraint failed: origin_kind <> 'fork' OR origin_entry_id IS NOT NULL", - "staleSeq": "seq 2 does not advance past the thread's latest entry" + "staleSeq": "seq 3 does not advance past the thread's latest entry" } diff --git a/packages/harness/test/__artifacts__/thread.fork.json b/packages/harness/test/__artifacts__/thread.fork.json index 07735f90..2149cd76 100644 --- a/packages/harness/test/__artifacts__/thread.fork.json +++ b/packages/harness/test/__artifacts__/thread.fork.json @@ -9,23 +9,29 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", - "5 session.llm.started.1 T1", - "6 session.llm.ended.1 T1", - "7 session.turn.ended.1 T1", - "8 session.prompt.admitted.1 T1", - "9 session.prompt.promoted.1 T1", - "10 session.llm.started.1 T1", - "11 session.llm.ended.1 T1", - "12 session.turn.ended.1 T1", - "13 session.thread.forked.1 T2", - "14 session.thread.forked.1 T3", - "15 session.thread.forked.1 T4", - "16 session.prompt.admitted.1 T1", - "17 session.prompt.promoted.1 T1", - "18 session.llm.started.1 T1", - "19 session.llm.failed.1 T1" + "3 session.execution.started.1 T1", + "4 session.prompt.promoted.1 T1", + "5 session.thread.system.changed.1 T1", + "6 session.llm.started.1 T1", + "7 session.llm.ended.1 T1", + "8 session.turn.ended.1 T1", + "9 session.execution.succeeded.1 T1", + "10 session.prompt.admitted.1 T1", + "11 session.execution.started.1 T1", + "12 session.prompt.promoted.1 T1", + "13 session.llm.started.1 T1", + "14 session.llm.ended.1 T1", + "15 session.turn.ended.1 T1", + "16 session.execution.succeeded.1 T1", + "17 session.thread.forked.1 T2", + "18 session.thread.forked.1 T3", + "19 session.thread.forked.1 T4", + "20 session.prompt.admitted.1 T1", + "21 session.execution.started.1 T1", + "22 session.prompt.promoted.1 T1", + "23 session.llm.started.1 T1", + "24 session.llm.failed.1 T1", + "25 session.execution.interrupted.1 T1" ], "threads": [ { @@ -37,30 +43,30 @@ { "thread": "T2", "origin": "fork T1 at E5", - "createdSeq": 13, + "createdSeq": 17, "title": null }, { "thread": "T3", "origin": "fork T1 at E1", - "createdSeq": 14, + "createdSeq": 18, "title": null }, { "thread": "T4", "origin": "fork T1 at E3", - "createdSeq": 15, + "createdSeq": 19, "title": null } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", - "E3 T1 assistant seq=5 committed=7 committed", - "E4 T1 user seq=9 committed=9 committed", - "E5 T1 assistant seq=10 committed=12 committed", - "E6 T1 user seq=17 committed=17 committed", - "E7 T1 assistant seq=18 committed=19 aborted" + "E1 T1 user seq=4 committed=4 committed", + "E2 T1 system seq=5 committed=5 committed", + "E3 T1 assistant seq=6 committed=8 committed", + "E4 T1 user seq=12 committed=12 committed", + "E5 T1 assistant seq=13 committed=15 committed", + "E6 T1 user seq=22 committed=22 committed", + "E7 T1 assistant seq=23 committed=24 aborted" ], "documents": [ { @@ -74,21 +80,21 @@ "thread": "T2", "kind": "agent", "revisions": [ - "13 {}" + "17 {}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "14 {}" + "18 {}" ] }, { "thread": "T4", "kind": "agent", "revisions": [ - "15 {}" + "19 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.lifecycle.json b/packages/harness/test/__artifacts__/thread.lifecycle.json new file mode 100644 index 00000000..2060ee4d --- /dev/null +++ b/packages/harness/test/__artifacts__/thread.lifecycle.json @@ -0,0 +1,139 @@ +{ + "log": [ + "3 session.execution.started.1 T1 {\"resumed\":false}", + "6 session.llm.started.1 T1", + "9 session.execution.succeeded.1 T1", + "11 session.execution.started.1 T1 {\"resumed\":false}", + "13 session.llm.started.1 T1", + "15 session.execution.interrupted.1 T1 {\"reason\":\"user\"}", + "17 session.execution.started.1 T1 {\"resumed\":false}", + "18 session.execution.interrupted.1 T1 {\"reason\":\"crash\"}", + "19 session.execution.started.1 T1 {\"resumed\":true}", + "20 session.llm.started.1 T1", + "24 session.llm.started.1 T1", + "27 session.execution.succeeded.1 T1", + "29 session.execution.started.1 T2 {\"resumed\":false}", + "30 session.execution.interrupted.1 T2 {\"reason\":\"shutdown\"}" + ], + "states": [ + { + "step": "created", + "thread": "T1", + "state": "idle", + "stateAt": null, + "startedAt": null, + "endedAt": null, + "outcome": null, + "error": null, + "execution": null + }, + { + "step": "succeeded", + "thread": "T1", + "state": "idle", + "stateAt": "9 session.execution.succeeded.1", + "startedAt": "3 session.execution.started.1", + "endedAt": "9 session.execution.succeeded.1", + "outcome": "succeeded", + "error": null, + "execution": null + }, + { + "step": "in flight", + "thread": "T1", + "state": "running", + "stateAt": "13 session.llm.started.1", + "startedAt": "11 session.execution.started.1", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "stopped", + "thread": "T1", + "state": "idle", + "stateAt": "15 session.execution.interrupted.1", + "startedAt": "11 session.execution.started.1", + "endedAt": "15 session.execution.interrupted.1", + "outcome": "interrupted", + "error": { + "_tag": "interrupted", + "reason": "user" + }, + "execution": null + }, + { + "step": "died", + "thread": "T1", + "state": "starting", + "stateAt": "17 session.execution.started.1", + "startedAt": "17 session.execution.started.1", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "swept", + "thread": "T1", + "state": "idle", + "stateAt": "18 session.execution.interrupted.1", + "startedAt": "17 session.execution.started.1", + "endedAt": "18 session.execution.interrupted.1", + "outcome": "interrupted", + "error": { + "_tag": "interrupted", + "reason": "crash" + }, + "execution": "running" + }, + { + "step": "resumed", + "thread": "T1", + "state": "running", + "stateAt": "20 session.llm.started.1", + "startedAt": "19 session.execution.started.1", + "endedAt": null, + "outcome": null, + "error": null, + "execution": "running" + }, + { + "step": "finished", + "thread": "T1", + "state": "idle", + "stateAt": "27 session.execution.succeeded.1", + "startedAt": "19 session.execution.started.1", + "endedAt": "27 session.execution.succeeded.1", + "outcome": "succeeded", + "error": null, + "execution": null + }, + { + "step": "swept after shutdown", + "thread": "T1", + "state": "idle", + "stateAt": "27 session.execution.succeeded.1", + "startedAt": "19 session.execution.started.1", + "endedAt": "27 session.execution.succeeded.1", + "outcome": "succeeded", + "error": null, + "execution": null + }, + { + "step": "swept after shutdown", + "thread": "T2", + "state": "idle", + "stateAt": "30 session.execution.interrupted.1", + "startedAt": "29 session.execution.started.1", + "endedAt": "30 session.execution.interrupted.1", + "outcome": "interrupted", + "error": { + "_tag": "interrupted", + "reason": "shutdown" + }, + "execution": "closed" + } + ] +} diff --git a/packages/harness/test/__artifacts__/thread.recovery.json b/packages/harness/test/__artifacts__/thread.recovery.json index f5bdc287..5d83c547 100644 --- a/packages/harness/test/__artifacts__/thread.recovery.json +++ b/packages/harness/test/__artifacts__/thread.recovery.json @@ -2,20 +2,24 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", - "5 session.llm.started.1 T1", - "6 session.llm.ended.1 T1", - "7 session.turn.ended.1 T1", - "8 session.thread.forked.1 T2", - "9 session.prompt.admitted.1 T2", - "10 session.prompt.promoted.1 T2", - "11 session.llm.started.1 T2", - "12 session.llm.failed.1 T2", - "13 session.thread.system.changed.1 T2", - "14 session.llm.started.1 T2", - "15 session.llm.ended.1 T2", - "16 session.turn.ended.1 T2" + "3 session.execution.started.1 T1", + "4 session.prompt.promoted.1 T1", + "5 session.thread.system.changed.1 T1", + "6 session.llm.started.1 T1", + "7 session.llm.ended.1 T1", + "8 session.turn.ended.1 T1", + "9 session.execution.succeeded.1 T1", + "10 session.thread.forked.1 T2", + "11 session.prompt.admitted.1 T2", + "12 session.prompt.promoted.1 T2", + "13 session.llm.started.1 T2", + "14 session.execution.started.1 T2", + "15 session.llm.failed.1 T2", + "16 session.thread.system.changed.1 T2", + "17 session.llm.started.1 T2", + "18 session.llm.ended.1 T2", + "19 session.turn.ended.1 T2", + "20 session.execution.succeeded.1 T2" ], "threads": [ { @@ -27,18 +31,18 @@ { "thread": "T2", "origin": "fork T1 at E1", - "createdSeq": 8, + "createdSeq": 10, "title": "cut off" } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", - "E3 T1 assistant seq=5 committed=7 committed", - "E4 T2 user seq=10 committed=10 committed", - "E5 T2 assistant seq=11 committed=12 aborted", - "E6 T2 system seq=13 committed=13 committed", - "E7 T2 assistant seq=14 committed=16 committed" + "E1 T1 user seq=4 committed=4 committed", + "E2 T1 system seq=5 committed=5 committed", + "E3 T1 assistant seq=6 committed=8 committed", + "E4 T2 user seq=12 committed=12 committed", + "E5 T2 assistant seq=13 committed=15 aborted", + "E6 T2 system seq=16 committed=16 committed", + "E7 T2 assistant seq=17 committed=19 committed" ], "documents": [ { @@ -52,7 +56,7 @@ "thread": "T2", "kind": "agent", "revisions": [ - "8 {}" + "10 {}" ] } ] diff --git a/packages/harness/test/__artifacts__/thread.reset.json b/packages/harness/test/__artifacts__/thread.reset.json index f26abe59..cc43d967 100644 --- a/packages/harness/test/__artifacts__/thread.reset.json +++ b/packages/harness/test/__artifacts__/thread.reset.json @@ -10,17 +10,21 @@ "events": [ "1 session.created.1 T1", "2 session.prompt.admitted.1 T1", - "3 session.prompt.promoted.1 T1", - "4 session.thread.system.changed.1 T1", - "5 session.llm.started.1 T1", - "6 session.llm.ended.1 T1", - "7 session.turn.ended.1 T1", - "9 session.prompt.admitted.1 T1", - "10 session.prompt.promoted.1 T1", - "11 session.thread.system.changed.1 T1", - "12 session.llm.started.1 T1", - "13 session.llm.ended.1 T1", - "14 session.turn.ended.1 T1" + "3 session.execution.started.1 T1", + "4 session.prompt.promoted.1 T1", + "5 session.thread.system.changed.1 T1", + "6 session.llm.started.1 T1", + "7 session.llm.ended.1 T1", + "8 session.turn.ended.1 T1", + "9 session.execution.succeeded.1 T1", + "11 session.prompt.admitted.1 T1", + "12 session.execution.started.1 T1", + "13 session.prompt.promoted.1 T1", + "14 session.thread.system.changed.1 T1", + "15 session.llm.started.1 T1", + "16 session.llm.ended.1 T1", + "17 session.turn.ended.1 T1", + "18 session.execution.succeeded.1 T1" ], "threads": [ { @@ -31,13 +35,13 @@ } ], "entries": [ - "E1 T1 user seq=3 committed=3 committed", - "E2 T1 system seq=4 committed=4 committed", - "E3 T1 assistant seq=5 committed=7 committed", - "E4 T1 reset seq=8 committed=8 committed", - "E5 T1 user seq=10 committed=10 committed", - "E6 T1 system seq=11 committed=11 committed", - "E7 T1 assistant seq=12 committed=14 committed" + "E1 T1 user seq=4 committed=4 committed", + "E2 T1 system seq=5 committed=5 committed", + "E3 T1 assistant seq=6 committed=8 committed", + "E4 T1 reset seq=10 committed=10 committed", + "E5 T1 user seq=13 committed=13 committed", + "E6 T1 system seq=14 committed=14 committed", + "E7 T1 assistant seq=15 committed=17 committed" ], "documents": [ { diff --git a/packages/harness/test/__artifacts__/thread.walkthrough.json b/packages/harness/test/__artifacts__/thread.walkthrough.json index fc5be405..381262f3 100644 --- a/packages/harness/test/__artifacts__/thread.walkthrough.json +++ b/packages/harness/test/__artifacts__/thread.walkthrough.json @@ -3,29 +3,35 @@ "1 session.created.1 T1", "2 session.thread.configured.1 T1", "3 session.prompt.admitted.1 T1", - "4 session.prompt.promoted.1 T1", - "5 session.thread.system.changed.1 T1", - "6 session.llm.started.1 T1", - "7 session.llm.ended.1 T1", - "8 session.thread.configured.1 T1", - "9 session.tool.settled.1 T1", - "10 session.turn.ended.1 T1", - "11 session.llm.started.1 T1", - "12 session.llm.ended.1 T1", - "13 session.turn.ended.1 T1", - "14 session.prompt.admitted.1 T1", - "15 session.prompt.promoted.1 T1", - "16 session.llm.started.1 T1", - "17 session.llm.ended.1 T1", - "18 session.turn.ended.1 T1", - "19 session.thread.forked.1 T2", - "20 session.prompt.admitted.1 T2", - "21 session.prompt.promoted.1 T2", - "22 session.llm.started.1 T2", - "23 session.llm.ended.1 T2", - "24 session.turn.ended.1 T2", - "25 session.thread.configured.1 T2", - "26 session.thread.forked.1 T3" + "4 session.execution.started.1 T1", + "5 session.prompt.promoted.1 T1", + "6 session.thread.system.changed.1 T1", + "7 session.llm.started.1 T1", + "8 session.llm.ended.1 T1", + "9 session.thread.configured.1 T1", + "10 session.tool.settled.1 T1", + "11 session.turn.ended.1 T1", + "12 session.llm.started.1 T1", + "13 session.llm.ended.1 T1", + "14 session.turn.ended.1 T1", + "15 session.execution.succeeded.1 T1", + "16 session.prompt.admitted.1 T1", + "17 session.execution.started.1 T1", + "18 session.prompt.promoted.1 T1", + "19 session.llm.started.1 T1", + "20 session.llm.ended.1 T1", + "21 session.turn.ended.1 T1", + "22 session.execution.succeeded.1 T1", + "23 session.thread.forked.1 T2", + "24 session.prompt.admitted.1 T2", + "25 session.execution.started.1 T2", + "26 session.prompt.promoted.1 T2", + "27 session.llm.started.1 T2", + "28 session.llm.ended.1 T2", + "29 session.turn.ended.1 T2", + "30 session.execution.succeeded.1 T2", + "31 session.thread.configured.1 T2", + "32 session.thread.forked.1 T3" ], "threads": [ { @@ -37,25 +43,25 @@ { "thread": "T2", "origin": "fork T1 at E4", - "createdSeq": 19, + "createdSeq": 23, "title": "Use OAuth instead" }, { "thread": "T3", "origin": "fork T1 at E3", - "createdSeq": 26, + "createdSeq": 32, "title": null } ], "entries": [ - "E1 T1 user seq=4 committed=4 committed", - "E2 T1 system seq=5 committed=5 committed", - "E3 T1 assistant seq=6 committed=10 committed", - "E4 T1 assistant seq=11 committed=13 committed", - "E5 T1 user seq=15 committed=15 committed", - "E6 T1 assistant seq=16 committed=18 committed", - "E7 T2 user seq=21 committed=21 committed", - "E8 T2 assistant seq=22 committed=24 committed" + "E1 T1 user seq=5 committed=5 committed", + "E2 T1 system seq=6 committed=6 committed", + "E3 T1 assistant seq=7 committed=11 committed", + "E4 T1 assistant seq=12 committed=14 committed", + "E5 T1 user seq=18 committed=18 committed", + "E6 T1 assistant seq=19 committed=21 committed", + "E7 T2 user seq=26 committed=26 committed", + "E8 T2 assistant seq=27 committed=29 committed" ], "documents": [ { @@ -64,22 +70,22 @@ "revisions": [ "1 {}", "2 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"}}", - "8 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "9 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T2", "kind": "agent", "revisions": [ - "19 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", - "25 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "23 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}", + "31 {\"model\":{\"provider\":\"anthropic\",\"id\":\"claude-sonnet-4-5\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] }, { "thread": "T3", "kind": "agent", "revisions": [ - "26 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" + "32 {\"model\":{\"provider\":\"openai\",\"id\":\"gpt-5.4\"},\"modelTools\":{\"add\":[\"web_fetch\"]}}" ] } ], diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json index c585146b..2835325c 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.fork.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.fork.json @@ -87,13 +87,13 @@ "entries": [ { "id": "E1", - "seq": 26, + "seq": 31, "type": "user", "state": "committed" }, { "id": "E2", - "seq": 27, + "seq": 32, "type": "system", "state": "committed", "data": { @@ -133,13 +133,13 @@ }, { "id": "E3", - "seq": 28, + "seq": 33, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 32, + "seq": 37, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.load.json b/packages/harness/test/__artifacts__/tools.toolsearch.load.json index 330d91e8..ceb21455 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.load.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.load.json @@ -114,71 +114,79 @@ }, { "seq": 3, - "type": "session.prompt.promoted.1" + "type": "session.execution.started.1" }, { "seq": 4, - "type": "session.thread.system.changed.1" + "type": "session.prompt.promoted.1" }, { "seq": 5, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 6, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 7, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 8, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 9, - "type": "session.thread.system.changed.1" + "type": "session.turn.ended.1" }, { "seq": 10, - "type": "session.llm.started.1" + "type": "session.thread.system.changed.1" }, { "seq": 11, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 12, - "type": "session.tool.settled.1" + "type": "session.llm.ended.1" }, { "seq": 13, - "type": "session.turn.ended.1" + "type": "session.tool.settled.1" }, { "seq": 14, - "type": "session.llm.started.1" + "type": "session.turn.ended.1" }, { "seq": 15, - "type": "session.llm.ended.1" + "type": "session.llm.started.1" }, { "seq": 16, + "type": "session.llm.ended.1" + }, + { + "seq": 17, "type": "session.turn.ended.1" + }, + { + "seq": 18, + "type": "session.execution.succeeded.1" } ], "entries": [ { "id": "E1", - "seq": 3, + "seq": 4, "type": "user", "state": "committed" }, { "id": "E2", - "seq": 4, + "seq": 5, "type": "system", "state": "committed", "data": { @@ -194,13 +202,13 @@ }, { "id": "E3", - "seq": 5, + "seq": 6, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 9, + "seq": 10, "type": "system", "state": "committed", "data": { @@ -240,13 +248,13 @@ }, { "id": "E5", - "seq": 10, + "seq": 11, "type": "assistant", "state": "committed" }, { "id": "E6", - "seq": 14, + "seq": 15, "type": "assistant", "state": "committed" } @@ -257,7 +265,7 @@ "value": {} }, { - "seq": 7, + "seq": 8, "value": { "modelTools": { "add": [ diff --git a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json index c8151453..378358d0 100644 --- a/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json +++ b/packages/harness/test/__artifacts__/tools.toolsearch.unavailable.json @@ -59,13 +59,13 @@ "entries": [ { "id": "E1", - "seq": 4, + "seq": 5, "type": "user", "state": "committed" }, { "id": "E2", - "seq": 5, + "seq": 6, "type": "system", "state": "committed", "data": { @@ -85,13 +85,13 @@ }, { "id": "E3", - "seq": 6, + "seq": 7, "type": "assistant", "state": "committed" }, { "id": "E4", - "seq": 12, + "seq": 13, "type": "assistant", "state": "committed" } diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index a1e56edf..1a2f6265 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -198,7 +198,8 @@ describe("runner loop โ€” aikit input/output", () => { yield* execution.resume(threadId); yield* execution.resume(threadId); - expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 3 }]); + // After session.created, prompt.admitted and execution.started. + expect(yield* delivered(threadId)).toEqual([{ id: "msg_once", promotedSeq: 4 }]); expect((yield* threads.line(threadId)).map((item) => item.entry.type)).toEqual([ "user", "system", @@ -266,8 +267,13 @@ describe("runner loop โ€” aikit input/output", () => { const failure = Cause.findErrorOption(exit.cause); expect(Option.isSome(failure) && failure.value._tag).toBe("Runner.SandboxDirectoryNotFoundError"); } - const durable = yield* sql`SELECT type FROM event WHERE aggregate_id = ${session.id}`; - expect(durable).toEqual([{ type: "session.created.1" }]); + const durable = yield* sql`SELECT type FROM event WHERE aggregate_id = ${session.id} ORDER BY seq`; + // Only the run's own lifecycle: nothing of the loop. + expect(durable).toEqual([ + { type: "session.created.1" }, + { type: "session.execution.started.1" }, + { type: "session.execution.failed.1" }, + ]); }), ); }); diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index 60b2383d..c04b9741 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -1,6 +1,6 @@ import "./utils/env.ts"; import { createAssistantMessageEventStream, Message } from "@codeworksh/aikit"; -import { Cause, DateTime, Deferred, Effect, Exit, Layer, Option, Schema } from "effect"; +import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema } from "effect"; import { SqlClient } from "effect/sql"; import { describe, expect } from "vite-plus/test"; import { Context } from "../src/context/context.ts"; @@ -10,6 +10,7 @@ import { Document } from "../src/document/document.ts"; import { Event } from "../src/event/event.ts"; import { EventList } from "../src/event/list.ts"; import { EventRegistry } from "../src/event/registry.ts"; +import { Inbox } from "../src/inbox/inbox.ts"; import { Global } from "../src/global.ts"; import { builtins } from "../src/plugin/builtin.ts"; import type { Plugin } from "../src/plugin/plugin.ts"; @@ -616,4 +617,169 @@ describe("threads", () => { }); }), ); + + it( + "the stored run follows a run, a stop, a dead process, the startup sweep and the resume", + Effect.gen(function* () { + yield* reset; + const threads = yield* Thread.Service; + const control = yield* Control.Service; + const events = yield* Event.Service; + const inputs = yield* Inbox.make; + const sql = yield* SqlClient.SqlClient; + const { sessionId, main } = yield* createSession; + + const lifecycle = [ + "session.execution.started.1", + "session.execution.succeeded.1", + "session.execution.failed.1", + "session.execution.interrupted.1", + "session.run.failed.1", + "session.llm.started.1", + ]; + const journal = Effect.gen(function* () { + const rows = yield* sql<{ seq: number; type: string; data: string }>` + SELECT seq, type, data FROM event WHERE aggregate_id = ${sessionId} ORDER BY seq + `; + return rows + .filter((row) => lifecycle.includes(row.type)) + .map((row) => ({ seq: row.seq, type: row.type, data: JSON.parse(row.data) as Record })); + }); + const name = aliases(); + name("T", main); + // A stored time, named by the thread's event of `types` it came from: the latest one at that instant. + const at = ( + log: ReadonlyArray<{ seq: number; type: string; data: Record }>, + threadId: string, + time: Option.Option, + types: ReadonlyArray, + ) => + Option.match(time, { + onNone: () => null, + onSome: (value) => { + const from = log.findLast( + (event) => + event.data["threadId"] === threadId && + types.includes(event.type) && + event.data["timestamp"] === DateTime.toEpochMillis(value), + ); + return from === undefined ? `unmatched ${DateTime.toEpochMillis(value)}` : `${from.seq} ${from.type}`; + }, + }); + const terminals = lifecycle.slice(1, 5); + const stateTypes = { starting: [lifecycle[0]!], running: ["session.llm.started.1"], idle: terminals }; + const states: Array = []; + const record = (step: string) => + Effect.gen(function* () { + const log = yield* journal; + const rows = [...(yield* threads.list(sessionId))].sort((a, b) => + name("T", a.id).localeCompare(name("T", b.id)), + ); + for (const row of rows) { + states.push({ + step, + thread: name("T", row.id), + state: row.runState, + stateAt: at(log, row.id, row.runStateAt, stateTypes[row.runState]), + startedAt: at(log, row.id, row.runStartedAt, [lifecycle[0]!]), + endedAt: at(log, row.id, row.runEndedAt, terminals), + outcome: Option.getOrNull(row.runOutcome), + error: Option.getOrNull(row.runError), + execution: Option.getOrNull(row.execution), + }); + } + }); + const gated = (threadId: ThreadSchema.ID) => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + const opened = yield* Deferred.make(); + script.set(threadId, [{ _tag: "gate", gate, opened }]); + // Held until its draft is in: `llm.started` committed. + const drafted = Deferred.await(opened).pipe( + Effect.andThen(threads.draft(threadId)), + Effect.tap(() => Effect.yieldNow), + Effect.repeat({ until: Option.isSome }), + ); + return { gate, drafted }; + }); + + yield* record("created"); + yield* run(sessionId, main, "first"); + yield* record("succeeded"); + + const second = yield* gated(main); + yield* control.prompt({ sessionId, threadId: main, prompt: PromptSchema.Prompt.fromText("second") }); + yield* second.drafted; + yield* record("in flight"); + yield* control.interrupt(main, { awaitSettlement: true }); + yield* record("stopped"); + + // What a process dying mid-run leaves: a follow-up admitted, its run started and marked, never ended. + const died = (threadId: ThreadSchema.ID) => + Effect.gen(function* () { + yield* events.publish(EventList.ExecutionStarted, { + timestamp: yield* DateTime.now, + sessionId, + threadId, + resumed: false, + }); + yield* threads.markRunning(threadId); + }); + yield* inputs.admit({ + id: SessionMessageSchema.ID.make("prompt_cut"), + sessionId, + threadId: main, + prompt: PromptSchema.Prompt.fromText("third"), + delivery: "followUp", + }); + yield* died(main); + yield* record("died"); + + yield* control.sweep; + yield* record("swept"); + const swept = Option.getOrThrow(yield* threads.get(main)); + expect(swept.runState).toBe("idle"); + expect(Option.getOrUndefined(swept.runOutcome)).toBe("interrupted"); + expect(Option.getOrUndefined(swept.runError)).toEqual({ _tag: "interrupted", reason: "crash" }); + // Nothing is left open, so a second sweep writes nothing. + const before = (yield* journal).length; + yield* control.sweep; + expect((yield* journal).length).toBe(before); + + const third = yield* gated(main); + const resuming = yield* Effect.forkChild(control.resumeUnfinished); + yield* third.drafted; + yield* record("resumed"); + expect(Option.getOrThrow(yield* threads.get(main)).runState).toBe("running"); + yield* Deferred.succeed(third.gate, undefined); + yield* Fiber.join(resuming); + yield* record("finished"); + + // A shutdown that closed the run and died before publishing its end: the sweep says `shutdown`. + const [first] = yield* threads.line(main); + const fork = ThreadSchema.ID.make( + (yield* threads.fork({ threadId: main, entryId: first!.entry.id })).thread.id, + ); + name("T", fork); + yield* died(fork); + yield* threads.endExecution(fork, "closed"); + yield* control.sweep; + yield* record("swept after shutdown"); + expect(Option.getOrUndefined(Option.getOrThrow(yield* threads.get(fork)).runError)).toEqual({ + _tag: "interrupted", + reason: "shutdown", + }); + + const log = yield* journal; + const resumed = log.filter((event) => event.type === lifecycle[0] && event.data["resumed"] === true); + expect(resumed.map((event) => event.data["threadId"])).toEqual([main]); + yield* artifact("thread.lifecycle.json", { + log: log.map((event) => { + const { timestamp: _t, sessionId: _s, threadId, message: _m, messageId: _id, ...rest } = event.data; + return `${event.seq} ${event.type} ${name("T", String(threadId))}${Object.keys(rest).length === 0 ? "" : ` ${JSON.stringify(rest)}`}`; + }), + states, + }); + }), + ); }); From 10bd78540598100a7da745ecd3d4d800a51d74bd Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 18:56:32 +0530 Subject: [PATCH 03/56] feat(harness): store-wide revision and tombstones A single-row store table holds the database's id and its last revision. Project, space, session, thread, entry and inbox rows carry the rev of the write transaction that last touched them, enforced by insert and update triggers. Store.write runs a write transaction (or joins the enclosing one), Store.rev takes its revision on first use, and a commit that took one rings a store-wide doorbell (Store.wake). Every write path goes through it: event projectors, plain-SQL thread, session, project and space writes. Part writes stamp their entry, document writes stamp the threads they configure, and deleting a session leaves a removed row. Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/db/db.ts | 5 +- packages/harness/src/db/migrations.ts | 52 ++- packages/harness/src/db/schema.sql.ts | 9 + packages/harness/src/db/store.ts | 84 ++++ packages/harness/src/document/document.ts | 94 +++-- packages/harness/src/effect.ts | 1 + packages/harness/src/event/event.ts | 5 +- packages/harness/src/inbox/inbox.ts | 226 ++++++----- packages/harness/src/project/project.ts | 26 +- packages/harness/src/runner/execute.ts | 3 + packages/harness/src/sandbox/control.ts | 8 +- packages/harness/src/session/session.ts | 97 +++-- packages/harness/src/space/space.ts | 117 +++--- packages/harness/src/thread/thread.ts | 366 ++++++++++-------- .../test/__artifacts__/store.revision.json | 271 +++++++++++++ packages/harness/test/db/db.test.ts | 28 +- packages/harness/test/fixtures/space.ts | 23 +- packages/harness/test/project.test.ts | 4 +- packages/harness/test/runner.loop.test.ts | 7 +- packages/harness/test/sandbox.destroy.test.ts | 6 +- .../harness/test/sandbox.instance.test.ts | 14 +- .../harness/test/session.input.flow.test.ts | 4 +- packages/harness/test/space.test.ts | 6 +- packages/harness/test/store.test.ts | 344 ++++++++++++++++ packages/harness/test/thread.test.ts | 16 +- 25 files changed, 1375 insertions(+), 441 deletions(-) create mode 100644 packages/harness/src/db/store.ts create mode 100644 packages/harness/test/__artifacts__/store.revision.json create mode 100644 packages/harness/test/store.test.ts diff --git a/packages/harness/src/db/db.ts b/packages/harness/src/db/db.ts index 86686968..a8a12c91 100644 --- a/packages/harness/src/db/db.ts +++ b/packages/harness/src/db/db.ts @@ -5,6 +5,7 @@ import { Global } from "../global.ts"; import { fileSystem } from "../host.ts"; import { posix } from "../util/posix.ts"; import { migrations } from "./migrations.ts"; +import { Store } from "./store.ts"; export { SqlClient, SqlSchema } from "effect/sql"; @@ -26,7 +27,8 @@ const setup = Effect.gen(function* () { }); // Provides `SqlClient` (and `SqliteClient`) for the database at `location`, -// with column-name transforms so camelCase fields map to snake_case columns. +// with column-name transforms so camelCase fields map to snake_case columns, +// and the database's `Store`: one per database, so its doorbell hears every write. export function layer(location: string) { const client = Layer.unwrap( Effect.gen(function* () { @@ -43,6 +45,7 @@ export function layer(location: string) { return client.pipe( Layer.tap((context) => setup.pipe(Effect.provide(context))), Layer.orDie, + (sqlite) => Layer.provideMerge(Store.layer, sqlite), ); } diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index e3c9f3a1..73fbb27c 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -1,5 +1,6 @@ import { Effect } from "effect"; import { SqlClient } from "effect/sql"; +import { uuidv7 } from "uuidv7"; // Migrations ship as code (Migrator.fromRecord) rather than .sql files on // disk, so they survive bundling and need no runtime path resolution. @@ -16,6 +17,27 @@ export const migrations = { "202610290001_init": Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // One row: the store's identity and its last revision taken (db/store.ts). A recreated database + // gets a new id; a restored backup keeps its id with a lower rev. + yield* sql` + CREATE TABLE store ( + id TEXT NOT NULL, + rev INTEGER NOT NULL + ) + `; + yield* sql`INSERT INTO store (id, rev) VALUES (${uuidv7()}, 0)`; + + // A deleted row, at the revision that deleted it. Kept, never pruned. + yield* sql` + CREATE TABLE removed ( + kind TEXT NOT NULL, + id TEXT NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (kind, id) + ) + `; + yield* sql`CREATE INDEX removed_rev_idx ON removed (rev)`; + // Head of one aggregate's log (aggregate = session), and who holds it. yield* sql` CREATE TABLE sequence ( @@ -73,7 +95,8 @@ export const migrations = { name TEXT NOT NULL, status TEXT NOT NULL, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + rev INTEGER NOT NULL ) `; @@ -87,7 +110,8 @@ export const migrations = { env TEXT REFERENCES sandbox(id) ON DELETE RESTRICT CHECK (env IS NULL OR env <> 'local'), status TEXT NOT NULL, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + rev INTEGER NOT NULL ) `; yield* sql`CREATE UNIQUE INDEX space_location_idx ON space (coalesce(env, 'local'), location)`; @@ -106,7 +130,8 @@ export const migrations = { tag TEXT, metadata TEXT, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + rev INTEGER NOT NULL ) `; yield* sql`CREATE UNIQUE INDEX session_slug_idx ON session (slug)`; @@ -133,6 +158,7 @@ export const migrations = { created_seq INTEGER NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, + rev INTEGER NOT NULL, UNIQUE (session_id, id), -- the source thread lives in the same session FOREIGN KEY (session_id, origin_thread_id) REFERENCES thread(session_id, id), @@ -169,6 +195,7 @@ export const migrations = { metadata TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, + rev INTEGER NOT NULL, UNIQUE (thread_id, id), FOREIGN KEY (session_id, thread_id) REFERENCES thread(session_id, id) ON DELETE CASCADE, -- the fork cut reads committed_seq: set exactly when the entry stops changing @@ -218,6 +245,7 @@ export const migrations = { promoted_seq INTEGER, cancelled_seq INTEGER, created_at INTEGER NOT NULL, + rev INTEGER NOT NULL, FOREIGN KEY (session_id, thread_id) REFERENCES thread(session_id, id) ON DELETE CASCADE ) `; @@ -254,5 +282,23 @@ export const migrations = { PRIMARY KEY (document_id, seq) ) `; + + yield* sql`CREATE INDEX project_rev_idx ON project (rev)`; + yield* sql`CREATE INDEX space_rev_idx ON space (rev)`; + yield* sql`CREATE INDEX session_rev_idx ON session (rev)`; + yield* sql`CREATE INDEX thread_rev_idx ON thread (rev)`; + yield* sql`CREATE INDEX entry_thread_rev_idx ON entry (thread_id, rev)`; + yield* sql`CREATE INDEX inbox_thread_rev_idx ON inbox (thread_id, rev)`; + // Every insert and update carries the current transaction's revision, so a write path that + // forgets fails loudly. A row written twice in one transaction keeps its rev, which this allows. + for (const table of ["project", "space", "session", "thread", "entry", "inbox"]) { + for (const event of ["insert", "update"]) { + yield* sql.unsafe(` + CREATE TRIGGER ${table}_rev_${event} BEFORE ${event.toUpperCase()} ON ${table} + WHEN NEW.rev <> (SELECT rev FROM store) + BEGIN SELECT RAISE(ABORT, 'rev is not the current revision'); END + `); + } + } }), }; diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index 6213f195..cc35b8a0 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -21,6 +21,9 @@ export const Timestamps = { const Metadata = Schema.Record(Schema.String, Schema.String); +// The revision of the write transaction that last wrote the row (db/store.ts). +const Rev = NonNegativeInt; + // Session entry discriminant. Frozen once shipped โ€” values live in persisted // rows, so a rename is a data migration. Additions are code-only (the column is unconstrained TEXT). export const entryTypes = ["user", "assistant", "synthetic", "system", "compaction", "reset", "custom"] as const; @@ -74,6 +77,7 @@ export class ProjectRow extends Model.Class("ProjectRow")({ name: Schema.String, status: ProjectSchema.Status, ...Timestamps, + rev: Rev, }) {} /** @@ -115,6 +119,7 @@ export class SpaceRow extends Model.Class("SpaceRow")({ env: Model.FieldOption(SandboxInstance.ID), status: SpaceSchema.Status, ...Timestamps, + rev: Rev, }) {} export class SessionRow extends Model.Class("SessionRow")({ @@ -129,6 +134,7 @@ export class SessionRow extends Model.Class("SessionRow")({ tag: Model.FieldOption(Schema.String), metadata: Model.FieldOption(Model.JsonFromString(Metadata)), ...Timestamps, + rev: Rev, }) {} // One linear transcript of a session. `origin*` is the flattened `ThreadSchema.Origin`. @@ -154,6 +160,7 @@ export class ThreadRow extends Model.Class("ThreadRow")({ tokensCacheWrite: Model.GeneratedByDb(Schema.Int), createdSeq: NonNegativeInt, ...Timestamps, + rev: Rev, }) {} // Work addressed to one thread, from admission to delivery. `kind` + `data` is the flattened item; @@ -169,6 +176,7 @@ export class InboxRow extends Model.Class("InboxRow")({ promotedSeq: Model.FieldOption(Schema.Int), cancelledSeq: Model.FieldOption(Schema.Int), createdAt: Model.DateTimeInsertFromNumber, + rev: Rev, }) {} // One transcript record in one thread. Immutable except a draft assistant, whose `data` and @@ -188,6 +196,7 @@ export class EntryRow extends Model.Class("EntryRow")({ label: Model.FieldOption(Schema.String), metadata: Model.FieldOption(Model.JsonFromString(Metadata)), ...Timestamps, + rev: Rev, }) {} // One aikit message part. `data` is the verbatim aikit part JSON and is authoritative; diff --git a/packages/harness/src/db/store.ts b/packages/harness/src/db/store.ts new file mode 100644 index 00000000..b950f13f --- /dev/null +++ b/packages/harness/src/db/store.ts @@ -0,0 +1,84 @@ +import { Context, Effect, Layer, PubSub, type Scope } from "effect"; +import { SqlClient, type SqlError } from "effect/sql"; + +/** The store's identity and its last revision taken. */ +export interface Head { + readonly id: string; + readonly rev: number; +} + +export interface Interface { + /** + * Runs `effect` as one write transaction, or inside the enclosing one. Every write to project, + * space, session, thread, entry or inbox goes through it. After a commit that took a revision, + * the doorbell rings. + */ + readonly write: (effect: Effect.Effect) => Effect.Effect; + /** The enclosing write's revision, taken on first use; every row the write touches is stamped with it. */ + readonly rev: Effect.Effect; + readonly head: Effect.Effect; + /** + * A doorbell for the caller's scope, rung after every commit that took a revision. Register it + * before reading `head`, so no commit falls between the read and the first ring. + */ + readonly wake: Effect.Effect, never, Scope.Scope>; +} + +export class Service extends Context.Service()("@codeworksh/harness/db/store/Service") {} + +// The revision taken by the write transaction in progress; absent outside `write`. +const Taken = Context.Reference<{ rev: number | undefined } | undefined>("@codeworksh/harness/db/store/Taken", { + defaultValue: () => undefined, +}); + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const wakes = new Set>(); + + const ring = Effect.forEach(wakes, (wake) => PubSub.publish(wake, undefined), { discard: true }); + + const write = (effect: Effect.Effect): Effect.Effect => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + if ((yield* Taken) !== undefined) return yield* restore(effect); + const taken: { rev: number | undefined } = { rev: undefined }; + const result = yield* sql.withTransaction(restore(Effect.provideService(effect, Taken, taken))); + if (taken.rev !== undefined) yield* ring; + return result; + }), + ); + + const rev = Effect.gen(function* () { + const taken = yield* Taken; + if (taken === undefined) return yield* Effect.die(new Error("Store.rev outside Store.write")); + if (taken.rev === undefined) { + const [row] = yield* sql<{ readonly rev: number }>`UPDATE store SET rev = rev + 1 RETURNING rev`; + if (row === undefined) return yield* Effect.die(new Error("store row is missing")); + taken.rev = row.rev; + } + return taken.rev; + }).pipe(Effect.orDie); + + const head = Effect.gen(function* () { + const [row] = yield* sql`SELECT id, rev FROM store`; + if (row === undefined) return yield* Effect.die(new Error("store row is missing")); + return row; + }).pipe(Effect.orDie); + + const wake = Effect.gen(function* () { + const pubsub = yield* PubSub.sliding(1); + const subscription = yield* PubSub.subscribe(pubsub); + yield* Effect.acquireRelease( + Effect.sync(() => wakes.add(pubsub)), + () => Effect.sync(() => wakes.delete(pubsub)).pipe(Effect.andThen(PubSub.shutdown(pubsub))), + ); + return subscription; + }); + + return Service.of({ write, rev, head, wake }); + }), +); + +export * as Store from "./store.ts"; diff --git a/packages/harness/src/document/document.ts b/packages/harness/src/document/document.ts index b932421f..dd32eb84 100644 --- a/packages/harness/src/document/document.ts +++ b/packages/harness/src/document/document.ts @@ -2,6 +2,7 @@ import { Context, Effect, Layer, Option, Schema } from "effect"; import { SqlClient, SqlSchema } from "effect/sql"; import { uuidv7 } from "uuidv7"; import { DocumentRevisionRow, DocumentRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import type { SessionSchema } from "../session/schema.ts"; import type { ThreadSchema } from "../thread/schema.ts"; import { DocumentSchema } from "./schema.ts"; @@ -56,6 +57,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const insertDocument = SqlSchema.void({ Request: DocumentRow.insert, @@ -98,30 +100,46 @@ export const layer = Layer.effect( value: row.value, }); - const write = Effect.fn("Document.write")(function* (documentId: DocumentSchema.ID, revision: Revision) { - yield* insertRevision({ documentId, ...revision }); - const document = yield* findById(documentId); - if (Option.isSome(document) && Option.getOrUndefined(document.value.history) === "latest") { - yield* sql`DELETE FROM revision WHERE document_id = ${documentId} AND seq < ${revision.seq}`; - } - }, Effect.orDie); - - const create = Effect.fn("Document.create")(function* (input: CreateInput) { - const { definition } = input; - const row = DocumentRow.insert.make({ - id: DocumentSchema.ID.make(uuidv7()), - sessionId: input.sessionId, - threadId: Option.fromUndefinedOr(input.threadId), - kind: definition.kind, - history: Option.fromUndefinedOr(definition.history), - fork: Option.fromUndefinedOr(definition.fork), - createdSeq: input.seq, - retiredSeq: Option.none(), - }); - yield* insertDocument(row); - yield* write(row.id, { seq: input.seq, version: definition.version, value: input.value }); - return new DocumentRow(row); - }, Effect.orDie); + // A document configures threads, so writing one stamps them: the thread for a thread-scoped + // document, every thread of the session for a session-scoped one. + const write = Effect.fn("Document.write")( + function* (documentId: DocumentSchema.ID, revision: Revision) { + yield* insertRevision({ documentId, ...revision }); + const document = yield* findById(documentId); + if (Option.isNone(document)) return yield* Effect.die(`document ${documentId} not found`); + if (Option.getOrUndefined(document.value.history) === "latest") { + yield* sql`DELETE FROM revision WHERE document_id = ${documentId} AND seq < ${revision.seq}`; + } + const rev = yield* store.rev; + yield* Option.match(document.value.threadId, { + onNone: () => sql`UPDATE thread SET rev = ${rev} WHERE session_id = ${document.value.sessionId}`, + onSome: (threadId) => sql`UPDATE thread SET rev = ${rev} WHERE id = ${threadId}`, + }); + }, + store.write, + Effect.orDie, + ); + + const create = Effect.fn("Document.create")( + function* (input: CreateInput) { + const { definition } = input; + const row = DocumentRow.insert.make({ + id: DocumentSchema.ID.make(uuidv7()), + sessionId: input.sessionId, + threadId: Option.fromUndefinedOr(input.threadId), + kind: definition.kind, + history: Option.fromUndefinedOr(definition.history), + fork: Option.fromUndefinedOr(definition.fork), + createdSeq: input.seq, + retiredSeq: Option.none(), + }); + yield* insertDocument(row); + yield* write(row.id, { seq: input.seq, version: definition.version, value: input.value }); + return new DocumentRow(row); + }, + store.write, + Effect.orDie, + ); const find = Effect.fn("Document.find")(function* (threadId: string, kind: string) { return yield* findLive({ threadId, kind }); @@ -148,20 +166,20 @@ export const layer = Layer.effect( const agent = Effect.fn("Document.agent")((threadId: string) => readAgent(threadId, Number.MAX_SAFE_INTEGER)); const agentAsOf = Effect.fn("Document.agentAsOf")(readAgent); - const configure = Effect.fn("Document.configure")(function* ( - threadId: string, - change: DocumentSchema.AgentChange, - seq: number, - ) { - const document = yield* find(threadId, DocumentSchema.agent.kind); - if (Option.isNone(document)) return yield* Effect.die(`thread ${threadId} has no agent document`); - const next = DocumentSchema.applyAgentChange(yield* readAgent(threadId, Number.MAX_SAFE_INTEGER), change); - yield* write(document.value.id, { - seq, - version: DocumentSchema.agent.version, - value: yield* encodeAgent(next).pipe(Effect.orDie), - }); - }); + const configure = Effect.fn("Document.configure")( + function* (threadId: string, change: DocumentSchema.AgentChange, seq: number) { + const document = yield* find(threadId, DocumentSchema.agent.kind); + if (Option.isNone(document)) return yield* Effect.die(`thread ${threadId} has no agent document`); + const next = DocumentSchema.applyAgentChange(yield* readAgent(threadId, Number.MAX_SAFE_INTEGER), change); + yield* write(document.value.id, { + seq, + version: DocumentSchema.agent.version, + value: yield* encodeAgent(next).pipe(Effect.orDie), + }); + }, + store.write, + Effect.orDie, + ); return Service.of({ create, find, ofThread, current, asOf, write, agent, agentAsOf, configure }); }), diff --git a/packages/harness/src/effect.ts b/packages/harness/src/effect.ts index adb237dc..fad546d3 100644 --- a/packages/harness/src/effect.ts +++ b/packages/harness/src/effect.ts @@ -3,6 +3,7 @@ export { Sandbox } from "./effect/sandbox.ts"; export { Session } from "./effect/session.ts"; export { Control } from "./control.ts"; export { Database } from "./db/db.ts"; +export { Store } from "./db/store.ts"; export { Event } from "./event/event.ts"; export { EventList } from "./event/list.ts"; export { EventRegistry } from "./event/registry.ts"; diff --git a/packages/harness/src/event/event.ts b/packages/harness/src/event/event.ts index 808f9b80..a6558a63 100644 --- a/packages/harness/src/event/event.ts +++ b/packages/harness/src/event/event.ts @@ -1,6 +1,7 @@ import { Cause, Context, Effect, Layer, Option, PubSub, Queue, Ref, Stream } from "effect"; import { SqlClient, SqlSchema } from "effect/sql"; import { EventRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; // Uncomment with `decodeSerializedEvent` below โ€” the manifest is what it looks // stored types up in. // import { Durable } from "./manifest.ts"; @@ -209,6 +210,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const pubsub = { // Per-aggregate doorbells for `log`. A wake carries no payload -- the @@ -479,7 +481,8 @@ export const layer = Layer.effect( }), ); const encoded = (yield* Schema.encodeUnknownEffect(definition.data)(event.data)) as Record; - const committed = yield* sql.withTransaction( + // The projectors write the views, so the commit is a store write: one revision for all of them. + const committed = yield* store.write( Effect.gen(function* () { const { seq } = yield* bumpSequence(aggregateId); const existing = yield* findEventById(event.id); diff --git a/packages/harness/src/inbox/inbox.ts b/packages/harness/src/inbox/inbox.ts index ba95065e..da2798c7 100644 --- a/packages/harness/src/inbox/inbox.ts +++ b/packages/harness/src/inbox/inbox.ts @@ -2,6 +2,7 @@ import { DateTime, Effect, Option, Schema } from "effect"; import { VariantSchema } from "effect/schema"; import { SqlClient, SqlSchema } from "effect/sql"; import { InboxRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; import { SessionMessageSchema } from "../session/message/schema.ts"; @@ -156,6 +157,7 @@ export interface Interface { export const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const events = yield* Event.Service; const findRow = SqlSchema.findOneOption({ @@ -182,11 +184,12 @@ export const make = Effect.gen(function* () { threadId: Schema.String, data: Schema.NullOr(Schema.fromJsonString(Prompt)), delivery: Schema.NullOr(Delivery), + rev: Schema.Int, }), Result: Schema.Struct({ id: Schema.String }), // A null field is one the update left alone. - execute: ({ id, threadId, data, delivery }) => sql` - UPDATE inbox SET data = COALESCE(${data}, data), delivery = COALESCE(${delivery}, delivery) + execute: ({ id, threadId, data, delivery, rev }) => sql` + UPDATE inbox SET data = COALESCE(${data}, data), delivery = COALESCE(${delivery}, delivery), rev = ${rev} WHERE id = ${id} AND thread_id = ${threadId} AND promoted_seq IS NULL AND cancelled_seq IS NULL RETURNING id `, @@ -201,12 +204,13 @@ export const make = Effect.gen(function* () { delivery: Delivery, admittedSeq: Schema.Int, createdAt: DateTimeUtcFromMillis, + rev: Schema.Int, }), Result: Schema.Struct({ id: Schema.String }), - execute: ({ id, threadId, data, delivery, admittedSeq, createdAt }) => sql` + execute: ({ id, threadId, data, delivery, admittedSeq, createdAt, rev }) => sql` UPDATE inbox SET data = ${data}, delivery = ${delivery}, admitted_seq = ${admittedSeq}, created_at = ${createdAt}, - cancelled_seq = NULL + cancelled_seq = NULL, rev = ${rev} WHERE id = ${id} AND thread_id = ${threadId} AND promoted_seq IS NULL AND cancelled_seq IS NOT NULL RETURNING id `, @@ -219,10 +223,11 @@ export const make = Effect.gen(function* () { id: Schema.String, threadId: Schema.String, promotedSeq: Schema.Int, + rev: Schema.Int, }), Result: InboxRow, - execute: ({ id, threadId, promotedSeq }) => sql` - UPDATE inbox SET promoted_seq = ${promotedSeq} + execute: ({ id, threadId, promotedSeq, rev }) => sql` + UPDATE inbox SET promoted_seq = ${promotedSeq}, rev = ${rev} WHERE id = ${id} AND thread_id = ${threadId} AND promoted_seq IS NULL AND cancelled_seq IS NULL RETURNING * `, @@ -230,10 +235,10 @@ export const make = Effect.gen(function* () { // The same compare-and-set as promotion: whichever of the two commits first wins the row. const cancelRow = SqlSchema.findAll({ - Request: Schema.Struct({ id: Schema.String, threadId: Schema.String, cancelledSeq: Schema.Int }), + Request: Schema.Struct({ id: Schema.String, threadId: Schema.String, cancelledSeq: Schema.Int, rev: Schema.Int }), Result: Schema.Struct({ id: Schema.String }), - execute: ({ id, threadId, cancelledSeq }) => sql` - UPDATE inbox SET cancelled_seq = ${cancelledSeq} + execute: ({ id, threadId, cancelledSeq, rev }) => sql` + UPDATE inbox SET cancelled_seq = ${cancelledSeq}, rev = ${rev} WHERE id = ${id} AND thread_id = ${threadId} AND promoted_seq IS NULL AND cancelled_seq IS NULL RETURNING id `, @@ -343,97 +348,118 @@ export const make = Effect.gen(function* () { ); }); - const projectAdmitted = Effect.fn("Inbox.projectAdmitted")(function* (input: ProjectAdmittedInput) { - // Safe to read first here: the commit transaction already opened with the - // event's sequence bump, so this cannot upgrade a read lock. - const entry = yield* selectEntry(input.id); - if (entry.length > 0) return yield* Effect.die(new LifecycleConflict({ id: input.id })); - const stored = yield* insertRow({ - id: input.id, - sessionId: input.sessionId, - threadId: input.threadId, - kind: "prompt", - data: input.prompt, - delivery: input.delivery, - admittedSeq: input.admittedSeq, - promotedSeq: Option.none(), - cancelledSeq: Option.none(), - createdAt: VariantSchema.Override(input.timeCreated), - }); - if (stored.length > 0) return; - const revived = yield* reviveRow({ - id: input.id, - threadId: input.threadId, - data: input.prompt, - delivery: input.delivery, - admittedSeq: input.admittedSeq, - createdAt: input.timeCreated, - }); - if (revived.length === 0) return yield* Effect.die(new LifecycleConflict({ id: input.id })); - }, Effect.orDie); - - const projectPrompted = Effect.fn("Inbox.projectPrompted")(function* (input: ProjectPromptedInput) { - const updated = yield* promoteRow({ - id: input.id, - threadId: input.threadId, - promotedSeq: input.promotedSeq, - }); - const promoted = updated[0]; - if (promoted) { - if (!matchesProjection(fromRow(promoted), input)) - return yield* Effect.die(new LifecycleConflict({ id: input.id })); - return; - } - - // Nothing moved: either this exact promotion is being replayed, or the row - // is already promoted at a different sequence, which is a conflict. - const stored = yield* find(input.id); - if (Option.isSome(stored)) { - if ( - !matchesProjection(stored.value, input) || - stored.value.promotedSeq !== input.promotedSeq || - stored.value.cancelledSeq !== undefined - ) - return yield* Effect.die(new LifecycleConflict({ id: input.id })); - return; - } - - // Promotion of an input this store never admitted -- replayed onto an empty - // projection. Both sequences collapse onto the promotion. - yield* insertRow({ - id: input.id, - sessionId: input.sessionId, - threadId: input.threadId, - kind: "prompt", - data: input.prompt, - delivery: input.delivery, - admittedSeq: input.promotedSeq, - promotedSeq: Option.some(input.promotedSeq), - cancelledSeq: Option.none(), - createdAt: VariantSchema.Override(input.timeCreated), - }); - }, Effect.orDie); - - const projectCancelled = Effect.fn("Inbox.projectCancelled")(function* (input: ProjectCancelledInput) { - const updated = yield* cancelRow(input); - if (updated.length > 0) return; - // Nothing moved: this exact cancel being replayed, or the row was promoted or cancelled first. - const stored = yield* find(input.id); - if (Option.isSome(stored) && stored.value.cancelledSeq === input.cancelledSeq) return; - return yield* Effect.die(new LifecycleConflict({ id: input.id })); - }, Effect.orDie); - - const projectUpdated = Effect.fn("Inbox.projectUpdated")(function* (input: ProjectUpdatedInput) { - const updated = yield* updateRow({ - id: input.id, - threadId: input.threadId, - data: input.prompt ?? null, - delivery: input.delivery ?? null, - }); - if (updated.length > 0) return; - // Promoted or cancelled first: the update lost, and its event rolls back. - return yield* Effect.die(new LifecycleConflict({ id: input.id })); - }, Effect.orDie); + const projectAdmitted = Effect.fn("Inbox.projectAdmitted")( + function* (input: ProjectAdmittedInput) { + // Safe to read first here: the commit transaction already opened with the + // event's sequence bump, so this cannot upgrade a read lock. + const entry = yield* selectEntry(input.id); + if (entry.length > 0) return yield* Effect.die(new LifecycleConflict({ id: input.id })); + const stored = yield* insertRow({ + id: input.id, + sessionId: input.sessionId, + threadId: input.threadId, + kind: "prompt", + data: input.prompt, + delivery: input.delivery, + admittedSeq: input.admittedSeq, + promotedSeq: Option.none(), + cancelledSeq: Option.none(), + createdAt: VariantSchema.Override(input.timeCreated), + rev: yield* store.rev, + }); + if (stored.length > 0) return; + const revived = yield* reviveRow({ + id: input.id, + threadId: input.threadId, + data: input.prompt, + delivery: input.delivery, + admittedSeq: input.admittedSeq, + createdAt: input.timeCreated, + rev: yield* store.rev, + }); + if (revived.length === 0) return yield* Effect.die(new LifecycleConflict({ id: input.id })); + }, + store.write, + Effect.orDie, + ); + + const projectPrompted = Effect.fn("Inbox.projectPrompted")( + function* (input: ProjectPromptedInput) { + const updated = yield* promoteRow({ + id: input.id, + threadId: input.threadId, + promotedSeq: input.promotedSeq, + rev: yield* store.rev, + }); + const promoted = updated[0]; + if (promoted) { + if (!matchesProjection(fromRow(promoted), input)) + return yield* Effect.die(new LifecycleConflict({ id: input.id })); + return; + } + + // Nothing moved: either this exact promotion is being replayed, or the row + // is already promoted at a different sequence, which is a conflict. + const stored = yield* find(input.id); + if (Option.isSome(stored)) { + if ( + !matchesProjection(stored.value, input) || + stored.value.promotedSeq !== input.promotedSeq || + stored.value.cancelledSeq !== undefined + ) + return yield* Effect.die(new LifecycleConflict({ id: input.id })); + return; + } + + // Promotion of an input this store never admitted -- replayed onto an empty + // projection. Both sequences collapse onto the promotion. + yield* insertRow({ + id: input.id, + sessionId: input.sessionId, + threadId: input.threadId, + kind: "prompt", + data: input.prompt, + delivery: input.delivery, + admittedSeq: input.promotedSeq, + promotedSeq: Option.some(input.promotedSeq), + cancelledSeq: Option.none(), + createdAt: VariantSchema.Override(input.timeCreated), + rev: yield* store.rev, + }); + }, + store.write, + Effect.orDie, + ); + + const projectCancelled = Effect.fn("Inbox.projectCancelled")( + function* (input: ProjectCancelledInput) { + const updated = yield* cancelRow({ ...input, rev: yield* store.rev }); + if (updated.length > 0) return; + // Nothing moved: this exact cancel being replayed, or the row was promoted or cancelled first. + const stored = yield* find(input.id); + if (Option.isSome(stored) && stored.value.cancelledSeq === input.cancelledSeq) return; + return yield* Effect.die(new LifecycleConflict({ id: input.id })); + }, + store.write, + Effect.orDie, + ); + + const projectUpdated = Effect.fn("Inbox.projectUpdated")( + function* (input: ProjectUpdatedInput) { + const updated = yield* updateRow({ + id: input.id, + threadId: input.threadId, + data: input.prompt ?? null, + delivery: input.delivery ?? null, + rev: yield* store.rev, + }); + if (updated.length > 0) return; + // Promoted or cancelled first: the update lost, and its event rolls back. + return yield* Effect.die(new LifecycleConflict({ id: input.id })); + }, + store.write, + Effect.orDie, + ); const pendingOf = (threadId: ThreadSchema.ID, id: SessionMessageSchema.ID) => find(id).pipe( diff --git a/packages/harness/src/project/project.ts b/packages/harness/src/project/project.ts index 033c3602..689a7b76 100644 --- a/packages/harness/src/project/project.ts +++ b/packages/harness/src/project/project.ts @@ -2,6 +2,7 @@ import { Context, Effect, Layer, Option, Schema, Semaphore } from "effect"; import { SqlClient, SqlSchema } from "effect/sql"; import { Database } from "../db/db.ts"; import { ProjectRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { Git } from "../git/git.ts"; import { DirectoryNotFoundError, NotDirectoryError, type Error as LocationError } from "../location/error.ts"; import { Repo } from "../repo/repo.ts"; @@ -56,6 +57,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const fs = yield* SandboxIO.FileSystem; const repos = yield* Repo.Service; const worktrees = yield* Worktree.Service; @@ -83,7 +85,7 @@ export const layer = Layer.effect( Request: ProjectRow.insert, execute: (row) => sql` INSERT INTO project ${sql.insert(row)} - ON CONFLICT(id) DO UPDATE SET status = 'active', updated_at = excluded.updated_at + ON CONFLICT(id) DO UPDATE SET status = 'active', updated_at = excluded.updated_at, rev = excluded.rev `, }); @@ -165,10 +167,15 @@ export const layer = Layer.effect( const worktreeSpaceId = Space.id(env, worktree); - yield* sql - .withTransaction( + yield* store + .write( Effect.gen(function* () { - const row = yield* ProjectRow.insert.makeEffect({ id: projectId, name, status: "active" }); + const row = yield* ProjectRow.insert.makeEffect({ + id: projectId, + name, + status: "active", + rev: yield* store.rev, + }); yield* upsertProject(row); for (const member of members) { @@ -234,14 +241,17 @@ export const rename = Effect.fn("Project.rename")(function* (input: { readonly name: string; }) { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); const update = SqlSchema.findOneOption({ - Request: Schema.Struct({ id: Schema.String, name: Schema.String }), + Request: Schema.Struct({ id: Schema.String, name: Schema.String, rev: Schema.Int }), Result: ProjectRow, - execute: ({ id, name }) => - sql`UPDATE project SET name = ${name}, updated_at = ${now} WHERE id = ${id} RETURNING *`, + execute: ({ id, name, rev }) => + sql`UPDATE project SET name = ${name}, updated_at = ${now}, rev = ${rev} WHERE id = ${id} RETURNING *`, }); - const row = yield* update({ id: input.projectId, name: input.name }).pipe(Effect.orDie); + const row = yield* Effect.gen(function* () { + return yield* update({ id: input.projectId, name: input.name, rev: yield* store.rev }); + }).pipe(store.write, Effect.orDie); if (Option.isNone(row)) return yield* new ProjectNotFoundError({ projectId: input.projectId }); return new ProjectSchema.Info({ id: ProjectSchema.ID.make(row.value.id), diff --git a/packages/harness/src/runner/execute.ts b/packages/harness/src/runner/execute.ts index 7847dbc4..56e34813 100644 --- a/packages/harness/src/runner/execute.ts +++ b/packages/harness/src/runner/execute.ts @@ -5,6 +5,7 @@ import { Cause, Context, DateTime, Effect, Exit, Layer, Option } from "effect"; import { SqlClient } from "effect/sql"; +import { Store } from "../db/store.ts"; import { Event } from "../event/event.ts"; import { EventList } from "../event/list.ts"; import { Location } from "../location/location.ts"; @@ -31,6 +32,7 @@ export const layer = Layer.effect( const threads = yield* Thread.Service; const sandbox = yield* SandboxController.Controller; const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; // Captured here, not requested inside `drain`: `RunCoordinator.make` // requires the drain's `R` channel to be `never`, and a `Runner.Service.use` // in the callback would leave the tag in it. @@ -163,6 +165,7 @@ export const layer = Layer.effect( const location = Location.layerMounted().pipe( Layer.provide(Layer.succeedContext(mountContext)), Layer.provide(Layer.succeed(SqlClient.SqlClient, sql)), + Layer.provide(Layer.succeed(Store.Service, store)), ); const locationContext = yield* Layer.build(location).pipe( Effect.catchTag( diff --git a/packages/harness/src/sandbox/control.ts b/packages/harness/src/sandbox/control.ts index 0c721d4e..8fd5338c 100644 --- a/packages/harness/src/sandbox/control.ts +++ b/packages/harness/src/sandbox/control.ts @@ -13,6 +13,7 @@ import { } from "effect"; import { SqlClient } from "effect/sql"; import { SandboxInstanceRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { Space } from "../space/space.ts"; import { Telemetry } from "../telemetry/telemetry.ts"; import { SandboxDriver } from "./driver.ts"; @@ -179,6 +180,7 @@ const timedShell = (shell: ISandboxExe, driver: SandboxDriver.Name): ISandboxExe export const make = Effect.fn("Sandbox.Controller.make")(function* (options: Options) { const sql = yield* SqlClient.SqlClient; + const db = yield* Store.Service; const registry = yield* SandboxDriverRegistry.Registry; const store = yield* SandboxStore.make; const gate = yield* Semaphore.make(1); @@ -913,8 +915,8 @@ export const make = Effect.fn("Sandbox.Controller.make")(function* (options: Opt const now = yield* Effect.clockWith((clock) => clock.currentTimeMillis); // Tombstone and archive the env's spaces atomically (D-ENV). The UPDATE // goes first so the DEFERRED transaction takes its write lock up front. - yield* sql - .withTransaction( + yield* db + .write( Effect.gen(function* () { yield* sql` UPDATE sandbox @@ -929,7 +931,7 @@ export const make = Effect.fn("Sandbox.Controller.make")(function* (options: Opt yield* Space.archiveEnv(id); }), ) - .pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.orDie); + .pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.provideService(Store.Service, db), Effect.orDie); }); const createAndMount: Interface["createAndMount"] = (input, mountOptions) => { diff --git a/packages/harness/src/session/session.ts b/packages/harness/src/session/session.ts index aa8cdca9..b0e3a51c 100644 --- a/packages/harness/src/session/session.ts +++ b/packages/harness/src/session/session.ts @@ -16,6 +16,7 @@ import { type ToolStatus, toolStatuses, } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { Document } from "../document/document.ts"; import { DocumentSchema } from "../document/schema.ts"; import { Event } from "../event/event.ts"; @@ -203,6 +204,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const events = yield* Event.Service; const threads = yield* Thread.Service; const documents = yield* Document.Service; @@ -268,35 +270,37 @@ export const layer = Layer.effect( const epochNow = Effect.map(DateTime.now, DateTime.toEpochMillis); - const projectCreated = Effect.fn("Session.projectCreated")(function* ( - data: EventList.SessionCreated["data"], - seq: number, - ) { - const row = yield* SessionRow.insert.makeEffect({ - id: data.sessionId, - spaceId: data.spaceId, - slug: data.slug, - directory: data.directory, - hostDir: Option.fromUndefinedOr(data.hostDir), - tag: Option.fromUndefinedOr(data.tag), - metadata: Option.fromUndefinedOr(data.metadata), - }); - yield* insertSession(row); - yield* threads.insert({ - id: data.threadId, - sessionId: data.sessionId, - origin: { kind: "main" }, - ...(data.title === undefined ? {} : { title: data.title }), - seq, - }); - yield* documents.create({ - sessionId: data.sessionId, - threadId: data.threadId, - definition: DocumentSchema.agent, - value: yield* Schema.encodeEffect(DocumentSchema.Agent)(data.agent), - seq, - }); - }, Effect.orDie); + const projectCreated = Effect.fn("Session.projectCreated")( + function* (data: EventList.SessionCreated["data"], seq: number) { + const row = yield* SessionRow.insert.makeEffect({ + id: data.sessionId, + spaceId: data.spaceId, + slug: data.slug, + directory: data.directory, + hostDir: Option.fromUndefinedOr(data.hostDir), + tag: Option.fromUndefinedOr(data.tag), + metadata: Option.fromUndefinedOr(data.metadata), + rev: yield* store.rev, + }); + yield* insertSession(row); + yield* threads.insert({ + id: data.threadId, + sessionId: data.sessionId, + origin: { kind: "main" }, + ...(data.title === undefined ? {} : { title: data.title }), + seq, + }); + yield* documents.create({ + sessionId: data.sessionId, + threadId: data.threadId, + definition: DocumentSchema.agent, + value: yield* Schema.encodeEffect(DocumentSchema.Agent)(data.agent), + seq, + }); + }, + store.write, + Effect.orDie, + ); const create = Effect.fn("Session.create")(function* (input: CreateSession) { const id = input.id ?? SessionSchema.ID.create(); @@ -385,10 +389,12 @@ export const layer = Layer.effect( const session = yield* findSession(input.sessionId).pipe(Effect.orDie); if (Option.isNone(session)) return yield* new SessionNotFoundError({ sessionId: input.sessionId }); const now = yield* epochNow; - yield* sql` - UPDATE session SET host_dir = ${input.hostDir}, updated_at = ${now} - WHERE id = ${input.sessionId} - `.pipe(Effect.orDie); + yield* Effect.gen(function* () { + yield* sql` + UPDATE session SET host_dir = ${input.hostDir}, updated_at = ${now}, rev = ${yield* store.rev} + WHERE id = ${input.sessionId} + `; + }).pipe(store.write, Effect.orDie); const updated = yield* findSession(input.sessionId).pipe(Effect.orDie); if (Option.isNone(updated)) return yield* Effect.die(new Error("session link did not persist")); return updated.value; @@ -426,10 +432,13 @@ export const layer = Layer.effect( if (!directoryWithin(target.value.location, directory)) return yield* reject("directory_outside_space"); if (input.threadId === undefined) { const now = yield* epochNow; - yield* sql` - UPDATE session SET space_id = ${input.spaceId}, directory = ${directory}, updated_at = ${now} - WHERE id = ${input.sessionId} - `.pipe(Effect.orDie); + yield* Effect.gen(function* () { + yield* sql` + UPDATE session SET space_id = ${input.spaceId}, directory = ${directory}, updated_at = ${now}, + rev = ${yield* store.rev} + WHERE id = ${input.sessionId} + `; + }).pipe(store.write, Effect.orDie); } else { yield* events.publish(EventList.ThreadConfigured, { timestamp: yield* DateTime.now, @@ -443,14 +452,20 @@ export const layer = Layer.effect( return updated.value; }); - // The log has no foreign key to the session, so both go in one transaction. + // The log has no foreign key to the session, so both go in one transaction. The tombstone lives + // outside the log, which goes with the session. const remove = Effect.fn("Session.delete")(function* (sessionId: SessionSchema.ID) { - return yield* sql - .withTransaction( + return yield* store + .write( Effect.gen(function* () { const deleted = yield* sql`DELETE FROM session WHERE id = ${sessionId} RETURNING id`; yield* sql`DELETE FROM sequence WHERE aggregate_id = ${sessionId}`; - return deleted.length > 0; + if (deleted.length === 0) return false; + yield* sql` + INSERT INTO removed (kind, id, rev) VALUES ('session', ${sessionId}, ${yield* store.rev}) + ON CONFLICT (kind, id) DO UPDATE SET rev = excluded.rev + `; + return true; }), ) .pipe(Effect.orDie); diff --git a/packages/harness/src/space/space.ts b/packages/harness/src/space/space.ts index 4ec3f60d..7f08a738 100644 --- a/packages/harness/src/space/space.ts +++ b/packages/harness/src/space/space.ts @@ -1,6 +1,7 @@ import { Clock, Context, Effect, Layer, Option, Schema } from "effect"; import { SqlClient, SqlError, SqlSchema } from "effect/sql"; import { SpaceRow } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { ProjectSchema } from "../project/schema.ts"; import { SandboxInstance } from "../sandbox/instance.ts"; import { SandboxIO } from "../sandbox/io.ts"; @@ -55,20 +56,22 @@ export const fromRow = (row: SpaceRow): SpaceSchema.Info => /** * Archive every space of an env, then every touched project left without an - * active space in any env (ยง5.5). Requires only `SqlClient` so the sandbox - * controller can run it inside its destroy tombstone transaction (D-ENV). + * active space in any env (ยง5.5). Requires only `SqlClient` and `Store` so the + * sandbox controller can run it inside its destroy tombstone transaction (D-ENV). * Write-first so a DEFERRED transaction takes the write lock immediately. */ export const archiveEnv = Effect.fn("Space.archiveEnv")(function* (env: SandboxInstance.ID) { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const column = SandboxInstance.toColumn(env); const now = yield* Clock.currentTimeMillis; - yield* sql - .withTransaction( + yield* store + .write( Effect.gen(function* () { - yield* sql`UPDATE space SET status = 'archived', updated_at = ${now} WHERE env IS ${column}`; + const rev = yield* store.rev; + yield* sql`UPDATE space SET status = 'archived', updated_at = ${now}, rev = ${rev} WHERE env IS ${column}`; yield* sql` - UPDATE project SET status = 'archived', updated_at = ${now} + UPDATE project SET status = 'archived', updated_at = ${now}, rev = ${rev} WHERE id IN (SELECT project_id FROM space WHERE env IS ${column}) AND NOT EXISTS (SELECT 1 FROM space WHERE space.project_id = project.id AND status = 'active') `; @@ -81,6 +84,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const fs = yield* SandboxIO.FileSystem; const { id: env } = yield* SandboxIO.Current; // NULL for the host; compared with `IS`, never `=`. @@ -106,16 +110,17 @@ export const layer = Layer.effect( project_id = excluded.project_id, kind = excluded.kind, status = 'active', - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + rev = excluded.rev `, }); // Projects with no active space in any env are archived, never deleted. - const archiveOrphanProjects = (projectIds: ReadonlyArray, now: number) => + const archiveOrphanProjects = (projectIds: ReadonlyArray, now: number, rev: number) => Effect.forEach( projectIds, (projectId) => sql` - UPDATE project SET status = 'archived', updated_at = ${now} + UPDATE project SET status = 'archived', updated_at = ${now}, rev = ${rev} WHERE id = ${projectId} AND NOT EXISTS (SELECT 1 FROM space WHERE project_id = ${projectId} AND status = 'active') `, @@ -154,14 +159,15 @@ export const layer = Layer.effect( .map((entry) => entry.row.id); const now = yield* Clock.currentTimeMillis; - yield* sql - .withTransaction( + yield* store + .write( Effect.gen(function* () { + const rev = yield* store.rev; if (gone.length > 0) { - yield* sql`UPDATE space SET status = 'archived', updated_at = ${now} WHERE ${sql.in("id", gone)}`; + yield* sql`UPDATE space SET status = 'archived', updated_at = ${now}, rev = ${rev} WHERE ${sql.in("id", gone)}`; } if (returned.length > 0) { - yield* sql`UPDATE space SET status = 'active', updated_at = ${now} WHERE ${sql.in("id", returned)}`; + yield* sql`UPDATE space SET status = 'active', updated_at = ${now}, rev = ${rev} WHERE ${sql.in("id", returned)}`; } const primary = yield* sql` @@ -173,11 +179,11 @@ export const layer = Layer.effect( // before promoting. Only a copy qualifies โ€” a linked worktree's // store died with the primary, and plain never needs one. yield* sql` - UPDATE space SET kind = 'copy', updated_at = ${now} + UPDATE space SET kind = 'copy', updated_at = ${now}, rev = ${rev} WHERE project_id = ${projectId} AND env IS ${envColumn} AND kind = 'primary' `; yield* sql` - UPDATE space SET kind = 'primary', updated_at = ${now} + UPDATE space SET kind = 'primary', updated_at = ${now}, rev = ${rev} WHERE id = ( SELECT id FROM space WHERE project_id = ${projectId} AND env IS ${envColumn} AND status = 'active' AND kind = 'copy' @@ -186,7 +192,7 @@ export const layer = Layer.effect( `; } - yield* archiveOrphanProjects([projectId], now); + yield* archiveOrphanProjects([projectId], now, rev); }), ) .pipe(Effect.orDie); @@ -194,44 +200,59 @@ export const layer = Layer.effect( return yield* list(projectId); }); - const rehome = Effect.fn("Space.rehome")(function* (input: RehomeInput) { - const write = (kind: SpaceSchema.Kind) => - SpaceRow.insert - .makeEffect({ - id: input.id, - projectId: input.projectId, - location: input.location, - kind, - env: SandboxInstance.toField(input.env), - status: "active", - }) - .pipe(Effect.flatMap(insertSpace)); - - // A concurrent primary claim cannot happen under the resolve lock, but - // the guard is cheap: the loser becomes a copy. - yield* write(input.kind).pipe( - Effect.catchIf( - (error) => - input.kind === "primary" && - SqlError.isSqlError(error) && - error.reason._tag === "UniqueViolation" && - error.reason.constraint.includes("space_primary_idx"), - () => write("copy"), - ), - Effect.orDie, - ); - }); + const rehome = Effect.fn("Space.rehome")( + function* (input: RehomeInput) { + const rev = yield* store.rev; + const write = (kind: SpaceSchema.Kind) => + SpaceRow.insert + .makeEffect({ + id: input.id, + projectId: input.projectId, + location: input.location, + kind, + env: SandboxInstance.toField(input.env), + status: "active", + rev, + }) + .pipe(Effect.flatMap(insertSpace)); - const absorb = Effect.fn("Space.absorb")(function* (input: AbsorbInput) { - yield* sql`UPDATE session SET space_id = ${input.into} WHERE space_id = ${input.strayId}`; - yield* sql`DELETE FROM space WHERE id = ${input.strayId}`; - }, Effect.orDie); + // A concurrent primary claim cannot happen under the resolve lock, but + // the guard is cheap: the loser becomes a copy. + yield* write(input.kind).pipe( + Effect.catchIf( + (error) => + input.kind === "primary" && + SqlError.isSqlError(error) && + error.reason._tag === "UniqueViolation" && + error.reason.constraint.includes("space_primary_idx"), + () => write("copy"), + ), + Effect.orDie, + ); + }, + store.write, + Effect.orDie, + ); + + const absorb = Effect.fn("Space.absorb")( + function* (input: AbsorbInput) { + const rev = yield* store.rev; + yield* sql`UPDATE session SET space_id = ${input.into}, rev = ${rev} WHERE space_id = ${input.strayId}`; + yield* sql`DELETE FROM space WHERE id = ${input.strayId}`; + }, + store.write, + Effect.orDie, + ); return Service.of({ get, list, refresh, - archiveEnv: (target) => archiveEnv(target).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + archiveEnv: (target) => + archiveEnv(target).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + Effect.provideService(Store.Service, store), + ), rehome, absorb, }); diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index 6c6ada30..f7afa05b 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -11,6 +11,7 @@ import { ThreadRow, type ToolStatus, } from "../db/schema.sql.ts"; +import { Store } from "../db/store.ts"; import { Document } from "../document/document.ts"; import { DocumentSchema } from "../document/schema.ts"; import { Event } from "../event/event.ts"; @@ -240,6 +241,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const events = yield* Event.Service; const documents = yield* Document.Service; @@ -285,11 +287,11 @@ export const layer = Layer.effect( Request: EntryRow.insert, Result: Schema.Struct({ id: Schema.String }), execute: (row) => sql` - INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, label, metadata, created_at, updated_at) + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, label, metadata, created_at, updated_at, rev) SELECT ${row.id}, ${row.sessionId}, ${row.threadId}, ${row.seq}, ${row.committedSeq ?? null}, ${row.type}, ${row.state}, ${row.data}, ${row.label ?? null}, ${row.metadata ?? null}, - ${row.createdAt}, ${row.updatedAt} + ${row.createdAt}, ${row.updatedAt}, ${row.rev} WHERE NOT EXISTS ( SELECT 1 FROM entry WHERE thread_id = ${row.threadId} AND seq >= ${row.seq} ) @@ -405,22 +407,27 @@ export const layer = Layer.effect( return yield* selectUnsettled(threadId).pipe(Effect.orDie); }); - const insert = Effect.fn("Thread.insert")(function* (input: InsertThread) { - const origin = input.origin; - const row = yield* ThreadRow.insert.makeEffect({ - id: input.id, - sessionId: input.sessionId, - originKind: origin.kind, - originThreadId: origin.kind === "main" ? Option.none() : Option.some(origin.threadId), - originEntryId: origin.kind === "main" ? Option.none() : Option.fromUndefinedOr(origin.entryId), - originCallId: origin.kind === "subagent" ? Option.some(origin.callId) : Option.none(), - title: Option.fromUndefinedOr(input.title), - execution: Option.none(), - attempt: Option.none(), - createdSeq: input.seq, - }); - yield* insertThread(row); - }, Effect.orDie); + const insert = Effect.fn("Thread.insert")( + function* (input: InsertThread) { + const origin = input.origin; + const row = yield* ThreadRow.insert.makeEffect({ + id: input.id, + sessionId: input.sessionId, + originKind: origin.kind, + originThreadId: origin.kind === "main" ? Option.none() : Option.some(origin.threadId), + originEntryId: origin.kind === "main" ? Option.none() : Option.fromUndefinedOr(origin.entryId), + originCallId: origin.kind === "subagent" ? Option.some(origin.callId) : Option.none(), + title: Option.fromUndefinedOr(input.title), + execution: Option.none(), + attempt: Option.none(), + createdSeq: input.seq, + rev: yield* store.rev, + }); + yield* insertThread(row); + }, + store.write, + Effect.orDie, + ); const writePart = Effect.fnUntraced(function* ( entryId: string, @@ -457,8 +464,10 @@ export const layer = Layer.effect( delta?: { cost: number; input: number; output: number; cacheRead: number; cacheWrite: number }, ) { const now = yield* epochNow; + const rev = yield* store.rev; yield* sql` UPDATE thread SET + rev = ${rev}, updated_at = ${now}, cost = cost + ${delta?.cost ?? 0}, tokens_input = tokens_input + ${delta?.input ?? 0}, @@ -467,7 +476,7 @@ export const layer = Layer.effect( tokens_cache_write = tokens_cache_write + ${delta?.cacheWrite ?? 0} WHERE id = ${thread.id} `; - yield* sql`UPDATE session SET updated_at = ${now} WHERE id = ${thread.sessionId}`; + yield* sql`UPDATE session SET updated_at = ${now}, rev = ${rev} WHERE id = ${thread.sessionId}`; }); type AppendTxResult = @@ -522,8 +531,8 @@ export const layer = Layer.effect( ) : undefined; - const result: AppendTxResult = yield* sql - .withTransaction( + const result: AppendTxResult = yield* store + .write( Effect.gen(function* () { const thread = yield* findThread(input.threadId); if (Option.isNone(thread)) return { _tag: "threadNotFound" } as const; @@ -549,6 +558,7 @@ export const layer = Layer.effect( data: input.data, label: Option.none(), metadata: Option.fromUndefinedOr(input.metadata), + rev: yield* store.rev, }); const inserted = yield* insertEntry(entryRow); if (inserted.length === 0) { @@ -618,7 +628,7 @@ export const layer = Layer.effect( }); const replaceAssistant = Effect.fn("Thread.replaceAssistant")(function* (input: ReplaceAssistant) { - yield* sql.withTransaction( + yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "replaceAssistant"); const delta = yield* usageDelta(entry.data, input.data); @@ -626,7 +636,7 @@ export const layer = Layer.effect( yield* sql`DELETE FROM part WHERE entry_id = ${input.id}`; yield* writeParts(input.id, input.threadId, input.parts); yield* sql` - UPDATE entry SET data = ${input.data}, updated_at = ${now} + UPDATE entry SET data = ${input.data}, updated_at = ${now}, rev = ${yield* store.rev} WHERE id = ${input.id} AND state = 'draft' `; yield* touch(thread, delta); @@ -635,7 +645,7 @@ export const layer = Layer.effect( }, Effect.orDie); const progress = Effect.fn("Thread.progress")(function* (input: ProgressAssistant) { - yield* sql.withTransaction( + yield* store.write( Effect.gen(function* () { yield* requireDraft(input.id, input.threadId, "progress"); const now = yield* epochNow; @@ -661,18 +671,19 @@ export const layer = Layer.effect( WHERE entry_id = ${input.id} AND part_index = ${change.partIndex} `; } - yield* sql`UPDATE entry SET updated_at = ${now} WHERE id = ${input.id}`; + yield* sql`UPDATE entry SET updated_at = ${now}, rev = ${yield* store.rev} WHERE id = ${input.id}`; }), ); }, Effect.orDie); - const settleToolCall = Effect.fn("Thread.settleToolCall")(function* (input: SettleToolCall) { - const data = yield* decodeJsonObject(input.data).pipe(Effect.orDie); - if (data["type"] !== "toolCall" || data["callID"] !== input.callId || data["status"] !== input.status) { - return yield* Effect.die(`settled tool payload does not match call ${input.callId}`); - } - const now = yield* epochNow; - const rows = yield* sql` + const settleToolCall = Effect.fn("Thread.settleToolCall")( + function* (input: SettleToolCall) { + const data = yield* decodeJsonObject(input.data).pipe(Effect.orDie); + if (data["type"] !== "toolCall" || data["callID"] !== input.callId || data["status"] !== input.status) { + return yield* Effect.die(`settled tool payload does not match call ${input.callId}`); + } + const now = yield* epochNow; + const rows = yield* sql` UPDATE part SET data = ${input.data}, status = ${input.status}, updated_at = ${now} WHERE entry_id = ${input.entryId} @@ -682,15 +693,21 @@ export const layer = Layer.effect( AND EXISTS (SELECT 1 FROM entry WHERE id = ${input.entryId} AND state = 'draft') RETURNING id `; - if (rows.length !== 1) { - return yield* Effect.die(`settleToolCall requires one unsettled call ${input.entryId}/${input.callId}`); - } - }, Effect.orDie); - - const commit = Effect.fn("Thread.commit")(function* (input: EndAssistant) { - const now = yield* epochNow; - const rows = yield* sql` - UPDATE entry SET state = 'committed', committed_seq = ${input.seq}, updated_at = ${now} + if (rows.length !== 1) { + return yield* Effect.die(`settleToolCall requires one unsettled call ${input.entryId}/${input.callId}`); + } + // The entry's rev covers its parts. + yield* sql`UPDATE entry SET updated_at = ${now}, rev = ${yield* store.rev} WHERE id = ${input.entryId}`; + }, + store.write, + Effect.orDie, + ); + + const commit = Effect.fn("Thread.commit")( + function* (input: EndAssistant) { + const now = yield* epochNow; + const rows = yield* sql` + UPDATE entry SET state = 'committed', committed_seq = ${input.seq}, updated_at = ${now}, rev = ${yield* store.rev} WHERE id = ${input.id} AND thread_id = ${input.threadId} AND type = 'assistant' @@ -700,12 +717,15 @@ export const layer = Layer.effect( ) RETURNING id `; - if (rows.length !== 1) return yield* Effect.die(`commit requires a fully-settled draft ${input.id}`); - yield* sql`UPDATE thread SET updated_at = ${now} WHERE id = ${input.threadId}`; - }, Effect.orDie); + if (rows.length !== 1) return yield* Effect.die(`commit requires a fully-settled draft ${input.id}`); + yield* sql`UPDATE thread SET updated_at = ${now}, rev = ${yield* store.rev} WHERE id = ${input.threadId}`; + }, + store.write, + Effect.orDie, + ); const abort = Effect.fn("Thread.abort")(function* (input: ReplaceAssistant & EndAssistant) { - yield* sql.withTransaction( + yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "abort"); const delta = yield* usageDelta(entry.data, input.data); @@ -714,7 +734,8 @@ export const layer = Layer.effect( yield* writeParts(input.id, input.threadId, input.parts); yield* sql` UPDATE entry - SET state = 'aborted', committed_seq = ${input.seq}, data = ${input.data}, updated_at = ${now} + SET state = 'aborted', committed_seq = ${input.seq}, data = ${input.data}, updated_at = ${now}, + rev = ${yield* store.rev} WHERE id = ${input.id} AND state = 'draft' `; yield* touch(thread, delta); @@ -764,122 +785,155 @@ export const layer = Layer.effect( return { thread: created.value, ...(text === undefined ? {} : { text }) } satisfies Forked; }); - const projectForked = Effect.fn("Thread.projectForked")(function* ( - input: EventList.ThreadForked["data"], - seq: number, - ) { - // Checked again inside the commit: the session may have been deleted since the request was read. - const point = yield* findEntry(input.entryId).pipe(Effect.orDie); - if ( - Option.isNone(point) || - point.value.threadId !== input.originThreadId || - point.value.state !== "committed" || - Option.isNone(point.value.committedSeq) - ) { - return yield* Effect.die(`fork point ${input.entryId} is not a committed entry of ${input.originThreadId}`); - } - const cut = point.value.committedSeq.value; - yield* insert({ - id: input.threadId, - sessionId: input.sessionId, - origin: { kind: "fork", threadId: input.originThreadId, entryId: input.entryId }, - seq, - }); - // The documents in effect at the cut are the owner's: every thread between the source and - // the owner was created after it. - for (const document of yield* documents.ofThread(input.sourceThreadId)) { - const policy = Option.getOrUndefined(document.fork); - if (policy === undefined || policy === "initial") continue; - const owned = - policy === "asOf" ? yield* documents.find(input.originThreadId, document.kind) : Option.some(document); - if (Option.isNone(owned)) continue; - const value = - policy === "asOf" - ? yield* documents.asOf(owned.value.id, cut) - : yield* documents.current(owned.value.id); - if (Option.isNone(value)) continue; - yield* documents.create({ + const projectForked = Effect.fn("Thread.projectForked")( + function* (input: EventList.ThreadForked["data"], seq: number) { + // Checked again inside the commit: the session may have been deleted since the request was read. + const point = yield* findEntry(input.entryId).pipe(Effect.orDie); + if ( + Option.isNone(point) || + point.value.threadId !== input.originThreadId || + point.value.state !== "committed" || + Option.isNone(point.value.committedSeq) + ) { + return yield* Effect.die( + `fork point ${input.entryId} is not a committed entry of ${input.originThreadId}`, + ); + } + const cut = point.value.committedSeq.value; + yield* insert({ + id: input.threadId, sessionId: input.sessionId, - threadId: input.threadId, - definition: { - kind: document.kind, - version: value.value.version, - scope: "thread", - ...Option.match(document.history, { onNone: () => ({}), onSome: (history) => ({ history }) }), - fork: policy, - }, - value: value.value.value, + origin: { kind: "fork", threadId: input.originThreadId, entryId: input.entryId }, seq, }); - } - // Every thread has an agent document, whatever its source held. - if (Option.isNone(yield* documents.find(input.threadId, DocumentSchema.agent.kind))) { - yield* documents.create({ - sessionId: input.sessionId, - threadId: input.threadId, - definition: DocumentSchema.agent, - value: {}, - seq, - }); - } - }); - - const retitle = Effect.fn("Thread.retitle")(function* (threadId: ThreadSchema.ID, title: string) { - const changed = yield* sql<{ readonly id: string }>` - UPDATE thread SET title = ${title} WHERE id = ${threadId} AND title IS NULL RETURNING id - `.pipe(Effect.orDie); - return changed.length > 0; - }); - - const markRunning = Effect.fn("Thread.markRunning")(function* (threadId: ThreadSchema.ID) { - yield* sql`UPDATE thread SET execution = 'running' WHERE id = ${threadId}`; - }, Effect.orDie); - - const endExecution = Effect.fn("Thread.endExecution")(function* ( - threadId: ThreadSchema.ID, - end: "finished" | "stopped" | "closed", - stoppedAt?: number, - ) { - // A run that ended in this process has no request in flight: only a dead host leaves one. - if (end === "closed") { - yield* sql`UPDATE thread SET execution = 'closed', attempt = NULL WHERE id = ${threadId}`; - return; - } - yield* sql`UPDATE thread SET attempt = NULL WHERE id = ${threadId}`; - // Pending prompts that keep the mark: any for a finished run; for a stop, only those sent - // after it, since what was queued before it stays parked by the user's choice. - const after = end === "stopped" ? (stoppedAt ?? Number.MAX_SAFE_INTEGER) : -1; - yield* sql` - UPDATE thread SET execution = NULL, restarts = 0 + // The documents in effect at the cut are the owner's: every thread between the source and + // the owner was created after it. + for (const document of yield* documents.ofThread(input.sourceThreadId)) { + const policy = Option.getOrUndefined(document.fork); + if (policy === undefined || policy === "initial") continue; + const owned = + policy === "asOf" + ? yield* documents.find(input.originThreadId, document.kind) + : Option.some(document); + if (Option.isNone(owned)) continue; + const value = + policy === "asOf" + ? yield* documents.asOf(owned.value.id, cut) + : yield* documents.current(owned.value.id); + if (Option.isNone(value)) continue; + yield* documents.create({ + sessionId: input.sessionId, + threadId: input.threadId, + definition: { + kind: document.kind, + version: value.value.version, + scope: "thread", + ...Option.match(document.history, { onNone: () => ({}), onSome: (history) => ({ history }) }), + fork: policy, + }, + value: value.value.value, + seq, + }); + } + // Every thread has an agent document, whatever its source held. + if (Option.isNone(yield* documents.find(input.threadId, DocumentSchema.agent.kind))) { + yield* documents.create({ + sessionId: input.sessionId, + threadId: input.threadId, + definition: DocumentSchema.agent, + value: {}, + seq, + }); + } + }, + store.write, + Effect.orDie, + ); + + const retitle = Effect.fn("Thread.retitle")( + function* (threadId: ThreadSchema.ID, title: string) { + const changed = yield* sql<{ readonly id: string }>` + UPDATE thread SET title = ${title}, rev = ${yield* store.rev} WHERE id = ${threadId} AND title IS NULL RETURNING id + `; + return changed.length > 0; + }, + store.write, + Effect.orDie, + ); + + const markRunning = Effect.fn("Thread.markRunning")( + function* (threadId: ThreadSchema.ID) { + yield* sql`UPDATE thread SET execution = 'running', rev = ${yield* store.rev} WHERE id = ${threadId}`; + }, + store.write, + Effect.orDie, + ); + + const endExecution = Effect.fn("Thread.endExecution")( + function* (threadId: ThreadSchema.ID, end: "finished" | "stopped" | "closed", stoppedAt?: number) { + // A run that ended in this process has no request in flight: only a dead host leaves one. + const rev = yield* store.rev; + if (end === "closed") { + yield* sql`UPDATE thread SET execution = 'closed', attempt = NULL, rev = ${rev} WHERE id = ${threadId}`; + return; + } + yield* sql`UPDATE thread SET attempt = NULL, rev = ${rev} WHERE id = ${threadId}`; + // Pending prompts that keep the mark: any for a finished run; for a stop, only those sent + // after it, since what was queued before it stays parked by the user's choice. + const after = end === "stopped" ? (stoppedAt ?? Number.MAX_SAFE_INTEGER) : -1; + yield* sql` + UPDATE thread SET execution = NULL, restarts = 0, rev = ${rev} WHERE id = ${threadId} AND NOT EXISTS ( SELECT 1 FROM inbox WHERE thread_id = ${threadId} AND promoted_seq IS NULL AND cancelled_seq IS NULL AND admitted_seq > ${after} ) `; - }, Effect.orDie); - - const claimRestart = Effect.fn("Thread.claimRestart")(function* (threadId: ThreadSchema.ID) { - const rows = yield* sql<{ restarts: number }>` + }, + store.write, + Effect.orDie, + ); + + const claimRestart = Effect.fn("Thread.claimRestart")( + function* (threadId: ThreadSchema.ID) { + const rows = yield* sql<{ restarts: number }>` UPDATE thread - SET restarts = restarts + (CASE WHEN execution = 'running' THEN 1 ELSE 0 END), execution = 'running' + SET restarts = restarts + (CASE WHEN execution = 'running' THEN 1 ELSE 0 END), execution = 'running', + rev = ${yield* store.rev} WHERE id = ${threadId} AND execution IS NOT NULL RETURNING restarts `; - return Option.fromUndefinedOr(rows[0]?.restarts); - }, Effect.orDie); - - const resetRestarts = Effect.fn("Thread.resetRestarts")(function* (threadId: ThreadSchema.ID) { - yield* sql`UPDATE thread SET restarts = 0 WHERE id = ${threadId} AND restarts > 0`; - }, Effect.orDie); - - const setAttempt = Effect.fn("Thread.setAttempt")(function* (threadId: ThreadSchema.ID, attempt: number | null) { - yield* sql`UPDATE thread SET attempt = ${attempt} WHERE id = ${threadId}`; - }, Effect.orDie); - - const idle = Effect.fn("Thread.idle")(function* (threadId: ThreadSchema.ID) { - yield* sql`UPDATE thread SET execution = NULL, restarts = 0, attempt = NULL WHERE id = ${threadId}`; - }, Effect.orDie); + return Option.fromUndefinedOr(rows[0]?.restarts); + }, + store.write, + Effect.orDie, + ); + + const resetRestarts = Effect.fn("Thread.resetRestarts")( + function* (threadId: ThreadSchema.ID) { + yield* sql`UPDATE thread SET restarts = 0, rev = ${yield* store.rev} WHERE id = ${threadId} AND restarts > 0`; + }, + store.write, + Effect.orDie, + ); + + const setAttempt = Effect.fn("Thread.setAttempt")( + function* (threadId: ThreadSchema.ID, attempt: number | null) { + yield* sql`UPDATE thread SET attempt = ${attempt}, rev = ${yield* store.rev} WHERE id = ${threadId}`; + }, + store.write, + Effect.orDie, + ); + + const idle = Effect.fn("Thread.idle")( + function* (threadId: ThreadSchema.ID) { + yield* sql` + UPDATE thread SET execution = NULL, restarts = 0, attempt = NULL, rev = ${yield* store.rev} WHERE id = ${threadId} + `; + }, + store.write, + Effect.orDie, + ); const findUnfinished = SqlSchema.findAll({ Request: Schema.Void, @@ -895,12 +949,14 @@ export const layer = Layer.effect( entryId: string; label: string | null; }) { - const now = yield* epochNow; - const rows = yield* sql` - UPDATE entry SET label = ${input.label}, updated_at = ${now} - WHERE id = ${input.entryId} AND thread_id = ${input.threadId} - RETURNING id - `.pipe(Effect.orDie); + const rows = yield* Effect.gen(function* () { + const now = yield* epochNow; + return yield* sql` + UPDATE entry SET label = ${input.label}, updated_at = ${now}, rev = ${yield* store.rev} + WHERE id = ${input.entryId} AND thread_id = ${input.threadId} + RETURNING id + `; + }).pipe(store.write, Effect.orDie); if (rows.length === 0) { return yield* new EntryNotFoundError({ threadId: input.threadId, entryId: input.entryId }); } diff --git a/packages/harness/test/__artifacts__/store.revision.json b/packages/harness/test/__artifacts__/store.revision.json new file mode 100644 index 00000000..6a64e31c --- /dev/null +++ b/packages/harness/test/__artifacts__/store.revision.json @@ -0,0 +1,271 @@ +{ + "store": { + "idIsSet": true, + "startRev": 0, + "sameId": true, + "head": 22 + }, + "doorbellRang": true, + "steps": [ + { + "step": "start", + "head": 0 + }, + { + "step": "seed project and space", + "head": 1 + }, + { + "step": "session.create", + "head": 2 + }, + { + "step": "thread.configure (thread-scoped agent document)", + "head": 3 + }, + { + "step": "prompt (tool call, settle, reply)", + "head": 19 + }, + { + "step": "fork", + "head": 20 + }, + { + "step": "session-scoped document", + "head": 21 + }, + { + "step": "session.delete", + "head": 22 + } + ], + "commits": [ + { + "rev": 1, + "by": "plain write", + "writes": [ + "project P1 insert", + "space SP1 insert" + ] + }, + { + "rev": 2, + "by": "session.created", + "writes": [ + "session S1 insert", + "thread T1 insert", + "thread T1 update" + ] + }, + { + "rev": 3, + "by": "session.thread.configured", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 4, + "by": "session.prompt.admitted", + "writes": [ + "inbox E1 insert" + ] + }, + { + "rev": 5, + "by": "plain write", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 6, + "by": "plain write", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 7, + "by": "plain write", + "writes": [ + "project P1 update", + "space SP1 update" + ] + }, + { + "rev": 8, + "by": "session.prompt.promoted", + "writes": [ + "inbox E1 update", + "entry E1 insert", + "thread T1 update", + "session S1 update", + "thread T1 update" + ] + }, + { + "rev": 9, + "by": "session.thread.system.changed", + "writes": [ + "entry E2 insert", + "thread T1 update", + "session S1 update" + ] + }, + { + "rev": 10, + "by": "plain write", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 11, + "by": "session.llm.started", + "writes": [ + "entry E3 insert", + "thread T1 update", + "session S1 update" + ] + }, + { + "rev": 12, + "by": "session.llm.ended", + "writes": [ + "entry E3 update", + "thread T1 update", + "session S1 update", + "thread T1 update" + ] + }, + { + "rev": 13, + "by": "session.tool.settled", + "writes": [ + "entry E3 update" + ] + }, + { + "rev": 14, + "by": "session.turn.ended", + "writes": [ + "entry E3 update", + "thread T1 update", + "thread T1 update" + ] + }, + { + "rev": 15, + "by": "plain write", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 16, + "by": "session.llm.started", + "writes": [ + "entry E4 insert", + "thread T1 update", + "session S1 update" + ] + }, + { + "rev": 17, + "by": "session.llm.ended", + "writes": [ + "entry E4 update", + "thread T1 update", + "session S1 update", + "thread T1 update" + ] + }, + { + "rev": 18, + "by": "session.turn.ended", + "writes": [ + "entry E4 update", + "thread T1 update", + "thread T1 update" + ] + }, + { + "rev": 19, + "by": "plain write", + "writes": [ + "thread T1 update", + "thread T1 update" + ] + }, + { + "rev": 20, + "by": "session.thread.forked", + "writes": [ + "thread T2 insert", + "thread T2 update" + ] + }, + { + "rev": 21, + "by": "plain write", + "writes": [ + "thread T1 update", + "thread T2 update" + ] + }, + { + "rev": 22, + "by": "plain write", + "writes": [ + "removed S1" + ] + } + ], + "rowsBeforeDelete": { + "project": [ + "P1 rev=7" + ], + "space": [ + "SP1 rev=7" + ], + "session": [ + "S1 rev=17" + ], + "thread": [ + "T1 rev=21", + "T2 rev=21" + ], + "entry": [ + "E1 rev=8", + "E2 rev=9", + "E3 rev=14", + "E4 rev=18" + ], + "inbox": [ + "E1 rev=8" + ] + }, + "rowsAfterDelete": { + "project": [ + "P1 rev=7" + ], + "space": [ + "SP1 rev=7" + ], + "session": [], + "thread": [], + "entry": [], + "inbox": [] + }, + "removed": [ + "session S1 rev=22" + ], + "rejected": { + "staleRev": "rev is not the current revision", + "forgotStamp": "rev is not the current revision", + "insertWithoutRevision": "rev is not the current revision", + "headUnchanged": true + } +} diff --git a/packages/harness/test/db/db.test.ts b/packages/harness/test/db/db.test.ts index f058ecae..d0f923d3 100644 --- a/packages/harness/test/db/db.test.ts +++ b/packages/harness/test/db/db.test.ts @@ -70,6 +70,8 @@ const space = (input: { kind: input.kind, env: instance(input.env), status: "active", + // Nothing here takes a revision, so the store is still at 0. + rev: 0, }); const registerInstances = (...ids: ReadonlyArray) => @@ -102,11 +104,13 @@ describe("Database", () => { "inbox", "part", "project", + "removed", "revision", "sandbox", "sequence", "session", "space", + "store", "thread", ]); }), @@ -132,7 +136,7 @@ describe("Database", () => { expect(orphanExit._tag).toBe("Failure"); yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active" }), + yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active", rev: 0 }), ); yield* db.insertSpace( yield* space({ @@ -188,7 +192,7 @@ describe("Database", () => { yield* registerInstances("sandbox-1", "sandbox-2", "sandbox-orphan"); yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active" }), + yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active", rev: 0 }), ); yield* db.insertSpace( yield* space({ @@ -216,7 +220,7 @@ describe("Database", () => { yield* registerInstances("sandbox-1", "sandbox-2", "sandbox-orphan"); yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active" }), + yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active", rev: 0 }), ); yield* sql.withTransaction( @@ -226,7 +230,7 @@ describe("Database", () => { // old drizzle wrapper had to forbid this yield* Effect.promise(() => Promise.resolve()); yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-2", name: "widget", status: "active" }), + yield* ProjectRow.insert.makeEffect({ id: "project-2", name: "widget", status: "active", rev: 0 }), ); }), ); @@ -244,18 +248,28 @@ describe("Database", () => { yield* registerInstances("sandbox-1", "sandbox-2", "sandbox-orphan"); yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active" }), + yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "codework", status: "active", rev: 0 }), ); const exit = yield* sql .withTransaction( Effect.gen(function* () { yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-2", name: "widget", status: "active" }), + yield* ProjectRow.insert.makeEffect({ + id: "project-2", + name: "widget", + status: "active", + rev: 0, + }), ); // duplicate primary key forces the transaction to fail yield* db.insertProject( - yield* ProjectRow.insert.makeEffect({ id: "project-1", name: "dupe", status: "active" }), + yield* ProjectRow.insert.makeEffect({ + id: "project-1", + name: "dupe", + status: "active", + rev: 0, + }), ); }), ) diff --git a/packages/harness/test/fixtures/space.ts b/packages/harness/test/fixtures/space.ts index baa40c44..6c8044c0 100644 --- a/packages/harness/test/fixtures/space.ts +++ b/packages/harness/test/fixtures/space.ts @@ -1,5 +1,6 @@ import { Effect } from "effect"; import { SqlClient } from "effect/sql"; +import { Store } from "../../src/db/store.ts"; import { ProjectSchema } from "../../src/project/schema.ts"; import { SandboxInstance } from "../../src/sandbox/instance.ts"; import { AbsolutePath } from "../../src/schema.ts"; @@ -28,18 +29,24 @@ export interface SeededSpace { */ export const seedSpace = Effect.fn("seedSpace")(function* (options: SeedSpaceOptions = {}) { const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; const location = AbsolutePath.make(options.location ?? "/repo"); const env = options.env ?? SandboxInstance.ID.local; const projectId = ProjectSchema.ID.make(options.projectId ?? "local"); const spaceId = Space.id(env, location); - yield* sql` - INSERT OR IGNORE INTO project (id, name, status, created_at, updated_at) - VALUES (${projectId}, ${projectId}, 'active', 0, 0) - `; - yield* sql` - INSERT OR IGNORE INTO space (id, project_id, location, kind, env, status, created_at, updated_at) - VALUES (${spaceId}, ${projectId}, ${location}, ${options.kind ?? "plain"}, ${SandboxInstance.toColumn(env)}, ${options.status ?? "active"}, 0, 0) - `; + yield* store.write( + Effect.gen(function* () { + const rev = yield* store.rev; + yield* sql` + INSERT OR IGNORE INTO project (id, name, status, created_at, updated_at, rev) + VALUES (${projectId}, ${projectId}, 'active', 0, 0, ${rev}) + `; + yield* sql` + INSERT OR IGNORE INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) + VALUES (${spaceId}, ${projectId}, ${location}, ${options.kind ?? "plain"}, ${SandboxInstance.toColumn(env)}, ${options.status ?? "active"}, 0, 0, ${rev}) + `; + }), + ); const seeded: SeededSpace = { spaceId, projectId, location, env }; return seeded; }); diff --git a/packages/harness/test/project.test.ts b/packages/harness/test/project.test.ts index 019f3d98..e1f79bc5 100644 --- a/packages/harness/test/project.test.ts +++ b/packages/harness/test/project.test.ts @@ -77,8 +77,8 @@ const seedSession = (id: string, space: string, directory: string) => Effect.flatMap( SqlClient.SqlClient, (sql) => sql` - INSERT INTO session (id, space_id, slug, directory, created_at, updated_at) - VALUES (${id}, ${space}, ${id}, ${directory}, 0, 0) + INSERT INTO session (id, space_id, slug, directory, created_at, updated_at, rev) + VALUES (${id}, ${space}, ${id}, ${directory}, 0, 0, (SELECT rev FROM store)) `, ); diff --git a/packages/harness/test/runner.loop.test.ts b/packages/harness/test/runner.loop.test.ts index a1e56edf..77d3c3f4 100644 --- a/packages/harness/test/runner.loop.test.ts +++ b/packages/harness/test/runner.loop.test.ts @@ -12,6 +12,7 @@ import { Context } from "../src/context/context.ts"; import { Control } from "../src/control.ts"; import { Database } from "../src/db/db.ts"; import { InboxRow } from "../src/db/schema.sql.ts"; +import { Store } from "../src/db/store.ts"; import { Event } from "../src/event/event.ts"; import { EventList } from "../src/event/list.ts"; import { EventRegistry } from "../src/event/registry.ts"; @@ -981,7 +982,7 @@ describe("runner loop โ€” queued input", () => { /** Cancels `id` the first time a turn loads settings: inside the window between the pending check and promotion. */ const cancelBeforePromotion = Effect.fnUntraced(function* (threadId: ThreadSchema.ID, id: string) { - const services = yield* Effect.context(); + const services = yield* Effect.context(); beforeSettings.run = Effect.gen(function* () { beforeSettings.run = Effect.void; const inputs = yield* Inbox.make; @@ -1321,7 +1322,7 @@ describe("runner loop โ€” resume after restart", () => { : Effect.void, ); yield* crashedMidReply(threadId, "A"); - yield* sql`UPDATE thread SET restarts = ${Control.RESTART_LIMIT - 1} WHERE id = ${threadId}`; + yield* sql`UPDATE thread SET restarts = ${Control.RESTART_LIMIT - 1}, rev = (SELECT rev FROM store) WHERE id = ${threadId}`; yield* control.resumeUnfinished; @@ -1350,7 +1351,7 @@ describe("runner loop โ€” resume after restart", () => { const sql = yield* SqlClient.SqlClient; const execution = yield* RunnerExecution.Service; const { threadId } = yield* seedSession(); - yield* sql`UPDATE thread SET restarts = 3 WHERE id = ${threadId}`; + yield* sql`UPDATE thread SET restarts = 3, rev = (SELECT rev FROM store) WHERE id = ${threadId}`; const streaming = yield* hanging; const services = yield* Effect.context(); script.push("answer", "hang"); diff --git a/packages/harness/test/sandbox.destroy.test.ts b/packages/harness/test/sandbox.destroy.test.ts index e31d842e..25c14cd3 100644 --- a/packages/harness/test/sandbox.destroy.test.ts +++ b/packages/harness/test/sandbox.destroy.test.ts @@ -26,11 +26,11 @@ const createStopped = Effect.gen(function* () { const seed = (project: string, spaces: ReadonlyArray<{ id: string; env: SandboxInstance.ID }>) => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* sql`INSERT INTO project (id, name, status, created_at, updated_at) VALUES (${project}, ${project}, 'active', 0, 0)`; + yield* sql`INSERT INTO project (id, name, status, created_at, updated_at, rev) VALUES (${project}, ${project}, 'active', 0, 0, (SELECT rev FROM store))`; for (const space of spaces) { yield* sql` - INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at) - VALUES (${space.id}, ${project}, ${`/${space.id}`}, 'plain', ${SandboxInstance.toColumn(space.env)}, 'active', 0, 0) + INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) + VALUES (${space.id}, ${project}, ${`/${space.id}`}, 'plain', ${SandboxInstance.toColumn(space.env)}, 'active', 0, 0, (SELECT rev FROM store)) `; } }); diff --git a/packages/harness/test/sandbox.instance.test.ts b/packages/harness/test/sandbox.instance.test.ts index 96123ed9..c2be1453 100644 --- a/packages/harness/test/sandbox.instance.test.ts +++ b/packages/harness/test/sandbox.instance.test.ts @@ -16,7 +16,7 @@ const sandboxStore = SandboxStore.make; const seedProject = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* sql`INSERT OR IGNORE INTO project (id, name, status, created_at, updated_at) VALUES ('p', 'p', 'active', 0, 0)`; + yield* sql`INSERT OR IGNORE INTO project (id, name, status, created_at, updated_at, rev) VALUES ('p', 'p', 'active', 0, 0, (SELECT rev FROM store))`; }); describe("SandboxInstance", () => { @@ -43,8 +43,8 @@ describe("SandboxInstance", () => { yield* seedProject; const space = (rowId: string, env: string | null) => sql` - INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at) - VALUES (${rowId}, 'p', ${`/workspace/${rowId}`}, 'plain', ${env}, 'active', 0, 0) + INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) + VALUES (${rowId}, 'p', ${`/workspace/${rowId}`}, 'plain', ${env}, 'active', 0, 0, (SELECT rev FROM store)) `; // positive control: NULL is the host and needs no registration at @@ -65,8 +65,8 @@ describe("SandboxInstance", () => { yield* seedProject; yield* store.register({ id: id("sbx_a"), driver: "memory", kind: "virtual", ownership: "managed" }); yield* sql` - INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at) - VALUES ('s', 'p', '/workspace', 'plain', 'sbx_a', 'active', 0, 0) + INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) + VALUES ('s', 'p', '/workspace', 'plain', 'sbx_a', 'active', 0, 0, (SELECT rev FROM store)) `; const deletion = sql`DELETE FROM sandbox WHERE id = 'sbx_a'`; @@ -85,8 +85,8 @@ describe("SandboxInstance", () => { yield* store.register({ id: id("sbx_a"), driver: "memory", kind: "virtual", ownership: "managed" }); const insert = (rowId: string, env: string | null) => sql` - INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at) - VALUES (${rowId}, 'p', '/workspace', 'plain', ${env}, 'active', 0, 0) + INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) + VALUES (${rowId}, 'p', '/workspace', 'plain', ${env}, 'active', 0, 0, (SELECT rev FROM store)) `; // one path, two namespaces โ€” the host and a registered one diff --git a/packages/harness/test/session.input.flow.test.ts b/packages/harness/test/session.input.flow.test.ts index 4f583ff3..c2278be3 100644 --- a/packages/harness/test/session.input.flow.test.ts +++ b/packages/harness/test/session.input.flow.test.ts @@ -59,8 +59,8 @@ describe("Inbox admission", () => { const input = yield* setup; const id = SessionMessageSchema.ID.create(); yield* sql` - INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at) - VALUES (${id}, ${sessionId}, ${threadId}, 1, 1, 'user', 'committed', '{}', 0, 0) + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at, rev) + VALUES (${id}, ${sessionId}, ${threadId}, 1, 1, 'user', 'committed', '{}', 0, 0, (SELECT rev FROM store)) `; const exit = yield* input.admit({ id, sessionId, threadId, prompt, delivery: "steer" }).pipe(Effect.exit); diff --git a/packages/harness/test/space.test.ts b/packages/harness/test/space.test.ts index 0bf30fce..d2e69ee8 100644 --- a/packages/harness/test/space.test.ts +++ b/packages/harness/test/space.test.ts @@ -38,7 +38,7 @@ const seedProject = (id: string) => Effect.flatMap( SqlClient.SqlClient, (sql) => - sql`INSERT INTO project (id, name, status, created_at, updated_at) VALUES (${id}, ${id}, 'active', 0, 0)`, + sql`INSERT INTO project (id, name, status, created_at, updated_at, rev) VALUES (${id}, ${id}, 'active', 0, 0, (SELECT rev FROM store))`, ); // The host is NULL; anything else must exist in sandbox first. @@ -65,9 +65,9 @@ const seedSpace = Effect.fnUntraced(function* (seed: Seed) { const env = seed.env ?? local; const id = Space.id(env, seed.location); yield* sql` - INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at) + INSERT INTO space (id, project_id, location, kind, env, status, created_at, updated_at, rev) VALUES (${id}, ${seed.projectId}, ${seed.location}, ${seed.kind}, ${SandboxInstance.toColumn(env)}, - ${seed.status ?? "active"}, ${seed.createdAt ?? 0}, 0) + ${seed.status ?? "active"}, ${seed.createdAt ?? 0}, 0, (SELECT rev FROM store)) `; return id; }); diff --git a/packages/harness/test/store.test.ts b/packages/harness/test/store.test.ts new file mode 100644 index 00000000..08603a63 --- /dev/null +++ b/packages/harness/test/store.test.ts @@ -0,0 +1,344 @@ +import "./utils/env.ts"; +import { realpathSync } from "node:fs"; +import { createAssistantMessageEventStream } from "@codeworksh/aikit"; +import { Cause, DateTime, Effect, Exit, Layer, PubSub, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import { describe, expect } from "vite-plus/test"; +import { Context } from "../src/context/context.ts"; +import { Control } from "../src/control.ts"; +import { Database } from "../src/db/db.ts"; +import { Store } from "../src/db/store.ts"; +import { Document } from "../src/document/document.ts"; +import { Event } from "../src/event/event.ts"; +import { EventList } from "../src/event/list.ts"; +import { EventRegistry } from "../src/event/registry.ts"; +import { Global } from "../src/global.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import type { Plugin } from "../src/plugin/plugin.ts"; +import { RunnerExecute } from "../src/runner/execute.ts"; +import { Hooks } from "../src/runner/hooks.ts"; +import { LLM } from "../src/runner/llm.ts"; +import { Loop } from "../src/runner/loop.ts"; +import { Project } from "../src/project/project.ts"; +import { SandboxController } from "../src/sandbox/control.ts"; +import { SandboxInstance } from "../src/sandbox/instance.ts"; +import { SandboxDriverRegistry } from "../src/sandbox/registry.ts"; +import { Process } from "../src/sandbox/utils/process.ts"; +import { Mcp } from "../src/mcp/mcp.ts"; +import { PromptSchema } from "../src/session/prompt/schema.ts"; +import { SessionProjector } from "../src/session/projector.ts"; +import { SessionRuntime } from "../src/session/runtime.ts"; +import { SessionSchema } from "../src/session/schema.ts"; +import { Session } from "../src/session/session.ts"; +import { Settings } from "../src/settings/settings.ts"; +import { State } from "../src/state/state.ts"; +import { ThreadSchema } from "../src/thread/schema.ts"; +import { Thread } from "../src/thread/thread.ts"; +import * as Tool from "../src/tool/tool.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { pooled } from "./fixtures/pool.ts"; +import { seedSpace } from "./fixtures/space.ts"; +import { tmpdir } from "./fixtures/tempdir.ts"; +import { testEffect } from "./utils/effect.ts"; + +// The first request calls `echo`, every one after replies: one turn settles a tool call. +let requests = 0; +const open: LLM.Open = (input) => + Effect.sync(() => { + requests += 1; + const call = requests === 1; + const message = assistant( + input, + requests, + call + ? { + stopReason: "toolUse", + parts: [ + { + type: "toolCall", + callID: "call_echo", + name: "echo", + arguments: { text: "hi" }, + status: "pending", + time: { start: 1, end: 1 }, + }, + ], + } + : {}, + ); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: call ? "toolUse" : "stop", message }); + return stream; + }); + +const echo: Plugin = { + id: "test.tool.echo", + kind: "tool", + setup: (ctx) => { + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name: "echo", + description: "echoes its text", + parameters: Schema.Struct({ text: Schema.String }), + success: Schema.String, + handler: ({ text }) => Effect.succeed(text), + }), + ), + ); + }, +}; + +const layer = () => { + const database = Database.layer(":memory:"); + const sandbox = SandboxController.layer({ hostCwd: "/" }).pipe( + Layer.provideMerge(SandboxDriverRegistry.layer()), + Layer.provideMerge(database), + ); + const seeded = pooled([...builtins, echo]); + return Control.layer.pipe( + Layer.provideMerge( + RunnerExecute.layer.pipe( + Layer.provide(Loop.layer({ request: LLM.make(open) }).pipe(Layer.provide(Hooks.layer))), + ), + ), + Layer.provideMerge(State.layer({}, seeded.ref, seeded.references, seeded.follow, seeded.rebuild)), + Layer.provideMerge(Mcp.layer({ hostCwd: "/", home: "/unused/home" }).pipe(Layer.provide(Process.host))), + Layer.provideMerge(SessionRuntime.layer), + Layer.provideMerge(Layer.succeed(Settings.Service, { load: () => Effect.succeed(Settings.defaults) })), + Layer.provideMerge(sandbox), + Layer.provideMerge(Context.layer), + Layer.provideMerge(SessionProjector.layer), + Layer.provideMerge(Session.storeLayer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(database), + Layer.provideMerge(Layer.succeed(Global.Service, Global.make({ home: "/unused/home" }))), + ); +}; + +const { effect: it } = testEffect(layer()); + +const wired = ["project", "space", "session", "thread", "entry", "inbox"] as const; + +// Test-only: every write to a wired row, and every tombstone, lands in `journal` with the rev it +// carries, in write order. Nothing in the store reads it. +const installJournal = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE journal (n INTEGER PRIMARY KEY AUTOINCREMENT, rev INTEGER, tbl TEXT, op TEXT, id TEXT)`; + for (const table of [...wired, "removed"]) { + for (const op of table === "removed" ? ["insert"] : ["insert", "update"]) { + yield* sql.unsafe(` + CREATE TRIGGER journal_${table}_${op} AFTER ${op.toUpperCase()} ON ${table} + BEGIN INSERT INTO journal (rev, tbl, op, id) VALUES (NEW.rev, '${table}', '${op}', NEW.id); END + `); + } + } + // Names the commit: an event row is the last write of its transaction, after its projectors. Every + // durable event of this script projects onto a wired row, so the head is its own revision. + yield* sql.unsafe(` + CREATE TRIGGER journal_event AFTER INSERT ON event + BEGIN INSERT INTO journal (rev, tbl, op, id) VALUES ((SELECT rev FROM store), 'event', NEW.type, NEW.id); END + `); +}); + +/** Ids in the order first seen, so the artifact reads `T1`, `E3` rather than uuids. */ +const aliases = () => { + const names = new Map(); + const counters = new Map(); + return (prefix: string, id: string) => { + const known = names.get(id); + if (known !== undefined) return known; + const next = (counters.get(prefix) ?? 0) + 1; + counters.set(prefix, next); + const name = `${prefix}${next}`; + names.set(id, name); + return name; + }; +}; +const prefixes: Record = { + project: "P", + space: "SP", + session: "S", + thread: "T", + entry: "E", + inbox: "I", +}; + +/** The trigger's message, off a rejected write. */ +const rejection = (exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return "accepted"; + let error: unknown = Cause.squash(exit.cause); + while (error instanceof Error && !error.message.includes("rev") && error.cause !== undefined) error = error.cause; + return error instanceof Error ? error.message : String(error); +}; + +describe("store", () => { + it( + "every write takes one revision in commit order; documents stamp threads; a delete leaves a tombstone", + Effect.gen(function* () { + requests = 0; + const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; + const sessions = yield* Session.Service; + const threads = yield* Thread.Service; + const documents = yield* Document.Service; + const events = yield* Event.Service; + const control = yield* Control.Service; + yield* installJournal; + const doorbell = yield* store.wake; + const name = aliases(); + const steps: Array<{ step: string; head: number }> = []; + const step = (label: string) => + store.head.pipe(Effect.map(({ rev }) => void steps.push({ step: label, head: rev }))); + + const before = yield* store.head; + yield* step("start"); + + // A plain directory, seeded under the id resolution gives it: the run's resolve then + // rewrites the same two rows instead of adopting strays. + const dir = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), + ); + const directory = realpathSync(dir.path); + const { spaceId, location } = yield* seedSpace({ + location: directory, + projectId: Project.provisional(SandboxInstance.ID.local, directory), + }); + yield* step("seed project and space"); + + const created = yield* sessions.create({ spaceId, directory: location }); + const sessionId = SessionSchema.ID.make(created.session.id); + const main = ThreadSchema.ID.make(created.thread.id); + yield* step("session.create"); + const rang = Exit.isSuccess(yield* PubSub.take(doorbell).pipe(Effect.timeout("1 second"), Effect.exit)); + + yield* events.publish(EventList.ThreadConfigured, { + timestamp: yield* DateTime.now, + sessionId, + threadId: main, + change: { model: { provider: "openai", id: "gpt-5.4" } }, + }); + yield* step("thread.configure (thread-scoped agent document)"); + + yield* control.run({ + sessionId, + threadId: main, + prompt: PromptSchema.Prompt.fromText("Say hi"), + delivery: "followUp", + }); + yield* step("prompt (tool call, settle, reply)"); + + const line = yield* threads.line(main); + const last = line.findLast(({ entry }) => entry.type === "assistant")!; + const forked = yield* threads.fork({ threadId: main, entryId: last.entry.id }); + yield* step("fork"); + + yield* documents.create({ + sessionId, + definition: { kind: "test.shared", version: 1, scope: "session" }, + value: { shared: true }, + seq: (yield* events.latestSequence(sessionId)) + 1, + }); + yield* step("session-scoped document"); + + const settled = yield* store.head; + const bad = { + // A plain write outside `Store.write`: it carries a stale rev. + staleRev: rejection( + yield* sql`UPDATE thread SET title = 'bad', rev = ${settled.rev - 1} WHERE id = ${main}`.pipe( + Effect.exit, + ), + ), + // Inside a write, an update that forgets to stamp keeps the old rev. + forgotStamp: rejection( + yield* store + .write( + Effect.gen(function* () { + yield* store.rev; + yield* sql`UPDATE thread SET title = 'bad' WHERE id = ${main}`; + }), + ) + .pipe(Effect.exit), + ), + // An insert without a revision taken. + insertWithoutRevision: rejection( + yield* sql` + INSERT INTO project (id, name, status, created_at, updated_at, rev) + VALUES ('bad', 'bad', 'active', 0, 0, ${settled.rev + 1}) + `.pipe(Effect.exit), + ), + // A rejected write rolls back with its revision. + headUnchanged: (yield* store.head).rev === settled.rev, + }; + + const rows = Effect.gen(function* () { + const out: Record> = {}; + for (const table of wired) { + const found = yield* sql.unsafe<{ id: string; rev: number }>( + `SELECT id, rev FROM ${table} ORDER BY rev, rowid`, + ); + out[table] = found.map((row) => `${name(prefixes[table]!, row.id)} rev=${row.rev}`); + } + return out; + }); + const beforeDelete = yield* rows; + + const deleted = yield* sessions.delete(sessionId); + yield* step("session.delete"); + const afterDelete = yield* rows; + const removed = yield* sql<{ kind: string; id: string; rev: number }>`SELECT kind, id, rev FROM removed`; + + const journal = yield* sql<{ rev: number; tbl: string; op: string; id: string }>` + SELECT rev, tbl, op, id FROM journal ORDER BY n + `; + const head = yield* store.head; + + // Revisions commit in order: the journal never steps back, and every step's writes carry + // revisions above the previous step's head. + const revs = journal.map((row) => row.rev); + expect(revs).toEqual(revs.toSorted((a, b) => a - b)); + expect(deleted).toBe(true); + expect(rang).toBe(true); + expect(forked.thread.id).toBeDefined(); + + // One commit per revision: the event that made it, or a plain write, and the rows it stamped. + const commits = new Map }>(); + for (const row of journal) { + const commit = commits.get(row.rev) ?? { rev: row.rev, by: "plain write", writes: [] }; + commits.set(row.rev, commit); + if (row.tbl === "event") { + expect(commit.writes.length).toBeGreaterThan(0); + commit.by = row.op.replace(/\.\d+$/, ""); + } else if (row.tbl === "removed") commit.writes.push(`removed ${name("S", row.id)}`); + else commit.writes.push(`${row.tbl} ${name(prefixes[row.tbl]!, row.id)} ${row.op}`); + } + + yield* Effect.promise(() => + expect( + `${JSON.stringify( + { + store: { + idIsSet: before.id.length > 0, + startRev: before.rev, + sameId: head.id === before.id, + head: head.rev, + }, + doorbellRang: rang, + steps, + commits: [...commits.values()], + rowsBeforeDelete: beforeDelete, + rowsAfterDelete: afterDelete, + removed: removed.map((row) => `${row.kind} ${name("S", row.id)} rev=${row.rev}`), + rejected: bad, + }, + null, + "\t", + )}\n`, + ).toMatchFileSnapshot("./__artifacts__/store.revision.json"), + ); + }), + ); +}); diff --git a/packages/harness/test/thread.test.ts b/packages/harness/test/thread.test.ts index 60b2383d..e4b363fe 100644 --- a/packages/harness/test/thread.test.ts +++ b/packages/harness/test/thread.test.ts @@ -584,23 +584,23 @@ describe("threads", () => { ); const results = { secondMain: yield* refused(sql` - INSERT INTO thread (id, session_id, origin_kind, created_seq, created_at, updated_at) - VALUES ('thr_second', ${sessionId}, 'main', 99, 0, 0) + INSERT INTO thread (id, session_id, origin_kind, created_seq, created_at, updated_at, rev) + VALUES ('thr_second', ${sessionId}, 'main', 99, 0, 0, (SELECT rev FROM store)) `), twoEntriesOneEvent: yield* refused(sql` - INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at) - VALUES ('entry_twin', ${sessionId}, ${main}, ${entry.seq}, ${entry.seq}, 'custom', 'committed', '{}', 0, 0) + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at, rev) + VALUES ('entry_twin', ${sessionId}, ${main}, ${entry.seq}, ${entry.seq}, 'custom', 'committed', '{}', 0, 0, (SELECT rev FROM store)) `), twoRevisionsOneEvent: yield* refused(sql` INSERT INTO revision (document_id, seq, version, value) VALUES (${document.id}, 1, 1, '{}') `), draftWithCommit: yield* refused(sql` - INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at) - VALUES ('entry_draft', ${sessionId}, ${main}, 500, 500, 'assistant', 'draft', '{}', 0, 0) + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at, rev) + VALUES ('entry_draft', ${sessionId}, ${main}, 500, 500, 'assistant', 'draft', '{}', 0, 0, (SELECT rev FROM store)) `), forkWithoutCut: yield* refused(sql` - INSERT INTO thread (id, session_id, origin_kind, origin_thread_id, created_seq, created_at, updated_at) - VALUES ('thr_nocut', ${sessionId}, 'fork', ${main}, 99, 0, 0) + INSERT INTO thread (id, session_id, origin_kind, origin_thread_id, created_seq, created_at, updated_at, rev) + VALUES ('thr_nocut', ${sessionId}, 'fork', ${main}, 99, 0, 0, (SELECT rev FROM store)) `), }; expect(Object.values(results).every((result) => result.includes("constraint failed"))).toBe(true); From 0e0654bbbb63a683ea0316b6cf0295df95656ef1 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 19:09:31 +0530 Subject: [PATCH 04/56] fix(harness): stamp rev on run-state projections Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/thread/thread.ts | 7 +- .../test/__artifacts__/store.revision.json | 82 +++++++++++-------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index 66acb2f2..6c3d4351 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -964,19 +964,20 @@ export const layer = Layer.effect( change: RunChange, ) { const time = DateTime.toEpochMillis(at); + const rev = yield* store.rev; const state: ThreadSchema.RunState = change._tag === "started" ? "starting" : change._tag === "running" ? "running" : "idle"; // SET reads the old row, so `run_state_at` moves only when the state does. const stateAt = sql`CASE WHEN run_state = ${state} THEN run_state_at ELSE ${time} END`; if (change._tag === "running") { - yield* sql`UPDATE thread SET run_state_at = ${stateAt}, run_state = ${state} WHERE id = ${threadId}`; + yield* sql`UPDATE thread SET run_state_at = ${stateAt}, run_state = ${state}, rev = ${rev} WHERE id = ${threadId}`; return; } if (change._tag === "started") { yield* sql` UPDATE thread SET run_state_at = ${stateAt}, run_state = ${state}, run_started_at = ${time}, - run_ended_at = NULL, run_outcome = NULL, run_error = NULL + run_ended_at = NULL, run_outcome = NULL, run_error = NULL, rev = ${rev} WHERE id = ${threadId} `; return; @@ -985,7 +986,7 @@ export const layer = Layer.effect( yield* sql` UPDATE thread SET run_state_at = ${stateAt}, run_state = ${state}, run_ended_at = ${time}, - run_outcome = ${change.outcome}, run_error = ${error} + run_outcome = ${change.outcome}, run_error = ${error}, rev = ${rev} WHERE id = ${threadId} `; }, Effect.orDie); diff --git a/packages/harness/test/__artifacts__/store.revision.json b/packages/harness/test/__artifacts__/store.revision.json index 6a64e31c..e306d91f 100644 --- a/packages/harness/test/__artifacts__/store.revision.json +++ b/packages/harness/test/__artifacts__/store.revision.json @@ -3,7 +3,7 @@ "idIsSet": true, "startRev": 0, "sameId": true, - "head": 22 + "head": 24 }, "doorbellRang": true, "steps": [ @@ -25,19 +25,19 @@ }, { "step": "prompt (tool call, settle, reply)", - "head": 19 + "head": 21 }, { "step": "fork", - "head": 20 + "head": 22 }, { "step": "session-scoped document", - "head": 21 + "head": 23 }, { "step": "session.delete", - "head": 22 + "head": 24 } ], "commits": [ @@ -88,6 +88,13 @@ }, { "rev": 7, + "by": "session.execution.started", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 8, "by": "plain write", "writes": [ "project P1 update", @@ -95,7 +102,7 @@ ] }, { - "rev": 8, + "rev": 9, "by": "session.prompt.promoted", "writes": [ "inbox E1 update", @@ -106,7 +113,7 @@ ] }, { - "rev": 9, + "rev": 10, "by": "session.thread.system.changed", "writes": [ "entry E2 insert", @@ -115,23 +122,24 @@ ] }, { - "rev": 10, + "rev": 11, "by": "plain write", "writes": [ "thread T1 update" ] }, { - "rev": 11, + "rev": 12, "by": "session.llm.started", "writes": [ + "thread T1 update", "entry E3 insert", "thread T1 update", "session S1 update" ] }, { - "rev": 12, + "rev": 13, "by": "session.llm.ended", "writes": [ "entry E3 update", @@ -141,14 +149,14 @@ ] }, { - "rev": 13, + "rev": 14, "by": "session.tool.settled", "writes": [ "entry E3 update" ] }, { - "rev": 14, + "rev": 15, "by": "session.turn.ended", "writes": [ "entry E3 update", @@ -157,23 +165,24 @@ ] }, { - "rev": 15, + "rev": 16, "by": "plain write", "writes": [ "thread T1 update" ] }, { - "rev": 16, + "rev": 17, "by": "session.llm.started", "writes": [ + "thread T1 update", "entry E4 insert", "thread T1 update", "session S1 update" ] }, { - "rev": 17, + "rev": 18, "by": "session.llm.ended", "writes": [ "entry E4 update", @@ -183,7 +192,7 @@ ] }, { - "rev": 18, + "rev": 19, "by": "session.turn.ended", "writes": [ "entry E4 update", @@ -192,7 +201,7 @@ ] }, { - "rev": 19, + "rev": 20, "by": "plain write", "writes": [ "thread T1 update", @@ -200,7 +209,14 @@ ] }, { - "rev": 20, + "rev": 21, + "by": "session.execution.succeeded", + "writes": [ + "thread T1 update" + ] + }, + { + "rev": 22, "by": "session.thread.forked", "writes": [ "thread T2 insert", @@ -208,7 +224,7 @@ ] }, { - "rev": 21, + "rev": 23, "by": "plain write", "writes": [ "thread T1 update", @@ -216,7 +232,7 @@ ] }, { - "rev": 22, + "rev": 24, "by": "plain write", "writes": [ "removed S1" @@ -225,34 +241,34 @@ ], "rowsBeforeDelete": { "project": [ - "P1 rev=7" + "P1 rev=8" ], "space": [ - "SP1 rev=7" + "SP1 rev=8" ], "session": [ - "S1 rev=17" + "S1 rev=18" ], "thread": [ - "T1 rev=21", - "T2 rev=21" + "T1 rev=23", + "T2 rev=23" ], "entry": [ - "E1 rev=8", - "E2 rev=9", - "E3 rev=14", - "E4 rev=18" + "E1 rev=9", + "E2 rev=10", + "E3 rev=15", + "E4 rev=19" ], "inbox": [ - "E1 rev=8" + "E1 rev=9" ] }, "rowsAfterDelete": { "project": [ - "P1 rev=7" + "P1 rev=8" ], "space": [ - "SP1 rev=7" + "SP1 rev=8" ], "session": [], "thread": [], @@ -260,7 +276,7 @@ "inbox": [] }, "removed": [ - "session S1 rev=22" + "session S1 rev=24" ], "rejected": { "staleRev": "rev is not the current revision", From b0e4ae79c5b1f6b7479e68afadc594f19265f60f Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 19:40:21 +0530 Subject: [PATCH 05/56] fix(harness): end an abandoned run with its failure only A run the loop ends itself (crashloop, interrupted, truncated) publishes session.run.failed and returns normally, so the coordinator also published session.execution.succeeded and the thread row lost the failure reason. Runner.run and abandon now return an outcome; the execution's settled hook publishes no terminal after a run that already ended itself. Co-Authored-By: Claude Opus 5.5 --- .../__artifacts__/recovery.crashloop.json | 16 ++++++++++++--- .../__artifacts__/recovery.interrupted.json | 16 ++++++++++++--- .../__artifacts__/recovery.toolcrash.json | 3 --- .../__artifacts__/recovery.truncated.json | 16 ++++++++++++--- packages/codework/test/recovery.test.ts | 20 ++++++++++++++++--- packages/harness/src/runner/execute.ts | 15 ++++++++++++-- packages/harness/src/runner/loop.ts | 18 +++++++++++------ packages/harness/src/runner/run.ts | 10 ++++++++-- 8 files changed, 89 insertions(+), 25 deletions(-) diff --git a/packages/codework/test/__artifacts__/recovery.crashloop.json b/packages/codework/test/__artifacts__/recovery.crashloop.json index 340b6a0b..a6202630 100644 --- a/packages/codework/test/__artifacts__/recovery.crashloop.json +++ b/packages/codework/test/__artifacts__/recovery.crashloop.json @@ -138,9 +138,6 @@ "reason": "crashloop", "message": "The run was interrupted 5 times in a row; it is not resumed again" }, - { - "type": "session.execution.succeeded" - }, { "type": "session.prompt.admitted", "delivery": "steer", @@ -228,6 +225,19 @@ ] } ], + "run": { + "state": "idle", + "stateAt": "30 session.run.failed", + "startedAt": "28 session.execution.started", + "endedAt": "30 session.run.failed", + "outcome": "failed", + "error": { + "_tag": "runFailed", + "reason": "crashloop", + "message": "The run was interrupted 5 times in a row; it is not resumed again" + }, + "execution": null + }, "requests": [ [ { diff --git a/packages/codework/test/__artifacts__/recovery.interrupted.json b/packages/codework/test/__artifacts__/recovery.interrupted.json index bf41c9f7..88ba0a37 100644 --- a/packages/codework/test/__artifacts__/recovery.interrupted.json +++ b/packages/codework/test/__artifacts__/recovery.interrupted.json @@ -70,9 +70,6 @@ "type": "session.run.failed", "reason": "interrupted", "message": "The host stopped while the model was answering, on attempt 2" - }, - { - "type": "session.execution.succeeded" } ], "history": [ @@ -102,6 +99,19 @@ "text": "Then this." } ], + "run": { + "state": "idle", + "stateAt": "16 session.run.failed", + "startedAt": "14 session.execution.started", + "endedAt": "16 session.run.failed", + "outcome": "failed", + "error": { + "_tag": "runFailed", + "reason": "interrupted", + "message": "The host stopped while the model was answering, on attempt 2" + }, + "execution": null + }, "requests": [ [ { diff --git a/packages/codework/test/__artifacts__/recovery.toolcrash.json b/packages/codework/test/__artifacts__/recovery.toolcrash.json index 0a0dc50b..fae15bb3 100644 --- a/packages/codework/test/__artifacts__/recovery.toolcrash.json +++ b/packages/codework/test/__artifacts__/recovery.toolcrash.json @@ -184,9 +184,6 @@ "type": "session.run.failed", "reason": "crashloop", "message": "The run was interrupted 5 times in a row; it is not resumed again" - }, - { - "type": "session.execution.succeeded" } ], "history": [ diff --git a/packages/codework/test/__artifacts__/recovery.truncated.json b/packages/codework/test/__artifacts__/recovery.truncated.json index 7df4fbdd..6da9d032 100644 --- a/packages/codework/test/__artifacts__/recovery.truncated.json +++ b/packages/codework/test/__artifacts__/recovery.truncated.json @@ -72,9 +72,6 @@ "type": "session.run.failed", "reason": "truncated", "message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended" - }, - { - "type": "session.execution.succeeded" } ], "history": [ @@ -110,6 +107,19 @@ ] } ], + "run": { + "state": "idle", + "stateAt": "14 session.run.failed", + "startedAt": "3 session.execution.started", + "endedAt": "14 session.run.failed", + "outcome": "failed", + "error": { + "_tag": "runFailed", + "reason": "truncated", + "message": "The response hit the output token limit with tool calls 2 times in a row; the run is ended" + }, + "execution": null + }, "requests": [ [ { diff --git a/packages/codework/test/recovery.test.ts b/packages/codework/test/recovery.test.ts index 3856a2fd..6a3e0d30 100644 --- a/packages/codework/test/recovery.test.ts +++ b/packages/codework/test/recovery.test.ts @@ -239,6 +239,13 @@ const runRow = (home: string, threadId: string, events: ReadonlyArray { + const row = runRow(home, ids.threadId, await log(url, ids)); + expect(row).toMatchObject({ state: "idle", outcome: "failed", error: { _tag: "runFailed", reason } }); + return row; +}; + describe("recovery after the host is killed", () => { it("the stored run of a killed host is swept as a crash before clients connect, then resumed", async () => { const resend = gate(); @@ -400,7 +407,7 @@ describe("recovery after the host is killed", () => { it("a request killed with no retry left ends the run as interrupted, its follow-up still queued", async () => { const fake = await streaming((_body, response) => hang(response)); - const { start } = setup(fake.url, { retry: { maxRetries: 1, baseDelayMs: 10, maxDelayMs: 1_000 } }); + const { home, start } = setup(fake.url, { retry: { maxRetries: 1, baseDelayMs: 10, maxDelayMs: 1_000 } }); const first = await start(); const created = await call(first.url, (rpc) => rpc["session.create"]({})); @@ -420,6 +427,7 @@ describe("recovery after the host is killed", () => { const events = await logged(third.url, ids, "session.run.failed"); await call(third.url, (rpc) => rpc["thread.wait"](ids)); const queue = await call(third.url, (rpc) => rpc["thread.queue"](ids)); + const run = await endedRow(third.url, home, ids, "interrupted"); expect(fake.requests).toHaveLength(2); expect(summary(events.find((event) => event.type === "session.run.failed")!)).toEqual({ @@ -434,6 +442,7 @@ describe("recovery after the host is killed", () => { log: events.map(summary), history: await history(third.url, ids), queue: queue.entries.map(({ delivery, text }) => ({ delivery, text })), + run, requests: fake.requests.map(sent), }), ).toMatchFileSnapshot("./__artifacts__/recovery.interrupted.json"); @@ -444,7 +453,7 @@ describe("recovery after the host is killed", () => { const fake = await streaming((_body, response) => answering ? answer(response, "Back to normal.") : hang(response), ); - const { start } = setup(fake.url, { retry: { maxRetries: 10, baseDelayMs: 10, maxDelayMs: 1_000 } }); + const { home, start } = setup(fake.url, { retry: { maxRetries: 10, baseDelayMs: 10, maxDelayMs: 1_000 } }); let server = await start(); const created = await call(server.url, (rpc) => rpc["session.create"]({})); @@ -463,6 +472,7 @@ describe("recovery after the host is killed", () => { expect(summary(ended.find((event) => event.type === "session.run.failed")!)).toMatchObject({ reason: "crashloop", }); + const run = await endedRow(server.url, home, ids, "crashloop"); answering = true; await call(server.url, (rpc) => rpc["thread.prompt"]({ ...ids, text: "Try again." })); @@ -474,6 +484,8 @@ describe("recovery after the host is killed", () => { artifact({ log: events.map(summary), history: await history(server.url, ids), + // Before the next prompt. + run, requests: fake.requests.map(sent), }), ).toMatchFileSnapshot("./__artifacts__/recovery.crashloop.json"); @@ -523,7 +535,7 @@ describe("recovery after the host is killed", () => { response.send(toolCalls([{ name: "bash", arguments: { command: "echo cut" } }])); response.finish("length"); }); - const { start } = setup(fake.url, retry); + const { home, start } = setup(fake.url, retry); const server = await start(); const created = await call(server.url, (rpc) => rpc["session.create"]({})); @@ -532,6 +544,7 @@ describe("recovery after the host is killed", () => { const events = await logged(server.url, ids, "session.run.failed"); await call(server.url, (rpc) => rpc["thread.wait"](ids)); + const run = await endedRow(server.url, home, ids, "truncated"); expect(fake.requests).toHaveLength(2); const settled = events.filter((event) => event.type === "session.tool.settled").map(summary); expect(settled).toHaveLength(2); @@ -546,6 +559,7 @@ describe("recovery after the host is killed", () => { artifact({ log: events.map(summary), history: await history(server.url, ids), + run, requests: fake.requests.map(sent), }), ).toMatchFileSnapshot("./__artifacts__/recovery.truncated.json"); diff --git a/packages/harness/src/runner/execute.ts b/packages/harness/src/runner/execute.ts index 13cfee11..f768eb22 100644 --- a/packages/harness/src/runner/execute.ts +++ b/packages/harness/src/runner/execute.ts @@ -77,6 +77,8 @@ export const layer = Layer.effect( const abandoning = new Map(); // The log position at each user stop: prompts admitted after it are new work, not parked by it. const stoppedAt = new Map(); + // Threads whose last drain ended its run itself, by publishing `session.run.failed`. + const ended = new Set(); const coordinator = yield* RunCoordinator.make({ // The run is recorded unfinished before it starts, so a process dying anywhere in it @@ -110,6 +112,9 @@ export const layer = Layer.effect( !interrupted ? "finished" : reason === "user" ? "stopped" : "closed", stop, ); + // One terminal per run: an abandoned run already ended with its failure and reason, which a + // success published after it would overwrite. + if (ended.delete(threadId) && Exit.isSuccess(exit)) return; const sessionId = yield* sessionOf(threadId); if (Option.isNone(sessionId)) return; const base = { sessionId: sessionId.value, threadId, timestamp: yield* DateTime.now }; @@ -127,6 +132,11 @@ export const layer = Layer.effect( }), ), drain: Effect.fnUntraced(function* (threadId: ThreadId, force) { + ended.delete(threadId); + const record = (outcome: Runner.Outcome) => + Effect.sync(() => { + if (outcome === "ended") ended.add(threadId); + }); const thread = yield* threads.get(threadId); // Deleted with its session after something woke it: nothing is left to run. if (Option.isNone(thread)) return; @@ -135,7 +145,8 @@ export const layer = Layer.effect( if (recovered && Option.isNone(thread.value.execution)) return; // Only the drain the abandon itself started: one it joined is a live run, which must stand. const abandoned = recovered && force ? abandoning.get(threadId) : undefined; - if (abandoned !== undefined) return yield* runner.abandon({ threadId, message: abandoned }); + if (abandoned !== undefined) + return yield* runner.abandon({ threadId, message: abandoned }).pipe(Effect.flatMap(record)); // The thread's placement, else the session's space and directory. A placement names its // space in a document no FK guards, so a space gone since is typed: `relink --thread` // is the repair. @@ -179,7 +190,7 @@ export const layer = Layer.effect( yield* mcp.lease({ sessionId, threadId }); return yield* runner .run({ threadId, force, recovered }) - .pipe(Effect.provideContext(Context.merge(locationContext, mountContext))); + .pipe(Effect.provideContext(Context.merge(locationContext, mountContext)), Effect.flatMap(record)); }); return yield* scopedRun.pipe( diff --git a/packages/harness/src/runner/loop.ts b/packages/harness/src/runner/loop.ts index a8fce6b3..d77256f1 100644 --- a/packages/harness/src/runner/loop.ts +++ b/packages/harness/src/runner/loop.ts @@ -277,6 +277,7 @@ export const layer = (options: Options = {}) => yield* threads.idle(threadId); }), ); + return "ended" as const; }); const settleTools = Effect.fn("Loop.settleTools", { level: "Info" })(function* ( @@ -616,7 +617,7 @@ export const layer = (options: Options = {}) => const hasSteer = yield* inputs.hasPending(input.threadId, "steer"); const hasFollowUp = hasSteer ? false : yield* inputs.hasPending(input.threadId, "followUp"); - if (!input.force && !hasSteer && !hasFollowUp) return; + if (!input.force && !hasSteer && !hasFollowUp) return "drained" as const; yield* Effect.annotateCurrentSpan({ "gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "codework", @@ -674,7 +675,7 @@ export const layer = (options: Options = {}) => }; const decision = completed === undefined ? undefined : yield* hooks.finishTurn(completed); // Queued steers and follow-ups stay for the next run. - if (decision === "end") return; + if (decision === "end") return "drained" as const; if (truncations >= TRUNCATION_LIMIT) { return yield* endRun( snapshot.sessionId, @@ -691,14 +692,14 @@ export const layer = (options: Options = {}) => // Only tool results outrank a follow-up. A forced continuation rides along with it: the // follow-up is the natural next request, and the continuation only its fallback. const followUp = !toolUse && !steer && (yield* inputs.hasPending(input.threadId, "followUp")); - if (!continuing && !steer && !followUp) return; + if (!continuing && !steer && !followUp) return "drained" as const; promotion = followUp ? "followUp" : "steer"; const next = holdServers(start, yield* state.snapshot(input.threadId)); snapshot = completed === undefined ? next : yield* hooks.prepareNextTurn(completed, next); } }); - yield* Telemetry.timed(drain, (seconds, exit) => + return yield* Telemetry.timed(drain, (seconds, exit) => Telemetry.record(Telemetry.agentDuration, seconds, Telemetry.withError({}, exit)), ); }); @@ -708,9 +709,14 @@ export const layer = (options: Options = {}) => readonly message: string; }) { const thread = yield* threads.get(input.threadId); - if (Option.isNone(thread)) return; + if (Option.isNone(thread)) return "drained" as const; yield* healDraft(input.threadId, Option.isSome(thread.value.attempt) ? input.message : undefined); - yield* endRun(SessionSchema.ID.make(thread.value.sessionId), input.threadId, "crashloop", input.message); + return yield* endRun( + SessionSchema.ID.make(thread.value.sessionId), + input.threadId, + "crashloop", + input.message, + ); }); return Runner.Service.of({ run, abandon }); diff --git a/packages/harness/src/runner/run.ts b/packages/harness/src/runner/run.ts index 51f95907..4b3a9d67 100644 --- a/packages/harness/src/runner/run.ts +++ b/packages/harness/src/runner/run.ts @@ -185,6 +185,9 @@ export type RunError = | SandboxMountError | SandboxFileSystem.FileSystemError; +/** `ended`: the run published its own terminal, `session.run.failed`; `drained`: it left that to its execution. */ +export type Outcome = "ended" | "drained"; + export interface Interface { /** Drains eligible durable work. Explicit runs perform one provider attempt even when no work is eligible. */ readonly run: (input: { @@ -192,12 +195,15 @@ export interface Interface { readonly force: boolean; /** Resuming a run a previous process left unfinished: its cut-off reply is answered first. */ readonly recovered?: boolean; - }) => Effect.Effect; + }) => Effect.Effect; /** * Ends a run the server no longer resumes: settles what the dead process left of its reply, * records `session.run.failed` with `message`, and leaves the thread idle, its queue kept. */ - readonly abandon: (input: { readonly threadId: ThreadSchema.ID; readonly message: string }) => Effect.Effect; + readonly abandon: (input: { + readonly threadId: ThreadSchema.ID; + readonly message: string; + }) => Effect.Effect; } export class Service extends Context.Service()("@codeworksh/harness/runner/run/Service") {} From cb6788d287c93dda21e106ee040748cefd04f24a Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 20:00:01 +0530 Subject: [PATCH 06/56] feat(harness): read the index by revision, with fork cuts and full session and thread rows Co-Authored-By: Claude Opus 5.5 --- packages/harness/src/effect.ts | 1 + packages/harness/src/effect/session.ts | 77 +++++++++-- packages/harness/src/effect/sync.ts | 171 +++++++++++++++++++++++++ packages/harness/src/thread/thread.ts | 17 +++ 4 files changed, 256 insertions(+), 10 deletions(-) create mode 100644 packages/harness/src/effect/sync.ts diff --git a/packages/harness/src/effect.ts b/packages/harness/src/effect.ts index fad546d3..8a59f60b 100644 --- a/packages/harness/src/effect.ts +++ b/packages/harness/src/effect.ts @@ -1,6 +1,7 @@ export { Harness } from "./effect/harness.ts"; export { Sandbox } from "./effect/sandbox.ts"; export { Session } from "./effect/session.ts"; +export { Sync } from "./effect/sync.ts"; export { Control } from "./control.ts"; export { Database } from "./db/db.ts"; export { Store } from "./db/store.ts"; diff --git a/packages/harness/src/effect/session.ts b/packages/harness/src/effect/session.ts index 6fdb59b2..67a6eee1 100644 --- a/packages/harness/src/effect/session.ts +++ b/packages/harness/src/effect/session.ts @@ -99,6 +99,8 @@ type PromptResult = Effect.Effect, Effect.Error>; + /** Epoch milliseconds. */ + readonly createdAt: number; /** Last activity in any of its threads, epoch milliseconds. */ readonly updatedAt: number; readonly mainThreadId: ThreadSchema.ID; } -export interface ThreadInfo { +/** How a thread came to exist; `cut` is the committed seq its inherited line ends at. */ +export type ThreadOrigin = + | { readonly kind: "main" } + | { readonly kind: "fork"; readonly threadId: ThreadSchema.ID; readonly entryId: string; readonly cut: number } + | { + readonly kind: "subagent"; + readonly threadId: ThreadSchema.ID; + readonly callId: string; + readonly entryId?: string; + /** Present once `entryId` is committed. */ + readonly cut?: number; + }; + +/** A thread without its session. */ +export interface ThreadSummary { readonly id: ThreadSchema.ID; readonly sessionId: SessionSchema.ID; - readonly origin: ThreadSchema.Origin; + readonly origin: ThreadOrigin; /** Absent until its first own prompt; clients show a fork's source title until then. */ readonly title?: string; /** Last activity, epoch milliseconds: every appended entry moves it. */ @@ -141,6 +161,17 @@ export interface ThreadInfo { readonly outcome?: ThreadSchema.RunOutcome; readonly error?: ThreadSchema.RunError; }; + /** Automatic restarts of the current run. */ + readonly restarts: number; + /** The 1-based provider request in flight; absent when none is. */ + readonly attempt?: number; + /** The seq of the event that created it. */ + readonly createdSeq: number; + /** Epoch milliseconds. */ + readonly createdAt: number; +} + +export interface ThreadInfo extends ThreadSummary { readonly session: Info; } @@ -259,30 +290,44 @@ export const info = Effect.fn("Session.info")(function* (id: SessionSchema.ID) { : Option.getOrUndefined(yield* sandboxes.get(space.env)); const hostDir = Option.getOrUndefined(row.hostDir); const project = Option.getOrUndefined(yield* sessions.project(id)); + const tag = Option.getOrUndefined(row.tag); + const metadata = Option.getOrUndefined(row.metadata); return { id, + spaceId: row.spaceId, + slug: row.slug, directory: row.directory, ...(hostDir === undefined ? {} : { hostDir }), hasHostLink: hostDir !== undefined, ...(sandbox === undefined ? {} : { sandbox }), ...(project === undefined ? {} : { project: { id: project.id, name: project.name } }), + ...(tag === undefined ? {} : { tag }), + ...(metadata === undefined ? {} : { metadata }), + createdAt: DateTime.toEpochMillis(row.createdAt), updatedAt: DateTime.toEpochMillis(row.updatedAt), mainThreadId: ThreadSchema.ID.make(main.value.id), } satisfies Info; }); -const originOf = (row: ThreadRow): ThreadSchema.Origin => { +const originOf = (row: ThreadRow, cut: number | undefined): ThreadOrigin => { const threadId = Option.getOrUndefined(row.originThreadId); const entryId = Option.getOrUndefined(row.originEntryId); const callId = Option.getOrUndefined(row.originCallId); - if (row.originKind === "fork" && threadId !== undefined && entryId !== undefined) - return { kind: "fork", threadId, entryId }; + // A fork is only ever cut at a committed entry (`Thread.projectForked`). + if (row.originKind === "fork" && threadId !== undefined && entryId !== undefined && cut !== undefined) + return { kind: "fork", threadId, entryId, cut }; if (row.originKind === "subagent" && threadId !== undefined && callId !== undefined) - return { kind: "subagent", threadId, callId, ...(entryId === undefined ? {} : { entryId }) }; + return { + kind: "subagent", + threadId, + callId, + ...(entryId === undefined ? {} : { entryId }), + ...(cut === undefined ? {} : { cut }), + }; return { kind: "main" }; }; -export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadSchema.ID) { +export const threadSummary = Effect.fn("Session.threadSummary")(function* (id: ThreadSchema.ID) { const threads = yield* Thread.Service; const documents = yield* Document.Service; const found = yield* threads.get(id); @@ -297,10 +342,14 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS const endedAt = millis(row.runEndedAt); const outcome = Option.getOrUndefined(row.runOutcome); const error = Option.getOrUndefined(row.runError); + const attempt = Option.getOrUndefined(row.attempt); return { id, sessionId: SessionSchema.ID.make(row.sessionId), - origin: originOf(row), + origin: originOf( + row, + Option.isSome(row.originEntryId) ? Option.getOrUndefined(yield* threads.cut(id)) : undefined, + ), ...(title === undefined ? {} : { title }), updatedAt: DateTime.toEpochMillis(row.updatedAt), ...(agent.model === undefined ? {} : { model: agent.model }), @@ -326,8 +375,16 @@ export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadS ...(error === undefined ? {} : { error }), }, }), - session: yield* info(SessionSchema.ID.make(row.sessionId)), - } satisfies ThreadInfo; + restarts: row.restarts, + ...(attempt === undefined ? {} : { attempt }), + createdSeq: row.createdSeq, + createdAt: DateTime.toEpochMillis(row.createdAt), + } satisfies ThreadSummary; +}); + +export const threadInfo = Effect.fn("Session.threadInfo")(function* (id: ThreadSchema.ID) { + const summary = yield* threadSummary(id); + return { ...summary, session: yield* info(summary.sessionId) } satisfies ThreadInfo; }); const makeThread = Effect.fn("Session.makeThread")(function* (sessionId: SessionSchema.ID, id: ThreadSchema.ID) { diff --git a/packages/harness/src/effect/sync.ts b/packages/harness/src/effect/sync.ts new file mode 100644 index 00000000..117876bf --- /dev/null +++ b/packages/harness/src/effect/sync.ts @@ -0,0 +1,171 @@ +import { DateTime, Effect, Option, Schema } from "effect"; +import { SqlClient, SqlSchema, type Statement } from "effect/sql"; +import { ProjectRow, SpaceRow } from "../db/schema.sql.ts"; +import type { ProjectSchema } from "../project/schema.ts"; +import type { AbsolutePath } from "../schema.ts"; +import type { SpaceSchema } from "../space/schema.ts"; +import { SessionSchema } from "../session/schema.ts"; +import { ThreadSchema } from "../thread/schema.ts"; +import { type Info, info, type ThreadSummary, threadSummary } from "./session.ts"; + +/** Where a read in revision order stands: past revision `rev` whole, or past `key` within it. */ +export interface Cursor { + readonly rev: number; + readonly key?: { readonly kind: string; readonly id: string }; +} + +export interface Page { + readonly items: ReadonlyArray; + /** Where the next page starts; absent once the range is read. */ + readonly next?: Cursor; +} + +const Row = Schema.Struct({ kind: Schema.String, id: Schema.String, rev: Schema.Int }); +type Row = typeof Row.Type; +const decodeRows = Schema.decodeUnknownEffect(Schema.Array(Row)); + +/** + * One page of the rows `select` names, as `kind`, `id` and `rev`, past `from` and up to revision + * `upto`, in `(rev, kind, id)` order. `select` filters each table on the revision range it is given, + * so every table reads through its `rev` index. Rows of one revision can span pages. + */ +export const page = Effect.fn("Sync.page")(function* ( + select: (range: Statement.Fragment) => Statement.Fragment, + from: Cursor, + upto: number, + limit: number, +) { + const sql = yield* SqlClient.SqlClient; + const after = + from.key === undefined + ? sql`rev > ${from.rev}` + : sql`(rev, kind, id) > (${from.rev}, ${from.key.kind}, ${from.key.id})`; + const rows = yield* sql` + SELECT kind, id, rev FROM (${select(sql`rev >= ${from.rev} AND rev <= ${upto}`)}) + WHERE ${after} + ORDER BY rev, kind, id + LIMIT ${limit} + `.pipe(Effect.flatMap(decodeRows), Effect.orDie); + const last = rows.at(-1); + return { + items: rows, + ...(last === undefined || rows.length < limit + ? {} + : { next: { rev: last.rev, key: { kind: last.kind, id: last.id } } }), + } satisfies Page; +}); + +export interface ProjectInfo { + readonly id: string; + readonly name: string; + readonly status: ProjectSchema.Status; + /** Epoch milliseconds. */ + readonly createdAt: number; + readonly updatedAt: number; +} + +export interface SpaceInfo { + readonly id: string; + readonly projectId: string; + readonly location: AbsolutePath; + readonly kind: SpaceSchema.Kind; + /** The sandbox it is in; absent for the host. */ + readonly env?: string; + readonly status: SpaceSchema.Status; + /** Epoch milliseconds. */ + readonly createdAt: number; + readonly updatedAt: number; +} + +/** A row of the index (every project, space, session and thread) at the revision that last wrote it. */ +export type IndexChange = + | { readonly _tag: "Project"; readonly project: ProjectInfo; readonly rev: number } + | { readonly _tag: "Space"; readonly space: SpaceInfo; readonly rev: number } + | { readonly _tag: "Session"; readonly session: Info; readonly rev: number } + | { readonly _tag: "Thread"; readonly thread: ThreadSummary; readonly rev: number } + /** Only sessions are deleted; their threads go with them. */ + | { readonly _tag: "Removed"; readonly kind: "session"; readonly id: string; readonly rev: number }; + +/** + * One page of the index rows past `from`, up to revision `upto`, read as one snapshot so every row + * the page names is still there to read. + */ +export const index = Effect.fn("Sync.index")(function* (from: Cursor, upto: number, limit: number) { + const sql = yield* SqlClient.SqlClient; + const findProject = SqlSchema.findOne({ + Request: Schema.String, + Result: ProjectRow, + execute: (id) => sql`SELECT * FROM project WHERE id = ${id}`, + }); + const findSpace = SqlSchema.findOne({ + Request: Schema.String, + Result: SpaceRow, + execute: (id) => sql`SELECT * FROM space WHERE id = ${id}`, + }); + const select = (range: Statement.Fragment) => sql` + SELECT 'project' AS kind, id, rev FROM project WHERE ${range} + UNION ALL SELECT 'space', id, rev FROM space WHERE ${range} + UNION ALL SELECT 'session', id, rev FROM session WHERE ${range} + UNION ALL SELECT 'thread', id, rev FROM thread WHERE ${range} + UNION ALL SELECT 'removed', id, rev FROM removed WHERE kind = 'session' AND ${range} + `; + const change = Effect.fnUntraced(function* ({ kind, id, rev }: Row) { + switch (kind) { + case "project": { + const row = yield* findProject(id); + return { + _tag: "Project", + project: { + id: row.id, + name: row.name, + status: row.status, + createdAt: DateTime.toEpochMillis(row.createdAt), + updatedAt: DateTime.toEpochMillis(row.updatedAt), + }, + rev, + } satisfies IndexChange; + } + case "space": { + const row = yield* findSpace(id); + const env = Option.getOrUndefined(row.env); + return { + _tag: "Space", + space: { + id: row.id, + projectId: row.projectId, + location: row.location, + kind: row.kind, + ...(env === undefined ? {} : { env }), + status: row.status, + createdAt: DateTime.toEpochMillis(row.createdAt), + updatedAt: DateTime.toEpochMillis(row.updatedAt), + }, + rev, + } satisfies IndexChange; + } + case "session": + return { _tag: "Session", session: yield* info(SessionSchema.ID.make(id)), rev } satisfies IndexChange; + case "thread": + return { + _tag: "Thread", + thread: yield* threadSummary(ThreadSchema.ID.make(id)), + rev, + } satisfies IndexChange; + default: + return { _tag: "Removed", kind: "session", id, rev } satisfies IndexChange; + } + }); + return yield* sql + .withTransaction( + Effect.gen(function* () { + const { items, next } = yield* page(select, from, upto, limit); + return { + items: yield* Effect.forEach(items, change), + ...(next === undefined ? {} : { next }), + } satisfies Page; + }), + ) + .pipe(Effect.orDie); +}); + +export * as Sync from "./sync.ts"; diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index 6c3d4351..7dbcdad5 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -12,6 +12,7 @@ import { type ToolStatus, } from "../db/schema.sql.ts"; import { Store } from "../db/store.ts"; +import { NonNegativeInt } from "../schema.ts"; import { Document } from "../document/document.ts"; import { DocumentSchema } from "../document/schema.ts"; import { Event } from "../event/event.ts"; @@ -160,6 +161,8 @@ export interface Interface { /** Threads newest first: of one session, or of the whole database. */ readonly list: (sessionId?: SessionSchema.ID) => Effect.Effect>; readonly entry: (entryId: string) => Effect.Effect>; + /** Where the thread's inherited line ends: its origin entry's committed seq; none when it inherits nothing. */ + readonly cut: (threadId: string) => Effect.Effect>; /** * The thread's line: its own entries, then its source's through the fork entry, recursively, in * seq order. Every state is included; context assembly drops what is not committed. @@ -294,6 +297,15 @@ export const layer = Layer.effect( execute: (id) => sql`SELECT * FROM entry WHERE id = ${id}`, }); + const findCut = SqlSchema.findOneOption({ + Request: Schema.String, + Result: Schema.Struct({ cut: NonNegativeInt }), + execute: (threadId) => sql` + SELECT e.committed_seq AS cut FROM thread t JOIN entry e ON e.id = t.origin_entry_id + WHERE t.id = ${threadId} AND e.committed_seq IS NOT NULL + `, + }); + // The WHERE NOT EXISTS keeps a thread's positions strictly advancing: seq comes from the // event that produced the entry, and a stale one would sort the thread out of order. // Inserting zero rows is the signal, checked by the callers. @@ -406,6 +418,10 @@ export const layer = Layer.effect( return Option.some(hydrate([found.value], parts)[0]!); }); + const cut = Effect.fn("Thread.cut")(function* (threadId: string) { + return Option.map(yield* findCut(threadId).pipe(Effect.orDie), (row) => row.cut); + }); + const line = Effect.fn("Thread.line")(function* (threadId: ThreadSchema.ID) { const entries = yield* selectLine(threadId).pipe(Effect.orDie); return hydrate(entries, yield* partsFor(entries)); @@ -1021,6 +1037,7 @@ export const layer = Layer.effect( main, list, entry, + cut, line, draft, unsettled, From c7249f00f4a1481c2ef9c31a1d00590b9bd4a7f7 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Thu, 5 Nov 2026 20:00:01 +0530 Subject: [PATCH 07/56] feat(codework): store identity on server.info and the index.sync stream Co-Authored-By: Claude Opus 5.5 --- packages/codework/src/server/contract.ts | 105 +++++++++- packages/codework/src/server/handlers.ts | 55 +++++- packages/codework/src/server/rows.ts | 42 ++++ .../test/__artifacts__/sync.index.json | 185 ++++++++++++++++++ packages/codework/test/server.test.ts | 7 +- packages/codework/test/sync.test.ts | 137 +++++++++++++ 6 files changed, 521 insertions(+), 10 deletions(-) create mode 100644 packages/codework/src/server/rows.ts create mode 100644 packages/codework/test/__artifacts__/sync.index.json create mode 100644 packages/codework/test/sync.test.ts diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index 4aee2a16..84112438 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -39,15 +39,47 @@ export const SandboxInfo = Schema.Struct({ }); export type SandboxInfo = typeof SandboxInfo.Type; +/** One logical codebase, whatever env it is checked out in. */ +export const ProjectInfo = Schema.Struct({ + id: Schema.String, + name: Schema.String, + status: Schema.Literals(["active", "archived"]), + /** Epoch milliseconds. */ + createdAt: Schema.Finite, + updatedAt: Schema.Finite, +}); +export type ProjectInfo = typeof ProjectInfo.Type; + +/** One directory of a project in one env; sessions attach to it. */ +export const SpaceInfo = Schema.Struct({ + id: Schema.String, + projectId: Schema.String, + location: Session.AbsolutePath, + kind: Schema.Literals(["primary", "linked", "copy", "plain"]), + /** The sandbox it is in; absent for the host. */ + env: optional(Schema.String), + status: Schema.Literals(["active", "archived"]), + /** Epoch milliseconds. */ + createdAt: Schema.Finite, + updatedAt: Schema.Finite, +}); +export type SpaceInfo = typeof SpaceInfo.Type; + /** The container: where its threads run by default, and which host project it reads. */ export const SessionInfo = Schema.Struct({ id: Session.SessionSchema.ID, + spaceId: Schema.String, + slug: Schema.String, directory: Session.AbsolutePath, /** The host directory this session is anchored to. Absent when it has none. */ hostDir: optional(Session.AbsolutePath), sandbox: optional(SandboxInfo), /** The project the session's space belongs to. */ project: optional(Schema.Struct({ id: Schema.String, name: Schema.String })), + tag: optional(Schema.String), + metadata: optional(Schema.Record(Schema.String, Schema.String)), + /** Epoch milliseconds. */ + createdAt: Schema.Finite, /** Last activity in any of its threads, epoch milliseconds. */ updatedAt: Schema.Finite, /** Where unaddressed prompts go. */ @@ -55,14 +87,17 @@ export const SessionInfo = Schema.Struct({ }); export type SessionInfo = typeof SessionInfo.Type; +/** `cut` is the committed seq of `entryId`: the inherited line ends there. */ export const ThreadOrigin = Schema.Union([ Schema.Struct({ kind: Schema.Literal("main") }), - Schema.Struct({ kind: Schema.Literal("fork"), threadId: Schema.String, entryId: Schema.String }), + Schema.Struct({ kind: Schema.Literal("fork"), threadId: Schema.String, entryId: Schema.String, cut: Schema.Int }), Schema.Struct({ kind: Schema.Literal("subagent"), threadId: Schema.String, callId: Schema.String, entryId: optional(Schema.String), + /** Present once `entryId` is committed. */ + cut: optional(Schema.Int), }), ]); @@ -77,8 +112,16 @@ export const ThreadUsage = Schema.Struct({ }); export type ThreadUsage = typeof ThreadUsage.Type; -/** One linear transcript of a session, with the session embedded. */ -export const ThreadInfo = Schema.Struct({ +/** The last run of a thread; `error` is set unless it succeeded or is still going. */ +export const ThreadRun = Schema.Struct({ + /** Epoch milliseconds. */ + startedAt: Schema.Finite, + endedAt: optional(Schema.Finite), + outcome: optional(ThreadSchema.RunOutcome), + error: optional(ThreadSchema.RunError), +}); + +const ThreadFields = { id: ThreadSchema.ID, origin: ThreadOrigin, /** Absent until its first own prompt; clients show a fork's source title until then. */ @@ -94,10 +137,42 @@ export const ThreadInfo = Schema.Struct({ /** Where it runs instead of the session's space and directory. */ placement: optional(Schema.Struct({ spaceId: Schema.String, directory: Session.AbsolutePath })), usage: ThreadUsage, - session: SessionInfo, -}); + /** Where its run is, as the harness stores it: live whenever the server is up. */ + runState: ThreadSchema.RunState, + /** When `runState` last changed, epoch milliseconds; absent before the first run. */ + runStateAt: optional(Schema.Finite), + /** Absent before the first run. */ + lastRun: optional(ThreadRun), + /** Automatic restarts of the current run. */ + restarts: Schema.Int, + /** The 1-based provider request in flight; absent when none is. */ + attempt: optional(Schema.Int), + /** The seq of the session event that created it. */ + createdSeq: Schema.Int, + /** Epoch milliseconds. */ + createdAt: Schema.Finite, +}; + +/** One linear transcript of a session, with the session embedded. */ +export const ThreadInfo = Schema.Struct({ ...ThreadFields, session: SessionInfo }); export type ThreadInfo = typeof ThreadInfo.Type; +/** A thread as `index.sync` sends it: its session is a row of its own. */ +export const ThreadSummary = Schema.Struct({ ...ThreadFields, sessionId: Session.SessionSchema.ID }); +export type ThreadSummary = typeof ThreadSummary.Type; + +/** An `index.sync` item: a row with the revision that last wrote it, a deletion, or progress. */ +export const IndexItem = Schema.Union([ + Schema.TaggedStruct("Project", { project: ProjectInfo, rev: Schema.Int }), + Schema.TaggedStruct("Space", { space: SpaceInfo, rev: Schema.Int }), + Schema.TaggedStruct("Session", { session: SessionInfo, rev: Schema.Int }), + Schema.TaggedStruct("Thread", { thread: ThreadSummary, rev: Schema.Int }), + Schema.TaggedStruct("Removed", { kind: Schema.Literal("session"), id: Schema.String, rev: Schema.Int }), + /** Every row with a revision up to `rev` has been sent. */ + Schema.TaggedStruct("Synced", { storeId: Schema.String, rev: Schema.Int }), +]); +export type IndexItem = typeof IndexItem.Type; + /** A thread of a session: `threadId`, or the session's `main` thread when absent. */ const ThreadRef = { sessionId: Session.SessionSchema.ID, @@ -316,7 +391,25 @@ export const Api = RpcGroup.make( */ Rpc.make("server.info", { payload: {}, - success: Schema.Struct({ pid: Schema.Int, version: Schema.String }), + success: Schema.Struct({ + pid: Schema.Int, + version: Schema.String, + /** The harness store this server serves; a sync cursor is only valid against it. */ + storeId: Schema.String, + /** The store's head: the last revision taken. */ + rev: Schema.Int, + }), + }), + /** + * The index of this machine: every project, space, session and thread whose revision is past + * `after`, in revision order, then `Synced` at the head read when the call began; then the same + * live after each commit. `after: 0` reads the whole index. A row written again moves to a later + * revision and is sent again; the latest wins. A deleted session arrives as `Removed`. + */ + Rpc.make("index.sync", { + payload: { after: Schema.Int }, + success: IndexItem, + stream: true, }), Rpc.make("auth.save", { payload: { credentials: OAuthCredentials }, diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index b3a8f109..6345c763 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -14,6 +14,8 @@ import { SessionStore, Settings, State, + Store, + Sync, ThreadSchema, ThreadStore, } from "@codeworksh/harness/effect"; @@ -21,11 +23,13 @@ import { DateTime, Effect, Option, Schema, Stream } from "effect"; import { Contract, type HistoryEntry, + type IndexItem, type LogSynced, type RuntimeConfig, type SandboxInfo, type SessionInfo, type ThreadInfo, + type ThreadSummary, } from "./contract.ts"; import { type EncodingError, Envelope, type EventEnvelope } from "./envelope.ts"; import { Clones } from "./clones.ts"; @@ -34,6 +38,7 @@ import { Host } from "./host.ts"; import { Identity } from "./identity.ts"; import { OAuth } from "./oauth.ts"; import { MessagePart } from "./part.ts"; +import { Rows } from "./rows.ts"; const toSandboxInfo = (info: Sandbox.Info): SandboxInfo => ({ id: info.id, @@ -50,16 +55,22 @@ const toSandboxInfo = (info: Sandbox.Info): SandboxInfo => ({ const toSessionInfo = (info: Session.Info): SessionInfo => ({ id: info.id, + spaceId: info.spaceId, + slug: info.slug, directory: info.directory, ...(info.hostDir === undefined ? {} : { hostDir: info.hostDir }), ...(info.sandbox === undefined ? {} : { sandbox: toSandboxInfo(info.sandbox) }), ...(info.project === undefined ? {} : { project: info.project }), + ...(info.tag === undefined ? {} : { tag: info.tag }), + ...(info.metadata === undefined ? {} : { metadata: info.metadata }), + createdAt: info.createdAt, updatedAt: info.updatedAt, mainThreadId: info.mainThreadId, }); -const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ +const toThreadSummary = (info: Session.ThreadSummary): ThreadSummary => ({ id: info.id, + sessionId: info.sessionId, origin: info.origin, ...(info.title === undefined ? {} : { title: info.title }), updatedAt: info.updatedAt, @@ -68,9 +79,31 @@ const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => ({ ...(info.foundation === undefined ? {} : { foundation: info.foundation }), ...(info.placement === undefined ? {} : { placement: info.placement }), usage: info.usage, - session: toSessionInfo(info.session), + runState: info.runState, + ...(info.runStateAt === undefined ? {} : { runStateAt: info.runStateAt }), + ...(info.lastRun === undefined ? {} : { lastRun: info.lastRun }), + restarts: info.restarts, + ...(info.attempt === undefined ? {} : { attempt: info.attempt }), + createdSeq: info.createdSeq, + createdAt: info.createdAt, }); +const toThreadInfo = (info: Session.ThreadInfo): ThreadInfo => { + const { sessionId: _, ...summary } = toThreadSummary(info); + return { ...summary, session: toSessionInfo(info.session) }; +}; + +const toIndexItem = (change: Sync.IndexChange): IndexItem => { + switch (change._tag) { + case "Session": + return { _tag: "Session", session: toSessionInfo(change.session), rev: change.rev }; + case "Thread": + return { _tag: "Thread", thread: toThreadSummary(change.thread), rev: change.rev }; + default: + return change; + } +}; + // Wire configs name only the keys they set, so `undefined` never reaches the harness under // exactOptionalPropertyTypes. Creating takes choices only; `null` clears nothing there. const runtimeInput = (runtime: RuntimeConfig | undefined): Session.RuntimeInput => ({ @@ -132,13 +165,29 @@ export const layer = Contract.Api.toLayer( const mcp = yield* Mcp.Service; const host = yield* Host.Service; const clones = yield* Clones.Service; + const store = yield* Store.Service; const mcpControl = (operation: "connect" | "disconnect") => Effect.fnUntraced(function* ({ server, ...ref }: Control.ThreadRef & { readonly server: string }) { return yield* mcp[operation](yield* resolve(ref), server); }); return Contract.Api.of({ - "server.info": () => Effect.succeed({ pid: identity.pid, version: identity.version }), + "server.info": Effect.fnUntraced(function* () { + const head = yield* store.head; + return { pid: identity.pid, version: identity.version, storeId: head.id, rev: head.rev }; + }), + "index.sync": ({ after }) => + Rows.follow( + after, + (from, upto) => + Sync.index(from, upto, Rows.batch).pipe( + Effect.map(({ items, next }) => ({ + items: items.map(toIndexItem), + ...(next === undefined ? {} : { next }), + })), + ), + (head): IndexItem => ({ _tag: "Synced", storeId: head.id, rev: head.rev }), + ), "auth.save": ({ credentials }) => oauth.save(credentials), "auth.status": ({ provider }) => oauth.status(provider), "auth.refresh": ({ provider }) => oauth.refresh(provider), diff --git a/packages/codework/src/server/rows.ts b/packages/codework/src/server/rows.ts new file mode 100644 index 00000000..f921eb84 --- /dev/null +++ b/packages/codework/src/server/rows.ts @@ -0,0 +1,42 @@ +import { Store, type Sync } from "@codeworksh/harness/effect"; +import { Effect, Option, Stream } from "effect"; + +/** Rows per page; each page is one read transaction, which writes wait behind. */ +export const batch = 500; + +/** + * A revision-ordered stream: every row past `after`, page by page up to the head, then `synced` at + * that head; then the same from the last head after each commit. The doorbell is registered before + * the first read, so no commit falls between. + */ +export const follow = ( + after: number, + page: (from: Sync.Cursor, upto: number) => Effect.Effect, never, R>, + synced: (head: Store.Head) => A, +): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + const store = yield* Store.Service; + const wake = yield* store.wake; + let sent: number | undefined; + const round = Stream.unwrap( + Effect.gen(function* () { + const head = yield* store.head; + if (head.rev === sent) return Stream.empty; + const from = sent ?? after; + sent = head.rev; + return Stream.paginate({ rev: from }, (cursor) => + page(cursor, head.rev).pipe( + Effect.map(({ items, next }) => [items, Option.fromUndefinedOr(next)] as const), + ), + ).pipe(Stream.concat(Stream.make(synced(head)))); + }), + ); + return Stream.make(undefined).pipe( + Stream.concat(Stream.fromSubscription(wake)), + Stream.flatMap(() => round), + ); + }), + ); + +export * as Rows from "./rows.ts"; diff --git a/packages/codework/test/__artifacts__/sync.index.json b/packages/codework/test/__artifacts__/sync.index.json new file mode 100644 index 00000000..b4ad102f --- /dev/null +++ b/packages/codework/test/__artifacts__/sync.index.json @@ -0,0 +1,185 @@ +{ + "info": { + "storeId": "", + "rev": "" + }, + "configured": { + "_tag": "Thread", + "thread": { + "id": "
", + "origin": { + "kind": "main" + }, + "updatedAt": "
" + }, + "rev": "" + }, + { + "_tag": "Thread", + "thread": { + "id": "
", + "origin": { + "kind": "main" + }, + "title": "Main prompt", + "updatedAt": "
", + "entryId": "", + "cut": 9 + }, + "updatedAt": "
", "
", "", ""]); expect(histories.main).toHaveLength(2); diff --git a/packages/codework/test/sync.test.ts b/packages/codework/test/sync.test.ts new file mode 100644 index 00000000..fa23b9e1 --- /dev/null +++ b/packages/codework/test/sync.test.ts @@ -0,0 +1,137 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- starts the real server in a temporary home. */ +import { Effect, Layer, Queue, Stream } from "effect"; +import { HttpServer } from "effect/http"; +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vite-plus/test"; +import { immediateOpen } from "../../harness/test/fixtures/llm.ts"; +import { Client } from "../src/server/client.ts"; +import type { IndexItem } from "../src/server/contract.ts"; +import { Server } from "../src/server/server.ts"; + +type Synced = Extract; + +it("index.sync sends the index, resumes from a revision, and follows configure, rename and delete", async () => { + const home = realpathSync(mkdtempSync(join(tmpdir(), "codework-sync-index-"))); + try { + const artifact = await Effect.gen(function* () { + const listener = yield* HttpServer.HttpServer; + if (listener.address._tag === "UnixPathAddress") return yield* Effect.die("Expected TCP listener"); + const context = yield* Layer.build(Client.layer(`ws://127.0.0.1:${listener.address.port}/rpc/v1`)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + + // Up to and including the first `Synced`: one catch-up. + const catchUp = (after: number) => + rpc["index.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ); + + const live = yield* Queue.unbounded(); + yield* rpc["index.sync"]({ after: 0 }).pipe( + Stream.runForEach((item) => Queue.offer(live, item)), + Effect.forkScoped, + ); + const until = (match: (item: IndexItem) => item is A) => + Effect.gen(function* () { + while (true) { + const item = yield* Queue.take(live); + if (match(item)) return item; + } + }); + const syncedAt = (rev: number) => until((item): item is Synced => item._tag === "Synced" && item.rev >= rev); + + const empty = yield* until((item): item is Synced => item._tag === "Synced"); + + const created = yield* rpc["session.create"]({ directory: home, project: { name: "Alpha" } }); + const sessionId = created.session.id; + yield* rpc["thread.configure"]({ sessionId, runtime: { thinkingLevel: "high" } }); + const configured = yield* until( + (item): item is Extract => + item._tag === "Thread" && item.thread.id === created.id && item.thread.thinkingLevel === "high", + ); + yield* rpc["thread.prompt"]({ sessionId, text: "Main prompt" }); + yield* rpc["thread.wait"]({ sessionId }); + const reply = (yield* rpc["thread.history"]({ sessionId })).entries.at(-1); + if (reply === undefined) return yield* Effect.die("main thread has no reply"); + const fork = yield* rpc["thread.fork"]({ sessionId, entryId: reply.id }); + const forked = yield* syncedAt((yield* rpc["server.info"]({})).rev); + + const projectId = created.session.project?.id; + if (projectId === undefined) return yield* Effect.die("session has no project"); + yield* rpc["project.rename"]({ projectId, name: "Beta" }); + const doomed = yield* rpc["session.create"]({ directory: home }); + yield* rpc["session.delete"]({ sessionId: doomed.session.id }); + const removed = yield* until( + (item): item is Extract => + item._tag === "Removed" && item.id === doomed.session.id, + ); + const info = yield* rpc["server.info"]({}); + const last = yield* syncedAt(info.rev); + + const snapshot = yield* catchUp(0); + const resumed = yield* catchUp(forked.rev); + + const labels = new Map([ + [home, ""], + [info.storeId, ""], + [projectId, ""], + [created.session.spaceId, ""], + [sessionId, ""], + [created.session.slug, ""], + [created.id, "
"], + [doomed.session.id, ""], + [doomed.session.slug, ""], + [doomed.id, ""], + [fork.thread.id, ""], + [reply.id, ""], + ]); + // Revisions by rank within the artifact: their order is the contract, their values are not. + const revs = [...new Set([...snapshot, ...resumed].map((item) => item.rev))].sort((a, b) => a - b); + const normalize = (value: unknown, key?: string): unknown => { + if (typeof value === "string") return labels.get(value) ?? value; + if (typeof value === "number") { + if (key === "rev") return revs.includes(value) ? `` : ""; + return key?.endsWith("At") ? "
", + "seq": "", + "role": "user", + "size": 37 + }, + { + "id": "assistant_1", + "threadId": "
", + "seq": "", + "role": "assistant", + "size": 35 + }, + { + "id": "", + "threadId": "
", + "seq": "", + "role": "user", + "size": 38 + }, + { + "id": "assistant_2", + "threadId": "
", + "seq": "", + "role": "assistant", + "size": 35 + }, + { + "id": "", + "threadId": "", + "seq": "", + "role": "user", + "size": 36 + }, + { + "id": "assistant_3", + "threadId": "", + "seq": "", + "role": "assistant", + "size": 35 + } + ], + "opened": [ + { + "_tag": "Entry", + "entry": { + "id": "", + "threadId": "
", + "seq": "", + "role": "user", + "state": "committed", + "createdAt": "
", + "seq": "", + "role": "assistant", + "state": "committed", + "createdAt": "(match: (item: ThreadItem) => item is A) => + Effect.gen(function* () { + while (true) { + const item = yield* Queue.take(queue); + if (match(item)) return item; + } + }); + return { items, until }; + }); + const isEntry = + (id: string, state?: Entry["entry"]["state"]) => + (item: ThreadItem): item is Entry => + item._tag === "Entry" && item.entry.id === id && (state === undefined || item.entry.state === state); + + // (a) First open: the newest page of the fork's line, then `Synced` with its floor. + const live = yield* follow(forkId); + const openedAt = yield* live.until((item): item is ThreadSynced => item._tag === "Synced"); + const opened = [...live.items]; + + // (b) A prompt held mid-reply, a follow-up queued behind it, then a resume. + yield* rpc["thread.prompt"]({ ...fork, text: "Busy prompt" }); + yield* live.until(isEntry("assistant_4", "draft")); + yield* rpc["thread.prompt"]({ ...fork, text: "Queued prompt", delivery: "followUp" }); + yield* live.until( + (item): item is Extract => + item._tag === "Inbox" && item.prompt.text === "Queued prompt", + ); + const busy = yield* catchUp(openedAt.rev); + const busyAt = busy.at(-1); + if (busyAt?._tag !== "Synced") return yield* Effect.die("resume did not end at Synced"); + + // (c) The held reply streams live; then its follow-up runs. + release(); + yield* live.until(isEntry("assistant_4", "committed")); + yield* rpc["thread.wait"](fork); + const settled = yield* catchUp(busyAt.rev); + + // (d) A reply on the main thread streams on its own stream only. + const head = (yield* rpc["server.info"]({})).rev; + const main = yield* follow(mainId, head); + yield* main.until((item): item is ThreadSynced => item._tag === "Synced"); + yield* rpc["thread.prompt"]({ sessionId, text: "Main prompt" }); + yield* rpc["thread.wait"]({ sessionId }); + yield* main.until(isEntry("assistant_6", "committed")); + // The feed is one ordered queue, so the fork's next delta comes after any of main's. + yield* rpc["thread.prompt"]({ ...fork, text: "Last prompt" }); + yield* live.until((item): item is Delta => item._tag === "Delta" && item.delta.entryId === "assistant_7"); + yield* rpc["thread.wait"](fork); + + const deltasOf = (items: ReadonlyArray, entryId: string) => + items.flatMap((item) => (item._tag === "Delta" && item.delta.entryId === entryId ? [item] : [])); + + const info = yield* rpc["server.info"]({}); + const histories = [ + ...(yield* rpc["thread.history"]({ sessionId })).entries, + ...(yield* rpc["thread.history"](fork)).entries, + ]; + const labels = new Map([ + [home, ""], + [info.storeId, ""], + [sessionId, ""], + [mainId, "
"], + [forkId, ""], + ]); + let users = 0; + for (const entry of histories) + if (!labels.has(entry.id) && entry.role === "user") labels.set(entry.id, ``); + const all = [...opened, ...busy, ...settled]; + // Revisions and seqs by rank: their order is the contract, their values are not. + const rank = (values: ReadonlyArray) => [...new Set(values)].sort((a, b) => a - b); + const revs = rank(all.flatMap((item) => ("rev" in item ? [item.rev] : []))); + const seqs = rank(histories.map((entry) => entry.seq)); + const normalize = (value: unknown, key?: string): unknown => { + if (typeof value === "string") return labels.get(value) ?? value; + if (typeof value === "number") { + if (key === "rev") return revs.includes(value) ? `` : ""; + if (key === "seq" || key === "floor") + return seqs.includes(value) ? `` : ""; + return key?.endsWith("At") ? "
", + "seq": "", + "role": "user", + "size": 37 + }, + { + "id": "assistant_1", + "threadId": "
", + "seq": "", + "role": "assistant", + "size": 35 + }, + { + "id": "", + "threadId": "
", + "seq": "", + "role": "user", + "size": 38 + }, + { + "id": "assistant_2", + "threadId": "
", + "seq": "", + "role": "assistant", + "size": 35 + }, + { + "id": "", + "threadId": "
", + "seq": "", + "role": "user", + "size": 37 + }, + { + "id": "assistant_3", + "threadId": "
", + "seq": "", + "role": "assistant", + "size": 35 + }, + { + "id": "", + "threadId": "", + "seq": "", + "role": "user", + "size": 36 + }, + { + "id": "assistant_4", + "threadId": "", + "seq": "", + "role": "assistant", + "size": 35 + } + ], + "opened": [ + { + "_tag": "Entry", + "entry": { + "id": "", + "threadId": "", + "seq": "", + "role": "user", + "state": "committed", + "createdAt": "
", + "seq": "", + "role": "assistant", + "state": "committed", + "createdAt": "