diff --git a/apps/contract/src/contract.ts b/apps/contract/src/contract.ts index 15f32ef4..c9fa6edf 100644 --- a/apps/contract/src/contract.ts +++ b/apps/contract/src/contract.ts @@ -1,31 +1,59 @@ import { Schema } from "effect"; import { Rpc, RpcGroup } from "effect/rpc"; -import { Clone, ClonePhase, CloneStage, Folder, ModelList, Repository } from "@codeworksh/widget/codework"; +import { + CodeworkUnavailable, + LineRequest, + LineThread, + SyncError, + SyncThread, + ThreadLine, +} from "@codeworksh/widget/backend"; +import { + Clone, + ClonePhase, + CloneStage, + Folder, + ModelList, + Repository, + RunState, + SpaceInfo, +} from "@codeworksh/widget/codework"; import { Event, EventTimeout, Publish, Request, Synced } from "@codeworksh/widget/event"; -/** A conversation with the agent, as listed under its project. */ -export const SessionSummary = Schema.Struct({ +export { CodeworkUnavailable, LineRequest, LineThread, SyncError, SyncThread, ThreadLine }; + +/** The organization the UI shows until auth exists; every synced project starts in it. */ +export const PERSONAL_ORGANIZATION = "personal"; + +/** A session as the sidebar lists it: its space's kind, its color and its main thread. */ +export const SidebarSession = Schema.Struct({ id: Schema.String, + /** Its main thread. */ + threadId: Schema.String, + spaceKind: SpaceInfo.fields.kind, + /** Hex; null is the default. */ + color: Schema.NullOr(Schema.String), + /** The main thread's title, or what an untitled one is called. */ title: Schema.String, - /** Last activity, epoch milliseconds. */ - updated: Schema.Number, - /** The model chosen for the session; absent when settings decide. */ model: Schema.optionalKey(Schema.Struct({ provider: Schema.String, id: Schema.String })), + runState: RunState, + /** The session's last activity: the newest change to any of its threads, forks included; epoch milliseconds. */ + updated: Schema.Finite, + /** The host folder it is anchored to, else its directory. */ + path: Schema.String, }); -export type SessionSummary = typeof SessionSummary.Type; +export type SidebarSession = typeof SidebarSession.Type; -export const Project = Schema.Struct({ +/** An active project with its sessions, newest first, and the machine it is on. */ +export const SidebarProject = Schema.Struct({ + machine: Schema.Struct({ id: Schema.String, name: Schema.String }), id: Schema.String, name: Schema.String, + /** Its newest session's folder, else its primary space's location: where it was last worked on. */ path: Schema.String, - sessions: Schema.Array(SessionSummary), + sessions: Schema.Array(SidebarSession), }); -export type Project = typeof Project.Type; - -/** The CodeWork server that owns projects and sessions is not running, or did not answer. */ -export class CodeworkUnavailable extends Schema.TaggedError()("Server.CodeworkUnavailable", { - message: Schema.String, -}) {} +export type SidebarProject = typeof SidebarProject.Type; /** A host folder operation that failed: unreadable, not a directory, taken, or git/gh refused. */ export class HostError extends Schema.TaggedError()("Server.HostError", { @@ -143,16 +171,16 @@ export const CliEnvVar = Schema.Struct({ }); export type CliEnvVar = typeof CliEnvVar.Type; -/** Only servers on this machine for now; remote ones come later. */ -export const ServerKind = Schema.Literals(["local"]); +/** A server on this computer (local), or one hosted elsewhere that the app reaches at its URL (remote). */ +export const ServerKind = Schema.Literals(["local", "remote"]); /** The server the app always has, which cannot be removed. */ export const LOCAL_SERVER = "local"; -/** A server and how it is set up, with its executable and process as found when listed. */ -export const ServerStatus = Schema.Struct({ +/** A local server and how it is set up, with its executable and process as found when listed. */ +export const LocalServerStatus = Schema.Struct({ id: Schema.String, - kind: ServerKind, + kind: Schema.Literal("local"), name: Schema.String, /** The one the app connects to. */ isDefault: Schema.Boolean, @@ -168,6 +196,19 @@ export const ServerStatus = Schema.Struct({ args: Schema.String, env: Schema.Array(CliEnvVar), }); +export type LocalServerStatus = typeof LocalServerStatus.Type; + +/** A remote server: only where to reach it; it has no executable or home here. */ +export const RemoteServerStatus = Schema.Struct({ + id: Schema.String, + kind: Schema.Literal("remote"), + name: Schema.String, + isDefault: Schema.Boolean, + url: Schema.String, +}); +export type RemoteServerStatus = typeof RemoteServerStatus.Type; + +export const ServerStatus = Schema.Union([LocalServerStatus, RemoteServerStatus]); export type ServerStatus = typeof ServerStatus.Type; /** How often every server is checked again in the background. */ @@ -182,18 +223,25 @@ export const DEFAULT_SERVER_HEALTH: ServerHealth = { interval: 300 }; export const CliAction = Schema.Literals(["start", "restart", "stop"]); export type CliAction = typeof CliAction.Type; -/** A new local server: it needs a home of its own, since a home names one server. */ -export const ServerAdd = Schema.Struct({ - name: Schema.String, - executable: Schema.NullOr(Schema.String), - home: Schema.String, - args: Schema.String, -}); +/** + * A new server. A local one needs a home of its own, since a home names one server; a remote one + * needs its WebSocket URL. + */ +export const ServerAdd = Schema.Union([ + Schema.Struct({ + kind: Schema.Literal("local"), + name: Schema.String, + executable: Schema.NullOr(Schema.String), + home: Schema.String, + args: Schema.String, + }), + Schema.Struct({ kind: Schema.Literal("remote"), name: Schema.String, url: Schema.String }), +]); export type ServerAdd = typeof ServerAdd.Type; /** * Changes to a server; an absent field is left as it is. A `null` path goes back to the one found; - * `env` replaces every variable. + * `env` replaces every variable. `url` is for a remote server only, the rest but `name` for a local one. */ export const ServerUpdate = Schema.Struct({ id: Schema.String, @@ -202,6 +250,7 @@ export const ServerUpdate = Schema.Struct({ home: Schema.optionalKey(Schema.NullOr(Schema.String)), args: Schema.optionalKey(Schema.String), env: Schema.optionalKey(Schema.Array(CliEnvVar)), + url: Schema.optionalKey(Schema.String), }); export type ServerUpdate = typeof ServerUpdate.Type; @@ -211,13 +260,181 @@ export class ServersError extends Schema.TaggedError()("Server.Ser message: Schema.String, }) {} +/** The app bus topics the sync publishes on. */ +export const syncTopics = { + /** `SyncChanged` after each applied batch; stored. */ + changed: "sync/changed", + /** `SyncStatus` whenever a thread's status moves; stored, in the transaction that moved it. */ + status: "sync/status", + /** A thread's live text and thinking deltas (`LiveDelta`) that continue its draft; never stored. */ + delta: (threadId: string) => `sync/delta/${threadId}`, + /** `SyncTracking` whenever the user edits a desktop column other than status; stored, in the transaction that wrote it. */ + tracking: "sync/tracking", +}; + +/** + * The rows one applied batch wrote, by table, for one machine. Projects and spaces are keyed by + * `machineId` and id, the rest by id. `threadId` names the thread whose detail the batch was; + * `items` counts the rows it carried; `full` marks the first batch of a full sync. + */ +export const SyncChanged = Schema.Struct({ + machineId: Schema.String, + threadId: Schema.optionalKey(Schema.String), + full: Schema.Boolean, + items: Schema.Int, + project: Schema.Array(Schema.String), + space: Schema.Array(Schema.String), + session: Schema.Array(Schema.String), + thread: Schema.Array(Schema.String), + entry: Schema.Array(Schema.String), + inbox: Schema.Array(Schema.String), + /** Sessions soft-deleted, with their threads. */ + removed: Schema.Array(Schema.String), +}); +export type SyncChanged = typeof SyncChanged.Type; + +export const ThreadStatus = Schema.Literals(["working", "settled", "attention", "cancelled"]); +export type ThreadStatus = typeof ThreadStatus.Type; + +/** A thread's status moved; `at` is the app server's clock. */ +export const SyncStatus = Schema.Struct({ + threadId: Schema.String, + status: ThreadStatus, + previous: ThreadStatus, + at: Schema.Number, +}); +export type SyncStatus = typeof SyncStatus.Type; + +/** Linear's scale: 0 none, 1 urgent, 2 high, 3 medium, 4 low. */ +export const Priority = Schema.Literals([0, 1, 2, 3, 4]); +export type Priority = typeof Priority.Type; + +/** A custom group of an organization's threads, in its own `ordering`. */ +export const Grouping = Schema.Struct({ + id: Schema.String, + organizationId: Schema.String, + name: Schema.String, + /** Hex, `#rrggbb`. */ + color: Schema.NullOr(Schema.String), + ordering: Schema.Number, +}); +export type Grouping = typeof Grouping.Type; + +/** Where a moved thread or group goes in its organization's list: first, last, or between two neighbours. */ +export const Position = Schema.Union([ + Schema.Literals(["first", "last"]), + Schema.Struct({ after: Schema.String, before: Schema.String }), +]); +export type Position = typeof Position.Type; + +/** + * A user edit of a desktop column: a thread's (only the field that changed; a `null` group is + * ungrouped, a `null` color falls back to the session's), a session's color, a group created or + * changed (the whole row), or a group deleted with the threads it left ungrouped. + */ +export const SyncTracking = Schema.Union([ + Schema.TaggedStruct("Thread", { + threadId: Schema.String, + priority: Schema.optionalKey(Priority), + groupingId: Schema.optionalKey(Schema.NullOr(Schema.String)), + ordering: Schema.optionalKey(Schema.Number), + color: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + Schema.TaggedStruct("Session", { sessionId: Schema.String, color: Schema.NullOr(Schema.String) }), + Schema.TaggedStruct("Group", Grouping.fields), + Schema.TaggedStruct("GroupDeleted", { groupingId: Schema.String, threads: Schema.Array(Schema.String) }), +]); +export type SyncTracking = typeof SyncTracking.Type; + +/** What the dashboard splits an organization's threads by. */ +export const DashboardBy = Schema.Literals(["status", "priority", "grouping"]); +export type DashboardBy = typeof DashboardBy.Type; + +/** A thread as the dashboard lists it, with its desktop columns. */ +export const DashboardThread = Schema.Struct({ + id: Schema.String, + sessionId: Schema.String, + originKind: Schema.Literals(["main", "fork"]), + /** Its own title, else its nearest titled ancestor's, else what an untitled one is called. */ + title: Schema.String, + project: Schema.Struct({ id: Schema.String, name: Schema.String }), + runState: RunState, + status: ThreadStatus, + priority: Priority, + groupingId: Schema.NullOr(Schema.String), + ordering: Schema.Number, + /** Its own color, else its session's; null is the default. */ + color: Schema.NullOr(Schema.String), + /** Last activity, epoch milliseconds. */ + updatedAt: Schema.Number, +}); +export type DashboardThread = typeof DashboardThread.Type; + +/** One list of the dashboard: the threads with one status, one priority, or in one group (`null` is ungrouped), in `ordering`. */ +export const DashboardList = Schema.Union([ + Schema.TaggedStruct("Status", { status: ThreadStatus, threads: Schema.Array(DashboardThread) }), + Schema.TaggedStruct("Priority", { priority: Priority, threads: Schema.Array(DashboardThread) }), + Schema.TaggedStruct("Grouping", { + groupingId: Schema.NullOr(Schema.String), + threads: Schema.Array(DashboardThread), + }), +]); +export type DashboardList = typeof DashboardList.Type; + +/** + * An organization's threads split into lists, each present even when empty: by status, by priority + * (urgent first, none last), or by group (groups in their `ordering`, ungrouped last). The + * organization's groups come with it, in their `ordering`, whatever the split. + */ +export const Dashboard = Schema.Struct({ + lists: Schema.Array(DashboardList), + groups: Schema.Array(Grouping), +}); +export type Dashboard = typeof Dashboard.Type; + +/** + * The topics a client follows to keep each read current; it reads again when one fires. + * - `sidebar.list`: `sync/changed` naming a project, space, session or thread, or a removed session; + * `sync/tracking` for a session's color. + * - `threads.line`: `sync/changed` whose `threadId` is the thread (its entries and inbox) or that + * names a thread (the thread itself, or the ancestor an untitled fork takes its title from); + * `sync/delta/` streams its draft between checkpoints. + * - `dashboard.list`: `sync/changed` naming a thread or a removed session; every `sync/status` and + * `sync/tracking`. + */ +export const readTopics = { + sidebar: [syncTopics.changed, syncTopics.tracking], + dashboard: [syncTopics.changed, syncTopics.status, syncTopics.tracking], + line: (threadId: string) => [syncTopics.changed, syncTopics.delta(threadId)], +}; + +/** A desktop edit refused: a value out of range, or a thread, session, group or organization not found. */ +export class TrackingError extends Schema.TaggedError()("Server.TrackingError", { + message: Schema.String, +}) {} + /** Package-scoped JSON storage: `package` namespaces keys so packages never collide. */ const KvKey = { package: Schema.String, key: Schema.String }; export const Api = RpcGroup.make( - Rpc.make("projects.list", { - success: Schema.Array(Project), - error: CodeworkUnavailable, + /** + * An organization's active projects, newest activity first, each with its sessions that are not + * deleted: on the default machine only, or on every machine. Read from app.db alone. + */ + Rpc.make("sidebar.list", { + payload: { organizationId: Schema.String, machines: Schema.Literals(["default", "all"]) }, + success: Schema.Array(SidebarProject), + }), + /** One page of a thread's line with its pending inbox, read from app.db alone; below `floor`, `sync.page` first. */ + Rpc.make("threads.line", { + payload: LineRequest, + success: ThreadLine, + error: SyncError, + }), + /** An organization's threads that are not deleted, subagents left out, split by `by` into lists in `ordering`. Read from app.db alone. */ + Rpc.make("dashboard.list", { + payload: { organizationId: Schema.String, by: DashboardBy }, + success: Dashboard, }), /** The models a new session can use; `hostDir` is its folder, whose project settings apply. */ Rpc.make("models.list", { @@ -361,18 +578,104 @@ export const Api = RpcGroup.make( success: Schema.Void, error: ServersError, }), - /** The app reconnects through the new default's home at once. */ + /** The app reconnects through the new default's home at once; only a local server can be the default. */ Rpc.make("servers.setDefault", { payload: ServerId, success: Schema.Void, error: ServersError, }), - /** Runs `codework service ` with the server's executable, home, arguments and variables. */ + /** Runs `codework service ` with a local server's executable, home, arguments and variables. */ Rpc.make("servers.service", { payload: { id: Schema.String, action: CliAction }, success: Schema.Void, error: ServersError, }), + /** + * Keeps a thread's detail in sync while it is open: the first open stores its newest page, a + * reopen resumes from where it stopped. Opens are counted; done once the detail is current, or + * at once while its machine is unreachable. + */ + Rpc.make("sync.open", { + payload: SyncThread, + success: Schema.Void, + }), + /** Stops syncing a thread's detail once every open is closed; what is stored stays. */ + Rpc.make("sync.close", { + payload: SyncThread, + success: Schema.Void, + }), + /** Stores up to `limit` entries of an opened thread's line below its floor; `floor` is the new one. */ + Rpc.make("sync.page", { + payload: { ...SyncThread.fields, limit: Schema.Int }, + success: Schema.Struct({ entries: Schema.Int, floor: Schema.NullOr(Schema.Int) }), + error: Schema.Union([CodeworkUnavailable, SyncError]), + }), + /** Sets a thread's status until its next run change; published on `sync/status`. */ + Rpc.make("threads.setStatus", { + payload: { threadId: Schema.String, status: ThreadStatus }, + success: Schema.Void, + error: TrackingError, + }), + /** `priority` must be one of `Priority`. */ + Rpc.make("threads.setPriority", { + payload: { threadId: Schema.String, priority: Schema.Int }, + success: Schema.Void, + error: TrackingError, + }), + /** A hex `#rrggbb` color over the session's; `null` falls back to it. */ + Rpc.make("threads.setColor", { + payload: { threadId: Schema.String, color: Schema.NullOr(Schema.String) }, + success: Schema.Void, + error: TrackingError, + }), + /** Puts a thread in a group of its organization; `null` ungroups it. */ + Rpc.make("threads.setGroup", { + payload: { threadId: Schema.String, groupingId: Schema.NullOr(Schema.String) }, + success: Schema.Void, + error: TrackingError, + }), + Rpc.make("threads.move", { + payload: { threadId: Schema.String, to: Position }, + success: Schema.Void, + error: TrackingError, + }), + /** A hex `#rrggbb` color; `null` is the default. */ + Rpc.make("sessions.setColor", { + payload: { sessionId: Schema.String, color: Schema.NullOr(Schema.String) }, + success: Schema.Void, + error: TrackingError, + }), + /** A new group, last in its organization. */ + Rpc.make("groups.create", { + payload: { + organizationId: Schema.String, + name: Schema.String, + color: Schema.optionalKey(Schema.NullOr(Schema.String)), + }, + success: Grouping, + error: TrackingError, + }), + /** Renames or recolors a group; an absent field is left as it is. */ + Rpc.make("groups.update", { + payload: { + groupingId: Schema.String, + name: Schema.optionalKey(Schema.String), + color: Schema.optionalKey(Schema.NullOr(Schema.String)), + }, + success: Schema.Void, + error: TrackingError, + }), + Rpc.make("groups.move", { + payload: { groupingId: Schema.String, to: Position }, + success: Schema.Void, + error: TrackingError, + }), + /** Its threads are left ungrouped. */ + Rpc.make("groups.delete", { + payload: { groupingId: Schema.String }, + success: Schema.Void, + error: TrackingError, + }), Rpc.make("kv.get", { payload: KvKey, success: Schema.OptionFromNullOr(Schema.Json), diff --git a/apps/server/package.json b/apps/server/package.json index 323f8d91..2e7a87c6 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -18,6 +18,7 @@ "effect": "4.0.5" }, "devDependencies": { + "@codeworksh/aikit": "workspace:*", "@types/node": "catalog:", "typescript": "catalog:", "vite-plus": "catalog:" diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.ts index 2d651fc7..4da40b21 100644 --- a/apps/server/src/cli.ts +++ b/apps/server/src/cli.ts @@ -101,7 +101,7 @@ const found = Effect.gen(function* () { }); /** The `codework` a server runs: the one chosen for it, else the one found. */ -const executableOf = (row: Servers.Row, foundOnce: typeof found) => { +const executableOf = (row: Servers.LocalRow, foundOnce: typeof found) => { const { executable } = row; if (executable === null) return foundOnce; const command = split(executable); @@ -151,7 +151,7 @@ export const awaitConnection = Effect.gen(function* () { ); }); -/** Every server and how it is set up, with its executable and process as found now. */ +/** Every server and how it is set up, with a local one's executable and process as found now. */ export const statuses = Effect.gen(function* () { const servers = yield* Servers.Service; const foundOnce = yield* Effect.cached(found); @@ -159,6 +159,7 @@ export const statuses = Effect.gen(function* () { yield* servers.list, (row) => Effect.gen(function* () { + if (row.kind === "remote") return row satisfies ServerStatus; const home = servers.homeOf(row); const [cli, server] = yield* Effect.all( [executableOf(row, foundOnce), serverAt(home.path, row.isDefault)], @@ -186,7 +187,7 @@ export const statuses = Effect.gen(function* () { // The CLI owns a server's lifecycle; the app only asks it, with the server's variables and in its // home, which a variable never overrides. -export const service = Effect.fn("Cli.service")(function* (row: Servers.Row, action: CliAction) { +export const service = Effect.fn("Cli.service")(function* (row: Servers.LocalRow, action: CliAction) { const servers = yield* Servers.Service; const cli = yield* executableOf(row, found); if (cli === null || !("version" in cli)) diff --git a/apps/server/src/codework.ts b/apps/server/src/codework.ts index 115686d8..093ede64 100644 --- a/apps/server/src/codework.ts +++ b/apps/server/src/codework.ts @@ -1,7 +1,7 @@ import { Codework as Mirror } from "@codeworksh/widget/codework"; import { Codework as Client } from "@codeworksh/widget/backend"; import { NodeSocket } from "@effect/platform-node"; -import { Deferred, Duration, Effect, FileSystem, Layer, Schedule, Schema, Stream } from "effect"; +import { Deferred, Duration, Effect, FileSystem, Layer, Option, Schedule, Schema, Stream } from "effect"; import { RpcClient, RpcSerialization } from "effect/rpc"; import { Socket } from "effect/socket"; /* @effect-diagnostics nodeBuiltinImport:off -- a home's registration is a file path. */ @@ -9,11 +9,6 @@ import path from "node:path"; import { CodeworkUnavailable, HostError } from "@codeworksh/contract"; import { Servers } from "./servers.ts"; -export type ThreadInfo = Mirror.ThreadInfo; - -/** What an untitled thread is called until its first prompt names it. */ -export const UNTITLED = Mirror.UNTITLED; - /** How a running server advertises itself: `/service.json`, written by `codework serve`. */ const Registration = Schema.fromJsonString( Schema.Struct({ pid: Schema.Int, version: Schema.String, url: Schema.String, port: Schema.Int }), @@ -62,37 +57,69 @@ const endOnHomeChange = (socket: Socket.Socket, changes: Stream.Stream): }), }); +/** A client over `socket`, which reconnects on its own. */ +const connect = (socket: Layer.Layer) => + Layer.effect(Client, RpcClient.make(Mirror.Api, { flatten: true })).pipe( + Layer.provide(RpcClient.layerProtocolSocket({ retryTransientErrors: true })), + Layer.provide(socket.pipe(Layer.provide(NodeSocket.layerWebSocketConstructor))), + Layer.provide(RpcSerialization.layerNdjson), + Layer.orDie, + ); + /** - * The app's one connection to a CodeWork server: the default server's. The registration is read on - * every connect, so a server restarted on a new port, a new home or a new default is picked up - * without restarting this app; until one is registered, connecting waits. + * The URL `registered` reads, waited for until there is one. The socket reads its URL while opening, + * which Effect runs uninterruptibly; waiting there for a server to register would otherwise keep the + * app server from ever shutting down. */ -export const layer = Layer.effect(Client, RpcClient.make(Mirror.Api, { flatten: true })).pipe( - Layer.provide(RpcClient.layerProtocolSocket({ retryTransientErrors: true })), - Layer.provide( +const waitFor = (registered: Effect.Effect>) => + registered.pipe( + Effect.flatMap(Effect.fromOption), + Effect.map((found) => found.url), + Effect.retry(Schedule.spaced("1 second")), + Effect.orDie, + Effect.interruptible, + ); + +/** + * The app's connection to the default server. The registration is read on every connect, so a server + * restarted on a new port, a new home or a new default is picked up without restarting this app; + * until one is registered, connecting waits. + */ +export const layer = connect( + Layer.effect( + Socket.Socket, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const servers = yield* Servers.Service; + const url = waitFor( + registration.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Servers.Service, servers), + ), + ); + return endOnHomeChange(yield* Socket.makeWebSocket(url), servers.homeChanges); + }), + ), +); + +/** + * A connection to one machine's server: a remote one at its URL, a local one at what its home + * registers, read on every connect as {@link layer} does. + */ +export const machine = (row: Servers.Row) => + connect( Layer.effect( Socket.Socket, Effect.gen(function* () { + if (row.kind === "remote") return yield* Socket.makeWebSocket(row.url); const fs = yield* FileSystem.FileSystem; - const servers = yield* Servers.Service; - const url = registration.pipe( - Effect.flatMap(Effect.fromOption), - Effect.map((registered) => registered.url), - Effect.retry(Schedule.spaced("1 second")), - Effect.orDie, - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Servers.Service, servers), - // The socket reads its URL while opening, which Effect runs uninterruptibly; waiting there - // for a server to register would otherwise keep the app server from ever shutting down. - Effect.interruptible, + const home = (yield* Servers.Service).homeOf(row).path; + return yield* Socket.makeWebSocket( + waitFor(registrationAt(home).pipe(Effect.provideService(FileSystem.FileSystem, fs))), ); - return endOnHomeChange(yield* Socket.makeWebSocket(url), servers.homeChanges); }), - ).pipe(Layer.provide(NodeSocket.layerWebSocketConstructor)), - ), - Layer.provide(RpcSerialization.layerNdjson), - Layer.orDie, -); + ), + ); export interface CallOptions { /** How long the server has to answer; 5 seconds unless given. */ diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index 74d0b3eb..3a24b33b 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -17,17 +17,18 @@ const migrations = { PRIMARY KEY (package, key) ) `; + // `tab` is what the workspace's tab last read, so a workspace not yet opened since a restart names what it shows. yield* sql` CREATE TABLE workspaces ( id TEXT PRIMARY KEY, bit INTEGER NOT NULL UNIQUE, pos INTEGER NOT NULL, name TEXT NOT NULL, - icon TEXT NOT NULL, layout TEXT NOT NULL, mfact REAL NOT NULL, auto INTEGER NOT NULL DEFAULT 0, - created INTEGER NOT NULL + created INTEGER NOT NULL, + tab TEXT ) `; // AUTOINCREMENT so a seq is never reused, even after the newest rows are deleted. @@ -60,12 +61,11 @@ const migrations = { created INTEGER NOT NULL ) `; - }), - // The built-in local server takes over its settings, kept until now as single kv entries. - "202610180001_servers": Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; + + // The sync's copy of each machine's CodeWork: core columns are written only by the sync, + // desktop columns only by this app. yield* sql` - CREATE TABLE servers ( + CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL, @@ -73,43 +73,192 @@ const migrations = { home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', + url TEXT, + namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, - created INTEGER NOT NULL + store_id TEXT UNIQUE, + cursor INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ) + `; + yield* sql` + CREATE TABLE organization ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `; + yield* sql` + CREATE TABLE grouping ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organization(id), + name TEXT NOT NULL, + color TEXT, + ordering REAL NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `; + yield* sql`CREATE INDEX grouping_ordering_idx ON grouping (organization_id, ordering)`; + // Core ids are repo hashes, so the same repo on two machines collides: keyed by machine too. + yield* sql` + CREATE TABLE project ( + machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + id TEXT NOT NULL, + organization_id TEXT NOT NULL REFERENCES organization(id), + namespace TEXT NOT NULL, + name TEXT NOT NULL, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (machine_id, id) ) `; - const rows = yield* sql<{ key: string; value: string }>` - SELECT key, value FROM kv WHERE package = 'codework' AND key LIKE 'servers.local.%' + yield* sql`CREATE INDEX project_organization_idx ON project (organization_id, namespace, status)`; + yield* sql` + CREATE TABLE space ( + machine_id TEXT NOT NULL, + id TEXT NOT NULL, + organization_id TEXT NOT NULL REFERENCES organization(id), + project_id TEXT NOT NULL, + location TEXT NOT NULL, + kind TEXT NOT NULL, + env TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (machine_id, id), + FOREIGN KEY (machine_id, project_id) REFERENCES project(machine_id, id) ON DELETE CASCADE + ) `; - const saved = new Map(rows.map(({ key, value }) => [key.slice("servers.local.".length), JSON.parse(value)])); - const text = (key: string) => { - const value: unknown = saved.get(key); - return typeof value === "string" && value !== "" ? value : null; - }; - const env: unknown = saved.get("env"); - yield* sql`INSERT INTO servers ${sql.insert({ + yield* sql`CREATE INDEX space_project_idx ON space (machine_id, project_id)`; + yield* sql` + CREATE TABLE session ( + id TEXT PRIMARY KEY, + machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + organization_id TEXT NOT NULL REFERENCES organization(id), + space_id TEXT NOT NULL, + slug TEXT NOT NULL, + dir TEXT NOT NULL, + hostdir TEXT, + tag TEXT, + metadata TEXT, + color TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + deleted_at INTEGER, + FOREIGN KEY (machine_id, space_id) REFERENCES space(machine_id, id) + ) + `; + yield* sql`CREATE INDEX session_space_idx ON session (machine_id, space_id)`; + yield* sql`CREATE INDEX session_list_idx ON session (organization_id, updated_at) WHERE deleted_at IS NULL`; + // No FK on origin_thread_id: a source can arrive after its fork. + yield* sql` + CREATE TABLE thread ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organization(id), + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + origin_kind TEXT NOT NULL, + origin_thread_id TEXT, + origin_entry_id TEXT, + origin_call_id TEXT, + origin_committed_seq INTEGER, + title TEXT, + model_provider TEXT, + model_id TEXT, + thinking_level TEXT, + foundation TEXT, + placement_space_id TEXT, + placement_dir TEXT, + restarts INTEGER NOT NULL DEFAULT 0, + attempt INTEGER, + run_state TEXT NOT NULL DEFAULT 'idle', + run_state_at INTEGER, + run_started_at INTEGER, + run_ended_at INTEGER, + run_outcome TEXT, + run_error TEXT, + cost REAL NOT NULL DEFAULT 0, + tokens_input INTEGER NOT NULL DEFAULT 0, + tokens_output INTEGER NOT NULL DEFAULT 0, + tokens_cache_read INTEGER NOT NULL DEFAULT 0, + tokens_cache_write INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + status_at INTEGER NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + grouping_id TEXT REFERENCES grouping(id) ON DELETE SET NULL, + ordering REAL NOT NULL, + color TEXT, + cursor INTEGER, + floor INTEGER, + created_seq INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + deleted_at INTEGER + ) + `; + yield* sql`CREATE UNIQUE INDEX thread_main_idx ON thread (session_id) WHERE origin_kind = 'main'`; + yield* sql`CREATE INDEX thread_session_idx ON thread (session_id, updated_at) WHERE deleted_at IS NULL`; + yield* sql`CREATE INDEX thread_origin_idx ON thread (origin_thread_id) WHERE origin_thread_id IS NOT NULL`; + yield* sql`CREATE INDEX thread_status_idx ON thread (organization_id, status, ordering) WHERE deleted_at IS NULL`; + yield* sql`CREATE INDEX thread_grouping_idx ON thread (organization_id, grouping_id, ordering) WHERE deleted_at IS NULL`; + yield* sql`CREATE INDEX thread_priority_idx ON thread (organization_id, priority, ordering) WHERE deleted_at IS NULL`; + yield* sql` + CREATE TABLE entry ( + id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL REFERENCES thread(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + state TEXT NOT NULL, + error TEXT, + size INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `; + yield* sql`CREATE UNIQUE INDEX entry_thread_idx ON entry (thread_id, seq)`; + yield* sql`CREATE INDEX entry_draft_idx ON entry (thread_id) WHERE state = 'draft'`; + yield* sql` + CREATE TABLE part ( + entry_id TEXT NOT NULL REFERENCES entry(id) ON DELETE CASCADE, + part_index INTEGER NOT NULL, + type TEXT NOT NULL, + status TEXT, + call_id TEXT, + tool_name TEXT, + data TEXT NOT NULL, + PRIMARY KEY (entry_id, part_index) + ) + `; + yield* sql`CREATE UNIQUE INDEX part_call_idx ON part (entry_id, call_id) WHERE call_id IS NOT NULL`; + yield* sql`CREATE INDEX part_unsettled_idx ON part (entry_id) WHERE status IN ('pending', 'running')`; + yield* sql` + CREATE TABLE inbox ( + id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL REFERENCES thread(id) ON DELETE CASCADE, + delivery TEXT NOT NULL, + text TEXT NOT NULL, + admitted_seq INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `; + yield* sql`CREATE UNIQUE INDEX inbox_thread_idx ON inbox (thread_id, admitted_seq)`; + + const now = yield* Clock.currentTimeMillis; + // The built-in local machine runs the CLI home's CodeWork: a null executable and home are the ones found. + yield* sql`INSERT INTO machine ${sql.insert({ id: "local", kind: "local", - name: text("name") ?? "CodeWork", - executable: text("executable"), - home: text("home"), - args: text("args") ?? "", - env: JSON.stringify(Array.isArray(env) ? env : []), + name: "CodeWork", + namespace: "codework", isDefault: 1, pos: 0, - created: yield* Clock.currentTimeMillis, + createdAt: now, })}`; - yield* sql`DELETE FROM kv WHERE package = 'codework' AND key LIKE 'servers.local.%'`; - }), - // Workspace tabs all show the window icon now. - "202610230001_drop_workspace_icon": Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`ALTER TABLE workspaces DROP COLUMN icon`; - }), - // What the workspace's tab last read, so a workspace not yet opened since a restart names what it shows. - "202610240001_workspace_tab": Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`ALTER TABLE workspaces ADD COLUMN tab TEXT`; + // The one organization the UI uses until auth exists. + yield* sql`INSERT INTO organization ${sql.insert({ id: "personal", name: "Personal", createdAt: now, updatedAt: now })}`; }), }; diff --git a/apps/server/src/event.ts b/apps/server/src/event.ts index e0e199a4..331083ec 100644 --- a/apps/server/src/event.ts +++ b/apps/server/src/event.ts @@ -11,7 +11,7 @@ import { Stream, SubscriptionRef, } from "effect"; -import { SqlClient } from "effect/sql"; +import { SqlClient, type SqlError } from "effect/sql"; import { WidgetEvent, type Event, type Publish, type Request, type Synced } from "@codeworksh/widget/event"; import { Db } from "./db.ts"; @@ -29,6 +29,13 @@ export interface Interface { }) => Stream.Stream; /** How many subscriptions follow `topic`: the current count, then each change. */ readonly watchers: (topic: string) => Stream.Stream; + /** + * Runs `body` in one database transaction; what it appends is stored in that transaction and + * handed to live subscribers once it commits, so a row and the event announcing it land together. + */ + readonly transact: ( + body: (append: (input: Publish) => Effect.Effect) => Effect.Effect, + ) => Effect.Effect; } export class Service extends Context.Service()("@codeworksh/server/event/Service") {} @@ -98,38 +105,68 @@ export const layer = Layer.effect( ORDER BY seq `); - const publish = Effect.fn("Event.publish")(function* ({ ephemeral, ...input }: Publish) { + const prepare = Effect.fnUntraced(function* ({ ephemeral, ...input }: Publish) { const id = input.id ?? (yield* Effect.sync(() => crypto.randomUUID())); - const created = yield* Clock.currentTimeMillis; - const event: Event = { ...input, id, created }; - if (ephemeral === true) { + const event: Event = { ...input, id, created: yield* Clock.currentTimeMillis }; + return { event, ephemeral: ephemeral === true }; + }); + + // Run under `lock`: the seq it takes is broadcast before any later one. + const insert = (event: Event) => + Effect.gen(function* () { + const text = yield* encodePayload(event.payload).pipe(Effect.orDie); + const rows = yield* sql<{ seq: number }>` + INSERT INTO event ${sql.insert({ + id: event.id, + topic: event.topic, + source: event.source, + cause: event.cause ?? null, + payload: text, + created: event.created, + })} RETURNING seq + `.pipe(Effect.orDie); + return { ...event, seq: rows[0]!.seq } satisfies Event; + }); + + const publish = Effect.fn("Event.publish")(function* (input: Publish) { + const { event, ephemeral } = yield* prepare(input); + if (ephemeral) { yield* PubSub.publish(live, event); return event; } - const text = yield* encodePayload(input.payload).pipe(Effect.orDie); return yield* lock.withPermit( Effect.gen(function* () { - const stored = yield* byId(id); + const stored = yield* byId(event.id); if (stored !== undefined) return stored; - const rows = yield* sql<{ seq: number }>` - INSERT INTO event ${sql.insert({ - id, - topic: input.topic, - source: input.source, - cause: input.cause ?? null, - payload: text, - created, - })} RETURNING seq - `.pipe(Effect.orDie); - const committed: Event = { ...event, seq: rows[0]!.seq }; + const committed = yield* insert(event); yield* PubSub.publish(live, committed); return committed; }), ); }); + // The lock is taken before the transaction, as `publish` takes it before the connection: the + // other order would deadlock against a publish waiting for the connection. + const transact: Interface["transact"] = (body) => + lock.withPermit( + Effect.gen(function* () { + const pending: Array = []; + const result = yield* sql.withTransaction( + body((input) => + Effect.gen(function* () { + const { event, ephemeral } = yield* prepare(input); + pending.push(ephemeral ? event : yield* insert(event)); + }), + ), + ); + yield* PubSub.publishAll(live, pending); + return result; + }), + ); + return Service.of({ publish, + transact, watchers: (topic) => SubscriptionRef.changes(watching).pipe( Stream.map((counts) => counts.get(topic) ?? 0), diff --git a/apps/server/src/handlers.ts b/apps/server/src/handlers.ts index fb573e16..886425bd 100644 --- a/apps/server/src/handlers.ts +++ b/apps/server/src/handlers.ts @@ -1,58 +1,29 @@ import { Effect, Stream } from "effect"; +import { SqlClient } from "effect/sql"; import { Cli } from "./cli.ts"; import { Codework } from "./codework.ts"; -import { - type CliAction, - Contract, - ModelsError, - type Project, - type ServerAdd, - type ServerUpdate, -} from "@codeworksh/contract"; +import { type CliAction, Contract, ModelsError, type ServerAdd, type ServerUpdate } from "@codeworksh/contract"; import { Desk } from "./desk.ts"; import { Event } from "./event.ts"; import { Health } from "./health.ts"; import { Kv } from "./kv.ts"; +import { Reads } from "./reads.ts"; import { Servers } from "./servers.ts"; +import { Sync } from "./sync.ts"; +import { Tracking } from "./tracking.ts"; import { Workspaces } from "./workspaces.ts"; -/** - * Sessions grouped by project, both newest first, each shown as its main thread. A session whose - * project row is missing gets a group of its own folder, so it still shows rather than vanishing. - */ -const byProject = (threads: ReadonlyArray): Project[] => { - const groups = new Map(); - const mains = threads.filter((thread) => thread.origin.kind === "main"); - for (const thread of [...mains].sort((a, b) => b.updatedAt - a.updatedAt)) { - const session = thread.session; - const folder = session.hostDir ?? session.directory; - const key = session.project?.id ?? folder; - const group = groups.get(key) ?? { - id: key, - name: session.project?.name ?? folder.split("/").at(-1) ?? folder, - // The newest session's folder: where the project was last worked on. - path: folder, - sessions: [], - }; - group.sessions.push({ - id: session.id, - title: thread.title ?? Codework.UNTITLED, - updated: thread.updatedAt, - ...(thread.model === undefined ? {} : { model: thread.model }), - }); - groups.set(key, group); - } - return [...groups.values()]; -}; - export const layer = Contract.Api.toLayer( Effect.gen(function* () { const workspaces = yield* Workspaces.handlers; const desk = yield* Desk.handlers; + const tracking = yield* Tracking.handlers; const events = yield* Event.Service; const kv = yield* Kv.Service; const servers = yield* Servers.Service; const health = yield* Health.Service; + const sync = yield* Sync.Service; + const sql = yield* SqlClient.SqlClient; // A failed call is the first sign the default server went down. const call = (use: Parameters>[0], options?: Codework.CallOptions) => Codework.call(use, options).pipe(Effect.tapError(() => health.poke)); @@ -71,10 +42,11 @@ export const layer = Contract.Api.toLayer( return Contract.Api.of({ ...workspaces, ...desk, + ...tracking, - "projects.list": Effect.fn("projects.list")(function* () { - return byProject(yield* call((client) => client("thread.list", {}))); - }), + "sidebar.list": (input) => Reads.sidebar(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "dashboard.list": (input) => Reads.dashboard(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "threads.line": (input) => Reads.line(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), "host.info": Effect.fn("host.info")(function* () { return yield* call((client) => client("host.info", {})); @@ -159,9 +131,13 @@ export const layer = Contract.Api.toLayer( readonly action: CliAction; }) { // Checked again whether it worked or not, so the status shows what the CLI left. - yield* Cli.service(yield* servers.get(id), action).pipe(Effect.ensuring(health.check)); + yield* Cli.service(yield* servers.getLocal(id), action).pipe(Effect.ensuring(health.check)); }), + "sync.open": (ref) => sync.open(ref), + "sync.close": (ref) => sync.close(ref), + "sync.page": (input) => sync.page(input), + "kv.get": ({ package: pkg, key }) => kv.get(pkg, key), "kv.set": ({ package: pkg, key, value }) => kv.set(pkg, key, value), diff --git a/apps/server/src/health.ts b/apps/server/src/health.ts index 2bb0b449..7863f51c 100644 --- a/apps/server/src/health.ts +++ b/apps/server/src/health.ts @@ -52,8 +52,8 @@ export const layer = Layer.effect( const checkDefault = (asked: boolean) => Effect.gen(function* () { const before = (yield* SubscriptionRef.get(current))?.find((status) => status.isDefault); - if (before === undefined) return; - const home = servers.homeOf(yield* servers.get(before.id)).path; + if (before === undefined || before.kind === "remote") return; + const home = servers.homeOf(yield* servers.getLocal(before.id)).path; if (!asked && before.server._tag === "Running") { const live = yield* Cli.liveAt(home).pipe(Effect.provideContext(context)); if (Option.isNone(live)) yield* check; diff --git a/apps/server/src/mirror.ts b/apps/server/src/mirror.ts new file mode 100644 index 00000000..753ffe9c --- /dev/null +++ b/apps/server/src/mirror.ts @@ -0,0 +1,335 @@ +import { Effect, String as Str } from "effect"; +import { SqlClient } from "effect/sql"; +import type { Publish } from "@codeworksh/widget/event"; +import type { Codework } from "@codeworksh/widget/codework"; +import { Ordering } from "./ordering.ts"; +import { + PERSONAL_ORGANIZATION as ORGANIZATION, + syncTopics, + type SyncChanged, + type SyncStatus, + type ThreadStatus, +} from "@codeworksh/contract"; + +/* + * The writes that copy a machine's CodeWork rows into app.db. Each runs inside the caller's + * transaction; core columns come from the rows, desktop columns are set once, on insert. + */ + +type IndexRow = Exclude; +type ThreadRow = Exclude; +type Append = (input: Publish) => Effect.Effect; + +/** The rows a batch wrote, for its `SyncChanged` notice. */ +export class Changes { + readonly project = new Set(); + readonly space = new Set(); + readonly session = new Set(); + readonly thread = new Set(); + readonly entry = new Set(); + readonly inbox = new Set(); + readonly removed = new Set(); + + notice(input: { machineId: string; threadId?: string; full: boolean; items: number }): SyncChanged { + return { + ...input, + project: [...this.project], + space: [...this.space], + session: [...this.session], + thread: [...this.thread], + entry: [...this.entry], + inbox: [...this.inbox], + removed: [...this.removed], + }; + } +} + +/** Inserts `core` and `created`, or updates only `core` when the row at `keys` exists. */ +const upsert = (table: string, keys: ReadonlyArray, core: Record, created = {}) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const set = Object.keys(core) + .map(Str.camelToSnake) + .filter((column) => !keys.includes(column)) + .map((column) => `${column} = excluded.${column}`) + .join(", "); + yield* sql` + INSERT INTO ${sql(table)} ${sql.insert({ ...core, ...created })} + ON CONFLICT (${sql.literal(keys.join(", "))}) DO UPDATE SET ${sql.literal(set)} + `; + }); + +/** The organization of the row `query` finds, else the one every project starts in. */ +const organizationOf = (query: Effect.Effect, unknown>) => + query.pipe(Effect.map((rows) => rows[0]?.organizationId ?? ORGANIZATION)); + +/** Where a thread goes when its run moves, by the incoming row against the stored one (section 5 of the design). */ +const moved = ( + stored: { readonly runStartedAt: number | null; readonly runEndedAt: number | null }, + thread: Codework.ThreadSummary, +): ThreadStatus | undefined => { + const run = thread.lastRun; + if (run === undefined) return undefined; + if (run.startedAt !== stored.runStartedAt && thread.runState !== "idle") return "working"; + if (run.endedAt === undefined || run.endedAt === stored.runEndedAt) return undefined; + if (run.outcome === "succeeded" || run.outcome === "failed") return "attention"; + if (run.error?._tag === "interrupted" && run.error.reason === "user") return "cancelled"; + // Shutdown or crash: a resume or a `run.failed` follows and decides. + return undefined; +}; + +const firstSeen = (thread: Codework.ThreadSummary): ThreadStatus => + thread.runState !== "idle" ? "working" : thread.lastRun?.outcome === "failed" ? "attention" : "settled"; + +const threadCore = (thread: Codework.ThreadSummary) => { + const origin = thread.origin; + const run = thread.lastRun; + return { + sessionId: thread.sessionId, + originKind: origin.kind, + originThreadId: origin.kind === "main" ? null : origin.threadId, + originEntryId: origin.kind === "main" ? null : (origin.entryId ?? null), + originCallId: origin.kind === "subagent" ? origin.callId : null, + originCommittedSeq: origin.kind === "main" ? null : (origin.cut ?? null), + title: thread.title ?? null, + modelProvider: thread.model?.provider ?? null, + modelId: thread.model?.id ?? null, + thinkingLevel: thread.thinkingLevel ?? null, + foundation: thread.foundation ?? null, + placementSpaceId: thread.placement?.spaceId ?? null, + placementDir: thread.placement?.directory ?? null, + restarts: thread.restarts, + attempt: thread.attempt ?? null, + runState: thread.runState, + runStateAt: thread.runStateAt ?? null, + runStartedAt: run?.startedAt ?? null, + runEndedAt: run?.endedAt ?? null, + runOutcome: run?.outcome ?? null, + runError: run?.error === undefined ? null : JSON.stringify(run.error), + cost: thread.usage.cost, + tokensInput: thread.usage.input, + tokensOutput: thread.usage.output, + tokensCacheRead: thread.usage.cacheRead, + tokensCacheWrite: thread.usage.cacheWrite, + createdSeq: thread.createdSeq, + createdAt: thread.createdAt, + updatedAt: thread.updatedAt, + // The store has the row, so a soft delete from an earlier full sync (a restored store) is undone. + deletedAt: null, + }; +}; + +const storeThread = (thread: Codework.ThreadSummary, now: number, append: Append) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const core = threadCore(thread); + const [stored] = yield* sql<{ status: ThreadStatus; runStartedAt: number | null; runEndedAt: number | null }>` + SELECT status, run_started_at, run_ended_at FROM thread WHERE id = ${thread.id} + `; + if (stored === undefined) { + const organizationId = yield* organizationOf( + sql`SELECT organization_id FROM session WHERE id = ${thread.sessionId}`, + ); + yield* sql`INSERT INTO thread ${sql.insert({ + id: thread.id, + ...core, + organizationId, + status: firstSeen(thread), + statusAt: now, + priority: 0, + ordering: yield* Ordering.edge("thread", organizationId, "first"), + })}`; + return; + } + yield* sql`UPDATE thread SET ${sql.update(core)} WHERE id = ${thread.id}`; + const status = moved(stored, thread); + if (status === undefined || status === stored.status) return; + yield* sql`UPDATE thread SET status = ${status}, status_at = ${now} WHERE id = ${thread.id}`; + const payload: SyncStatus = { threadId: thread.id, status, previous: stored.status, at: now }; + yield* append({ topic: syncTopics.status, source: "sync", payload }); + }); + +/** Applies one `index.sync` batch of a machine's rows. */ +export const index = ( + machine: { readonly id: string; readonly namespace: string }, + rows: ReadonlyArray, + now: number, + append: Append, +) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const changes = new Changes(); + for (const row of rows) { + switch (row._tag) { + case "Project": { + const { project } = row; + yield* upsert( + "project", + ["machine_id", "id"], + { + machineId: machine.id, + id: project.id, + namespace: machine.namespace, + name: project.name, + status: project.status, + createdAt: project.createdAt, + updatedAt: project.updatedAt, + }, + { organizationId: ORGANIZATION }, + ); + changes.project.add(project.id); + break; + } + case "Space": { + const { space } = row; + const organizationId = yield* organizationOf( + sql`SELECT organization_id FROM project WHERE machine_id = ${machine.id} AND id = ${space.projectId}`, + ); + yield* upsert( + "space", + ["machine_id", "id"], + { + machineId: machine.id, + id: space.id, + projectId: space.projectId, + location: space.location, + kind: space.kind, + env: space.env ?? null, + status: space.status, + createdAt: space.createdAt, + updatedAt: space.updatedAt, + }, + { organizationId }, + ); + changes.space.add(space.id); + break; + } + case "Session": { + const { session } = row; + const organizationId = yield* organizationOf( + sql`SELECT organization_id FROM space WHERE machine_id = ${machine.id} AND id = ${session.spaceId}`, + ); + yield* upsert( + "session", + ["id"], + { + id: session.id, + machineId: machine.id, + spaceId: session.spaceId, + slug: session.slug, + dir: session.directory, + hostdir: session.hostDir ?? null, + tag: session.tag ?? null, + metadata: session.metadata === undefined ? null : JSON.stringify(session.metadata), + createdAt: session.createdAt, + updatedAt: session.updatedAt, + deletedAt: null, + }, + { organizationId }, + ); + changes.session.add(session.id); + break; + } + case "Thread": + yield* storeThread(row.thread, now, append); + changes.thread.add(row.thread.id); + break; + case "Removed": + yield* sql`UPDATE session SET deleted_at = ${now} WHERE id = ${row.id} AND deleted_at IS NULL`; + yield* sql`UPDATE thread SET deleted_at = ${now} WHERE session_id = ${row.id} AND deleted_at IS NULL`; + changes.removed.add(row.id); + break; + } + } + return changes; + }); + +/** + * Ends a full sync's first batch: the machine's sessions and threads it did not send are gone from + * the store, and every thread's detail starts over from a first open. + */ +export const unseen = (machineId: string, seen: Changes, now: number) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const sessions = [...seen.session]; + const threads = [...seen.thread]; + const gone = yield* sql<{ id: string }>` + SELECT id FROM session WHERE machine_id = ${machineId} AND deleted_at IS NULL + ${sessions.length === 0 ? sql.literal("") : sql`AND id NOT IN ${sql.in(sessions)}`} + `; + for (const { id } of gone) seen.removed.add(id); + yield* sql` + UPDATE session SET deleted_at = ${now} WHERE machine_id = ${machineId} AND deleted_at IS NULL + ${sessions.length === 0 ? sql.literal("") : sql`AND id NOT IN ${sql.in(sessions)}`} + `; + yield* sql` + UPDATE thread SET deleted_at = ${now} + WHERE deleted_at IS NULL AND session_id IN (SELECT id FROM session WHERE machine_id = ${machineId}) + ${threads.length === 0 ? sql.literal("") : sql`AND id NOT IN ${sql.in(threads)}`} + `; + yield* sql` + UPDATE thread SET cursor = NULL WHERE session_id IN (SELECT id FROM session WHERE machine_id = ${machineId}) + `; + }); + +/** Stores entries of a thread's line, each under the thread that wrote it, with its parts. */ +export const entries = (list: ReadonlyArray, changes: Changes) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const entry of list) { + yield* upsert("entry", ["id"], { + id: entry.id, + threadId: entry.threadId, + seq: entry.seq, + type: entry.role, + state: entry.state, + error: entry.error ?? null, + size: entry.size, + createdAt: entry.createdAt, + updatedAt: entry.updatedAt, + }); + yield* sql`DELETE FROM part WHERE entry_id = ${entry.id}`; + if (entry.parts.length > 0) + yield* sql`INSERT INTO part ${sql.insert( + entry.parts.map((part, partIndex) => ({ + entryId: entry.id, + partIndex, + type: part.type, + status: part.type === "toolCall" ? part.status : null, + callId: part.type === "toolCall" ? part.callID : null, + toolName: part.type === "toolCall" ? part.name : null, + data: JSON.stringify(part), + })), + )}`; + changes.entry.add(entry.id); + } + }); + +/** Applies one `thread.sync` batch of a thread's rows; deltas are never stored. */ +export const thread = (threadId: string, rows: ReadonlyArray) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const changes = new Changes(); + yield* entries( + rows.flatMap((row) => (row._tag === "Entry" ? [row.entry] : [])), + changes, + ); + for (const row of rows) { + if (row._tag !== "Inbox") continue; + const { prompt } = row; + if (row.state === "pending") + yield* upsert("inbox", ["id"], { + id: prompt.id, + threadId, + delivery: prompt.delivery, + text: prompt.text, + admittedSeq: prompt.admittedSeq, + createdAt: prompt.createdAt, + }); + else yield* sql`DELETE FROM inbox WHERE id = ${prompt.id}`; + changes.inbox.add(row.prompt.id); + } + return changes; + }); + +export * as Mirror from "./mirror.ts"; diff --git a/apps/server/src/ordering.ts b/apps/server/src/ordering.ts new file mode 100644 index 00000000..95183b4e --- /dev/null +++ b/apps/server/src/ordering.ts @@ -0,0 +1,27 @@ +import { Effect } from "effect"; +import { SqlClient } from "effect/sql"; + +/* + * Manual order on threads and groups, per organization. Fractional, so a move writes one row: + * between two neighbours is their midpoint, first is the lowest minus one, last the highest plus one. + */ + +export type Table = "thread" | "grouping"; + +/** The value that puts a row first or last among its organization's `table` rows. */ +export const edge = (table: Table, organizationId: string, end: "first" | "last") => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const value = + end === "first" + ? sql.literal("COALESCE(MIN(ordering), 1) - 1") + : sql.literal("COALESCE(MAX(ordering), -1) + 1"); + const [row] = yield* sql<{ ordering: number }>` + SELECT ${value} AS ordering FROM ${sql(table)} WHERE organization_id = ${organizationId} + `; + return row?.ordering ?? 0; + }); + +export const midpoint = (after: number, before: number) => (after + before) / 2; + +export * as Ordering from "./ordering.ts"; diff --git a/apps/server/src/reads.ts b/apps/server/src/reads.ts new file mode 100644 index 00000000..104f5875 --- /dev/null +++ b/apps/server/src/reads.ts @@ -0,0 +1,408 @@ +import { Effect, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import { + type Dashboard, + type DashboardBy, + type DashboardList, + type DashboardThread, + type Grouping, + type LineRequest, + LineThread, + type Priority, + type SidebarProject, + type SidebarSession, + SyncError, + type ThreadLine, + type ThreadStatus, +} from "@codeworksh/contract"; +import { HistoryEntry, Part, QueuedPrompt, RunError, UNTITLED } from "@codeworksh/widget/codework"; + +/* + * What the UI reads from app.db: the sidebar and a thread's line. Neither calls CodeWork, so both + * answer from the copy while a machine is away. + */ + +/** A first page's bytes, as `thread.sync`'s. */ +const BUDGET = 1 << 20; + +const decodePart = Schema.decodeUnknownEffect(Schema.fromJsonString(Part)); +const decodeRunError = Schema.decodeUnknownEffect(Schema.fromJsonString(RunError)); +const decodeEntry = Schema.decodeUnknownEffect(HistoryEntry); +const decodePrompt = Schema.decodeUnknownEffect(QueuedPrompt); +const decodeThread = Schema.decodeUnknownEffect(LineThread); + +const model = (provider: string | null, id: string | null) => + provider === null || id === null ? {} : { model: { provider, id } }; + +/** The machine filter of `sidebar.list`. */ +const onMachines = (sql: SqlClient.SqlClient, machines: "default" | "all") => + machines === "all" ? sql.literal("") : sql`AND machine.is_default = 1`; + +export const sidebar = Effect.fn("Reads.sidebar")(function* ({ + organizationId, + machines, +}: { + readonly organizationId: string; + readonly machines: "default" | "all"; +}) { + const sql = yield* SqlClient.SqlClient; + const projects = yield* sql<{ + machineId: string; + machineName: string; + id: string; + name: string; + location: string | null; + updatedAt: number; + }>` + SELECT project.machine_id, machine.name AS machine_name, project.id, project.name, project.updated_at, + (SELECT location FROM space WHERE space.machine_id = project.machine_id AND space.project_id = project.id + ORDER BY space.kind = 'primary' DESC, space.created_at LIMIT 1) AS location + FROM project JOIN machine ON machine.id = project.machine_id + WHERE project.organization_id = ${organizationId} AND project.status = 'active' ${onMachines(sql, machines)} + ORDER BY project.updated_at DESC + `; + const sessions = yield* sql<{ + id: string; + machineId: string; + projectId: string; + spaceKind: SidebarSession["spaceKind"]; + color: string | null; + dir: string; + hostdir: string | null; + threadId: string; + title: string | null; + modelProvider: string | null; + modelId: string | null; + runState: SidebarSession["runState"]; + updatedAt: number; + }>` + SELECT session.id, session.machine_id, space.project_id, space.kind AS space_kind, session.color, + session.dir, session.hostdir, thread.id AS thread_id, thread.title, thread.model_provider, + thread.model_id, thread.run_state, + (SELECT MAX(activity.updated_at) FROM thread AS activity + WHERE activity.session_id = session.id AND activity.deleted_at IS NULL) AS updated_at + FROM session + JOIN space ON space.machine_id = session.machine_id AND space.id = session.space_id + JOIN thread ON thread.session_id = session.id AND thread.origin_kind = 'main' AND thread.deleted_at IS NULL + JOIN machine ON machine.id = session.machine_id + WHERE session.organization_id = ${organizationId} AND session.deleted_at IS NULL ${onMachines(sql, machines)} + ORDER BY updated_at DESC + `; + const byProject = new Map>(); + for (const row of sessions) { + const key = `${row.machineId}/${row.projectId}`; + const list = byProject.get(key) ?? []; + list.push({ + id: row.id, + threadId: row.threadId, + spaceKind: row.spaceKind, + color: row.color, + title: row.title ?? UNTITLED, + ...model(row.modelProvider, row.modelId), + runState: row.runState, + updated: row.updatedAt, + path: row.hostdir ?? row.dir, + }); + byProject.set(key, list); + } + const listed = projects.map((project): SidebarProject => { + const list = byProject.get(`${project.machineId}/${project.id}`) ?? []; + return { + machine: { id: project.machineId, name: project.machineName }, + id: project.id, + name: project.name, + path: list[0]?.path ?? project.location ?? project.name, + sessions: list, + }; + }); + // Newest activity first; a project without sessions by its own last change. + const activity = (project: SidebarProject) => project.sessions[0]?.updated ?? -1; + return listed.toSorted((a, b) => activity(b) - activity(a)); +}, Effect.orDie); + +/** The status lists in the order the dashboard shows them: what needs the user first. */ +const STATUSES: ReadonlyArray = ["working", "attention", "settled", "cancelled"]; +/** Urgent first and none last, as Linear. */ +const PRIORITIES: ReadonlyArray = [1, 2, 3, 4, 0]; + +/** Each `by` reads its list column then `ordering`, as its index (`thread__idx`) holds them. */ +const SPLIT = { + status: "thread.status, thread.ordering", + priority: "thread.priority, thread.ordering", + grouping: "thread.grouping_id, thread.ordering", +} satisfies Record; + +export const dashboard = Effect.fn("Reads.dashboard")(function* ({ + organizationId, + by, +}: { + readonly organizationId: string; + readonly by: DashboardBy; +}) { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ + id: string; + sessionId: string; + originKind: DashboardThread["originKind"]; + originThreadId: string | null; + title: string | null; + projectId: string; + projectName: string; + runState: DashboardThread["runState"]; + status: ThreadStatus; + priority: Priority; + groupingId: string | null; + ordering: number; + color: string | null; + updatedAt: number; + }>` + SELECT thread.id, thread.session_id, thread.origin_kind, thread.origin_thread_id, thread.title, + project.id AS project_id, project.name AS project_name, thread.run_state, thread.status, thread.priority, + thread.grouping_id, thread.ordering, COALESCE(thread.color, session.color) AS color, thread.updated_at + FROM thread + JOIN session ON session.id = thread.session_id + JOIN space ON space.machine_id = session.machine_id AND space.id = session.space_id + JOIN project ON project.machine_id = space.machine_id AND project.id = space.project_id + WHERE thread.organization_id = ${organizationId} AND thread.deleted_at IS NULL AND thread.origin_kind <> 'subagent' + ORDER BY ${sql.literal(SPLIT[by])} + `; + const groups = yield* sql` + SELECT id, organization_id, name, color, ordering FROM grouping + WHERE organization_id = ${organizationId} ORDER BY ordering + `; + // An untitled fork reads as its nearest titled ancestor, as its line does. + const byId = new Map(rows.map((row) => [row.id, row])); + const titleOf = (row: (typeof rows)[number]) => { + const seen = new Set(); + for (let at: typeof row | undefined = row; at !== undefined && !seen.has(at.id);) { + if (at.title !== null) return at.title; + seen.add(at.id); + at = at.originThreadId === null ? undefined : byId.get(at.originThreadId); + } + return UNTITLED; + }; + const threads = rows.map((row): DashboardThread => ({ + id: row.id, + sessionId: row.sessionId, + originKind: row.originKind, + title: titleOf(row), + project: { id: row.projectId, name: row.projectName }, + runState: row.runState, + status: row.status, + priority: row.priority, + groupingId: row.groupingId, + ordering: row.ordering, + color: row.color, + updatedAt: row.updatedAt, + })); + const lists: Array = + by === "status" + ? STATUSES.map((status) => ({ + _tag: "Status", + status, + threads: threads.filter((thread) => thread.status === status), + })) + : by === "priority" + ? PRIORITIES.map((priority) => ({ + _tag: "Priority", + priority, + threads: threads.filter((thread) => thread.priority === priority), + })) + : [...groups.map((group) => group.id), null].map((groupingId) => ({ + _tag: "Grouping", + groupingId, + threads: threads.filter((thread) => thread.groupingId === groupingId), + })); + return { lists, groups } satisfies Dashboard; +}, Effect.orDie); + +/** + * A thread and the sources its line inherits from, as `(thread_id, upto, depth)`: each source's + * entries count up to the cut of the thread forked from it, as the harness walks it. + */ +const walk = (sql: SqlClient.SqlClient, threadId: string) => sql` + WITH RECURSIVE line(thread_id, upto, depth) AS ( + SELECT ${threadId}, NULL, 0 + UNION ALL + SELECT thread.origin_thread_id, thread.origin_committed_seq, line.depth + 1 + FROM line JOIN thread ON thread.id = line.thread_id + WHERE thread.origin_thread_id IS NOT NULL AND thread.origin_committed_seq IS NOT NULL + ) +`; + +interface EntryRow { + readonly id: string; + readonly threadId: string; + readonly seq: number; + readonly type: string; + readonly state: string; + readonly error: string | null; + readonly size: number; + readonly createdAt: number; + readonly updatedAt: number; +} + +const ENTRY_COLUMNS = + "entry.id, entry.thread_id, entry.seq, entry.type, entry.state, entry.error, entry.size, entry.created_at, entry.updated_at"; + +/** Entry rows as the wire's entries, with their parts. */ +const withParts = (rows: ReadonlyArray) => + Effect.gen(function* () { + if (rows.length === 0) return []; + const sql = yield* SqlClient.SqlClient; + const parts = yield* sql<{ entryId: string; data: string }>` + SELECT entry_id, data FROM part WHERE entry_id IN ${sql.in(rows.map((row) => row.id))} + ORDER BY entry_id, part_index + `; + const byEntry = new Map>(); + for (const part of parts) { + const list = byEntry.get(part.entryId) ?? []; + list.push(yield* decodePart(part.data)); + byEntry.set(part.entryId, list); + } + return yield* Effect.forEach(rows, ({ type, error, ...row }) => + decodeEntry({ + ...row, + role: type, + parts: byEntry.get(row.id) ?? [], + ...(error === null ? {} : { error }), + }), + ); + }); + +/** The stored entries with these ids, oldest first. */ +export const entries = Effect.fn("Reads.entries")(function* (ids: ReadonlyArray) { + if (ids.length === 0) return []; + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql` + SELECT ${sql.literal(ENTRY_COLUMNS)} FROM entry WHERE entry.id IN ${sql.in(ids)} ORDER BY entry.seq + `; + return yield* withParts(rows); +}, Effect.orDie); + +const head = (sql: SqlClient.SqlClient, { sessionId, threadId }: LineRequest) => + sql<{ + id: string; + sessionId: string; + machineId: string; + title: string | null; + modelProvider: string | null; + modelId: string | null; + thinkingLevel: string | null; + runState: string; + runStartedAt: number | null; + runEndedAt: number | null; + runOutcome: string | null; + runError: string | null; + updatedAt: number; + floor: number | null; + dir: string; + hostdir: string | null; + projectId: string; + projectName: string; + removed: number; + }>` + SELECT thread.id, thread.session_id, session.machine_id, thread.title, thread.model_provider, thread.model_id, + thread.thinking_level, thread.run_state, thread.run_started_at, thread.run_ended_at, thread.run_outcome, + thread.run_error, thread.updated_at, thread.floor, session.dir, session.hostdir, + project.id AS project_id, project.name AS project_name, + thread.deleted_at IS NOT NULL OR session.deleted_at IS NOT NULL AS removed + FROM thread + JOIN session ON session.id = thread.session_id + JOIN space ON space.machine_id = session.machine_id AND space.id = session.space_id + JOIN project ON project.machine_id = space.machine_id AND project.id = space.project_id + WHERE thread.session_id = ${sessionId} + AND ${threadId === undefined ? sql`thread.origin_kind = 'main'` : sql`thread.id = ${threadId}`} + `; + +export const line = Effect.fn("Reads.line")(function* (request: LineRequest) { + const sql = yield* SqlClient.SqlClient; + const [row] = yield* head(sql, request).pipe(Effect.orDie); + const name = `thread ${request.threadId ?? "(main)"} of session ${request.sessionId}`; + if (row === undefined) return yield* new SyncError({ message: `No ${name} is synced.` }); + if (row.removed === 1) return yield* new SyncError({ message: `The ${name} was deleted.`, removed: true }); + return yield* Effect.gen(function* () { + const lineOf = walk(sql, row.id); + const [titled] = + row.title === null + ? yield* sql<{ title: string }>` + ${lineOf} + SELECT thread.title FROM line JOIN thread ON thread.id = line.thread_id + WHERE thread.title IS NOT NULL ORDER BY line.depth LIMIT 1 + ` + : [{ title: row.title }]; + const { before, limit, budget = BUDGET } = request; + // The budget is a running total over the newest `limit` entries, so it reads no further; the newest always counts. + const page = + limit <= 0 + ? [] + : yield* sql` + SELECT id, thread_id, seq, type, state, error, size, created_at, updated_at FROM ( + SELECT *, SUM(size) OVER newest AS total, ROW_NUMBER() OVER newest AS n FROM ( + ${lineOf} + SELECT ${sql.literal(ENTRY_COLUMNS)} + FROM line JOIN entry ON entry.thread_id = line.thread_id + AND (line.upto IS NULL OR entry.seq <= line.upto) + ${before === undefined ? sql.literal("") : sql`WHERE entry.seq < ${before}`} + ORDER BY entry.seq DESC LIMIT ${limit} + ) + WINDOW newest AS (ORDER BY seq DESC) + ) + WHERE n = 1 OR total <= ${budget} + ORDER BY seq + `; + const below = page[0]?.seq ?? before; + const [older] = + below === undefined + ? [{ found: 0 }] + : yield* sql<{ found: number }>` + ${lineOf} + SELECT EXISTS ( + SELECT 1 FROM line JOIN entry ON entry.thread_id = line.thread_id + AND (line.upto IS NULL OR entry.seq <= line.upto) + WHERE entry.seq < ${below} + ) AS found + `; + const inbox = yield* sql<{ + id: string; + delivery: string; + text: string; + admittedSeq: number; + createdAt: number; + }>`SELECT id, delivery, text, admitted_seq, created_at FROM inbox WHERE thread_id = ${row.id} ORDER BY admitted_seq`; + const lastRun = + row.runStartedAt === null + ? {} + : { + lastRun: { + startedAt: row.runStartedAt, + ...(row.runEndedAt === null ? {} : { endedAt: row.runEndedAt }), + ...(row.runOutcome === null ? {} : { outcome: row.runOutcome }), + ...(row.runError === null ? {} : { error: yield* decodeRunError(row.runError) }), + }, + }; + const thread = yield* decodeThread({ + id: row.id, + sessionId: row.sessionId, + machineId: row.machineId, + ...(titled === undefined ? {} : { title: titled.title }), + ...model(row.modelProvider, row.modelId), + ...(row.thinkingLevel === null ? {} : { thinkingLevel: row.thinkingLevel }), + runState: row.runState, + ...lastRun, + updatedAt: row.updatedAt, + directory: row.dir, + ...(row.hostdir === null ? {} : { hostDir: row.hostdir }), + project: { id: row.projectId, name: row.projectName }, + }); + return { + thread, + entries: yield* withParts(page), + inbox: yield* Effect.forEach(inbox, (prompt) => decodePrompt(prompt)), + more: older !== undefined && older.found === 1, + floor: row.floor, + } satisfies ThreadLine; + }).pipe(Effect.orDie); +}); + +export * as Reads from "./reads.ts"; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8d3b9f15..16f74b38 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -13,6 +13,7 @@ import { Handlers } from "./handlers.ts"; import { Health } from "./health.ts"; import { Kv } from "./kv.ts"; import { Servers } from "./servers.ts"; +import { Sync } from "./sync.ts"; import { Widgets } from "./widgets.ts"; /** @@ -89,7 +90,11 @@ export const layer = Layer.mergeAll( exitWithParent, ).pipe( // The servers' status is checked from launch, so the first page already has it. - Layer.provide(Health.layer.pipe(Layer.provideMerge(Layer.mergeAll(Event.layer, Kv.layer, Codework.layer)))), + Layer.provide( + Layer.mergeAll(Health.layer, Sync.layer).pipe( + Layer.provideMerge(Layer.mergeAll(Event.layer, Kv.layer, Codework.layer)), + ), + ), // The app's servers decide which one its connection goes to, and the handlers manage them too. Layer.provideMerge(Servers.layer), Layer.provideMerge(announceReady), diff --git a/apps/server/src/servers.ts b/apps/server/src/servers.ts index 9178aa0f..a650c265 100644 --- a/apps/server/src/servers.ts +++ b/apps/server/src/servers.ts @@ -18,9 +18,9 @@ export type StoredEnv = typeof StoredEnv.Type; export const splitArgs = (input: string) => [...input.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)].map((match) => match[1] ?? match[2] ?? match[3] ?? ""); -/** A server as stored. A `null` executable or home is the one found. An executable is a path, or a +/** A local server as stored. A `null` executable or home is the one found. An executable is a path, or a * command whose first word is one (`node --conditions=development …/index.ts`, running from source). */ -export interface Row { +export interface LocalRow { readonly id: string; readonly kind: "local"; readonly name: string; @@ -31,6 +31,17 @@ export interface Row { readonly isDefault: boolean; } +/** A remote server as stored: where the app reaches it. */ +export interface RemoteRow { + readonly id: string; + readonly kind: "remote"; + readonly name: string; + readonly url: string; + readonly isDefault: boolean; +} + +export type Row = LocalRow | RemoteRow; + const decodeEnv = Schema.decodeUnknownEffect(Schema.fromJsonString(StoredEnv)); const fail = (message: string) => new ServersError({ message }); @@ -47,6 +58,14 @@ const chosenPath = (input: string | null) => return path.normalize(expanded); }); +/** A remote server's URL typed in Settings: CodeWork answers over a WebSocket. */ +const chosenUrl = (input: string) => { + const trimmed = input.trim(); + return URL.canParse(trimmed) && ["ws:", "wss:"].includes(new URL(trimmed).protocol) + ? Effect.succeed(trimmed) + : Effect.fail(fail("Use a WebSocket URL, such as wss://codework.example.com/rpc.")); +}; + const quoted = (word: string) => (/\s/.test(word) ? `"${word}"` : word); /** @@ -93,15 +112,18 @@ const nextEnv = (chosen: ReadonlyArray, saved: StoredEnv) => /** * The servers the app knows, and the one it connects to: the default, whose home's `service.json` - * names the server. The built-in local server is always there. + * names the server. The built-in local server is always there. The default is always a local one: + * the connection finds its server through a home. */ export class Service extends Context.Service< Service, { readonly list: Effect.Effect>; readonly get: (id: string) => Effect.Effect; - /** The home a server uses; `fallback` is the built-in server's when none is chosen. */ - readonly homeOf: (row: Row) => { + /** The local server `id`; a remote one is refused. */ + readonly getLocal: (id: string) => Effect.Effect; + /** The home a local server uses; `fallback` is the built-in server's when none is chosen. */ + readonly homeOf: (row: LocalRow) => { readonly path: string; readonly custom: boolean; readonly fallback: string | null; @@ -110,6 +132,8 @@ export class Service extends Context.Service< readonly defaultHome: Effect.Effect; /** The default server's home each time it changes, without the current one. */ readonly homeChanges: Stream.Stream; + /** Every server now, then again after each change to any of them. */ + readonly watch: Stream.Stream>; readonly add: (input: ServerAdd) => Effect.Effect; readonly update: (input: ServerUpdate) => Effect.Effect; readonly remove: (id: string) => Effect.Effect; @@ -124,18 +148,28 @@ export const layer = Layer.effect( const fs = yield* FileSystem.FileSystem; const fallback = yield* cliHome.pipe(Effect.orDie); - const list = sql & { kind: string; env: string; isDefault: number }>` - SELECT id, kind, name, executable, home, args, env, is_default FROM servers ORDER BY pos + const list = sql< + Omit & { + kind: string; + env: string; + url: string | null; + isDefault: number; + } + >` + SELECT id, kind, name, executable, home, args, env, url, is_default FROM machine ORDER BY pos `.pipe( Effect.flatMap((rows) => - Effect.forEach(rows, (row) => - decodeEnv(row.env).pipe( - Effect.map((env): Row => ({ ...row, kind: "local", env, isDefault: row.isDefault === 1 })), - ), + Effect.forEach(rows, ({ kind, url, env, isDefault, ...row }) => + kind === "remote" && url !== null + ? Effect.succeed({ id: row.id, kind, name: row.name, url, isDefault: isDefault === 1 }) + : decodeEnv(env).pipe( + Effect.map((env): Row => ({ ...row, kind: "local", env, isDefault: isDefault === 1 })), + ), ), ), Effect.orDie, ); + const locals = list.pipe(Effect.map((rows) => rows.filter((row): row is LocalRow => row.kind === "local"))); const get = (id: string) => list.pipe( @@ -143,26 +177,37 @@ export const layer = Layer.effect( Effect.flatMap((row) => (row === undefined ? Effect.fail(fail(`No server ${id}`)) : Effect.succeed(row))), ); - const homeOf = (row: Row) => ({ + const getLocal = (id: string) => + get(id).pipe( + Effect.flatMap((row) => + row.kind === "local" ? Effect.succeed(row) : Effect.fail(fail(`${row.name} is a remote server.`)), + ), + ); + + const homeOf = (row: LocalRow) => ({ path: row.home ?? fallback, custom: row.home !== null, fallback: row.id === LOCAL_SERVER ? fallback : null, }); - const defaultHome = list.pipe(Effect.map((rows) => homeOf(rows.find((row) => row.isDefault) ?? rows[0]!).path)); + const defaultHome = locals.pipe(Effect.map((rows) => homeOf(rows.find((row) => row.isDefault) ?? rows[0]!).path)); const current = yield* SubscriptionRef.make(yield* defaultHome); - // Told after every write, so a new default or a new home for it reaches the connection. + const machines = yield* SubscriptionRef.make(yield* list); + // Told after every write, so a new default or a new home for it reaches the connection, and + // every change of a server reaches the sync. const announce = defaultHome.pipe( Effect.flatMap((home) => SubscriptionRef.get(current).pipe( Effect.flatMap((was) => (was === home ? Effect.void : SubscriptionRef.set(current, home))), ), ), + Effect.andThen(list), + Effect.flatMap((rows) => SubscriptionRef.set(machines, rows)), ); // A home names one server, so no two servers share one. const ensureOwnHome = (id: string | null, home: string) => - list.pipe( + locals.pipe( Effect.flatMap((rows) => { const taken = rows.find((row) => row.id !== id && homeOf(row).path === home); return taken === undefined @@ -182,32 +227,36 @@ export const layer = Layer.effect( return Service.of({ list, get, + getLocal, homeOf, defaultHome, homeChanges: SubscriptionRef.changes(current).pipe(Stream.drop(1)), + watch: SubscriptionRef.changes(machines), add: Effect.fn("Servers.add")(function* (input) { - const home = yield* chosenPath(input.home); - if (home === null) return yield* fail("A new server needs a home of its own."); const id = yield* Effect.sync(() => crypto.randomUUID().slice(0, 8)); + const home = input.kind === "local" ? yield* chosenPath(input.home) : null; + if (input.kind === "local" && home === null) return yield* fail("A new server needs a home of its own."); + const route = + input.kind === "local" + ? { executable: yield* chosenCommand(fs, input.executable), home, args: input.args.trim() } + : { url: yield* chosenUrl(input.url) }; yield* transaction( Effect.gen(function* () { - yield* ensureOwnHome(null, home); - const [last] = yield* sql<{ pos: number }>`SELECT COALESCE(MAX(pos), 0) AS pos FROM servers`; - yield* sql`INSERT INTO servers ${sql.insert({ + if (home !== null) yield* ensureOwnHome(null, home); + const [last] = yield* sql<{ pos: number }>`SELECT COALESCE(MAX(pos), 0) AS pos FROM machine`; + yield* sql`INSERT INTO machine ${sql.insert({ id, - kind: "local", + kind: input.kind, name: yield* name(input.name), - executable: yield* chosenCommand(fs, input.executable), - home, - args: input.args.trim(), - env: "[]", - isDefault: 0, + ...route, + namespace: "codework", pos: (last?.pos ?? 0) + 1, - created: yield* Clock.currentTimeMillis, + createdAt: yield* Clock.currentTimeMillis, })}`; }), ); + yield* announce; return id; }), @@ -217,35 +266,44 @@ export const layer = Layer.effect( const row = yield* get(id); const fields: Record = {}; if (patch.name !== undefined) fields.name = yield* name(patch.name); - if (patch.executable !== undefined) fields.executable = yield* chosenCommand(fs, patch.executable); - if (patch.args !== undefined) fields.args = patch.args.trim(); - if (patch.env !== undefined) fields.env = JSON.stringify(yield* nextEnv(patch.env, row.env)); - if (patch.home !== undefined) { - const home = yield* chosenPath(patch.home); - if (home === null && row.id !== LOCAL_SERVER) - return yield* fail("An added server keeps a home of its own."); - yield* ensureOwnHome(id, home ?? fallback); - fields.home = home; + if (row.kind === "remote") { + if ([patch.executable, patch.home, patch.args, patch.env].some((value) => value !== undefined)) + return yield* fail("A remote server has no executable, home, arguments or variables."); + if (patch.url !== undefined) fields.url = yield* chosenUrl(patch.url); + } else { + if (patch.url !== undefined) return yield* fail("Only a remote server has a URL."); + if (patch.executable !== undefined) fields.executable = yield* chosenCommand(fs, patch.executable); + if (patch.args !== undefined) fields.args = patch.args.trim(); + if (patch.env !== undefined) fields.env = JSON.stringify(yield* nextEnv(patch.env, row.env)); + if (patch.home !== undefined) { + const home = yield* chosenPath(patch.home); + if (home === null && row.id !== LOCAL_SERVER) + return yield* fail("An added server keeps a home of its own."); + yield* ensureOwnHome(id, home ?? fallback); + fields.home = home; + } } if (Object.keys(fields).length > 0) - yield* sql`UPDATE servers SET ${sql.update(fields)} WHERE id = ${id}`; + yield* sql`UPDATE machine SET ${sql.update(fields)} WHERE id = ${id}`; }), ); yield* announce; }), + // Everything synced from the server goes with it: its rows cascade. remove: Effect.fn("Servers.remove")(function* (id) { const row = yield* get(id); if (row.id === LOCAL_SERVER) return yield* fail("The built-in local server cannot be removed."); if (row.isDefault) return yield* fail("Make another server the default before removing this one."); - yield* sql`DELETE FROM servers WHERE id = ${id}`.pipe(Effect.orDie); + yield* sql`DELETE FROM machine WHERE id = ${id}`.pipe(Effect.orDie); + yield* announce; }), setDefault: Effect.fn("Servers.setDefault")(function* (id) { yield* transaction( Effect.gen(function* () { - yield* get(id); - yield* sql`UPDATE servers SET is_default = CASE WHEN id = ${id} THEN 1 ELSE 0 END`; + yield* getLocal(id); + yield* sql`UPDATE machine SET is_default = CASE WHEN id = ${id} THEN 1 ELSE 0 END`; }), ); yield* announce; diff --git a/apps/server/src/sync.ts b/apps/server/src/sync.ts new file mode 100644 index 00000000..a4f3134f --- /dev/null +++ b/apps/server/src/sync.ts @@ -0,0 +1,441 @@ +import { Codework as Client } from "@codeworksh/widget/backend"; +import type { Codework as Wire } from "@codeworksh/widget/codework"; +import { Clock, Context, Deferred, Effect, FiberMap, FileSystem, Layer, Option, Stream, SubscriptionRef } from "effect"; +import { SqlClient } from "effect/sql"; +import type { Publish } from "@codeworksh/widget/event"; +import { CodeworkUnavailable, SyncError, type SyncThread, syncTopics } from "@codeworksh/contract"; +import { Codework } from "./codework.ts"; +import { Event } from "./event.ts"; +import { Mirror } from "./mirror.ts"; +import { Servers } from "./servers.ts"; + +type Connection = typeof Client.Service; +type IndexSynced = Extract; +type IndexRow = Exclude; +type ThreadSynced = Extract; +type ThreadRow = Exclude; + +/** A machine whose index is current: its connection and the threads whose detail it syncs. */ +interface Live { + readonly client: Connection; + readonly threads: FiberMap.FiberMap; +} + +export interface Interface { + /** Syncs a thread's detail until every open is closed; done once it is current, or at once while its machine is away. */ + readonly open: (ref: SyncThread) => Effect.Effect; + readonly close: (ref: SyncThread) => Effect.Effect; + /** Stores up to `limit` entries below an opened thread's floor, read from its machine. */ + readonly page: ( + ref: SyncThread & { readonly limit: number }, + ) => Effect.Effect<{ readonly entries: number; readonly floor: number | null }, CodeworkUnavailable | SyncError>; + /** The connection to a machine's server, for commands to its sessions. */ + readonly client: (machineId: string) => Effect.Effect; + /** The machine a synced session is on. */ + readonly machineOf: (sessionId: string) => Effect.Effect>; + /** + * Done once the machine a session is on (the default one for a session not synced) has applied + * its index up to the head its store has now; at once while that machine is away. + */ + readonly settle: (sessionId: string) => Effect.Effect; +} + +/** + * Keeps app.db a copy of every machine's CodeWork: each machine over its own connection, its index + * through `index.sync` from `machine.cursor`, and the detail of opened threads through `thread.sync` + * from `thread.cursor`. A new or restored store is read again from zero. + */ +export class Service extends Context.Service()("@codeworksh/server/sync/Service") {} + +const unavailable = (machineId: string) => + new CodeworkUnavailable({ message: `The CodeWork server of machine ${machineId} is not connected.` }); + +/** The route a connection is made from: a change reconnects the machine. */ +const routeOf = (row: Servers.Row) => + row.kind === "remote" ? `remote ${row.url}` : `local ${row.home ?? ""} ${row.executable ?? ""}`; + +/** + * Runs `apply` on each batch up to its `Synced`, in order; `onItem` sees every other item as it + * arrives and says whether the batch keeps it. + */ +const batches = ( + stream: Stream.Stream, + isSynced: (item: R | S) => item is S, + onItem: (item: R) => Effect.Effect, + apply: (rows: ReadonlyArray, synced: S) => Effect.Effect, +) => + Effect.suspend(() => { + let rows: Array = []; + return stream.pipe( + Stream.runForEach((item) => { + if (!isSynced(item)) + return onItem(item).pipe( + Effect.map((keep) => { + if (keep) rows.push(item); + }), + ); + const batch = rows; + rows = []; + return apply(batch, item); + }), + ); + }); + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const events = yield* Event.Service; + const servers = yield* Servers.Service; + const fs = yield* FileSystem.FileSystem; + + const clients = new Map(); + const lives = new Map(); + // Open counts by thread: windows open and close a thread independently. + const opened = new Map(); + // Done once a thread's stream applied its first `Synced`; present while its stream runs. + const readiness = new Map>(); + // The open drafts of opened threads, by thread, entry and part: their text as last known. + const drafts = new Map>>(); + // Each live machine's index cursor as last applied, including batches that carried no rows. + const applied = yield* SubscriptionRef.make>(new Map()); + + // Each batch in one transaction, foreign keys checked at its commit: rows arrive in revision + // order, so a session can come before the latest version of its space. + const transact = ( + body: (append: (input: Publish) => Effect.Effect) => Effect.Effect, + ) => + events.transact((append) => + sql`PRAGMA defer_foreign_keys = ON`.pipe( + Effect.andThen(body(append)), + Effect.provideService(SqlClient.SqlClient, sql), + ), + ); + + const machineOf = (sessionId: string) => + sql<{ machineId: string }>`SELECT machine_id FROM session WHERE id = ${sessionId}`.pipe( + Effect.map((rows) => Option.fromNullishOr(rows[0]?.machineId)), + Effect.orDie, + ); + + const draftsOf = (threadId: string) => { + const found = drafts.get(threadId); + if (found !== undefined) return found; + const made = new Map>(); + drafts.set(threadId, made); + return made; + }; + + // A checkpoint replaces the held text only when it is longer: the held text is the checkpoint + // plus deltas that continued it exactly, so the longer of the two is the newer. + const noteEntry = (threadId: string, entry: Wire.HistoryEntry) => + Effect.sync(() => { + const held = draftsOf(threadId); + if (entry.state !== "draft") return held.delete(entry.id); + const parts = held.get(entry.id) ?? new Map(); + entry.parts.forEach((part, index) => { + const text = part.type === "text" ? part.text : part.type === "thinking" ? part.thinking : undefined; + if (text !== undefined && text.length >= (parts.get(index)?.length ?? 0)) parts.set(index, text); + }); + held.set(entry.id, parts); + }); + + /** Folds a delta into its draft when it continues the held text exactly, and forwards it; drops it otherwise. */ + const fold = (threadId: string, delta: Wire.LiveDelta) => + Effect.suspend(() => { + const held = draftsOf(threadId); + const parts = held.get(delta.entryId) ?? (delta.offset === 0 ? new Map() : undefined); + const text = parts?.get(delta.partIndex) ?? ""; + if (parts === undefined || text.length !== delta.offset) return Effect.void; + parts.set(delta.partIndex, text + delta.delta); + held.set(delta.entryId, parts); + return events.publish({ + topic: syncTopics.delta(threadId), + source: "sync", + payload: delta, + ephemeral: true, + }); + }); + + const syncThread = (machineId: string, live: Live, sessionId: string, threadId: string) => + Effect.gen(function* () { + const [row] = yield* sql<{ cursor: number | null }>`SELECT cursor FROM thread WHERE id = ${threadId}`; + if (row === undefined) return; + yield* batches( + live.client("thread.sync", { + sessionId, + threadId, + ...(row.cursor === null ? {} : { after: row.cursor }), + }), + (item) => item._tag === "Synced", + (item) => + item._tag === "Delta" + ? fold(threadId, item.delta).pipe(Effect.as(false)) + : item._tag === "Entry" + ? noteEntry(threadId, item.entry).pipe(Effect.as(true)) + : Effect.succeed(true), + (rows, synced) => + transact((append) => + Effect.gen(function* () { + const changes = yield* Mirror.thread(threadId, rows); + yield* sql` + UPDATE thread SET cursor = ${synced.rev}, floor = COALESCE(${synced.floor ?? null}, floor) + WHERE id = ${threadId} + `; + if (rows.length > 0 || synced.floor !== undefined) + yield* append({ + topic: syncTopics.changed, + source: "sync", + payload: changes.notice({ machineId, threadId, full: false, items: rows.length }), + }); + }), + ).pipe(Effect.andThen(Effect.suspend(() => readyOf(threadId)))), + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Thread sync stopped", cause).pipe(Effect.annotateLogs({ machineId, threadId })), + ), + ); + + const readyOf = (threadId: string) => { + const ready = readiness.get(threadId); + return ready === undefined ? Effect.void : Deferred.succeed(ready, undefined).pipe(Effect.asVoid); + }; + + // Starts the opened threads of a machine whose detail is not syncing yet; known by their session. + const startOpened = (machineId: string, live: Live) => + Effect.gen(function* () { + const ids = [...opened.keys()]; + if (ids.length === 0) return; + const rows = yield* sql<{ id: string; sessionId: string }>` + SELECT thread.id, thread.session_id FROM thread JOIN session ON session.id = thread.session_id + WHERE session.machine_id = ${machineId} AND thread.deleted_at IS NULL AND thread.id IN ${sql.in(ids)} + `.pipe(Effect.orDie); + for (const { id, sessionId } of rows) { + const ready = yield* Deferred.make(); + // Checked and claimed in one step, so two starts of a thread cannot both run it. + const fresh = yield* Effect.sync(() => { + if (readiness.has(id)) return false; + readiness.set(id, ready); + return true; + }); + if (!fresh) continue; + yield* FiberMap.run( + live.threads, + id, + syncThread(machineId, live, sessionId, id).pipe( + Effect.ensuring( + Deferred.succeed(ready, undefined).pipe( + Effect.andThen( + Effect.sync(() => { + if (readiness.get(id) === ready) readiness.delete(id); + }), + ), + ), + ), + ), + { onlyIfMissing: true }, + ); + } + }); + + /** One connection's sync: decides between resume and full sync, then follows the index. */ + const connected = (machineId: string, client: Connection) => + Effect.gen(function* () { + const info = yield* client("server.info", {}); + const [machine] = yield* sql<{ storeId: string | null; cursor: number; namespace: string }>` + SELECT store_id, cursor, namespace FROM machine WHERE id = ${machineId} + `; + if (machine === undefined) return yield* Effect.never; + const [owner] = yield* sql<{ id: string }>` + SELECT id FROM machine WHERE store_id = ${info.storeId} AND id <> ${machineId} + `; + if (owner !== undefined) { + yield* Effect.logWarning( + `Not syncing machine ${machineId}: its store is already synced as machine ${owner.id}`, + ); + return yield* Effect.never; + } + let full = info.storeId !== machine.storeId || info.rev < machine.cursor; + const live: Live = { client, threads: yield* FiberMap.make() }; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (lives.get(machineId) === live) lives.delete(machineId); + }), + ); + yield* batches( + client("index.sync", { after: full ? 0 : machine.cursor }), + (item) => item._tag === "Synced", + () => Effect.succeed(true), + (rows, synced) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const wasFull = full; + yield* transact((append) => + Effect.gen(function* () { + const changes = yield* Mirror.index( + { id: machineId, namespace: machine.namespace }, + rows, + now, + append, + ); + if (wasFull) yield* Mirror.unseen(machineId, changes, now); + yield* sql` + UPDATE machine SET cursor = ${synced.rev}, store_id = ${synced.storeId} WHERE id = ${machineId} + `; + if (rows.length > 0 || wasFull) + yield* append({ + topic: syncTopics.changed, + source: "sync", + payload: changes.notice({ machineId, full: wasFull, items: rows.length }), + }); + }), + ); + full = false; + lives.set(machineId, live); + yield* SubscriptionRef.update(applied, (cursors) => new Map(cursors).set(machineId, synced.rev)); + yield* startOpened(machineId, live); + }), + ); + return yield* Effect.fail("The index stream ended"); + }).pipe(Effect.scoped); + + /** A machine for as long as it is configured: connected, and reconnected after any drop. */ + const follow = (row: Servers.Row) => + Effect.gen(function* () { + const context = yield* Layer.build(Codework.machine(row)).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Servers.Service, servers), + ); + const client = Context.get(context, Client); + clients.set(row.id, client); + yield* Effect.addFinalizer(() => Effect.sync(() => clients.delete(row.id))); + yield* connected(row.id, client).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Machine sync stopped; reconnecting", cause).pipe( + Effect.annotateLogs({ machineId: row.id }), + ), + ), + Effect.andThen(Effect.sleep("1 second")), + Effect.forever, + ); + }).pipe(Effect.scoped); + + const machines = yield* FiberMap.make(); + const routes = new Map(); + yield* servers.watch.pipe( + Stream.runForEach((rows) => + Effect.gen(function* () { + for (const row of rows) { + const route = routeOf(row); + if (routes.get(row.id) === route) continue; + routes.set(row.id, route); + yield* FiberMap.run(machines, row.id, follow(row)); + } + for (const id of routes.keys()) { + if (rows.some((row) => row.id === id)) continue; + routes.delete(id); + yield* FiberMap.remove(machines, id); + } + }), + ), + Effect.forkScoped, + ); + + return Service.of({ + machineOf, + settle: Effect.fn("Sync.settle")(function* (sessionId) { + const known = yield* machineOf(sessionId); + const machineId = Option.isSome(known) + ? known.value + : (yield* sql<{ id: string }>`SELECT id FROM machine WHERE is_default = 1`.pipe(Effect.orDie))[0]?.id; + // Its connection, live or still on its first batch (the app just started); none once removed. + const client = + machineId === undefined ? undefined : (lives.get(machineId)?.client ?? clients.get(machineId)); + if (machineId === undefined || client === undefined) return; + // An unreachable machine fails this within the call's timeout. + const info = yield* Codework.call((client) => client("server.info", {})).pipe( + Effect.provideService(Client, client), + Effect.option, + ); + if (Option.isNone(info)) return; + const head = info.value.rev; + // Bounded: a machine that drops mid-wait sends no more batches. + yield* SubscriptionRef.changes(applied).pipe( + Stream.filter((cursors) => (cursors.get(machineId) ?? -1) >= head), + Stream.runHead, + Effect.timeoutOption("10 seconds"), + ); + }), + client: (machineId) => + Effect.suspend(() => { + const client = clients.get(machineId); + return client === undefined ? Effect.fail(unavailable(machineId)) : Effect.succeed(client); + }), + + open: Effect.fn("Sync.open")(function* ({ sessionId, threadId }) { + yield* Effect.sync(() => { + opened.set(threadId, (opened.get(threadId) ?? 0) + 1); + }); + const machineId = yield* machineOf(sessionId); + const live = Option.isSome(machineId) ? lives.get(machineId.value) : undefined; + if (Option.isNone(machineId) || live === undefined) return; + yield* startOpened(machineId.value, live); + const ready = readiness.get(threadId); + if (ready !== undefined) yield* Deferred.await(ready); + }), + + close: Effect.fn("Sync.close")(function* ({ threadId }) { + const left = (opened.get(threadId) ?? 0) - 1; + yield* Effect.sync(() => { + if (left > 0) opened.set(threadId, left); + else { + opened.delete(threadId); + drafts.delete(threadId); + } + }); + if (left > 0) return; + yield* Effect.forEach(lives.values(), (live) => FiberMap.remove(live.threads, threadId), { discard: true }); + }), + + page: Effect.fn("Sync.page")(function* ({ sessionId, threadId, limit }) { + const [row] = yield* sql<{ cursor: number | null; floor: number | null; machineId: string }>` + SELECT thread.cursor, thread.floor, session.machine_id FROM thread + JOIN session ON session.id = thread.session_id + WHERE thread.id = ${threadId} AND thread.session_id = ${sessionId} + `.pipe(Effect.orDie); + if (row === undefined || row.cursor === null) + return yield* new SyncError({ message: `The detail of thread ${threadId} is not synced.` }); + if (row.floor === null) return { entries: 0, floor: null }; + const live = lives.get(row.machineId); + if (live === undefined) return yield* unavailable(row.machineId); + const before = row.floor; + const { entries } = yield* Codework.call( + (client) => client("thread.page", { sessionId, threadId, before, limit }), + { timeout: "30 seconds" }, + ).pipe(Effect.provideService(Client, live.client)); + const floor = Math.min(before, ...entries.map((entry) => entry.seq)); + yield* transact((append) => + Effect.gen(function* () { + const changes = new Mirror.Changes(); + yield* Mirror.entries(entries, changes); + yield* sql`UPDATE thread SET floor = ${floor} WHERE id = ${threadId}`; + yield* append({ + topic: syncTopics.changed, + source: "sync", + payload: changes.notice({ + machineId: row.machineId, + threadId, + full: false, + items: entries.length, + }), + }); + }), + ).pipe(Effect.orDie); + return { entries: entries.length, floor }; + }), + }); + }), +); + +export * as Sync from "./sync.ts"; diff --git a/apps/server/src/tracking.ts b/apps/server/src/tracking.ts new file mode 100644 index 00000000..70c0ef1d --- /dev/null +++ b/apps/server/src/tracking.ts @@ -0,0 +1,279 @@ +import { Clock, Effect, Schema } from "effect"; +import { SqlClient, type SqlError } from "effect/sql"; +import type { Publish } from "@codeworksh/widget/event"; +import { + type Grouping, + Position, + Priority, + type SyncStatus, + type SyncTracking, + syncTopics, + type ThreadStatus, + TrackingError, +} from "@codeworksh/contract"; +import { Event } from "./event.ts"; +import { Ordering } from "./ordering.ts"; + +/* + * The user's edits of the desktop columns: status, priority, group, order and color. Each is written + * with its bus event in one transaction, so every window and widget follows it. + */ + +type Append = (input: Publish) => Effect.Effect; +type Fields = Omit, "_tag" | "threadId">; + +const SOURCE = "tracking"; +const HEX = /^#[0-9a-f]{6}$/i; +const isPriority = Schema.is(Priority); + +const fail = (message: string) => Effect.fail(new TrackingError({ message })); +const missing = (what: string, id: string) => fail(`No ${what} ${id}.`); + +const checkColor = (color: string | null | undefined) => + color === null || color === undefined || HEX.test(color) ? Effect.void : fail(`${color} is not a #rrggbb color.`); + +const checkName = (name: string) => (name.trim() === "" ? fail("A group needs a name.") : Effect.succeed(name.trim())); + +const tracking = (append: Append, payload: SyncTracking) => + append({ topic: syncTopics.tracking, source: SOURCE, payload }); + +/** Tracking RPC handlers; `apps/server` is the only writer of the desktop columns. */ +export const handlers = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const events = yield* Event.Service; + + const transact = ( + body: (append: Append) => Effect.Effect, + ) => + events + .transact((append) => body(append).pipe(Effect.provideService(SqlClient.SqlClient, sql))) + .pipe(Effect.catchTag("SqlError", Effect.die)); + + /** Writes `fields` on a thread and announces them; `guard` narrows which row may take them. */ + const updateThread = ( + threadId: string, + fields: Fields, + append: Append, + guard = sql.literal(""), + notFound = `No thread ${threadId}.`, + ) => + Effect.gen(function* () { + const rows = yield* sql`UPDATE thread SET ${sql.update(fields)} WHERE id = ${threadId} ${guard} RETURNING id`; + if (rows.length === 0) return yield* fail(notFound); + yield* tracking(append, { _tag: "Thread", threadId, ...fields }); + }); + + /** The `ordering` that puts a row of `table` at `to` in its organization's list. */ + const place = (table: Ordering.Table, id: string, to: Position) => + Effect.gen(function* () { + const [row] = yield* sql<{ + organizationId: string; + }>`SELECT organization_id FROM ${sql(table)} WHERE id = ${id}`; + if (row === undefined) return yield* missing(table, id); + if (typeof to === "string") return yield* Ordering.edge(table, row.organizationId, to); + const neighbours = yield* sql<{ id: string; ordering: number }>` + SELECT id, ordering FROM ${sql(table)} + WHERE organization_id = ${row.organizationId} AND id IN ${sql.in([to.after, to.before])} + `; + const after = neighbours.find((neighbour) => neighbour.id === to.after); + const before = neighbours.find((neighbour) => neighbour.id === to.before); + if (after === undefined) return yield* missing(`${table} in its organization`, to.after); + if (before === undefined) return yield* missing(`${table} in its organization`, to.before); + return Ordering.midpoint(after.ordering, before.ordering); + }); + + const groupRow = (rows: ReadonlyArray, id: string) => + rows[0] === undefined ? missing("group", id) : Effect.succeed(rows[0]); + + return { + "threads.setStatus": Effect.fn("threads.setStatus")(function* ({ + threadId, + status, + }: { + readonly threadId: string; + readonly status: ThreadStatus; + }) { + yield* transact((append) => + Effect.gen(function* () { + const [stored] = yield* sql<{ status: ThreadStatus }>`SELECT status FROM thread WHERE id = ${threadId}`; + if (stored === undefined) return yield* missing("thread", threadId); + if (stored.status === status) return; + const at = yield* Clock.currentTimeMillis; + yield* sql`UPDATE thread SET status = ${status}, status_at = ${at} WHERE id = ${threadId}`; + const payload: SyncStatus = { threadId, status, previous: stored.status, at }; + yield* append({ topic: syncTopics.status, source: SOURCE, payload }); + }), + ); + }), + + "threads.setPriority": Effect.fn("threads.setPriority")(function* ({ + threadId, + priority, + }: { + readonly threadId: string; + readonly priority: number; + }) { + if (!isPriority(priority)) return yield* fail(`${priority} is not a priority: 0 none, 1 urgent to 4 low.`); + yield* transact((append) => updateThread(threadId, { priority }, append)); + }), + + "threads.setColor": Effect.fn("threads.setColor")(function* ({ + threadId, + color, + }: { + readonly threadId: string; + readonly color: string | null; + }) { + yield* checkColor(color); + yield* transact((append) => updateThread(threadId, { color }, append)); + }), + + "threads.setGroup": Effect.fn("threads.setGroup")(function* ({ + threadId, + groupingId, + }: { + readonly threadId: string; + readonly groupingId: string | null; + }) { + // Only a group of the thread's own organization takes it. + const guard = + groupingId === null + ? sql.literal("") + : sql`AND EXISTS (SELECT 1 FROM grouping WHERE grouping.id = ${groupingId} AND grouping.organization_id = thread.organization_id)`; + const notFound = + groupingId === null + ? `No thread ${threadId}.` + : `No thread ${threadId} with a group ${groupingId} in its organization.`; + yield* transact((append) => updateThread(threadId, { groupingId }, append, guard, notFound)); + }), + + "threads.move": Effect.fn("threads.move")(function* ({ + threadId, + to, + }: { + readonly threadId: string; + readonly to: Position; + }) { + yield* transact((append) => + Effect.gen(function* () { + yield* updateThread(threadId, { ordering: yield* place("thread", threadId, to) }, append); + }), + ); + }), + + "sessions.setColor": Effect.fn("sessions.setColor")(function* ({ + sessionId, + color, + }: { + readonly sessionId: string; + readonly color: string | null; + }) { + yield* checkColor(color); + yield* transact((append) => + Effect.gen(function* () { + const rows = yield* sql`UPDATE session SET color = ${color} WHERE id = ${sessionId} RETURNING id`; + if (rows.length === 0) return yield* missing("session", sessionId); + yield* tracking(append, { _tag: "Session", sessionId, color }); + }), + ); + }), + + "groups.create": Effect.fn("groups.create")(function* ({ + organizationId, + name, + color = null, + }: { + readonly organizationId: string; + readonly name: string; + readonly color?: string | null; + }) { + const trimmed = yield* checkName(name); + yield* checkColor(color); + return yield* transact((append) => + Effect.gen(function* () { + const id = crypto.randomUUID(); + const now = yield* Clock.currentTimeMillis; + const ordering = yield* Ordering.edge("grouping", organizationId, "last"); + // Selected from the organization, so a missing one inserts nothing. + const [group] = yield* sql` + INSERT INTO grouping (id, organization_id, name, color, ordering, created_at, updated_at) + SELECT ${id}, id, ${trimmed}, ${color}, ${ordering}, ${now}, ${now} FROM organization WHERE id = ${organizationId} + RETURNING id, organization_id, name, color, ordering + `; + if (group === undefined) return yield* missing("organization", organizationId); + yield* tracking(append, { _tag: "Group", ...group }); + return group; + }), + ); + }), + + "groups.update": Effect.fn("groups.update")(function* ({ + groupingId, + name, + color, + }: { + readonly groupingId: string; + readonly name?: string; + readonly color?: string | null; + }) { + const fields = { + ...(name === undefined ? {} : { name: yield* checkName(name) }), + ...(color === undefined ? {} : { color: yield* checkColor(color).pipe(Effect.as(color)) }), + }; + yield* transact((append) => + Effect.gen(function* () { + const updatedAt = yield* Clock.currentTimeMillis; + const group = yield* groupRow( + yield* sql` + UPDATE grouping SET ${sql.update({ ...fields, updatedAt })} WHERE id = ${groupingId} + RETURNING id, organization_id, name, color, ordering + `, + groupingId, + ); + yield* tracking(append, { _tag: "Group", ...group }); + }), + ); + }), + + "groups.move": Effect.fn("groups.move")(function* ({ + groupingId, + to, + }: { + readonly groupingId: string; + readonly to: Position; + }) { + yield* transact((append) => + Effect.gen(function* () { + const ordering = yield* place("grouping", groupingId, to); + const updatedAt = yield* Clock.currentTimeMillis; + const group = yield* groupRow( + yield* sql` + UPDATE grouping SET ordering = ${ordering}, updated_at = ${updatedAt} WHERE id = ${groupingId} + RETURNING id, organization_id, name, color, ordering + `, + groupingId, + ); + yield* tracking(append, { _tag: "Group", ...group }); + }), + ); + }), + + "groups.delete": Effect.fn("groups.delete")(function* ({ groupingId }: { readonly groupingId: string }) { + yield* transact((append) => + Effect.gen(function* () { + // Named before the delete, whose foreign key ungroups them. + const threads = yield* sql<{ id: string }>`SELECT id FROM thread WHERE grouping_id = ${groupingId}`; + const rows = yield* sql`DELETE FROM grouping WHERE id = ${groupingId} RETURNING id`; + if (rows.length === 0) return yield* missing("group", groupingId); + yield* tracking(append, { + _tag: "GroupDeleted", + groupingId, + threads: threads.map((thread) => thread.id), + }); + }), + ); + }), + }; +}); + +export * as Tracking from "./tracking.ts"; diff --git a/apps/server/src/widgets.ts b/apps/server/src/widgets.ts index d3f96c65..f1c1ea03 100644 --- a/apps/server/src/widgets.ts +++ b/apps/server/src/widgets.ts @@ -1,8 +1,11 @@ import { backends } from "@codeworksh/web/backends"; -import { Bus, Codework, Kv as WidgetKv } from "@codeworksh/widget/backend"; -import { Effect, Layer } from "effect"; +import { Bus, Codework, Kv as WidgetKv, Sync as WidgetSync } from "@codeworksh/widget/backend"; +import { Effect, Layer, Option } from "effect"; +import { SqlClient } from "effect/sql"; import { Event } from "./event.ts"; import { Kv } from "./kv.ts"; +import { Reads } from "./reads.ts"; +import { Sync } from "./sync.ts"; /** A topic outside the publisher's package is a bug in the widget, not a runtime condition. */ const owned = (pkg: string, topic: string) => @@ -14,6 +17,28 @@ export const layer = Layer.effectDiscard( const events = yield* Event.Service; const kv = yield* Kv.Service; const codework = yield* Codework; + const sync = yield* Sync.Service; + const sql = yield* SqlClient.SqlClient; + const read = (effect: Effect.Effect) => + Effect.provideService(effect, SqlClient.SqlClient, sql); + const synced = WidgetSync.of({ + open: sync.open, + close: sync.close, + page: sync.page, + line: (input) => read(Reads.line(input)), + entries: (ids) => read(Reads.entries(ids)), + settle: sync.settle, + // A session the index has not brought yet was just created, on the default machine. + machine: (sessionId) => + sync.machineOf(sessionId).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeed(codework), + onSome: (machineId) => sync.client(machineId), + }), + ), + ), + }); // Third-party packages will join the built-in backends at load time. yield* Effect.forEach(backends, (backend) => backend.run.pipe( @@ -40,6 +65,7 @@ export const layer = Layer.effectDiscard( }), ), Effect.provideService(Codework, codework), + Effect.provideService(WidgetSync, synced), Effect.forkScoped, ), ); diff --git a/apps/server/test/__artifacts__/dashboard.grouping.json b/apps/server/test/__artifacts__/dashboard.grouping.json new file mode 100644 index 00000000..10ce4aef --- /dev/null +++ b/apps/server/test/__artifacts__/dashboard.grouping.json @@ -0,0 +1,123 @@ +{ + "lists": [ + { + "_tag": "Grouping", + "groupingId": "", + "threads": [ + { + "id": "", + "sessionId": "", + "originKind": "main", + "title": "Thread B", + "project": { + "id": "", + "name": "Board" + }, + "runState": "idle", + "status": "settled", + "priority": 0, + "groupingId": "", + "ordering": -1, + "color": "#0000cc", + "updatedAt": "", + "ordering": -4 + } + }, + { + "table": "thread", + "name": "B", + "to": { + "after": "", + "before": "" + }, + "list": [ + { + "name": "A", + "ordering": -4 + }, + { + "name": "B", + "ordering": -3.5 + }, + { + "name": "D", + "ordering": -3 + }, + { + "name": "C", + "ordering": -2 + } + ], + "event": { + "_tag": "Thread", + "threadId": "", + "ordering": -3.5 + } + }, + { + "table": "thread", + "name": "D", + "to": "last", + "list": [ + { + "name": "A", + "ordering": -4 + }, + { + "name": "B", + "ordering": -3.5 + }, + { + "name": "C", + "ordering": -2 + }, + { + "name": "D", + "ordering": -1 + } + ], + "event": { + "_tag": "Thread", + "threadId": "", + "ordering": -1 + } + }, + { + "table": "grouping", + "name": "G3", + "to": "first", + "list": [ + { + "name": "G3", + "ordering": -1 + }, + { + "name": "G1", + "ordering": 0 + }, + { + "name": "G2", + "ordering": 1 + } + ], + "event": { + "_tag": "Group", + "id": "", + "organizationId": "personal", + "name": "G3", + "color": null, + "ordering": -1 + } + }, + { + "table": "grouping", + "name": "G2", + "to": { + "after": "", + "before": "" + }, + "list": [ + { + "name": "G3", + "ordering": -1 + }, + { + "name": "G2", + "ordering": -0.5 + }, + { + "name": "G1", + "ordering": 0 + } + ], + "event": { + "_tag": "Group", + "id": "", + "organizationId": "personal", + "name": "G2", + "color": null, + "ordering": -0.5 + } + }, + { + "table": "grouping", + "name": "G3", + "to": "last", + "list": [ + { + "name": "G2", + "ordering": -0.5 + }, + { + "name": "G1", + "ordering": 0 + }, + { + "name": "G3", + "ordering": 1 + } + ], + "event": { + "_tag": "Group", + "id": "", + "organizationId": "personal", + "name": "G3", + "color": null, + "ordering": 1 + } + } + ] +} diff --git a/apps/server/test/__artifacts__/tracking.rejected.json b/apps/server/test/__artifacts__/tracking.rejected.json new file mode 100644 index 00000000..6d56853c --- /dev/null +++ b/apps/server/test/__artifacts__/tracking.rejected.json @@ -0,0 +1,39 @@ +{ + "rejected": { + "priority 5": { + "_tag": "Server.TrackingError", + "message": "5 is not a priority: 0 none, 1 urgent to 4 low." + }, + "priority -1": { + "_tag": "Server.TrackingError", + "message": "-1 is not a priority: 0 none, 1 urgent to 4 low." + }, + "thread color orange": { + "_tag": "Server.TrackingError", + "message": "orange is not a #rrggbb color." + }, + "session color #12345": { + "_tag": "Server.TrackingError", + "message": "#12345 is not a #rrggbb color." + }, + "group color #ggg000": { + "_tag": "Server.TrackingError", + "message": "#ggg000 is not a #rrggbb color." + }, + "missing thread": { + "_tag": "Server.TrackingError", + "message": "No thread thr_missing." + }, + "missing group": { + "_tag": "Server.TrackingError", + "message": "No thread with a group missing in its organization." + } + }, + "thread": { + "status": "cancelled", + "priority": 1, + "grouping_id": null, + "ordering": 0, + "color": null + } +} diff --git a/apps/server/test/__artifacts__/tracking.status.json b/apps/server/test/__artifacts__/tracking.status.json new file mode 100644 index 00000000..a18fcc8a --- /dev/null +++ b/apps/server/test/__artifacts__/tracking.status.json @@ -0,0 +1,56 @@ +{ + "steps": [ + { + "step": "first seen", + "status": "settled", + "thinking_level": null + }, + { + "step": "run succeeded", + "status": "attention", + "thinking_level": null + }, + { + "step": "user settled", + "status": "settled", + "thinking_level": null + }, + { + "step": "row without a run change", + "status": "settled", + "thinking_level": "high" + }, + { + "step": "next run succeeded", + "status": "attention", + "thinking_level": "high" + } + ], + "moves": [ + { + "source": "sync", + "previous": "settled", + "status": "working" + }, + { + "source": "sync", + "previous": "working", + "status": "attention" + }, + { + "source": "tracking", + "previous": "attention", + "status": "settled" + }, + { + "source": "sync", + "previous": "settled", + "status": "working" + }, + { + "source": "sync", + "previous": "working", + "status": "attention" + } + ] +} diff --git a/apps/server/test/dashboard.test.ts b/apps/server/test/dashboard.test.ts new file mode 100644 index 00000000..0248c83b --- /dev/null +++ b/apps/server/test/dashboard.test.ts @@ -0,0 +1,271 @@ +import { Effect, Stream } from "effect"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract, type Dashboard, type DashboardBy } from "@codeworksh/contract"; +import { + addMachine, + app, + type AppRpc, + cleanup, + codework, + freePort, + normalizer, + type Row, + query, + snapshot as match, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`dashboard.${name}`, artifact); + +const read = (rpc: AppRpc, by: DashboardBy) => + rpc["dashboard.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, by }); + +/** Each list as its key and its threads' names, in the order read: what the dashboard draws. */ +const names = (dashboard: Dashboard, label: (id: string) => string) => + dashboard.lists.map((list) => ({ + list: list._tag === "Status" ? list.status : list._tag === "Priority" ? list.priority : list.groupingId, + names: list.threads.map((thread) => label(thread.id)), + })); + +/** Times as `( + read: () => A | undefined | Promise, + what: string, + timeout = 20_000, +) => { + const until = Date.now() + timeout; + while (Date.now() < until) { + const value = await read(); + if (value !== undefined) return value; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`timed out waiting for ${what}`); +}; + +/** A CodeWork server of the fixture in `home` on `port`, with its scripted model. */ +export const codework = async (home: string, port: number) => { + const proc = spawn(process.execPath, ["--conditions=development", fixture, home, String(port)], { + stdio: ["pipe", "pipe", "inherit"], + }); + processes.add(proc); + await ready(proc); + const url = `ws://127.0.0.1:${port}/rpc/v1`; + return { + url, + proc, + database: join(home, "data", "codework.db"), + /** The oldest held reply takes its next step. */ + release: () => proc.stdin!.write("release\n"), + run: (use: (rpc: Client.Client) => Effect.Effect) => + Effect.gen(function* () { + return yield* use(yield* Client.make); + }).pipe(Effect.scoped, Effect.provide(Client.layer(url)), Effect.runPromise), + }; +}; +export type Codework = Awaited>; + +const appClient = RpcClient.make(Contract.Api); +export type AppRpc = Effect.Success; + +/** The app server with its database in `home`; its own CLI home names no CodeWork server. */ +export const app = async (home: string) => { + const proc = spawn(process.execPath, [appEntry], { + env: { ...process.env, CODEWORK_DATABASE: join(home, "app.db"), CODEWORK_HOME_DIR: join(home, "cli") }, + stdio: ["ignore", "pipe", "inherit"], + }); + processes.add(proc); + const { port } = await ready(proc); + const protocol = RpcClient.layerProtocolSocket().pipe( + Layer.provide(Socket.layerWebSocket(`ws://127.0.0.1:${port}/rpc`)), + Layer.provide(Socket.layerWebSocketConstructorGlobal), + Layer.provide(RpcSerialization.layerNdjson), + ); + return { + proc, + run: (use: (rpc: AppRpc) => Effect.Effect) => + Effect.gen(function* () { + return yield* use(yield* appClient); + }).pipe(Effect.scoped, Effect.provide(protocol), Effect.runPromise), + /** Runs `use` in the background on a connection of its own until `stop` is called. */ + background: (use: (rpc: AppRpc) => Effect.Effect) => { + const controller = new AbortController(); + const done = Effect.gen(function* () { + return yield* use(yield* appClient); + }).pipe(Effect.scoped, Effect.provide(protocol), (effect) => + Effect.runPromise(effect, { signal: controller.signal }).catch(() => undefined), + ); + return { stop: () => controller.abort(), done }; + }, + }; +}; + +/** Reads app.db while the app runs; each read opens the file afresh. */ +export const query = (home: string, statement: string, ...params: Array): Array => { + const db = new DatabaseSync(join(home, "app.db")); + try { + return db.prepare(statement).all(...params) as Array; + } finally { + db.close(); + } +}; +export const write = (home: string, statement: string, ...params: Array) => { + const db = new DatabaseSync(join(home, "app.db")); + try { + db.prepare(statement).run(...params); + } finally { + db.close(); + } +}; + +export const machineRow = (home: string, id: string) => + query(home, "SELECT store_id, cursor FROM machine WHERE id = ?", id)[0] as { + store_id: string | null; + cursor: number; + }; + +/** Adds the CodeWork server as a remote machine, then waits until the app holds its head. */ +export const addMachine = async (home: string, server: Codework, run: Awaited>["run"]) => { + const id = await run((rpc) => rpc["servers.add"]({ kind: "remote", name: "Scripted", url: server.url })); + await synced(home, id, server); + return id; +}; + +/** Waits until the app's cursor for `machineId` reached the server's head. */ +export const synced = async (home: string, machineId: string, server: Codework) => { + const { storeId, rev } = await server.run((rpc) => rpc["server.info"]({})); + await waitFor(() => { + const row = machineRow(home, machineId); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, `machine ${machineId} at rev ${rev}`); + return { storeId, rev }; +}; + +/** The payloads of the sync's bus events after `after` (an event seq). */ +export const notices = (home: string, topic: string, after = 0) => + query(home, "SELECT payload FROM event WHERE topic = ? AND seq > ? ORDER BY seq", topic, after).map( + (row) => JSON.parse(row.payload as string) as Row, + ); +export const lastSeq = (home: string) => + (query(home, "SELECT COALESCE(MAX(seq), 0) AS seq FROM event")[0]!.seq as number) ?? 0; + +// Ids by first appearance, seqs and revisions by rank, times and temp paths elided: what the +// artifact keeps is the shape and the relations, which are the contract. +const ID_COLUMNS = new Set([ + "id", + "machine_id", + "store_id", + "project_id", + "space_id", + "session_id", + "thread_id", + "entry_id", + "origin_thread_id", + "origin_entry_id", + "placement_space_id", + "machineId", + "threadId", + "entryId", + "sessionId", + "grouping_id", + "groupingId", + "slug", +]); +const ID_LISTS = new Set(["project", "space", "session", "thread", "entry", "inbox", "removed", "entries", "threads"]); +const SEQ_KEYS = new Set(["seq", "floor", "created_seq", "origin_committed_seq", "admitted_seq"]); +const REV_KEYS = new Set(["cursor", "rev"]); + +export const normalizer = (homes: ReadonlyArray, fixed: ReadonlyArray = []) => { + const labels = new Map(fixed); + const counts = new Map(); + const label = (value: string) => { + const known = labels.get(value); + if (known !== undefined) return known; + const kind = /^([a-z]+)_/.exec(value)?.[1] ?? "id"; + const count = (counts.get(kind) ?? 0) + 1; + counts.set(kind, count); + const made = `<${kind} ${count}>`; + labels.set(value, made); + return made; + }; + return (artifact: unknown) => { + const numbers = (keys: ReadonlySet) => { + const found = new Set(); + const walk = (value: unknown, key?: string) => { + if (typeof value === "number" && key !== undefined && keys.has(key)) found.add(value); + else if (Array.isArray(value)) value.forEach((item) => walk(item)); + else if (typeof value === "object" && value !== null) + for (const [k, v] of Object.entries(value)) walk(v, k); + }; + walk(artifact); + return [...found].sort((a, b) => a - b); + }; + const seqs = numbers(SEQ_KEYS); + const revs = numbers(REV_KEYS); + const walk = (value: unknown, key?: string): unknown => { + if (typeof value === "string") { + if (key !== undefined && (ID_COLUMNS.has(key) || ID_LISTS.has(key))) return label(value); + // Fixed labels are also replaced inside text, such as an error message naming an id. + const text = homes.reduce((text, home) => text.split(home).join(""), value); + return fixed.reduce((text, [raw, made]) => text.split(raw).join(made), text); + } + if (typeof value === "number" && key !== undefined) { + if (key.endsWith("_at") || key === "at") return " }; + +let appHome = ""; +let local: Codework; +let desk: Awaited>; +/** Machines in the app by name: the Mac's built-in local one and each added Linux machine. */ +const members = new Map(); + +const snapshot = (name: string, artifact: unknown) => match(`sync.machines.${name}`, artifact); +const normalize = (fixed: ReadonlyArray = []) => + normalizer([appHome, userHome], [...[...members].map(([name, id]) => [id, ``] as const), ...fixed]); + +const idOf = (m: Machine) => members.get(m.name)!; + +/** Waits until the app's cursor for `machineId` reached the server's head; the head it reached. */ +const caughtUp = async (machineId: string, server: Server, timeout = 120_000) => { + const { storeId, rev } = await server.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = machineRow(appHome, machineId); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, + `machine ${machineId} at rev ${rev}`, + timeout, + ); + return { storeId, rev }; +}; + +const add = async (m: Machine) => { + const id = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", name: m.name, url: m.url })); + members.set(m.name, id); + return id; +}; + +/** A session in `directory` with one prompt answered. */ +const session = (server: Server, directory: string, text: string) => + server.run((rpc) => + Effect.gen(function* () { + const created = yield* rpc["session.create"]({ directory }); + yield* rpc["thread.prompt"]({ sessionId: created.session.id, text }); + yield* rpc["thread.wait"]({ sessionId: created.session.id }); + return created; + }), + ); + +const work = (m: Machine) => `${m.cwHome}/work`; + +/** A machine with a fresh store and its server running. */ +const fresh = async (m: Machine) => { + await m.net("restore"); + await m.net("fast"); + await m.stop("KILL"); + await m.wipe(); + await m.sh(`mkdir -p ${work(m)}`); + await m.start(); +}; + +// What app.db holds for a machine, and the harness rows it mirrors. +const appTables = (machineId: string) => ({ + project: query(appHome, "SELECT * FROM project WHERE machine_id = ? ORDER BY created_at, name", machineId), + space: query(appHome, "SELECT * FROM space WHERE machine_id = ? ORDER BY created_at, location", machineId), + session: query(appHome, "SELECT * FROM session WHERE machine_id = ? ORDER BY created_at, slug", machineId), + thread: query( + appHome, + `SELECT thread.* FROM thread JOIN session ON session.id = thread.session_id + WHERE session.machine_id = ? ORDER BY thread.created_at, thread.created_seq`, + machineId, + ), +}); +const harnessTables = async (m: Machine) => ({ + project: await m.sql("SELECT id, name, status, created_at, updated_at, rev FROM project ORDER BY created_at, name"), + space: await m.sql( + "SELECT id, project_id, location, kind, status, created_at, updated_at, rev FROM space ORDER BY created_at, location", + ), + session: await m.sql( + "SELECT id, space_id, slug, directory, created_at, updated_at, rev FROM session ORDER BY created_at, slug", + ), + thread: await m.sql( + `SELECT id, session_id, origin_kind, origin_thread_id, title, run_state, run_outcome, run_error, restarts, + created_seq, created_at, updated_at, rev FROM thread ORDER BY created_at, created_seq`, + ), + removed: await m.sql("SELECT kind, id, rev FROM removed ORDER BY rev"), +}); +type Harness = Awaited>; + +/** Where app.db differs from the harness rows: missing or extra ids, and core columns that disagree. */ +const differences = (copied: ReturnType, harness: Harness) => { + const found: Array = []; + const compare = ( + table: string, + mine: ReadonlyArray, + theirs: ReadonlyArray, + columns: ReadonlyArray, + ) => { + const live = mine.filter((row) => row.deleted_at === undefined || row.deleted_at === null); + const byId = new Map(live.map((row) => [row.id, row])); + for (const row of theirs) { + const copy = byId.get(row.id); + const id = String(row.id); + if (copy === undefined) found.push(`${table} ${id} missing`); + else + for (const column of columns) + if (copy[column] !== row[column]) + found.push( + `${table} ${id} ${column}: ${JSON.stringify(copy[column])} vs ${JSON.stringify(row[column])}`, + ); + byId.delete(row.id); + } + for (const id of byId.keys()) found.push(`${table} ${String(id)} extra`); + }; + compare("project", copied.project, harness.project, ["name", "status", "updated_at"]); + compare("space", copied.space, harness.space, ["project_id", "location", "kind", "status"]); + compare("session", copied.session, harness.session, ["space_id", "slug", "created_at"]); + compare("thread", copied.thread, harness.thread, [ + "session_id", + "origin_kind", + "origin_thread_id", + "title", + "run_state", + "run_outcome", + "created_seq", + "created_at", + "updated_at", + ]); + return found; +}; + +const changedSince = (machineId: string, after: number) => + notices(appHome, Contract.syncTopics.changed, after).filter((notice) => notice.machineId === machineId); + +// The shuffle: each move of the machine set, with the rows each machine holds in app.db around it. +const shuffle: Array = []; +const timings: Array = []; +/** Raw numbers that vary run to run: printed and kept in a temp file, never in an artifact. */ +const measured: Record = {}; +const measure = (name: string, value: unknown) => { + measured[name] = value; + console.log(name, JSON.stringify(value)); +}; +const rowsOf = (machineId: string) => + query( + appHome, + `SELECT (SELECT COUNT(*) FROM project WHERE machine_id = ?1) + (SELECT COUNT(*) FROM space WHERE machine_id = ?1) + + (SELECT COUNT(*) FROM session WHERE machine_id = ?1) + + (SELECT COUNT(*) FROM thread WHERE session_id IN (SELECT id FROM session WHERE machine_id = ?1)) AS n`, + machineId, + )[0]!.n as number; +const census = () => Object.fromEntries([...members].map(([name, id]) => [name, rowsOf(id)])); +const bucket = (ms: number) => (ms < 10_000 ? "<10s" : ms < 30_000 ? "<30s" : ms < 120_000 ? "<120s" : ">=120s"); +/** Runs a move of the machine set and records it; `move` returns the sync path each machine took. */ +/** + * Runs a move of the machine set and records it; `move` returns the sync path each machine took. A + * move that waits on something other than the sync (a machine booting) times the sync itself with + * `sync`: the artifact keeps that time, the printed timings keep both. + */ +const step = async ( + name: string, + move: (sync: (effect: () => Promise) => Promise) => Promise>, +) => { + const before = census(); + const started = performance.now(); + let synced: number | undefined; + const paths = await move(async (effect) => { + const from = performance.now(); + const result = await effect(); + synced = Math.round(performance.now() - from); + return result; + }); + const ms = Math.round(performance.now() - started); + timings.push(synced === undefined ? { step: name, ms } : { step: name, ms, sync: synced }); + shuffle.push({ + step: name, + machines: [...members.keys()], + before, + after: census(), + paths, + time: bucket(synced ?? ms), + }); +}; +/** The path a machine's sync took since `after`: a full sync, or a resume from its cursor. */ +const pathOf = (machineId: string, after: number) => + changedSince(machineId, after).some((notice) => notice.full === true) ? "full sync" : "resume"; + +const describeMachines = available ? describe : describe.skip; + +describeMachines("sync across OrbStack machines", () => { + beforeAll(async () => { + for (const m of pool) await ensure(m.name, m.name === "fedora" ? "fedora" : "ubuntu"); + await Promise.all(pool.map((m) => m.setup())); + await Promise.all(pool.map((m) => fresh(m))); + appHome = tempDir("machines-app"); + local = await codework(join(appHome, "cli"), await freePort()); + members.set("local", LOCAL_SERVER); + await session(local, appHome, "On the Mac"); + desk = await app(appHome); + await step("start: the Mac and fedora", async () => { + const at = lastSeq(appHome); + await add(fedora); + await caughtUp(LOCAL_SERVER, local); + await caughtUp(idOf(fedora), fedora); + return { local: pathOf(LOCAL_SERVER, at), fedora: pathOf(idOf(fedora), at) }; + }); + }, 1_800_000); + + afterAll(async () => { + measure("timings", timings); + writeFileSync(join(tmpdir(), "codework-machines-measured.json"), `${JSON.stringify(measured, null, "\t")}\n`); + await Promise.all( + pool.map((m) => + m + .net("fast") + .then(() => m.net("restore")) + .catch(() => undefined), + ), + ); + await cleanup(); + }, 120_000); + + it("1. snapshot, then watch: rows written during the first sync all arrive", async () => { + // Enough rows for several catch-up batches. + const before = 400; + await cwA.run((rpc) => + Effect.forEach( + Array.from({ length: before }, (_, n) => n), + () => rpc["session.create"]({ directory: work(cwA) }), + { concurrency: 8, discard: true }, + ), + ); + const head = (await cwA.run((rpc) => rpc["server.info"]({}))).rev; + + // Writes start with the add and run through its catch-up: new sessions, and prompts on old ones. + let firstSynced: number | undefined; + let added = ""; + const during = 30; + await step("add cw-a with writes during its first sync", async () => { + const at = lastSeq(appHome); + const writes = cwA.run((rpc) => + Effect.forEach( + Array.from({ length: during }, (_, n) => n), + (n) => + rpc["session.create"]({ directory: work(cwA) }).pipe( + Effect.tap((created) => + rpc["thread.prompt"]({ sessionId: created.session.id, text: `During ${n}` }).pipe( + Effect.andThen(rpc["thread.wait"]({ sessionId: created.session.id })), + ), + ), + ), + { concurrency: 2, discard: true }, + ), + ); + added = await add(cwA); + const watch = waitFor( + () => { + const row = machineRow(appHome, added); + return row !== undefined && row.cursor > 0 ? row.cursor : undefined; + }, + "the first Synced", + 120_000, + ).then((cursor) => (firstSynced = cursor)); + await Promise.all([writes, watch]); + await caughtUp(added, cwA); + return { "cw-a": pathOf(added, at) }; + }); + + const harness = await harnessTables(cwA); + const copied = appTables(added); + expect(differences(copied, harness)).toEqual([]); + const caughtUpRows = harness.thread.filter( + (row) => (row.rev as number) > head && (row.rev as number) <= firstSynced!, + ); + const liveRows = harness.thread.filter((row) => (row.rev as number) > firstSynced!); + measure("case 1", { head, firstSynced, writtenDuringCatchUp: caughtUpRows.length, afterSynced: liveRows.length }); + expect(copied.session).toHaveLength(before + during); + + await snapshot( + "snapshot", + normalize()({ + sessions: { before, during }, + rows: Object.fromEntries( + Object.entries(copied).map(([table, rows]) => [ + table, + { app: rows.length, harness: harness[table as keyof Harness].length }, + ]), + ), + missing: differences(copied, harness), + projects: copied.project.map(({ id, machine_id, name, status }) => ({ id, machine_id, name, status })), + }), + ); + }, 600_000); + + it("2. a partition resumes from the cursor with only the rows changed since", async () => { + const machineId = idOf(cwA); + const created = await session(cwA, work(cwA), "Before the cut"); + const sessionId = created.session.id; + const reply = await cwA.run((rpc) => + rpc["thread.history"]({ sessionId }).pipe(Effect.map((h) => h.entries.at(-1)!)), + ); + const { rev: cursor } = await caughtUp(machineId, cwA); + const [space] = await cwA.sql(`SELECT id, project_id FROM space WHERE id = '${created.session.spaceId}'`); + + const cutAt = performance.now(); + await cwA.net("cut"); + const [forked] = (await cwA.drive([ + ["thread.fork", { sessionId, entryId: reply.id }], + ["thread.prompt", { sessionId, text: "While cut" }], + ["thread.wait", { sessionId }], + ["project.rename", { projectId: space!.project_id, name: "Renamed while cut" }], + ])) as [{ thread: { id: string } }]; + // No RPC archives a space: a store write by the revision rule, as the harness's own would. + await cwA.sql(`BEGIN; UPDATE store SET rev = rev + 1; + UPDATE space SET status = 'archived', updated_at = 1, rev = (SELECT rev FROM store) WHERE id = '${String(space!.id)}'; COMMIT;`); + // Past the client's ping timeout, so the app has dropped the connection. + const held = 15_000 - (performance.now() - cutAt); + if (held > 0) await new Promise((resolve) => setTimeout(resolve, held)); + const tables = ["project", "space", "session", "thread", "removed"]; + const count = async (where: string) => + ( + await cwA.sql( + `SELECT ${tables.map((table) => `(SELECT COUNT(*) FROM ${table} ${where})`).join(" + ")} AS n`, + ) + )[0]!.n as number; + const sent = await count(`WHERE rev > ${cursor}`); + const total = await count(""); + + const at = lastSeq(appHome); + let resumeMs = 0; + await step("partition cw-a and restore", async () => { + await cwA.net("restore"); + const restoredAt = performance.now(); + await caughtUp(machineId, cwA); + resumeMs = Math.round(performance.now() - restoredAt); + return { "cw-a": pathOf(machineId, at) }; + }); + const changed = changedSince(machineId, at); + measure("case 2", { sent, total, resumeMs, notices: changed.length }); + expect(changed.every((notice) => notice.full === false)).toBe(true); + expect(changed.reduce((sum, notice) => sum + (notice.items as number), 0)).toBe(sent); + expect(sent).toBeLessThan(total); + const copied = appTables(machineId); + expect(differences(copied, await harnessTables(cwA))).toEqual([]); + const thread = copied.thread.find((row) => row.id === forked.thread.id); + expect(thread).toMatchObject({ origin_kind: "fork", origin_thread_id: created.id }); + expect(copied.space.find((row) => row.id === space!.id)?.status).toBe("archived"); + expect(copied.project.find((row) => row.id === space!.project_id)?.name).toBe("Renamed while cut"); + + await snapshot( + "partition", + normalize()({ + rows: { sent, total }, + changed, + project: copied.project.filter((row) => row.id === space!.project_id), + space: copied.space.filter((row) => row.id === space!.id), + thread: copied.thread.filter((row) => row.session_id === sessionId), + }), + ); + }, 300_000); + + it("3. a session deleted while away is soft-deleted on resume, its detail kept and hidden from reads", async () => { + const machineId = idOf(fedora); + const kept = await session(fedora, work(fedora), "Kept"); + const doomed = await session(fedora, work(fedora), "Doomed"); + await caughtUp(machineId, fedora); + await desk.run((rpc) => rpc["sync.open"]({ sessionId: doomed.session.id, threadId: doomed.id })); + await desk.run((rpc) => rpc["sync.close"]({ sessionId: doomed.session.id, threadId: doomed.id })); + + const cutAt = performance.now(); + await fedora.net("cut"); + await fedora.drive([["session.delete", { sessionId: doomed.session.id }]]); + const held = 15_000 - (performance.now() - cutAt); + if (held > 0) await new Promise((resolve) => setTimeout(resolve, held)); + const at = lastSeq(appHome); + await step("partition fedora, delete a session, restore", async () => { + await fedora.net("restore"); + await caughtUp(machineId, fedora); + return { fedora: pathOf(machineId, at) }; + }); + + const sessions = query( + appHome, + "SELECT * FROM session WHERE id IN (?, ?) ORDER BY created_at", + kept.session.id, + doomed.session.id, + ); + const threads = query(appHome, "SELECT * FROM thread WHERE session_id = ?", doomed.session.id); + const entries = query(appHome, "SELECT * FROM entry WHERE thread_id = ? ORDER BY seq", doomed.id); + // The sidebar's read: sessions not deleted. + const listed = query( + appHome, + "SELECT id FROM session WHERE machine_id = ? AND deleted_at IS NULL AND id IN (?, ?)", + machineId, + kept.session.id, + doomed.session.id, + ); + const changed = changedSince(machineId, at); + expect(sessions.find((row) => row.id === doomed.session.id)?.deleted_at).not.toBeNull(); + expect(threads.every((row) => row.deleted_at !== null)).toBe(true); + expect(entries).toHaveLength(2); + expect(listed).toEqual([{ id: kept.session.id }]); + expect(changed.at(-1)).toMatchObject({ full: false, removed: [doomed.session.id] }); + const removed = await fedora.sql(`SELECT kind, id, rev FROM removed WHERE id = '${doomed.session.id}'`); + expect(removed).toHaveLength(1); + + await snapshot( + "deleted", + normalize()({ changed, harness: { removed }, session: sessions, thread: threads, entry: entries, listed }), + ); + }, 300_000); + + it("4. a crash mid-run is swept before any client connects, then recovery resumes it", async () => { + await step("add cw-b live", async () => { + const at = lastSeq(appHome); + const id = await add(cwB); + await caughtUp(id, cwB); + return { "cw-b": pathOf(id, at) }; + }); + const machineId = idOf(cwA); + const created = await cwA.run((rpc) => rpc["session.create"]({ directory: work(cwA) })); + const ref = { sessionId: created.session.id, threadId: created.id }; + const row = () => + query( + appHome, + "SELECT status, run_state, run_started_at, run_outcome, run_error FROM thread WHERE id = ?", + ref.threadId, + )[0]!; + await cwA.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "hold" })); + const running = await waitFor(() => (row().run_state === "running" ? row() : undefined), "the held run"); + expect(running.status).toBe("working"); + const crashedAt = lastSeq(appHome); + + await cwA.stop("KILL"); + const harnessRow = `SELECT run_state, run_outcome, run_error, execution FROM thread WHERE id = '${ref.threadId}'`; + const open = await cwA.sql(harnessRow); + await cwA.sweep(); + const swept = await cwA.sql(harnessRow); + expect(swept[0]).toMatchObject({ run_state: "idle", run_outcome: "interrupted" }); + expect(JSON.parse(swept[0]!.run_error as string)).toEqual({ _tag: "interrupted", reason: "crash" }); + + const at = lastSeq(appHome); + await step("crash cw-a mid-run and restart it", async () => { + await cwA.start(); + await waitFor( + () => + row().run_started_at !== running.run_started_at && row().run_state === "running" ? row() : undefined, + "the resumed run", + 60_000, + ); + return { "cw-a": pathOf(machineId, at) }; + }); + const events = await cwA.sql( + `SELECT type, json_extract(data, '$.resumed') AS resumed, json_extract(data, '$.reason') AS reason FROM event + WHERE aggregate_id = '${ref.sessionId}' AND type LIKE 'session.execution.%' ORDER BY seq`, + ); + // Durable event types carry their version: `session.execution.started.1`. + expect(events.at(-1)).toMatchObject({ type: "session.execution.started.1", resumed: 1 }); + const moves = notices(appHome, Contract.syncTopics.status, crashedAt).filter( + (move) => move.threadId === ref.threadId, + ); + // Interrupted by a crash leaves the status alone; the resumed start keeps it working. + expect(moves).toEqual([]); + expect(row().status).toBe("working"); + const resumed = row(); + await cwA.run((rpc) => rpc["thread.interrupt"](ref)); + + await snapshot( + "crash", + normalize()({ + harness: { open: open[0], swept: swept[0], events }, + app: { running, resumed }, + statusMoves: moves, + }), + ); + }, 300_000); + + it("5. a restored store is read again in full; returning rows keep their desktop columns", async () => { + await step("add cw-c live", async () => { + const at = lastSeq(appHome); + const id = await add(cwC); + await caughtUp(id, cwC); + return { "cw-c": pathOf(id, at) }; + }); + const machineId = idOf(cwB); + const before = await session(cwB, work(cwB), "Before backup"); + await caughtUp(machineId, cwB); + write(appHome, "UPDATE thread SET priority = 2, color = '#ff8800', status = 'attention' WHERE id = ?", before.id); + write(appHome, "UPDATE session SET color = '#00aa88' WHERE id = ?", before.session.id); + + await cwB.stop("TERM"); + await cwB.sh(`cp ${cwB.database} ${cwB.cwHome}/backup.db`); + await cwB.start(); + const after = await session(cwB, work(cwB), "After backup"); + const { rev: head } = await caughtUp(machineId, cwB); + await cwB.stop("TERM"); + await cwB.sh(`rm -f ${cwB.database}-wal ${cwB.database}-shm && cp ${cwB.cwHome}/backup.db ${cwB.database}`); + + const at = lastSeq(appHome); + let fullMs = 0; + let restored = { storeId: "", rev: 0 }; + await step("restore cw-b's store from a backup", async () => { + await cwB.start(); + const startedAt = performance.now(); + restored = await cwB.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => (machineRow(appHome, machineId).cursor === restored.rev ? true : undefined), + "the full sync", + 120_000, + ); + fullMs = Math.round(performance.now() - startedAt); + return { "cw-b": pathOf(machineId, at) }; + }); + measure("case 5", { head, restoredRev: restored.rev, fullMs }); + expect(restored.rev).toBeLessThan(head); + const changed = changedSince(machineId, at); + expect(changed[0]).toMatchObject({ full: true, removed: [after.session.id] }); + const rows = appTables(machineId); + expect(rows.thread.find((row) => row.id === before.id)).toMatchObject({ + priority: 2, + color: "#ff8800", + status: "attention", + deleted_at: null, + }); + expect(rows.session.find((row) => row.id === before.session.id)).toMatchObject({ + color: "#00aa88", + deleted_at: null, + }); + expect(rows.session.find((row) => row.id === after.session.id)?.deleted_at).not.toBeNull(); + expect(differences(rows, await harnessTables(cwB))).toEqual([]); + + await snapshot( + "restored", + normalize([[restored.storeId, ""]])({ + heads: [{ rev: head }, { rev: restored.rev }], + full: changed.filter((notice) => notice.full === true), + ...rows, + }), + ); + }, 300_000); + + it("6. a new store on the same machine is read again in full", async () => { + const machineId = idOf(cwC); + const old = await session(cwC, work(cwC), "Old store"); + const first = await caughtUp(machineId, cwC); + const at = lastSeq(appHome); + let fresher = { storeId: "", rev: 0 }; + await step("replace cw-c's store with a new one", async () => { + await fresh(cwC); + fresher = await caughtUp(machineId, cwC); + return { "cw-c": pathOf(machineId, at) }; + }); + const created = await session(cwC, work(cwC), "New store"); + await caughtUp(machineId, cwC); + expect(fresher.storeId).not.toBe(first.storeId); + const changed = changedSince(machineId, at); + expect(changed.find((notice) => notice.full === true)).toMatchObject({ removed: [old.session.id] }); + const rows = appTables(machineId); + expect(rows.session.find((row) => row.id === old.session.id)?.deleted_at).not.toBeNull(); + expect(rows.session.find((row) => row.id === created.session.id)?.deleted_at).toBeNull(); + + await snapshot( + "store", + normalize([ + [first.storeId, ""], + [fresher.storeId, ""], + ])({ + machine: machineRow(appHome, machineId), + // Live batches after it split by doorbell timing, so only the full sync is kept. + full: changed.filter((notice) => notice.full === true), + session: rows.session, + thread: rows.thread, + }), + ); + }, 300_000); + + it("7. a slow link keeps thread.sync in order, drops deltas that do not continue the draft, and lands the reply", async () => { + const machineId = idOf(fedora); + const created = await fedora.run((rpc) => rpc["session.create"]({ directory: work(fedora) })); + const ref = { sessionId: created.session.id, threadId: created.id }; + await caughtUp(machineId, fedora); + await desk.run((rpc) => rpc["sync.open"](ref)); + const forwarded: Array<{ entryId: string; offset: number; delta: string }> = []; + let subscribed = false; + const follower = desk.background((rpc) => + rpc["event.subscribe"]({ topics: [Contract.syncTopics.delta(ref.threadId)] }).pipe( + Stream.runForEach((item) => + Effect.sync(() => { + if ("_tag" in item) subscribed = true; + else forwarded.push(item.payload as { entryId: string; offset: number; delta: string }); + }), + ), + ), + ); + await waitFor(() => (subscribed ? true : undefined), "the delta subscription"); + const cursors: Array = []; + let polling = true; + const poll = (async () => { + while (polling) { + const cursor = query(appHome, "SELECT cursor FROM thread WHERE id = ?", ref.threadId)[0]?.cursor as + | number + | null; + if (cursor !== null && cursor !== cursors.at(-1)) cursors.push(cursor); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + })(); + + const chunks = 400; + await fedora.net("slow"); + try { + await fedora.run((rpc) => rpc["thread.prompt"]({ ...ref, text: `long ${chunks}` })); + await waitFor(() => (forwarded.length >= 20 ? true : undefined), "deltas over the slow link", 60_000); + // Closed and reopened mid-reply: the held draft goes, and the stream restarts from the last checkpoint. + const reopenedAt = forwarded.length; + await desk.run((rpc) => rpc["sync.close"](ref)); + await desk.run((rpc) => rpc["sync.open"](ref)); + await fedora.run((rpc) => rpc["thread.wait"](ref)); + const committed = () => + query( + appHome, + "SELECT entry.id, entry.state FROM entry WHERE thread_id = ? AND type = 'assistant' AND state = 'committed'", + ref.threadId, + )[0]; + await waitFor(committed, "the committed reply", 120_000); + polling = false; + await poll; + follower.stop(); + + const final = query( + appHome, + `SELECT json_extract(part.data, '$.text') AS text FROM part JOIN entry ON entry.id = part.entry_id + WHERE entry.thread_id = ? AND entry.type = 'assistant'`, + ref.threadId, + )[0]!.text as string; + const harness = ( + await fedora.sql( + `SELECT json_extract(part.data, '$.text') AS text FROM part JOIN entry ON entry.id = part.entry_id + WHERE entry.thread_id = '${ref.threadId}' AND entry.type = 'assistant'`, + ) + )[0]?.text; + const expected = Array.from({ length: chunks }, (_, step) => + `chunk ${String(step).padStart(5, "0")} `.padEnd(32, "."), + ).join(""); + // Every forwarded delta sits at its place in the final text; a gap is a delta never folded. + const continues = forwarded.every( + (delta) => final.slice(delta.offset, delta.offset + delta.delta.length) === delta.delta, + ); + const offsets = forwarded.map((delta) => delta.offset); + const ordered = offsets.every((offset, index) => index === 0 || offset > offsets[index - 1]!); + const gaps = offsets.filter((offset, index) => index > 0 && offset !== offsets[index - 1]! + 32).length; + const afterReopen = offsets.slice(reopenedAt); + const gapAfterReopen = afterReopen.length === 0 || afterReopen[0] !== offsets[reopenedAt - 1]! + 32; + const cursorsInOrder = cursors.every((cursor, index) => index === 0 || cursor > cursors[index - 1]!); + measure("case 7", { + emitted: chunks, + forwarded: forwarded.length, + dropped: chunks - forwarded.length, + gaps, + cursors: cursors.length, + }); + expect(final).toBe(expected); + expect(harness).toBe(expected); + expect(continues).toBe(true); + expect(ordered).toBe(true); + expect(cursorsInOrder).toBe(true); + expect(forwarded.length).toBeLessThan(chunks); + + await snapshot( + "slow", + normalize()({ + link: "netem delay 300ms rate 256kbit", + emitted: chunks, + forwardedAreInOrder: ordered, + forwardedContinueTheFinalText: continues, + someDropped: forwarded.length < chunks, + gapAfterReopen, + cursorsInOrder, + finalEqualsHarness: final === harness, + entry: query( + appHome, + "SELECT id, thread_id, type, state, size FROM entry WHERE thread_id = ? ORDER BY seq", + ref.threadId, + ), + }), + ); + } finally { + polling = false; + follower.stop(); + await fedora.net("fast"); + await desk.run((rpc) => rpc["sync.close"](ref)); + } + }, 300_000); + + it("8. a fork opened while its source keeps writing holds the source's line up to the cut, stored once", async () => { + const machineId = idOf(cwA); + const created = await session(cwA, work(cwA), "Source one"); + const sessionId = created.session.id; + const source = { sessionId, threadId: created.id }; + await cwA.run((rpc) => + rpc["thread.prompt"]({ ...source, text: "Source two" }).pipe(Effect.andThen(rpc["thread.wait"](source))), + ); + const history = await cwA.run((rpc) => rpc["thread.history"]({ sessionId })); + const at = history.entries[1]!; + const fork = (await cwA.run((rpc) => rpc["thread.fork"]({ sessionId, entryId: at.id }))).thread; + const forkRef = { sessionId, threadId: fork.id }; + + // The source keeps going while the fork is opened. + const busy = cwA.run((rpc) => + Effect.forEach( + ["Source three", "Source four", "Source five"], + (text) => rpc["thread.prompt"]({ ...source, text }).pipe(Effect.andThen(rpc["thread.wait"](source))), + { discard: true }, + ), + ); + await caughtUp(machineId, cwA); + await desk.run((rpc) => rpc["sync.open"](forkRef)); + await busy; + await cwA.run((rpc) => + rpc["thread.prompt"]({ ...forkRef, text: "Fork one" }).pipe(Effect.andThen(rpc["thread.wait"](forkRef))), + ); + await caughtUp(machineId, cwA); + const line = (await cwA.run((rpc) => rpc["thread.history"](forkRef))).entries; + const [threadRow] = query(appHome, "SELECT origin_committed_seq, cursor FROM thread WHERE id = ?", fork.id); + const cut = threadRow!.origin_committed_seq as number; + await waitFor( + () => (query(appHome, "SELECT id FROM entry WHERE thread_id = ?", fork.id).length === 2 ? true : undefined), + "the fork's own entries", + ); + const held = () => + query( + appHome, + "SELECT id, thread_id, seq, type, state FROM entry WHERE thread_id IN (?, ?) ORDER BY seq", + created.id, + fork.id, + ); + const opened = held(); + const sourceHeld = opened.filter((row) => row.thread_id === created.id); + expect(opened.map((row) => row.id)).toEqual(line.map((entry) => entry.id)); + expect(sourceHeld.every((row) => (row.seq as number) <= cut)).toBe(true); + expect(cut).toBe(fork.origin.kind === "fork" ? fork.origin.cut : undefined); + + // Opening the source as well stores its later entries; the inherited ones stay one row each. + await desk.run((rpc) => rpc["sync.open"](source)); + const both = held(); + const sourceLine = (await cwA.run((rpc) => rpc["thread.history"](source))).entries; + expect(both.filter((row) => row.thread_id === created.id).map((row) => row.id)).toEqual( + sourceLine.map((entry) => entry.id), + ); + expect(new Set(both.map((row) => row.id)).size).toBe(both.length); + await desk.run((rpc) => rpc["sync.close"](source)); + await desk.run((rpc) => rpc["sync.close"](forkRef)); + + await snapshot( + "fork", + normalize()({ + cut: { origin_committed_seq: cut }, + forkLine: opened, + afterSourceOpened: both, + }), + ); + }, 300_000); + + it("9. several machines at once: one repository on two machines, commands to each machine, removal cascades", async () => { + const repo = `${userHome}/codework`; + const onFedora = await session(fedora, repo, "Repo on fedora"); + const onA = await session(cwA, repo, "Repo on cw-a"); + await caughtUp(idOf(fedora), fedora); + await caughtUp(idOf(cwA), cwA); + const projects = query( + appHome, + `SELECT project.machine_id, project.id, project.name FROM project JOIN space + ON space.machine_id = project.machine_id AND space.project_id = project.id + WHERE space.location = ? ORDER BY project.machine_id = ?`, + repo, + idOf(cwA), + ); + expect(projects).toHaveLength(2); + expect(projects[0]!.id).toBe(projects[1]!.id); + + // Each thread's detail comes from its own machine. + const machineOf = (sessionId: string) => + query(appHome, "SELECT machine_id FROM session WHERE id = ?", sessionId)[0]!.machine_id; + const detail = async (server: Server, created: typeof onA) => { + const ref = { sessionId: created.session.id, threadId: created.id }; + await desk.run((rpc) => rpc["sync.open"](ref)); + await desk.run((rpc) => rpc["sync.close"](ref)); + const line = (await server.run((rpc) => rpc["thread.history"](ref))).entries.map((entry) => entry.id); + const held = query(appHome, "SELECT id FROM entry WHERE thread_id = ? ORDER BY seq", created.id).map( + (row) => row.id, + ); + return { + machine: machineOf(created.session.id), + sameAsItsMachine: JSON.stringify(line) === JSON.stringify(held), + }; + }; + const routed = { fedora: await detail(fedora, onFedora), "cw-a": await detail(cwA, onA) }; + expect(routed).toEqual({ + fedora: { machine: idOf(fedora), sameAsItsMachine: true }, + "cw-a": { machine: idOf(cwA), sameAsItsMachine: true }, + }); + + const fedoraId = idOf(fedora); + const left = () => ({ + project: query(appHome, "SELECT COUNT(*) AS n FROM project WHERE machine_id = ?", fedoraId)[0]!.n, + space: query(appHome, "SELECT COUNT(*) AS n FROM space WHERE machine_id = ?", fedoraId)[0]!.n, + session: query(appHome, "SELECT COUNT(*) AS n FROM session WHERE machine_id = ?", fedoraId)[0]!.n, + thread: query(appHome, "SELECT COUNT(*) AS n FROM thread WHERE id = ?", onFedora.id)[0]!.n, + entry: query(appHome, "SELECT COUNT(*) AS n FROM entry WHERE thread_id = ?", onFedora.id)[0]!.n, + }); + const beforeRemove = left(); + await step("remove fedora", async () => { + await desk.run((rpc) => rpc["servers.remove"]({ id: fedoraId })); + return { fedora: "cascade" }; + }); + members.delete("fedora"); + const afterRemove = left(); + expect(Object.values(afterRemove).every((n) => n === 0)).toBe(true); + + await snapshot( + "several", + normalize([[fedoraId, ""]])({ projects, routed, beforeRemove, afterRemove }), + ); + }, 300_000); + + it("shuffle: a stopped machine resumes, a removed one cascades, a removed one comes back", async () => { + const machineId = idOf(cwC); + await step("stop cw-c (orb stop) and start it again", async (sync) => { + const at = lastSeq(appHome); + await orb(["stop", "cw-c"], { timeout: 120_000 }); + await orb(["start", "cw-c"], { timeout: 300_000 }); + await cwC.start(); + // The machine's name answers again a moment after it boots, sometimes after minutes. + const bootedAt = performance.now(); + await waitFor( + () => cwC.run((rpc) => rpc["server.info"]({}).pipe(Effect.timeout("5 seconds"))).catch(() => undefined), + "cw-c reachable from the Mac", + 480_000, + ); + measure("cw-c reachable after boot (ms)", Math.round(performance.now() - bootedAt)); + // The resume: from the server answering again to the app's cursor at its head. + await sync(() => caughtUp(machineId, cwC)); + await session(cwC, work(cwC), "After a machine restart"); + await caughtUp(machineId, cwC); + return { "cw-c": pathOf(machineId, at) }; + }); + const cwBId = idOf(cwB); + await step("remove cw-b", async () => { + await desk.run((rpc) => rpc["servers.remove"]({ id: cwBId })); + return { "cw-b": "cascade" }; + }); + members.delete("cw-b"); + expect(rowsOf(cwBId)).toBe(0); + await step("add fedora again", async () => { + const at = lastSeq(appHome); + const id = await add(fedora); + await caughtUp(id, fedora); + return { fedora: pathOf(id, at) }; + }); + expect(differences(appTables(idOf(fedora)), await harnessTables(fedora))).toEqual([]); + }, 900_000); + + it("10. concurrent writes take strictly increasing revisions; a write without the current one is refused", async () => { + const machineId = idOf(fedora); + const created = await session(fedora, work(fedora), "Revisions"); + const sessionId = created.session.id; + const reply = (await fedora.run((rpc) => rpc["thread.history"]({ sessionId }))).entries.at(-1)!; + const fork = (await fedora.run((rpc) => rpc["thread.fork"]({ sessionId, entryId: reply.id }))).thread; + const main = { sessionId, threadId: created.id }; + const forked = { sessionId, threadId: fork.id }; + const { rev: head } = await caughtUp(machineId, fedora); + const projectId = created.session.project?.id; + + await fedora.run((rpc) => + Effect.all( + [ + rpc["thread.prompt"]({ ...main, text: "Concurrent main" }).pipe( + Effect.andThen(rpc["thread.wait"](main)), + ), + rpc["thread.prompt"]({ ...forked, text: "Concurrent fork" }).pipe( + Effect.andThen(rpc["thread.wait"](forked)), + ), + rpc["thread.configure"]({ ...forked, runtime: { thinkingLevel: "high" } }), + rpc["project.rename"]({ projectId: projectId!, name: "Renamed concurrently" }), + ], + { concurrency: "unbounded", discard: true }, + ), + ); + const items = await fedora.run((rpc) => + rpc["index.sync"]({ after: head }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ); + const revs = items.map((item) => item.rev); + const synced = items.at(-1)!; + const rows = items.filter((item) => item._tag !== "Synced"); + const increasing = revs.every((rev, index) => index === 0 || rev >= revs[index - 1]!); + // One revision per write transaction: no two distinct write paths share one. + const touched = await fedora.sql( + `SELECT 'project' AS t, rev FROM project WHERE rev > ${head} UNION ALL SELECT 'thread', rev FROM thread WHERE rev > ${head} + UNION ALL SELECT 'entry', rev FROM entry WHERE rev > ${head} ORDER BY rev`, + ); + const [store] = await fedora.sql("SELECT rev FROM store"); + const configured = rows.find( + (item) => item._tag === "Thread" && item.thread.id === fork.id && item.thread.thinkingLevel === "high", + ); + await caughtUp(machineId, fedora); + const appThread = query(appHome, "SELECT thinking_level FROM thread WHERE id = ?", fork.id)[0]; + // The main thread's row is older than the head: rewriting it with its own rev is a write without the current revision. + const [stale] = await fedora.sql(`SELECT rev FROM thread WHERE id = '${created.id}'`); + expect(stale!.rev as number).toBeLessThan(store!.rev as number); + const refused = await fedora.refused( + `UPDATE thread SET title = 'Not at the current revision' WHERE id = '${created.id}';`, + ); + // A write that stamps the head without taking a new revision is not caught: the trigger compares with + // store.rev only, so it cannot tell a forgotten increment from a second write in the same transaction. + const atHead = await fedora.refused( + `UPDATE thread SET title = 'At the head, untaken', rev = (SELECT rev FROM store) WHERE id = '${created.id}';`, + ); + const [headAfter] = await fedora.sql("SELECT rev FROM store"); + const refusedInsert = await fedora.refused( + `INSERT INTO project (id, name, status, created_at, updated_at, rev) VALUES ('stray', 'Stray', 'active', 1, 1, 0);`, + ); + const untouched = await fedora.sql(`SELECT title FROM thread WHERE id = '${created.id}'`); + measure("case 10", { head, items: rows.length, revs: new Set(revs).size, store: store!.rev }); + expect(increasing).toBe(true); + expect(revs.every((rev) => rev > head)).toBe(true); + expect(synced).toMatchObject({ _tag: "Synced", rev: store!.rev }); + expect(touched.every((row) => (row.rev as number) > head && (row.rev as number) <= (store!.rev as number))).toBe( + true, + ); + expect(configured).toBeDefined(); + expect(appThread).toEqual({ thinking_level: "high" }); + expect(refused).toContain("rev is not the current revision"); + expect(refusedInsert).toContain("rev is not the current revision"); + expect(untouched[0]!.title).toBe(atHead === undefined ? "At the head, untaken" : "Revisions"); + expect(headAfter!.rev).toBe(store!.rev); + + await snapshot( + "revisions", + normalize()({ + itemsInRevOrder: increasing, + itemTags: [...new Set(rows.map((item) => item._tag))].sort(), + syncedAtStoreHead: synced.rev === store!.rev, + configureReachedIndexSync: configured !== undefined, + app: appThread, + refused: { update: refused, insert: refusedInsert }, + headWithoutIncrement: { accepted: atHead === undefined, storeRevMoved: headAfter!.rev !== store!.rev }, + }), + ); + }, 300_000); + it("shuffle: the machine set's moves, rows and sync paths", async () => { + expect(shuffle.length).toBeGreaterThan(0); + await snapshot("shuffle", normalize()(shuffle)); + }); +}); diff --git a/apps/server/test/sync.test.ts b/apps/server/test/sync.test.ts new file mode 100644 index 00000000..a2187204 --- /dev/null +++ b/apps/server/test/sync.test.ts @@ -0,0 +1,625 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite stops, copies and removes CodeWork stores in temp dirs. */ +import { Effect, Stream } from "effect"; +import { copyFileSync, existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { + addMachine, + app, + cleanup, + type Codework, + codework, + freePort, + lastSeq, + machineRow, + normalizer, + notices, + query, + type Row, + snapshot as match, + stop, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const tables = { + machine: "SELECT id, kind, store_id, cursor FROM machine ORDER BY pos", + project: "SELECT * FROM project ORDER BY created_at, id", + space: "SELECT * FROM space ORDER BY created_at, id", + session: "SELECT * FROM session ORDER BY created_at, id", + thread: "SELECT * FROM thread ORDER BY created_seq, created_at, id", + entry: "SELECT * FROM entry ORDER BY seq", + part: `SELECT part.entry_id, part.part_index, part.type, part.status, json_extract(part.data, '$.text') AS text + FROM part JOIN entry ON entry.id = part.entry_id ORDER BY entry.seq, part.part_index`, + inbox: "SELECT * FROM inbox ORDER BY admitted_seq", +}; +type Table = keyof typeof tables; +const dump = (home: string, ...names: ReadonlyArray) => + Object.fromEntries(names.map((name) => [name, query(home, tables[name])])); + +const snapshot = (name: string, artifact: unknown) => match(`sync.${name}`, artifact); + +/** A session in `home` with one prompt answered, as the CodeWork client sees it. */ +const session = (server: Codework, directory: string, text: string, project?: string) => + server.run((rpc) => + Effect.gen(function* () { + const created = yield* rpc["session.create"]({ + directory, + ...(project === undefined ? {} : { project: { name: project } }), + }); + yield* rpc["thread.prompt"]({ sessionId: created.session.id, text }); + yield* rpc["thread.wait"]({ sessionId: created.session.id }); + return created; + }), + ); + +it("first sync copies a session with a fork and a prompt, with first-seen desktop values", async () => { + const cwHome = tempDir("first-cw"); + const appHome = tempDir("first-app"); + const server = await codework(cwHome, await freePort()); + const created = await session(server, cwHome, "First prompt", "Alpha"); + const sessionId = created.session.id; + const fork = await server.run((rpc) => + Effect.gen(function* () { + const reply = (yield* rpc["thread.history"]({ sessionId })).entries.at(-1)!; + const forked = yield* rpc["thread.fork"]({ sessionId, entryId: reply.id }); + yield* rpc["thread.prompt"]({ sessionId, threadId: forked.thread.id, text: "Fork prompt" }); + yield* rpc["thread.wait"]({ sessionId, threadId: forked.thread.id }); + return forked.thread; + }), + ); + + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const { storeId, rev } = await synced(appHome, machineId, server); + const threads = await server.run((rpc) => rpc["thread.list"]({ sessionId })); + + const rows = dump(appHome, "machine", "project", "space", "session", "thread"); + const remote = rows.machine!.find((row) => row.id === machineId)!; + expect(remote).toMatchObject({ kind: "remote", store_id: storeId, cursor: rev }); + const [stored] = rows.session!; + expect(stored).toMatchObject({ + id: sessionId, + machine_id: machineId, + organization_id: "personal", + space_id: created.session.spaceId, + slug: created.session.slug, + dir: created.session.directory, + created_at: created.session.createdAt, + deleted_at: null, + }); + expect(rows.thread).toHaveLength(threads.length); + for (const thread of threads) { + const row = rows.thread!.find((candidate) => candidate.id === thread.id)!; + expect(row).toMatchObject({ + session_id: sessionId, + organization_id: "personal", + origin_kind: thread.origin.kind, + origin_thread_id: thread.origin.kind === "main" ? null : thread.origin.threadId, + origin_committed_seq: thread.origin.kind === "main" ? null : (thread.origin.cut ?? null), + title: thread.title ?? null, + run_state: thread.runState, + run_started_at: thread.lastRun?.startedAt ?? null, + run_ended_at: thread.lastRun?.endedAt ?? null, + run_outcome: thread.lastRun?.outcome ?? null, + created_seq: thread.createdSeq, + created_at: thread.createdAt, + updated_at: thread.updatedAt, + tokens_input: thread.usage.input, + status: "settled", + priority: 0, + cursor: null, + }); + } + // Each new thread goes first in its organization. + expect(rows.thread!.map((row) => row.ordering)).toEqual([0, -1]); + expect(rows.thread!.find((row) => row.id === fork.id)?.origin_kind).toBe("fork"); + + await snapshot("first", normalizer([cwHome, appHome], [[storeId, ""]])(rows)); +}, 90_000); + +it("a restart of CodeWork and the app resumes from the cursor with only the rows changed since", async () => { + const cwHome = tempDir("resume-cw"); + const appHome = tempDir("resume-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const created = await session(server, cwHome, "First prompt"); + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const before = machineRow(appHome, machineId); + + await stop(desk.proc); + await stop(server.proc); + server = await codework(cwHome, port); + await server.run((rpc) => + Effect.gen(function* () { + yield* rpc["thread.prompt"]({ sessionId: created.session.id, text: "Second prompt" }); + yield* rpc["thread.wait"]({ sessionId: created.session.id }); + }), + ); + // What a resume from the app's cursor carries, read from CodeWork itself. + const expected = await server.run((rpc) => + rpc["index.sync"]({ after: before.cursor }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + const after = await synced(appHome, machineId, server); + const changed = notices(appHome, Contract.syncTopics.changed, restartedAt); + + const items = expected.filter((item) => item._tag !== "Synced"); + expect(after.storeId).toBe(before.store_id); + expect(changed).toHaveLength(1); + expect(changed[0]).toMatchObject({ machineId, full: false, items: items.length }); + expect(items.length).toBeGreaterThan(0); + expect(items.every((item) => item.rev > before.cursor)).toBe(true); + + await snapshot( + "resume", + normalizer( + [cwHome, appHome], + [[after.storeId, ""]], + )({ + cursors: [{ rev: before.cursor }, { rev: machineRow(appHome, machineId).cursor }], + resumed: { items: items.length, tags: items.map((item) => item._tag) }, + changed: changed, + thread: dump(appHome, "thread").thread, + }), + ); +}, 90_000); + +it("a session deleted while the app was down is soft-deleted with its detail kept", async () => { + const cwHome = tempDir("removed-cw"); + const appHome = tempDir("removed-app"); + const server = await codework(cwHome, await freePort()); + const kept = await session(server, cwHome, "Kept prompt"); + const doomed = await session(server, cwHome, "Doomed prompt"); + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + await desk.run((rpc) => rpc["sync.open"]({ sessionId: doomed.session.id, threadId: doomed.id })); + await stop(desk.proc); + + await server.run((rpc) => rpc["session.delete"]({ sessionId: doomed.session.id })); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + const { storeId } = await synced(appHome, machineId, server); + + const rows = dump(appHome, "session", "thread", "entry"); + const doomedRow = rows.session!.find((row) => row.id === doomed.session.id)!; + expect(doomedRow.deleted_at).not.toBeNull(); + expect(rows.session!.find((row) => row.id === kept.session.id)?.deleted_at).toBeNull(); + expect(rows.thread!.find((row) => row.id === doomed.id)?.deleted_at).not.toBeNull(); + expect(rows.entry!.filter((row) => row.thread_id === doomed.id)).toHaveLength(2); + const changed = notices(appHome, Contract.syncTopics.changed, restartedAt); + expect(changed.at(-1)).toMatchObject({ removed: [doomed.session.id] }); + + await snapshot( + "removed", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )({ + changed: changed, + ...rows, + }), + ); +}, 90_000); + +it("an opened thread keeps its detail: first page with a floor, resume after close, older pages below the floor", async () => { + const cwHome = tempDir("detail-cw"); + const appHome = tempDir("detail-app"); + const server = await codework(cwHome, await freePort()); + // Two large prompts: the first page, about 1 MiB, leaves the oldest out. + const big = (tag: string) => `${tag}\n${"x".repeat(300_000)}`; + const created = await session(server, cwHome, big("First")); + const sessionId = created.session.id; + const threadId = created.id; + await server.run((rpc) => + Effect.gen(function* () { + for (const text of [big("Second"), "Third prompt"]) { + yield* rpc["thread.prompt"]({ sessionId, text }); + yield* rpc["thread.wait"]({ sessionId }); + } + }), + ); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const ref = { sessionId, threadId }; + const threadRow = () => + query(appHome, "SELECT cursor, floor FROM thread WHERE id = ?", threadId)[0] as { + cursor: number | null; + floor: number | null; + }; + const entryIds = () => query(appHome, "SELECT id FROM entry ORDER BY seq").map((row) => row.id as string); + + const openedAt = lastSeq(appHome); + await desk.run((rpc) => rpc["sync.open"](ref)); + const opened = { thread: threadRow(), entries: entryIds() }; + const line = (await server.run((rpc) => rpc["thread.history"]({ sessionId }))).entries; + // The newest entries within 1 MiB, the newest always: what CodeWork's first page holds. + const within = (entries: typeof line) => { + let total = 0; + return entries.toReversed().filter((entry, index) => (total += entry.size) <= 1 << 20 || index === 0); + }; + const firstPage = within(line).toReversed(); + expect(firstPage.length).toBeLessThan(line.length); + expect(opened.thread.cursor).not.toBeNull(); + expect(opened.entries).toEqual(firstPage.map((entry) => entry.id)); + expect(opened.thread.floor).toBe(firstPage[0]?.seq); + const first = notices(appHome, Contract.syncTopics.changed, openedAt).filter( + (notice) => notice.threadId === threadId, + ); + + await desk.run((rpc) => rpc["sync.close"](ref)); + await server.run((rpc) => + Effect.gen(function* () { + yield* rpc["thread.prompt"]({ sessionId, text: "While closed" }); + yield* rpc["thread.wait"]({ sessionId }); + }), + ); + // Closed: the stream is off, so nothing of the new prompt arrives. + await synced(appHome, machineId, server); + expect(entryIds()).toEqual(opened.entries); + const reopenedAt = lastSeq(appHome); + await desk.run((rpc) => rpc["sync.open"](ref)); + const resumed = notices(appHome, Contract.syncTopics.changed, reopenedAt).filter( + (notice) => notice.threadId === threadId, + ); + const reopened = { thread: threadRow(), entries: entryIds() }; + expect(resumed).toHaveLength(1); + expect(resumed[0]!.entry).toHaveLength(2); + expect(reopened.entries).toEqual([...opened.entries, ...(resumed[0]!.entry as Array)]); + + const pagedAt = lastSeq(appHome); + const page = await desk.run((rpc) => rpc["sync.page"]({ ...ref, limit: 10 })); + const paged = { thread: threadRow(), entries: entryIds() }; + const below = within(line.filter((entry) => entry.seq < firstPage[0]!.seq).slice(-10)).toReversed(); + expect(page).toEqual({ entries: below.length, floor: below[0]?.seq }); + expect(below[0]?.id).toBe(line[0]?.id); + expect(paged.thread.floor).toBe(line[0]?.seq); + expect(paged.entries.slice(0, below.length)).toEqual(below.map((entry) => entry.id)); + const pagedNotices = notices(appHome, Contract.syncTopics.changed, pagedAt); + await desk.run((rpc) => rpc["sync.close"](ref)); + + const { storeId } = await synced(appHome, machineId, server); + await snapshot( + "detail", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )({ + opened: { ...opened, changed: first }, + reopened: { ...reopened, changed: resumed }, + paged: { ...paged, page, changed: pagedNotices }, + ...dump(appHome, "entry", "part"), + }), + ); +}, 120_000); + +it("a follow-up queued behind a reply is a pending inbox row until it is promoted", async () => { + const cwHome = tempDir("inbox-cw"); + const appHome = tempDir("inbox-app"); + const server = await codework(cwHome, await freePort()); + const created = await server.run((rpc) => rpc["session.create"]({ directory: cwHome })); + const ref = { sessionId: created.session.id, threadId: created.id }; + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + await desk.run((rpc) => rpc["sync.open"](ref)); + + const queue = () => server.run((rpc) => rpc["thread.queue"](ref)).then((queued) => queued.entries); + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "gate" })); + // The held reply has taken its own prompt, so the queue holds only the follow-up. + await waitFor(async () => ((await queue()).length === 0 ? true : undefined), "the held reply"); + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "Queued", delivery: "followUp" })); + const [prompt] = await queue(); + expect(prompt).toBeDefined(); + const pending = await waitFor( + () => query(appHome, "SELECT * FROM inbox WHERE id = ?", prompt!.id)[0], + "the pending inbox row", + ); + expect(pending).toEqual({ + id: prompt!.id, + thread_id: ref.threadId, + delivery: "followUp", + text: "Queued", + admitted_seq: prompt!.admittedSeq, + created_at: prompt!.createdAt, + }); + + for (let step = 0; step < 4; step++) server.release(); + await server.run((rpc) => rpc["thread.wait"](ref)); + const promoted = () => query(appHome, "SELECT id FROM entry WHERE id = ?", prompt!.id).length === 1; + await waitFor( + () => (query(appHome, tables.inbox).length === 0 && promoted() ? true : undefined), + "the promoted follow-up", + ); + + const { storeId } = await synced(appHome, machineId, server); + await snapshot( + "inbox", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )({ + pending: [pending], + promoted: dump(appHome, "inbox", "entry"), + }), + ); +}, 90_000); + +it("a restored store is read again in full: returning rows keep their desktop columns, the rest are soft-deleted", async () => { + const cwHome = tempDir("restored-cw"); + const appHome = tempDir("restored-app"); + const backup = tempDir("restored-backup"); + const port = await freePort(); + let server = await codework(cwHome, port); + const before = await session(server, cwHome, "Before backup"); + // A clean stop leaves the database whole in its main file. + await stop(server.proc, "SIGTERM"); + copyFileSync(server.database, join(backup, "codework.db")); + server = await codework(cwHome, port); + + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + write(appHome, "UPDATE thread SET priority = 2, color = '#ff8800', status = 'attention' WHERE id = ?", before.id); + const after = await session(server, cwHome, "After backup"); + const { rev: head } = await synced(appHome, machineId, server); + + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + copyFileSync(server.database, join(backup, "later.db")); + for (const suffix of ["-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + copyFileSync(join(backup, "codework.db"), server.database); + server = await codework(cwHome, port); + const restored = await server.run((rpc) => rpc["server.info"]({})); + expect(restored.rev).toBeLessThan(head); + + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await waitFor(() => { + const row = machineRow(appHome, machineId); + return row.cursor === restored.rev ? row : undefined; + }, "the full sync of the restored store"); + const rows = dump(appHome, "machine", "session", "thread"); + const changed = notices(appHome, Contract.syncTopics.changed, restartedAt); + expect(changed[0]).toMatchObject({ machineId, full: true, removed: [after.session.id] }); + expect(rows.thread!.find((row) => row.id === before.id)).toMatchObject({ + priority: 2, + color: "#ff8800", + status: "attention", + deleted_at: null, + }); + expect(rows.session!.find((row) => row.id === after.session.id)?.deleted_at).not.toBeNull(); + expect(rows.thread!.find((row) => row.id === after.id)?.deleted_at).not.toBeNull(); + + // Restored again, to the later copy that has the soft-deleted session: past the cursor, so a resume brings it back. + write(appHome, "UPDATE session SET color = '#0088ff' WHERE id = ?", after.session.id); + write(appHome, "UPDATE thread SET priority = 3, color = '#0088ff' WHERE id = ?", after.id); + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + for (const suffix of ["-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + copyFileSync(join(backup, "later.db"), server.database); + server = await codework(cwHome, port); + const later = await server.run((rpc) => rpc["server.info"]({})); + expect(later.storeId).toBe(restored.storeId); + expect(later.rev).toBeGreaterThan(restored.rev); + const againAt = lastSeq(appHome); + desk = await app(appHome); + await synced(appHome, machineId, server); + const again = dump(appHome, "session", "thread"); + const changedAgain = notices(appHome, Contract.syncTopics.changed, againAt); + expect(changedAgain.every((notice) => notice.full === false)).toBe(true); + expect(again.session!.find((row) => row.id === after.session.id)).toMatchObject({ + color: "#0088ff", + deleted_at: null, + }); + expect(again.thread!.find((row) => row.id === after.id)).toMatchObject({ + priority: 3, + color: "#0088ff", + deleted_at: null, + }); + + await snapshot( + "restored", + normalizer( + [cwHome, appHome], + [[restored.storeId, ""]], + )({ + heads: [{ rev: head }, { rev: restored.rev }], + changed: changed, + ...rows, + again: { head: { rev: later.rev }, changed: changedAgain, ...again }, + }), + ); +}, 120_000); + +it("a new store in the same home is read again in full", async () => { + const cwHome = tempDir("store-cw"); + const appHome = tempDir("store-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const old = await session(server, cwHome, "Old store"); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const first = machineRow(appHome, machineId); + + // The app stays up: it reconnects to the new store on its own. + await stop(server.proc, "SIGTERM"); + for (const suffix of ["", "-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + server = await codework(cwHome, port); + const fresh = await session(server, cwHome, "New store"); + const startedAt = lastSeq(appHome); + const { storeId } = await synced(appHome, machineId, server); + expect(existsSync(server.database)).toBe(true); + expect(storeId).not.toBe(first.store_id); + + const rows = dump(appHome, "machine", "session", "thread"); + const changed = notices(appHome, Contract.syncTopics.changed, startedAt); + expect(changed.find((notice) => notice.full === true)).toMatchObject({ removed: [old.session.id] }); + expect(rows.session!.find((row) => row.id === old.session.id)?.deleted_at).not.toBeNull(); + expect(rows.session!.find((row) => row.id === fresh.session.id)?.deleted_at).toBeNull(); + + await snapshot( + "store", + normalizer( + [cwHome, appHome], + [ + [first.store_id!, ""], + [storeId, ""], + ], + )({ changed: changed, ...rows }), + ); +}, 120_000); + +it("a live reply folds deltas that continue the draft, drops one that does not, and lands its checkpoint", async () => { + const cwHome = tempDir("live-cw"); + const appHome = tempDir("live-app"); + const server = await codework(cwHome, await freePort()); + const created = await server.run((rpc) => rpc["session.create"]({ directory: cwHome })); + const ref = { sessionId: created.session.id, threadId: created.id }; + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + await desk.run((rpc) => rpc["sync.open"](ref)); + + const forwarded: Array = []; + let subscribed = false; + const follower = desk.background((rpc) => + rpc["event.subscribe"]({ topics: [Contract.syncTopics.delta(ref.threadId)] }).pipe( + Stream.runForEach((item) => + Effect.sync(() => { + if ("_tag" in item) subscribed = true; + else forwarded.push(item.payload as Row); + }), + ), + ), + ); + await waitFor(() => (subscribed ? true : undefined), "the delta subscription"); + + const draftText = () => + query( + appHome, + `SELECT entry.state, json_extract(part.data, '$.text') AS text FROM entry + JOIN part ON part.entry_id = entry.id WHERE entry.thread_id = ? AND entry.type = 'assistant'`, + ref.threadId, + )[0] as { state: string; text: string } | undefined; + + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "gate" })); + await waitFor(() => (forwarded.length > 0 ? forwarded : undefined), "the first delta"); + // The checkpoint, at most a second later, holds the first chunk. + await waitFor(() => (draftText()?.text === "Hello " ? true : undefined), "the first checkpoint"); + + // Closed while "wor" streams, reopened before the next checkpoint: "ld" no longer continues the draft. + await desk.run((rpc) => rpc["sync.close"](ref)); + server.release(); + await desk.run((rpc) => rpc["sync.open"](ref)); + server.release(); + await waitFor(() => (draftText()?.text === "Hello world" ? true : undefined), "the second checkpoint"); + server.release(); + await waitFor(() => (forwarded.length >= 2 ? true : undefined), "the delta after the checkpoint"); + // The reply ends only now, so the committed entry cannot overtake that delta. + server.release(); + await server.run((rpc) => rpc["thread.wait"](ref)); + const landed = await waitFor( + () => (draftText()?.state === "committed" ? draftText() : undefined), + "the committed reply", + ); + follower.stop(); + + expect(forwarded.map((delta) => [delta.offset, delta.delta])).toEqual([ + [0, "Hello "], + [11, "!"], + ]); + expect(landed).toEqual({ state: "committed", text: "Hello world!" }); + + const { storeId } = await synced(appHome, machineId, server); + await snapshot( + "live", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )({ + forwarded, + ...dump(appHome, "entry", "part"), + }), + ); +}, 90_000); + +it("thread status follows its runs: started, succeeded, user interrupt, crash sweep", async () => { + const cwHome = tempDir("status-cw"); + const appHome = tempDir("status-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const created = await server.run((rpc) => rpc["session.create"]({ directory: cwHome })); + const ref = { sessionId: created.session.id, threadId: created.id }; + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const status = () => + query(appHome, "SELECT status, run_outcome, run_error FROM thread WHERE id = ?", ref.threadId)[0]!; + const reach = (value: string) => + waitFor(() => (status().status === value ? status() : undefined), value).catch((error: Error) => { + throw new Error(`${error.message}; the thread is ${JSON.stringify(status())}`); + }); + const steps: Array = [{ step: "first seen", ...status() }]; + + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "hold" })); + steps.push({ step: "run started", ...(await reach("working")) }); + await server.run((rpc) => rpc["thread.interrupt"](ref)); + steps.push({ step: "user interrupt", ...(await reach("cancelled")) }); + // Held after its first chunk, so its start and its end arrive as two rows. + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "gate" })); + steps.push({ step: "run started again", ...(await reach("working")) }); + for (let step = 0; step < 4; step++) server.release(); + await server.run((rpc) => rpc["thread.wait"](ref)); + steps.push({ step: "succeeded", ...(await reach("attention")) }); + + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "hold" })); + await reach("working"); + await synced(appHome, machineId, server); + const run = () => + query(appHome, "SELECT run_state, run_started_at FROM thread WHERE id = ?", ref.threadId)[0] as { + run_state: string; + run_started_at: number; + }; + const crashed = run(); + const crashedAt = lastSeq(appHome); + await stop(server.proc); + server = await codework(cwHome, port); + // The sweep ends the run as crashed and recovery resumes it at once: the row arrives with the resumed run. + await waitFor(() => { + const now = run(); + return now.run_started_at !== crashed.run_started_at && now.run_state === "running" ? now : undefined; + }, "the resumed run"); + steps.push({ step: "crash sweep and resume", ...status() }); + const sinceCrash = notices(appHome, Contract.syncTopics.status, crashedAt); + expect(sinceCrash).toEqual([]); + expect(status().status).toBe("working"); + + const moves = notices(appHome, Contract.syncTopics.status); + expect(moves.map((move) => [move.previous, move.status])).toEqual([ + ["settled", "working"], + ["working", "cancelled"], + ["cancelled", "working"], + ["working", "attention"], + ["attention", "working"], + ]); + await server.run((rpc) => rpc["thread.interrupt"](ref)); + + const { storeId } = await synced(appHome, machineId, server); + await snapshot( + "status", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )({ + steps, + moves: moves, + }), + ); +}, 120_000); diff --git a/apps/server/test/tracking.test.ts b/apps/server/test/tracking.test.ts new file mode 100644 index 00000000..42d1b2df --- /dev/null +++ b/apps/server/test/tracking.test.ts @@ -0,0 +1,305 @@ +import { Effect, Stream } from "effect"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { + addMachine, + app, + type AppRpc, + cleanup, + type Codework, + codework, + freePort, + lastSeq, + normalizer, + query, + type Row, + snapshot as match, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`tracking.${name}`, artifact); +const TOPICS = [Contract.syncTopics.status, Contract.syncTopics.tracking]; + +/** The stored tracking and status events after `after`, as the bus holds them. */ +const stored = (home: string, after = 0) => + query( + home, + "SELECT topic, source, payload FROM event WHERE topic IN (?, ?) AND seq > ? ORDER BY seq", + ...TOPICS, + after, + ).map((row) => ({ + topic: row.topic as string, + source: row.source as string, + payload: JSON.parse(row.payload as string) as Row, + })); + +/** A CodeWork server with `count` new sessions, synced into a fresh app server. */ +const setup = async (name: string, count: number) => { + const cwHome = tempDir(`${name}-cw`); + const appHome = tempDir(`${name}-app`); + const server = await codework(cwHome, await freePort()); + const sessions = []; + for (let index = 0; index < count; index++) { + const created = await server.run((rpc) => rpc["session.create"]({ directory: cwHome })); + sessions.push({ sessionId: created.session.id, threadId: created.id }); + } + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + return { cwHome, appHome, server, desk, machineId, sessions }; +}; + +/** A second window: another connection following the tracking topics from now on. */ +const follow = async (desk: Awaited>) => { + const received: Array = []; + let subscribed = false; + const window = desk.background((rpc) => + rpc["event.subscribe"]({ topics: TOPICS }).pipe( + Stream.runForEach((item) => + Effect.sync(() => { + if ("_tag" in item) subscribed = true; + else received.push({ topic: item.topic, source: item.source, payload: item.payload as Row }); + }), + ), + ), + ); + await waitFor(() => (subscribed ? true : undefined), "the second window's subscription"); + return { received, stop: window.stop }; +}; + +const threadRow = (home: string, threadId: string) => + query(home, "SELECT status, priority, grouping_id, ordering, color FROM thread WHERE id = ?", threadId)[0]!; + +it("every edit changes its row with its event in the same transaction, and a second window receives it", async () => { + const { cwHome, appHome, desk, sessions } = await setup("edits", 1); + const { sessionId, threadId } = sessions[0]!; + const window = await follow(desk); + const sessionColor = () => query(appHome, "SELECT color FROM session WHERE id = ?", sessionId)[0]!.color; + const groups = () => + query(appHome, "SELECT id, organization_id, name, color, ordering FROM grouping ORDER BY ordering"); + + const steps: Array = []; + /** Runs one edit; once it returns, its row and exactly one event are there together. */ + const edit = async (step: string, call: Parameters[0]) => { + const before = lastSeq(appHome); + await desk.run(call); + const written = stored(appHome, before); + expect(written).toHaveLength(1); + steps.push({ + step, + thread: threadRow(appHome, threadId), + session: { color: sessionColor() }, + grouping: groups(), + event: written[0], + }); + return written[0]!; + }; + + await edit("status cancelled", (rpc) => rpc["threads.setStatus"]({ threadId, status: "cancelled" })); + await edit("priority urgent", (rpc) => rpc["threads.setPriority"]({ threadId, priority: 1 })); + await edit("thread color", (rpc) => rpc["threads.setColor"]({ threadId, color: "#ff8800" })); + await edit("session color", (rpc) => rpc["sessions.setColor"]({ sessionId, color: "#00aaff" })); + const created = await edit("group created", (rpc) => + rpc["groups.create"]({ organizationId: "personal", name: "Backlog", color: "#123456" }), + ); + const groupingId = created.payload.id as string; + await edit("group renamed and color cleared", (rpc) => + rpc["groups.update"]({ groupingId, name: "Next", color: null }), + ); + await edit("thread grouped", (rpc) => rpc["threads.setGroup"]({ threadId, groupingId })); + await edit("thread ungrouped", (rpc) => rpc["threads.setGroup"]({ threadId, groupingId: null })); + await edit("thread color cleared", (rpc) => rpc["threads.setColor"]({ threadId, color: null })); + await edit("session color cleared", (rpc) => rpc["sessions.setColor"]({ sessionId, color: null })); + + expect(threadRow(appHome, threadId)).toEqual({ + status: "cancelled", + priority: 1, + grouping_id: null, + ordering: 0, + color: null, + }); + const all = stored(appHome); + await waitFor(() => (window.received.length >= all.length ? true : undefined), "the second window's events"); + window.stop(); + expect(window.received).toEqual(all); + + // Refused edits: typed errors, and neither the row nor the bus changes. + const beforeRejected = { seq: lastSeq(appHome), thread: threadRow(appHome, threadId) }; + const refuse = ( + call: (rpc: AppRpc) => Effect.Effect, + ) => desk.run((rpc) => Effect.flip(call(rpc)).pipe(Effect.map(({ _tag, message }) => ({ _tag, message })))); + const rejected = { + "priority 5": await refuse((rpc) => rpc["threads.setPriority"]({ threadId, priority: 5 })), + "priority -1": await refuse((rpc) => rpc["threads.setPriority"]({ threadId, priority: -1 })), + "thread color orange": await refuse((rpc) => rpc["threads.setColor"]({ threadId, color: "orange" })), + "session color #12345": await refuse((rpc) => rpc["sessions.setColor"]({ sessionId, color: "#12345" })), + "group color #ggg000": await refuse((rpc) => + rpc["groups.create"]({ organizationId: "personal", name: "Bad", color: "#ggg000" }), + ), + "missing thread": await refuse((rpc) => rpc["threads.setPriority"]({ threadId: "thr_missing", priority: 2 })), + "missing group": await refuse((rpc) => rpc["threads.setGroup"]({ threadId, groupingId: "missing" })), + }; + for (const error of Object.values(rejected)) expect(error._tag).toBe("Server.TrackingError"); + expect(lastSeq(appHome)).toBe(beforeRejected.seq); + expect(threadRow(appHome, threadId)).toEqual(beforeRejected.thread); + + const normalize = normalizer( + [cwHome, appHome], + [ + [groupingId, ""], + [threadId, ""], + [sessionId, ""], + ], + ); + await snapshot("edits", normalize({ steps, received: window.received })); + await snapshot("rejected", normalize({ rejected, thread: threadRow(appHome, threadId) })); +}, 90_000); + +it("a user-set status stays through rows without a run change, then the next run's transitions win", async () => { + const { cwHome, appHome, server, desk, sessions } = await setup("status", 1); + const ref = sessions[0]!; + const status = () => query(appHome, "SELECT status, thinking_level FROM thread WHERE id = ?", ref.threadId)[0]!; + const reach = (value: string) => waitFor(() => (status().status === value ? status() : undefined), value); + /** A reply held after its first chunk, so its start and its end arrive as two rows. */ + const run = async (server: Codework) => { + await server.run((rpc) => rpc["thread.prompt"]({ ...ref, text: "gate" })); + await reach("working"); + for (let step = 0; step < 4; step++) server.release(); + await server.run((rpc) => rpc["thread.wait"](ref)); + return reach("attention"); + }; + const steps: Array = [{ step: "first seen", ...status() }]; + + steps.push({ step: "run succeeded", ...(await run(server)) }); + await desk.run((rpc) => rpc["threads.setStatus"]({ threadId: ref.threadId, status: "settled" })); + steps.push({ step: "user settled", ...status() }); + await server.run((rpc) => rpc["thread.configure"]({ ...ref, runtime: { thinkingLevel: "high" } })); + const configured = await waitFor( + () => (status().thinking_level === "high" ? status() : undefined), + "the configured thread row", + ); + expect(configured.status).toBe("settled"); + steps.push({ step: "row without a run change", ...configured }); + steps.push({ step: "next run succeeded", ...(await run(server)) }); + + const moves = stored(appHome).map((event) => ({ + source: event.source, + previous: event.payload.previous, + status: event.payload.status, + })); + expect(moves).toEqual([ + { source: "sync", previous: "settled", status: "working" }, + { source: "sync", previous: "working", status: "attention" }, + { source: "tracking", previous: "attention", status: "settled" }, + { source: "sync", previous: "settled", status: "working" }, + { source: "sync", previous: "working", status: "attention" }, + ]); + await snapshot("status", normalizer([cwHome, appHome])({ steps, moves })); +}, 90_000); + +it("moving a thread or a group first, between two, or last writes one row with the midpoint values", async () => { + const { cwHome, appHome, desk, sessions } = await setup("ordering", 4); + // Each new thread goes first: A 0, B -1, C -2, D -3. + const names = new Map(sessions.map((ref, index): [string, string] => [ref.threadId, "ABCD"[index]!])); + const ids = new Map([...names].map(([id, name]) => [name, id])); + const list = (table: "thread" | "grouping", label: (id: string) => string) => + query(appHome, `SELECT id, ordering FROM ${table} ORDER BY ordering`).map((row) => ({ + name: label(row.id as string), + ordering: row.ordering as number, + })); + + const groupNames = new Map(); + for (const name of ["G1", "G2", "G3"]) { + const group = await desk.run((rpc) => rpc["groups.create"]({ organizationId: "personal", name })); + groupNames.set(group.id, name); + } + const groupIds = new Map([...groupNames].map(([id, name]) => [name, id])); + + const moves: Array = []; + const move = async ( + table: "thread" | "grouping", + name: string, + to: Contract.Position, + call: Parameters[0], + ) => { + const label = (id: string) => (table === "thread" ? names : groupNames).get(id)!; + const before = list(table, label); + const seq = lastSeq(appHome); + await desk.run(call); + const after = list(table, label); + const changed = after.filter((row) => before.find((prior) => prior.name === row.name)?.ordering !== row.ordering); + expect(changed.map((row) => row.name)).toEqual([name]); + expect(stored(appHome, seq)).toHaveLength(1); + moves.push({ table, name, to, list: after, event: stored(appHome, seq)[0]!.payload }); + }; + const thread = (name: string, to: Contract.Position) => + move("thread", name, to, (rpc) => rpc["threads.move"]({ threadId: ids.get(name)!, to })); + const group = (name: string, to: Contract.Position) => + move("grouping", name, to, (rpc) => rpc["groups.move"]({ groupingId: groupIds.get(name)!, to })); + const between = (after: string, before: string, of: Map) => ({ + after: of.get(after)!, + before: of.get(before)!, + }); + + expect(list("thread", (id) => names.get(id)!).map((row) => row.ordering)).toEqual([-3, -2, -1, 0]); + await thread("A", "first"); + await thread("B", between("A", "D", ids)); + await thread("D", "last"); + expect(list("thread", (id) => names.get(id)!)).toEqual([ + { name: "A", ordering: -4 }, + { name: "B", ordering: -3.5 }, + { name: "C", ordering: -2 }, + { name: "D", ordering: -1 }, + ]); + + expect(list("grouping", (id) => groupNames.get(id)!).map((row) => row.ordering)).toEqual([0, 1, 2]); + await group("G3", "first"); + await group("G2", between("G3", "G1", groupIds)); + await group("G3", "last"); + expect(list("grouping", (id) => groupNames.get(id)!)).toEqual([ + { name: "G2", ordering: -0.5 }, + { name: "G1", ordering: 0 }, + { name: "G3", ordering: 1 }, + ]); + + const labels = [...names, ...groupNames].map(([id, name]): [string, string] => [id, `<${name}>`]); + await snapshot("ordering", normalizer([cwHome, appHome], labels)({ moves })); +}, 90_000); + +it("deleting a group leaves its threads ungrouped and names them in its event", async () => { + const { cwHome, appHome, desk, sessions } = await setup("delete", 3); + const [a, b, c] = sessions.map((ref) => ref.threadId as string) as [string, string, string]; + const doomed = await desk.run((rpc) => rpc["groups.create"]({ organizationId: "personal", name: "Doomed" })); + const kept = await desk.run((rpc) => rpc["groups.create"]({ organizationId: "personal", name: "Kept" })); + for (const threadId of [a, b]) await desk.run((rpc) => rpc["threads.setGroup"]({ threadId, groupingId: doomed.id })); + await desk.run((rpc) => rpc["threads.setGroup"]({ threadId: c, groupingId: kept.id })); + + const seq = lastSeq(appHome); + await desk.run((rpc) => rpc["groups.delete"]({ groupingId: doomed.id })); + const events = stored(appHome, seq); + const threads = query(appHome, "SELECT id, grouping_id FROM thread ORDER BY created_seq"); + const groups = query(appHome, "SELECT id, name FROM grouping ORDER BY ordering"); + + expect(events).toHaveLength(1); + expect(events[0]!.payload).toEqual({ _tag: "GroupDeleted", groupingId: doomed.id, threads: [a, b] }); + expect(threads).toEqual([ + { id: a, grouping_id: null }, + { id: b, grouping_id: null }, + { id: c, grouping_id: kept.id }, + ]); + expect(groups).toEqual([{ id: kept.id, name: "Kept" }]); + + const normalize = normalizer( + [cwHome, appHome], + [ + [a, ""], + [b, ""], + [c, ""], + [doomed.id, ""], + [kept.id, ""], + ], + ); + await snapshot("delete", normalize({ events, threads, groups })); +}, 90_000); diff --git a/apps/web/src/kernel/servers.ts b/apps/web/src/kernel/servers.ts index 1b006102..72ac4443 100644 --- a/apps/web/src/kernel/servers.ts +++ b/apps/web/src/kernel/servers.ts @@ -1,4 +1,4 @@ -import type { ServerStatus } from "@codeworksh/contract"; +import type { LocalServerStatus, ServerStatus } from "@codeworksh/contract"; import { Effect, Schedule, Stream } from "effect"; import { AsyncResult, Atom, Reactivity } from "effect/reactivity"; @@ -15,7 +15,7 @@ export interface Problem { } /** The default server's problem from its status; undefined while the app can work with it. */ -function problemOf(status: ServerStatus): Problem | undefined { +function problemOf(status: LocalServerStatus): Problem | undefined { const { cli, server } = status; // An already running, compatible server is usable even without a local CLI. if (server._tag === "Running") return undefined; @@ -48,6 +48,10 @@ function problemOf(status: ServerStatus): Problem | undefined { } } +// Only a local server can be the default. +const defaultOf = (statuses: ReadonlyArray) => + statuses.find((status): status is LocalServerStatus => status.kind === "local" && status.isDefault); + /** * Every server's status, as the app server last checked it, kept current as it checks again. When * the default server becomes usable or its home changes, its queries load again. @@ -58,7 +62,7 @@ export const serversAtom = Server.runtime let previous: { readonly home: string | undefined; readonly usable: boolean } | undefined; return Stream.unwrap(Server.use((client) => Effect.succeed(client("servers.watch", undefined)))).pipe( Stream.tap((statuses) => { - const current = statuses.find((status) => status.isDefault); + const current = defaultOf(statuses); const home = current?.home.path; const usable = current !== undefined && problemOf(current) === undefined; const refresh = usable && previous !== undefined && (!previous.usable || previous.home !== home); @@ -75,6 +79,6 @@ export const serversAtom = Server.runtime /** The default server's problem once its status is known; undefined until then, and while there is none. */ export const problemAtom = Atom.make((get) => { const result = get(serversAtom); - const status = AsyncResult.isSuccess(result) ? result.value.find((candidate) => candidate.isDefault) : undefined; + const status = AsyncResult.isSuccess(result) ? defaultOf(result.value) : undefined; return status === undefined ? undefined : problemOf(status); }); diff --git a/apps/web/src/landing/landing.tsx b/apps/web/src/landing/landing.tsx index bd4c756a..3d2600b2 100644 --- a/apps/web/src/landing/landing.tsx +++ b/apps/web/src/landing/landing.tsx @@ -1,4 +1,4 @@ -import { RegistryContext, useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { RegistryContext, useAtomValue } from "@effect/atom-react"; import { Option, Schema } from "effect"; import { Folder } from "lucide-react"; import { useContext, useMemo, useState } from "react"; @@ -7,6 +7,7 @@ import { NoticeSlot } from "@/components/banner"; import { Composer, ComposerSubmit } from "@/components/composer"; import { makeHost } from "../kernel/host"; import { sessionAddress } from "../widgets/session/address"; +import { useSent } from "../widgets/session/display"; import { handoffs } from "../widgets/session/handoff"; import { Json, type Place as Where, topics } from "../widgets/session/schema"; import { useOpenElsewhere } from "../wm/open"; @@ -15,7 +16,6 @@ import { Arcade } from "./arcade"; import { cloneBlock, useClone, useCloneNotice } from "../shell/clones"; import { useAddProject } from "../shell/palette"; import { placeAtom, placeLabel } from "./place"; -import { projectsAtom } from "../widgets/session/projects"; const encodeCreate = Schema.encodeSync(Json.Create); const decodeCreated = Schema.decodeUnknownOption(Json.Created); @@ -29,7 +29,7 @@ export function Landing() { // The start page asks the Session backend as the "landing" source, outside any widget frame. const bus = useMemo(() => makeHost({ registry, package: "codework", instanceId: "landing" }).bus, [registry]); const openElsewhere = useOpenElsewhere(); - const refreshProjects = useAtomRefresh(projectsAtom); + const inFlight = useSent(); const [query, setQuery] = useState(""); const [playing, setPlaying] = useState(false); const place = useAtomValue(placeAtom); @@ -70,8 +70,9 @@ export function Landing() { setError(created?.message ?? "The CodeWork server is not reachable."); return; } + // Pending under the main thread's key (its session's id) until its row lands in app.db. + inFlight.prompt(created.sessionId, prompt.id); handoffs.set(created.sessionId, prompt); - refreshProjects(); await openElsewhere(sessionAddress(created.sessionId), placeLabel(place), "replace"); }; diff --git a/apps/web/src/landing/sidebar.tsx b/apps/web/src/landing/sidebar.tsx index 2b339c6e..a744a941 100644 --- a/apps/web/src/landing/sidebar.tsx +++ b/apps/web/src/landing/sidebar.tsx @@ -3,7 +3,7 @@ import { PlusSignIcon, Search01Icon, Settings02Icon, SquarePenIcon } from "@huge import { HugeiconsIcon } from "@hugeicons/react"; import { Link, useNavigate, useParams } from "@tanstack/react-router"; import { AsyncResult } from "effect/reactivity"; -import type { Project } from "@codeworksh/contract"; +import type { SidebarProject } from "@codeworksh/contract"; import { type MouseEvent, memo, useCallback, useLayoutEffect, useMemo, useRef, useState } from "react"; import { @@ -16,7 +16,7 @@ import { } from "@/components/ui/sidebar"; import { isMac, openMode } from "@codeworksh/widget/view"; import { sessionAddress } from "../widgets/session/address"; -import { projectsAtom, workingAtom } from "../widgets/session/projects"; +import { projectsAtom } from "../widgets/session/projects"; import { focusedAtom } from "../wm/chrome"; import { deskAtom } from "../wm/instances"; import { useOpenElsewhere } from "../wm/open"; @@ -28,7 +28,8 @@ import { problemAtom } from "../kernel/servers"; import { Empty, EmptyAction } from "./empty"; import { useAddProject } from "../shell/palette"; import { placeAtom } from "./place"; -import { PixelLoader } from "@/components/loader"; +import { useRunDisplay } from "../widgets/session/display"; +import { RunMark } from "../widgets/session/run"; /** * Recent sessions across every project, newest first, beside every page. With none yet it invites @@ -37,7 +38,6 @@ import { PixelLoader } from "@/components/loader"; export function SessionsSidebar() { const result = useAtomValue(projectsAtom); const problem = useAtomValue(problemAtom); - const working = useAtomValue(workingAtom); const [query, setQuery] = useState(""); const openElsewhere = useOpenElsewhere(); const [place, setPlace] = useAtom(placeAtom); @@ -153,7 +153,6 @@ export function SessionsSidebar() { key={session.id} session={session} current={sessionAddress(session.id) === current} - working={working.has(session.id)} onOpen={open} /> ))} @@ -174,36 +173,34 @@ export function SessionsSidebar() { ); } -type Listed = Project["sessions"][number] & { readonly project: Project }; +type Listed = SidebarProject["sessions"][number] & { readonly project: SidebarProject }; /** Off-screen rows skip layout and paint until scrolled near; memoized so a loader or focus change repaints one row. */ const SessionRow = memo(function SessionRow({ session, current, - working, onOpen, }: { readonly session: Listed; readonly current: boolean; - readonly working: boolean; readonly onOpen: (session: Listed, event: MouseEvent) => void; }) { + // The main thread's key is its session's id. + const display = useRunDisplay([{ key: session.id, runState: session.runState }]); return (
  • + ))} + + {by === "grouping" && ( + + )} + + {AsyncResult.match(result, { + onInitial: () => null, + onFailure: () =>

    Could not load the dashboard.

    , + onSuccess: ({ value }) => ( + setRenaming(undefined)} + onRename={setRenaming} + /> + ), + })} + + ); +} + +type Drag = { readonly kind: "thread" | "group"; readonly id: string }; +type Drop = { readonly list: string; readonly index: number }; + +/** `list` with `thread` taken out of wherever it was and put at `index` of the list keyed `to`, as it will be. */ +const moved = (lists: ReadonlyArray, thread: DashboardThread, to: string, index: number) => + lists.map((list): DashboardList => { + const threads = list.threads.filter((candidate) => candidate.id !== thread.id); + if (listKey(list) !== to) return { ...list, threads }; + const placed = + list._tag === "Status" + ? { ...thread, status: list.status } + : list._tag === "Priority" + ? { ...thread, priority: list.priority } + : { ...thread, groupingId: list.groupingId }; + return { ...list, threads: threads.toSpliced(index, 0, placed) }; + }); + +/** The thread as the session view names it: its session alone for the main thread. */ +const refOf = (thread: DashboardThread): ThreadRef => + thread.originKind === "main" + ? { sessionId: thread.sessionId } + : { sessionId: thread.sessionId, threadId: thread.id }; + +function Lists({ + board, + renaming, + onRename, + onRenamed, +}: { + readonly board: Board; + readonly renaming: string | undefined; + readonly onRename: (groupingId: string) => void; + readonly onRenamed: () => void; +}) { + const shell = useShell(); + const status = useAtomSet(setStatus, { mode: "promise" }); + const priority = useAtomSet(setPriority, { mode: "promise" }); + const group = useAtomSet(setGroup, { mode: "promise" }); + const move = useAtomSet(moveThread, { mode: "promise" }); + const reorder = useAtomSet(moveGroup, { mode: "promise" }); + // Drawn at once while the writes go out, until the read after them arrives. + const [optimistic, setOptimistic] = useState<{ + readonly base: Board; + readonly lists: ReadonlyArray; + }>(); + const lists = optimistic?.base === board ? optimistic.lists : board.lists; + const [drag, setDrag] = useState(); + const [drop, setDrop] = useState(); + const groups = new Map(board.groups.map((candidate) => [candidate.id, candidate])); + const all = lists.flatMap((list) => list.threads).toSorted((a, b) => a.ordering - b.ordering); + + const open = (thread: DashboardThread, event: MouseEvent) => { + if (event.button <= 1) shell.open(threadAddress(refOf(thread)), { mode: openMode(event) }); + }; + + /** Puts the thread in the list keyed `to`: the setter of the column the board splits by. */ + const assign = (thread: DashboardThread, list: DashboardList) => + list._tag === "Status" + ? status({ payload: { threadId: thread.id, status: list.status } }) + : list._tag === "Priority" + ? priority({ payload: { threadId: thread.id, priority: list.priority } }) + : group({ payload: { threadId: thread.id, groupingId: list.groupingId } }); + + const dropThread = async (target: DashboardList, index: number) => { + const thread = all.find((candidate) => candidate.id === drag?.id); + if (thread === undefined) return; + const source = lists.find((list) => list.threads.some((candidate) => candidate.id === thread.id)); + const same = source !== undefined && listKey(source) === listKey(target); + if (same && source.threads.findIndex((candidate) => candidate.id === thread.id) === index) return; + setOptimistic({ base: board, lists: moved(lists, thread, listKey(target), index) }); + const to = threadPosition( + all.filter((candidate) => candidate.id !== thread.id), + target.threads.filter((candidate) => candidate.id !== thread.id), + index, + ); + try { + if (!same) await assign(thread, target); + if (to !== undefined) await move({ payload: { threadId: thread.id, to } }); + } catch (error) { + setOptimistic(undefined); + failed(error); + } + }; + + /** Drops the dragged group above or below the group whose header is under the pointer. */ + const dropGroup = (event: DragEvent, onto: string) => { + const dragged = drag?.id; + const others = board.groups.filter((candidate) => candidate.id !== dragged); + const rect = event.currentTarget.getBoundingClientRect(); + const index = + others.findIndex((candidate) => candidate.id === onto) + (event.clientY > rect.top + rect.height / 2 ? 1 : 0); + if ( + dragged === undefined || + dragged === onto || + board.groups.findIndex((candidate) => candidate.id === dragged) === index + ) + return; + void reorder({ payload: { groupingId: dragged, to: groupPosition(others, index) } }).catch(failed); + }; + + /** Where a drag over a row of `list` at `at` lands, as an index of the list without the dragged thread. */ + const over = (event: DragEvent, list: DashboardList, at: number) => { + if (drag?.kind !== "thread") return; + event.preventDefault(); + event.stopPropagation(); + const rect = event.currentTarget.getBoundingClientRect(); + const below = event.clientY > rect.top + rect.height / 2; + const from = list.threads.findIndex((candidate) => candidate.id === drag.id); + const index = at + (below ? 1 : 0); + const next = { list: listKey(list), index: from !== -1 && from < index ? index - 1 : index }; + if (drop?.list !== next.list || drop.index !== next.index) setDrop(next); + }; + + const end = () => { + setDrag(undefined); + setDrop(undefined); + }; + + return ( +
    + {lists.map((list) => { + const key = listKey(list); + // A group's own list: its header drags to reorder the groups. + const groupId = list._tag === "Grouping" ? list.groupingId : null; + const without = list.threads.filter((candidate) => candidate.id !== drag?.id); + return ( +
    { + if (drag?.kind !== "thread") return; + event.preventDefault(); + if (drop?.list !== key) setDrop({ list: key, index: without.length }); + }} + onDrop={(event) => { + event.preventDefault(); + if (drag?.kind === "thread" && drop?.list === key) void dropThread(list, drop.index); + end(); + }} + > + groupId !== null && setDrag({ kind: "group", id: groupId })} + onDragOver={(event) => { + // A thread over the header lands at the top of the list; its drop is the section's. + if (drag?.kind === "thread") { + event.preventDefault(); + event.stopPropagation(); + if (drop?.list !== key || drop.index !== 0) setDrop({ list: key, index: 0 }); + return; + } + if (drag?.kind !== "group" || groupId === null) return; + event.preventDefault(); + event.stopPropagation(); + }} + onDrop={(event) => { + if (drag?.kind !== "group" || groupId === null) return; + event.preventDefault(); + event.stopPropagation(); + dropGroup(event, groupId); + end(); + }} + /> +
      + {list.threads.map((thread, at) => { + const index = without.findIndex((candidate) => candidate.id === thread.id); + return ( + setDrag({ kind: "thread", id: thread.id })} + onDragOver={(event) => over(event, list, at)} + /> + ); + })} + {drop?.list === key && drop.index === without.length && ( +
    • + )} +
    +
    + ); + })} +
    + ); +} + +/** What a list is called and how it is marked. */ +const heading = (list: DashboardList, groups: ReadonlyMap): { name: string; mark: ReactNode } => { + if (list._tag === "Status") { + const { label, icon: Icon } = STATUS[list.status]; + return { name: label, mark: }; + } + if (list._tag === "Priority") { + const { label, icon: Icon } = PRIORITY[list.priority]; + return { name: label, mark: }; + } + const group = list.groupingId === null ? undefined : groups.get(list.groupingId); + return { name: group?.name ?? "No group", mark: }; +}; + +function Dot({ color }: { readonly color: string | null }) { + return ( + + ); +} + +function ListHeader({ + list, + groups, + renaming, + onRename, + onRenamed, + draggable, + onDragStart, + onDragOver, + onDrop, +}: { + readonly list: DashboardList; + readonly groups: ReadonlyMap; + readonly renaming: string | undefined; + readonly onRename: (groupingId: string) => void; + readonly onRenamed: () => void; + readonly draggable: boolean; + readonly onDragStart: () => void; + readonly onDragOver: (event: DragEvent) => void; + readonly onDrop: (event: DragEvent) => void; +}) { + const update = useAtomSet(updateGroup, { mode: "promise" }); + const remove = useAtomSet(deleteGroup, { mode: "promise" }); + const { name, mark } = heading(list, groups); + const group = list._tag === "Grouping" && list.groupingId !== null ? groups.get(list.groupingId) : undefined; + const rename = (value: string) => { + onRenamed(); + if (group !== undefined && value.trim() !== "" && value.trim() !== group.name) + void update({ payload: { groupingId: group.id, name: value } }).catch(failed); + }; + return ( +
    { + event.dataTransfer.effectAllowed = "move"; + onDragStart(); + }} + onDragOver={onDragOver} + onDrop={onDrop} + className="group/header sticky top-0 z-10 flex items-center gap-2 bg-frame px-3 pt-3 pb-1 text-xs font-medium" + > + {draggable && ( + + )} + {mark} + {group !== undefined && renaming === group.id ? ( + event.currentTarget.select()} + onBlur={(event) => rename(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") event.currentTarget.blur(); + if (event.key === "Escape") onRenamed(); + }} + className="min-w-0 rounded-sm bg-ink/5 px-1 outline-none focus-visible:ring-1 focus-visible:ring-ring" + /> + ) : ( + {name} + )} + {list.threads.length} + {group !== undefined && ( + + + + + + onRename(group.id)}>Rename + +
    + {[...COLORS, null].map((color) => ( + void update({ payload: { groupingId: group.id, color } }).catch(failed)} + className="grid size-6 min-h-0 place-items-center p-0" + > + + + ))} +
    + + void remove({ payload: { groupingId: group.id } }).catch(failed)} + > + Delete group + +     +     + )} +
    + ); +} + +function Row({ + thread, + groups, + dragging, + above, + onOpen, + onDragStart, + onDragOver, +}: { + readonly thread: DashboardThread; + readonly groups: ReadonlyArray; + readonly dragging: boolean; + /** The drop would land just above it. */ + readonly above: boolean; + readonly onOpen: (thread: DashboardThread, event: MouseEvent) => void; + readonly onDragStart: () => void; + readonly onDragOver: (event: DragEvent) => void; +}) { + const status = useAtomSet(setStatus, { mode: "promise" }); + const priority = useAtomSet(setPriority, { mode: "promise" }); + const group = useAtomSet(setGroup, { mode: "promise" }); + const display = useRunDisplay([{ key: threadKey(refOf(thread)), runState: thread.runState }]); + const grouping = groups.find((candidate) => candidate.id === thread.groupingId); + const PriorityIcon = PRIORITY[thread.priority].icon; + const StatusIcon = STATUS[thread.status].icon; + return ( +
  • { + event.dataTransfer.effectAllowed = "move"; + event.dataTransfer.setData("text/plain", thread.title); + onDragStart(); + }} + onDragOver={onDragOver} + data-thread={thread.id} + className={`relative ${dragging ? "opacity-40" : ""}`} + > + {above && } +
    + + } + value={String(thread.priority)} + options={PRIORITIES.map((value) => ({ value: String(value), label: PRIORITY[value].label }))} + onPick={(value) => + void priority({ payload: { threadId: thread.id, priority: Number(value) } }).catch(failed) + } + /> + + + + + + + {grouping !== undefined && {grouping.name}} + + } + value={thread.groupingId ?? "none"} + options={[ + ...groups.map((candidate) => ({ value: candidate.id, label: candidate.name })), + { value: "none", label: "No group" }, + ]} + onPick={(value) => + void group({ payload: { threadId: thread.id, groupingId: value === "none" ? null : value } }).catch( + failed, + ) + } + /> + } + value={thread.status} + options={STATUSES.map((value) => ({ value, label: STATUS[value].label }))} + onPick={(value) => { + const next = STATUSES.find((candidate) => candidate === value); + if (next !== undefined) void status({ payload: { threadId: thread.id, status: next } }).catch(failed); + }} + /> + + {ago(thread.updatedAt)} + +
    +
    • + ); +} + +/** A small menu button that picks one of `options`. */ +function Pick({ + label, + icon, + value, + options, + onPick, +}: { + readonly label: string; + readonly icon: ReactNode; + readonly value: string; + readonly options: ReadonlyArray<{ readonly value: string; readonly label: string }>; + readonly onPick: (value: string) => void; +}) { + return ( + + + {icon} + + + next !== value && onPick(next)}> + {options.map((option) => ( + + {option.label} + + ))} + + + + ); +} diff --git a/apps/web/src/widgets/dashboard/data.ts b/apps/web/src/widgets/dashboard/data.ts new file mode 100644 index 00000000..392ce254 --- /dev/null +++ b/apps/web/src/widgets/dashboard/data.ts @@ -0,0 +1,93 @@ +import { + type DashboardBy, + type DashboardList, + type DashboardThread, + type Grouping, + PERSONAL_ORGANIZATION, + type Position, + readTopics, + SyncChanged, + syncTopics, +} from "@codeworksh/contract"; +import type { Event } from "@codeworksh/widget/event"; +import { Effect, Option, Schedule, Schema, Stream } from "effect"; +import { Atom } from "effect/reactivity"; + +import { Server } from "../../kernel/rpc"; + +const decodeChanged = Schema.decodeUnknownOption(SyncChanged); + +/** Whether an event changes what the dashboard shows: a thread, its project, a removed session, or any desktop edit. */ +const touches = (event: Event) => + event.topic !== syncTopics.changed || + Option.exists( + decodeChanged(event.payload), + (notice) => notice.thread.length + notice.project.length + notice.removed.length > 0, + ); + +/** The organization's threads split by `by`: read when subscribed, and again after each change to them. */ +export const dashboardAtom = Atom.family((by: DashboardBy) => + Server.runtime.atom( + Stream.unwrap( + Server.use((client) => + Effect.succeed( + client("event.subscribe", { topics: readTopics.dashboard }).pipe( + // `Synced` first, once subscribed: the first read can miss no change. + Stream.filter((item) => "_tag" in item || touches(item)), + Stream.debounce("50 millis"), + Stream.mapEffect(() => client("dashboard.list", { organizationId: PERSONAL_ORGANIZATION, by })), + ), + ), + ), + ).pipe(Stream.retry(Schedule.spaced("1 second"))), + ), +); + +export const setStatus = Server.mutation("threads.setStatus"); +export const setPriority = Server.mutation("threads.setPriority"); +export const setGroup = Server.mutation("threads.setGroup"); +export const moveThread = Server.mutation("threads.move"); +export const createGroup = Server.mutation("groups.create"); +export const updateGroup = Server.mutation("groups.update"); +export const moveGroup = Server.mutation("groups.move"); +export const deleteGroup = Server.mutation("groups.delete"); + +/** What names a list, for drag and drop: its status, priority or group (`none` is ungrouped). */ +export const listKey = (list: DashboardList) => + list._tag === "Status" + ? list.status + : list._tag === "Priority" + ? String(list.priority) + : (list.groupingId ?? "none"); + +/** + * Where a thread goes to land at `index` of `list`, both without it. Every view splits one order, so + * at a list's edge the neighbour beyond is the next thread in the whole order (`all`, in `ordering`, + * without it too). An empty list needs no move. + */ +export const threadPosition = ( + all: ReadonlyArray, + list: ReadonlyArray, + index: number, +): Position | undefined => { + const after = list[index - 1]; + const before = list[index]; + if (after !== undefined && before !== undefined) return { after: after.id, before: before.id }; + if (before !== undefined) { + const beyond = all[all.findIndex((thread) => thread.id === before.id) - 1]; + return beyond === undefined ? "first" : { after: beyond.id, before: before.id }; + } + if (after !== undefined) { + const beyond = all[all.findIndex((thread) => thread.id === after.id) + 1]; + return beyond === undefined ? "last" : { after: after.id, before: beyond.id }; + } + return undefined; +}; + +/** Where a group goes to land at `index` of `groups`, without it. */ +export const groupPosition = (groups: ReadonlyArray, index: number): Position => { + const after = groups[index - 1]; + const before = groups[index]; + if (after !== undefined && before !== undefined) return { after: after.id, before: before.id }; + return before !== undefined ? "first" : "last"; +}; diff --git a/apps/web/src/widgets/dashboard/index.ts b/apps/web/src/widgets/dashboard/index.ts new file mode 100644 index 00000000..0261f3a3 --- /dev/null +++ b/apps/web/src/widgets/dashboard/index.ts @@ -0,0 +1,22 @@ +import { KanbanSquare } from "lucide-react"; +import { Schema } from "effect"; + +import { defineWidget } from "@codeworksh/widget/view"; +import { Dashboard } from "./dashboard"; + +/** What opens the Dashboard widget, from the command palette. */ +export const dashboardAddress = "codework://dashboard"; + +/** Every thread by status, priority or group, in its manual order; picking one opens it beside the board. */ +export const dashboard = defineWidget({ + kind: "codework:dashboard", + title: "Dashboard", + icon: KanbanSquare, + minWidth: 320, + minHeight: 200, + cardinality: "one", + priority: "builtin", + opens: [dashboardAddress], + props: Schema.Struct({}), + component: Dashboard, +}); diff --git a/apps/web/src/widgets/index.ts b/apps/web/src/widgets/index.ts index 06a15f93..b78bd28b 100644 --- a/apps/web/src/widgets/index.ts +++ b/apps/web/src/widgets/index.ts @@ -1,8 +1,9 @@ import type { Widget } from "@codeworksh/widget/view"; +import { dashboard } from "./dashboard"; import { projects } from "./projects"; import { session } from "./session"; /** Widgets that ship with the app. Registry-installed widgets join these by kind. */ -export const builtins: readonly Widget[] = [projects, session]; +export const builtins: readonly Widget[] = [projects, dashboard, session]; export const registry: ReadonlyMap = new Map(builtins.map((widget) => [widget.kind, widget])); diff --git a/apps/web/src/widgets/projects/projects.tsx b/apps/web/src/widgets/projects/projects.tsx index 46c720c1..517e8488 100644 --- a/apps/web/src/widgets/projects/projects.tsx +++ b/apps/web/src/widgets/projects/projects.tsx @@ -23,7 +23,7 @@ export function Projects() { onSuccess: ({ value: projects }) => (