diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..54a6852 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +# Cancel superseded runs on the same ref. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check: + name: typecheck · lint · test · build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Reads the pnpm version from the root package.json "packageManager" field. + - name: Set up pnpm + uses: pnpm/action-setup@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Persist the Turborepo local cache across runs to skip unchanged tasks. + - name: Cache Turborepo + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ runner.os }}-${{ github.sha }} + restore-keys: | + turbo-${{ runner.os }}- + + - name: Typecheck + run: pnpm typecheck + + - name: Lint + run: pnpm lint + + - name: Test + run: pnpm test + + - name: Build + run: pnpm build diff --git a/README.md b/README.md index 9bc2a5a..dc82ba1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ **An autonomous engineering-management & delivery tool for a single cross-functional engineering org unit** — a persistent org of role-based AI agents that takes work from a one-line epic all the way to a shipped change, with the human as the accountable lead at the approval gates. +[![CI](https://github.com/tobyvee/engineering/actions/workflows/ci.yml/badge.svg)](https://github.com/tobyvee/engineering/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE) ![TypeScript](https://img.shields.io/badge/TypeScript-strict%2C%20ESM-3178c6) ![Temporal](https://img.shields.io/badge/durable-Temporal-000) @@ -121,6 +122,16 @@ pnpm db:generate # drizzle-kit generate pnpm db:migrate # drizzle-kit migrate ``` +## Continuous integration + +[`.github/workflows/ci.yml`](./.github/workflows/ci.yml) runs `typecheck` · `lint` · `test` · `build` +on every pull request to `main` (and on pushes to `main`), using the Node version pinned in `.nvmrc` +with a frozen lockfile and pnpm + Turborepo caching. Unit tests need no live services. + +To make a failing check **block merges**, enable branch protection on `main` +(GitHub → Settings → Branches → Add rule) and mark the **`typecheck · lint · test · build`** status +check as required. This is a one-time repo setting and can't be committed from the codebase. + ## Key invariants 1. **Goal traceability** is enforced in `core` — no work item exists without a parent goal; the chain is injected into every agent's context. diff --git a/apps/server/package.json b/apps/server/package.json index af7ac3a..a396c43 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -22,6 +22,7 @@ "hono": "^4.12.26" }, "devDependencies": { + "@temporalio/testing": "^1.18.1", "tsx": "^4.22.4", "typescript": "^6.0.3" } diff --git a/apps/server/src/temporal/workflows.test.ts b/apps/server/src/temporal/workflows.test.ts new file mode 100644 index 0000000..9ea58c0 --- /dev/null +++ b/apps/server/src/temporal/workflows.test.ts @@ -0,0 +1,223 @@ +import { fileURLToPath } from "node:url" +import { TestWorkflowEnvironment } from "@temporalio/testing" +import { bundleWorkflowCode, Worker, type WorkflowBundle } from "@temporalio/worker" +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest" +import { + approveSignal, + epicDecomposition, + epicShaping, + heartbeat, + roadmapSignal, + ticketLifecycle, +} from "./workflows" + +/** + * ENG-002 — exercises the durable orchestration layer with Temporal's time-skipping test + * environment and mocked activities. Time-skipping fast-forwards the CI/deploy poll `sleep`s; the + * approval gates (Signals) are driven explicitly. All GitHub/DB/agent I/O is mocked, so these run + * with no live services. The workflow code is bundled once (`beforeAll`) and reused per test. + */ + +const TQ = "test" +const workflowsPath = fileURLToPath(new URL("./workflows.ts", import.meta.url)) + +let env: TestWorkflowEnvironment +let bundle: WorkflowBundle +let wfId = 0 +const nextId = (p: string) => `${p}-${++wfId}` + +beforeAll(async () => { + env = await TestWorkflowEnvironment.createTimeSkipping() + bundle = await bundleWorkflowCode({ workflowsPath }) +}, 120_000) + +afterAll(async () => { + await env?.teardown() +}) + +/** Happy-path activity mocks; pass overrides to force failure branches. */ +function makeActivities(overrides: Record = {}) { + return { + transitionTicket: vi.fn(async (_id: string, _status: string) => {}), + implementTicket: vi.fn(async (_id: string) => ({ + number: 1, + url: "http://pr/1", + branch: "ticket/t", + })), + verifyTicket: vi.fn(async (_id: string) => true), + checkDeliveryStatus: vi.fn(async (_pr: unknown) => "success"), + mergeDelivery: vi.fn(async (_id: string, _pr: unknown) => true), + startDeploy: vi.fn(async (_id: string) => 42), + checkDeployStatus: vi.fn(async (_runId: number) => "success"), + recordDeploy: vi.fn(async (_id: string, _state: string) => {}), + pickUpBacklog: vi.fn(async () => 0), + decomposeEpic: vi.fn(async (_epicId: string) => ["ticket-1"]), + runShapingStage: vi.fn(async (_epicId: string, _stageKey: string) => {}), + requestRoadmapSignoff: vi.fn(async (_epicId: string) => {}), + recordRoadmapApproval: vi.fn(async (_epicId: string) => {}), + ...overrides, + } +} + +type Activities = ReturnType + +/** Run `cb` with a worker bound to `activities` for the duration, then shut it down. */ +async function withWorker(activities: Activities, cb: () => Promise): Promise { + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: TQ, + workflowBundle: bundle, + activities, + }) + return worker.runUntil(cb) +} + +describe("ticketLifecycle", () => { + it("reaches done when both gates are approved", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(ticketLifecycle, { + taskQueue: TQ, + workflowId: nextId("ticket"), + args: ["t1"], + }) + await handle.signal(approveSignal, "merge") + await handle.signal(approveSignal, "deploy") + await handle.result() + }) + + expect(acts.mergeDelivery).toHaveBeenCalledTimes(1) + expect(acts.recordDeploy).toHaveBeenCalledWith("t1", "success") + expect(acts.transitionTicket).toHaveBeenCalledWith("t1", "done") + expect(acts.transitionTicket).not.toHaveBeenCalledWith("t1", "blocked") + }, 30_000) + + it("blocks on the merge gate until approved", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(ticketLifecycle, { + taskQueue: TQ, + workflowId: nextId("ticket"), + args: ["t1"], + }) + // Let the workflow run to the merge gate; with no signal it stays blocked (no timer to skip). + await env.sleep("1 day") + expect(acts.verifyTicket).toHaveBeenCalledTimes(1) + expect(acts.mergeDelivery).not.toHaveBeenCalled() + + await handle.signal(approveSignal, "merge") + await handle.signal(approveSignal, "deploy") + await handle.result() + }) + + expect(acts.transitionTicket).toHaveBeenCalledWith("t1", "done") + }, 30_000) + + it("blocks the ticket when QA fails (before any gate)", async () => { + const acts = makeActivities({ verifyTicket: vi.fn(async (_id: string) => false) }) + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(ticketLifecycle, { + taskQueue: TQ, + workflowId: nextId("ticket"), + args: ["t1"], + }) + await handle.result() // returns without any signal — QA fail short-circuits + }) + + expect(acts.transitionTicket).toHaveBeenCalledWith("t1", "blocked") + expect(acts.mergeDelivery).not.toHaveBeenCalled() + expect(acts.transitionTicket).not.toHaveBeenCalledWith("t1", "done") + }, 30_000) + + it("blocks the ticket when the merge fails", async () => { + const acts = makeActivities({ + mergeDelivery: vi.fn(async (_id: string, _pr: unknown) => false), + }) + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(ticketLifecycle, { + taskQueue: TQ, + workflowId: nextId("ticket"), + args: ["t1"], + }) + await handle.signal(approveSignal, "merge") + await handle.result() + }) + + expect(acts.transitionTicket).toHaveBeenCalledWith("t1", "blocked") + expect(acts.startDeploy).not.toHaveBeenCalled() + expect(acts.transitionTicket).not.toHaveBeenCalledWith("t1", "done") + }, 30_000) + + it("blocks the ticket when the deploy fails", async () => { + const acts = makeActivities({ + checkDeployStatus: vi.fn(async (_runId: number) => "failure"), + }) + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(ticketLifecycle, { + taskQueue: TQ, + workflowId: nextId("ticket"), + args: ["t1"], + }) + await handle.signal(approveSignal, "merge") + await handle.signal(approveSignal, "deploy") + await handle.result() + }) + + expect(acts.recordDeploy).toHaveBeenCalledWith("t1", "failure") + expect(acts.transitionTicket).toHaveBeenCalledWith("t1", "blocked") + expect(acts.transitionTicket).not.toHaveBeenCalledWith("t1", "done") + }, 30_000) +}) + +describe("epicDecomposition", () => { + it("blocks on the roadmap gate until signed off, then decomposes", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(epicDecomposition, { + taskQueue: TQ, + workflowId: nextId("epic"), + args: ["e1"], + }) + await env.sleep("1 day") + expect(acts.requestRoadmapSignoff).toHaveBeenCalledWith("e1") + expect(acts.decomposeEpic).not.toHaveBeenCalled() + + await handle.signal(roadmapSignal) + await handle.result() + }) + + expect(acts.recordRoadmapApproval).toHaveBeenCalledWith("e1") + expect(acts.decomposeEpic).toHaveBeenCalledWith("e1") + }, 30_000) +}) + +describe("epicShaping", () => { + it("runs every shaping stage in order", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + await env.client.workflow.execute(epicShaping, { + taskQueue: TQ, + workflowId: nextId("shape"), + args: ["e1"], + }) + }) + // One call per stage (PM → UX → Architect → System Design). + expect(acts.runShapingStage.mock.calls.length).toBeGreaterThanOrEqual(4) + expect(acts.runShapingStage).toHaveBeenCalledWith("e1", "discovery") + expect(acts.runShapingStage).toHaveBeenCalledWith("e1", "system_design") + }, 30_000) +}) + +describe("heartbeat", () => { + it("picks up the backlog", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + await env.client.workflow.execute(heartbeat, { + taskQueue: TQ, + workflowId: nextId("hb"), + args: [], + }) + }) + expect(acts.pickUpBacklog).toHaveBeenCalledTimes(1) + }, 30_000) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f24c5e..db200e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,6 +60,9 @@ importers: specifier: ^4.12.26 version: 4.12.26 devDependencies: + '@temporalio/testing': + specifier: ^1.18.1 + version: 1.18.1(esbuild@0.28.1)(tslib@2.8.1) tsx: specifier: ^4.22.4 version: 4.22.4 @@ -1251,6 +1254,10 @@ packages: resolution: {integrity: sha512-pLa1JkmVwGdoFPCmb9DJ4P0z2mk3Zyu1VW6pXvJiYjt6W52Vm7YYi0caqfcSs/Wb/ZamdMbGqMcezftzVdzkYg==} engines: {node: '>= 20.0.0'} + '@temporalio/testing@1.18.1': + resolution: {integrity: sha512-fleEok2IWrc64lhuOMln4BhrjF5WgPYoh/t5ZH5M9nOPGgnQtD56k1KYlZ0WH8FKLJ4ZZgzo6zURHp8SovxHxw==} + engines: {node: '>= 20.0.0'} + '@temporalio/worker@1.18.1': resolution: {integrity: sha512-ydeTHhlZF4G9ye5YnyBNgZT8/7lpGdEaF4LwJLUsW33AtLiqUp5zkzwefktpn/vyHWmNWjDdacxZ3l8WgdN+0w==} engines: {node: '>= 20.0.0'} @@ -3159,6 +3166,31 @@ snapshots: long: 5.3.2 protobufjs: 7.5.8 + '@temporalio/testing@1.18.1(esbuild@0.28.1)(tslib@2.8.1)': + dependencies: + '@temporalio/activity': 1.18.1 + '@temporalio/client': 1.18.1 + '@temporalio/common': 1.18.1 + '@temporalio/core-bridge': 1.18.1 + '@temporalio/proto': 1.18.1 + '@temporalio/worker': 1.18.1(esbuild@0.28.1)(tslib@2.8.1) + '@temporalio/workflow': 1.18.1 + transitivePeerDependencies: + - '@minify-html/node' + - '@swc/css' + - '@swc/helpers' + - '@swc/html' + - clean-css + - cssnano + - csso + - esbuild + - html-minifier-terser + - lightningcss + - postcss + - tslib + - uglify-js + - webpack-cli + '@temporalio/worker@1.18.1(esbuild@0.28.1)(tslib@2.8.1)': dependencies: '@grpc/grpc-js': 1.14.4 diff --git a/project/ENG-001-repo-aware-coding-agents.md b/project/ENG-001-repo-aware-coding-agents.md new file mode 100644 index 0000000..7f69b71 --- /dev/null +++ b/project/ENG-001-repo-aware-coding-agents.md @@ -0,0 +1,60 @@ +# ENG-001 — Give coding agents real repository context + +- **Status:** backlog +- **Priority:** P0 (Critical — blocks a usable autonomous run) +- **Stage:** implementation +- **Assignee role:** staff_engineer (with lead_system_design input) +- **Area:** packages/agents + +## Problem + +Neither `Worker` backend can see the repository it is supposed to modify, so the coding agent writes +blind, hallucinated diffs that are then committed via the Git Data API. This is the single biggest +gap between the current "demo" state and the documented "one `ANTHROPIC_API_KEY` away from a live +autonomous run" claim. + +- The **API backend** is a single tool-less `messages.create` — the Staff Engineer agent imagines + file contents from the prompt and never reads real source. +- The **CLI backend** is full Claude Code but is deliberately confined to an empty throwaway dir + under `workspaces/`, so it also cannot read the target codebase. + +## Evidence + +- `packages/agents/src/backends/api.ts` — `ApiBackend.run` sends `messages.create` with no `tools`. +- `packages/agents/src/backends/cli.ts` — `createSandbox` / `resolveWorkspaceRoot` create an empty + per-run dir; the agent's cwd has no repo in it. +- `apps/server/src/temporal/activities.ts` — `implementTicket` feeds only `traceContext` (the + mission→goal→epic→ticket "why"), never repository files, to `proposeFileChanges`. + +## Proposed approach + +Build on the **working-code workspace** provisioned by ENG-013 (the persistent shared root holding the +cloned target repo — distinct from the throwaway agent-state sandbox). Then thread repo context into +`implementTicket`: + +1. **CLI backend, repo-aware:** run `claude -p` with its cwd in the cloned target repo inside the + working-code workspace, so the agent reads and edits against real source. Preserve the "never + mutate this product's own source" guarantee by operating on the cloned target repo (a per-ticket + branch/worktree), never this repo's working tree. +2. **API backend, context injection:** retrieve relevant files from the clone (by path heuristics / + the ticket description) and include them in the prompt, or give the API backend a read-only file + surface over the working-code workspace. + +Surface a knob (e.g. `AGENT_REPO_CONTEXT`) and keep the no-op/skipped audit path for when no repo is +configured. + +## Acceptance criteria + +- [ ] A coding agent run can read existing files from the target repository before proposing changes. +- [ ] Proposed `FileChange`s are diffs against actual file content, not invented whole-file rewrites + of files the agent never saw. +- [ ] The repo this product lives in is never mutated by an agent (clone/copy isolation preserved). +- [ ] Behaviour is config-gated and audited (`*_skipped` when no repo is available). +- [ ] A test demonstrates the agent reading a seeded file and editing it. + +## Notes / risks + +- **Depends on ENG-013** (repo provisioning + the two-root workspace model) to produce the clone the + agents work against. +- Interacts with ENG-005 (sandbox env scrubbing) and ENG-009 (structured outputs). +- Decide whether the clone is per-ticket (fresh) or cached per-repo for speed. diff --git a/project/ENG-002-test-orchestration-layer.md b/project/ENG-002-test-orchestration-layer.md new file mode 100644 index 0000000..0cd0a0b --- /dev/null +++ b/project/ENG-002-test-orchestration-layer.md @@ -0,0 +1,52 @@ +# ENG-002 — Test the Temporal orchestration layer + +- **Status:** done +- **Priority:** P1 (High) +- **Stage:** review +- **Assignee role:** qa_test +- **Area:** apps/server + +> **Outcome (Wave 0):** `apps/server/src/temporal/workflows.test.ts` drives the workflows through +> `@temporalio/testing` time-skipping with mocked activities (no live services). 8 tests cover: +> happy-path → `done` (both gates), merge-gate-blocks-until-approved, QA-fail → `blocked`, merge-fail +> → `blocked`, deploy-fail → `blocked`, the roadmap gate (blocks until `roadmapApprove`, then +> decomposes), all shaping stages run, and heartbeat picks up the backlog. The bundle compiles once +> in `beforeAll`. Runs inside `pnpm test` (59 tests total); typecheck + lint green. Added +> `@temporalio/testing` to `apps/server` devDependencies. + +## Problem + +The most complex, most stateful code in the system — the durable workflows and their activities — has +zero automated tests. All 51 existing tests live in `core` / `agents` / `integrations`; `apps/server/src` +has no `.test.ts` files at all. The approval gates, block-on-failure paths, and CI/deploy polling loops +are exactly the logic most likely to regress and hardest to verify by hand. + +## Evidence + +- `apps/server/src/temporal/workflows.ts` — `ticketLifecycle`, `epicShaping`, `epicDecomposition`: + no tests. +- `apps/server/src/temporal/activities.ts` — `implementTicket`, `verifyTicket`, deploy/merge + activities: no tests. +- `find apps/server/src -name '*.test.ts'` → none. + +## Proposed approach + +Use Temporal's official test harness (`@temporalio/testing` → `TestWorkflowEnvironment`) with mocked +activities to drive the workflows deterministically: + +- Assert `ticketLifecycle` reaches `done` only after both the merge and deploy signals. +- Assert a QA fail (`verifyTicket → false`) routes to `blocked` and stops. +- Assert a failed merge / failed deploy routes to `blocked` (no longer reaches `done`). +- Assert `epicDecomposition` blocks until the `roadmapApprove` signal, then decomposes. +- Use time-skipping to cover the CI/deploy poll loops without real waits. + +## Acceptance criteria + +- [ ] `@temporalio/testing` wired into the Vitest setup for `apps/server`. +- [ ] Tests cover: both gates, QA-fail block, merge-fail block, deploy-fail block, roadmap gate. +- [ ] Tests run in `pnpm test` and in CI (see ENG-003). +- [ ] Poll loops are exercised via time-skipping, not wall-clock sleeps. + +## Notes / risks + +- Pairs naturally with ENG-003 (CI) so these tests actually gate merges. diff --git a/project/ENG-003-ci-for-this-repo.md b/project/ENG-003-ci-for-this-repo.md new file mode 100644 index 0000000..18710d8 --- /dev/null +++ b/project/ENG-003-ci-for-this-repo.md @@ -0,0 +1,49 @@ +# ENG-003 — Add CI for this repository + +- **Status:** done +- **Priority:** P1 (High) +- **Stage:** implementation +- **Assignee role:** lead_engineer +- **Area:** repo / .github + +> **Outcome (Wave 0):** `.github/workflows/ci.yml` runs typecheck · lint · test · build on PRs/pushes +> to `main` (Node from `.nvmrc`, `--frozen-lockfile`, pnpm + Turborepo caching). All four checks +> verified green locally (51 tests). CI badge + branch-protection note added to the root `README.md`. +> Branch protection itself is a one-time GitHub repo setting (cannot be committed). + +## Problem + +There is no continuous integration for this repository itself — `.github/` does not exist, and +`docs/OVERVIEW.md` explicitly lists "No remote / CI for this repo itself" as a known gap. For a +product whose entire premise is CI-gated delivery, dogfooding a CI pipeline is both low-effort and +high-signal, and it prevents the quality gates (`typecheck` / `lint` / `test` / `build`) from +silently regressing. + +## Evidence + +- `.github/` — absent. +- `docs/OVERVIEW.md` → "Not yet built (honest gaps)" → "No remote / CI for this repo itself." +- `package.json` already defines `typecheck`, `lint`, `test`, `build` scripts (Turborepo). + +## Proposed approach + +Add a GitHub Actions workflow (`.github/workflows/ci.yml`) that, on push/PR: + +- sets up Node (per `.nvmrc`) + pnpm, +- `pnpm install --frozen-lockfile`, +- runs `pnpm typecheck`, `pnpm lint`, `pnpm test`, `pnpm build`. + +Cache pnpm + Turborepo for speed. Consider a Postgres/Temporal services matrix only once ENG-002 adds +integration-level tests that need them (unit tests should not require live services). + +## Acceptance criteria + +- [ ] `.github/workflows/ci.yml` runs typecheck + lint + test + build on every PR to `main`. +- [ ] The workflow uses the Node version from `.nvmrc` and a frozen lockfile. +- [ ] A red check blocks merge (branch protection note added to README/CLAUDE.md). +- [ ] Build/test caching configured so CI is reasonably fast. + +## Notes / risks + +- Once green, this becomes the reference `GITHUB_DEPLOY_WORKFLOW`-adjacent example for the product's + own delivery loop. diff --git a/project/ENG-004-api-auth-and-approval-identity.md b/project/ENG-004-api-auth-and-approval-identity.md new file mode 100644 index 0000000..a5d7104 --- /dev/null +++ b/project/ENG-004-api-auth-and-approval-identity.md @@ -0,0 +1,43 @@ +# ENG-004 — API authentication + approval identity + +- **Status:** backlog +- **Priority:** P0 (Critical — do before any live/shared deployment) +- **Stage:** architecture +- **Assignee role:** lead_system_design (with lead_architect sign-off) +- **Area:** apps/server + +## Problem + +The HTTP API has no authentication or authorization. Anyone who can reach the server can approve +merges and deploys, release the roadmap gate, start workflows, and create tickets. The approval gates +are the product's core value (invariant #4: "the human is in the loop"), yet there is no identity +behind the human — approval events are audited as the literal actor `"human"` with no `decidedBy`. +This undermines both invariant #4 and invariant #2 (audit-log integrity: who actually decided?). + +## Evidence + +- `apps/server/src/app.ts` — `new Hono()` with no auth middleware; `POST /api/tickets/:id/approve` + and `POST /api/epics/:id/approve-roadmap` are unauthenticated. +- `apps/server/src/temporal/activities.ts` — `recordRoadmapApproval` appends + `{ actor: "human" }` with no identity; merge/deploy approvals similarly anonymous. +- `packages/core/src/schema.ts` — `Approval.decidedBy` exists in the schema but is never populated. + +## Proposed approach + +- Add auth middleware to the Hono app (token / session / OIDC — choose per deployment model). +- Require an authenticated principal on all mutating routes, especially the approval gates. +- Thread the principal's identity into approval audit events (`decidedBy`), so the audit log records + *who* released each gate. +- Keep read views (Board/Roadmap/Audit) and write routes appropriately scoped. + +## Acceptance criteria + +- [ ] All mutating endpoints reject unauthenticated requests. +- [ ] Approval endpoints record the authenticated identity in the audit event (`decidedBy` populated). +- [ ] The dashboard authenticates and attaches the principal to approval actions. +- [ ] Tests cover: unauthenticated approval rejected; authenticated approval records identity. + +## Notes / risks + +- Pairs with ENG-006 (first-class approvals) — identity should live on the persisted approval record. +- Decide whether identity is per-user accounts or a single shared operator token for v1. diff --git a/project/ENG-005-scrub-cli-sandbox-env.md b/project/ENG-005-scrub-cli-sandbox-env.md new file mode 100644 index 0000000..4c46c34 --- /dev/null +++ b/project/ENG-005-scrub-cli-sandbox-env.md @@ -0,0 +1,47 @@ +# ENG-005 — Scrub the environment for CLI agent sandboxes + +- **Status:** backlog +- **Priority:** P1 (High — security hardening) +- **Stage:** implementation +- **Assignee role:** staff_engineer (security review by lead_system_design) +- **Area:** packages/agents + +## Problem + +The CLI backend confines an agent's *cwd* to a throwaway sandbox, but it does not scrub the *process +environment*. `claude -p` is full Claude Code with bash + filesystem tools, and the child inherits the +host environment — so an agent can read `ANTHROPIC_API_KEY`, `GITHUB_TOKEN`, `DATABASE_URL`, etc. via +its bash tool and potentially exfiltrate them. The sandbox boundary is cwd-only; env and network are +not contained. + +## Evidence + +- `packages/agents/src/backends/cli.ts` — `spawn(this.bin, args, { cwd, signal })` passes no `env`, + so the child inherits `process.env` in full. +- The CLAUDE.md / OVERVIEW describe the sandbox as cwd-confined; env/network isolation is not claimed + but is implicitly assumed by "can never write to repo source." + +## Proposed approach + +- Pass an explicit minimal `env` to `spawn` (allow-list only what the agent legitimately needs). +- Use a scoped/short-lived `GITHUB_TOKEN` (least-privilege) rather than the full host token. +- Consider network egress restrictions for the sandbox where feasible. +- Document the sandbox's actual trust boundary (cwd + env + network) honestly. + +## Acceptance criteria + +- [ ] CLI-backend children run with an allow-listed environment, not the full host env. +- [ ] Host secrets (`ANTHROPIC_API_KEY`, `DATABASE_URL`, broad `GITHUB_TOKEN`) are not present in the + agent's environment unless explicitly required and scoped. +- [ ] The documented sandbox boundary matches the implemented one. +- [ ] A test asserts a sensitive env var is absent from the spawned child's environment. + +## Notes / risks + +- Interacts with ENG-001 (repo-aware agents) and ENG-013 (provisioning + two-root workspace) — keep + all three consistent in the `CliBackend.run` spawn path. +- **Controlled exception:** the PM agent's `git`/`gh` tools (ENG-013) need credentials. Grant a + *scoped, least-privilege* `GITHUB_TOKEN`/`GH_TOKEN` to that sandbox only (repo create + read/list + scope) — never the full host token, and not to other roles. +- Env-scrubbing applies to **both** roots — the ephemeral agent-state sandbox and the persistent + working-code workspace. diff --git a/project/ENG-006-first-class-approvals.md b/project/ENG-006-first-class-approvals.md new file mode 100644 index 0000000..8de0cfc --- /dev/null +++ b/project/ENG-006-first-class-approvals.md @@ -0,0 +1,43 @@ +# ENG-006 — Make approval gates first-class, persisted entities + +- **Status:** backlog +- **Priority:** P2 (Medium) +- **Stage:** design +- **Assignee role:** lead_system_design +- **Area:** packages/core · apps/server + +## Problem + +Pending approvals are not modelled as real entities. The `/api/approvals` endpoint reconstructs +pending gates by replaying the audit log and **only surfaces roadmap gates** — the merge and deploy +gates exist solely as in-memory booleans inside the running Temporal workflow, so they cannot be +listed, queried, or shown in the UI, and "what's pending" does not survive cleanly as a queryable +view. Meanwhile the `Approval` / `ApprovalKind` zod schemas already designed in `core` are unused. + +## Evidence + +- `apps/server/src/app.ts` (`GET /api/approvals`) — derives pending state from audit events and only + emits `kind: "roadmap"`. +- `apps/server/src/temporal/workflows.ts` — `ticketLifecycle` holds `approved = { merge, deploy }` + as workflow-local state; nothing persists "a merge gate is pending." +- `packages/core/src/schema.ts` — `Approval` and `ApprovalKind` (`roadmap` · `design_signoff` · + `architecture_decision` · `pr_merge` · `deploy`) defined but unused. + +## Proposed approach + +- Persist a pending-approval record when any gate is reached (roadmap / merge / deploy), resolve it + on signal, and record the deciding identity (see ENG-004). +- Back `/api/approvals` with the persisted records instead of audit replay, covering all gate kinds. +- Update the Approvals page to show every pending gate, not just roadmap. + +## Acceptance criteria + +- [ ] Reaching any gate creates a queryable pending `Approval` record using the existing schema. +- [ ] `/api/approvals` returns roadmap, merge, and deploy gates. +- [ ] Approving a gate resolves the record (status + `decidedBy` + `decidedAt`). +- [ ] The Approvals UI lists all pending gates across tickets and epics. + +## Notes / risks + +- Keep the audit log as the append-only source of truth; the approval record is a derived, + mutable read-model — do not mutate audit history (invariant #2). diff --git a/project/ENG-007-budget-model-reset-and-reservation.md b/project/ENG-007-budget-model-reset-and-reservation.md new file mode 100644 index 0000000..78ff595 --- /dev/null +++ b/project/ENG-007-budget-model-reset-and-reservation.md @@ -0,0 +1,45 @@ +# ENG-007 — Fix the budget model (reset window + reservation) + +- **Status:** backlog +- **Priority:** P2 (Medium) +- **Stage:** implementation +- **Assignee role:** staff_engineer (with lead_system_design input) +- **Area:** packages/db + +## Problem + +Budgets are softer than invariant #3 ("budgets are enforced by the orchestrator") implies: + +1. **No reset window.** Despite the field name `monthlyBudgetCents`, `addSpend` only ever increments + `spentCents`; there is no monthly (or any) reset, so spend accumulates forever until the role is + silently exhausted. +2. **Read-then-spend TOCTOU.** Remaining budget is read *before* the agent runs with no reservation, + so two concurrent runs for the same role both observe the full remaining budget and can jointly + overspend. The per-call `affordableMaxTokens` cap mitigates a single run but is not a global ceiling. + +## Evidence + +- `packages/db/src/repo.ts` — `addSpend` does `spentCents + c` (atomic increment, good) but never + resets; `getBudgetRemaining` reads `limit - spent` with no reservation. +- `packages/core/src/roles.ts` — `monthlyBudgetCents` implies a monthly window that doesn't exist. +- `apps/server/src/temporal/activities.ts` — each agent activity reads `getBudgetRemaining` then + `addSpend` after the run, with no debit-on-start. + +## Proposed approach + +- Introduce an explicit budget period (e.g. a `periodStart` / rolling window) and reset/roll spend, + or rename to honest lifetime semantics if no reset is intended. +- Add a reservation/debit pattern: reserve an estimated cost on start, reconcile to actual on + completion — closing the TOCTOU gap for concurrent same-role runs. + +## Acceptance criteria + +- [ ] Budget semantics are explicit: either a real reset window or clearly-named lifetime budgets. +- [ ] Concurrent agent runs for one role cannot jointly exceed the role's remaining budget. +- [ ] Spend reconciliation (estimate → actual) is recorded and audited. +- [ ] Tests cover: exhaustion blocks further runs; concurrent runs respect the ceiling; reset (if any). + +## Notes / risks + +- Keep `addSpend`'s atomic SQL increment; the change is about *reservation* and *windowing*, not the + write itself. diff --git a/project/ENG-008-cyclic-rework-routing.md b/project/ENG-008-cyclic-rework-routing.md new file mode 100644 index 0000000..d39d275 --- /dev/null +++ b/project/ENG-008-cyclic-rework-routing.md @@ -0,0 +1,42 @@ +# ENG-008 — Implement cyclic review/blocked rework routing + +- **Status:** backlog +- **Priority:** P2 (Medium) +- **Stage:** architecture +- **Assignee role:** lead_system_design (with lead_architect input) +- **Area:** apps/server + +## Problem + +The architecture states the lifecycle is "a durable state machine, not a batch DAG … review can +bounce work back and blockers loop," but the implementation does not loop. When QA fails or a +merge/deploy fails, `ticketLifecycle` simply sets the ticket to `blocked` and returns. There is no +feedback path back to the Staff Engineer to address the QA findings and re-attempt — so the promised +cyclic routing is not yet real. + +## Evidence + +- `apps/server/src/temporal/workflows.ts` — + - QA fail: `if (!qaOk) { transitionTicket(ticketId, "blocked"); return }` + - merge fail: `if (!merged) { ...blocked; return }` + - deploy fail: `if (state === "failure") { ...blocked; return }` +- `packages/agents/src/assess.ts` produces a verdict + summary that is currently terminal, never fed + back into a re-implementation step. + +## Proposed approach + +- Add a bounded rework loop: on QA fail, route the QA summary back to `proposeFileChanges` as feedback, + re-open a revision, and re-run QA — up to a max attempt count, then `blocked` for human attention. +- Decide the same for recoverable merge/CI failures (e.g. failing checks → fix → re-push). +- Keep every transition audited and the loop bounded (no infinite cycles; respect budgets, ENG-007). + +## Acceptance criteria + +- [ ] A QA fail produces actionable feedback that drives a re-implementation attempt. +- [ ] The rework loop is bounded (configurable max attempts) and falls back to `blocked`. +- [ ] CI-failure rework path defined (even if minimal for v1). +- [ ] Workflow tests (ENG-002) cover the rework loop and the eventual `blocked` fallback. + +## Notes / risks + +- Bounding is essential — combine with budget enforcement so a stuck ticket can't burn a role's budget. diff --git a/project/ENG-009-structured-outputs-for-agents.md b/project/ENG-009-structured-outputs-for-agents.md new file mode 100644 index 0000000..ae132cb --- /dev/null +++ b/project/ENG-009-structured-outputs-for-agents.md @@ -0,0 +1,45 @@ +# ENG-009 — Use structured outputs for agent response parsing + +- **Status:** backlog +- **Priority:** P1 (High) +- **Stage:** implementation +- **Assignee role:** staff_engineer +- **Area:** packages/agents + +## Problem + +The agent helpers (`proposeFileChanges`, `proposeTickets`, `assess`) rely on prompting the model to +"Respond with ONLY a JSON object" and then best-effort `extractJson` parsing. A chatty or +slightly-off response silently degrades: `parseProposal` falls back to `files: []`, which the +workflow reads as "no file changes proposed" and skips delivery entirely — a silent failure on the +most important path. + +## Evidence + +- `packages/agents/src/propose.ts` — `CONTRACT` string + `extractJson` / `parseProposal` best-effort + regex/brace extraction; empty/garbled output → `files: []`. +- `apps/server/src/temporal/activities.ts` — `implementTicket` treats `proposed.files.length === 0` + as `delivery_skipped` ("no file changes proposed"). +- `packages/agents/src/backends/api.ts` — the API backend sends no `output_config`. + +## Proposed approach + +- For the **API backend**, use structured outputs (`output_config: { format: { type: "json_schema", + schema } }`) with a zod-derived JSON schema for each shape (proposal, tickets, QA verdict). On Opus + 4.8 this guarantees a parseable, schema-conformant response and removes the regex extraction. +- Keep the prose fallback only for the CLI backend (which can't easily use structured outputs); + consider a stricter contract / retry there. +- Distinguish "genuinely no change needed" from "parse failure" so the latter is audited as an error, + not silently skipped. + +## Acceptance criteria + +- [ ] API-backend agent calls return schema-validated JSON via structured outputs (no regex parsing). +- [ ] Parse failures are audited distinctly from a legitimate empty change set. +- [ ] Existing `parseProposal` / `parseTickets` tests pass or are updated for the new path. +- [ ] CLI-backend fallback path retains a tested contract. + +## Notes / risks + +- Structured outputs are incompatible with a few features (e.g. citations) — not a concern here. +- Directly improves whether autonomous runs actually produce committed code (relates to ENG-001). diff --git a/project/ENG-010-dashboard-budgets-and-cost.md b/project/ENG-010-dashboard-budgets-and-cost.md new file mode 100644 index 0000000..a284e61 --- /dev/null +++ b/project/ENG-010-dashboard-budgets-and-cost.md @@ -0,0 +1,37 @@ +# ENG-010 — Surface budgets and cost in the dashboard + +- **Status:** backlog +- **Priority:** P2 (Medium) +- **Stage:** design +- **Assignee role:** ux_design (build by staff_engineer) +- **Area:** apps/web + +## Problem + +The dashboard is the accountable lead's primary surface, but it shows no budget or cost view even +though all the data already exists: per-role `limitCents` / `spentCents` in the budgets table, and +per-event `costCents` on audit records. The lead governs budgets (a stated core responsibility) with +no visibility into spend. + +## Evidence + +- `packages/db/src/repo.ts` — `getBudgetRemaining` / `addSpend` maintain per-scope limit/spent. +- `apps/server/src/temporal/activities.ts` — agent steps record `costCents` in audit payloads. +- `apps/web/src/pages/` — Board / Roadmap / Approvals / Audit exist; no budgets/cost page. + +## Proposed approach + +- Add a budgets/cost view: per-role limit · spent · remaining, plus a rollup of total spend. +- Optionally trend cost over time from the audit log's `costCents` events. +- Expose a read endpoint for budgets (the data is currently Postgres-direct; add `/api/budgets`). + +## Acceptance criteria + +- [ ] A dashboard view shows per-role limit / spent / remaining. +- [ ] A total-spend summary is visible to the lead. +- [ ] Data comes from a typed API endpoint, consistent with the existing `core`-shared schema pattern. +- [ ] Empty/zero states render cleanly before any spend has occurred. + +## Notes / risks + +- Depends partly on ENG-007 if budget semantics (reset window) change — align the labels. diff --git a/project/ENG-011-decide-initial-issue-tracker.md b/project/ENG-011-decide-initial-issue-tracker.md new file mode 100644 index 0000000..cdd9eed --- /dev/null +++ b/project/ENG-011-decide-initial-issue-tracker.md @@ -0,0 +1,62 @@ +# ENG-011 — Decide the initial issue tracker + +- **Status:** done (decision recorded below) +- **Priority:** P3 (Low) +- **Stage:** discovery +- **Assignee role:** lead_architect (with PM input) +- **Area:** packages/core · packages/integrations + +## Decision (Wave 0 spike outcome) + +**v1 default: GitHub Issues/Projects** (with the **Postgres** backend for standalone/local runs). +Linear/Jira are **deferred** — addable later behind the unchanged `IssueTracker` port if a concrete +need arises. + +**Why GitHub:** +- **Already built** — `GitHubIssueTracker` exists, plus the Mission→Goal→Epic hierarchy as native + GitHub **sub-issues** and a repo-docs KB. It's the lowest-effort path; no new integration to write. +- **Where the work already lives** — the operator is an engineering unit already on GitHub for code, + PRs, CI, and the Actions deploy. Keeping issues there gives native ticket ↔ PR ↔ commit linkage and + one fewer external system. +- **No lock-in** — selection is governed by `PERSISTENCE_BACKEND` behind the `IssueTracker` port, so + swapping to Linear/Jira later never touches `core` (invariant #5). + +**Interface sufficiency:** confirmed for GitHub — the port already round-trips ticket fields (via a +metadata block) and models the goal hierarchy through sub-issues. For Linear/Jira the port would need +re-validation (hierarchy/acceptance-criteria mapping) **at the time an adapter is built**, not now. + +**Follow-up:** none required for v1. If/when an external tracker is requested, open an adapter ticket +that (1) re-checks the `IssueTracker` interface against that tracker and (2) implements it behind the +factory. CLAUDE.md's "Still open" note can be updated to reflect this decision (a one-line lead edit). + +## Problem + +The initial issue-tracker choice is the one remaining "still open" decision in CLAUDE.md: GitHub +Issues/Projects vs. an external tracker (Linear/Jira) behind the same `IssueTracker` interface. +Postgres and GitHub backends exist; no external-tracker adapter does. This is a decision-and-maybe-a- +spike rather than a defect. + +## Evidence + +- `CLAUDE.md` → Decisions → "Still open: Initial issue tracker: GitHub Issues/Projects vs. external + (Linear/Jira) behind the same interface." +- `packages/core/src/tracker.ts` — `IssueTracker` port; `packages/db` + `packages/integrations/github` + implement it; no Linear/Jira implementation. + +## Proposed approach + +- Capture the decision criteria (who the operator is, where their work already lives, API ergonomics, + sub-issue/hierarchy support, cost). +- Confirm the `IssueTracker` interface is sufficient for an external tracker (hierarchy, acceptance + criteria round-trip), and note any gaps. +- Decide v1 default; if external, scope a thin adapter spike behind the existing interface. + +## Acceptance criteria + +- [ ] A written decision (with rejected alternatives) recorded in CLAUDE.md / OVERVIEW. +- [ ] Confirmation that `IssueTracker` covers the chosen tracker's needs, or a list of interface gaps. +- [ ] If external is chosen: a scoped follow-up ticket for the adapter. + +## Notes / risks + +- Swapping backends must not touch `core` (invariant #5) — validate the interface holds. diff --git a/project/ENG-012-research-a2a-protocol.md b/project/ENG-012-research-a2a-protocol.md new file mode 100644 index 0000000..ca41493 --- /dev/null +++ b/project/ENG-012-research-a2a-protocol.md @@ -0,0 +1,115 @@ +# ENG-012 — Research Google's A2A (Agent-to-Agent) protocol + +- **Status:** done (findings + recommendation recorded below) +- **Priority:** P2 (Medium — discovery spike, decision-oriented) +- **Stage:** discovery +- **Assignee role:** lead_architect +- **Area:** architecture (cross-cutting) + +## Findings & recommendation (Wave 0 spike) + +**What A2A is (verified, June 2026).** Google's Agent2Agent protocol (announced Apr 2025; donated to +the **Linux Foundation** Jun 2025; **v1.0 early 2026**, 150+ orgs) is an open standard for *agent↔agent* +interoperability: agents from different vendors/frameworks **discover** each other via an **Agent Card** +(a signed JSON document at `/.well-known/agent.json` describing skills, auth, endpoints), then +**delegate tasks** and exchange **messages/artifacts** over JSON-RPC/HTTP with SSE streaming. v1.0's +headline addition is **signed Agent Cards** (cryptographic issuer verification). + +**A2A vs MCP.** They are complementary, not competing: **MCP is vertical** (an agent → tools/data — +Anthropic's protocol, Claude-native), **A2A is horizontal** (agent ↔ agent coordination). Industry +guidance: a single agent with tools needs only MCP; networked *multi-vendor* multi-agent systems use +both, "MCP first." + +**Fit for this project.** The decisive point: this unit's multi-agent coordination is **intra-unit and +already solved** — the seven roles are stages in a *durable Temporal state machine*, handed artifacts +stage-to-stage behind the `Worker` interface, with human gates and an append-only audit log. That is +stronger than networked peer messaging (durable, gated, traceable). **A2A solves a problem the project +does not currently have internally**, and pulling a vendor-neutral peer-messaging transport into the +core would cut against the deliberate Claude-first / no-provider-agnostic-orchestrator stance (the same +reasoning that rejected LangChain) — unless it's confined to the *edge*. + +A2A is only relevant at the unit's **boundary**, in two future directions: +- **Expose** — publish the unit (or individual roles) as A2A-discoverable agents (signed Agent Card), + so external orgs/frameworks can delegate work to this unit. +- **Consume** — treat an external A2A agent as extra "headcount" behind the `Worker` interface. + +**Recommendation: decline now; monitor; keep the boundary port-shaped.** +- **Do not adopt A2A for the internal lifecycle** — Temporal + `Worker` + staged handoff already do this + better; A2A would be redundant and stance-violating internally. +- **Monitor the spec** — LF-governed, v1.0, 150+ orgs: it is the emerging interop standard, worth + tracking, but adoption is not justified for a single self-contained unit today. +- **If cross-org / cross-framework interop becomes a goal**, add it *behind a port* (an A2A adapter + implementing `Worker` for *consume*; an A2A server exposing signed Agent Cards for *expose*), keeping + `core` framework-agnostic (invariant #5) — never in the orchestration loop. +- **MCP is the higher-value near-term interop bet** for a Claude-first project: if agents should reach + external *tools*, MCP (vertical, Claude-native) beats A2A (horizontal) for this project's current + shape. Consider an MCP spike before an A2A one. + +### Sources +- A2A overview / Linux Foundation / Agent Card — https://www.ibm.com/think/topics/agent2agent-protocol +- A2A growth + v1.0 signed Agent Cards — https://stellagent.ai/insights/a2a-protocol-google-agent-to-agent +- A2A vs MCP (vertical vs horizontal; "MCP first") — https://onereach.ai/blog/guide-choosing-mcp-vs-a2a-protocols/ + +## Problem + +Evaluate Google's **A2A (Agent-to-Agent) protocol** and determine whether it is appropriate to adopt +in this project, and if so where. A2A is an open protocol (introduced by Google in 2025, subsequently +moved to open governance under the Linux Foundation) for interoperability *between* agents — capability +discovery, task delegation, and message/artifact exchange across agents that may be built on different +frameworks or vendors. This is distinct from MCP, which standardizes an agent's access to *tools and +context*. The goal of this ticket is a clear, evidence-based recommendation — not a commitment to adopt. + +> Treat the description above as background to verify, not settled fact. Confirm the current spec +> version, governance, and shape from primary sources as part of this spike. + +## Why it matters here (project context) + +This system already models a unit's "headcount" as seven role personas behind a uniform `Worker` +interface, orchestrated durably by Temporal, with delivery behind a `DeliveryAdapter`. A2A is +potentially relevant in two directions: + +1. **Expose** — present each role agent (PM, Architect, Staff Eng, QA, …) as an A2A-discoverable agent + (via an Agent Card), so external agents or other frameworks could interoperate with this unit. +2. **Consume** — let the unit delegate to external A2A agents as additional, vendor-neutral + "headcount" behind the existing orchestration. + +There is also a **design-tension to examine head-on:** this project is deliberately Claude-first and +explicitly rejected provider-agnostic orchestration abstractions (e.g. LangChain) to avoid a +vendor-neutral layer through the middle of the product. A2A is intentionally vendor-neutral, so the +spike must judge whether adopting it conflicts with that decision or complements it (e.g. as an +*edge* interop protocol rather than the internal orchestration model). + +## Research questions + +- What problem does A2A actually solve, and how does it differ from / compose with MCP? +- Core concepts and wire format: Agent Cards / capability discovery, tasks vs. messages vs. artifacts, + transport (HTTP/JSON-RPC), streaming (e.g. SSE), and authentication model. Confirm current versions. +- Governance, maturity, ecosystem, and SDK/library support (especially TypeScript/Node). +- Security model: authN/authZ between agents, trust boundaries, and how it would interact with this + project's approval gates, budgets, and append-only audit log. +- Where (if anywhere) it maps onto our seams: `Worker`, the role personas (roles-as-config, + invariant #6), Temporal orchestration, `DeliveryAdapter`. Does it sit at the *edge* (interop) or + would it intrude on `core` (which must stay framework-agnostic, invariant #5)? +- Does it conflict with the Claude-first / no-provider-agnostic-orchestrator stance, or is it + orthogonal (interop at the boundary, not the internal loop)? + +## Proposed approach + +- Read primary sources (spec + reference implementations/SDKs); build a minimal proof-of-concept only + if cheap and clarifying (e.g. an Agent Card for one role behind a feature flag). +- Produce a short ADR-style recommendation: adopt / adopt-narrowly / defer / decline, with rationale, + a candidate integration seam, and a rough effort estimate. + +## Acceptance criteria + +- [ ] A written findings doc / ADR in `docs/` summarizing A2A, its fit, and the MCP relationship. +- [ ] An explicit recommendation (adopt / adopt-narrowly / defer / decline) with reasoning tied to + this project's invariants and the Claude-first decision. +- [ ] If "adopt"/"adopt-narrowly": the specific integration seam (expose vs. consume), a proposed + interface boundary that keeps `core` framework-agnostic, and a follow-up implementation ticket. +- [ ] Primary-source citations and the spec version reviewed. + +## Notes / risks + +- Time-box the spike; the deliverable is a decision, not an implementation. +- Keep any A2A surface at the boundary — do not let a vendor-neutral abstraction leak into `core`. diff --git a/project/ENG-013-repo-provisioning-and-shared-workspace.md b/project/ENG-013-repo-provisioning-and-shared-workspace.md new file mode 100644 index 0000000..06146f6 --- /dev/null +++ b/project/ENG-013-repo-provisioning-and-shared-workspace.md @@ -0,0 +1,96 @@ +# ENG-013 — Repository provisioning via the PM agent (git + gh) & shared workspace + +- **Status:** backlog +- **Priority:** P0 (Critical — prerequisite for ENG-001) +- **Stage:** discovery → implementation +- **Assignee role:** pm (owns repo create / list / clone); supporting backend work (per-role tool + gating, scoped auth) by staff_engineer with lead_system_design input +- **Area:** packages/core (role config) · packages/agents (CLI backend tool gating + auth) · + apps/server (shared workspace + audit) + +## Decision + +Per the product owner's direction, the **PM agent owns repository provisioning** and is granted two +command-line tools: + +- **`git`** — create local repos (`git init`) and perform local git operations. +- **`gh`** — list remote repos on the GitHub account, and create/clone remote repos. + +The PM creates a repo if it does not already exist, lists existing repos, and clones the target into +the **shared `./workspace`** so the other agents can inspect and work on it. This supersedes the +earlier "port + activities" proposal; provisioning is driven by the PM agent via CLI, with the +safeguards below so it stays reliable and secure. + +## Problem + +For agents to work on a codebase it must exist locally first. Today there is no way to create a repo +when missing, discover existing repos, or clone a target into a workspace agents can read. ENG-001 +("give agents real repository context") depends on this provisioning step existing. + +## Evidence / context + +- `packages/core/src/roles.ts` — `RolePersona.tools` is the per-role allow-list. PM is currently + `tools: ["tracker", "docs"]`; this ticket adds `"git"` and `"gh"`. Roles are config (invariant #6), + so this is a data change, not orchestrator branching. +- `packages/agents/src/backends/cli.ts` — the CLI backend runs full Claude Code (`claude -p`) with + bash; today it does **not** pass a per-role tool allow-list, so the `tools` array is *advisory* + only. The API backend has no tools at all. +- `packages/agents/src/backends/cli.ts` — `resolveWorkspaceRoot` / `createSandbox` create *throwaway, + per-run* role sandboxes (`workspaces/-xxxx`). The shared target-repo clone is a **separate** + artifact. + +## Proposed approach + +1. **Grant the tools (config):** add `"git"` and `"gh"` to `ROLES.pm.tools`, and extend the PM system + prompt to describe when to create / list / clone (create-if-missing; clone into the shared + workspace; prefer reuse of an existing repo). +2. **Make the grant real (enforcement):** the CLI backend must enforce the per-role allow-list (e.g. + `claude -p --allowedTools` / `--disallowedTools` derived from `role.tools`) so the PM gets + `git`/`gh` and other roles do not. Without this, every CLI agent already has git/gh via bash and + the grant is meaningless. *(If this turns out large, split it into its own ticket.)* +3. **Scoped auth:** `gh` needs a token and `git` needs credentials for private repos. Provide the PM + sandbox a **least-privilege** `GITHUB_TOKEN` / `GH_TOKEN` (repo create + read/list scope) — the + controlled exception to ENG-005's env-scrubbing, never the full host token. +4. **Two separate sandboxed roots (explicit):** + - **Agent-state sandbox** — the existing per-run, throwaway role scratch dir + (`workspaces/-xxxx`, via `AGENT_WORKSPACE_DIR`); ephemeral, cleaned up after each run, where + the agent reasons and scratches. **No product code lives here.** + - **Working-code workspace** — a new, *persistent* shared root holding cloned target repos that + agents inspect and modify (e.g. `./workspace` or `workspaces/repos//`, behind a new + env var such as `AGENT_CODE_WORKSPACE`). Use per-ticket branches/worktrees for concurrency. + + Naming gotcha: the existing throwaway root is plural `workspaces/`; pick a clearly distinct name + for the code workspace so the two roots aren't conflated. +5. **Idempotency + verification:** create-if-missing must be safe to repeat. Because this runs as an + LLM + CLI step (less deterministic than a plain activity), the workflow/activity should **verify** + the repo/clone actually exists and **audit** the provisioning actions, so a flaky agent turn can't + silently leave no repo. +6. **Repo selection stays a planning decision:** *which* repo (new vs. existing) is associated with + the Goal/Epic at roadmap authoring; the PM proposes it and the human approves at the existing + roadmap gate. + +## Acceptance criteria + +- [ ] `ROLES.pm.tools` includes `git` and `gh`; the PM prompt explains create/list/clone duties. +- [ ] The CLI backend enforces the per-role tool allow-list so non-PM roles do **not** receive + `git`/`gh` (the grant is real, not advisory). +- [ ] The PM agent can: create a repo when missing, list remote repos (`gh`), and clone a repo into + the shared workspace. +- [ ] Two distinct roots exist and are documented — an **ephemeral agent-state sandbox** and a + **persistent working-code workspace**; product code only ever lives in the latter, with a + defined concurrency model. +- [ ] `gh`/`git` auth uses a scoped, least-privilege token provided only to the PM sandbox (documented + ENG-005 exception). +- [ ] Provisioning is idempotent and audited; the workflow verifies the repo/clone exists before + downstream stages depend on it. +- [ ] No-op / `*_skipped` audit when GitHub or the token is unconfigured. + +## Notes / risks + +- **Unblocks ENG-001** — coding agents read/write against the clone this ticket produces. +- **Reliability:** an LLM-driven CLI step is less deterministic than a plain activity — mitigate with + idempotent commands (`git init` / `gh repo create` guarded by existence checks), post-step + verification, and audit. +- **Security:** scoped token only (see ENG-005); never expose the full host token to agent bash. +- **Concurrency:** multiple agents in one working tree will corrupt git state — prefer per-ticket + worktrees/branches. diff --git a/project/ENG-014-decision-log-and-provenance-tree.md b/project/ENG-014-decision-log-and-provenance-tree.md new file mode 100644 index 0000000..92d58f6 --- /dev/null +++ b/project/ENG-014-decision-log-and-provenance-tree.md @@ -0,0 +1,114 @@ +# ENG-014 — Per-agent decision logs & a traceable decision tree + +- **Status:** backlog +- **Priority:** P2 (Medium — governance / explainability) +- **Stage:** architecture → implementation +- **Assignee role:** lead_system_design (schema + storage port) with lead_architect sign-off + (cross-cutting concern) +- **Area:** packages/core (schema + port) · packages/db · packages/integrations · apps/server + (emit on each step) · apps/web (optional view) + +## Problem + +Every agent should emit a structured **decision log** — what it decided, why, what alternatives it +weighed, what it consumed and produced — and those logs should link into a **decision graph that is +traceable all the way up to the original request**. Today agents emit only a free-text `summary`, the +audit log is a flat event stream, and the work hierarchy traces *work items* (not decisions and their +rationale). There is no queryable, linked provenance of *why* the unit did what it did. + +This is the same "make an implicit thing first-class" pattern as ENG-006 (approvals), applied to +decision provenance. + +## What this is (and is not) + +- **Not** a replacement for the append-only audit log (`AuditLog` port / `auditLog` table) — that + stays the event substrate ("what happened, when, cost"). Decisions *reference* audit events; they do + not duplicate them (invariant #2). +- **Not** the work hierarchy — Mission→Goal→Epic→Ticket gives the work-item spine; decisions hang off + it and off each other. +- **It is** a semantic layer linking decision → parent decision(s) → work item → original request. +- **It is a DAG, not a strict tree:** a decision can build on several prior decisions (e.g. System + Design depends on both PM requirements and the Architect ADR). Model parents as a set of edges. + +## Evidence / context + +- `packages/core/src/schema.ts` — `AuditEvent` (flat stream) and the Mission/Goal/Epic/Ticket schemas + (work-item traceability, invariant #1). No decision/provenance entity exists. +- `packages/db/src/repo.ts` — `getTraceContext` / `getEpicContext` already walk the hierarchy to root; + the decision graph reuses this spine. +- `apps/server/src/temporal/activities.ts` — `runShapingStage`, `decomposeEpic`, `implementTicket`, + `verifyTicket` each currently record a free-text `summary` in the audit payload; these are the + emit points for structured decisions. +- `packages/core/src/knowledge.ts` (+ GitHub/DB KB impls) — the existing document substrate for + artifact bodies (shaping artifacts already stored this way). + +## Storage — options & recommendation + +Anchor on what exists: the append-only audit log + the hierarchy spine. The decision graph is a layer +on top of them. + +| Option | Pros | Cons | +|--------|------|------| +| **A. Relational** (Postgres `decisions` table: self-ref `parent_decision_id` edges + FKs to work item + `root_request_id`; rationale in JSONB; recursive CTE to root) | Already the system of record; FK integrity; transactional; append-only; Drizzle in use | Graph traversal via recursive CTE (fine at this scale) | +| **B. Document store** (existing `KnowledgeBase` port — Postgres `kb_docs` or GitHub repo docs; one doc per decision) | Human-readable; on the GitHub backend decisions become **PR-reviewable files next to the code**; reuses the KB port | Weak tree/query layer — parent links in frontmatter, graph rebuilt in app code; no integrity | +| **C. Graph DB** (Neo4j, …) | Native lineage traversal | New infra + ops burden; against the lean-Postgres stack decision; overkill — **not now** | +| **D. SurrealDB** (multi-model: relational + document + **graph** in one engine; native `RELATE` edges + `->` traversal; live queries) | Best technical fit for this DAG *and* the work hierarchy *and* the KB body — all in one store; pitched explicitly for "AI agent memory" | BSL 1.1 (source-available, **not** OSI — see ENG-015 license note); does not remove Postgres (Temporal still needs it); loses Drizzle/TS-native types; new datastore. Being evaluated separately — **see ENG-015** | + +**Recommendation — hybrid (A as the index, B for the body):** store the **decision node + edges** +(structure, refs, parent links, `root_request_id`) as **append-only rows in Postgres** for queryable, +integrity-checked traversal; store the **rich rationale body** inline as JSONB or as a KB document +referenced by path (human-readable, and PR-reviewable on the GitHub backend). Put it behind a new +`core` port (e.g. `DecisionLog` / `ProvenanceGraph`, alongside `AuditLog` / `Hierarchy`) so backends +stay pluggable (invariant #5). Do **not** stand up a graph DB. + +**Modeling lens:** this is structurally an OpenTelemetry trace — `root_request_id` ≈ trace id, each +decision ≈ a span with a `parent_decision_id`. Keep field names trace-compatible so the graph could +later be emitted to a tracing/observability backend without remodeling. + +> If the SurrealDB evaluation (ENG-015) goes ahead, this feature is the natural **pilot**: it is the +> most graph-shaped, highest-upside use case. The `DecisionLog` port keeps the body backend-agnostic, +> so it can land on the Postgres+KB hybrid first and move to a `surreal` backend later without +> reworking the schema. + +## Proposed decision-node schema (sketch) + +- `id` +- `rootRequestId` — the originating human request / workflow kickoff (the root every chain reaches) +- `parentDecisionIds[]` — causal parents (DAG edges) +- work-item refs: `missionId` / `goalId` / `epicId` / `ticketId` (the hierarchy spine) +- `actor` (RoleId / agent), `stage` (LifecycleStage), `at` +- `statement` (what was decided), `rationale` (why) +- `alternatives[]` (considered + why rejected) +- `inputs` (artifacts / prior decisions / files consumed) +- `outputs` (KB doc paths, file changes, PR, created ticket ids) +- `confidence`, `costCents` +- `auditEventId` — link back to the audit-log event substrate + +## Proposed approach + +1. Add a `Decision` zod schema in `core` and a `DecisionLog` port (`record`, `get`, + `listByRoot` / `byWorkItem`, `traverseToRoot`), with Postgres + GitHub-doc implementations. +2. Establish a **root request node** at each kickoff (epic shaping / roadmap authoring / workflow + start) capturing the human's original ask. +3. Have each agent step emit a structured decision via **structured outputs (ENG-009)** — shaping + draft, decomposition, implementation, QA verdict — linked to its parent decision(s) and work item. +4. Keep it append-only; reference (don't copy) audit events and KB artifacts. +5. (Optional) a dashboard decision-tree view (ENG-010 adjacent). + +## Acceptance criteria + +- [ ] Every agent step (shape / decompose / implement / QA) emits a structured decision record linked + to its parent(s) and its work item. +- [ ] Each decision chains via parent edges + the work hierarchy to the **original request** (root), + and this is verifiable by a traversal API. +- [ ] A `core` `DecisionLog` port persists decisions, is backend-pluggable (Postgres + GitHub-doc), + and is append-only. +- [ ] Decision bodies are human-readable (and PR-reviewable on the GitHub backend). +- [ ] Decisions reference audit events / KB artifacts rather than duplicating them. + +## Notes / risks + +- **Enabled by ENG-009** — without structured outputs, extracting decisions from prose is brittle. +- It is a **DAG, not a strict tree** — support multiple parents. +- Keep field names OpenTelemetry-trace-compatible to allow later emit to a tracing backend. +- Same first-class-entity pattern as ENG-006 (approvals); a natural dashboard view pairs with ENG-010. diff --git a/project/ENG-015-evaluate-surrealdb-backend.md b/project/ENG-015-evaluate-surrealdb-backend.md new file mode 100644 index 0000000..f8d8bca --- /dev/null +++ b/project/ENG-015-evaluate-surrealdb-backend.md @@ -0,0 +1,115 @@ +# ENG-015 — Evaluate SurrealDB as a persistence backend (decision spike) + +- **Status:** done (decision recorded; one item awaits the lead's ratification — see below) +- **Priority:** P2 (Medium — decision spike; gates a potentially large change) +- **Stage:** discovery → architecture +- **Assignee role:** lead_architect (tech direction / cross-cutting) with lead_system_design +- **Area:** packages/core (ports) · packages/db · packages/integrations · infra + +## Decision (Wave 0 spike outcome) + +**No-go on a wholesale "store everything in SurrealDB" replacement.** Two independent blockers, either +of which is sufficient: + +1. **It doesn't consolidate the stack.** Temporal needs a supported backing store (PostgreSQL / MySQL / + Cassandra / SQLite); SurrealDB is not one. So Postgres stays regardless — this is *additive* + infrastructure, not the one-datastore simplification it appears to be. +2. **It requires relaxing the project's license bar.** SurrealDB core is **BSL 1.1** — source-available + but **not** OSI open source, the same category the project used to reject Inngest (SSPL) "on license + purity." In practice BSL's only restriction (no competing managed DBaaS) would never bind a + self-hosted internal tool, so it's far milder than SSPL — but adopting it is a conscious reversal of + a settled decision. + +**Conditional future option (deferred):** add a `surreal` backend behind the existing +`PERSISTENCE_BACKEND` ports as a *narrow pilot* (the ENG-014 decision graph — the most graph-shaped, +highest-upside feature), **only if both** (a) the lead consciously relaxes the OSI-only license bar to +accept BSL, **and** (b) graph/lineage queries become central enough to justify a second datastore. +SurrealDB's multi-model fit (relational + document + native graph) and "AI agent memory" positioning +make it a genuinely strong technical candidate *for that pilot* if those conditions are met. + +**Action now:** keep Postgres + Drizzle as the default and as Temporal's store. Build **ENG-014 on the +Postgres + KB hybrid behind its port**, so a later migration to a `surreal` backend (if the pilot ever +runs) needs no `core` changes. + +**Awaiting the lead's ratification:** the one decision the spike can't make unilaterally — *do we relax +the OSI-only license bar to accept BSL 1.1?* Recommendation: **keep the bar (decline) for now**, since +no current need exists that Postgres can't meet. Revisit only alongside a concrete graph-query need. + +## Question + +Should this project store its state in **SurrealDB** — potentially "everything" — instead of / in +addition to Postgres? This ticket evaluates that and produces a go/no-go, rather than committing to a +big-bang migration. The trigger is that the project's data is naturally multi-model: the +Mission→Goal→Epic→Ticket hierarchy and the ENG-014 decision DAG are **graph-shaped**, the KB artifacts +are **document-shaped**, and tickets/budgets are **relational** — all three of which SurrealDB handles +in one engine. + +## Verified findings (June 2026) + +- **License: BSL 1.1.** Free to use, embed, and self-host internally at any scale; the only + restriction is offering SurrealDB itself as a managed DBaaS. Each release converts to Apache 2.0 + four years after launch (3.0 → Apache 2.0 on 2030-01-01). **Source-available, but not OSI-approved + open source.** +- **Maturity:** SurrealDB 3.0 reached GA on 2026-02-17, with a $23M raise and enterprise users; + explicitly positioned as a database for **"AI agent memory"** — this project's domain. +- **Model:** multi-model (relational + document + graph) with native `RELATE` edges, `->` graph + traversal, and live queries (real-time subscriptions). + +## Why it's attractive here + +- One engine natively covers the hierarchy (graph), the decision DAG (graph — ENG-014), the KB + (document), and tickets/budgets (relational) — no recursive-CTE / FK-join gymnastics for traversal. +- Live queries could drive the dashboard read-views without polling. +- "AI agent memory" positioning aligns with the KB / decision-log / provenance direction. + +## Why NOT a wholesale replacement (the hard constraints) + +1. **It does not remove Postgres.** Temporal — the chosen durable engine — needs its own backing store + and supports only PostgreSQL / MySQL / Cassandra / SQLite. **SurrealDB is not a Temporal + persistence option.** So "everything in SurrealDB" still leaves Postgres running under Temporal; + this is "add a store," not "remove one." +2. **License-purity tension.** BSL 1.1 is source-available but **not OSI open source** — the same + top-line category the project used to reject Inngest ("SSPL … ruled out on license purity"). In + practice BSL's restriction (no competing DBaaS) will never bind a self-hosted internal tool, so it + is far milder than SSPL — but adopting it means **consciously relaxing the OSI-only bar** the + project set. This needs an explicit, recorded decision. +3. **Cost of leaving Postgres+Drizzle.** Loses TS-native query types end-to-end, the Drizzle migration + tooling (`0000`–`0002`), and Postgres operational maturity, for a DB whose current major version is + months old. + +## Recommended approach (don't big-bang) + +Use the seam that already exists — the pluggable `core` persistence ports +(`IssueTracker` · `KnowledgeBase` · `Hierarchy` · `AuditLog`) selected by `PERSISTENCE_BACKEND` +(`postgres` | `github`). Add a **`surreal`** backend implementing those ports (plus the `DecisionLog` +port from ENG-014) **behind the same factory** — `core` and the workflow stay untouched (invariant +#5). Pilot with **ENG-014's decision graph** (most graph-shaped, highest upside), compare against the +Postgres+KB hybrid, then decide on broader adoption. Keep `postgres` as default and as Temporal's +backing store regardless. + +## Acceptance criteria + +- [ ] An explicit, recorded license decision: does the project relax its OSI-only bar to accept BSL + 1.1? (Documented in CLAUDE.md / OVERVIEW with rationale, alongside the Inngest rejection.) +- [ ] Confirmation in writing that Temporal continues to run on a supported store (Postgres) — i.e. + SurrealDB is additive to the stack, not a replacement of it. +- [ ] A `surreal` backend implementing at least `KnowledgeBase` + the ENG-014 `DecisionLog` port, + selectable via `PERSISTENCE_BACKEND`, without touching `core` or the workflow. +- [ ] A written comparison (modelling fit, query ergonomics for the hierarchy + decision DAG, types + story, ops/backup, maturity) vs. the Postgres+Drizzle baseline. +- [ ] A go/no-go recommendation on broader adoption (decision-log only · all agent state · not now), + with rejected alternatives recorded in the project's decision log. + +## Notes / risks + +- Pilot scope is deliberately the decision graph (ENG-014), not the whole persistence layer — keeps + the spike cheap and reversible. +- If go: budgets and the lifecycle's transactional writes need careful thought (cross-store + transactions between SurrealDB and Temporal/Postgres are not atomic). +- If no-go: the Postgres+KB hybrid in ENG-014 stands; this ticket still leaves a recorded decision. + +## Sources + +- SurrealDB license FAQ — https://surrealdb.com/license +- SurrealDB licensing repo — https://github.com/surrealdb/license +- SurrealDB 3.0 GA / "AI agent memory" — https://surrealdb.com/blog/introducing-surrealdb-3-0--the-future-of-ai-agent-memory diff --git a/project/ENG-016-implement-kappa-consensus.md b/project/ENG-016-implement-kappa-consensus.md new file mode 100644 index 0000000..2bca6f0 --- /dev/null +++ b/project/ENG-016-implement-kappa-consensus.md @@ -0,0 +1,102 @@ +# ENG-016 — Implement Kappa-style consensus (PRD-001) + +- **Status:** backlog +- **Priority:** P2 (Medium — feature; not a live-run blocker) +- **Stage:** implementation +- **Assignee role:** staff_engineer (schema + agreement function design with lead_system_design; + gate/flow with lead_architect) +- **Area:** packages/core (schema + agreement fn) · packages/agents (rater + candidate gen) · + apps/server (workflow + activities + tie-break gate) · apps/web (optional view, later phase) + +## Summary + +Build the consensus mechanism specified in [PRD-001](PRD-001-kappa-consensus.md): at the +architecture/system-design stage, three role agents (**Lead System Design, Lead Architect, Lead +Engineer**) independently rate candidate implementation directions; an inter-rater agreement +coefficient gates whether a clear winner is adopted or sent to a **tie-breaker** (default: human +escalation via the existing approval gate). All rounds are budget-enforced, audited, and recorded in +the decision log. + +## Dependencies + +- **ENG-009 (structured outputs)** — *hard*: raters must emit schema-conformant ratings, not prose. +- **ENG-014 (decision log)** — *soft*: outcomes feed the decision graph; audit-log in the interim if + ENG-014 isn't built yet. +- **ENG-002 (workflow test harness)** — for testing the consensus / tie-break branches. +- Existing: shaping stages (`SHAPING_STAGES` / `epicShaping`), approval-gate signal pattern, `ROLES`, + central budgets, audit log. + +## Build breakdown + +1. **Schema & config (core).** `ConsensusRound`, `Rating`, `Outcome` zod schemas in + `packages/core/src/schema.ts`; a `ConsensusConfig` (criteria rubric · `inputMode: "pick" | "rank"` + · threshold · tie-breaker policy). Shared with server/web via the existing core export. +2. **Agreement coefficients (core, pure).** An `agreement(ratings, mode)` module returning + `{ coefficient, metric }`. Implement **Krippendorff's alpha** as the primary (spans nominal / + ordinal / ranked, ≥3 fixed raters, missing data); support **Fleiss' / Light's kappa** (nominal + single-pick) and **Kendall's W** (ranked) per the PRD decision tree. Pure and heavily unit-tested. +3. **Aggregation/decision (core, pure).** `decide(ratings, config)` → `{ winner, consensusReached }` + via Borda count (ranked) or mean score, gated on `coefficient ≥ threshold`. +4. **Rater + candidate agents (agents).** A `rateDirections({ role, candidates, criteria, … })` helper + (sibling to `assess` / `draft` / `proposeTickets`) emitting a structured `Rating` via structured + outputs (ENG-009). Candidate generation (`proposeDirections`, or reuse the System Design draft) to + enumerate 2–4 genuinely distinct directions. +5. **Consensus workflow (server).** A durable step — a sub-stage of `epicShaping` or a dedicated + `directionConsensus(epicId, featureId)` workflow + activities: generate candidates → run the three + rater activities **independently in parallel (barrier)** → compute agreement + decide → branch. + Budget-enforced per rater (invariant #3); every step audited (invariant #2). +6. **Tie-breaker (server).** Default: human escalation — a new gate reusing + `ApprovalKind.architecture_decision`, with a workflow Signal + client fn + API endpoint mirroring + `approveRoadmap` / `approveTicket` in `temporal/client.ts` + `app.ts`. Optional (later): a + domain-weighted casting-vote role or a dedicated tie-breaker persona (roles-as-config, invariant #6). +7. **Persistence / provenance.** Persist round/ratings/outcome; append audit events; emit a decision + node to ENG-014 (or audit-only interim), traceable to the originating request (invariant #1). +8. **Config & roles.** Criteria rubric, input mode, threshold, and tie-breaker policy as per-unit + config (not hardcoded — invariant #6). +9. **Tests.** Agreement-fn unit tests with fixtures for perfect / zero / negative / **degenerate** + agreement (see risks); rater-output parsing; workflow-branch tests (consensus-reached vs + tie-break→gate) via the ENG-002 harness. + +## Phasing (mirrors the PRD rollout) + +- **Phase 1** — single-pick + Fleiss'/Krippendorff; human-escalation tie-breaker; behind a flag and + **advisory** (record the round + score, don't block). +- **Phase 2** — ranked input + Kendall's W / Krippendorff's alpha + Borda; promote to a real gate. +- **Phase 3** — optional agent casting-vote / tie-breaker persona; optional dashboard view (fold into + ENG-010). + +## Acceptance criteria + +- [ ] `core` schemas + `ConsensusConfig` for round/rating/outcome, shared via zod. +- [ ] A pure, unit-tested agreement function — Krippendorff's alpha primary, with Fleiss'/Light's and + Kendall's W — covering perfect / none / partial / degenerate agreement. +- [ ] A pure aggregation/decision function (winner + `consensusReached` vs threshold). +- [ ] A rater agent helper emits **structured** ratings (ENG-009) for each of the three roles, + **independently** (no rater sees another's input); candidate generation yields 2–4 distinct + directions. +- [ ] A durable consensus step runs the parallel raters, computes agreement, and branches on the + threshold; budget-enforced and audited. +- [ ] Tie-break default = human escalation via the `architecture_decision` gate (Signal + endpoint, + mirroring `approveRoadmap`); resolution recorded with the deciding identity. +- [ ] Round / ratings / outcome persisted and linked into the decision log (ENG-014) — or audit-logged + if ENG-014 isn't yet available — traceable to the originating request. +- [ ] Config-driven (criteria · input mode · threshold · tie-breaker policy) per unit; no-op/skipped + path when the agent runtime is unavailable. +- [ ] Workflow tests cover both the consensus-reached and tie-break→gate paths. + +## Risks / implementation notes + +- **Degenerate coefficient cases:** when all raters rate identically (or every candidate scores the + same), the chance-agreement term can make kappa **undefined** (0/0). Handle explicitly — unanimous + agreement must map to "consensus", not `NaN`. +- **Correlated raters (all Claude):** independence is procedural; reinforce with distinct rubrics/lenses + per rater and genuinely distinct candidates, and watch for suspiciously high agreement (PRD risk). +- **Cost/latency:** N rater runs per decision — bound candidates to 2–4 and only trigger when ≥2 + viable options exist; per-role budgets cap spend. +- **Cross-store atomicity:** if ENG-015 (SurrealDB) lands, the round/decision writes and the + Temporal/Postgres state are not transactional together — make writes idempotent and audit-anchored. + +## Out of scope + +- The dashboard consensus-round view beyond a minimal read (Phase 3 / ENG-010). +- Non-`architecture_decision` tie-breaker policies beyond the configurable default. diff --git a/project/PRD-001-kappa-consensus.md b/project/PRD-001-kappa-consensus.md new file mode 100644 index 0000000..3662cd5 --- /dev/null +++ b/project/PRD-001-kappa-consensus.md @@ -0,0 +1,203 @@ +# PRD-001 — Kappa-style consensus for implementation-direction decisions + +- **Status:** Draft +- **Author role:** PM (with Lead Architect + Lead System Design input) +- **Lifecycle stage touched:** architecture + system design (shaping) +- **Date:** 2026-06-27 +- **Related:** existing shaping stages (`epicShaping` / `SHAPING_STAGES`) · approval gates + (`ApprovalKind.architecture_decision`) · ENG-008 (cyclic rework) · ENG-009 (structured outputs) · + ENG-014 (decision log) + +## TL;DR + +At the architecture / system-design stage, when a product feature has more than one viable +implementation direction, introduce a structured **consensus step**. The three senior technical role +agents — **Lead System Design, Lead Architect, Lead Engineer** — independently evaluate the candidate +directions; an **inter-rater agreement coefficient** (Cohen's/Fleiss' kappa family) quantifies how +much they actually agree, corrected for chance. If there is a clear winner *and* agreement clears a +configurable threshold, that direction is adopted. Otherwise a **tie-breaker** (default: escalate to +the human lead via the existing approval gate) resolves it. Every candidate, rating, rationale, +agreement score, and outcome is audited and recorded in the decision log (ENG-014), traceable to the +originating request. + +## Interpretation note + +"Kappa-style" is read here as **inter-rater reliability** — Cohen's kappa / Fleiss' kappa, the +statistics for agreement among multiple raters corrected for chance agreement. This is deliberately +*not* "Kappa architecture" (a streaming-data pattern) and *not* a distributed-consensus protocol +(Raft/Paxos). The fit is: several role agents are *raters*, and we want a principled measure of their +agreement plus a tie-breaker. If a different meaning was intended, flag it before build. + +## Problem & motivation + +- Shaping currently runs **sequentially** (PM → UX → Architect → System Design), each agent drafting + one artifact handed downstream. There is no point at which the senior technical roles *converge on a + direction* or surface and resolve disagreement. +- A single agent's chosen direction can be plausible-but-wrong, and there is no independent + cross-check and no signal of how aligned/confident the unit actually is. +- When roles would disagree, nothing makes that disagreement visible or resolves it explicitly. + +## Goals + +- Make implementation-direction decisions the product of **independent** senior assessments, not one + agent's pick. +- Produce a **quantified agreement signal** and require an explicit resolution path when agreement is + low. +- Provide a **configurable tie-breaker**, defaulting to human-in-the-loop. +- Leave an **auditable, traceable** record that feeds the decision log (ENG-014). + +## Non-goals + +- Not a distributed-systems consensus protocol. +- Not applied to every ticket — scoped to arch/system-design-stage *direction* decisions where + multiple viable options genuinely exist. +- Not a removal of human authority — the approval gate remains the backstop. + +## Users & stakeholders + +- **Human lead** — accountable; ultimate tie-breaker; reviews outcomes. +- **Lead System Design, Lead Architect, Lead Engineer** agents — the raters. +- **Optional tie-breaker persona** — a neutral fourth agent (config-only to add; invariant #6). +- **Staff Eng / QA** — downstream consumers of the chosen direction. + +## Background — agreement coefficients (and how to use them honestly) + +| Coefficient | Raters | Input | Fit here | +|-------------|--------|-------|----------| +| Cohen's kappa | 2 | categorical | n/a (we have 3 raters) | +| **Fleiss' kappa** | ≥3 | categorical | each rater **picks one** direction | +| **Kendall's W** (concordance) | ≥3 | rankings | each rater **ranks** the directions — richest signal | +| Krippendorff's alpha | any | any level | general-purpose fallback (ordinal scores, missing data) | + +All correct for chance agreement, so they distinguish *genuine* agreement from coincidence. +Interpretation follows the Landis–Koch bands (≤0 poor … 0.61–0.80 substantial … 0.81–1 almost +perfect). + +**Fleiss', not Cohen's — and why.** Cohen's kappa is defined for exactly **two** raters, so with our +three fixed raters it is ruled out for the panel as a whole. Fleiss' generalises to ≥3 raters but +technically assumes *interchangeable* raters; because our three are the **same identified roles** every +round, the more principled *nominal* measure is **Light's kappa** (the mean of the three pairwise +Cohen's kappas — so Cohen's survives as the pairwise building block). Fleiss' ≈ Light's in practice and +is far more widely implemented, so Fleiss' is an acceptable pragmatic stand-in for single-pick mode. +For the **recommended ranked** mode use **Kendall's W**; for ordinal per-criterion scores, +**Krippendorff's alpha**. **Implementation recommendation:** standardise on **Krippendorff's alpha** as +the single primary coefficient — it spans nominal / ordinal / ranked input, ≥3 fixed raters, and +missing ratings, so the coefficient need not change between Phase 1 (single-pick) and Phase 2 (ranked). +Decision tree: *2 raters → Cohen's · 3+ categorical → Fleiss' (Light's for fixed raters) · 3+ ranked → +Kendall's W · one coefficient for all → Krippendorff's alpha.* + +> **Honest caveat that shapes the design:** with only **3 raters** and a handful of candidates, any +> chance-corrected coefficient is statistically **noisy** — a single flipped vote swings it a lot. +> Therefore the coefficient is used as a **confidence signal feeding a decision rule**, *not* as a +> hard statistical inference. The primary decision is the aggregate winner (Borda/mean); the +> coefficient gates whether that winner is trusted or sent to the tie-breaker. + +## Proposed solution + +### Flow + +1. **Candidate generation** — produce 2–4 candidate implementation directions for the feature (each: + summary, key tradeoffs, rough cost/risk). Source: the System Design agent enumerates options, or a + fan-out where each of the three roles proposes one and duplicates are merged. +2. **Independent rating** — each of the three raters, *independently and without seeing the others' + input*, scores or ranks the candidates against a shared rubric. Independence is what makes the + agreement measure meaningful (this mirrors the judge-panel / independent-perspectives pattern). +3. **Aggregate + measure agreement** — aggregate into a ranked outcome (Borda count for rankings, or + mean score) and compute the agreement coefficient (Kendall's W for rankings; Fleiss' kappa for + single-picks). +4. **Decision rule** — *consensus* = (a clear top candidate) **AND** (agreement ≥ configurable + threshold, default ≈ 0.6 "substantial"). If met → adopt that direction. +5. **Tie-breaker** — if there is no clear winner **OR** agreement < threshold → invoke the configured + tie-breaker (below). +6. **Record + gate** — persist the chosen direction, all ratings + rationale, the agreement score, and + any tie-break to the audit log + decision log (ENG-014). Optionally require human sign-off via the + existing `architecture_decision` approval gate. + +### Rating criteria + +A shared, explicit rubric (configurable per unit), each criterion on a fixed **ordinal scale** so +ratings are comparable. Starting set: *simplicity · scalability · delivery risk · ADR alignment · +cost-to-build · reversibility · testability*. The rubric is part of config, not code. + +### Tie-breaker (the explicit requirement) + +Configurable, with a recommended default: + +- **(Default) Human escalation** via the existing approval gate (`ApprovalKind.architecture_decision`) + — safest, aligns with invariant #4. The human sees every rating and the disagreement, then decides. +- **Casting-vote role** — a domain-weighted deciding vote: Lead Architect casts on + architecture-dominant questions; Lead System Design on interface/data-design-dominant ones. For + higher autonomy. +- **Dedicated tie-breaker agent** — a neutral fourth persona with a *distinct* rubric/lens (e.g. + risk-first / devil's-advocate) producing a casting assessment. Adding it is a config change + (invariant #6). + +**Recommendation:** default to human escalation; allow opt-in to an agent casting vote or tie-breaker +persona for unattended runs, configured per unit. + +### Workflow integration + +- A new durable step at the architecture/system-design point of shaping — either a sub-stage of + `epicShaping` or a dedicated `directionConsensus(epicId, featureId)` Temporal workflow. +- Reuses the multi-agent fan-out: independent rater activities run in **parallel** (barrier to collect + all), then compute agreement, branch on the threshold, and (on tie-break) Signal the human gate. +- Budgets enforced per rater run (invariant #3); all events audited (invariant #2); outcome feeds the + decision log (ENG-014) and is goal-traceable (invariant #1). Raters emit **structured** ratings via + schema (ENG-009) so scores are parseable, not prose. + +### Data model (sketch) + +- **ConsensusRound** — `id`, `epicId`/`featureId`, `candidates[]`, `criteria[]`, `raterRoleIds[]`, + `status`. +- **Rating** — `roundId`, `raterRole`, per-candidate scores/rank, `rationale`, `costCents`. +- **Outcome** — `roundId`, `chosenCandidate`, `aggregateMethod`, `agreementCoefficient` (+ which + metric), `consensusReached` (bool), `tieBreaker` `{type, decidedBy}`, `decisionId` (→ ENG-014). + +## Success metrics / acceptance criteria + +- [ ] Direction decisions at the arch/system-design stage produce ≥2 candidates and 3 independent + ratings. +- [ ] An agreement coefficient is computed and recorded per round, against a configurable threshold. +- [ ] When consensus is not reached, the configured tie-breaker resolves it (default: human gate) and + the resolution is recorded with the deciding identity. +- [ ] Every round (candidates, ratings, rationale, score, outcome, tie-break) is audited and linked + into the decision log (ENG-014), traceable to the originating request. +- [ ] Raters operate independently — no rater sees another's input before submitting. +- [ ] Budgets enforced; audited no-op/skipped path when the agent runtime is unavailable. +- **Health metrics to watch:** tie-break rate (too high → criteria unclear or candidates too similar; + too low → raters may be correlated), and human-override rate at the gate. + +## Rollout / phasing + +1. **Phase 1** — categorical single-pick + Fleiss' kappa; human-escalation tie-breaker only; behind a + flag and **advisory** (record the round and score but don't block). +2. **Phase 2** — ranked input + Kendall's W + Borda aggregation; promote to a real gate. +3. **Phase 3** — optional agent casting-vote / tie-breaker persona for unattended runs. + +## Risks & mitigations + +- **Statistical noise (3 raters):** the coefficient is a confidence signal feeding the decision rule, + not a hard inference; the Borda/majority winner is primary. +- **Correlated raters (all Claude):** independence is procedural (no shared draft), but the models may + still correlate and inflate agreement; mitigate with *distinct rubrics/lenses* per rater, genuinely + distinct candidates, and (optionally) a devil's-advocate rater. Watch for suspiciously high + agreement. +- **Ambiguous criteria → unstable ratings:** require an explicit fixed rubric and ordinal scale; + iterate the criteria over time. +- **Cost / latency:** N rater runs per decision — bound candidates to 2–4, gate to genuinely + multi-option decisions, and rely on per-role budgets for the ceiling. +- **Over-triggering:** only invoke when ≥2 viable candidates exist; otherwise shaping proceeds + normally. + +## Open questions + +- Default input mode: single-pick (Fleiss) vs ranked (Kendall's W)? (Recommend ranked.) +- Default consensus threshold value, and per-unit configurability. +- Does the consensus outcome *always* pass through the human `architecture_decision` gate, or only on + tie-break? +- If a tie-breaker agent is used, does it need a different model / effort tier than the raters? + +## Next step + +Implementation is tracked in **[ENG-016](ENG-016-implement-kappa-consensus.md)**, depending on ENG-009 +(structured rater outputs) and feeding ENG-014 (decision log). diff --git a/project/README.md b/project/README.md new file mode 100644 index 0000000..f67f99d --- /dev/null +++ b/project/README.md @@ -0,0 +1,35 @@ +# Project tickets + +Backlog of work items derived from a codebase review (2026-06-27). Each ticket is self-contained +and uses the project's own vocabulary — `status` / `stage` / `assignee role` / `acceptance criteria` +mirror the `TicketStatus`, `LifecycleStage`, and `RoleId` enums in `packages/core/src/schema.ts`. + +**See [ROADMAP.md](ROADMAP.md) for the recommended implementation order** (dependency-sequenced waves ++ critical path). + +| # | Title | Priority | Area | +|---|-------|----------|------| +| [ENG-001](ENG-001-repo-aware-coding-agents.md) | Give coding agents real repository context | P0 | packages/agents | +| [ENG-002](ENG-002-test-orchestration-layer.md) | Test the Temporal orchestration layer | P1 | apps/server | +| [ENG-003](ENG-003-ci-for-this-repo.md) | Add CI for this repository | P1 | repo / .github | +| [ENG-004](ENG-004-api-auth-and-approval-identity.md) | API authentication + approval identity | P0 | apps/server | +| [ENG-005](ENG-005-scrub-cli-sandbox-env.md) | Scrub the environment for CLI agent sandboxes | P1 | packages/agents | +| [ENG-006](ENG-006-first-class-approvals.md) | Make approval gates first-class, persisted entities | P2 | core / server | +| [ENG-007](ENG-007-budget-model-reset-and-reservation.md) | Fix the budget model (reset + reservation) | P2 | packages/db | +| [ENG-008](ENG-008-cyclic-rework-routing.md) | Implement cyclic review/blocked rework routing | P2 | apps/server | +| [ENG-009](ENG-009-structured-outputs-for-agents.md) | Use structured outputs for agent response parsing | P1 | packages/agents | +| [ENG-010](ENG-010-dashboard-budgets-and-cost.md) | Surface budgets and cost in the dashboard | P2 | apps/web | +| [ENG-011](ENG-011-decide-initial-issue-tracker.md) | Decide the initial issue tracker | P3 | core / integrations | +| [ENG-012](ENG-012-research-a2a-protocol.md) | Research Google's A2A (Agent-to-Agent) protocol | P2 | architecture | +| [ENG-013](ENG-013-repo-provisioning-and-shared-workspace.md) | Repository provisioning via the PM agent (git + gh) & two-root workspace | P0 | core · agents · server | +| [ENG-014](ENG-014-decision-log-and-provenance-tree.md) | Per-agent decision logs & a traceable decision tree | P2 | core · db · server | +| [ENG-015](ENG-015-evaluate-surrealdb-backend.md) | Evaluate SurrealDB as a persistence backend (decision spike) | P2 | core · db · infra | +| [ENG-016](ENG-016-implement-kappa-consensus.md) | Implement Kappa-style consensus (PRD-001) | P2 | core · agents · server | + +**Priority key:** P0 = do before any live autonomous run · P1 = high · P2 = medium · P3 = low. + +## Product requirements (PRDs) + +| Doc | Summary | +|-----|---------| +| [PRD-001](PRD-001-kappa-consensus.md) | Kappa-style consensus for implementation-direction decisions (Lead System Design · Lead Architect · Lead Engineer + tie-breaker) | diff --git a/project/ROADMAP.md b/project/ROADMAP.md new file mode 100644 index 0000000..553ad5e --- /dev/null +++ b/project/ROADMAP.md @@ -0,0 +1,75 @@ +# Implementation roadmap + +Recommended order for the [tickets](README.md), derived from their dependencies, priorities, and +leverage. Ordering principles, in order: **(1)** foundations that de-risk everything first; **(2)** +respect hard dependencies; **(3)** the two P0 "before a live run" concerns early; **(4)** pull +high-leverage enablers forward even when labelled P1/P2. + +## Dependency map (B must precede A) + +- **ENG-001** ← ENG-013, ENG-009, ENG-005 +- **ENG-013** ← ENG-005 +- **ENG-006** ← ENG-004 +- **ENG-010** ← ENG-007 +- **ENG-008** ← ENG-007, ENG-002 +- **ENG-014** ← ENG-009, (ENG-015 *decision*) +- **ENG-016** ← ENG-009, ENG-002, ENG-014 +- Roots (no inbound deps): ENG-002, ENG-003, ENG-004, ENG-005, ENG-007, ENG-009, ENG-011, ENG-012, ENG-015 + +## Waves (items within a wave parallelise) + +### Wave 0 — Foundations — ✅ complete (2026-06-27) +- **ENG-003** ✅ — CI for this repo (`.github/workflows/ci.yml`; badge + branch-protection note). +- **ENG-002** ✅ — Temporal workflow test harness (8 tests via `@temporalio/testing`; 59 total green). +- *Discovery (decisions recorded in the tickets):* **ENG-012** ✅ (A2A — decline now, monitor, keep + port-shaped), **ENG-011** ✅ (tracker — GitHub Issues default; Postgres for local), **ENG-015** ✅ + (SurrealDB — no-go on wholesale replacement; one item awaits the lead: relax the OSI-only license + bar? recommended *no* for now). + +### Wave 1 — Safety + enablers (independent, high-leverage) +- **ENG-004** — API auth + approval identity (P0 security gate before any live/shared run). +- **ENG-009** — Structured outputs (enabler for ENG-001 quality, ENG-014, ENG-016). +- **ENG-005** — Scrub the CLI sandbox env (security; prerequisite for the repo-context chain). + +### Wave 2 — Repo-context critical path +- **ENG-013** — Repo provisioning (PM `git`/`gh`) + two-root workspace. Needs ENG-005. +- **ENG-001** — Repo-aware coding agents. Needs ENG-013; benefits from ENG-009. + +### Wave 3 — Governance / observability +- **ENG-007** — Budget model (reset + reservation). Do first in this wave — gates 010 and 008. +- **ENG-006** — First-class approvals. Builds on ENG-004's identity. +- **ENG-010** — Budget/cost dashboard. Needs ENG-007. +- **ENG-008** — Cyclic rework routing. Needs ENG-007 + ENG-002. + +### Wave 4 — Provenance + consensus +- **ENG-014** — Decision log / provenance tree. Needs ENG-009; build on the Postgres+KB default behind + its port (see flag 3) rather than waiting on a SurrealDB migration. +- **ENG-016** — Kappa-style consensus (PRD-001). Needs ENG-009 + ENG-002; soft-needs ENG-014. + +## Critical path (to a safe, useful autonomous run) + +``` +ENG-003/002 → ENG-009 + ENG-005 + ENG-004 → ENG-013 → ENG-001 +(foundations) (Wave 1) (Wave 2 — usable code) +``` + +Everything else branches off the sides of this spine. Minimum viable sequencing: Wave 0 → Wave 1; add +Wave 2 to make autonomous runs produce usable code. + +## Single-threaded sequence (if not parallelising) + +1. ENG-003 · 2. ENG-002 · 3. ENG-004 · 4. ENG-009 · 5. ENG-005 · 6. ENG-013 · 7. ENG-001 · +8. ENG-007 · 9. ENG-006 · 10. ENG-010 · 11. ENG-008 · 12. ENG-015 · 13. ENG-014 · 14. ENG-016 + +(ENG-011 and ENG-012 slot in as parallel discovery whenever there's slack.) + +## Ordering flags + +1. **ENG-005 (P1) gates a P0.** ENG-013 depends on it — treat it as Wave 1, not "later with the P1s." +2. **ENG-009 is the highest-leverage single item.** P1, but unlocks a P0 (ENG-001 quality) plus ENG-014 + and ENG-016 — pulling it early is the biggest momentum multiplier. +3. **Don't let ENG-015 block ENG-014.** Make the SurrealDB *decision* early, but build ENG-014 on the + Postgres+KB default behind its port; migrate later only if ENG-015 says "go" — no rework, thanks to + the port abstraction. +4. **Ship auth (ENG-004) before enriching it (ENG-006).** Auth + a recorded principal is the P0; + first-class persisted approval records are the P2 enhancement on top.