diff --git a/.env.example b/.env.example index 5a75c30..27174f9 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,11 @@ GITHUB_DEPLOY_REF=main # Persistence backend for agent state (tracker + knowledge base): postgres (default) or github. PERSISTENCE_BACKEND=postgres GITHUB_DOCS_PREFIX=docs + +# Kappa-style consensus (ENG-016 / PRD-001). Off by default; trigger a round per epic via +# POST /api/epics/:id/consensus. Phase 1 is advisory (records the round + agreement score, no gate). +CONSENSUS_ENABLED=false +# CONSENSUS_INPUT_MODE=rank # "rank" (Borda + Kendall/Krippendorff) or "pick" (Fleiss) +# CONSENSUS_THRESHOLD=0.6 # agreement gate (Landis–Koch "substantial") +# CONSENSUS_ADVISORY=true # false → promote to a real architecture_decision gate (Phase 2) +# CONSENSUS_TIEBREAKER=human # "human" (escalate to the gate) or "casting_vote" diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 85f486c..013e0fc 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -4,8 +4,10 @@ import { type AppEnv, authMiddleware } from "./auth" import { persistenceFromEnv } from "./persistence" import { artifactPath, SHAPING_STAGES } from "./shaping" import { + approveArchitecture, approveRoadmap, approveTicket, + startDirectionConsensus, startEpicDecomposition, startEpicShaping, startTicketLifecycle, @@ -105,6 +107,29 @@ app.get("/api/epics/:id/artifacts", async (c) => { return c.json({ artifacts }) }) +// Kappa-style consensus (ENG-016): the senior technical roles independently rate candidate +// implementation directions; the agreement coefficient gates adopt-vs-tie-break. +app.post("/api/epics/:id/consensus", async (c) => { + const id = c.req.param("id") + await startDirectionConsensus(id) + return c.json({ epicId: id, consensus: true }) +}) + +// Architecture-decision tie-break (invariant #4): release the gate so the consensus round resolves. +app.post("/api/epics/:id/approve-architecture", async (c) => { + const id = c.req.param("id") + const signaled = await approveArchitecture(id) + const by = c.get("actor") + await resolveApproval({ kind: "architecture_decision", epicId: id, decidedBy: by }) + await persistence.audit.append({ + actor: "human", + kind: "approval_decided", + ticketId: null, + payload: { epicId: id, gate: "architecture", by, signaled }, + }) + return c.json({ epicId: id, gate: "architecture", signaled, by }) +}) + // Agent-driven decomposition: the Lead Engineer breaks the epic into backlog tickets — but behind a // roadmap sign-off gate (the workflow blocks until the human approves the plan, below). app.post("/api/epics/:id/decompose", async (c) => { diff --git a/apps/server/src/consensus.ts b/apps/server/src/consensus.ts new file mode 100644 index 0000000..ab498a3 --- /dev/null +++ b/apps/server/src/consensus.ts @@ -0,0 +1,30 @@ +import { type ConsensusConfig, DEFAULT_CONSENSUS_CONFIG } from "@eng/core" + +/** + * Per-unit consensus configuration (ENG-016), resolved from env so the rubric / mode / threshold / + * tie-breaker are config, not code (invariant #6). Phase 1 ships **advisory** (record the round + score + * but don't block) and behind `CONSENSUS_ENABLED`. + */ +export function consensusEnabled(): boolean { + return process.env.CONSENSUS_ENABLED === "true" +} + +export function consensusConfigFromEnv(): ConsensusConfig { + return { + ...DEFAULT_CONSENSUS_CONFIG, + inputMode: + process.env.CONSENSUS_INPUT_MODE === "pick" + ? "pick" + : process.env.CONSENSUS_INPUT_MODE === "rank" + ? "rank" + : DEFAULT_CONSENSUS_CONFIG.inputMode, + threshold: process.env.CONSENSUS_THRESHOLD + ? Number(process.env.CONSENSUS_THRESHOLD) + : DEFAULT_CONSENSUS_CONFIG.threshold, + // Advisory by default (Phase 1); set CONSENSUS_ADVISORY=false to promote to a real gate (Phase 2). + advisory: process.env.CONSENSUS_ADVISORY + ? process.env.CONSENSUS_ADVISORY !== "false" + : DEFAULT_CONSENSUS_CONFIG.advisory, + tieBreaker: process.env.CONSENSUS_TIEBREAKER === "casting_vote" ? "casting_vote" : "human", + } +} diff --git a/apps/server/src/temporal/activities.ts b/apps/server/src/temporal/activities.ts index 0f8c5d7..923c545 100644 --- a/apps/server/src/temporal/activities.ts +++ b/apps/server/src/temporal/activities.ts @@ -3,15 +3,23 @@ import { draft, ensureRepoCloned, estimateRunCostCents, + proposeDirections, proposeFileChanges, proposeTickets, + rateDirections, repoWorkspacePath, } from "@eng/agents" import { + type ConsensusCandidate, + type ConsensusConfig, + type ConsensusInputMode, type DeliveryAdapter, type DeployState, + decide, type PullRequestRef, + type Rating, ROLES, + type RoleId, type TicketStatus, } from "@eng/core" import { createApproval, getBudgetRemaining, reconcileSpend, reserveBudget } from "@eng/db" @@ -20,6 +28,7 @@ import { LocalGitDeliveryAdapter, localGitBranchFiles, } from "@eng/integrations" +import { consensusConfigFromEnv } from "../consensus" import { persistenceFromEnv } from "../persistence" import { artifactPath, SHAPING_STAGES } from "../shaping" import { startTicketLifecycle } from "./client" @@ -661,3 +670,197 @@ export async function recordDeploy(ticketId: string, state: DeployState): Promis payload: { state }, }) } + +// --- Kappa-style consensus (ENG-016 / PRD-001) ------------------------------- + +/** Candidates + the resolved per-unit config — the workflow needs both to fan out the raters. */ +export interface ConsensusKickoff { + candidates: ConsensusCandidate[] + config: ConsensusConfig +} + +/** + * Generate 2–4 distinct candidate implementation directions (Lead System Design), informed by the + * epic's shaping artifacts. Budget-enforced + audited; returns null (no-op) when the runtime is + * unavailable or the budget is exhausted, so the lifecycle proceeds normally. + */ +export async function generateDirections(epicId: string): Promise { + const config = consensusConfigFromEnv() + const role = ROLES.lead_system_design + const remaining = (await getBudgetRemaining(role.id)) ?? role.monthlyBudgetCents + const held = await reserveBudget(role.id, estimateRunCostCents(remaining)) + if (held === null) { + await persistence.audit.append({ + actor: "system", + kind: "consensus_skipped", + ticketId: null, + payload: { epicId, stage: "candidates", reason: "budget_exhausted" }, + }) + return null + } + const [epicCtx, artifacts] = await Promise.all([ + persistence.hierarchy.epicContext(epicId), + epicArtifacts(epicId), + ]) + try { + const proposed = await proposeDirections({ + role: role.id, + systemPrompt: role.systemPrompt, + goalContext: [epicCtx, ...artifacts].join("\n\n"), + task: "Enumerate the candidate implementation directions for this epic's core feature.", + budgetCentsRemaining: remaining, + }) + await reconcileSpend(role.id, held, proposed.costCents) + const candidates: ConsensusCandidate[] = proposed.candidates.map((c, i) => ({ + id: `c${i + 1}`, + title: c.title, + summary: c.summary, + tradeoffs: c.tradeoffs, + })) + await persistence.audit.append({ + actor: role.id, + kind: "consensus_candidates", + ticketId: null, + payload: { epicId, candidates: candidates.length, costCents: proposed.costCents }, + }) + return { candidates, config } + } catch (err) { + await reconcileSpend(role.id, held, 0) + await persistence.audit.append({ + actor: "system", + kind: "consensus_skipped", + ticketId: null, + payload: { epicId, stage: "candidates", error: String(err) }, + }) + return null + } +} + +/** + * One rater scores/ranks the candidates **independently** (no rater sees another's input — that's + * what makes the agreement coefficient meaningful). Budget-enforced per role; audited. Returns null + * when the runtime is unavailable so `decide` simply has one fewer rating. + */ +export async function rateDirection( + epicId: string, + candidates: ConsensusCandidate[], + raterRole: RoleId, + inputMode: ConsensusInputMode, +): Promise { + const role = ROLES[raterRole] + const remaining = (await getBudgetRemaining(role.id)) ?? role.monthlyBudgetCents + const held = await reserveBudget(role.id, estimateRunCostCents(remaining)) + if (held === null) { + await persistence.audit.append({ + actor: "system", + kind: "consensus_skipped", + ticketId: null, + payload: { epicId, stage: "rating", raterRole, reason: "budget_exhausted" }, + }) + return null + } + try { + const epicCtx = await persistence.hierarchy.epicContext(epicId) + const { rating, costCents } = await rateDirections({ + role: role.id, + systemPrompt: role.systemPrompt, + goalContext: epicCtx, + candidates: candidates.map((c) => ({ id: c.id, title: c.title, summary: c.summary })), + criteria: consensusConfigFromEnv().criteria, + inputMode, + budgetCentsRemaining: remaining, + }) + await reconcileSpend(role.id, held, costCents) + await persistence.audit.append({ + actor: role.id, + kind: "consensus_rating", + ticketId: null, + payload: { epicId, pick: rating.pick, ranking: rating.ranking, costCents }, + }) + return rating + } catch (err) { + await reconcileSpend(role.id, held, 0) + await persistence.audit.append({ + actor: "system", + kind: "consensus_skipped", + ticketId: null, + payload: { epicId, stage: "rating", raterRole, error: String(err) }, + }) + return null + } +} + +/** + * Aggregate the ratings, compute the agreement coefficient, and decide. Persists the outcome to the + * audit log and the decision graph (ENG-014), traceable to the originating request. Returns whether + * consensus was reached (the workflow branches on it for the tie-breaker). + */ +export async function decideConsensus( + epicId: string, + candidates: ConsensusCandidate[], + ratings: (Rating | null)[], +): Promise<{ consensusReached: boolean; winner: string | null }> { + const config = consensusConfigFromEnv() + const valid = ratings.filter((r): r is Rating => r !== null) + const decision = decide(candidates, valid, config) + const winnerCand = candidates.find((c) => c.id === decision.winner) ?? null + const totalCost = valid.reduce((s, r) => s + (r.costCents ?? 0), 0) + + const event = await persistence.audit.append({ + actor: "lead_architect", + kind: "consensus_decided", + ticketId: null, + payload: { + epicId, + winner: decision.winner, + consensusReached: decision.consensusReached, + coefficient: decision.coefficient, + metric: decision.metric, + method: decision.aggregateMethod, + threshold: config.threshold, + advisory: config.advisory, + raters: valid.length, + }, + }) + await recordStepDecision({ + epicId, + actor: "lead_architect", + stage: "architecture", + statement: winnerCand + ? `Consensus direction: ${winnerCand.title}` + : "No clear consensus direction", + rationale: + `${decision.aggregateMethod} winner from ${valid.length} independent raters; agreement ` + + `${decision.metric}=${decision.coefficient.toFixed(2)} vs threshold ${config.threshold} → ` + + `${decision.consensusReached ? "consensus reached" : "tie-break required"}`, + alternatives: candidates + .filter((c) => c.id !== decision.winner) + .map((c) => ({ option: c.title, rejectedBecause: "not the consensus winner" })), + confidence: Math.max(0, Math.min(1, decision.coefficient)), + costCents: totalCost, + auditEventId: event.id, + }) + return { consensusReached: decision.consensusReached, winner: decision.winner } +} + +/** Record that an epic's implementation direction awaits the human architecture sign-off (tie-break). + * Creates a first-class pending approval (ENG-006) plus the audit event. */ +export async function requestArchitectureApproval(epicId: string): Promise { + await createApproval({ kind: "architecture_decision", epicId, requestedByRole: "lead_architect" }) + await persistence.audit.append({ + actor: "lead_architect", + kind: "architecture_requested", + ticketId: null, + payload: { epicId }, + }) +} + +/** Record the human architecture sign-off (the tie-break gate released the consensus). */ +export async function recordArchitectureApproval(epicId: string): Promise { + await persistence.audit.append({ + actor: "human", + kind: "architecture_approved", + ticketId: null, + payload: { epicId }, + }) +} diff --git a/apps/server/src/temporal/client.ts b/apps/server/src/temporal/client.ts index 6bb9187..d81ffe6 100644 --- a/apps/server/src/temporal/client.ts +++ b/apps/server/src/temporal/client.ts @@ -64,6 +64,34 @@ export async function startEpicDecomposition(epicId: string): Promise { } } +/** Start Kappa-style direction consensus for an epic (idempotent while one is already running). */ +export async function startDirectionConsensus(epicId: string): Promise { + const client = await getTemporalClient() + try { + await client.workflow.start("directionConsensus", { + taskQueue: TASK_QUEUE, + workflowId: `epic-consensus-${epicId}`, + args: [epicId], + }) + } catch (err) { + if ((err as { name?: string })?.name === "WorkflowExecutionAlreadyStartedError") return + throw err + } +} + +/** Release the architecture-decision tie-break gate for an epic's consensus round (ENG-016). Returns + * false if no consensus round is waiting (advisory, already resolved, or never started). */ +export async function approveArchitecture(epicId: string): Promise { + const client = await getTemporalClient() + const handle = client.workflow.getHandle(`epic-consensus-${epicId}`) + try { + await handle.signal("architectureApprove") + return true + } catch { + return false + } +} + /** Release the roadmap sign-off gate for an epic. Returns false if no decomposition is waiting * (already decomposed, or never started) — a benign no-op for the UI. */ export async function approveRoadmap(epicId: string): Promise { diff --git a/apps/server/src/temporal/workflows.test.ts b/apps/server/src/temporal/workflows.test.ts index ff27917..9e4aa2c 100644 --- a/apps/server/src/temporal/workflows.test.ts +++ b/apps/server/src/temporal/workflows.test.ts @@ -4,6 +4,8 @@ import { bundleWorkflowCode, Worker, type WorkflowBundle } from "@temporalio/wor import { afterAll, beforeAll, describe, expect, it, vi } from "vitest" import { approveSignal, + architectureSignal, + directionConsensus, epicDecomposition, epicShaping, heartbeat, @@ -56,6 +58,34 @@ function makeActivities(overrides: Record = {}) { requestRoadmapSignoff: vi.fn(async (_epicId: string) => {}), recordRoadmapApproval: vi.fn(async (_epicId: string) => {}), requestApproval: vi.fn(async (_kind: string, _ticketId: string) => {}), + // Consensus (ENG-016): two candidates, three raters, a clear non-advisory winner by default. + generateDirections: vi.fn(async (_epicId: string) => ({ + candidates: [ + { id: "c1", title: "A", summary: "", tradeoffs: [] }, + { id: "c2", title: "B", summary: "", tradeoffs: [] }, + ], + config: { + inputMode: "rank", + threshold: 0.6, + tieBreaker: "human", + criteria: [], + raterRoles: ["lead_system_design", "lead_architect", "lead_engineer"], + advisory: false, + }, + })), + rateDirection: vi.fn(async (_e: string, _c: unknown, role: string, _m: string) => ({ + raterRole: role, + pick: null, + ranking: ["c1", "c2"], + rationale: "", + costCents: 0, + })), + decideConsensus: vi.fn(async (_e: string, _c: unknown, _r: unknown) => ({ + consensusReached: true, + winner: "c1", + })), + requestArchitectureApproval: vi.fn(async (_epicId: string) => {}), + recordArchitectureApproval: vi.fn(async (_epicId: string) => {}), ...overrides, } } @@ -235,6 +265,97 @@ describe("epicShaping", () => { }, 30_000) }) +describe("directionConsensus", () => { + it("runs the raters in parallel and adopts a clear winner without a tie-break gate", async () => { + const acts = makeActivities() + await withWorker(acts, async () => { + await env.client.workflow.execute(directionConsensus, { + taskQueue: TQ, + workflowId: nextId("consensus"), + args: ["e1"], + }) + }) + expect(acts.generateDirections).toHaveBeenCalledWith("e1") + expect(acts.rateDirection).toHaveBeenCalledTimes(3) // one per rater role, in parallel + expect(acts.decideConsensus).toHaveBeenCalledTimes(1) + expect(acts.requestArchitectureApproval).not.toHaveBeenCalled() // consensus reached → no gate + }, 30_000) + + it("escalates to the architecture gate on low consensus, then resolves on sign-off", async () => { + const acts = makeActivities({ + decideConsensus: vi.fn(async () => ({ consensusReached: false, winner: null })), + }) + await withWorker(acts, async () => { + const handle = await env.client.workflow.start(directionConsensus, { + taskQueue: TQ, + workflowId: nextId("consensus"), + args: ["e1"], + }) + await env.sleep("1 day") + expect(acts.requestArchitectureApproval).toHaveBeenCalledWith("e1") + expect(acts.recordArchitectureApproval).not.toHaveBeenCalled() // still blocked + + await handle.signal(architectureSignal) + await handle.result() + }) + expect(acts.recordArchitectureApproval).toHaveBeenCalledWith("e1") + }, 30_000) + + it("is advisory: records the round but never blocks even when consensus is not reached", async () => { + const acts = makeActivities({ + generateDirections: vi.fn(async () => ({ + candidates: [ + { id: "c1", title: "A", summary: "", tradeoffs: [] }, + { id: "c2", title: "B", summary: "", tradeoffs: [] }, + ], + config: { + inputMode: "rank", + threshold: 0.6, + tieBreaker: "human", + criteria: [], + raterRoles: ["lead_system_design", "lead_architect", "lead_engineer"], + advisory: true, + }, + })), + decideConsensus: vi.fn(async () => ({ consensusReached: false, winner: null })), + }) + await withWorker(acts, async () => { + await env.client.workflow.execute(directionConsensus, { + taskQueue: TQ, + workflowId: nextId("consensus"), + args: ["e1"], + }) + }) + expect(acts.decideConsensus).toHaveBeenCalledTimes(1) + expect(acts.requestArchitectureApproval).not.toHaveBeenCalled() // advisory → never gates + }, 30_000) + + it("no-ops when fewer than two viable candidates are generated", async () => { + const acts = makeActivities({ + generateDirections: vi.fn(async () => ({ + candidates: [{ id: "c1", title: "A", summary: "", tradeoffs: [] }], + config: { + inputMode: "rank", + threshold: 0.6, + tieBreaker: "human", + criteria: [], + raterRoles: ["lead_system_design", "lead_architect", "lead_engineer"], + advisory: false, + }, + })), + }) + await withWorker(acts, async () => { + await env.client.workflow.execute(directionConsensus, { + taskQueue: TQ, + workflowId: nextId("consensus"), + args: ["e1"], + }) + }) + expect(acts.rateDirection).not.toHaveBeenCalled() + expect(acts.decideConsensus).not.toHaveBeenCalled() + }, 30_000) +}) + describe("heartbeat", () => { it("picks up the backlog", async () => { const acts = makeActivities() diff --git a/apps/server/src/temporal/workflows.ts b/apps/server/src/temporal/workflows.ts index 440afac..819055f 100644 --- a/apps/server/src/temporal/workflows.ts +++ b/apps/server/src/temporal/workflows.ts @@ -17,6 +17,11 @@ const { requestRoadmapSignoff, recordRoadmapApproval, requestApproval, + generateDirections, + rateDirection, + decideConsensus, + requestArchitectureApproval, + recordArchitectureApproval, } = proxyActivities({ startToCloseTimeout: "10 minutes", retry: { maximumAttempts: 3 }, @@ -59,6 +64,43 @@ export async function epicDecomposition(epicId: string): Promise { await decomposeEpic(epicId) } +/** Architecture sign-off gate (ENG-016 tie-breaker): released by the human when consensus is low. */ +export const architectureSignal = defineSignal("architectureApprove") + +/** + * Kappa-style consensus over candidate implementation directions (ENG-016 / PRD-001). A durable step: + * the Lead System Design agent enumerates 2–4 candidates, then the senior technical roles rate them + * **independently and in parallel** (the barrier is what makes the agreement coefficient meaningful); + * the agreement coefficient gates whether the aggregate winner is adopted or sent to the tie-breaker. + * + * Phase 1 is **advisory** (record the round + score, don't block); when promoted to a real gate + * (`CONSENSUS_ADVISORY=false`) a low-agreement outcome escalates to the human via the + * `architecture_decision` approval gate (invariant #4). No-op when <2 viable candidates exist. + */ +export async function directionConsensus(epicId: string): Promise { + let approved = false + setHandler(architectureSignal, () => { + approved = true + }) + + const kickoff = await generateDirections(epicId) + if (!kickoff || kickoff.candidates.length < 2) return // not enough viable options → shaping proceeds + const { candidates, config } = kickoff + + // Independent raters, concurrently (barrier): collect every rating before measuring agreement. + const ratings = await Promise.all( + config.raterRoles.map((role) => rateDirection(epicId, candidates, role, config.inputMode)), + ) + + const { consensusReached } = await decideConsensus(epicId, candidates, ratings) + if (config.advisory || consensusReached) return // advisory (record-only), or a trusted winner + + // Tie-break: escalate to the human architecture_decision gate and block until signed off. + await requestArchitectureApproval(epicId) + await condition(() => approved) + await recordArchitectureApproval(epicId) +} + /** * Human approval gates — the dashboard signals these to release the durable waits. One signal * carries which gate is being approved: `merge` (the review/merge gate) or `deploy` (the ship gate). diff --git a/packages/agents/src/consensus.test.ts b/packages/agents/src/consensus.test.ts new file mode 100644 index 0000000..ee0ec06 --- /dev/null +++ b/packages/agents/src/consensus.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest" +import { parseDirections, parseRating } from "./consensus" + +describe("parseDirections", () => { + it("extracts candidates and bounds them to max", () => { + const text = JSON.stringify({ + candidates: [ + { title: "A", summary: "sa", tradeoffs: ["t1"] }, + { title: "B", summary: "sb", tradeoffs: [] }, + { title: "C", summary: "sc", tradeoffs: ["t2", "t3"] }, + ], + }) + const out = parseDirections(text, 2) + expect(out).toHaveLength(2) + expect(out[0]?.title).toBe("A") + expect(out[0]?.tradeoffs).toEqual(["t1"]) + }) + + it("tolerates a fenced JSON block and missing tradeoffs", () => { + const text = '```json\n{"candidates":[{"title":"X","summary":"y"}]}\n```' + const out = parseDirections(text) + expect(out).toEqual([{ title: "X", summary: "y", tradeoffs: [] }]) + }) + + it("returns [] on unparseable output", () => { + expect(parseDirections("no json here")).toEqual([]) + }) +}) + +describe("parseRating", () => { + const ids = ["a", "b", "c"] + + it("parses a single pick constrained to known ids", () => { + const r = parseRating('{"pick":"b","rationale":"clearest"}', "lead_architect", ids, "pick", 1.5) + expect(r.pick).toBe("b") + expect(r.ranking).toEqual([]) + expect(r.rationale).toBe("clearest") + expect(r.costCents).toBe(1.5) + expect(r.raterRole).toBe("lead_architect") + }) + + it("drops a pick that isn't a known candidate (abstention)", () => { + const r = parseRating('{"pick":"z","rationale":"?"}', "lead_engineer", ids, "pick", 0) + expect(r.pick).toBeNull() + }) + + it("parses a ranking, filtering unknowns and de-duplicating", () => { + const r = parseRating( + '{"ranking":["c","a","c","z","b"],"rationale":"r"}', + "lead_system_design", + ids, + "rank", + 2, + ) + expect(r.ranking).toEqual(["c", "a", "b"]) + expect(r.pick).toBeNull() + }) + + it("falls back to an abstention on unparseable output", () => { + const r = parseRating("garbage", "lead_architect", ids, "pick", 0) + expect(r.pick).toBeNull() + expect(r.ranking).toEqual([]) + }) +}) diff --git a/packages/agents/src/consensus.ts b/packages/agents/src/consensus.ts new file mode 100644 index 0000000..ac10ffa --- /dev/null +++ b/packages/agents/src/consensus.ts @@ -0,0 +1,210 @@ +import type { ConsensusInputMode, Rating, RoleId, StoppedReason } from "@eng/core" +import { ClaudeWorker } from "./claude-worker" +import { extractJson } from "./propose" + +/** + * Candidate generation + rater agents for Kappa-style consensus (ENG-016). `proposeDirections` + * enumerates 2–4 genuinely distinct implementation directions; `rateDirections` has one role agent + * **independently** score/rank them via structured outputs (ENG-009). Independence is procedural: + * each rater is a separate run that never sees another rater's input — what makes the agreement + * coefficient meaningful (the judge-panel pattern). + */ + +export interface ProposeDirectionsInput { + role: RoleId + systemPrompt: string + goalContext: string + /** What feature/decision to enumerate directions for. */ + task: string + budgetCentsRemaining: number + /** Upper bound on candidates (PRD: 2–4). Default 4. */ + maxCandidates?: number +} + +export interface ProposedDirection { + title: string + summary: string + tradeoffs: string[] +} + +export interface ProposedDirections { + candidates: ProposedDirection[] + costCents: number + stoppedReason: StoppedReason +} + +const DIRECTIONS_SCHEMA: Record = { + type: "object", + additionalProperties: false, + required: ["candidates"], + properties: { + candidates: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["title", "summary", "tradeoffs"], + properties: { + title: { type: "string" }, + summary: { type: "string" }, + tradeoffs: { type: "array", items: { type: "string" } }, + }, + }, + }, + }, +} + +/** Run an agent to enumerate distinct candidate implementation directions for a feature. */ +export async function proposeDirections( + input: ProposeDirectionsInput, +): Promise { + const max = input.maxCandidates ?? 4 + const contract = + `Propose between 2 and ${max} GENUINELY DISTINCT implementation directions — not variations of ` + + "one idea. For each give a short title, a one-paragraph summary, and its key tradeoffs. Respond " + + 'with ONLY {"candidates":[{"title":string,"summary":string,"tradeoffs":[string]}]}.' + const result = await new ClaudeWorker().run({ + role: input.role, + systemPrompt: input.systemPrompt, + tools: [], + goalContext: input.goalContext, + task: `${input.task}\n\n${contract}`, + budgetCentsRemaining: input.budgetCentsRemaining, + outputSchema: DIRECTIONS_SCHEMA, + }) + return { + candidates: parseDirections(result.summary, max), + costCents: result.costCents, + stoppedReason: result.stoppedReason, + } +} + +/** Best-effort extraction of candidate directions; bounded to `max`. */ +export function parseDirections(text: string, max = 4): ProposedDirection[] { + const json = extractJson(text) + if (!json) return [] + try { + const obj = JSON.parse(json) as { candidates?: unknown } + if (!Array.isArray(obj.candidates)) return [] + return obj.candidates + .filter( + (c): c is ProposedDirection => + !!c && + typeof (c as ProposedDirection).title === "string" && + typeof (c as ProposedDirection).summary === "string", + ) + .map((c) => ({ + title: c.title, + summary: c.summary, + tradeoffs: Array.isArray(c.tradeoffs) + ? c.tradeoffs.filter((t) => typeof t === "string") + : [], + })) + .slice(0, max) + } catch { + return [] + } +} + +export interface RateDirectionsInput { + role: RoleId + systemPrompt: string + goalContext: string + candidates: { id: string; title: string; summary: string }[] + criteria: string[] + inputMode: ConsensusInputMode + budgetCentsRemaining: number +} + +export interface RaterResult { + rating: Rating + costCents: number + stoppedReason: StoppedReason +} + +/** Schema constraining the rater to the actual candidate ids (single-pick or full ranking). */ +function ratingSchema(ids: string[], mode: ConsensusInputMode): Record { + if (mode === "pick") { + return { + type: "object", + additionalProperties: false, + required: ["pick", "rationale"], + properties: { pick: { type: "string", enum: ids }, rationale: { type: "string" } }, + } + } + return { + type: "object", + additionalProperties: false, + required: ["ranking", "rationale"], + properties: { + ranking: { type: "array", items: { type: "string", enum: ids } }, + rationale: { type: "string" }, + }, + } +} + +/** Run ONE rater independently over the candidates, returning a structured `Rating`. */ +export async function rateDirections(input: RateDirectionsInput): Promise { + const ids = input.candidates.map((c) => c.id) + const list = input.candidates.map((c) => `[${c.id}] ${c.title}: ${c.summary}`).join("\n") + const instruction = + input.inputMode === "pick" + ? 'Pick the single best candidate. Respond with ONLY {"pick":"","rationale":string}.' + : 'Rank ALL candidates from best to worst by id. Respond with ONLY {"ranking":["",...],"rationale":string}.' + const task = + `Independently evaluate these candidate implementation directions against the rubric ` + + `(${input.criteria.join(", ")}). Judge on the merits; you are not seeing other reviewers' views.\n\n` + + `Candidates:\n${list}\n\n${instruction}` + + const result = await new ClaudeWorker().run({ + role: input.role, + systemPrompt: input.systemPrompt, + tools: [], + goalContext: input.goalContext, + task, + budgetCentsRemaining: input.budgetCentsRemaining, + outputSchema: ratingSchema(ids, input.inputMode), + }) + return { + rating: parseRating(result.summary, input.role, ids, input.inputMode, result.costCents), + costCents: result.costCents, + stoppedReason: result.stoppedReason, + } +} + +/** Best-effort rating extraction, constrained to known candidate ids. Defaults to an abstention. */ +export function parseRating( + text: string, + role: RoleId, + ids: string[], + mode: ConsensusInputMode, + costCents: number, +): Rating { + const base: Rating = { + raterRole: role, + pick: null, + ranking: [], + rationale: text.trim(), + costCents, + } + const json = extractJson(text) + if (!json) return base + try { + const obj = JSON.parse(json) as { pick?: unknown; ranking?: unknown; rationale?: unknown } + const rationale = typeof obj.rationale === "string" ? obj.rationale : base.rationale + if (mode === "pick") { + const pick = typeof obj.pick === "string" && ids.includes(obj.pick) ? obj.pick : null + return { ...base, pick, rationale } + } + const ranking = Array.isArray(obj.ranking) + ? [ + ...new Set( + obj.ranking.filter((r): r is string => typeof r === "string" && ids.includes(r)), + ), + ] + : [] + return { ...base, ranking, rationale } + } catch { + return base + } +} diff --git a/packages/agents/src/index.ts b/packages/agents/src/index.ts index a62e2ff..e32de5e 100644 --- a/packages/agents/src/index.ts +++ b/packages/agents/src/index.ts @@ -4,6 +4,17 @@ import { ClaudeWorker, type ClaudeWorkerOptions } from "./claude-worker" export { type AssessInput, type Assessment, assess, parseVerdict } from "./assess" export type { ClaudeWorkerOptions, WorkerBackend, WorkerMode } from "./claude-worker" export { ClaudeWorker } from "./claude-worker" +export { + type ProposeDirectionsInput, + type ProposedDirection, + type ProposedDirections, + parseDirections, + parseRating, + proposeDirections, + type RateDirectionsInput, + type RaterResult, + rateDirections, +} from "./consensus" export { type ProposedTicket, type ProposedTickets, diff --git a/packages/core/src/agreement.test.ts b/packages/core/src/agreement.test.ts new file mode 100644 index 0000000..2fb7bd0 --- /dev/null +++ b/packages/core/src/agreement.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest" +import { agreement, fleissKappa, kendallW, krippendorffAlpha } from "./agreement" + +describe("krippendorffAlpha", () => { + it("is 1 for perfect agreement (all raters identical)", () => { + // 3 units (candidates) × 3 raters, everyone selects candidate 0 + const m = [ + [1, 1, 1], + [0, 0, 0], + [0, 0, 0], + ] + expect(krippendorffAlpha(m, "nominal")).toBe(1) + }) + + it("is 1 (degenerate-safe) when there is no variation at all", () => { + expect(krippendorffAlpha([[5, 5, 5]], "nominal")).toBe(1) + expect(krippendorffAlpha([[0, 0, 0]], "ordinal")).toBe(1) + }) + + it("goes negative for maximal disagreement (everyone picks a different option)", () => { + const m = [ + [1, 0, 0], + [0, 1, 0], + [0, 0, 1], + ] + expect(krippendorffAlpha(m, "nominal")).toBeLessThan(0) + }) + + it("is between full and none for partial agreement (2 of 3 agree)", () => { + const m = [ + [1, 1, 0], + [0, 0, 1], + [0, 0, 0], + ] + const a = krippendorffAlpha(m, "nominal") + expect(a).toBeGreaterThan(0) + expect(a).toBeLessThan(1) + }) + + it("handles missing values (null cells) without NaN", () => { + const a = krippendorffAlpha( + [ + [1, 1, null], + [0, null, 0], + ], + "nominal", + ) + expect(Number.isNaN(a)).toBe(false) + }) +}) + +describe("fleissKappa (single-pick)", () => { + it("is 1 when all raters pick the same candidate", () => { + expect(fleissKappa([0, 0, 0], 3)).toBe(1) + }) + + it("is negative when all three raters pick different candidates", () => { + expect(fleissKappa([0, 1, 2], 3)).toBeCloseTo(-0.5, 5) + }) + + it("is degenerate-safe for <2 raters or <2 candidates", () => { + expect(fleissKappa([0], 3)).toBe(1) + expect(fleissKappa([0, 0], 1)).toBe(1) + }) +}) + +describe("kendallW (ranked)", () => { + it("is 1 when all raters give the identical ranking", () => { + const w = kendallW([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + ]) + expect(w).toBeCloseTo(1, 5) + }) + + it("is ~0 for perfectly opposed rankings (two raters)", () => { + const w = kendallW([ + [1, 2, 3], + [3, 2, 1], + ]) + expect(w).toBeCloseTo(0, 5) + }) + + it("is between for partial concordance", () => { + const w = kendallW([ + [1, 2, 3], + [1, 3, 2], + ]) + expect(w).toBeGreaterThan(0) + expect(w).toBeLessThan(1) + }) + + it("is degenerate-safe for <2 items", () => { + expect(kendallW([[1], [1]])).toBe(1) + }) +}) + +describe("agreement (primary = Krippendorff)", () => { + it("labels the metric and computes from a selection matrix (pick)", () => { + const res = agreement({ + mode: "pick", + selectionMatrix: [ + [1, 1, 1], + [0, 0, 0], + ], + }) + expect(res.metric).toBe("krippendorff_alpha") + expect(res.coefficient).toBe(1) + }) + + it("computes from a rank matrix (rank, ordinal)", () => { + const res = agreement({ + mode: "rank", + rankMatrix: [ + [1, 1, 1], + [2, 2, 2], + [3, 3, 3], + ], + }) + expect(res.coefficient).toBe(1) + }) +}) diff --git a/packages/core/src/agreement.ts b/packages/core/src/agreement.ts new file mode 100644 index 0000000..00ebd4c --- /dev/null +++ b/packages/core/src/agreement.ts @@ -0,0 +1,149 @@ +/** + * Inter-rater agreement coefficients (ENG-016 / PRD-001), corrected for chance. Pure functions over + * plain numeric matrices so they're trivially unit-testable and free of any schema/agent coupling. + * + * Honest caveat (PRD): with only ~3 raters and a few candidates these coefficients are *noisy* — a + * single flipped vote swings them. They are used as a confidence signal feeding a decision rule, not + * as a hard statistical inference. Degenerate cases (no variation / unanimity) map to 1, never NaN. + */ + +export type AgreementMetric = "krippendorff_alpha" | "fleiss_kappa" | "kendall_w" +export type DiffMetric = "nominal" | "ordinal" | "interval" + +export interface AgreementResult { + coefficient: number + metric: AgreementMetric +} + +/** + * Krippendorff's alpha over a reliability matrix `units × raters` (`null` = a missing rating). The + * single primary coefficient (PRD): it spans nominal / ordinal / interval data and ≥3 fixed raters, + * so it need not change between single-pick and ranked input. Returns 1 for perfect agreement + * (including the degenerate no-variation case) and can go negative for systematic disagreement. + */ +export function krippendorffAlpha( + matrix: ReadonlyArray>, + metric: DiffMetric = "nominal", +): number { + const valueSet = new Set() + for (const unit of matrix) for (const v of unit) if (v != null) valueSet.add(v) + const levels = [...valueSet].sort((a, b) => a - b) + if (levels.length <= 1) return 1 // no variation → perfect agreement (degenerate-safe) + + const indexOf = new Map(levels.map((v, i) => [v, i])) + const L = levels.length + const o: number[][] = Array.from({ length: L }, () => new Array(L).fill(0)) + + for (const unit of matrix) { + const present: number[] = [] + for (const v of unit) if (v != null) present.push(v) + const m = present.length + if (m < 2) continue + for (let a = 0; a < m; a++) { + for (let b = 0; b < m; b++) { + if (a === b) continue + const c = indexOf.get(present[a] ?? 0) ?? 0 + const k = indexOf.get(present[b] ?? 0) ?? 0 + const row = o[c] + if (row) row[k] = (row[k] ?? 0) + 1 / (m - 1) + } + } + } + + const nC = o.map((row) => row.reduce((s, x) => s + x, 0)) + const n = nC.reduce((s, x) => s + x, 0) + if (n < 2) return 1 // nothing pairable → nothing to disagree on + + const delta2 = (c: number, k: number): number => { + if (c === k) return 0 + if (metric === "nominal") return 1 + if (metric === "interval") return ((levels[c] ?? 0) - (levels[k] ?? 0)) ** 2 + // ordinal: (Σ marginals spanning c..k, less half of each endpoint)² + const lo = Math.min(c, k) + const hi = Math.max(c, k) + let s = 0 + for (let g = lo; g <= hi; g++) s += nC[g] ?? 0 + s -= ((nC[lo] ?? 0) + (nC[hi] ?? 0)) / 2 + return s * s + } + + let Do = 0 + let De = 0 + for (let c = 0; c < L; c++) { + for (let k = 0; k < L; k++) { + const ock = o[c]?.[k] ?? 0 + const d = delta2(c, k) + Do += ock * d + De += (nC[c] ?? 0) * (nC[k] ?? 0) * d + } + } + Do /= n + De /= n * (n - 1) + if (De === 0) return 1 // no expected disagreement → perfect (degenerate-safe) + return 1 - Do / De +} + +/** + * Fleiss' kappa for single-pick agreement (PRD): subjects = candidates, categories = {selected, not}, + * `n` raters each selecting exactly one candidate. (Light's kappa — the mean pairwise Cohen's — is the + * more principled measure for fixed raters but ≈ Fleiss' in practice; Fleiss' is the pragmatic + * stand-in.) `picks[r]` is the candidate index rater r chose. Unanimity → 1; degenerate-safe. + */ +export function fleissKappa(picks: ReadonlyArray, candidateCount: number): number { + const n = picks.length + if (n < 2 || candidateCount < 2) return 1 + const votes = new Array(candidateCount).fill(0) + for (const p of picks) if (p >= 0 && p < candidateCount) votes[p] = (votes[p] ?? 0) + 1 + + let pBar = 0 + for (let i = 0; i < candidateCount; i++) { + const v = votes[i] ?? 0 + pBar += (v * v + (n - v) * (n - v) - n) / (n * (n - 1)) + } + pBar /= candidateCount + + const pSelected = 1 / candidateCount // each rater picks one of `candidateCount` + const pNot = 1 - pSelected + const pE = pSelected * pSelected + pNot * pNot + if (pE >= 1) return 1 + return (pBar - pE) / (1 - pE) +} + +/** + * Kendall's W (coefficient of concordance) for ranked agreement (PRD): `m` raters each producing a + * strict ranking over `n` items. `rankings[r][i]` is the rank rater r gave item i (1 = best). Returns + * W in [0,1] — 1 = perfect concordance, 0 = none. Degenerate-safe (n < 2 → 1). + */ +export function kendallW(rankings: ReadonlyArray>): number { + const m = rankings.length + if (m === 0) return 1 + const n = rankings[0]?.length ?? 0 + if (n < 2) return 1 + + const rankSum = new Array(n).fill(0) + for (const r of rankings) for (let i = 0; i < n; i++) rankSum[i] = (rankSum[i] ?? 0) + (r[i] ?? 0) + + const rBar = (m * (n + 1)) / 2 + let S = 0 + for (let i = 0; i < n; i++) S += ((rankSum[i] ?? 0) - rBar) ** 2 + + const denom = (m * m * (n * n * n - n)) / 12 + if (denom === 0) return 1 + return S / denom +} + +/** + * The primary agreement coefficient for a consensus round: Krippendorff's alpha over the selection + * matrix (nominal, single-pick) or the rank matrix (ordinal, ranked). Both are `candidates × raters`. + */ +export function agreement( + input: + | { mode: "pick"; selectionMatrix: ReadonlyArray> } + | { mode: "rank"; rankMatrix: ReadonlyArray> }, +): AgreementResult { + const coefficient = + input.mode === "pick" + ? krippendorffAlpha(input.selectionMatrix, "nominal") + : krippendorffAlpha(input.rankMatrix, "ordinal") + return { coefficient, metric: "krippendorff_alpha" } +} diff --git a/packages/core/src/consensus.test.ts b/packages/core/src/consensus.test.ts new file mode 100644 index 0000000..f62cdef --- /dev/null +++ b/packages/core/src/consensus.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest" +import { type ConsensusConfig, DEFAULT_CONSENSUS_CONFIG, decide, type Rating } from "./consensus" + +const cands = [{ id: "a" }, { id: "b" }, { id: "c" }] + +const pickConfig: ConsensusConfig = { + ...DEFAULT_CONSENSUS_CONFIG, + inputMode: "pick", + threshold: 0.6, + advisory: true, +} +const rankConfig: ConsensusConfig = { + ...DEFAULT_CONSENSUS_CONFIG, + inputMode: "rank", + threshold: 0.6, +} + +const pick = (role: Rating["raterRole"], c: string): Rating => ({ + raterRole: role, + pick: c, + ranking: [], + rationale: "", + costCents: null, +}) +const rank = (role: Rating["raterRole"], order: string[]): Rating => ({ + raterRole: role, + pick: null, + ranking: order, + rationale: "", + costCents: null, +}) + +describe("decide — single-pick", () => { + it("reaches consensus when all raters pick the same candidate", () => { + const out = decide( + cands, + [pick("lead_architect", "b"), pick("lead_system_design", "b"), pick("lead_engineer", "b")], + pickConfig, + ) + expect(out.winner).toBe("b") + expect(out.consensusReached).toBe(true) + expect(out.aggregateMethod).toBe("votes") + expect(out.coefficient).toBe(1) + }) + + it("does NOT reach consensus when raters split evenly (no clear winner, low agreement)", () => { + const out = decide( + cands, + [pick("lead_architect", "a"), pick("lead_system_design", "b"), pick("lead_engineer", "c")], + pickConfig, + ) + expect(out.consensusReached).toBe(false) // three-way tie → tie-breaker + }) + + it("has a clear plurality winner but withholds consensus when agreement is below threshold", () => { + const out = decide( + cands, + [pick("lead_architect", "a"), pick("lead_system_design", "a"), pick("lead_engineer", "b")], + pickConfig, + ) + expect(out.winner).toBe("a") // 2 vs 1 — clear plurality + expect(out.consensusReached).toBe(false) // but agreement < 0.6 → tie-breaker + }) +}) + +describe("decide — ranked (Borda + Kendall/Krippendorff)", () => { + it("reaches consensus when all raters rank identically", () => { + const out = decide( + cands, + [ + rank("lead_architect", ["a", "b", "c"]), + rank("lead_system_design", ["a", "b", "c"]), + rank("lead_engineer", ["a", "b", "c"]), + ], + rankConfig, + ) + expect(out.winner).toBe("a") + expect(out.consensusReached).toBe(true) + expect(out.aggregateMethod).toBe("borda") + expect(out.coefficient).toBeGreaterThanOrEqual(0.6) + }) + + it("picks the Borda winner but routes to tie-break when rankings disagree", () => { + const out = decide( + cands, + [ + rank("lead_architect", ["a", "b", "c"]), + rank("lead_system_design", ["c", "b", "a"]), + rank("lead_engineer", ["b", "c", "a"]), + ], + rankConfig, + ) + expect(out.consensusReached).toBe(false) + }) + + it("respects a lower threshold (advisory tuning)", () => { + const out = decide( + cands, + [ + rank("lead_architect", ["a", "b", "c"]), + rank("lead_system_design", ["a", "c", "b"]), + rank("lead_engineer", ["a", "b", "c"]), + ], + { ...rankConfig, threshold: -1 }, + ) + expect(out.winner).toBe("a") + expect(out.consensusReached).toBe(true) // clear winner + threshold trivially met + }) +}) diff --git a/packages/core/src/consensus.ts b/packages/core/src/consensus.ts new file mode 100644 index 0000000..1b25c48 --- /dev/null +++ b/packages/core/src/consensus.ts @@ -0,0 +1,194 @@ +import { z } from "zod" +import { type AgreementMetric, agreement } from "./agreement" +import { RoleId } from "./schema" + +/** + * Kappa-style consensus (ENG-016 / PRD-001): at the architecture / system-design stage the senior + * technical roles independently rate candidate implementation directions; an inter-rater agreement + * coefficient gates whether a clear winner is adopted or sent to a tie-breaker. Schemas + the pure + * aggregation/decision rule live here; the agreement math is in `./agreement`. + */ + +const id = z.string() + +/** One candidate implementation direction for a feature. */ +export const ConsensusCandidate = z.object({ + id, + title: z.string(), + summary: z.string(), + tradeoffs: z.array(z.string()), +}) +export type ConsensusCandidate = z.infer + +export const ConsensusInputMode = z.enum(["pick", "rank"]) +export type ConsensusInputMode = z.infer + +export const ConsensusTieBreaker = z.enum(["human", "casting_vote"]) +export type ConsensusTieBreaker = z.infer + +/** One rater's assessment — a single pick or a full ranking, plus their rationale. */ +export const Rating = z.object({ + raterRole: RoleId, + /** Single-pick mode: the chosen candidate id (null if abstained / wrong mode). */ + pick: id.nullable(), + /** Ranked mode: candidate ids best→worst (empty in pick mode). */ + ranking: z.array(id), + rationale: z.string(), + costCents: z.number().nullable(), +}) +export type Rating = z.infer + +/** Per-unit consensus configuration (config, not code — invariant #6). */ +export const ConsensusConfig = z.object({ + inputMode: ConsensusInputMode, + /** Agreement gate; Landis–Koch "substantial" ≈ 0.6 by default. */ + threshold: z.number(), + tieBreaker: ConsensusTieBreaker, + /** Rubric criteria keys raters score against. */ + criteria: z.array(z.string()), + raterRoles: z.array(RoleId), + /** Phase 1: record the round + score but don't block the lifecycle (advisory). */ + advisory: z.boolean(), +}) +export type ConsensusConfig = z.infer + +export const ConsensusRoundStatus = z.enum(["rating", "decided", "tie_break", "resolved"]) +export type ConsensusRoundStatus = z.infer + +export const ConsensusRound = z.object({ + id, + epicId: id, + featureId: id.nullable(), + candidates: z.array(ConsensusCandidate), + criteria: z.array(z.string()), + raterRoles: z.array(RoleId), + status: ConsensusRoundStatus, +}) +export type ConsensusRound = z.infer + +export const ConsensusOutcome = z.object({ + roundId: id, + winner: id.nullable(), + consensusReached: z.boolean(), + coefficient: z.number(), + metric: z.string(), + aggregateMethod: z.string(), + tieBreaker: z.object({ type: ConsensusTieBreaker, decidedBy: z.string().nullable() }).nullable(), +}) +export type ConsensusOutcome = z.infer + +export interface CandidateScore { + candidateId: string + score: number +} + +export interface ConsensusDecision { + winner: string | null + consensusReached: boolean + coefficient: number + metric: AgreementMetric + aggregateMethod: "votes" | "borda" + scores: CandidateScore[] +} + +/** The default rubric + thresholds; per-unit overridable (invariant #6). Phase 1 = advisory. */ +export const DEFAULT_CONSENSUS_CONFIG: ConsensusConfig = { + inputMode: "rank", + threshold: 0.6, + tieBreaker: "human", + criteria: [ + "simplicity", + "scalability", + "delivery_risk", + "adr_alignment", + "cost_to_build", + "reversibility", + "testability", + ], + raterRoles: ["lead_system_design", "lead_architect", "lead_engineer"], + advisory: true, +} + +/** + * Pure aggregation + agreement gate (PRD §decision rule). The **aggregate winner** (vote tally for + * single-pick, Borda count for rankings) is primary; the **coefficient** gates whether that winner is + * trusted (consensus) or routed to the tie-breaker. `consensusReached` requires both a *clear* winner + * (no tie for the top) and `coefficient ≥ threshold`. + */ +export function decide( + candidates: ReadonlyArray<{ id: string }>, + ratings: ReadonlyArray, + config: ConsensusConfig, +): ConsensusDecision { + const ids = candidates.map((c) => c.id) + const colOf = new Map(ids.map((cid, i) => [cid, i])) + const N = ids.length + + if (config.inputMode === "pick") { + const selectionMatrix: (number | null)[][] = ids.map(() => ratings.map(() => 0)) + const votes = new Array(N).fill(0) + ratings.forEach((r, ri) => { + const col = r.pick != null ? colOf.get(r.pick) : undefined + if (col != null) { + votes[col] = (votes[col] ?? 0) + 1 + const row = selectionMatrix[col] + if (row) row[ri] = 1 + } + }) + const { coefficient, metric } = agreement({ mode: "pick", selectionMatrix }) + const { winner, clear } = topCandidate(ids, votes) + return { + winner, + consensusReached: clear && coefficient >= config.threshold, + coefficient, + metric, + aggregateMethod: "votes", + scores: ids.map((cid, i) => ({ candidateId: cid, score: votes[i] ?? 0 })), + } + } + + // Ranked: Borda count for the winner; Krippendorff (ordinal) over the rank matrix for agreement. + const rankMatrix: (number | null)[][] = ids.map(() => ratings.map(() => null as number | null)) + const borda = new Array(N).fill(0) + ratings.forEach((r, ri) => { + r.ranking.forEach((cid, pos) => { + const col = colOf.get(cid) + if (col != null) { + const row = rankMatrix[col] + if (row) row[ri] = pos + 1 // 1 = best + borda[col] = (borda[col] ?? 0) + (N - (pos + 1)) // best gets N-1 points + } + }) + }) + const { coefficient, metric } = agreement({ mode: "rank", rankMatrix }) + const { winner, clear } = topCandidate(ids, borda) + return { + winner, + consensusReached: clear && coefficient >= config.threshold, + coefficient, + metric, + aggregateMethod: "borda", + scores: ids.map((cid, i) => ({ candidateId: cid, score: borda[i] ?? 0 })), + } +} + +/** Argmax with a strict-uniqueness check: a tie for the top is not a clear winner. */ +function topCandidate( + ids: ReadonlyArray, + scores: ReadonlyArray, +): { winner: string | null; clear: boolean } { + let best = Number.NEGATIVE_INFINITY + let bestIdx = -1 + let tie = false + scores.forEach((s, i) => { + if (s > best) { + best = s + bestIdx = i + tie = false + } else if (s === best) { + tie = true + } + }) + const winner = bestIdx >= 0 ? (ids[bestIdx] ?? null) : null + return { winner, clear: bestIdx >= 0 && !tie } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 887f96d..79cb4a0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,5 +1,7 @@ +export * from "./agreement" export * from "./audit" export * from "./budget" +export * from "./consensus" export * from "./decision" export * from "./delivery" export * from "./hierarchy" diff --git a/project/ENG-016-implement-kappa-consensus.md b/project/ENG-016-implement-kappa-consensus.md index 2bca6f0..267e837 100644 --- a/project/ENG-016-implement-kappa-consensus.md +++ b/project/ENG-016-implement-kappa-consensus.md @@ -1,6 +1,6 @@ # ENG-016 — Implement Kappa-style consensus (PRD-001) -- **Status:** backlog +- **Status:** done (Phase 1 — advisory; ranked Borda + Krippendorff/Kendall/Fleiss; human-escalation tie-break) - **Priority:** P2 (Medium — feature; not a live-run blocker) - **Stage:** implementation - **Assignee role:** staff_engineer (schema + agreement function design with lead_system_design; diff --git a/project/ROADMAP.md b/project/ROADMAP.md index ecda46f..3af2a74 100644 --- a/project/ROADMAP.md +++ b/project/ROADMAP.md @@ -48,10 +48,12 @@ high-leverage enablers forward even when labelled P1/P2. - **ENG-010** ✅ — `/api/budgets` + web Budgets page (per-role limit/spent/remaining + total). - **ENG-008** ✅ — Bounded implement→review→QA rework loop with QA feedback, then blocked. -### Wave 4 — Provenance + consensus -- **ENG-014** — Decision log / provenance tree. Needs ENG-009; build on the Postgres+KB default behind - its port (see flag 3) rather than waiting on a SurrealDB migration. -- **ENG-016** — Kappa-style consensus (PRD-001). Needs ENG-009 + ENG-002; soft-needs ENG-014. +### Wave 4 — Provenance + consensus ✅ +- **ENG-014** ✅ — Decision log / provenance DAG on the Postgres+KB hybrid behind a `DecisionLog` port; + every agent step (shape/decompose/implement/QA) emits a decision traceable to the originating request. +- **ENG-016** ✅ — Kappa-style consensus (PRD-001), Phase 1: advisory `directionConsensus` workflow — + candidate generation → parallel independent raters → Krippendorff/Fleiss/Kendall agreement + Borda → + human-escalation tie-break via the `architecture_decision` gate; outcomes feed the decision log. ## Critical path (to a safe, useful autonomous run)