diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d95a768..2383f982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,29 @@ new version heading in the same commit. ## [Unreleased] +## [0.471.0] - 2026-11-19 +### Fixed +- **A busy console can no longer starve the server: the sessions poll stops reading the whole connectors + folder.** `resumableIds` listed every file in `/connectors/` on each `/api/sessions/summary` (the + console's 1.5s poll). On live globex that folder had reached 61k files / 677 MB, so each poll cost ~50ms. + With eight people's tabs open, plus a box that was also CPU- and swap-starved, `/health` stopped answering + and requests queued 460 deep. The summary path now stats only the ≤ few hundred rows it returns; the full + list reads the folder once and reuses it until the folder changes. +### Added +- **Connector-file retention.** Nothing but an explicit session delete ever removed a launch's files, and + `.mcp.json` carries each run's resolved connector credentials (`.env` its session secret), so months of + stale credential copies sat on disk. A session that ended more than 30 days ago, and whose pane is gone, + now keeps only `session-.log` (the run's record). The sweep runs every 10 min, at most 500 sessions a + pass, and logs `connectors.pruned` to the audit trail. Past that point an old session is no longer one-click + resumable; Take over still revives it from its conversation id. +- **The process janitor reaps a `claude` whose pane is gone.** On globex a member stopped a session 35s in. + The pane died but the `claude` inside ignored the hangup and ran for 70 days, reaching 23 GB RSS and + filling swap. Six more were found the same day, 8–50 days old at ~300 MB each. Neither the session reaper + (it reaps panes) nor the shell predicate (the parent was still alive) could see them. The janitor now + identifies a session's `claude` by its `--mcp-config …/session-.mcp.json`. It asks the tenant that owns + that folder whether the pane is alive, and reaps on a definite "no" seen on two sweeps past the 10-min + grace; unknown liveness is never a kill. Counted as `claude` in `orphan.reaped`. + ## [0.470.1] - 2026-11-12 ### Fixed - **New agents are born on the tenant's own ruleset, and a hand edit to `agent.json` no longer gets diff --git a/package-lock.json b/package-lock.json index 7f88bc6f..94ae1af2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index e2e8dbe3..07366a75 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", @@ -28,7 +28,7 @@ "dev": "ts-node src/cli.ts serve", "demo:dev": "ts-node src/demo.ts", "test:governance": "node scripts/governance-isolated.cjs", - "test:governance:run": "node scripts/version-sync-test.cjs && node scripts/agent-manifest-disk-test.cjs && node scripts/router-dispatch-test.cjs && node scripts/prompt-credential-guard-test.cjs && node scripts/standing-instructions-test.cjs && node scripts/session-trust-test.cjs && node scripts/video-gen-mapping-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/git-ceiling-test.cjs && node scripts/cron-supersede-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/script-body-gate-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/connector-auth-stub-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-pool-signed-out-test.cjs && node scripts/login-expiry-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/resume-transcript-stage-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/unblock-run-dedupe-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/session-search-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/launch-window-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/protected-read-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/goal-heartbeat-test.cjs && node scripts/question-durability-test.cjs && node scripts/ask-block-link-test.cjs && node scripts/answer-pinning-test.cjs && node scripts/bets-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", + "test:governance:run": "node scripts/version-sync-test.cjs && node scripts/agent-manifest-disk-test.cjs && node scripts/router-dispatch-test.cjs && node scripts/prompt-credential-guard-test.cjs && node scripts/standing-instructions-test.cjs && node scripts/session-trust-test.cjs && node scripts/video-gen-mapping-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/git-ceiling-test.cjs && node scripts/cron-supersede-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/script-body-gate-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/connector-auth-stub-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-pool-signed-out-test.cjs && node scripts/login-expiry-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/resume-transcript-stage-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/unblock-run-dedupe-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/session-search-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/launch-window-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/protected-read-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/connectors-retention-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/goal-heartbeat-test.cjs && node scripts/question-durability-test.cjs && node scripts/ask-block-link-test.cjs && node scripts/answer-pinning-test.cjs && node scripts/bets-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", "test:alert-staleness": "node scripts/alert-staleness-test.cjs", "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs", "test:dm-continuity": "node scripts/dm-continuity-test.cjs", diff --git a/scripts/connectors-retention-test.cjs b/scripts/connectors-retention-test.cjs new file mode 100644 index 00000000..1a12f70d --- /dev/null +++ b/scripts/connectors-retention-test.cjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node +/* CONNECTORS FOLDER — retention + the resumable lookup that read it on every console poll. + * + * Live globex, 2026-11-19: `/connectors/` held 61k files (677 MB) for 11k sessions because nothing + * but an explicit session DELETE ever removed a launch's files, and `resumableIds` read the whole folder on + * every `/api/sessions/summary` (the console's 1.5s poll) — ~50ms a call, enough with eight tabs open to + * keep `/health` from answering once the box was also CPU-starved. Pinned here: + * - the by-id (summary) path stats only the rows it returns, never reads the folder; + * - the full-list path reads the folder once and reuses it until the folder changes; + * - retention prunes ONLY ended + old + pane-less sessions, keeps `.log` (the run's record), honours the + * per-pass bound, and does nothing when liveness is unknown. + * Isolated home; no tmux, no claude (the backend's liveness poll is stubbed). */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'aos-connectors-retention-test-')); +process.env.AGENT_OS_HOME = HOME; +process.env.AGENT_OS_TENANT = 'testco'; +process.env.AOS_NO_TTYD = '1'; +delete process.env.AGENT_OS_SECRET_KEY; + +let pass = 0, fail = 0; +const assert = (c, name, d) => c ? (pass++, console.log(` \x1b[32m✓\x1b[0m ${name}`)) : (fail++, console.log(` \x1b[31m✗ ${name}\x1b[0m${d ? ' — ' + d : ''}`)); + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { TerminalManager, CONNECTOR_FILE_RETENTION_DAYS } = require(path.join(ROOT, 'dist/terminal.js')); + +const aos = loadAgentOS(); +const tm = new TerminalManager(aos, 'http://127.0.0.1:0', path.join(HOME, 'tmux.sock')); +const CONN = aos.paths.connectors; +fs.mkdirSync(CONN, { recursive: true }); + +let alive = new Set(); +tm.backend.aliveNames = () => alive; + +const DAY = 86_400_000; +const OLD = Date.now() - (CONNECTOR_FILE_RETENTION_DAYS + 5) * DAY; +const RECENT = Date.now() - 2 * DAY; +const mkRun = (id, status, at) => aos.db.prepare(`INSERT INTO term_sessions (id,agent,title,task,tmux,status,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?)`).run(id, 'engineer', id, 'task', 'aos-' + id, status, at, at); +const LAUNCH = ['mcp.json', 'company.md', 'task', 'env', 'stopped', 'log']; +const mkFiles = (id, exts = LAUNCH) => { + for (const e of exts) fs.writeFileSync(path.join(CONN, `session-${id}.${e}`), 'x'); + fs.mkdirSync(path.join(CONN, `session-${id}.d`, 'bin'), { recursive: true }); +}; +const filesOf = (id) => fs.readdirSync(CONN).filter((f) => f.startsWith(`session-${id}.`)).sort(); + +mkRun('ses_olddone', 'done', OLD); mkFiles('ses_olddone'); +mkRun('ses_oldstop', 'stopped', OLD); mkFiles('ses_oldstop'); +mkRun('ses_oldcrash', 'crashed', OLD); mkFiles('ses_oldcrash'); +mkRun('ses_recent', 'done', RECENT); mkFiles('ses_recent'); +mkRun('ses_running', 'running', OLD); mkFiles('ses_running'); +mkRun('ses_livepane', 'stopped', OLD); mkFiles('ses_livepane'); +mkFiles('ses_norow'); // a deleted session's leftovers +for (const f of filesOf('ses_norow')) fs.utimesSync(path.join(CONN, f), OLD / 1000, OLD / 1000); +mkFiles('ses_norow_new'); // a deleted session's leftovers, but recent + +// ── resumableIds ────────────────────────────────────────────────────────────────────────────────── +{ + const realReaddir = fs.readdirSync; + let reads = 0; + fs.readdirSync = function (p, ...rest) { if (path.resolve(String(p)) === path.resolve(CONN)) reads++; return realReaddir.call(fs, p, ...rest); }; + try { + const byId = tm.listSessions(undefined, undefined, ['ses_recent', 'ses_running']); + assert(reads === 0, 'the by-id (summary) path never reads the connectors folder', `reads=${reads}`); + assert(byId.every((s) => s.resumable === true), 'by-id rows with a .env are resumable'); + const full1 = tm.listSessions(); + const full2 = tm.listSessions(); + assert(reads === 1, 'the full list reads the folder once and reuses it while nothing changed', `reads=${reads}`); + assert(full1.length === full2.length && full1.find((s) => s.id === 'ses_olddone')?.resumable === true, 'full-list resumable is correct'); + fs.rmSync(path.join(CONN, 'session-ses_recent.env')); + const full3 = tm.listSessions(); + assert(reads === 2, 'removing a file invalidates the memo', `reads=${reads}`); + assert(full3.find((s) => s.id === 'ses_recent')?.resumable === false, 'a removed .env reads non-resumable straight away'); + fs.writeFileSync(path.join(CONN, 'session-ses_recent.env'), 'x'); + } finally { fs.readdirSync = realReaddir; } +} + +// ── retention ───────────────────────────────────────────────────────────────────────────────────── +alive = new Set(['aos-ses_livepane']); +{ + alive = null; + const r = tm.pruneConnectorFiles(); + assert(r.sessions === 0, 'liveness unknown → prunes nothing'); + alive = new Set(['aos-ses_livepane']); +} +{ + const r = tm.pruneConnectorFiles(Date.now(), 2); + assert(r.sessions === 2, 'a pass is bounded by maxSessions', JSON.stringify(r)); + const r2 = tm.pruneConnectorFiles(); + assert(r.sessions + r2.sessions === 4, 'the rest drains on the next pass (3 old ended + 1 old orphan)', JSON.stringify([r, r2])); + for (const id of ['ses_olddone', 'ses_oldstop', 'ses_oldcrash', 'ses_norow']) { + assert(JSON.stringify(filesOf(id)) === JSON.stringify([`session-${id}.log`]), `${id}: everything but .log removed (incl. the .d dir)`, filesOf(id).join(',')); + } + for (const id of ['ses_recent', 'ses_running', 'ses_livepane', 'ses_norow_new']) { + assert(filesOf(id).length === LAUNCH.length + 1, `${id}: untouched`, filesOf(id).join(',')); + } + assert(tm.pruneConnectorFiles().sessions === 0, 'a settled folder is a no-op pass'); + assert(tm.listSessions().find((s) => s.id === 'ses_olddone')?.resumable === false, 'a pruned session is no longer one-click resumable'); +} + +// ── sessionPaneAlive (the process janitor's oracle) ─────────────────────────────────────────────── +assert(tm.sessionPaneAlive('ses_livepane') === true, 'pane alive → true'); +assert(tm.sessionPaneAlive('ses_olddone') === false, 'pane gone → false'); +assert(tm.sessionPaneAlive('ses_norow') === false, 'no row → checks aos-'); +alive = null; +assert(tm.sessionPaneAlive('ses_olddone') === null, 'liveness unknown → null (never a kill)'); + +fs.rmSync(HOME, { recursive: true, force: true }); +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/process-janitor-test.cjs b/scripts/process-janitor-test.cjs index fa1b74ad..b5abdd9c 100644 --- a/scripts/process-janitor-test.cjs +++ b/scripts/process-janitor-test.cjs @@ -119,8 +119,51 @@ function harness(rows, liveSockets = []) { const realKill = process.kill; process.kill = () => { throw new Error('must not signal anything'); }; try { - assert.deepStrictEqual(j.sweep(), { ttyd: 0, tmux: 0, shell: 0, pending: 0 }, 'a failed ps is a no-op sweep'); + assert.deepStrictEqual(j.sweep(), { ttyd: 0, tmux: 0, shell: 0, claude: 0, pending: 0 }, 'a failed ps is a no-op sweep'); } finally { process.kill = realKill; } } -console.log('✓ process janitor: orphaned agent shells reaped; live tenants, other uids and in-flight tool calls untouched'); +// ------------------------------------------------- pane-less claude (2026-11-19, globex: 70 days, 23 GB) +{ + const CONN = '/home/u/tools/agent-os/data/connectors'; + const agentArgv = (id, extra = '') => `claude --session-id 0faa17b3 --settings .claude/aos-settings.json --mcp-config ${CONN}/session-${id}.mcp.json --strict-mcp-config ${extra}`; + const got = parseProcessTable(table( + row(300, 77, agentArgv('ses_dead')), + row(301, 77, agentArgv('ses_live')), + row(302, 77, 'claude -p --tools "" --strict-mcp-config --no-session-persistence'), // router one-shot: no session config + row(303, 77, agentArgv('ses_dead'), OLD, 502), // another uid + ), UID, SELF); + const c = got.find((x) => x.pid === 300); + assert.deepStrictEqual([c.kind, c.session, c.connectors], ['claude', 'ses_dead', CONN], 'session + folder parsed off --mcp-config'); + assert.deepStrictEqual(got.map((x) => x.pid).sort(), [300, 301], 'a claude with no session MCP config, or another uid, is never a candidate'); + + const answers = { ses_dead: false, ses_live: true, ses_unknown: null }; + const asked = []; + const sent = []; + const j = new ProcessJanitor(() => new Set(), (conn, id) => { asked.push(`${conn}|${id}`); return answers[id]; }); + j.scan = () => parseProcessTable(table( + row(300, 77, agentArgv('ses_dead')), + row(301, 77, agentArgv('ses_live')), + row(304, 77, agentArgv('ses_unknown')), + row(305, 77, agentArgv('ses_dead'), YOUNG), + ), UID, SELF); + const realKill = process.kill; + process.kill = (pid, sig) => { sent.push(`${pid}:${sig}`); }; + try { + let r = j.sweep(); + assert.deepStrictEqual([r.claude, sent.length], [0, 0], 'first sighting of a pane-less claude is a strike'); + r = j.sweep(); + assert.strictEqual(r.claude, 1, 'second sighting reaps the pane-less claude'); + assert.deepStrictEqual(sent, ['300:SIGTERM'], 'only the dead-pane, past-grace process is signalled — live pane, unknown liveness and a young one are not'); + } finally { process.kill = realKill; } + assert.ok(asked.includes(`${CONN}|ses_dead`), 'the owning tenant is asked by connectors folder + session id'); + + // No pane oracle (the old constructor) → the claude kind is never reaped. + const j2 = new ProcessJanitor(() => new Set()); + j2.scan = () => parseProcessTable(table(row(300, 77, agentArgv('ses_dead'))), UID, SELF); + process.kill = () => { throw new Error('must not signal without a pane oracle'); }; + try { j2.sweep(); const r = j2.sweep(); assert.strictEqual(r.claude, 0, 'unknown liveness is never a kill'); } + finally { process.kill = realKill; } +} + +console.log('✓ process janitor: orphaned agent shells + pane-less claude reaped; live tenants, other uids and in-flight tool calls untouched'); diff --git a/src/edge/process-janitor.ts b/src/edge/process-janitor.ts index f783e6e0..c07436cb 100644 --- a/src/edge/process-janitor.ts +++ b/src/edge/process-janitor.ts @@ -37,6 +37,17 @@ * exec's, so its argv is `node …`, not the snapshot line, and it is never a candidate. Reaping the orphaned * *shell* also does not kill such a child — the child has its own pid and reparents independently. * + * **Pane-less agents (`claude`)** — one layer further up, and the one that took globex down on + * 2026-11-19. A `claude` whose tmux pane is gone: a member pressed stop 35 s into a session, `stopSession` + * killed the pane, and the `claude` inside ignored the hangup and kept running — for **70 days**, growing to + * **23 GB RSS** until the box's swap was full and the server stopped answering. Six more were found the same + * day (~300 MB each, 8–50 days old). Neither the session reaper (it reaps PANES; this one had none) nor the + * shell predicate (its parent, the pane's launcher, was still alive) could see them. The predicate: our uid, + * a `claude` argv carrying `--mcp-config /session-.mcp.json` (minted per session by the launcher), + * and the owning tenant answering `false` to "is that session's pane alive" — `null` (liveness unknown, or a + * folder no live tenant owns) is never a kill. A pane-less `claude` cannot be attached to, gated through a + * human, resumed or reaped by any row-driven path, so it is unreachable by definition. + * * Killing a zombie tmux server may kill a claude still running inside it. That work is already unreachable * — no attach, no gate, no recovery — so reaping is right, but the caller MUST audit the count: a janitor * that quietly cleans up after a leak hides the leak. @@ -61,15 +72,20 @@ export interface OrphanSweepResult { tmux: number; /** Orphaned agent shells (claude-code Bash-tool shells reparented to init) killed this sweep. */ shell: number; + /** `claude` processes whose session pane is gone, killed this sweep. */ + claude: number; /** Candidates seen missing but not yet killed (first strike, or inside the age grace). */ pending: number; } export interface Candidate { pid: number; - kind: 'ttyd' | 'tmux' | 'shell'; - /** The tmux socket for `ttyd`/`tmux`. Empty for `shell`, whose predicate is PPID, not a socket. */ + kind: 'ttyd' | 'tmux' | 'shell' | 'claude'; + /** The tmux socket for `ttyd`/`tmux`. Empty for `shell`/`claude`, whose predicates aren't a socket. */ socket: string; + /** `claude` only: the session id and the connectors folder its `--mcp-config` lives in. */ + session?: string; + connectors?: string; ageMs: number; } @@ -83,10 +99,15 @@ export class ProcessJanitor { * @param liveSockets sockets currently claimed by a live runtime; never reaped even if the path check * somehow disagrees. Called fresh each sweep so a newly built tenant is respected. */ - constructor(private readonly liveSockets: () => Set) {} + constructor( + private readonly liveSockets: () => Set, + /** `claude` predicate: is session `id`'s pane alive, asked of the tenant owning `connectors`? + * `null` = unknown / no such tenant → never reaped. Omitted → the `claude` kind is never reaped. */ + private readonly paneAlive: (connectors: string, id: string) => boolean | null = () => null, + ) {} sweep(): OrphanSweepResult { - const out: OrphanSweepResult = { ttyd: 0, tmux: 0, shell: 0, pending: 0 }; + const out: OrphanSweepResult = { ttyd: 0, tmux: 0, shell: 0, claude: 0, pending: 0 }; let candidates: Candidate[]; try { candidates = this.scan(); @@ -97,7 +118,11 @@ export class ProcessJanitor { const nextStrikes = new Set(); const nextTermed = new Set(); for (const c of candidates) { - if (c.kind !== 'shell') { + if (c.kind === 'claude') { + let alive: boolean | null = null; + try { alive = this.paneAlive(c.connectors ?? '', c.session ?? ''); } catch { alive = null; } + if (alive !== false) continue; // alive, or we can't tell — not ours to kill + } else if (c.kind !== 'shell') { // Socket-backed kinds: reachable ⇒ not our business. An orphaned shell holds no socket; being // reparented to init IS its unreachability proof, so it skips straight to the age/strike guards. if (live.has(c.socket)) continue; // a running tenant owns it @@ -174,6 +199,13 @@ export function parseProcessTable(stdout: string, uid: number, self: number): Ca out.push({ pid, kind: ttyd ? 'ttyd' : 'tmux', socket, ageMs }); continue; } + // A claude-code agent process, identified by the per-session MCP config its launcher minted. Whether it + // is orphaned is the TENANT's call (is that session's pane alive?) — see ProcessJanitor.sweep. + const agent = args.match(/(?:^|\/)claude\s.*--mcp-config\s+(\S+)\/session-([A-Za-z0-9_-]+)\.mcp\.json(?:\s|$)/); + if (agent) { + out.push({ pid, kind: 'claude', socket: '', session: agent[2], connectors: agent[1], ageMs }); + continue; + } // An agent shell is a candidate ONLY once reparented to init: with a live parent it is a tool call in // flight, and the same argv shape is exactly what a healthy session looks like mid-Bash. if (ppid === 1 && AGENT_SHELL.test(args)) out.push({ pid, kind: 'shell', socket: '', ageMs }); diff --git a/src/server.ts b/src/server.ts index 3e924639..b206e55b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -507,13 +507,25 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve // Shared, process-wide upkeep timers — each fans out across every tenant runtime. // Idle GC (A5): reclaim idle members' uids/ttyds. No-op under the local backend, so always-on is safe. - const reaper = setInterval(() => registry.forEach((rt) => { + // Connector-file retention runs on the same sweep but only every 10 min — one readdir per tenant per pass. + let lastConnectorPrune = 0; + const reaper = setInterval(() => { + const pruneDue = Date.now() - lastConnectorPrune >= 10 * 60_000; + if (pruneDue) lastConnectorPrune = Date.now(); + registry.forEach((rt) => { + if (pruneDue) { + try { + const pr = requestMetrics.phase('reaper:connectorFiles', () => rt.tm.pruneConnectorFiles()); + if (pr.sessions) rt.os.audit.append({ ts: Date.now(), runId: '-', tenant: rt.os.tenant, principal: 'system', type: 'connectors.pruned', data: { sessions: pr.sessions, files: pr.files } }); + } catch { /* retention must never crash the sweep */ } + } try { requestMetrics.phase('reaper:idleSpaces', () => rt.tm.reapIdleSpaces()); } catch { /* never let the sweep crash */ } try { requestMetrics.phase('reaper:idleSessions', () => rt.tm.reapIdleSessions()); } catch { /* idle reaper (warm chat + unattended backstop) — never crash the sweep */ } // The inverse of the reaper: runs parked because every runtime account was rate-limited, launched as // soon as one resets. Same 60s cadence, because a limit reset is only ever known to the minute. try { requestMetrics.phase('reaper:capacityQueue', () => rt.tm.retryCapacityQueue()); } catch { /* a parked launch must never crash the sweep */ } - }), 60_000); + }); + }, 60_000); reaper.unref?.(); // Process janitor: reap ttyd/tmux left behind pointing at tmux sockets that no longer exist, plus agent // shells reparented to init by a dead tool call (see ProcessJanitor). Process-wide, not per-tenant — it @@ -523,6 +535,14 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve const live = new Set(); registry.forEach((rt) => { if (rt.os.paths?.tmuxSocket) live.add(rt.os.paths.tmuxSocket); }); return live; + }, (connectors, id) => { + // Only the tenant that owns this connectors folder may answer; a folder no live tenant owns (a test + // harness, a removed tenant) stays unknown → never reaped. + let answer: boolean | null = null; + registry.forEach((rt) => { + if (answer === null && rt.os.paths && path.resolve(rt.os.paths.connectors) === path.resolve(connectors)) answer = rt.tm.sessionPaneAlive(id); + }); + return answer; }); // Tool-usage flush. Counters accumulate in memory on the request path and land here — one write per // tenant per interval instead of one per agent tool call. 60s bounds what a crash loses to a single @@ -537,10 +557,10 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve const janitorTimer = setInterval(() => { try { const r = requestMetrics.phase('janitor:sweep', () => janitor.sweep()); - if (r.ttyd === 0 && r.tmux === 0 && r.shell === 0) return; + if (r.ttyd === 0 && r.tmux === 0 && r.shell === 0 && r.claude === 0) return; const os = registry.default()?.os; - console.log(` [janitor] reaped ${r.ttyd} orphaned ttyd + ${r.tmux} orphaned tmux (unreachable sockets) + ${r.shell} orphaned agent shells (reparented to init)`); - if (os) os.audit.append({ ts: Date.now(), runId: '-', tenant: os.tenant, principal: 'system', type: 'orphan.reaped', data: { ttyd: r.ttyd, tmux: r.tmux, shell: r.shell, pending: r.pending } }); + console.log(` [janitor] reaped ${r.ttyd} orphaned ttyd + ${r.tmux} orphaned tmux (unreachable sockets) + ${r.shell} orphaned agent shells (reparented to init) + ${r.claude} pane-less claude`); + if (os) os.audit.append({ ts: Date.now(), runId: '-', tenant: os.tenant, principal: 'system', type: 'orphan.reaped', data: { ttyd: r.ttyd, tmux: r.tmux, shell: r.shell, claude: r.claude, pending: r.pending } }); } catch { /* never let the janitor crash the process */ } }, 5 * 60_000); janitorTimer.unref?.(); diff --git a/src/terminal.ts b/src/terminal.ts index 847e4262..33c5bb0b 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -69,6 +69,10 @@ const WARM_CONFIRM_MS = 12_000; * for minutes; one still "busy" after this is wedged, and protecting it would mean never reclaiming * its pane. */ const MID_TURN_MAX_MS = 2 * 3600_000; +/** How long an ENDED session keeps its launch files (see `TerminalManager.pruneConnectorFiles`). */ +export const CONNECTOR_FILE_RETENTION_DAYS = 30; +/** Sessions pruned per pass — bounds the synchronous unlinks one pass costs the event loop (~6 files each). */ +const CONNECTOR_PRUNE_MAX_SESSIONS = 500; /** How long the LAUNCH-TIME `busy_since` prediction may stand before we stop believing a turn started. * `createSession` stamps `busy_since = created_at` because a launch normally seeds a prompt — but that * is a PREDICTION, not evidence, and the runtime has plenty of ways to open without ever running a turn @@ -1468,7 +1472,9 @@ export class TerminalManager { // Outcome + governance counts: cheap indexed lookups off the audit stream (unlike cost, which parses // a transcript), so this isn't budgeted — terminal rows are stamped once, live rows re-tallied. this.stampInsights(visible); - const resumable = this.resumableIds(); + // By-id fetch (the 1.5s summary poll, ≤ a few hundred rows) → one stat per VISIBLE row; the full list + // → one readdir, memoized on the folder's mtime. See resumableIds. + const resumable = this.resumableIds(ids ? visible.map((r) => r.id) : undefined); // One query resolves the whole list's blocked-on-human state (a pending ask/approval), instead of a // per-row check — so the console gets an authoritative `blocked` without re-deriving it from the feed. const blocked = this.blockedSessionIds(); @@ -2299,27 +2305,55 @@ export class TerminalManager { try { return { file, mtime: Math.floor(fs.statSync(file).mtimeMs) }; } catch { return {}; } } + /** {@link resumableIds}' full-folder scan, keyed on the connectors folder's mtime. */ + private resumableMemo: { mtimeMs: number; ids: Set } | null = null; + /** * Session ids that have a persisted launch env (`session-.env`) — i.e. a session the ttyd attach * wrapper can resurrect via `claude --resume` (see `writeEnvFile`/`terminal/attach.sh`). Every * claude-code launch writes one, unattended included; runtimes with no resurrect env (and sessions - * predating that change) are absent and correctly report `resumable:false`. One readdir serves the - * whole list; no data home (demo/tests) → nothing resumable. + * predating that change) are absent and correctly report `resumable:false`. No data home (demo/tests) → + * nothing resumable. + * + * This used to readdir the whole folder on EVERY call, and it is on the console's 1.5s summary poll. The + * folder is never emptied (until {@link pruneConnectorFiles}), so on live globex it held 61k files and the + * scan cost ~50ms a call; with eight people's tabs open that alone kept the event loop busy enough that, + * under CPU contention, `/health` stopped answering (2026-11-19). */ - private resumableIds(): Set { + private resumableIds(only?: string[]): Set { + if (!this.os.paths) return new Set(); + const dir = this.os.paths.connectors; + // A known, bounded id set: stat just those. `only` is always DB-sourced ids (the visible rows), never + // request input, so it can't be steered into a path outside the folder. + if (only) { + const ids = new Set(); + for (const id of only) if (fs.existsSync(path.join(dir, `session-${id}.env`))) ids.add(id); + return ids; + } + // The whole list: one readdir, reused until the folder's mtime moves (any file created, removed or + // renamed into it bumps it; rewriting a file in place does not, and doesn't change which ids are + // resumable either). + let mtimeMs: number; + try { + mtimeMs = fs.statSync(dir).mtimeMs; + } catch { + return new Set(); // connectors dir may not exist yet — nothing resumable + } + if (this.resumableMemo && this.resumableMemo.mtimeMs === mtimeMs) return this.resumableMemo.ids; const ids = new Set(); - if (!this.os.paths) return ids; try { - for (const f of fs.readdirSync(this.os.paths.connectors)) { + for (const f of fs.readdirSync(dir)) { const m = /^session-(.+)\.env$/.exec(f); if (m) ids.add(m[1]); } } catch { - /* connectors dir may not exist yet — nothing resumable */ + return ids; } + this.resumableMemo = { mtimeMs, ids }; return ids; } + /** * The per-member inbox visibility rule. owner/admin see everything; a member sees what THEY * spawned, plus sessions an automation they created fired. Used to scope the sessions list, the @@ -10556,6 +10590,83 @@ export class TerminalManager { } } + /** + * Connector-file RETENTION. Every launch writes `session-.{mcp.json,company.md,task,env,…}` (plus a + * `.d/` dir for ssh-capable runs) and nothing removed them except an explicit session DELETE, so the folder + * only grew: live globex reached 61k files / 677 MB across 11k sessions, and `resumableIds` read the whole + * folder on every console poll. Worse than the size, `.mcp.json` carries each run's RESOLVED connector + * credentials and `.env` its session secret — months of stale credential copies sitting on disk. + * + * So a session that ENDED (done / stopped / crashed) more than {@link CONNECTOR_FILE_RETENTION_DAYS} ago, + * and whose pane is not alive, loses everything except `session-.log` — the pane snapshot the console + * and `session_open` fall back to for a run with no transcript, i.e. the record of what happened. A file + * whose session row no longer exists goes on the same clock, measured by its own mtime. + * + * What it costs: past retention the session stops being one-click `resumable` (no `.env` for attach.sh). + * Take-over still revives it — that path relaunches from `claude_session_id` in the DB and writes fresh + * files — so nothing is lost but the shortcut. + * + * One readdir per pass, at most `maxSessions` sessions unlinked, so a first pass over a large backlog is + * spread across several ticks instead of stalling one. Never throws. + */ + pruneConnectorFiles(nowMs = Date.now(), maxSessions = CONNECTOR_PRUNE_MAX_SESSIONS): { sessions: number; files: number } { + const out = { sessions: 0, files: 0 }; + if (!this.os.paths) return out; + const dir = this.os.paths.connectors; + const cutoff = nowMs - CONNECTOR_FILE_RETENTION_DAYS * 86_400_000; + let names: string[]; + try { names = fs.readdirSync(dir); } catch { return out; } + const byId = new Map(); + for (const f of names) { + const m = /^session-([A-Za-z0-9_-]+)\.(.+)$/.exec(f); + if (!m || m[2] === 'log') continue; // the run's record — kept + const list = byId.get(m[1]); + if (list) list.push(f); else byId.set(m[1], [f]); + } + if (!byId.size) return out; + // Liveness unknown (launcher backend / failed poll) → prune nothing: a terminal row can still hold a pane. + const alive = this.backend.aliveNames(); + if (!alive) return out; + const ids = [...byId.keys()]; + const rows = new Map(); + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + for (const r of this.db.prepare(`SELECT id, status, tmux, COALESCE(updated_at, created_at) AS at FROM term_sessions WHERE id IN (${chunk.map(() => '?').join(',')})`).all<{ id: string; status: string; tmux: string; at: number }>(...chunk)) { + rows.set(r.id, r); + } + } + for (const [id, files] of byId) { + if (out.sessions >= maxSessions) break; + const r = rows.get(id); + if (r) { + if (r.status !== 'done' && r.status !== 'stopped' && r.status !== 'crashed') continue; + if (r.at >= cutoff || alive.has(r.tmux) || this.launching.has(id)) continue; + } else { + // No row (deleted session, or a file a harness left behind): age by the newest file's own mtime. + let newest = 0; + for (const f of files) { try { newest = Math.max(newest, fs.statSync(path.join(dir, f)).mtimeMs); } catch { /* gone */ } } + if (!newest || newest >= cutoff || alive.has(`aos-${id}`)) continue; + } + for (const f of files) { + try { fs.rmSync(path.join(dir, f), { recursive: true, force: true }); out.files++; } catch { /* best-effort */ } + } + out.sessions++; + } + return out; + } + + /** + * Is session `id`'s tmux pane alive? `null` when liveness can't be known (launcher backend, failed poll). + * The process janitor asks this about a `claude` process it found carrying this session's `--mcp-config`: + * `false` means nothing can ever attach to, gate-check or reap that process through a pane again. + */ + sessionPaneAlive(id: string): boolean | null { + const alive = this.backend.aliveNames(); + if (!alive) return null; + const r = this.db.prepare('SELECT tmux FROM term_sessions WHERE id = ?').get<{ tmux: string }>(id); + return alive.has(r?.tmux ?? `aos-${id}`); + } + private audit(sessionId: string, principal: string, type: string, data: Record): void { const ev: AuditEvent = { ts: Date.now(), runId: sessionId, tenant: this.os.tenant, principal, type, data }; this.os.audit.append(ev);