From c8138067321eaaaafb7d163034b828f14e3505e2 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Thu, 19 Nov 2026 13:21:18 +0530 Subject: [PATCH] fix(sessions): bounded resumable lookup, connector-file retention, reap pane-less claude (v0.471.0) The console's 1.5s sessions poll read the whole connectors folder (61k files on a live tenant, ~50ms a call); with several tabs open on a CPU/swap-starved box the server stopped answering. The summary path now stats only its rows and the full list memoizes on the folder mtime. Ended sessions older than 30 days with no live pane now keep only their .log, pruning stale credential copies (.mcp.json, .env) in bounded passes. The process janitor reaps a claude whose session pane is gone (one ran 70 days at 23 GB after its pane was stopped), asking the owning tenant for liveness and never killing on unknown. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 23 +++++ package-lock.json | 4 +- package.json | 4 +- scripts/connectors-retention-test.cjs | 113 +++++++++++++++++++++++ scripts/process-janitor-test.cjs | 47 +++++++++- src/edge/process-janitor.ts | 42 +++++++-- src/server.ts | 30 +++++-- src/terminal.ts | 125 ++++++++++++++++++++++++-- 8 files changed, 365 insertions(+), 23 deletions(-) create mode 100644 scripts/connectors-retention-test.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d95a768..2383f982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,29 @@ new version heading in the same commit. ## [Unreleased] +## [0.471.0] - 2026-11-19 +### Fixed +- **A busy console can no longer starve the server: the sessions poll stops reading the whole connectors + folder.** `resumableIds` listed every file in `/connectors/` on each `/api/sessions/summary` (the + console's 1.5s poll). On live globex that folder had reached 61k files / 677 MB, so each poll cost ~50ms. + With eight people's tabs open, plus a box that was also CPU- and swap-starved, `/health` stopped answering + and requests queued 460 deep. The summary path now stats only the ≤ few hundred rows it returns; the full + list reads the folder once and reuses it until the folder changes. +### Added +- **Connector-file retention.** Nothing but an explicit session delete ever removed a launch's files, and + `.mcp.json` carries each run's resolved connector credentials (`.env` its session secret), so months of + stale credential copies sat on disk. A session that ended more than 30 days ago, and whose pane is gone, + now keeps only `session-.log` (the run's record). The sweep runs every 10 min, at most 500 sessions a + pass, and logs `connectors.pruned` to the audit trail. Past that point an old session is no longer one-click + resumable; Take over still revives it from its conversation id. +- **The process janitor reaps a `claude` whose pane is gone.** On globex a member stopped a session 35s in. + The pane died but the `claude` inside ignored the hangup and ran for 70 days, reaching 23 GB RSS and + filling swap. Six more were found the same day, 8–50 days old at ~300 MB each. Neither the session reaper + (it reaps panes) nor the shell predicate (the parent was still alive) could see them. The janitor now + identifies a session's `claude` by its `--mcp-config …/session-.mcp.json`. It asks the tenant that owns + that folder whether the pane is alive, and reaps on a definite "no" seen on two sweeps past the 10-min + grace; unknown liveness is never a kill. Counted as `claude` in `orphan.reaped`. + ## [0.470.1] - 2026-11-12 ### Fixed - **New agents are born on the tenant's own ruleset, and a hand edit to `agent.json` no longer gets diff --git a/package-lock.json b/package-lock.json index 7f88bc6f..94ae1af2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index e2e8dbe3..07366a75 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.470.1", + "version": "0.471.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", @@ -28,7 +28,7 @@ "dev": "ts-node src/cli.ts serve", "demo:dev": "ts-node src/demo.ts", "test:governance": "node scripts/governance-isolated.cjs", - "test:governance:run": "node scripts/version-sync-test.cjs && node scripts/agent-manifest-disk-test.cjs && node scripts/router-dispatch-test.cjs && node scripts/prompt-credential-guard-test.cjs && node scripts/standing-instructions-test.cjs && node scripts/session-trust-test.cjs && node scripts/video-gen-mapping-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/git-ceiling-test.cjs && node scripts/cron-supersede-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/script-body-gate-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/connector-auth-stub-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-pool-signed-out-test.cjs && node scripts/login-expiry-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/resume-transcript-stage-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/unblock-run-dedupe-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/session-search-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/launch-window-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/protected-read-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/goal-heartbeat-test.cjs && node scripts/question-durability-test.cjs && node scripts/ask-block-link-test.cjs && node scripts/answer-pinning-test.cjs && node scripts/bets-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", + "test:governance:run": "node scripts/version-sync-test.cjs && node scripts/agent-manifest-disk-test.cjs && node scripts/router-dispatch-test.cjs && node scripts/prompt-credential-guard-test.cjs && node scripts/standing-instructions-test.cjs && node scripts/session-trust-test.cjs && node scripts/video-gen-mapping-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/git-ceiling-test.cjs && node scripts/cron-supersede-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/script-body-gate-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/connector-auth-stub-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-pool-signed-out-test.cjs && node scripts/login-expiry-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/resume-transcript-stage-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/unblock-run-dedupe-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/session-search-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/launch-window-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/protected-read-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/connectors-retention-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/goal-heartbeat-test.cjs && node scripts/question-durability-test.cjs && node scripts/ask-block-link-test.cjs && node scripts/answer-pinning-test.cjs && node scripts/bets-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", "test:alert-staleness": "node scripts/alert-staleness-test.cjs", "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs", "test:dm-continuity": "node scripts/dm-continuity-test.cjs", diff --git a/scripts/connectors-retention-test.cjs b/scripts/connectors-retention-test.cjs new file mode 100644 index 00000000..1a12f70d --- /dev/null +++ b/scripts/connectors-retention-test.cjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node +/* CONNECTORS FOLDER — retention + the resumable lookup that read it on every console poll. + * + * Live globex, 2026-11-19: `/connectors/` held 61k files (677 MB) for 11k sessions because nothing + * but an explicit session DELETE ever removed a launch's files, and `resumableIds` read the whole folder on + * every `/api/sessions/summary` (the console's 1.5s poll) — ~50ms a call, enough with eight tabs open to + * keep `/health` from answering once the box was also CPU-starved. Pinned here: + * - the by-id (summary) path stats only the rows it returns, never reads the folder; + * - the full-list path reads the folder once and reuses it until the folder changes; + * - retention prunes ONLY ended + old + pane-less sessions, keeps `.log` (the run's record), honours the + * per-pass bound, and does nothing when liveness is unknown. + * Isolated home; no tmux, no claude (the backend's liveness poll is stubbed). */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'aos-connectors-retention-test-')); +process.env.AGENT_OS_HOME = HOME; +process.env.AGENT_OS_TENANT = 'testco'; +process.env.AOS_NO_TTYD = '1'; +delete process.env.AGENT_OS_SECRET_KEY; + +let pass = 0, fail = 0; +const assert = (c, name, d) => c ? (pass++, console.log(` \x1b[32m✓\x1b[0m ${name}`)) : (fail++, console.log(` \x1b[31m✗ ${name}\x1b[0m${d ? ' — ' + d : ''}`)); + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { TerminalManager, CONNECTOR_FILE_RETENTION_DAYS } = require(path.join(ROOT, 'dist/terminal.js')); + +const aos = loadAgentOS(); +const tm = new TerminalManager(aos, 'http://127.0.0.1:0', path.join(HOME, 'tmux.sock')); +const CONN = aos.paths.connectors; +fs.mkdirSync(CONN, { recursive: true }); + +let alive = new Set(); +tm.backend.aliveNames = () => alive; + +const DAY = 86_400_000; +const OLD = Date.now() - (CONNECTOR_FILE_RETENTION_DAYS + 5) * DAY; +const RECENT = Date.now() - 2 * DAY; +const mkRun = (id, status, at) => aos.db.prepare(`INSERT INTO term_sessions (id,agent,title,task,tmux,status,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?)`).run(id, 'engineer', id, 'task', 'aos-' + id, status, at, at); +const LAUNCH = ['mcp.json', 'company.md', 'task', 'env', 'stopped', 'log']; +const mkFiles = (id, exts = LAUNCH) => { + for (const e of exts) fs.writeFileSync(path.join(CONN, `session-${id}.${e}`), 'x'); + fs.mkdirSync(path.join(CONN, `session-${id}.d`, 'bin'), { recursive: true }); +}; +const filesOf = (id) => fs.readdirSync(CONN).filter((f) => f.startsWith(`session-${id}.`)).sort(); + +mkRun('ses_olddone', 'done', OLD); mkFiles('ses_olddone'); +mkRun('ses_oldstop', 'stopped', OLD); mkFiles('ses_oldstop'); +mkRun('ses_oldcrash', 'crashed', OLD); mkFiles('ses_oldcrash'); +mkRun('ses_recent', 'done', RECENT); mkFiles('ses_recent'); +mkRun('ses_running', 'running', OLD); mkFiles('ses_running'); +mkRun('ses_livepane', 'stopped', OLD); mkFiles('ses_livepane'); +mkFiles('ses_norow'); // a deleted session's leftovers +for (const f of filesOf('ses_norow')) fs.utimesSync(path.join(CONN, f), OLD / 1000, OLD / 1000); +mkFiles('ses_norow_new'); // a deleted session's leftovers, but recent + +// ── resumableIds ────────────────────────────────────────────────────────────────────────────────── +{ + const realReaddir = fs.readdirSync; + let reads = 0; + fs.readdirSync = function (p, ...rest) { if (path.resolve(String(p)) === path.resolve(CONN)) reads++; return realReaddir.call(fs, p, ...rest); }; + try { + const byId = tm.listSessions(undefined, undefined, ['ses_recent', 'ses_running']); + assert(reads === 0, 'the by-id (summary) path never reads the connectors folder', `reads=${reads}`); + assert(byId.every((s) => s.resumable === true), 'by-id rows with a .env are resumable'); + const full1 = tm.listSessions(); + const full2 = tm.listSessions(); + assert(reads === 1, 'the full list reads the folder once and reuses it while nothing changed', `reads=${reads}`); + assert(full1.length === full2.length && full1.find((s) => s.id === 'ses_olddone')?.resumable === true, 'full-list resumable is correct'); + fs.rmSync(path.join(CONN, 'session-ses_recent.env')); + const full3 = tm.listSessions(); + assert(reads === 2, 'removing a file invalidates the memo', `reads=${reads}`); + assert(full3.find((s) => s.id === 'ses_recent')?.resumable === false, 'a removed .env reads non-resumable straight away'); + fs.writeFileSync(path.join(CONN, 'session-ses_recent.env'), 'x'); + } finally { fs.readdirSync = realReaddir; } +} + +// ── retention ───────────────────────────────────────────────────────────────────────────────────── +alive = new Set(['aos-ses_livepane']); +{ + alive = null; + const r = tm.pruneConnectorFiles(); + assert(r.sessions === 0, 'liveness unknown → prunes nothing'); + alive = new Set(['aos-ses_livepane']); +} +{ + const r = tm.pruneConnectorFiles(Date.now(), 2); + assert(r.sessions === 2, 'a pass is bounded by maxSessions', JSON.stringify(r)); + const r2 = tm.pruneConnectorFiles(); + assert(r.sessions + r2.sessions === 4, 'the rest drains on the next pass (3 old ended + 1 old orphan)', JSON.stringify([r, r2])); + for (const id of ['ses_olddone', 'ses_oldstop', 'ses_oldcrash', 'ses_norow']) { + assert(JSON.stringify(filesOf(id)) === JSON.stringify([`session-${id}.log`]), `${id}: everything but .log removed (incl. the .d dir)`, filesOf(id).join(',')); + } + for (const id of ['ses_recent', 'ses_running', 'ses_livepane', 'ses_norow_new']) { + assert(filesOf(id).length === LAUNCH.length + 1, `${id}: untouched`, filesOf(id).join(',')); + } + assert(tm.pruneConnectorFiles().sessions === 0, 'a settled folder is a no-op pass'); + assert(tm.listSessions().find((s) => s.id === 'ses_olddone')?.resumable === false, 'a pruned session is no longer one-click resumable'); +} + +// ── sessionPaneAlive (the process janitor's oracle) ─────────────────────────────────────────────── +assert(tm.sessionPaneAlive('ses_livepane') === true, 'pane alive → true'); +assert(tm.sessionPaneAlive('ses_olddone') === false, 'pane gone → false'); +assert(tm.sessionPaneAlive('ses_norow') === false, 'no row → checks aos-'); +alive = null; +assert(tm.sessionPaneAlive('ses_olddone') === null, 'liveness unknown → null (never a kill)'); + +fs.rmSync(HOME, { recursive: true, force: true }); +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/process-janitor-test.cjs b/scripts/process-janitor-test.cjs index fa1b74ad..b5abdd9c 100644 --- a/scripts/process-janitor-test.cjs +++ b/scripts/process-janitor-test.cjs @@ -119,8 +119,51 @@ function harness(rows, liveSockets = []) { const realKill = process.kill; process.kill = () => { throw new Error('must not signal anything'); }; try { - assert.deepStrictEqual(j.sweep(), { ttyd: 0, tmux: 0, shell: 0, pending: 0 }, 'a failed ps is a no-op sweep'); + assert.deepStrictEqual(j.sweep(), { ttyd: 0, tmux: 0, shell: 0, claude: 0, pending: 0 }, 'a failed ps is a no-op sweep'); } finally { process.kill = realKill; } } -console.log('✓ process janitor: orphaned agent shells reaped; live tenants, other uids and in-flight tool calls untouched'); +// ------------------------------------------------- pane-less claude (2026-11-19, globex: 70 days, 23 GB) +{ + const CONN = '/home/u/tools/agent-os/data/connectors'; + const agentArgv = (id, extra = '') => `claude --session-id 0faa17b3 --settings .claude/aos-settings.json --mcp-config ${CONN}/session-${id}.mcp.json --strict-mcp-config ${extra}`; + const got = parseProcessTable(table( + row(300, 77, agentArgv('ses_dead')), + row(301, 77, agentArgv('ses_live')), + row(302, 77, 'claude -p --tools "" --strict-mcp-config --no-session-persistence'), // router one-shot: no session config + row(303, 77, agentArgv('ses_dead'), OLD, 502), // another uid + ), UID, SELF); + const c = got.find((x) => x.pid === 300); + assert.deepStrictEqual([c.kind, c.session, c.connectors], ['claude', 'ses_dead', CONN], 'session + folder parsed off --mcp-config'); + assert.deepStrictEqual(got.map((x) => x.pid).sort(), [300, 301], 'a claude with no session MCP config, or another uid, is never a candidate'); + + const answers = { ses_dead: false, ses_live: true, ses_unknown: null }; + const asked = []; + const sent = []; + const j = new ProcessJanitor(() => new Set(), (conn, id) => { asked.push(`${conn}|${id}`); return answers[id]; }); + j.scan = () => parseProcessTable(table( + row(300, 77, agentArgv('ses_dead')), + row(301, 77, agentArgv('ses_live')), + row(304, 77, agentArgv('ses_unknown')), + row(305, 77, agentArgv('ses_dead'), YOUNG), + ), UID, SELF); + const realKill = process.kill; + process.kill = (pid, sig) => { sent.push(`${pid}:${sig}`); }; + try { + let r = j.sweep(); + assert.deepStrictEqual([r.claude, sent.length], [0, 0], 'first sighting of a pane-less claude is a strike'); + r = j.sweep(); + assert.strictEqual(r.claude, 1, 'second sighting reaps the pane-less claude'); + assert.deepStrictEqual(sent, ['300:SIGTERM'], 'only the dead-pane, past-grace process is signalled — live pane, unknown liveness and a young one are not'); + } finally { process.kill = realKill; } + assert.ok(asked.includes(`${CONN}|ses_dead`), 'the owning tenant is asked by connectors folder + session id'); + + // No pane oracle (the old constructor) → the claude kind is never reaped. + const j2 = new ProcessJanitor(() => new Set()); + j2.scan = () => parseProcessTable(table(row(300, 77, agentArgv('ses_dead'))), UID, SELF); + process.kill = () => { throw new Error('must not signal without a pane oracle'); }; + try { j2.sweep(); const r = j2.sweep(); assert.strictEqual(r.claude, 0, 'unknown liveness is never a kill'); } + finally { process.kill = realKill; } +} + +console.log('✓ process janitor: orphaned agent shells + pane-less claude reaped; live tenants, other uids and in-flight tool calls untouched'); diff --git a/src/edge/process-janitor.ts b/src/edge/process-janitor.ts index f783e6e0..c07436cb 100644 --- a/src/edge/process-janitor.ts +++ b/src/edge/process-janitor.ts @@ -37,6 +37,17 @@ * exec's, so its argv is `node …`, not the snapshot line, and it is never a candidate. Reaping the orphaned * *shell* also does not kill such a child — the child has its own pid and reparents independently. * + * **Pane-less agents (`claude`)** — one layer further up, and the one that took globex down on + * 2026-11-19. A `claude` whose tmux pane is gone: a member pressed stop 35 s into a session, `stopSession` + * killed the pane, and the `claude` inside ignored the hangup and kept running — for **70 days**, growing to + * **23 GB RSS** until the box's swap was full and the server stopped answering. Six more were found the same + * day (~300 MB each, 8–50 days old). Neither the session reaper (it reaps PANES; this one had none) nor the + * shell predicate (its parent, the pane's launcher, was still alive) could see them. The predicate: our uid, + * a `claude` argv carrying `--mcp-config /session-.mcp.json` (minted per session by the launcher), + * and the owning tenant answering `false` to "is that session's pane alive" — `null` (liveness unknown, or a + * folder no live tenant owns) is never a kill. A pane-less `claude` cannot be attached to, gated through a + * human, resumed or reaped by any row-driven path, so it is unreachable by definition. + * * Killing a zombie tmux server may kill a claude still running inside it. That work is already unreachable * — no attach, no gate, no recovery — so reaping is right, but the caller MUST audit the count: a janitor * that quietly cleans up after a leak hides the leak. @@ -61,15 +72,20 @@ export interface OrphanSweepResult { tmux: number; /** Orphaned agent shells (claude-code Bash-tool shells reparented to init) killed this sweep. */ shell: number; + /** `claude` processes whose session pane is gone, killed this sweep. */ + claude: number; /** Candidates seen missing but not yet killed (first strike, or inside the age grace). */ pending: number; } export interface Candidate { pid: number; - kind: 'ttyd' | 'tmux' | 'shell'; - /** The tmux socket for `ttyd`/`tmux`. Empty for `shell`, whose predicate is PPID, not a socket. */ + kind: 'ttyd' | 'tmux' | 'shell' | 'claude'; + /** The tmux socket for `ttyd`/`tmux`. Empty for `shell`/`claude`, whose predicates aren't a socket. */ socket: string; + /** `claude` only: the session id and the connectors folder its `--mcp-config` lives in. */ + session?: string; + connectors?: string; ageMs: number; } @@ -83,10 +99,15 @@ export class ProcessJanitor { * @param liveSockets sockets currently claimed by a live runtime; never reaped even if the path check * somehow disagrees. Called fresh each sweep so a newly built tenant is respected. */ - constructor(private readonly liveSockets: () => Set) {} + constructor( + private readonly liveSockets: () => Set, + /** `claude` predicate: is session `id`'s pane alive, asked of the tenant owning `connectors`? + * `null` = unknown / no such tenant → never reaped. Omitted → the `claude` kind is never reaped. */ + private readonly paneAlive: (connectors: string, id: string) => boolean | null = () => null, + ) {} sweep(): OrphanSweepResult { - const out: OrphanSweepResult = { ttyd: 0, tmux: 0, shell: 0, pending: 0 }; + const out: OrphanSweepResult = { ttyd: 0, tmux: 0, shell: 0, claude: 0, pending: 0 }; let candidates: Candidate[]; try { candidates = this.scan(); @@ -97,7 +118,11 @@ export class ProcessJanitor { const nextStrikes = new Set(); const nextTermed = new Set(); for (const c of candidates) { - if (c.kind !== 'shell') { + if (c.kind === 'claude') { + let alive: boolean | null = null; + try { alive = this.paneAlive(c.connectors ?? '', c.session ?? ''); } catch { alive = null; } + if (alive !== false) continue; // alive, or we can't tell — not ours to kill + } else if (c.kind !== 'shell') { // Socket-backed kinds: reachable ⇒ not our business. An orphaned shell holds no socket; being // reparented to init IS its unreachability proof, so it skips straight to the age/strike guards. if (live.has(c.socket)) continue; // a running tenant owns it @@ -174,6 +199,13 @@ export function parseProcessTable(stdout: string, uid: number, self: number): Ca out.push({ pid, kind: ttyd ? 'ttyd' : 'tmux', socket, ageMs }); continue; } + // A claude-code agent process, identified by the per-session MCP config its launcher minted. Whether it + // is orphaned is the TENANT's call (is that session's pane alive?) — see ProcessJanitor.sweep. + const agent = args.match(/(?:^|\/)claude\s.*--mcp-config\s+(\S+)\/session-([A-Za-z0-9_-]+)\.mcp\.json(?:\s|$)/); + if (agent) { + out.push({ pid, kind: 'claude', socket: '', session: agent[2], connectors: agent[1], ageMs }); + continue; + } // An agent shell is a candidate ONLY once reparented to init: with a live parent it is a tool call in // flight, and the same argv shape is exactly what a healthy session looks like mid-Bash. if (ppid === 1 && AGENT_SHELL.test(args)) out.push({ pid, kind: 'shell', socket: '', ageMs }); diff --git a/src/server.ts b/src/server.ts index 3e924639..b206e55b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -507,13 +507,25 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve // Shared, process-wide upkeep timers — each fans out across every tenant runtime. // Idle GC (A5): reclaim idle members' uids/ttyds. No-op under the local backend, so always-on is safe. - const reaper = setInterval(() => registry.forEach((rt) => { + // Connector-file retention runs on the same sweep but only every 10 min — one readdir per tenant per pass. + let lastConnectorPrune = 0; + const reaper = setInterval(() => { + const pruneDue = Date.now() - lastConnectorPrune >= 10 * 60_000; + if (pruneDue) lastConnectorPrune = Date.now(); + registry.forEach((rt) => { + if (pruneDue) { + try { + const pr = requestMetrics.phase('reaper:connectorFiles', () => rt.tm.pruneConnectorFiles()); + if (pr.sessions) rt.os.audit.append({ ts: Date.now(), runId: '-', tenant: rt.os.tenant, principal: 'system', type: 'connectors.pruned', data: { sessions: pr.sessions, files: pr.files } }); + } catch { /* retention must never crash the sweep */ } + } try { requestMetrics.phase('reaper:idleSpaces', () => rt.tm.reapIdleSpaces()); } catch { /* never let the sweep crash */ } try { requestMetrics.phase('reaper:idleSessions', () => rt.tm.reapIdleSessions()); } catch { /* idle reaper (warm chat + unattended backstop) — never crash the sweep */ } // The inverse of the reaper: runs parked because every runtime account was rate-limited, launched as // soon as one resets. Same 60s cadence, because a limit reset is only ever known to the minute. try { requestMetrics.phase('reaper:capacityQueue', () => rt.tm.retryCapacityQueue()); } catch { /* a parked launch must never crash the sweep */ } - }), 60_000); + }); + }, 60_000); reaper.unref?.(); // Process janitor: reap ttyd/tmux left behind pointing at tmux sockets that no longer exist, plus agent // shells reparented to init by a dead tool call (see ProcessJanitor). Process-wide, not per-tenant — it @@ -523,6 +535,14 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve const live = new Set(); registry.forEach((rt) => { if (rt.os.paths?.tmuxSocket) live.add(rt.os.paths.tmuxSocket); }); return live; + }, (connectors, id) => { + // Only the tenant that owns this connectors folder may answer; a folder no live tenant owns (a test + // harness, a removed tenant) stays unknown → never reaped. + let answer: boolean | null = null; + registry.forEach((rt) => { + if (answer === null && rt.os.paths && path.resolve(rt.os.paths.connectors) === path.resolve(connectors)) answer = rt.tm.sessionPaneAlive(id); + }); + return answer; }); // Tool-usage flush. Counters accumulate in memory on the request path and land here — one write per // tenant per interval instead of one per agent tool call. 60s bounds what a crash loses to a single @@ -537,10 +557,10 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve const janitorTimer = setInterval(() => { try { const r = requestMetrics.phase('janitor:sweep', () => janitor.sweep()); - if (r.ttyd === 0 && r.tmux === 0 && r.shell === 0) return; + if (r.ttyd === 0 && r.tmux === 0 && r.shell === 0 && r.claude === 0) return; const os = registry.default()?.os; - console.log(` [janitor] reaped ${r.ttyd} orphaned ttyd + ${r.tmux} orphaned tmux (unreachable sockets) + ${r.shell} orphaned agent shells (reparented to init)`); - if (os) os.audit.append({ ts: Date.now(), runId: '-', tenant: os.tenant, principal: 'system', type: 'orphan.reaped', data: { ttyd: r.ttyd, tmux: r.tmux, shell: r.shell, pending: r.pending } }); + console.log(` [janitor] reaped ${r.ttyd} orphaned ttyd + ${r.tmux} orphaned tmux (unreachable sockets) + ${r.shell} orphaned agent shells (reparented to init) + ${r.claude} pane-less claude`); + if (os) os.audit.append({ ts: Date.now(), runId: '-', tenant: os.tenant, principal: 'system', type: 'orphan.reaped', data: { ttyd: r.ttyd, tmux: r.tmux, shell: r.shell, claude: r.claude, pending: r.pending } }); } catch { /* never let the janitor crash the process */ } }, 5 * 60_000); janitorTimer.unref?.(); diff --git a/src/terminal.ts b/src/terminal.ts index 847e4262..33c5bb0b 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -69,6 +69,10 @@ const WARM_CONFIRM_MS = 12_000; * for minutes; one still "busy" after this is wedged, and protecting it would mean never reclaiming * its pane. */ const MID_TURN_MAX_MS = 2 * 3600_000; +/** How long an ENDED session keeps its launch files (see `TerminalManager.pruneConnectorFiles`). */ +export const CONNECTOR_FILE_RETENTION_DAYS = 30; +/** Sessions pruned per pass — bounds the synchronous unlinks one pass costs the event loop (~6 files each). */ +const CONNECTOR_PRUNE_MAX_SESSIONS = 500; /** How long the LAUNCH-TIME `busy_since` prediction may stand before we stop believing a turn started. * `createSession` stamps `busy_since = created_at` because a launch normally seeds a prompt — but that * is a PREDICTION, not evidence, and the runtime has plenty of ways to open without ever running a turn @@ -1468,7 +1472,9 @@ export class TerminalManager { // Outcome + governance counts: cheap indexed lookups off the audit stream (unlike cost, which parses // a transcript), so this isn't budgeted — terminal rows are stamped once, live rows re-tallied. this.stampInsights(visible); - const resumable = this.resumableIds(); + // By-id fetch (the 1.5s summary poll, ≤ a few hundred rows) → one stat per VISIBLE row; the full list + // → one readdir, memoized on the folder's mtime. See resumableIds. + const resumable = this.resumableIds(ids ? visible.map((r) => r.id) : undefined); // One query resolves the whole list's blocked-on-human state (a pending ask/approval), instead of a // per-row check — so the console gets an authoritative `blocked` without re-deriving it from the feed. const blocked = this.blockedSessionIds(); @@ -2299,27 +2305,55 @@ export class TerminalManager { try { return { file, mtime: Math.floor(fs.statSync(file).mtimeMs) }; } catch { return {}; } } + /** {@link resumableIds}' full-folder scan, keyed on the connectors folder's mtime. */ + private resumableMemo: { mtimeMs: number; ids: Set } | null = null; + /** * Session ids that have a persisted launch env (`session-.env`) — i.e. a session the ttyd attach * wrapper can resurrect via `claude --resume` (see `writeEnvFile`/`terminal/attach.sh`). Every * claude-code launch writes one, unattended included; runtimes with no resurrect env (and sessions - * predating that change) are absent and correctly report `resumable:false`. One readdir serves the - * whole list; no data home (demo/tests) → nothing resumable. + * predating that change) are absent and correctly report `resumable:false`. No data home (demo/tests) → + * nothing resumable. + * + * This used to readdir the whole folder on EVERY call, and it is on the console's 1.5s summary poll. The + * folder is never emptied (until {@link pruneConnectorFiles}), so on live globex it held 61k files and the + * scan cost ~50ms a call; with eight people's tabs open that alone kept the event loop busy enough that, + * under CPU contention, `/health` stopped answering (2026-11-19). */ - private resumableIds(): Set { + private resumableIds(only?: string[]): Set { + if (!this.os.paths) return new Set(); + const dir = this.os.paths.connectors; + // A known, bounded id set: stat just those. `only` is always DB-sourced ids (the visible rows), never + // request input, so it can't be steered into a path outside the folder. + if (only) { + const ids = new Set(); + for (const id of only) if (fs.existsSync(path.join(dir, `session-${id}.env`))) ids.add(id); + return ids; + } + // The whole list: one readdir, reused until the folder's mtime moves (any file created, removed or + // renamed into it bumps it; rewriting a file in place does not, and doesn't change which ids are + // resumable either). + let mtimeMs: number; + try { + mtimeMs = fs.statSync(dir).mtimeMs; + } catch { + return new Set(); // connectors dir may not exist yet — nothing resumable + } + if (this.resumableMemo && this.resumableMemo.mtimeMs === mtimeMs) return this.resumableMemo.ids; const ids = new Set(); - if (!this.os.paths) return ids; try { - for (const f of fs.readdirSync(this.os.paths.connectors)) { + for (const f of fs.readdirSync(dir)) { const m = /^session-(.+)\.env$/.exec(f); if (m) ids.add(m[1]); } } catch { - /* connectors dir may not exist yet — nothing resumable */ + return ids; } + this.resumableMemo = { mtimeMs, ids }; return ids; } + /** * The per-member inbox visibility rule. owner/admin see everything; a member sees what THEY * spawned, plus sessions an automation they created fired. Used to scope the sessions list, the @@ -10556,6 +10590,83 @@ export class TerminalManager { } } + /** + * Connector-file RETENTION. Every launch writes `session-.{mcp.json,company.md,task,env,…}` (plus a + * `.d/` dir for ssh-capable runs) and nothing removed them except an explicit session DELETE, so the folder + * only grew: live globex reached 61k files / 677 MB across 11k sessions, and `resumableIds` read the whole + * folder on every console poll. Worse than the size, `.mcp.json` carries each run's RESOLVED connector + * credentials and `.env` its session secret — months of stale credential copies sitting on disk. + * + * So a session that ENDED (done / stopped / crashed) more than {@link CONNECTOR_FILE_RETENTION_DAYS} ago, + * and whose pane is not alive, loses everything except `session-.log` — the pane snapshot the console + * and `session_open` fall back to for a run with no transcript, i.e. the record of what happened. A file + * whose session row no longer exists goes on the same clock, measured by its own mtime. + * + * What it costs: past retention the session stops being one-click `resumable` (no `.env` for attach.sh). + * Take-over still revives it — that path relaunches from `claude_session_id` in the DB and writes fresh + * files — so nothing is lost but the shortcut. + * + * One readdir per pass, at most `maxSessions` sessions unlinked, so a first pass over a large backlog is + * spread across several ticks instead of stalling one. Never throws. + */ + pruneConnectorFiles(nowMs = Date.now(), maxSessions = CONNECTOR_PRUNE_MAX_SESSIONS): { sessions: number; files: number } { + const out = { sessions: 0, files: 0 }; + if (!this.os.paths) return out; + const dir = this.os.paths.connectors; + const cutoff = nowMs - CONNECTOR_FILE_RETENTION_DAYS * 86_400_000; + let names: string[]; + try { names = fs.readdirSync(dir); } catch { return out; } + const byId = new Map(); + for (const f of names) { + const m = /^session-([A-Za-z0-9_-]+)\.(.+)$/.exec(f); + if (!m || m[2] === 'log') continue; // the run's record — kept + const list = byId.get(m[1]); + if (list) list.push(f); else byId.set(m[1], [f]); + } + if (!byId.size) return out; + // Liveness unknown (launcher backend / failed poll) → prune nothing: a terminal row can still hold a pane. + const alive = this.backend.aliveNames(); + if (!alive) return out; + const ids = [...byId.keys()]; + const rows = new Map(); + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + for (const r of this.db.prepare(`SELECT id, status, tmux, COALESCE(updated_at, created_at) AS at FROM term_sessions WHERE id IN (${chunk.map(() => '?').join(',')})`).all<{ id: string; status: string; tmux: string; at: number }>(...chunk)) { + rows.set(r.id, r); + } + } + for (const [id, files] of byId) { + if (out.sessions >= maxSessions) break; + const r = rows.get(id); + if (r) { + if (r.status !== 'done' && r.status !== 'stopped' && r.status !== 'crashed') continue; + if (r.at >= cutoff || alive.has(r.tmux) || this.launching.has(id)) continue; + } else { + // No row (deleted session, or a file a harness left behind): age by the newest file's own mtime. + let newest = 0; + for (const f of files) { try { newest = Math.max(newest, fs.statSync(path.join(dir, f)).mtimeMs); } catch { /* gone */ } } + if (!newest || newest >= cutoff || alive.has(`aos-${id}`)) continue; + } + for (const f of files) { + try { fs.rmSync(path.join(dir, f), { recursive: true, force: true }); out.files++; } catch { /* best-effort */ } + } + out.sessions++; + } + return out; + } + + /** + * Is session `id`'s tmux pane alive? `null` when liveness can't be known (launcher backend, failed poll). + * The process janitor asks this about a `claude` process it found carrying this session's `--mcp-config`: + * `false` means nothing can ever attach to, gate-check or reap that process through a pane again. + */ + sessionPaneAlive(id: string): boolean | null { + const alive = this.backend.aliveNames(); + if (!alive) return null; + const r = this.db.prepare('SELECT tmux FROM term_sessions WHERE id = ?').get<{ tmux: string }>(id); + return alive.has(r?.tmux ?? `aos-${id}`); + } + private audit(sessionId: string, principal: string, type: string, data: Record): void { const ev: AuditEvent = { ts: Date.now(), runId: sessionId, tenant: this.os.tenant, principal, type, data }; this.os.audit.append(ev);