diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83fe38fc..3cef47da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,14 @@ jobs: run: docker network create --subnet 172.30.0.0/16 hack-dev - name: Run local and Docker E2E run: HACK_E2E_REQUIRE_DOCKER=1 HACK_E2E_REQUIRE_TMUX=1 bun run test:e2e:local:docker + - name: Smoke container resource metadata without optional fields + run: | + probe_id=$(docker create --network none --read-only alpine:3.20 true) + trap 'docker rm -f "$probe_id" >/dev/null' EXIT + bun scripts/inspect-container-resources.ts --container "$probe_id" > "$RUNNER_TEMP/resource-probe.json" + bun -e 'const r = await Bun.stdin.json(); if (r.probeStatus !== "not_running" || r.container.healthcheckIntervalNs !== null || r.container.writableLayerBytes !== null) throw new Error("Invalid optional resource metadata");' < "$RUNNER_TEMP/resource-probe.json" + bun scripts/inspect-container-resources.ts --container "$probe_id" --storage > "$RUNNER_TEMP/resource-probe.json" + bun -e 'const r = await Bun.stdin.json(); if (typeof r.container.writableLayerBytes !== "number") throw new Error("Missing writable-layer accounting");' < "$RUNNER_TEMP/resource-probe.json" test: runs-on: blacksmith-6vcpu-macos-15 @@ -99,3 +107,21 @@ jobs: run: bun run build - name: Build release smoke (no tests) run: bun run build:release --skip-tests --no-clean --out=dist/release-ci + + linux-process-lifetime: + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: "1.3.9" + - name: Require native regression tools + run: | + command -v python3 + command -v lsof + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Run process and terminal regressions + run: bun test tests/daemon-command.test.ts tests/daemon-orphan.test.ts tests/host-exec-lifetime.test.ts tests/host-exec-tty.test.ts tests/shell-observation.test.ts diff --git a/docs/env.md b/docs/env.md index 74e837c9..9c641cd9 100644 --- a/docs/env.md +++ b/docs/env.md @@ -247,7 +247,9 @@ the signal to the command. Commands run in an owned process group, so cancellation also stops their descendants, escalating to SIGKILL after two seconds if necessary. Hack returns 130 for SIGINT and 143 for SIGTERM. Interactive commands use a supervisor in their own foreground group on the same terminal, -preserving stdin, separate output streams, and Ctrl-Z/foreground resume. The +preserving stdin, separate output streams, and Ctrl-Z/foreground resume. Piped +stdin also retains the controlling terminal, so commands can open `/dev/tty` +for native authentication prompts. The supervisor holds group ownership until cancellation cleanup finishes. SIGKILL cannot be forwarded; supervisors must terminate the whole owned process tree when force-killing a wrapper. Commands have no implicit time limit, and normal completion preserves the command's exit status. @@ -405,3 +407,49 @@ If you are writing new docs or new project setup flows, document the YAML overla - [Sessions](sessions.md) - [CLI reference](cli.md) - [Pulumi-style env config design](plans/2026-03-27-pulumi-style-env-config-design.md) + + +### Host command lifetime and diagnostics + +`hack host exec` and `hack env exec` record payload-free execution metadata under +`$HACK_HOME/host-commands` (normally `~/.hack/host-commands`). Records contain the +executable basename, project, process identities, owned process group, intended +lifetime, elapsed time, exit/cancellation/timeout outcome, and final child CPU/RSS +accounting. Arguments, shell strings, environment values and command output are +not recorded. Completed records expire after seven days during later executions; +interrupted records remain available for review. A diagnostic storage failure +warns on stderr and preserves command execution and exit status. + +```sh +hack host exec --timeout 60 -- bun scripts/check.ts +bun scripts/inspect-container-resources.ts --container --runtime node +bun scripts/inspect-container-resources.ts --container --storage +``` + +The probe reports selected Docker metadata, cgroup memory/current/peak/anonymous +and file counters, OOM events, cumulative CPU, process RSS and inotify watch-entry +counts. It requires Bun or Node inside a running container. Unsupported runtime, +procfs/cgroup access or stopped containers are reported as unavailable; the script +never starts containers. Process enumeration is capped at 128 processes and 4096 +file descriptors with a three-second scan budget. The observer's RSS is reported +separately, but its allocation still affects cgroup totals. Watch entries are +counts, not necessarily unique files or proof that watching is expensive. No argv, +environment, application files or output logs are read by the probe. + +Compare at least two CPU/memory samples: cumulative counters alone cannot establish +current load or a leak. Inspect anonymous memory, file cache and OOM events before +changing limits. Frequent healthchecks against a full application route can trigger +rendering/database work; use measured request costs to decide whether an application +should provide a cheaper readiness route. + +## Watchers and caches + +Use mount metadata and actual watch counts before adding volumes or ignore rules. +Measure the same tracked source-file reads on the host and in the container, with +identical file counts and bytes. Repeated reads measure warmed behavior; do not +call the first observed read cold without controlling caches. Do not flush system +caches or write benchmark files into an active source tree as a default diagnostic. +A shared source bind across many services establishes fan-out, not its CPU cost. + +## Stopped containers + +```sh +hack projects prune --dry-run --json +hack projects prune --project my-project --dry-run --json +``` + +The preview uses existing registry/runtime ownership and missing-path checks, and +changes nothing. It reports candidates, not proof that their writable data is +safe to discard. A missing working directory might require further review of +worktree ownership or disconnected storage. Existing `projects prune --json` +without `--dry-run` applies cleanup, so use the preview explicitly. + +The resource probe's `--storage` option adds writable-layer size and changed-path +prefix counts. Sizes exclude named-volume data and are not exact reclaimed disk +space. Large changes outside declared volume destinations may be application data; +review or preserve them before removal. Successful dependency/setup containers can +legitimately be stopped. Cleanup can recover storage and reduce inventory work; +stopped containers do not execute CPU work. Never infer that broad pruning is a +CPU remedy. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7a9c92de..6cdfe733 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1122,6 +1122,7 @@ hack usage [options] | Option | Description | | --- | --- | | `--project ` | Target a registered project by name (from ~/.hack/projects.json) | +| `--details` | Show per-container usage and mount types | | `--include-global` | Include global infra projects under ~/.hack (e.g. logging stack) | | `--watch` | Refresh usage continuously | | `--interval ` | Refresh interval (ms) for --watch | @@ -1154,6 +1155,9 @@ hack projects [options] | --- | --- | | `--project ` | Target a registered project by name (from ~/.hack/projects.json) | | `--details` | Show per-project service tables | +| `--summary` | Return compact project counts with --json; load details with --project | +| `--timings` | Write numeric listing phase timings to stderr (requires --json) | +| `--no-daemon` | Read runtime directly instead of the daemon cache | | `--meta` | Include git/worktree/session/env metadata (implies --details) | | `--include-global` | Include global infra projects under ~/.hack (e.g. logging stack) | | `--all` | Include unregistered docker compose projects (best-effort) | @@ -1178,6 +1182,7 @@ hack projects prune [options] | --- | --- | | `--project ` | Target a registered project by name (from ~/.hack/projects.json) | | `--include-global` | Include global infra projects under ~/.hack (e.g. logging stack) | +| `--dry-run` | Report prune candidates without changing registry entries or containers | | `--json` | Output JSON (machine-readable) | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | | `--help, -h` | Show help | @@ -2250,6 +2255,8 @@ Inject the selected Hack env overlay directly into a one-off host command withou | `--service ` | Target scope (global or a discovered service name) | | `--target ` | Env view for host commands (default: host rewrites container-oriented addresses for local host execution) | | `--shell ` | Run a shell command string via /bin/sh -lc after env injection so `$VAR` expansion happens inside the child shell | +| `--timeout ` | Bound a non-TTY host command; terminate its process group and return 124 on expiry | +| `--lifetime ` | Declare intended lifetime for diagnostics (default: command; does not detach) | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | | `--help, -h` | Show help | | `--version, -v` | Show version | @@ -2350,6 +2357,7 @@ Use hack host when a command should run on your host machine, not inside the com | Command | Summary | | --- | --- | +| `hack host ps` | Inspect host command lifetimes, CPU and lifecycle ownership (read-only) | | `hack host exec [command...]` | Run a host command with project env injected | | `hack host shell` | Open a host shell with project env injected | @@ -2361,6 +2369,26 @@ Use hack host when a command should run on your host machine, not inside the com | `--help, -h` | Show help | | `--version, -v` | Show version | +## `hack host ps` + +Inspect host command lifetimes, CPU and lifecycle ownership (read-only) + +### Usage + +```bash +hack host ps [options] +``` + +### Options + +| Option | Description | +| --- | --- | +| `--project ` | Target a registered project by name (from ~/.hack/projects.json) | +| `--json` | Output JSON (machine-readable) | +| `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | +| `--help, -h` | Show help | +| `--version, -v` | Show version | + ## `hack host exec [command...]` Run a host command with project env injected @@ -2389,6 +2417,8 @@ Run a one-off command on the host with the selected Hack env overlay injected. U | `--scope ` | Resolve values for one env scope while still running the command on the host | | `--target ` | Env view for host commands (default: host rewrites container-oriented addresses for local host execution) | | `--shell ` | Run a shell command string via /bin/sh -lc after env injection so `$VAR` expansion happens inside the child shell | +| `--timeout ` | Bound a non-TTY host command; terminate its process group and return 124 on expiry | +| `--lifetime ` | Declare intended lifetime for diagnostics (default: command; does not detach) | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | | `--help, -h` | Show help | | `--version, -v` | Show version | diff --git a/scripts/inspect-container-resources.ts b/scripts/inspect-container-resources.ts new file mode 100644 index 00000000..9d86e37f --- /dev/null +++ b/scripts/inspect-container-resources.ts @@ -0,0 +1,115 @@ +import { parseArgs } from "node:util"; +import { isRecord } from "../src/lib/guards.ts"; +import { exec } from "../src/lib/shell.ts"; + +const { values } = parseArgs({ + args: Bun.argv.slice(2), + options: { + container: { type: "string" }, + runtime: { type: "string", default: "bun" }, + storage: { type: "boolean", default: false }, + }, +}); +if (!(values.container && /^[\w][\w.-]*$/.test(values.container))) { + throw new Error( + "Use --container [--runtime bun|node] [--storage]" + ); +} +if (values.runtime !== "bun" && values.runtime !== "node") { + throw new Error( + "--runtime must be bun or node (installed inside the container)" + ); +} + +const format = + '{"id":{{json .ID}},"name":{{json .Name}},"state":{{json .State.Status}},"oomKilled":{{.State.OOMKilled}},"restartCount":{{.RestartCount}},"project":{{json (index .Config.Labels "com.docker.compose.project")}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"mounts":{{json .Mounts}},"healthcheckIntervalNs":{{with .Config.Healthcheck}}{{json .Interval}}{{else}}null{{end}},"writableLayerBytes":{{json .SizeRw}}}'; +const metadata = await exec( + [ + "docker", + "inspect", + "--type", + "container", + "--format", + format, + ...(values.storage ? ["--size"] : []), + values.container, + ], + { stdin: "ignore", timeoutMs: 15_000 } +); +if (metadata.exitCode !== 0) { + throw new Error("Container inspection unavailable"); +} +const container: unknown = JSON.parse(metadata.stdout); +if (!isRecord(container)) { + throw new Error("Invalid container metadata"); +} + +// Runs under either Node or Bun. Fixed /proc and cgroup paths only: never reads argv, env or application files. +const probe = String.raw` +const fs = require("node:fs"); +const read = (path) => { try { return fs.readFileSync(path, "utf8"); } catch { return null; } }; +const numeric = (path) => { const value = read(path); return value === null ? null : value.trim() === "max" ? "max" : Number(value.trim()); }; +const counters = (path) => { const value = read(path); return value === null ? null : Object.fromEntries(value.trim().split("\n").map((line) => { const [key, number] = line.split(/\s+/); return [key, Number(number)]; })); }; +const processes = []; +const deadline = Date.now() + 3000; +let fdCount = 0, truncated = false; +if (process.platform === "linux") { + for (const pid of fs.readdirSync("/proc").filter((value) => /^\d+$/.test(value))) { + if (Number(pid) === process.pid) continue; + if (processes.length >= 128 || Date.now() > deadline) { truncated = true; break; } + const status = read("/proc/" + pid + "/status"); + if (!status) continue; + const fields = Object.fromEntries(status.trim().split("\n").map((line) => { const i = line.indexOf(":"); return [line.slice(0, i), line.slice(i + 1).trim()]; })); + let watches = 0, descriptors = 0, watchesAvailable = true; + try { + for (const fd of fs.readdirSync("/proc/" + pid + "/fdinfo")) { + if (++fdCount > 4096 || Date.now() > deadline) { truncated = true; watchesAvailable = false; break; } + const info = read("/proc/" + pid + "/fdinfo/" + fd); + if (info === null) { watchesAvailable = false; continue; } + const count = (info.match(/^inotify wd:/gm) || []).length; + if (count) { descriptors++; watches += count; } + } + } catch { watchesAvailable = false; } + processes.push({ pid: Number(pid), parentPid: Number(fields.PPid), name: fields.Name, rssBytes: fields.VmRSS ? parseInt(fields.VmRSS) * 1024 : null, threads: Number(fields.Threads), inotifyDescriptors: watchesAvailable ? descriptors : null, inotifyWatchEntries: watchesAvailable ? watches : null }); + } +} +console.log(JSON.stringify({ sampledAt: new Date().toISOString(), platform: process.platform, truncated, probeRssBytes: process.memoryUsage().rss, memoryCurrentBytes: numeric("/sys/fs/cgroup/memory.current"), memoryPeakBytes: numeric("/sys/fs/cgroup/memory.peak"), memoryLimitBytes: numeric("/sys/fs/cgroup/memory.max"), memoryStat: counters("/sys/fs/cgroup/memory.stat"), memoryEvents: counters("/sys/fs/cgroup/memory.events"), cpuStat: counters("/sys/fs/cgroup/cpu.stat"), processes })); +`; + +let resources: unknown = null; +let probeStatus = "not_running"; +if (container.state === "running") { + const child = Bun.spawn( + ["docker", "exec", "-i", values.container, values.runtime, "-"], + { stdin: "pipe", stdout: "pipe", stderr: "pipe", timeout: 10_000 } + ); + child.stdin.write(probe); + child.stdin.end(); + const [stdout, , exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + probeStatus = exitCode === 0 ? "available" : "unavailable"; + if (exitCode === 0) { + resources = JSON.parse(stdout); + } +} + +let changedPrefixes: Record | null = null; +if (values.storage) { + const diff = await exec(["docker", "diff", values.container], { + stdin: "ignore", + timeoutMs: 15_000, + }); + if (diff.exitCode === 0) { + changedPrefixes = {}; + for (const line of diff.stdout.split("\n").filter(Boolean)) { + const prefix = line.slice(2).split("/").slice(0, 3).join("/"); + changedPrefixes[prefix] = (changedPrefixes[prefix] ?? 0) + 1; + } + } +} +process.stdout.write( + `${JSON.stringify({ container, probeStatus, resources, changedPrefixes }, null, 2)}\n` +); diff --git a/src/commands/env.ts b/src/commands/env.ts index 404963fb..ddb38aa4 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -39,6 +39,7 @@ import { serializeEnvClassificationForJson, serializeEnvStorageForJson as serializeEnvStorageForJsonShape, } from "../lib/hack-env-status.ts"; +import { runObservedHostCommand } from "../lib/host-command-observation.ts"; import { canPrompt, confirmSafe, @@ -76,6 +77,7 @@ import { import { run } from "../lib/shell.ts"; import { display } from "../ui/display.ts"; import { logger } from "../ui/logger.ts"; +import { hostPsCommand } from "./host-ps.ts"; const optShowSecrets = defineOption({ name: "showSecrets", @@ -146,6 +148,23 @@ const optShellCommand = defineOption({ "Run a shell command string via /bin/sh -lc after env injection so `$VAR` expansion happens inside the child shell", } as const); +const optHostTimeout = defineOption({ + name: "timeout", + type: "number", + long: "--timeout", + valueHint: "", + description: + "Bound a non-TTY host command; terminate its process group and return 124 on expiry", +} as const); +const optHostLifetime = defineOption({ + name: "lifetime", + type: "string", + long: "--lifetime", + valueHint: "", + description: + "Declare intended lifetime for diagnostics (default: command; does not detach)", +} as const); + const SECRET_MASK = "***"; const MODERN_ENV_STATUS_CLASSIFICATION = { trust_model: "repo_managed_env_config", @@ -239,6 +258,8 @@ const execSpec = defineCommand({ optService, optTarget, optShellCommand, + optHostTimeout, + optHostLifetime, ], positionals: [{ name: "command", required: false, multiple: true }], subcommands: [], @@ -261,7 +282,16 @@ const hostExecSpec = defineCommand({ group: "Project", description: 'Run a one-off command on the host with the selected Hack env overlay injected. Use --scope when you want service-scoped values without running inside that service container. To inspect a value, prefer `printenv KEY` or `sh -lc \'printf "%s\\n" "$KEY"\'`; `echo $KEY` expands in your current shell before Hack injects env.', - options: [optPath, optProject, optEnv, optScope, optTarget, optShellCommand], + options: [ + optPath, + optProject, + optEnv, + optScope, + optTarget, + optShellCommand, + optHostTimeout, + optHostLifetime, + ], positionals: [{ name: "command", required: false, multiple: true }], subcommands: [], } as const); @@ -1885,7 +1915,35 @@ async function runHostCommandWithInjectedEnv(input: { readonly targetOpt: string | undefined; readonly command: readonly string[]; readonly shellCommandOpt?: string; + readonly timeout?: number; + readonly lifetime?: string; }): Promise { + if ( + input.timeout !== undefined && + (!Number.isFinite(input.timeout) || + input.timeout <= 0 || + input.timeout * 1000 > 2_147_483_647) + ) { + throw new CliUsageError( + "--timeout must be a positive number of seconds, at most 2147483." + ); + } + if ( + input.lifetime !== undefined && + !["command", "persistent"].includes(input.lifetime) + ) { + throw new CliUsageError("--lifetime must be command or persistent."); + } + if (input.timeout !== undefined && input.lifetime === "persistent") { + throw new CliUsageError( + "--timeout cannot be combined with --lifetime persistent." + ); + } + if (input.timeout !== undefined && process.stdin.isTTY) { + throw new CliUsageError( + "--timeout requires non-TTY stdin; pipe input or redirect from /dev/null for a finite job." + ); + } const project = await resolveProjectForEnv({ ctx: input.ctx, pathOpt: input.pathOpt, @@ -1917,17 +1975,23 @@ async function runHostCommandWithInjectedEnv(input: { }), target, }); - return await run( - shellCommand + const declaredLifetime = + input.lifetime === "persistent" ? "persistent" : "command"; + return await runObservedHostCommand({ + command: shellCommand ? resolveShellCommandCommand({ command: shellCommand }) : positionalCommand, - { + project: projectName, + projectRoot: project.projectRoot, + lifetime: input.timeout !== undefined ? "bounded" : declaredLifetime, + runOptions: { cwd: project.projectRoot, env: envState.env, stdin: "inherit", forwardSignals: true, - } - ); + timeoutMs: input.timeout === undefined ? undefined : input.timeout * 1000, + }, + }); } async function openHostShellWithInjectedEnv(input: { @@ -1982,6 +2046,8 @@ const handleEnvExec: CommandHandlerFor = async ({ targetOpt: args.options.target, command: args.positionals.command, shellCommandOpt: args.options.shellCommand, + timeout: args.options.timeout, + lifetime: args.options.lifetime, }); }; @@ -2012,6 +2078,8 @@ const handleHostExec: CommandHandlerFor = async ({ targetOpt: args.options.target, command: args.positionals.command, shellCommandOpt: args.options.shellCommand, + timeout: args.options.timeout, + lifetime: args.options.lifetime, }); }; @@ -2413,6 +2481,7 @@ export const hostCommand = defineCommand({ options: [], positionals: [], subcommands: [ + hostPsCommand, withHandler(hostExecSpec, handleHostExec), withHandler(hostShellSpec, handleHostShell), ], diff --git a/src/commands/host-ps.ts b/src/commands/host-ps.ts new file mode 100644 index 00000000..c8112473 --- /dev/null +++ b/src/commands/host-ps.ts @@ -0,0 +1,189 @@ +import { resolve } from "node:path"; +import { defineCommand, withHandler } from "../cli/command.ts"; +import { optJson, optProject } from "../cli/options.ts"; +import { + type ObservedProcess, + observeHostCommand, + readHostCommandRecords, + readObservedProcesses, +} from "../lib/host-command-observation.ts"; +import { + type LifecycleStateEntry, + readLifecycleState, +} from "../lib/lifecycle-runtime.ts"; +import { resolvePersistedLifecycleProcessGroupIds } from "../lib/project-lifecycle-processes.ts"; +import { inspectLifecycleSession } from "../lib/project-lifecycle-sessions.ts"; +import { + type ProjectsRegistry, + readProjectsRegistry, +} from "../lib/projects-registry.ts"; +import { getMuxBackends } from "../mux/mux-resolver.ts"; +import { display } from "../ui/display.ts"; + +const spec = defineCommand({ + name: "ps", + summary: + "Inspect host command lifetimes, CPU and lifecycle ownership (read-only)", + group: "Integrations", + options: [optProject, optJson], + positionals: [], + subcommands: [], +} as const); + +export const hostPsCommand = withHandler(spec, async ({ args }) => { + const [records, snapshot, registry] = await Promise.all([ + readHostCommandRecords(), + readObservedProcesses(), + readProjectsRegistry(), + ]); + const filter = args.options.project; + const commands = records + .filter((record) => !filter || record.project === filter) + .map((record) => observeHostCommand(record, snapshot)); + const lifecycle = await readLifecycleObservations({ + registry, + filter, + snapshot, + }); + if (args.options.json) { + process.stdout.write( + `${JSON.stringify({ generatedAt: new Date().toISOString(), snapshotAvailable: snapshot !== null, commands, lifecycle }, null, 2)}\n` + ); + return 0; + } + await display.table({ + columns: [ + "Project", + "Executable", + "PID", + "Lifetime", + "State", + "Elapsed (s)", + "CPU (s)", + "PGID", + ], + rows: commands.map((row) => [ + row.project, + row.executable, + row.child.pid, + row.lifetime, + row.status, + (row.elapsedMs / 1000).toFixed(1), + formatCpuSeconds(row.liveCpuTimeMs ?? row.cpuTimeMs), + row.processGroupId ?? "shared", + ]), + }); + if (lifecycle.length) { + await display.table({ + columns: ["Project", "Session", "Lifetime", "Ownership"], + rows: lifecycle.map((row) => [ + String(row.project), + String(row.session), + "persistent", + String(row.ownership), + ]), + }); + } + return 0; +}); + +async function readLifecycleObservations({ + registry, + filter, + snapshot, +}: { + readonly registry: ProjectsRegistry; + readonly filter: string | undefined; + readonly snapshot: readonly ObservedProcess[] | null; +}): Promise[]> { + const lifecycle: Record[] = []; + const directories = new Map(); + for (const project of registry.projects.filter( + (project) => !filter || project.name === filter + )) { + directories.set(project.projectDir, project.repoRoot); + for (const worktree of project.worktrees ?? []) { + directories.set( + resolve(worktree.path, project.projectDirName), + worktree.path + ); + } + } + for (const [projectDir, projectRoot] of directories) { + for (const entry of await readLifecycleState({ projectDir })) { + lifecycle.push( + await observeLifecycleEntry({ entry, projectRoot, snapshot }) + ); + } + } + return lifecycle; +} +async function observeLifecycleEntry({ + entry, + projectRoot, + snapshot, +}: { + readonly entry: LifecycleStateEntry; + readonly projectRoot: string; + readonly snapshot: readonly ObservedProcess[] | null; +}): Promise> { + const backends = getMuxBackends(); + const backend = backends.get(entry.backend); + const inspection = backend + ? await inspectLifecycleSession({ + backend, + entry, + expectedSessionName: entry.sessionName, + expectedProjectRoot: projectRoot, + expectedDefinitionHash: entry.definitionHash ?? "", + }).catch(() => null) + : null; + const owned = + inspection?.classification === "owned-healthy" || + inspection?.classification === "owned-stale" || + inspection?.classification === "legacy-owned"; + const groups = + owned && snapshot + ? resolvePersistedLifecycleProcessGroupIds({ + lifecycleEntry: entry, + snapshot, + }) + : []; + const members = + snapshot?.filter((row) => groups.includes(row.processGroupId)) ?? []; + return { + project: entry.projectName, + projectRoot, + branch: entry.branch, + session: entry.sessionName, + backend: entry.backend, + lifetime: "persistent", + ownership: inspection?.classification ?? "unknown", + definitionSource: "persisted", + processGroupIds: groups, + observedPids: members.map((row) => row.pid), + liveCpuTimeMs: + owned && snapshot + ? members.reduce((sum, row) => sum + row.cpuTimeMs, 0) + : null, + liveRssBytes: + owned && snapshot + ? members.reduce((sum, row) => sum + row.rssBytes, 0) + : null, + elapsedMs: members.length + ? Math.max(...members.map((row) => row.elapsedMs)) + : null, + attention: lifecycleAttention(inspection?.classification), + }; +} + +function lifecycleAttention(classification: string | undefined): string | null { + if (classification === "absent") { + return "session_absent_review_state"; + } + return classification === "foreign" ? "ownership_unverified" : null; +} + +function formatCpuSeconds(value: number | null): string { + return value === null ? "n/a" : (value / 1000).toFixed(2); +} diff --git a/src/commands/projects.ts b/src/commands/projects.ts index 106a487b..b92ec88c 100644 --- a/src/commands/projects.ts +++ b/src/commands/projects.ts @@ -1,6 +1,11 @@ import { resolve } from "node:path"; import type { CommandHandlerFor } from "../cli/command.ts"; -import { defineCommand, defineOption, withHandler } from "../cli/command.ts"; +import { + CliUsageError, + defineCommand, + defineOption, + withHandler, +} from "../cli/command.ts"; import { optJson, optProject } from "../cli/options.ts"; import { PROJECT_COMPOSE_FILENAME } from "../constants.ts"; import { requestDaemonJson } from "../daemon/client.ts"; @@ -10,6 +15,10 @@ import { } from "../lib/caddy-hosts.ts"; import { emitCliResult, okResult } from "../lib/cli-result.ts"; import { confirmSafe } from "../lib/interactivity.ts"; +import { + createOperationTimings, + type OperationTimings, +} from "../lib/operation-timings.ts"; import { findProjectContext } from "../lib/project.ts"; import { type ProjectMeta, resolveProjectMeta } from "../lib/project-meta.ts"; import { @@ -20,6 +29,7 @@ import { import type { ProjectView } from "../lib/project-views.ts"; import { buildProjectViews, + serializeProjectSummary, serializeProjectView, } from "../lib/project-views.ts"; import { @@ -71,9 +81,40 @@ const optMeta = defineOption({ description: "Include git/worktree/session/env metadata (implies --details)", } as const); +const optSummary = defineOption({ + name: "summary", + type: "boolean", + long: "--summary", + description: + "Return compact project counts with --json; load details with --project", +} as const); +const optTimings = defineOption({ + name: "timings", + type: "boolean", + long: "--timings", + description: + "Write numeric listing phase timings to stderr (requires --json)", +} as const); +const optNoDaemon = defineOption({ + name: "noDaemon", + type: "boolean", + long: "--no-daemon", + description: "Read runtime directly instead of the daemon cache", +} as const); +const optDryRun = defineOption({ + name: "dryRun", + type: "boolean", + long: "--dry-run", + description: + "Report prune candidates without changing registry entries or containers", +} as const); + const options = [ optProject, optDetails, + optSummary, + optTimings, + optNoDaemon, optMeta, optIncludeGlobal, optAll, @@ -99,7 +140,12 @@ const statusSpec = defineCommand({ expandInRootHelp: true, } as const); -const pruneOptions = [optProject, optIncludeGlobal, optJson] as const; +const pruneOptions = [ + optProject, + optIncludeGlobal, + optDryRun, + optJson, +] as const; const pruneSpec = defineCommand({ name: "prune", summary: "Remove stale registry entries and stop orphaned containers", @@ -123,13 +169,30 @@ const handleProjects: CommandHandlerFor = async ({ ctx, args, }): Promise => { + if ((args.options.summary || args.options.timings) && !args.options.json) { + throw new CliUsageError("--summary and --timings require --json."); + } + if (args.options.summary && (args.options.meta || args.options.details)) { + throw new CliUsageError( + "--summary cannot be combined with --meta or --details." + ); + } + const profiler = createOperationTimings(); + const started = performance.now(); const requestedProject = typeof args.options.project === "string" ? sanitizeName(args.options.project) : ""; const filter = requestedProject.length > 0 ? requestedProject : null; - await touchCwdProjectRegistration({ cwd: ctx.cwd }); + await profiler.measure("cwd_registration_ms", () => + touchCwdProjectRegistration({ cwd: ctx.cwd }) + ); return await runProjects({ + profiler, + started, + summary: args.options.summary === true, + timings: args.options.timings === true, + noDaemon: args.options.noDaemon === true, filter, includeGlobal: args.options.includeGlobal === true, includeUnregistered: args.options.all === true, @@ -236,6 +299,30 @@ const handlePrune: CommandHandlerFor = async ({ 0 ); + if (args.options.dryRun) { + const data = { + dryRun: true, + runtimeOk: runtimeResult.ok, + registryCandidates: candidates.map((entry) => ({ + name: entry.project.name, + projectDir: entry.project.projectDir, + reason: entry.reason, + })), + orphanedProjects: orphaned.map((entry) => ({ ...entry })), + candidateContainerCount: orphanedContainerCount, + }; + if (json) { + emitCliResult({ result: okResult({ data }) }); + } else { + await display.panel({ + title: "Prune preview", + tone: "info", + lines: [JSON.stringify(data, null, 2)], + }); + } + return 0; + } + if (json) { await applyPrune({ candidates, orphaned }); emitCliResult({ @@ -369,48 +456,57 @@ async function runProjects(opts: { readonly details: boolean; readonly meta: boolean; readonly json: boolean; + readonly profiler?: OperationTimings; + readonly started?: number; + readonly summary?: boolean; + readonly timings?: boolean; + readonly noDaemon?: boolean; }): Promise { - const daemonRuntimeMeta = opts.json - ? null - : await readDaemonRuntimeRecoveryMeta(); + const profiler = opts.profiler ?? createOperationTimings(); + const started = opts.started ?? performance.now(); + const daemonRuntimeMeta = + opts.json || opts.noDaemon ? null : await readDaemonRuntimeRecoveryMeta(); - if (opts.json) { - const daemon = await requestDaemonJson({ - path: "/v1/projects", - query: { - filter: opts.filter ?? null, - include_global: opts.includeGlobal, - include_unregistered: opts.includeUnregistered, - include_meta: opts.meta, - }, - }); - if (daemon?.ok && daemon.json) { - process.stdout.write(`${JSON.stringify(daemon.json, null, 2)}\n`); - return 0; - } + if ( + opts.json && + !opts.noDaemon && + (await outputDaemonProjects({ opts, profiler, started })) + ) { + return 0; } const runtime = await readRuntimeProjects({ includeGlobal: opts.includeGlobal, + profiler, }); if (runtime.ok) { - await autoRegisterRuntimeHackProjects({ runtime: runtime.runtime }); + await profiler.measure("auto_register_ms", () => + autoRegisterRuntimeHackProjects({ runtime: runtime.runtime }) + ); } - const registry = await readProjectsRegistry(); + const registry = await profiler.measure("registry_ms", readProjectsRegistry); - const views = await buildProjectViews({ - registryProjects: registry.projects, - runtime: runtime.runtime, - runtimeOk: runtime.ok, - filter: opts.filter, - includeUnregistered: opts.includeUnregistered, - }); + const views = await profiler.measure("project_views_ms", () => + buildProjectViews({ + registryProjects: registry.projects, + runtime: runtime.runtime, + runtimeOk: runtime.ok, + filter: opts.filter, + includeUnregistered: opts.includeUnregistered, + }) + ); const metaByName = opts.meta - ? await buildMetaByProjectName({ views }) + ? await profiler.measure("metadata_ms", () => + buildMetaByProjectName({ views }) + ) : new Map(); if (opts.json) { outputProjectsJson({ + summary: opts.summary === true, + profiler, + started, + timings: opts.timings === true, filter: opts.filter, includeGlobal: opts.includeGlobal, includeUnregistered: opts.includeUnregistered, @@ -476,6 +572,47 @@ async function runProjects(opts: { return 0; } +async function outputDaemonProjects({ + opts, + profiler, + started, +}: { + readonly opts: Parameters[0]; + readonly profiler: OperationTimings; + readonly started: number; +}): Promise { + const daemon = await profiler.measure("daemon_request_ms", () => + requestDaemonJson({ + path: "/v1/projects", + query: { + filter: opts.filter ?? null, + include_global: opts.includeGlobal, + include_unregistered: opts.includeUnregistered, + include_meta: opts.meta, + summary: opts.summary ?? false, + profile: opts.timings ?? false, + }, + }) + ); + if ( + daemon?.ok && + daemon.json && + (!opts.summary || daemon.json.detail_level === "summary") + ) { + const { profiling, ...payload } = daemon.json; + writeProfiledProjects({ + payload, + profiler, + started, + enabled: opts.timings === true, + source: "daemon", + daemonProfiling: profiling, + }); + return true; + } + return false; +} + type RuntimeRecoveryMeta = { readonly resetCount: number; readonly lastResetSummary: string | null; @@ -491,6 +628,10 @@ type RuntimeRecoveryNotice = { }; function outputProjectsJson(opts: { + readonly summary: boolean; + readonly profiler: OperationTimings; + readonly started: number; + readonly timings: boolean; readonly filter: string | null; readonly includeGlobal: boolean; readonly includeUnregistered: boolean; @@ -501,6 +642,7 @@ function outputProjectsJson(opts: { }): void { const runtimeMeta = formatRuntimeMeta({ runtime: opts.runtime }); const payload = { + ...(opts.summary ? { detail_level: "summary" } : {}), generated_at: new Date().toISOString(), filter: opts.filter, include_global: opts.includeGlobal, @@ -518,14 +660,43 @@ function outputProjectsJson(opts: { runtime_repair_action: runtimeMeta.lastRepairAction, runtime_repair_outcome: runtimeMeta.lastRepairOutcome, runtime_next_step: runtimeMeta.nextStep, - projects: opts.views.map((view) => ({ - ...serializeProjectView(view), - ...(opts.includeMeta - ? { meta: opts.metaByName.get(view.name) ?? null } - : {}), - })), + projects: opts.profiler.measureSync("projection_ms", () => + opts.views.map((view) => ({ + ...(opts.summary + ? serializeProjectSummary(view) + : serializeProjectView(view)), + ...(opts.includeMeta + ? { meta: opts.metaByName.get(view.name) ?? null } + : {}), + })) + ), }; - process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`); + writeProfiledProjects({ + payload, + profiler: opts.profiler, + started: opts.started, + enabled: opts.timings, + source: "direct", + }); +} + +function writeProfiledProjects(opts: { + readonly payload: Record; + readonly profiler: OperationTimings; + readonly started: number; + readonly enabled: boolean; + readonly source: "daemon" | "direct"; + readonly daemonProfiling?: unknown; +}): void { + const text = opts.profiler.measureSync("json_serialization_ms", () => + JSON.stringify(opts.payload, null, 2) + ); + process.stdout.write(`${text}\n`); + if (opts.enabled) { + process.stderr.write( + `${JSON.stringify({ source: opts.source, elapsed_ms: performance.now() - opts.started, phases_ms: opts.profiler.timings, response_bytes: Buffer.byteLength(text) + 1, daemon: opts.daemonProfiling ?? null })}\n` + ); + } } async function renderRuntimeNotices(opts: { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index 417d5c2e..9e896325 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -12,8 +12,18 @@ import { readControlPlaneConfig } from "../control-plane/sdk/config.ts"; import { resolveDaemonPaths } from "../daemon/paths.ts"; import { readDaemonPid } from "../daemon/process.ts"; import { resolveGlobalHackDir } from "../lib/config-paths.ts"; +import { + type HostCommandRecord, + type ObservedProcess, + observeHostCommand, + readHostCommandRecords, + readObservedProcesses, +} from "../lib/host-command-observation.ts"; import { sanitizeProjectSlug } from "../lib/project.ts"; -import type { RuntimeProject } from "../lib/runtime-projects.ts"; +import type { + RuntimeContainer, + RuntimeProject, +} from "../lib/runtime-projects.ts"; import { readRuntimeProjects } from "../lib/runtime-projects.ts"; import { exec } from "../lib/shell.ts"; import { display } from "../ui/display.ts"; @@ -56,8 +66,16 @@ const optNoHost = defineOption({ description: "Skip host process metrics", } as const); +const optDetails = defineOption({ + name: "details", + type: "boolean", + long: "--details", + description: "Show per-container usage and mount types", +} as const); + const options = [ optProject, + optDetails, optIncludeGlobal, optWatch, optInterval, @@ -102,6 +120,7 @@ const handleUsage: CommandHandlerFor = async ({ includeHost, intervalMs: watchIntervalMs, historySize: usageConfig.historySize, + details: args.options.details === true, }); return 0; } @@ -150,7 +169,15 @@ type UsageProjectRow = { readonly containers: number; }; +type ContainerUsageRow = DockerStatsSample & { + readonly project: string; + readonly service: string; + readonly name: string; + readonly mounts: RuntimeContainer["mounts"]; +}; + type UsageReport = { + readonly containerDetails?: readonly ContainerUsageRow[]; readonly projects: readonly UsageProjectRow[]; readonly total: UsageProjectRow | null; }; @@ -169,6 +196,7 @@ type HostUsageReport = { }; type ContainerIndex = { + readonly containerById: ReadonlyMap; readonly containerIds: readonly string[]; readonly projectByContainer: ReadonlyMap; }; @@ -186,6 +214,7 @@ async function runUsageWatch(opts: { readonly includeHost: boolean; readonly intervalMs: number; readonly historySize: number; + readonly details: boolean; }): Promise { let running = true; const cpuHistory: Array = []; @@ -210,6 +239,7 @@ async function runUsageWatch(opts: { const output = renderUsageSnapshot({ snapshot, + details: opts.details, intervalMs: opts.intervalMs, cpuHistory, memHistory, @@ -239,11 +269,15 @@ async function resolveUsageSnapshot(opts: { }); const runtime = runtimeResult.ok ? runtimeResult.runtime : []; const filtered = opts.filter - ? runtime.filter((project) => project.project === opts.filter) + ? runtime.filter( + (project) => + project.project === opts.filter || + project.project.startsWith(`${opts.filter}--`) + ) : runtime; const index = buildContainerIndex({ projects: filtered }); const host = opts.includeHost - ? await readHostUsage() + ? await readHostUsage({ filter: opts.filter }) : { rows: [], total: null }; const errors: string[] = []; if (!runtimeResult.ok) { @@ -272,23 +306,53 @@ async function resolveUsageSnapshot(opts: { }; } +function formatContainerMounts(mounts: RuntimeContainer["mounts"]): string { + return ( + mounts.map((mount) => `${mount.type}:${mount.destination}`).join(", ") || + "none" + ); +} + +function appendContainerDetails( + lines: string[], + snapshot: UsageSnapshot +): void { + if (snapshot.report.containerDetails?.length) { + lines.push( + "", + "Containers", + renderTable({ + columns: ["Container", "CPU", "Memory", "PIDs", "Mounts"], + rows: snapshot.report.containerDetails.map((row) => [ + row.name, + formatPercent({ percent: row.cpuPercent }), + formatBytesMaybe({ bytes: row.memUsedBytes }), + String(row.pids ?? "n/a"), + formatContainerMounts(row.mounts), + ]), + }) + ); + } +} + +function appendUsageErrors(lines: string[], errors: readonly string[]): void { + if (errors.length > 0) { + lines.push("", "Errors:", ...errors.map((error) => `- ${error}`)); + } +} + function renderUsageSnapshot(opts: { readonly snapshot: UsageSnapshot; readonly intervalMs: number; readonly cpuHistory: readonly (number | null)[]; readonly memHistory: readonly (number | null)[]; + readonly details: boolean; }): string { const lines: string[] = []; lines.push( `hack usage --watch (interval ${opts.intervalMs}ms) ${opts.snapshot.timestamp.toISOString()}` ); - if (opts.snapshot.errors.length > 0) { - lines.push(""); - lines.push("Errors:"); - opts.snapshot.errors.forEach((error) => { - lines.push(`- ${error}`); - }); - } + appendUsageErrors(lines, opts.snapshot.errors); if (opts.snapshot.report.projects.length > 0) { lines.push(""); @@ -314,6 +378,9 @@ function renderUsageSnapshot(opts: { lines.push("Projects: none"); } + if (opts.details) { + appendContainerDetails(lines, opts.snapshot); + } if (opts.snapshot.host.rows.length > 0) { lines.push(""); lines.push("Host processes"); @@ -450,14 +517,18 @@ function resolveIntervalMs(opts: { return Math.max(250, Math.floor(raw)); } -async function readHostUsage(): Promise { - const trackedPids = await resolveTrackedPids(); +async function readHostUsage(opts: { + readonly filter: string | null; +}): Promise { + const trackedPids = await resolveTrackedPids(opts); const samples = await readHostProcessSamples({ trackedPids }); return buildHostUsageReport({ samples }); } -async function resolveTrackedPids(): Promise> { - const tracked = new Map(); +async function resolveTrackedPids(opts: { + readonly filter: string | null; +}): Promise> { + const tracked = new Map(); const daemonPaths = resolveDaemonPaths({}); const daemonPid = await readDaemonPid({ pidPath: daemonPaths.pidPath }); if (daemonPid) { @@ -473,11 +544,73 @@ async function resolveTrackedPids(): Promise> { if (cloudflaredPid) { tracked.set(cloudflaredPid, "cloudflared"); } + const [records, snapshot] = await Promise.all([ + readHostCommandRecords(), + readObservedProcesses(), + ]); + for (const [pid, name] of collectTrackedHostPids({ + records, + snapshot, + filter: opts.filter, + })) { + tracked.set(pid, name); + } return tracked; } +function collectTrackedHostPids(opts: { + readonly records: readonly HostCommandRecord[]; + readonly snapshot: readonly ObservedProcess[] | null; + readonly filter: string | null; +}): Map { + const tracked = new Map(); + const parents = new Map( + (opts.snapshot ?? []).map((row) => [row.pid, row.ppid]) + ); + const records = [...opts.records].sort( + (left, right) => + processDepth(right.child.pid, parents) - + processDepth(left.child.pid, parents) + ); + for (const record of records) { + if (record.status !== "running") { + continue; + } + const included = + opts.filter === null || + record.project === opts.filter || + record.project.startsWith(`${opts.filter}--`); + const observed = observeHostCommand(record, opts.snapshot); + for (const pid of observed.observedPids) { + if (tracked.has(pid)) { + continue; + } + // Inner command roots take precedence over their enclosing command trees. + // A null entry also excludes known foreign commands from generic host heuristics. + tracked.set( + pid, + included ? `host:${record.project}:${record.executable}` : null + ); + } + } + return tracked; +} + +function processDepth( + pid: number, + parents: ReadonlyMap +): number { + const seen = new Set(); + let current = pid; + while (current > 0 && !seen.has(current)) { + seen.add(current); + current = parents.get(current) ?? 0; + } + return seen.size; +} + async function readHostProcessSamples(opts: { - readonly trackedPids: Map; + readonly trackedPids: Map; }): Promise { const res = await exec(["ps", "-axo", "pid=,pcpu=,rss=,command="], { stdin: "ignore", @@ -505,8 +638,9 @@ async function readHostProcessSamples(opts: { const rssKb = Number.parseInt(match[3] ?? "", 10); const command = match[4] ?? ""; - const trackedName = - opts.trackedPids.get(pid) ?? resolveHostProcessKind({ command }); + const trackedName = opts.trackedPids.has(pid) + ? opts.trackedPids.get(pid) + : resolveHostProcessKind({ command }); if (!trackedName) { continue; } @@ -631,14 +765,21 @@ function buildContainerIndex(opts: { projects: readonly RuntimeProject[]; }): ContainerIndex { const projectByContainer = new Map(); + const containerById = new Map(); const containerIds: string[] = []; for (const project of opts.projects) { for (const service of project.services.values()) { for (const container of service.containers) { - if (!container.id) { + if ( + !container.id || + container.state !== "running" || + container.labels?.["hack.lifecycle.process"] === "true" + ) { continue; } containerIds.push(container.id); + containerById.set(container.id, container); + containerById.set(container.id.slice(0, 12), container); projectByContainer.set(container.id, project.project); if (container.id.length >= 12) { projectByContainer.set(container.id.slice(0, 12), project.project); @@ -647,6 +788,7 @@ function buildContainerIndex(opts: { } } return { + containerById, containerIds: [...new Set(containerIds)], projectByContainer, }; @@ -800,11 +942,15 @@ async function runUsageOnce(opts: { }); const runtime = runtimeResult.ok ? runtimeResult.runtime : []; const filtered = opts.filter - ? runtime.filter((project) => project.project === opts.filter) + ? runtime.filter( + (project) => + project.project === opts.filter || + project.project.startsWith(`${opts.filter}--`) + ) : runtime; const index = buildContainerIndex({ projects: filtered }); const hostReport = opts.includeHost - ? await readHostUsage() + ? await readHostUsage({ filter: opts.filter }) : { rows: [], total: null }; const stats = index.containerIds.length === 0 @@ -949,6 +1095,9 @@ async function renderUsageSuccess(opts: { if (opts.args.options.json === true) { writeJson({ payload: { + ...(opts.args.options.details + ? { containers: opts.report.containerDetails ?? [] } + : {}), projects: opts.report.projects, total: opts.report.total, host: opts.hostReport.rows, @@ -960,6 +1109,18 @@ async function renderUsageSuccess(opts: { return 0; } + if (opts.args.options.details) { + await display.table({ + columns: ["Container", "CPU", "Memory", "PIDs", "Mounts"], + rows: (opts.report.containerDetails ?? []).map((row) => [ + row.name, + formatPercent({ percent: row.cpuPercent }), + formatBytesMaybe({ bytes: row.memUsedBytes }), + row.pids ?? "n/a", + formatContainerMounts(row.mounts), + ]), + }); + } await renderProjectUsageTable({ report: opts.report }); await renderHostUsageTable({ hostReport: opts.hostReport }); await renderTotalUsagePanels({ @@ -1093,9 +1254,26 @@ function buildUsageReport(opts: { } : null; + const containerDetails = opts.samples.flatMap((sample) => { + const container = sample.containerId + ? opts.index.containerById.get(sample.containerId) + : null; + return container + ? [ + { + ...sample, + project: container.project, + service: container.service, + name: container.name, + mounts: container.mounts, + }, + ] + : []; + }); return { projects, total, + containerDetails, }; } @@ -1278,3 +1456,10 @@ function getString(value: Record, key: string): string | null { const raw = value[key]; return typeof raw === "string" ? raw.trim() : null; } + +export const __testOnlyUsage = { + buildContainerIndex, + buildUsageReport, + collectTrackedHostPids, + buildHostUsageReport, +}; diff --git a/src/daemon/process.ts b/src/daemon/process.ts index 847e1745..fd677594 100644 --- a/src/daemon/process.ts +++ b/src/daemon/process.ts @@ -142,8 +142,12 @@ async function findDaemonSocketOwners(opts: { } const socketPaths = new Set( [...roots].flatMap((root) => - ["hackd.sock", "hackd.internal.sock", "gateway.internal.sock"].map( - (name) => resolve(root, name) + ["hackd.sock", "hackd.internal.sock", "gateway.internal.sock"].flatMap( + (name) => { + const path = resolve(root, name); + // Linux lsof includes the Unix socket type in its machine-readable name. + return [path, `${path} type=STREAM`]; + } ) ) ); diff --git a/src/daemon/runtime-cache.ts b/src/daemon/runtime-cache.ts index 3fd5c0af..28dbe83b 100644 --- a/src/daemon/runtime-cache.ts +++ b/src/daemon/runtime-cache.ts @@ -1,8 +1,10 @@ import { resolve } from "node:path"; import { PROJECT_COMPOSE_FILENAME } from "../constants.ts"; +import { createOperationTimings } from "../lib/operation-timings.ts"; import { resolveProjectMeta } from "../lib/project-meta.ts"; import { buildProjectViews, + serializeProjectSummary, serializeProjectView, } from "../lib/project-views.ts"; import { readProjectsRegistry } from "../lib/projects-registry.ts"; @@ -47,6 +49,8 @@ export type RuntimeSnapshot = { }; export type ProjectsPayload = { + readonly detail_level?: "summary"; + readonly profiling?: Record; readonly generated_at: string; readonly filter: string | null; readonly include_global: boolean; @@ -109,6 +113,8 @@ export interface RuntimeCache { readonly includeGlobal: boolean; readonly includeUnregistered: boolean; readonly includeMeta: boolean; + readonly summary?: boolean; + readonly profile?: boolean; }): Promise; getPsPayload(opts: { readonly composeProject: string; @@ -120,6 +126,7 @@ export interface RuntimeCache { } export type RuntimeCacheDiagnostics = { + readonly lastRefreshPhasesMs: Readonly>; readonly refreshInFlight: boolean; readonly lastRefreshDurationMs: number | null; readonly maxRefreshDurationMs: number | null; @@ -153,6 +160,7 @@ export function createRuntimeCache(opts: { resolveProjectMeta: opts.deps?.resolveProjectMeta ?? resolveProjectMeta, } as const; + let lastRefreshPhasesMs: Readonly> = {}; let snapshot: RuntimeSnapshot | null = null; let refreshTask: Promise | null = null; let pendingRefresh: QueuedRefresh | null = null; @@ -223,6 +231,7 @@ export function createRuntimeCache(opts: { forceInspect, }: QueuedRefresh): Promise { const startedAtMs = Date.now(); + const profiler = createOperationTimings(); try { const checkedAtMs = Date.now(); const previousSnapshot = snapshot; @@ -230,14 +239,17 @@ export function createRuntimeCache(opts: { includeGlobal: true, inspectCache, forceInspect, + profiler, }); - const refreshed = await resolveRefreshResult({ - checkedAtMs, - currentHealth: health, - previousSnapshot, - reason, - runtimeResult, - }); + const refreshed = await profiler.measure("runtime_identity_ms", () => + resolveRefreshResult({ + checkedAtMs, + currentHealth: health, + previousSnapshot, + reason, + runtimeResult, + }) + ); if (refreshed.repairReason) { queueRefresh({ forceInspect: true, @@ -248,9 +260,11 @@ export function createRuntimeCache(opts: { let nextSnapshot: RuntimeSnapshot; if (runtimeResult.ok) { - await autoRegisterRuntimeHackProjects({ - runtime: runtimeResult.runtime, - }); + await profiler.measure("auto_register_ms", () => + autoRegisterRuntimeHackProjects({ + runtime: runtimeResult.runtime, + }) + ); nextSnapshot = { runtime: runtimeResult.runtime, updatedAtMs: checkedAtMs, @@ -269,6 +283,7 @@ export function createRuntimeCache(opts: { opts.onRefresh?.(nextSnapshot); } finally { const durationMs = Math.max(0, Date.now() - startedAtMs); + lastRefreshPhasesMs = { ...profiler.timings }; lastRefreshDurationMs = durationMs; maxRefreshDurationMs = Math.max(maxRefreshDurationMs ?? 0, durationMs); } @@ -279,27 +294,37 @@ export function createRuntimeCache(opts: { includeGlobal, includeUnregistered, includeMeta, + summary = false, + profile = false, }: { readonly filter: string | null; readonly includeGlobal: boolean; readonly includeUnregistered: boolean; readonly includeMeta: boolean; + readonly summary?: boolean; + readonly profile?: boolean; }): Promise => { + const profiler = createOperationTimings(); if (!snapshot) { await refresh({ reason: "projects" }); } - const registry = await deps.readProjectsRegistry(); + const registry = await profiler.measure( + "registry_ms", + deps.readProjectsRegistry + ); const runtime = filterRuntimeProjects({ runtime: snapshot?.runtime ?? [], includeGlobal, }); - const views = await deps.buildProjectViews({ - registryProjects: registry.projects, - runtime, - runtimeOk: health.ok, - filter, - includeUnregistered, - }); + const views = await profiler.measure("project_views_ms", () => + deps.buildProjectViews({ + registryProjects: registry.projects, + runtime, + runtimeOk: health.ok, + filter, + includeUnregistered, + }) + ); const runtimeMeta = serializeRuntimeHealth({ health }); @@ -307,29 +332,55 @@ export function createRuntimeCache(opts: { registry.projects.map((p) => [p.name, p] as const) ); const metas = includeMeta - ? await Promise.all( - views.map(async (view) => { - if (view.kind !== "registered") { - return null; - } - const reg = registryByName.get(view.name) ?? null; - if (!reg) { - return null; - } - try { - return await deps.resolveProjectMeta({ - projectName: reg.name, - repoRoot: reg.repoRoot, - projectDir: reg.projectDir, - composeFile: resolve(reg.projectDir, PROJECT_COMPOSE_FILENAME), - }); - } catch { - return null; - } - }) + ? await profiler.measure("metadata_ms", () => + Promise.all( + views.map(async (view) => { + if (view.kind !== "registered") { + return null; + } + const reg = registryByName.get(view.name) ?? null; + if (!reg) { + return null; + } + try { + return await deps.resolveProjectMeta({ + projectName: reg.name, + repoRoot: reg.repoRoot, + projectDir: reg.projectDir, + composeFile: resolve( + reg.projectDir, + PROJECT_COMPOSE_FILENAME + ), + }); + } catch { + return null; + } + }) + ) ) : []; + const projects = profiler.measureSync("projection_ms", () => + views.map((view, i) => ({ + ...(summary + ? serializeProjectSummary(view) + : deps.serializeProjectView(view)), + ...(includeMeta ? { meta: metas[i] ?? null } : {}), + })) + ); return { + ...(summary ? { detail_level: "summary" as const } : {}), + ...(profile + ? { + profiling: { + source: "daemon", + cache_age_ms: snapshot?.updatedAtMs + ? Date.now() - snapshot.updatedAtMs + : null, + request_phases_ms: { ...profiler.timings }, + last_refresh_phases_ms: lastRefreshPhasesMs, + }, + } + : {}), generated_at: new Date().toISOString(), filter, include_global: includeGlobal, @@ -347,10 +398,7 @@ export function createRuntimeCache(opts: { runtime_repair_action: runtimeMeta.lastRepairAction, runtime_repair_outcome: runtimeMeta.lastRepairOutcome, runtime_next_step: runtimeMeta.nextStep, - projects: views.map((view, i) => ({ - ...deps.serializeProjectView(view), - ...(includeMeta ? { meta: metas[i] ?? null } : {}), - })), + projects, }; }; @@ -427,6 +475,7 @@ export function createRuntimeCache(opts: { cache: inspectCache, }); return { + lastRefreshPhasesMs, refreshInFlight: refreshTask !== null, lastRefreshDurationMs, maxRefreshDurationMs, diff --git a/src/daemon/server.ts b/src/daemon/server.ts index ee686f21..4816b03c 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -62,6 +62,8 @@ type DaemonMetrics = { refreshRequests: number; refreshRequestsCoalesced: number; refreshFailures: number; + lastProjectsSerializationMs: number | null; + lastProjectsResponseBytes: number | null; }; export async function runDaemon({ @@ -101,6 +103,8 @@ export async function runDaemon({ refreshRequests: 0, refreshRequestsCoalesced: 0, refreshFailures: 0, + lastProjectsSerializationMs: null, + lastProjectsResponseBytes: null, }; const cache = createRuntimeCache({ @@ -419,6 +423,9 @@ async function handleRequest({ refresh_requests_coalesced: metrics.refreshRequestsCoalesced, refresh_failures: metrics.refreshFailures, refresh_in_flight: diagnostics.refreshInFlight, + last_refresh_phases_ms: diagnostics.lastRefreshPhasesMs, + last_projects_serialization_ms: metrics.lastProjectsSerializationMs, + last_projects_response_bytes: metrics.lastProjectsResponseBytes, last_refresh_duration_ms: diagnostics.lastRefreshDurationMs, max_refresh_duration_ms: diagnostics.maxRefreshDurationMs, last_event_at: metrics.lastEventAtMs @@ -461,13 +468,29 @@ async function handleRequest({ const includeMeta = parseBoolean({ value: url.searchParams.get("include_meta"), }); + const summary = parseBoolean({ value: url.searchParams.get("summary") }); + if (summary && includeMeta) { + return jsonResponse({ error: "summary_incompatible_with_meta" }, 400); + } const payload = await cache.getProjectsPayload({ filter, includeGlobal, includeUnregistered, includeMeta, + summary, + profile: parseBoolean({ value: url.searchParams.get("profile") }), }); - return jsonResponse(payload); + const started = performance.now(); + const body = JSON.stringify(payload); + metrics.lastProjectsSerializationMs = performance.now() - started; + metrics.lastProjectsResponseBytes = Buffer.byteLength(body); + const response = new Response(body, { + headers: { + "Content-Type": "application/json", + "Cache-Control": "no-store", + }, + }); + return response; } if (url.pathname === "/v1/ps") { diff --git a/src/lib/host-command-observation.ts b/src/lib/host-command-observation.ts new file mode 100644 index 00000000..a3117b54 --- /dev/null +++ b/src/lib/host-command-observation.ts @@ -0,0 +1,328 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readdir, rename, unlink } from "node:fs/promises"; +import { basename, resolve } from "node:path"; +import { z } from "zod"; +import { resolveGlobalHackDir } from "./config-paths.ts"; +import { + collectDescendantProcessIds, + type ProcessSnapshotRow, +} from "./project-lifecycle-processes.ts"; +import { exec, type RunOptions, run } from "./shell.ts"; + +export type HostLifetime = "command" | "persistent" | "bounded"; +const identitySchema = z.object({ + pid: z.number().int().positive(), + birth: z.string().nullable(), +}); +const recordSchema = z.object({ + version: z.literal(1), + id: z.string().uuid(), + project: z.string(), + projectRoot: z.string(), + executable: z.string(), + wrapper: identitySchema, + child: identitySchema, + ownsProcessGroup: z.boolean(), + processGroupId: z.number().int().positive().nullable(), + lifetime: z.enum(["command", "persistent", "bounded"]), + timeoutMs: z.number().positive().nullable(), + startedAt: z.string().datetime(), + finishedAt: z.string().datetime().nullable(), + status: z.enum(["running", "exited", "cancelled", "timed_out"]), + exitCode: z.number().int().nullable(), + cpuTimeMs: z.number().nonnegative().nullable(), + maxRssBytes: z.number().nonnegative().nullable(), +}); +export type HostCommandRecord = z.infer; +type ProcessIdentity = z.infer; + +export type ObservedProcess = ProcessSnapshotRow & { + readonly birth: string; + readonly elapsedMs: number; + readonly cpuTimeMs: number; + readonly rssBytes: number; +}; +const PROCESS_DURATION = /^(?:\d+-)?\d+(?::\d+){1,2}(?:\.\d+)?$/; +const PROCESS_ROW = /^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(.+)$/; +const RECORD_NAME = /^[\da-f-]{36}\.json$/; +const RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +/** ps uses [[days-]hours:]minutes:seconds, with optional fractional seconds. */ +export function parseProcessDuration(value: string): number | null { + if (!PROCESS_DURATION.test(value)) { + return null; + } + const [days, clock] = value.includes("-") ? value.split("-") : ["0", value]; + const parts = (clock ?? "").split(":").map(Number); + const seconds = + parts.reduce((total, part) => total * 60 + part, 0) + Number(days) * 86_400; + return Number.isFinite(seconds) ? seconds * 1000 : null; +} + +export function parseObservedProcesses(output: string): ObservedProcess[] { + return output.split("\n").flatMap((line) => { + const match = PROCESS_ROW.exec(line.trim()); + if (!match) { + return []; + } + const elapsedMs = parseProcessDuration(match[4] ?? ""); + const cpuTimeMs = parseProcessDuration(match[5] ?? ""); + if (elapsedMs === null || cpuTimeMs === null) { + return []; + } + return [ + { + pid: Number(match[1]), + ppid: Number(match[2]), + processGroupId: Number(match[3]), + elapsedMs, + cpuTimeMs, + rssBytes: Number(match[6]) * 1024, + birth: (match[7] ?? "").replace(/\s+/g, " "), + }, + ]; + }); +} + +/** Payload-free snapshot; no argv, environment, open files or output is collected. */ +export async function readObservedProcesses( + pids?: readonly number[] +): Promise { + try { + const result = await exec( + [ + "ps", + ...(pids ? ["-p", pids.join(",")] : ["-A"]), + "-o", + "pid=,ppid=,pgid=,etime=,time=,rss=,lstart=", + ], + { stdin: "ignore", timeoutMs: 2000, env: { LC_ALL: "C" } } + ); + return result.exitCode === 0 ? parseObservedProcesses(result.stdout) : null; + } catch { + return null; + } +} + +function recordRoot(): string { + return resolve(resolveGlobalHackDir(), "host-commands"); +} + +async function saveRecord(record: HostCommandRecord): Promise { + const root = recordRoot(); + await mkdir(root, { recursive: true, mode: 0o700 }); + const path = resolve(root, `${record.id}.json`); + const temporary = `${path}.${randomUUID()}.tmp`; + try { + await Bun.write(temporary, JSON.stringify(record), { mode: 0o600 }); + await rename(temporary, path); + } finally { + await unlink(temporary).catch(() => undefined); + } +} + +/** Diagnostic records contain no arguments, shell strings, output, env names or values. */ +export async function runObservedHostCommand(opts: { + readonly command: readonly string[]; + readonly project: string; + readonly projectRoot: string; + readonly lifetime: HostLifetime; + readonly runOptions: RunOptions; +}): Promise { + let record: HostCommandRecord | null = null; + let warned = false; + const reportFailure = (): void => { + if (!warned) { + process.stderr.write( + "hack: host-command diagnostics unavailable; execution continues.\n" + ); + } + warned = true; + }; + return await run(opts.command, { + ...opts.runOptions, + onSpawn: async ({ pid, ownsProcessGroup, processGroupId }) => { + try { + const startedAt = new Date().toISOString(); + const snapshot = await readObservedProcesses([process.pid, pid]); + record = { + version: 1, + id: randomUUID(), + project: opts.project, + projectRoot: opts.projectRoot, + executable: basename(opts.command[0] ?? "unknown"), + wrapper: { + pid: process.pid, + birth: + snapshot?.find((row) => row.pid === process.pid)?.birth ?? null, + }, + child: { + pid, + birth: snapshot?.find((row) => row.pid === pid)?.birth ?? null, + }, + ownsProcessGroup, + processGroupId: ownsProcessGroup ? (processGroupId ?? pid) : null, + lifetime: opts.lifetime, + timeoutMs: opts.runOptions.timeoutMs ?? null, + startedAt, + finishedAt: null, + status: "running", + exitCode: null, + cpuTimeMs: null, + maxRssBytes: null, + }; + await saveRecord(record); + } catch { + reportFailure(); + } + }, + onExit: async (event) => { + if (!record) { + return; + } + try { + const completedStatus = event.cancelled ? "cancelled" : "exited"; + await saveRecord({ + ...record, + finishedAt: event.finishedAt, + status: event.timedOut ? "timed_out" : completedStatus, + exitCode: event.exitCode, + cpuTimeMs: event.cpuTimeMs, + maxRssBytes: event.maxRssBytes, + }); + await expireHostCommandRecords(); + } catch { + reportFailure(); + } + }, + }); +} + +/** Schema parsing strips unknown fields, including any accidental payload data. */ +export function parseHostCommandRecord( + value: unknown +): HostCommandRecord | null { + const parsed = recordSchema.safeParse(value); + return parsed.success ? parsed.data : null; +} + +export async function readHostCommandRecords(): Promise { + const root = recordRoot(); + const files = await readdir(root).catch(() => []); + const records: HostCommandRecord[] = []; + for (const name of files.filter((file) => RECORD_NAME.test(file))) { + try { + const record = parseHostCommandRecord( + await Bun.file(resolve(root, name)).json() + ); + if (record) { + records.push(record); + } + } catch { + /* A partial/unreadable record is not evidence of a running process. */ + } + } + return records.sort((left, right) => + right.startedAt.localeCompare(left.startedAt) + ); +} + +/** Only expires our completed diagnostics. Active and interrupted records are preserved. */ +export async function expireHostCommandRecords(): Promise { + const cutoff = Date.now() - RETENTION_MS; + for (const record of await readHostCommandRecords()) { + if (record.finishedAt && Date.parse(record.finishedAt) < cutoff) { + await unlink(resolve(recordRoot(), `${record.id}.json`)).catch( + () => undefined + ); + } + } +} + +export function observeHostCommand( + record: HostCommandRecord, + snapshot: readonly ObservedProcess[] | null +) { + const matches = (identity: ProcessIdentity): ObservedProcess | null => + identity.birth + ? (snapshot?.find( + (row) => row.pid === identity.pid && row.birth === identity.birth + ) ?? null) + : null; + const wrapper = matches(record.wrapper); + const child = matches(record.child); + const status = observedCommandStatus({ record, snapshot, wrapper, child }); + const treeIds = new Set( + child + ? collectDescendantProcessIds({ + snapshot: snapshot ?? [], + rootPids: [child.pid], + }) + : [] + ); + const members = child + ? (snapshot ?? []).filter( + (row) => + treeIds.has(row.pid) || + (record.ownsProcessGroup && + row.processGroupId === child.processGroupId) + ) + : []; + const unverifiedGroupPids = + !child && record.ownsProcessGroup + ? (snapshot ?? []) + .filter((row) => row.processGroupId === record.processGroupId) + .map((row) => row.pid) + : []; + return { + ...record, + status, + elapsedMs: Math.max( + 0, + (record.finishedAt ? Date.parse(record.finishedAt) : Date.now()) - + Date.parse(record.startedAt) + ), + observedPids: members.map((row) => row.pid), + unverifiedGroupPids, + liveCpuTimeMs: child + ? members.reduce((total, row) => total + row.cpuTimeMs, 0) + : null, + liveRssBytes: child + ? members.reduce((total, row) => total + row.rssBytes, 0) + : null, + cpuAccounting: record.finishedAt ? "reaped_child" : "live_tree_snapshot", + ownership: child ? "pid_and_start_time" : "unverified", + attention: unverifiedGroupPids.length + ? "group_members_require_review" + : commandAttention(status, record.lifetime), + }; +} + +function observedCommandStatus(opts: { + readonly record: HostCommandRecord; + readonly snapshot: readonly ObservedProcess[] | null; + readonly wrapper: ObservedProcess | null; + readonly child: ObservedProcess | null; +}): string { + if (opts.record.status !== "running") { + return opts.record.status; + } + if (!(opts.snapshot && opts.record.wrapper.birth)) { + return "unknown"; + } + if (opts.wrapper) { + return "running"; + } + return opts.child ? "orphaned" : "interrupted"; +} +function commandAttention( + status: string, + lifetime: HostLifetime +): string | null { + if (status === "orphaned") { + return lifetime === "persistent" + ? "persistent_wrapper_lost" + : "wrapper_lost"; + } + return status === "interrupted" ? "completion_unknown" : null; +} diff --git a/src/lib/operation-timings.ts b/src/lib/operation-timings.ts new file mode 100644 index 00000000..058f13a4 --- /dev/null +++ b/src/lib/operation-timings.ts @@ -0,0 +1,28 @@ +/** Numeric, request-local timings. Never records command arguments or payloads. */ +export function createOperationTimings() { + const timings: Record = {}; + function record(name: string, started: number): void { + timings[name] = (timings[name] ?? 0) + performance.now() - started; + } + return { + timings, + async measure(name: string, operation: () => Promise): Promise { + const started = performance.now(); + try { + return await operation(); + } finally { + record(name, started); + } + }, + measureSync(name: string, operation: () => T): T { + const started = performance.now(); + try { + return operation(); + } finally { + record(name, started); + } + }, + }; +} + +export type OperationTimings = ReturnType; diff --git a/src/lib/process-resource-usage.ts b/src/lib/process-resource-usage.ts new file mode 100644 index 00000000..6bbb0044 --- /dev/null +++ b/src/lib/process-resource-usage.ts @@ -0,0 +1,25 @@ +/** Bun versions expose CPU counters as either numbers or bigints despite older type declarations. */ +export function readSubprocessResourceUsage( + proc: Pick +): { cpuTimeMs: number | null; maxRssBytes: number | null } { + try { + const usage = proc.resourceUsage(); + if (!usage) { + return { cpuTimeMs: null, maxRssBytes: null }; + } + const cpuTimeMs = Number(usage.cpuTime.total) / 1000; + // Bun 1.3 exposes native ru_maxrss (KiB on Linux); Bun 1.4 normalizes to bytes. + const rssScale = + process.platform === "linux" && + Bun.semver.satisfies(Bun.version, "<1.4.0") + ? 1024 + : 1; + const maxRssBytes = Number(usage.maxRSS) * rssScale; + return { + cpuTimeMs: Number.isFinite(cpuTimeMs) ? cpuTimeMs : null, + maxRssBytes: Number.isFinite(maxRssBytes) ? maxRssBytes : null, + }; + } catch { + return { cpuTimeMs: null, maxRssBytes: null }; + } +} diff --git a/src/lib/project-lifecycle-processes.ts b/src/lib/project-lifecycle-processes.ts index 66a9459e..428b3e42 100644 --- a/src/lib/project-lifecycle-processes.ts +++ b/src/lib/project-lifecycle-processes.ts @@ -39,43 +39,44 @@ export function parseProcessSnapshotOutput(text: string): ProcessSnapshotRow[] { }); } -/** Collect distinct process groups reachable from the provided root PIDs. */ -export function collectDescendantProcessGroupIds(opts: { +/** Collect the process tree without interpreting names or taking ownership. */ +export function collectDescendantProcessIds(opts: { readonly snapshot: readonly ProcessSnapshotRow[]; readonly rootPids: readonly number[]; }): number[] { - const processByParent = new Map(); - const groups = new Set(); - const queue = [...opts.rootPids]; - const visited = new Set(); - + const children = new Map(); + const live = new Set(opts.snapshot.map((row) => row.pid)); for (const row of opts.snapshot) { - const siblings = processByParent.get(row.ppid) ?? []; - siblings.push(row); - processByParent.set(row.ppid, siblings); + const siblings = children.get(row.ppid) ?? []; + siblings.push(row.pid); + children.set(row.ppid, siblings); } - - while (queue.length > 0) { - const pid = queue.shift(); - if (!(pid && pid > 0) || visited.has(pid)) { + const queue = [...opts.rootPids]; + const visited = new Set(); + for (let index = 0; index < queue.length; index++) { + const pid = queue[index]; + if (pid === undefined || visited.has(pid) || !live.has(pid)) { continue; } visited.add(pid); - - const current = opts.snapshot.find((row) => row.pid === pid); - if (current) { - groups.add(current.processGroupId); - } - - for (const child of processByParent.get(pid) ?? []) { - groups.add(child.processGroupId); - if (!visited.has(child.pid)) { - queue.push(child.pid); - } - } + queue.push(...(children.get(pid) ?? [])); } + return [...visited]; +} - return [...groups].sort((left, right) => left - right); +/** Collect distinct process groups reachable from the provided root PIDs. */ +export function collectDescendantProcessGroupIds(opts: { + readonly snapshot: readonly ProcessSnapshotRow[]; + readonly rootPids: readonly number[]; +}): number[] { + const pids = new Set(collectDescendantProcessIds(opts)); + return [ + ...new Set( + opts.snapshot + .filter((row) => pids.has(row.pid)) + .map((row) => row.processGroupId) + ), + ].sort((left, right) => left - right); } /** Reconcile mux pane state with persisted lifecycle metadata to recover live groups. */ diff --git a/src/lib/project-runtime-hygiene.ts b/src/lib/project-runtime-hygiene.ts index 537ce321..603ca08c 100644 --- a/src/lib/project-runtime-hygiene.ts +++ b/src/lib/project-runtime-hygiene.ts @@ -79,7 +79,8 @@ export async function findOrphanRuntimeProjects(input: { const out: OrphanedRuntimeProject[] = []; for (const project of input.runtime) { const workingDir = project.workingDir; - if (!workingDir) { + const containerIds = collectContainerIds(project); + if (!workingDir || containerIds.length === 0) { continue; } if (!(await pathExists(workingDir))) { @@ -87,7 +88,7 @@ export async function findOrphanRuntimeProjects(input: { project: project.project, workingDir, reason: "missing working dir", - containerIds: collectContainerIds(project), + containerIds, }); continue; } @@ -97,7 +98,7 @@ export async function findOrphanRuntimeProjects(input: { project: project.project, workingDir, reason: "missing compose file", - containerIds: collectContainerIds(project), + containerIds, }); } } @@ -151,7 +152,10 @@ function collectContainerIds(project: RuntimeProject): readonly string[] { const out: string[] = []; for (const service of project.services.values()) { for (const container of service.containers) { - if (container.id.length > 0) { + if ( + container.id.length > 0 && + container.labels?.["hack.lifecycle.process"] !== "true" + ) { out.push(container.id); } } diff --git a/src/lib/project-views.ts b/src/lib/project-views.ts index 013a38bf..66415e44 100644 --- a/src/lib/project-views.ts +++ b/src/lib/project-views.ts @@ -330,6 +330,41 @@ function buildUnregisteredProjectView(opts: { }; } +/** Summary projection deliberately excludes container labels, mounts and command definitions. */ +export function serializeProjectSummary( + view: ProjectView +): Record { + const runtimes = [ + view.runtime, + ...view.branchRuntime.map((branch) => branch.runtime), + ].filter((runtime): runtime is RuntimeProject => runtime !== null); + const containers = runtimes.flatMap((runtime) => + [...runtime.services.values()].flatMap((service) => service.containers) + ); + return { + project_id: view.projectId ?? null, + name: view.name, + repo_root: view.repoRoot, + dev_host: view.devHost, + status: view.status, + runtime_status: view.runtimeStatus, + defined_service_count: view.definedServices?.length ?? null, + host_process_count: containers.filter( + (container) => container.labels?.["hack.lifecycle.process"] === "true" + ).length, + container_count: containers.filter( + (container) => container.labels?.["hack.lifecycle.process"] !== "true" + ).length, + running_container_count: containers.filter( + (container) => + container.state === "running" && + container.labels?.["hack.lifecycle.process"] !== "true" + ).length, + branch_count: view.branchRuntime.length, + session_count: view.sessions.length, + }; +} + export function serializeProjectView( view: ProjectView ): Record { diff --git a/src/lib/runtime-projects.ts b/src/lib/runtime-projects.ts index e0f1eeeb..03735b2b 100644 --- a/src/lib/runtime-projects.ts +++ b/src/lib/runtime-projects.ts @@ -9,6 +9,10 @@ import { pathExists } from "./fs.ts"; import { getString, isRecord } from "./guards.ts"; import { parseJsonLines } from "./json-lines.ts"; import { readLifecycleState } from "./lifecycle-runtime.ts"; +import { + createOperationTimings, + type OperationTimings, +} from "./operation-timings.ts"; import { upsertProjectRegistration } from "./projects-registry.ts"; import { exec, findExecutableInPath } from "./shell.ts"; @@ -143,7 +147,9 @@ export async function readRuntimeProjects(opts: { readonly includeGlobal: boolean; readonly inspectCache?: RuntimeInspectCache; readonly forceInspect?: boolean; + readonly profiler?: OperationTimings; }): Promise { + const profiler = opts.profiler ?? createOperationTimings(); const checkedAtMs = Date.now(); if (!findExecutableInPath("docker")) { return { @@ -156,17 +162,19 @@ export async function readRuntimeProjects(opts: { let res: Awaited>; try { - res = await exec( - [ - "docker", - "ps", - "-a", - "--filter", - "label=com.docker.compose.project", - "--format", - "json", - ], - { stdin: "ignore" } + res = await profiler.measure("docker_list_ms", () => + exec( + [ + "docker", + "ps", + "-a", + "--filter", + "label=com.docker.compose.project", + "--format", + "json", + ], + { stdin: "ignore" } + ) ); } catch (error: unknown) { return { @@ -193,13 +201,15 @@ export async function readRuntimeProjects(opts: { const ids = baseRows .map((row) => getString(row, "ID") ?? getString(row, "Id") ?? "") .filter((id) => id.length > 0); - const inspectById = opts.inspectCache - ? await readCachedContainerInspectData({ - cache: opts.inspectCache, - forceInspect: opts.forceInspect ?? true, - ids, - }) - : await readContainerInspectData({ ids }); + const inspectById = await profiler.measure("docker_inspect_ms", async () => + opts.inspectCache + ? await readCachedContainerInspectData({ + cache: opts.inspectCache, + forceInspect: opts.forceInspect ?? true, + ids, + }) + : await readContainerInspectData({ ids }) + ); const globalRoot = resolveGlobalHackDir(); @@ -286,9 +296,11 @@ export async function readRuntimeProjects(opts: { }); } - const runtimeWithLifecycle = await addLifecycleProcessServices({ - runtime: out, - }); + const runtimeWithLifecycle = await profiler.measure("lifecycle_ms", () => + addLifecycleProcessServices({ + runtime: out, + }) + ); return { ok: true, @@ -584,6 +596,10 @@ export async function readContainerLabels(opts: { return labelsById; } +// Request only fields used by the runtime model; Docker must not return env values or command arguments. +const RUNTIME_INSPECT_FORMAT = + '{"Id":{{json .Id}},"Config":{"Labels":{{json .Config.Labels}},"Image":{{json .Config.Image}}},"Mounts":{{json .Mounts}},"NetworkSettings":{"Networks":{{json .NetworkSettings.Networks}}}}'; + async function readContainerInspectData(opts: { readonly ids: readonly string[]; }): Promise> { @@ -591,15 +607,21 @@ async function readContainerInspectData(opts: { return new Map(); } - const res = await exec(["docker", "inspect", ...opts.ids], { - stdin: "ignore", - }); + const res = await exec( + ["docker", "inspect", "--format", RUNTIME_INSPECT_FORMAT, ...opts.ids], + { + stdin: "ignore", + } + ); let parsed: unknown; try { parsed = JSON.parse(res.stdout); } catch { - return new Map(); + parsed = parseJsonLines(res.stdout); + } + if (isRecord(parsed)) { + parsed = [parsed]; } if (!Array.isArray(parsed)) { return new Map(); diff --git a/src/lib/shell.ts b/src/lib/shell.ts index 7cdd2847..c246909c 100644 --- a/src/lib/shell.ts +++ b/src/lib/shell.ts @@ -1,3 +1,6 @@ +import { readSubprocessResourceUsage } from "./process-resource-usage.ts"; +import { hasControllingTerminal } from "./tty-process-group.ts"; + export interface ExecResult { readonly exitCode: number; readonly stdout: string; @@ -75,16 +78,30 @@ export interface RunOptions { readonly timeoutMs?: number; /** Forward cancellation to an owned command process group, preserving TTY input. */ readonly forwardSignals?: boolean; + readonly onSpawn?: (event: { + readonly pid: number; + readonly ownsProcessGroup: boolean; + readonly processGroupId?: number; + }) => Promise; + readonly onExit?: (event: RunExitEvent) => Promise; } +export type RunExitEvent = { + readonly finishedAt: string; + readonly exitCode: number; + readonly timedOut: boolean; + readonly cancelled: boolean; + readonly cpuTimeMs: number | null; + readonly maxRssBytes: number | null; +}; + export async function run( cmd: readonly string[], opts: RunOptions = {} ): Promise { if ( opts.forwardSignals && - process.stdin.isTTY && - (opts.stdin ?? "inherit") === "inherit" + (process.stdin.isTTY || hasControllingTerminal()) ) { const { runWithTerminalGroup } = await import("./tty-run.ts"); return await runWithTerminalGroup({ @@ -92,7 +109,10 @@ export async function run( cwd: opts.cwd, env: buildSpawnEnv(opts.env), stdout: opts.stdout, + stdin: opts.stdin, timeoutMs: opts.timeoutMs, + onSpawn: opts.onSpawn, + onExit: opts.onExit, }); } const ownsProcessGroup = @@ -114,13 +134,34 @@ export async function run( pid: proc.pid, }) : null; - try { - const exitCode = await proc.exited; - return cancellation?.exitCode() ?? (timeout.didTimeout() ? 124 : exitCode); - } finally { - timeout.dispose(); - cancellation?.dispose(); - } + // Observe completion immediately: diagnostic setup must not keep deadlines armed + // after the command has exited. Record callbacks still finish in spawn/exit order. + const completion = (async (): Promise => { + try { + const exitCode = await proc.exited; + const code = + cancellation?.exitCode() ?? (timeout.didTimeout() ? 124 : exitCode); + const usage = opts.onExit + ? readSubprocessResourceUsage(proc) + : { cpuTimeMs: null, maxRssBytes: null }; + return { + finishedAt: new Date().toISOString(), + exitCode: code, + timedOut: timeout.didTimeout(), + cancelled: cancellation?.exitCode() != null, + ...usage, + }; + } finally { + timeout.dispose(); + cancellation?.dispose(); + } + })(); + const [result] = await Promise.all([ + completion, + opts.onSpawn?.({ pid: proc.pid, ownsProcessGroup }), + ]); + await opts.onExit?.(result); + return result.exitCode; } /** Detached noninteractive children keep cancellation scoped to their group. */ diff --git a/src/lib/tty-process-group.ts b/src/lib/tty-process-group.ts index c44d45b9..78c6d10b 100644 --- a/src/lib/tty-process-group.ts +++ b/src/lib/tty-process-group.ts @@ -1,4 +1,5 @@ import { dlopen, FFIType, type Pointer } from "bun:ffi"; +import { closeSync, openSync } from "node:fs"; import { constants } from "node:os"; /** POSIX job control without a proxy PTY: all three command streams stay intact. */ @@ -18,6 +19,8 @@ export function openTerminalControl() { const library = process.platform === "darwin" ? "/usr/lib/libSystem.B.dylib" : "libc.so.6"; const libc = dlopen(library, symbols); + const controllingDescriptor = openControllingTerminal(); + const descriptor = controllingDescriptor ?? 0; function withoutBackgroundStop(operation: () => T): T { const previous = libc.symbols.signal( constants.signals.SIGTTOU, @@ -32,20 +35,29 @@ export function openTerminalControl() { return { createGroup: () => libc.symbols.setpgid(0, 0) === 0, group: () => libc.symbols.getpgrp(), - foreground: () => libc.symbols.tcgetpgrp(0), + foreground: () => libc.symbols.tcgetpgrp(descriptor), setForeground: (group: number) => - withoutBackgroundStop(() => libc.symbols.tcsetpgrp(0, group) === 0), + withoutBackgroundStop( + () => libc.symbols.tcsetpgrp(descriptor, group) === 0 + ), attributes: () => { // Opaque termios storage, larger than both Darwin and Linux structures. const value = new Uint8Array(256); - return libc.symbols.tcgetattr(0, value) === 0 ? value : null; + return libc.symbols.tcgetattr(descriptor, value) === 0 ? value : null; }, restoreAttributes: (value: Uint8Array | null) => { if (value) { - withoutBackgroundStop(() => libc.symbols.tcsetattr(0, 0, value)); + withoutBackgroundStop(() => + libc.symbols.tcsetattr(descriptor, 0, value) + ); } }, - close: () => libc.close(), + close: () => { + if (controllingDescriptor !== null) { + closeSync(controllingDescriptor); + } + libc.close(); + }, }; } @@ -56,3 +68,20 @@ export function signalOwnedGroup(pid: number, signal: NodeJS.Signals): void { // The owned group may already have exited. } } + +function openControllingTerminal(): number | null { + try { + return openSync("/dev/tty", "r+"); + } catch { + return null; + } +} + +export function hasControllingTerminal(): boolean { + const descriptor = openControllingTerminal(); + if (descriptor === null) { + return false; + } + closeSync(descriptor); + return true; +} diff --git a/src/lib/tty-run.ts b/src/lib/tty-run.ts index 509ede26..bb231e09 100644 --- a/src/lib/tty-run.ts +++ b/src/lib/tty-run.ts @@ -1,4 +1,5 @@ import { fileURLToPath } from "node:url"; +import type { RunExitEvent, RunOptions } from "./shell.ts"; import { openTerminalControl, signalOwnedGroup } from "./tty-process-group.ts"; import { TTY_SUPERVISOR_ARGUMENT } from "./tty-supervisor.ts"; @@ -8,7 +9,10 @@ export async function runWithTerminalGroup(opts: { readonly cwd?: string; readonly env: Record; readonly stdout?: "inherit" | "stderr"; + readonly stdin?: RunOptions["stdin"]; readonly timeoutMs?: number; + readonly onSpawn?: RunOptions["onSpawn"]; + readonly onExit?: RunOptions["onExit"]; }): Promise { const terminal = openTerminalControl(); const parentGroup = terminal.group(); @@ -20,6 +24,14 @@ export async function runWithTerminalGroup(opts: { let cancellationCode: number | null = null; let escalation: ReturnType | undefined; let timeout: ReturnType | undefined; + let spawnObservation: Promise = Promise.resolve(); + let observationError: unknown; + const measurements: { + accounting: Pick< + RunExitEvent, + "finishedAt" | "cpuTimeMs" | "maxRssBytes" + > | null; + } = { accounting: null }; const entrypoint = fileURLToPath(new URL("../../index.ts", import.meta.url)); const invocation = Bun.main.startsWith("/$bunfs/") ? [process.execPath] @@ -27,7 +39,7 @@ export async function runWithTerminalGroup(opts: { const child = Bun.spawn([...invocation, TTY_SUPERVISOR_ARGUMENT], { cwd: opts.cwd, env: opts.env, - stdin: "inherit", + stdin: opts.stdin ?? "inherit", stdout: opts.stdout === "stderr" ? 2 : "inherit", stderr: "inherit", ipc(message: unknown) { @@ -46,18 +58,42 @@ export async function runWithTerminalGroup(opts: { (message.signal === "SIGINT" || message.signal === "SIGTERM") ) { cancel(message.signal, message.signal === "SIGINT" ? 130 : 143, false); + } else if ( + message.kind === "spawn" && + "pid" in message && + typeof message.pid === "number" + ) { + observeSpawn(message.pid); } else if (message.kind === "done") { - if (cancellationCode !== null) { - signalOwnedGroup(child.pid, "SIGKILL"); - } else { - acknowledged = true; - child.send("ack"); - } + finishCommand(message); } else if (message.kind === "stop") { suspend(); } }, }); + function finishCommand(message: object): void { + clearTimeout(timeout); + measurements.accounting = readAccounting(message); + if (cancellationCode !== null) { + signalOwnedGroup(child.pid, "SIGKILL"); + } else { + acknowledged = true; + child.send("ack"); + } + } + function observeSpawn(pid: number): void { + spawnObservation = Promise.resolve() + .then(() => + opts.onSpawn?.({ + pid, + ownsProcessGroup: true, + processGroupId: child.pid, + }) + ) + .catch((error: unknown) => { + observationError = error; + }); + } function startCommand(): void { ready = true; if (cancellationCode !== null) { @@ -131,9 +167,18 @@ export async function runWithTerminalGroup(opts: { if (opts.timeoutMs !== undefined) { timeout = setTimeout(() => cancel("SIGTERM", 124, true), opts.timeoutMs); } + let result: RunExitEvent; try { const code = await child.exited; - return cancellationCode ?? code; + result = { + finishedAt: + measurements.accounting?.finishedAt ?? new Date().toISOString(), + exitCode: cancellationCode ?? code, + cancelled: cancellationCode === 130 || cancellationCode === 143, + timedOut: cancellationCode === 124, + cpuTimeMs: measurements.accounting?.cpuTimeMs ?? null, + maxRssBytes: measurements.accounting?.maxRssBytes ?? null, + }; } finally { clearTimeout(timeout); clearTimeout(escalation); @@ -150,4 +195,29 @@ export async function runWithTerminalGroup(opts: { restoreTerminal(); terminal.close(); } + await spawnObservation; + if (observationError) { + throw observationError; + } + await opts.onExit?.(result); + return result.exitCode; +} + +function readAccounting( + message: object +): Pick | null { + if (!("finishedAt" in message) || typeof message.finishedAt !== "string") { + return null; + } + return { + finishedAt: message.finishedAt, + cpuTimeMs: + "cpuTimeMs" in message && typeof message.cpuTimeMs === "number" + ? message.cpuTimeMs + : null, + maxRssBytes: + "maxRssBytes" in message && typeof message.maxRssBytes === "number" + ? message.maxRssBytes + : null, + }; } diff --git a/src/lib/tty-supervisor.ts b/src/lib/tty-supervisor.ts index 69441d67..7162a8c3 100644 --- a/src/lib/tty-supervisor.ts +++ b/src/lib/tty-supervisor.ts @@ -1,3 +1,4 @@ +import { readSubprocessResourceUsage } from "./process-resource-usage.ts"; import { openTerminalControl } from "./tty-process-group.ts"; export const TTY_SUPERVISOR_ARGUMENT = "--internal-tty-supervisor"; @@ -55,9 +56,14 @@ export async function runTtySupervisor(): Promise { stderr: "inherit", env: process.env, }); + process.send?.({ kind: "spawn", pid: child.pid }); child.exited.then((code) => { completed = code; - process.send?.({ kind: "done" }); + process.send?.({ + kind: "done", + finishedAt: new Date().toISOString(), + ...readSubprocessResourceUsage(child), + }); }); } catch (error) { process.stderr.write( diff --git a/tests/daemon-orphan.test.ts b/tests/daemon-orphan.test.ts index 797eac13..353102a1 100644 --- a/tests/daemon-orphan.test.ts +++ b/tests/daemon-orphan.test.ts @@ -72,6 +72,35 @@ test("preserves daemons belonging to another state directory", async () => { expect(orphans).toEqual([]); }); +test("accepts Linux lsof stream names without accepting foreign paths or suffix lookalikes", async () => { + expect( + await findOrphanDaemonProcesses({ + trackedPid: null, + daemonRoot: DAEMON_ROOT, + psLines: PS_LINES, + lsofLines: [ + "p123", + `n${DAEMON_ROOT}/hackd.sock type=STREAM`, + "p456", + `n${DAEMON_ROOT}/hackd.sock.backup type=STREAM`, + ], + }) + ).toEqual([123]); + expect( + await findOrphanDaemonProcesses({ + trackedPid: null, + daemonRoot: DAEMON_ROOT, + psLines: PS_LINES, + lsofLines: [ + "p123", + "n/tmp/foreign/hackd.sock type=STREAM", + "p456", + `n${DAEMON_ROOT}/hackd.sock type=STREAM extra`, + ], + }) + ).toEqual([]); +}); + test("does not authorize cleanup without socket ownership evidence", async () => { const orphans = await findOrphanDaemonProcesses({ trackedPid: null, diff --git a/tests/fixtures/tty-cancellation.py b/tests/fixtures/tty-cancellation.py index 6a471cdf..8bb40c62 100644 --- a/tests/fixtures/tty-cancellation.py +++ b/tests/fixtures/tty-cancellation.py @@ -25,14 +25,19 @@ def worker(root, behavior): write_json(root / 'tty.json', { 'stdin': os.isatty(0), 'stdout': os.isatty(1), 'stderr': os.isatty(2), 'devTty': os.isatty(tty.fileno()), - 'foreground': os.tcgetpgrp(0) == os.getpgrp(), + 'foreground': os.tcgetpgrp(tty.fileno()) == os.getpgrp(), }) # Publish the parent before spawning the grandchild: its readiness can never race this file. (root / 'child.pid').write_text(str(os.getpid())) if behavior != 'normal': subprocess.Popen([sys.executable, __file__, 'grandchild', str(root)]) - if behavior in ['normal', 'io']: - (root / 'input.txt').write_text(sys.stdin.readline()) + if behavior in ['normal', 'io', 'pipe']: + if behavior == 'pipe': + (root / 'pipe-input.txt').write_text(sys.stdin.readline()) + with open('/dev/tty') as tty: + (root / 'input.txt').write_text(tty.readline()) + else: + (root / 'input.txt').write_text(sys.stdin.readline()) print('child stdout', flush=True) print('child stderr', file=sys.stderr, flush=True) if behavior == 'normal': @@ -52,11 +57,17 @@ def launcher(root, bun, entrypoint, behavior): if wrapper == 0: for sig in [signal.SIGHUP, signal.SIGINT, signal.SIGTERM, signal.SIGTSTP]: signal.signal(sig, signal.SIG_DFL) - if behavior in ['normal', 'io']: + if behavior in ['normal', 'io', 'pipe']: for number, name in [(1, 'stdout.txt'), (2, 'stderr.txt')]: output = os.open(str(root / name), os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) os.dup2(output, number) os.close(output) + if behavior == 'pipe': + reader, writer = os.pipe() + os.write(writer, b'piped data\n') + os.close(writer) + os.dup2(reader, 0) + os.close(reader) env = dict(os.environ, HACK_HOME=str(root / 'state')) os.execve(bun, [bun, *([entrypoint] if entrypoint else []), 'host', 'exec', '--path', str(root), '--no-interactive', '--', sys.executable, __file__, 'worker', str(root), behavior], env) (root / 'wrapper.pid').write_text(str(wrapper)) @@ -134,7 +145,7 @@ def wait_for(names, seconds=8): (root / 'resume.request').write_text('resume') if not wait_for(['continued']): raise RuntimeError('Command did not resume after fg') - if behavior in ['normal', 'io']: + if behavior in ['normal', 'io', 'pipe']: os.write(fd, b'hello tty\n') if not wait_for(['input.txt']): raise RuntimeError('Command did not read terminal stdin') @@ -159,8 +170,20 @@ def wait_for(names, seconds=8): 'tty': json.loads((root / 'tty.json').read_text()), 'stopped': stopped, } - if behavior in ['normal', 'io']: + records = list((root / 'state' / 'host-commands').glob('*.json')) + if records: + record = json.loads(records[0].read_text()) + output['record'] = { + 'status': record['status'], 'exitCode': record['exitCode'], + 'ownsProcessGroup': record['ownsProcessGroup'], + 'childMatches': record['child']['pid'] == int((root / 'child.pid').read_text()), + 'groupDifferentFromChild': record['processGroupId'] != record['child']['pid'], + 'maxRssBytes': record['maxRssBytes'], 'cpuTimeMs': record['cpuTimeMs'], + } + if behavior in ['normal', 'io', 'pipe']: output.update({name: (root / path).read_text() for name, path in [('input', 'input.txt'), ('stdout', 'stdout.txt'), ('stderr', 'stderr.txt')]}) + if behavior == 'pipe': + output['pipeInput'] = (root / 'pipe-input.txt').read_text() print(json.dumps(output)) finally: # Every PID below was created by this fixture. Never signal the shared outer group. diff --git a/tests/host-command-observation.test.ts b/tests/host-command-observation.test.ts new file mode 100644 index 00000000..1a43b4b7 --- /dev/null +++ b/tests/host-command-observation.test.ts @@ -0,0 +1,95 @@ +import { expect, test } from "bun:test"; +import { + type HostCommandRecord, + observeHostCommand, + parseHostCommandRecord, + parseObservedProcesses, + parseProcessDuration, +} from "../src/lib/host-command-observation.ts"; + +const record: HostCommandRecord = { + version: 1, + id: "f7e49a0a-1355-4b60-8aba-b0a0467dc845", + project: "fixture", + projectRoot: "/fixture", + executable: "bun", + wrapper: { pid: 100, birth: "Tue Sep 8 13:00:00 2026" }, + child: { pid: 101, birth: "Tue Sep 8 13:00:01 2026" }, + ownsProcessGroup: true, + processGroupId: 101, + lifetime: "persistent", + timeoutMs: null, + startedAt: "2026-09-08T17:00:00.000Z", + finishedAt: null, + status: "running", + exitCode: null, + cpuTimeMs: null, + maxRssBytes: null, +}; + +test("process durations cover macOS fractional CPU and Linux day/hour elapsed fields", () => { + expect(parseProcessDuration("0:01.25")).toBe(1250); + expect(parseProcessDuration("2-03:04:05")).toBe(183_845_000); + expect(parseProcessDuration("bad")).toBeNull(); +}); + +test("process rows contain elapsed, cumulative CPU, RSS and stable identity without argv", () => { + const rows = parseObservedProcesses( + "101 100 101 00:02 0:01.25 1024 Tue Sep 8 13:00:01 2026\ninvalid" + ); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + pid: 101, + ppid: 100, + processGroupId: 101, + elapsedMs: 2000, + cpuTimeMs: 1250, + rssBytes: 1_048_576, + birth: record.child.birth, + }); + const observed = observeHostCommand(record, rows); + expect(observed.status).toBe("orphaned"); + expect(observed.attention).toBe("persistent_wrapper_lost"); + expect(observed.liveCpuTimeMs).toBe(1250); +}); + +test("PID reuse and unavailable snapshots never establish live ownership", () => { + const reused = parseObservedProcesses( + "101 1 101 00:01 0:00.01 128 Tue Sep 8 14:00:01 2026" + ); + expect(observeHostCommand(record, reused)).toMatchObject({ + status: "interrupted", + ownership: "unverified", + liveCpuTimeMs: null, + }); + expect(observeHostCommand(record, null).status).toBe("unknown"); +}); + +test("completed CPU accounting stays separate from live process snapshots", () => { + const completed = { + ...record, + status: "cancelled" as const, + finishedAt: "2026-09-08T17:00:03.000Z", + cpuTimeMs: 25, + exitCode: 143, + }; + expect(observeHostCommand(completed, [])).toMatchObject({ + status: "cancelled", + cpuTimeMs: 25, + liveCpuTimeMs: null, + elapsedMs: 3000, + cpuAccounting: "reaped_child", + }); +}); + +test("stored observation schema strips payload fields and rejects invalid metrics", () => { + expect( + parseHostCommandRecord({ + ...record, + argv: ["secret"], + env: { TOKEN: "secret" }, + }) + ).toEqual(record); + expect(parseHostCommandRecord({ ...record, cpuTimeMs: -1 })).toBeNull(); + expect(parseHostCommandRecord({ ...record, id: "../../outside" })).toBeNull(); +}); diff --git a/tests/host-exec-lifetime.test.ts b/tests/host-exec-lifetime.test.ts index 7c63ce2e..48facbca 100644 --- a/tests/host-exec-lifetime.test.ts +++ b/tests/host-exec-lifetime.test.ts @@ -1,5 +1,12 @@ import { afterEach, expect, test } from "bun:test"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; @@ -92,6 +99,96 @@ test("host exec preserves piped stdin and a normal nonzero exit status", async ( expect(await output).toBe("received:fixture-input\n"); }); +for (const ignores of [false, true]) { + test(`host timeout records bounded lifetime and stops descendants (ignores=${ignores})`, async () => { + const root = await createFixture({ childIgnoresSignals: ignores }); + const wrapper = Bun.spawn( + hostCommand({ root, flags: ["--timeout", "0.5"] }), + { + cwd: root, + env: fixtureEnv(root), + stdin: "ignore", + stdout: "ignore", + stderr: "pipe", + detached: true, + } + ); + wrappers.push(wrapper); + const child = await waitForPid(resolve(root, "child.pid")); + const grandchild = await waitForPid(resolve(root, "grandchild.pid")); + expect(await wrapper.exited).toBe(124); + await expectStopped(child); + await expectStopped(grandchild); + const directory = resolve(root, "hack-home", "host-commands"); + const files = (await readdir(directory)).filter((name) => + name.endsWith(".json") + ); + expect(files).toHaveLength(1); + const record = await Bun.file(resolve(directory, files[0] ?? "")).json(); + expect(record).toMatchObject({ + lifetime: "bounded", + status: "timed_out", + exitCode: 124, + ownsProcessGroup: true, + timeoutMs: 500, + }); + expect(record.cpuTimeMs).toBeGreaterThanOrEqual(0); + }, 10_000); +} + +test("persistent commands have no implicit deadline and orphan inspection leaves them alive", async () => { + const root = await createFixture({ childIgnoresSignals: false }); + const wrapper = Bun.spawn( + [ + ...hostCommand({ root, flags: ["--lifetime", "persistent"] }), + "sensitive-argument-fixture", + ], + { + cwd: root, + env: fixtureEnv(root), + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + detached: true, + } + ); + wrappers.push(wrapper); + const child = await waitForPid(resolve(root, "child.pid")); + await waitForPid(resolve(root, "grandchild.pid")); + const recordDir = resolve(root, "hack-home", "host-commands"); + for (let attempt = 0; attempt < 100; attempt++) { + const files = await readdir(recordDir).catch(() => []); + if (files.some((file) => file.endsWith(".json"))) { + break; + } + await Bun.sleep(20); + } + expect(wrapper.exitCode).toBeNull(); + signalPid(wrapper.pid, "SIGKILL"); + await wrapper.exited; + const query = Bun.spawn( + [process.execPath, entrypoint, "host", "ps", "--json"], + { + cwd: root, + env: fixtureEnv(root), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + } + ); + const output = await new Response(query.stdout).text(); + expect(await query.exited).toBe(0); + const record = JSON.parse(output).commands[0]; + expect(record).toMatchObject({ + lifetime: "persistent", + status: "orphaned", + attention: "persistent_wrapper_lost", + timeoutMs: null, + }); + expect(output).not.toContain("sensitive-argument-fixture"); + expect(process.kill(child, 0)).toBe(true); +}); + function startHostCommand({ root }: { readonly root: string }) { const wrapper = Bun.spawn(hostCommand({ root }), { cwd: root, @@ -105,7 +202,13 @@ function startHostCommand({ root }: { readonly root: string }) { return wrapper; } -function hostCommand({ root }: { readonly root: string }): string[] { +function hostCommand({ + root, + flags = [], +}: { + readonly root: string; + readonly flags?: readonly string[]; +}): string[] { return [ process.execPath, entrypoint, @@ -114,6 +217,7 @@ function hostCommand({ root }: { readonly root: string }): string[] { "--path", root, "--no-interactive", + ...flags, "--", process.execPath, resolve(root, "child.ts"), diff --git a/tests/host-exec-tty.test.ts b/tests/host-exec-tty.test.ts index 728c8871..80c3bc41 100644 --- a/tests/host-exec-tty.test.ts +++ b/tests/host-exec-tty.test.ts @@ -46,6 +46,13 @@ const cases = [ mode: "wrapper", code: 143, }, + { + name: "piped stdin retains the controlling terminal", + signal: "SIGTERM", + behavior: "pipe", + mode: "wrapper", + code: 143, + }, { name: "normal command return", signal: "SIGTERM", @@ -93,9 +100,28 @@ for (const scenario of cases) { childAlive: false, grandchildAlive: false, siblingAlive: true, - tty: { stdin: true, devTty: true, foreground: true }, + tty: { + stdin: scenario.behavior !== "pipe", + devTty: true, + foreground: true, + }, + }); + expect(outcome.record).toMatchObject({ + status: scenario.behavior === "normal" ? "exited" : "cancelled", + exitCode: scenario.code, + ownsProcessGroup: true, + childMatches: true, + groupDifferentFromChild: true, }); - if (scenario.behavior === "normal" || scenario.behavior === "io") { + if (scenario.behavior === "normal") { + expect(outcome.record.maxRssBytes).toBeGreaterThan(0); + expect(outcome.record.cpuTimeMs).toBeGreaterThan(0); + } + if ( + scenario.behavior === "normal" || + scenario.behavior === "io" || + scenario.behavior === "pipe" + ) { expect(outcome).toMatchObject({ input: "hello tty\n", stdout: "child stdout\n", @@ -103,6 +129,9 @@ for (const scenario of cases) { tty: { stdout: false, stderr: false }, }); } + if (scenario.behavior === "pipe") { + expect(outcome.pipeInput).toBe("piped data\n"); + } if (scenario.mode === "resume") { expect(outcome.stopped).toEqual({ foregroundRestored: true }); } diff --git a/tests/project-runtime-hygiene.test.ts b/tests/project-runtime-hygiene.test.ts index 63ec72da..f6b77f0b 100644 --- a/tests/project-runtime-hygiene.test.ts +++ b/tests/project-runtime-hygiene.test.ts @@ -120,6 +120,42 @@ test("findOrphanRuntimeProjects reports missing working dirs and compose files", ]); }); +test("orphan cleanup excludes lifecycle placeholders from mixed and lifecycle-only projects", async () => { + const root = await mkdtemp(join(tmpdir(), "hack-runtime-hygiene-")); + tempDirs.add(root); + const mixed = buildRuntimeProject({ + project: "mixed", + workingDir: root, + containerIds: ["docker-1", "lifecycle-1"], + }); + const lifecycle = buildRuntimeProject({ + project: "lifecycle", + workingDir: root, + containerIds: ["lifecycle-only"], + }); + for (const project of [mixed, lifecycle]) { + for (const service of project.services.values()) { + for (const container of service.containers) { + if (container.id.startsWith("lifecycle-")) { + Object.assign(container, { + labels: { "hack.lifecycle.process": "true" }, + }); + } + } + } + } + expect( + await findOrphanRuntimeProjects({ runtime: [mixed, lifecycle] }) + ).toEqual([ + { + project: "mixed", + workingDir: root, + reason: "missing compose file", + containerIds: ["docker-1"], + }, + ]); +}); + test("findIncompleteRuntimeProjects reports regular services stuck in Created", () => { const runtime = buildRuntimeProject({ project: "interrupted", diff --git a/tests/projects-prune.test.ts b/tests/projects-prune.test.ts index 83bfaece..cef5678e 100644 --- a/tests/projects-prune.test.ts +++ b/tests/projects-prune.test.ts @@ -173,3 +173,26 @@ function runtimeProject(project: string): RuntimeProject { isGlobal: false, }; } + +test("prune dry-run leaves a missing registration intact", async () => { + const dead = buildRegistration({ + id: "preview000001", + name: "preview", + repoRoot: join(tempDir ?? "", "gone"), + }); + await writeRegistry([dead]); + const { runCli } = await import("../src/cli/run.ts"); + expect( + await runCli([ + "projects", + "prune", + "--project", + "preview", + "--dry-run", + "--json", + ]) + ).toBe(0); + expect( + (await readProjectsRegistry()).projects.map((project) => project.id) + ).toEqual([dead.id]); +}); diff --git a/tests/runtime-cache.test.ts b/tests/runtime-cache.test.ts index fc0649b1..0a966352 100644 --- a/tests/runtime-cache.test.ts +++ b/tests/runtime-cache.test.ts @@ -730,3 +730,56 @@ async function waitFor(opts: { await Bun.sleep(1); } } + +test("summary payload keeps project identity and omits detailed runtime payloads", async () => { + const view: ProjectView = { + name: "summary-fixture", + devHost: "fixture.hack", + repoRoot: "/fixture", + projectDir: "/fixture/.hack", + definedServices: ["web"], + extensionsEnabled: null, + features: null, + serviceHosts: null, + runtimeConfigured: true, + runtimeStatus: "stopped", + runtime: null, + branchRuntime: [], + sessions: [], + lifecycle: null, + ownership: null, + worktrees: null, + kind: "registered", + status: "stopped", + }; + const cache = createRuntimeCache({ + deps: { + readProjectsRegistry: async () => ({ version: 1, projects: [] }), + buildProjectViews: async () => [view], + serializeProjectView: () => { + throw new Error("full serializer must not run for summary"); + }, + }, + }); + const payload = await cache.getProjectsPayload({ + filter: null, + includeGlobal: false, + includeUnregistered: false, + includeMeta: false, + summary: true, + profile: true, + }); + expect(payload.detail_level).toBe("summary"); + expect(payload.projects[0]).toMatchObject({ + name: "summary-fixture", + defined_service_count: 1, + container_count: 0, + running_container_count: 0, + }); + expect(payload.projects[0]).not.toHaveProperty("runtime"); + expect(payload.projects[0]).not.toHaveProperty("lifecycle"); + expect(payload.profiling?.source).toBe("daemon"); + expect(cache.getDiagnostics().lastRefreshPhasesMs).toHaveProperty( + "auto_register_ms" + ); +}); diff --git a/tests/runtime-projects.test.ts b/tests/runtime-projects.test.ts index 3f2d1f48..2fe046f9 100644 --- a/tests/runtime-projects.test.ts +++ b/tests/runtime-projects.test.ts @@ -22,13 +22,15 @@ const shellMock = await registerScopedModuleMock({ }; } if (command[1] === "inspect") { - const ids = [...command.slice(2)]; + expect(command[2]).toBe("--format"); + expect(command[3]).not.toContain(".Env"); + const ids = [...command.slice(4)]; inspectCalls.push(ids); const returnedIds = inspectExitCode === 0 ? ids : ids.slice(0, 1); return { - stdout: JSON.stringify( - returnedIds.map((id) => makeInspectRow({ id })) - ), + stdout: returnedIds + .map((id) => JSON.stringify(makeInspectRow({ id }))) + .join("\n"), stderr: inspectExitCode === 0 ? "" : "one inspected container disappeared", exitCode: inspectExitCode, diff --git a/tests/shell-observation.test.ts b/tests/shell-observation.test.ts new file mode 100644 index 00000000..fd41a475 --- /dev/null +++ b/tests/shell-observation.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test"; +import { type RunExitEvent, run } from "../src/lib/shell.ts"; + +test("slow diagnostics preserve an already-completed command's exit and callback order", async () => { + const calls: string[] = []; + let outcome: RunExitEvent | undefined; + let setupFinishedAt = 0; + const code = await run([process.execPath, "-e", "process.exit(7)"], { + stdin: "ignore", + timeoutMs: 1000, + onSpawn: async () => { + await Bun.sleep(1500); + setupFinishedAt = Date.now(); + calls.push("spawn"); + }, + onExit: async (event) => { + outcome = event; + calls.push("exit"); + }, + }); + expect(code).toBe(7); + expect(outcome).toMatchObject({ + exitCode: 7, + timedOut: false, + cancelled: false, + }); + expect(calls).toEqual(["spawn", "exit"]); + expect(Date.parse(outcome?.finishedAt ?? "")).toBeLessThan(setupFinishedAt); +}); + +test("completed process peak RSS is recorded in bytes on the executing platform", async () => { + let peakBytes: number | null = null; + const code = await run( + [ + process.execPath, + "-e", + "const bytes = Buffer.alloc(64 * 1024 * 1024, 1); await Bun.sleep(50); if (bytes[bytes.length - 1] !== 1) process.exit(1);", + ], + { + stdin: "ignore", + onExit: async (event) => { + peakBytes = event.maxRssBytes; + }, + } + ); + expect(code).toBe(0); + expect(peakBytes).not.toBeNull(); + expect(Number(peakBytes)).toBeGreaterThanOrEqual(64 * 1024 * 1024); + expect(Number(peakBytes)).toBeLessThan(2 * 1024 * 1024 * 1024); +}); diff --git a/tests/usage-details.test.ts b/tests/usage-details.test.ts new file mode 100644 index 00000000..afac77b2 --- /dev/null +++ b/tests/usage-details.test.ts @@ -0,0 +1,168 @@ +import { expect, test } from "bun:test"; +import { __testOnlyUsage } from "../src/commands/usage.ts"; +import type { + HostCommandRecord, + ObservedProcess, +} from "../src/lib/host-command-observation.ts"; +import type { + RuntimeContainer, + RuntimeProject, +} from "../src/lib/runtime-projects.ts"; + +function container( + id: string, + state: string, + lifecycle = false +): RuntimeContainer { + return { + id, + state, + project: "fixture--branch", + service: "web", + name: id, + status: state, + ports: "", + workingDir: "/fixture/.hack", + image: "fixture", + labels: lifecycle ? { "hack.lifecycle.process": "true" } : {}, + mounts: [ + { + type: "volume", + source: "/volume", + destination: "/app/node_modules", + mode: "rw", + rw: true, + }, + ], + networks: [], + }; +} + +test("usage samples only running Docker containers and retains service/mount attribution", () => { + const project: RuntimeProject = { + project: "fixture--branch", + workingDir: "/fixture/.hack", + isGlobal: false, + services: new Map([ + [ + "web", + { + service: "web", + containers: [ + container("running", "running"), + container("stopped", "exited"), + container("created", "created"), + container("host", "running", true), + ], + }, + ], + ]), + }; + const index = __testOnlyUsage.buildContainerIndex({ projects: [project] }); + expect(index.containerIds).toEqual(["running"]); + const report = __testOnlyUsage.buildUsageReport({ + projects: [project], + index, + samples: [ + { + containerId: "running", + cpuPercent: 12, + memUsedBytes: 1024, + memLimitBytes: 4096, + memPercent: 25, + netInputBytes: 0, + netOutputBytes: 0, + blockInputBytes: 0, + blockOutputBytes: 0, + pids: 4, + }, + ], + }); + expect(report.projects[0]?.containers).toBe(1); + expect(report.containerDetails?.[0]).toMatchObject({ + project: "fixture--branch", + service: "web", + name: "running", + memUsedBytes: 1024, + mounts: [{ type: "volume", destination: "/app/node_modules" }], + }); +}); + +test("project-scoped host usage includes branch descendants and excludes other projects from totals", () => { + const records: HostCommandRecord[] = [ + "alpha", + "alpha--branch", + "beta", + "alphabet", + "alpha", + ].map((project, index) => ({ + version: 1, + id: String(index), + project, + projectRoot: `/${project}`, + executable: "bun", + wrapper: { pid: 100 * (index + 1), birth: "start" }, + child: { pid: 100 * (index + 1) + 1, birth: "start" }, + ownsProcessGroup: true, + processGroupId: 100 * (index + 1) + 1, + lifetime: "command", + timeoutMs: null, + startedAt: "2026-09-08T17:00:00.000Z", + finishedAt: null, + status: index === 4 ? "exited" : "running", + exitCode: null, + cpuTimeMs: null, + maxRssBytes: null, + })); + const snapshot: ObservedProcess[] = records.flatMap((record) => + [0, 1].map((offset) => ({ + pid: record.child.pid + offset, + ppid: offset === 0 ? record.wrapper.pid : record.child.pid, + processGroupId: record.child.pid, + birth: "start", + elapsedMs: 1000, + cpuTimeMs: 10, + rssBytes: 100, + })) + ); + const tracked = __testOnlyUsage.collectTrackedHostPids({ + records, + snapshot, + filter: "alpha", + }); + expect( + [...tracked].filter(([, name]) => name !== null).map(([pid]) => pid) + ).toEqual([101, 102, 201, 202]); + expect(tracked.get(301)).toBeNull(); + const report = __testOnlyUsage.buildHostUsageReport({ + samples: [...tracked].flatMap(([pid, name]) => + name === null ? [] : [{ pid, name, cpuPercent: 1, memBytes: 100 }] + ), + }); + expect(report.rows.map((row) => row.name)).toEqual([ + "host:alpha--branch:bun", + "host:alpha:bun", + ]); + expect(report.total).toMatchObject({ + cpuPercent: 4, + memBytes: 400, + processes: 4, + }); + expect( + __testOnlyUsage.collectTrackedHostPids({ records, snapshot, filter: null }) + .size + ).toBe(8); + const nestedSnapshot = snapshot.map((row) => + row.pid === 301 ? { ...row, ppid: 102 } : row + ); + for (const filter of ["alpha", "beta", null]) { + const nested = __testOnlyUsage.collectTrackedHostPids({ + records, + snapshot: nestedSnapshot, + filter, + }); + expect(nested.get(301)).toBe(filter === "alpha" ? null : "host:beta:bun"); + expect(nested.get(302)).toBe(filter === "alpha" ? null : "host:beta:bun"); + expect(nested.get(101)).toBe(filter === "beta" ? null : "host:alpha:bun"); + } +});