From 67241574a4d1214af7471b934b60f282ab229ca6 Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 12:06:09 +0000 Subject: [PATCH 1/6] feat(bot): add production ops launcher --- apps/bot/README.md | 140 +++- scripts/ickb-bot-collect-incident.mjs | 816 +++++++++++++++++++ scripts/ickb-bot-collect-incident.test.mjs | 492 +++++++++++ scripts/ickb-bot-launcher.mjs | 441 ++++++++++ scripts/ickb-bot-launcher.test.mjs | 301 +++++++ scripts/ickb-bot-systemd-credential.sh | 19 +- scripts/ickb-bot-systemd-credential.test.mjs | 25 + scripts/ickb-bot-systemd-install.sh | 113 ++- scripts/ickb-bot-systemd-install.test.mjs | 96 +++ scripts/ickb-bot-systemd-update.sh | 99 ++- scripts/ickb-bot-systemd-update.test.mjs | 221 +++++ 11 files changed, 2745 insertions(+), 18 deletions(-) create mode 100644 scripts/ickb-bot-collect-incident.mjs create mode 100644 scripts/ickb-bot-collect-incident.test.mjs create mode 100644 scripts/ickb-bot-launcher.mjs create mode 100644 scripts/ickb-bot-launcher.test.mjs create mode 100644 scripts/ickb-bot-systemd-install.test.mjs create mode 100644 scripts/ickb-bot-systemd-update.test.mjs diff --git a/apps/bot/README.md b/apps/bot/README.md index a1481cb4..db9169c3 100644 --- a/apps/bot/README.md +++ b/apps/bot/README.md @@ -53,6 +53,8 @@ The start script writes NDJSON logs to stdout and tees one log file per run. Bal Every bot observability record is one JSON object on stdout with `version`, `app: "bot"`, `chain`, `runId`, `iterationId`, ISO `timestamp`, and `type`. Execution-log records also remain on stdout, and structured bot records can be selected with `app == "bot"`. +The stable event contract is the bot NDJSON object stream, not a particular file path. Production launchers may route stdout to journald, files, or another operator-owned log root; consumers should depend on records with `app: "bot"` and `bot.*` event types, not supervisor/tester output, launcher metadata, rotation layout, incident bundles, `/var/log`, or validation log directories. + Stable event types: - `bot.run.started` @@ -76,15 +78,43 @@ JSON `"maxIterations":1` makes `pnpm --filter ./apps/bot start` exit with code ` Structured events should contain public evidence and summaries needed to understand bot behavior. Do not add private keys, seed phrases, mnemonics, or other secrets to log payloads; omit noisy public fields at the call site instead of relying on redaction. +Bot-only log queries, using the production event file or any saved bot stdout NDJSON stream: + +```bash +LOG_DIR=/opt/ickb-stack-testnet/log/bot/testnet +jq -c 'select(.app == "bot")' "$LOG_DIR/bot.events.ndjson" +jq -r 'select(.app == "bot") | .type' "$LOG_DIR/bot.events.ndjson" | sort | uniq -c +jq -c 'select(.type == "launcher.child.exited") | {timestamp, status, signal, elapsedMs, logRoot, logDir, command}' "$LOG_DIR/launches.ndjson" +``` + ## Ubuntu systemd Deployment -For unattended Ubuntu 24.04 deployments, run testnet and mainnet as separate systemd services with separate users, deploy directories, and encrypted JSON config credentials. The production units should execute the built app directly with `node apps/bot/dist/index.js`; the package `start` script is for local JSON-config runs and tees app-local log files. +For unattended Ubuntu 24.04 deployments, run testnet and mainnet as separate systemd services with separate users, deploy directories, encrypted JSON config credentials, and bot-only file logs. The production units execute the built app through `scripts/ickb-bot-launcher.mjs`, which owns only process and file plumbing: it starts `/usr/bin/node apps/bot/dist/index.js`, writes bot stdout byte-for-byte to `bot.events.ndjson`, writes child stderr byte-for-byte to `bot.stderr.log`, writes launch metadata to `launches.ndjson`, and tees stdout/stderr to journald as a fallback. Launcher metadata records the executable basename and argument count, but not raw child argument values or environment. The package `start` script is for local JSON-config runs and tees app-local log files. + +The launcher resolves the log root in this order: explicit `--log-root`, runtime `ICKB_BOT_LOG_ROOT`, then `/log`. The systemd install script bakes an explicit `--log-root` into generated units only when `ICKB_BOT_LOG_ROOT` is set during install; relative values are resolved against that network's deploy directory. Without an explicit configured root, testnet defaults to `/opt/ickb-stack-testnet/log` and mainnet defaults to `/opt/ickb-stack-mainnet/log`. + +The launcher refuses empty paths, log directories outside the resolved log root, symlinked log roots or parent directories, and symlinked log files. It creates `bot.events.ndjson`, `bot.stderr.log`, and `launches.ndjson` with mode `0600` for service-user writes. + +Production log layout: + +```text +/bot/testnet/bot.events.ndjson +/bot/testnet/bot.stderr.log +/bot/testnet/launches.ndjson +/bot/mainnet/bot.events.ndjson +/bot/mainnet/bot.stderr.log +/bot/mainnet/launches.ndjson +``` + +These are production bot-only logs. They are separate from local live validation supervisor artifacts such as `logs/live-supervisor/...`. This layout keeps the workflow simple while avoiding accidental cross-network updates: ```text /opt/ickb-stack-testnet /opt/ickb-stack-mainnet +/opt/ickb-stack-testnet/log/bot/testnet/ +/opt/ickb-stack-mainnet/log/bot/mainnet/ /etc/ickb/credentials/ickb-bot-testnet-config.cred /etc/ickb/credentials/ickb-bot-mainnet-config.cred /etc/systemd/system/ickb-bot-testnet.service @@ -97,6 +127,12 @@ From a deployed checkout on the VM, install service users, deploy directories, a sudo scripts/ickb-bot-systemd-install.sh all ``` +To use one explicit log root for both networks, set it while installing or regenerating the units: + +```bash +sudo ICKB_BOT_LOG_ROOT=/path/to/ickb-log-root scripts/ickb-bot-systemd-install.sh all +``` + Populate and build each deploy directory before starting services. Clone or copy the same repo revision into both directories, then build as the matching service user: ```bash @@ -112,7 +148,7 @@ sudo -u ickb-bot-mainnet pnpm -C /opt/ickb-stack-mainnet bot:build If `/opt/ickb-stack-testnet` or `/opt/ickb-stack-mainnet` already exists from the install script, clone into a temporary path and move the checkout into place, or initialize the existing directory with your normal deployment tooling. The update script expects each deploy directory to be a clean git checkout. -Create encrypted config credentials on the VM. The tested Ubuntu 24.04 VM has `systemd-creds` and no TPM device, so host-key credentials are the compatible unattended option. If a future VM exposes a TPM, replace `--with-key=host` with the TPM-backed mode selected for that host. The helper prompts for the private key, RPC URL, sleep interval, and optional max iterations, validates the same strict JSON schema that the app reads, and encrypts that JSON as one systemd credential. +Create encrypted config credentials on the VM. The tested Ubuntu 24.04 VM has `systemd-creds` and no TPM device, so host-key credentials are the compatible unattended option. If a future VM exposes a TPM, replace `--with-key=host` with the TPM-backed mode selected for that host. The helper prompts for the private key, optional RPC URL, sleep interval, optional max iterations, and max retryable attempts, validates the same strict JSON schema that the app reads, and encrypts that JSON as one systemd credential. ```bash sudo systemd-creds setup @@ -135,14 +171,16 @@ Group=ickb-bot-testnet WorkingDirectory=/opt/ickb-stack-testnet Environment=BOT_CONFIG_FILE=%d/ickb-bot-testnet-config.json LoadCredentialEncrypted=ickb-bot-testnet-config.json:/etc/ickb/credentials/ickb-bot-testnet-config.cred -ExecStart=/usr/bin/node apps/bot/dist/index.js +ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs --network testnet -- /usr/bin/node apps/bot/dist/index.js Restart=on-failure RestartSec=10 RestartPreventExitStatus=2 +LimitCORE=0 NoNewPrivileges=true PrivateTmp=true ProtectProc=invisible ProtectSystem=strict +ReadWritePaths=/opt/ickb-stack-testnet/log ProtectHome=true [Install] @@ -164,14 +202,16 @@ Group=ickb-bot-mainnet WorkingDirectory=/opt/ickb-stack-mainnet Environment=BOT_CONFIG_FILE=%d/ickb-bot-mainnet-config.json LoadCredentialEncrypted=ickb-bot-mainnet-config.json:/etc/ickb/credentials/ickb-bot-mainnet-config.cred -ExecStart=/usr/bin/node apps/bot/dist/index.js +ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs --network mainnet -- /usr/bin/node apps/bot/dist/index.js Restart=on-failure RestartSec=10 RestartPreventExitStatus=2 +LimitCORE=0 NoNewPrivileges=true PrivateTmp=true ProtectProc=invisible ProtectSystem=strict +ReadWritePaths=/opt/ickb-stack-mainnet/log ProtectHome=true [Install] @@ -194,11 +234,14 @@ sudo systemctl status ickb-bot-testnet.service sudo systemctl status ickb-bot-mainnet.service sudo journalctl -u ickb-bot-testnet.service -f sudo journalctl -u ickb-bot-mainnet.service -f +sudo tail -f /opt/ickb-stack-testnet/log/bot/testnet/bot.events.ndjson +sudo tail -f /opt/ickb-stack-mainnet/log/bot/mainnet/bot.events.ndjson +jq -c 'select(.type == "launcher.child.exited")' /opt/ickb-stack-testnet/log/bot/testnet/launches.ndjson sudo systemctl restart ickb-bot-testnet.service sudo systemctl restart ickb-bot-mainnet.service ``` -Update testnet first, then mainnet after the same revision is validated. The update script pulls, installs, and builds before restarting the service, so a failed build leaves the currently running bot alone. +Update testnet first, then mainnet after the same revision is validated. Regenerate the units with `scripts/ickb-bot-systemd-install.sh` before updating when the documented unit shape changes. The update script refuses stale units that are missing the production launcher wiring or `LimitCORE=0`, then pulls, installs, and builds before restarting the service, so a failed build leaves the currently running bot alone. ```bash sudo scripts/ickb-bot-systemd-update.sh testnet @@ -207,7 +250,92 @@ sudo scripts/ickb-bot-systemd-update.sh mainnet Use `scripts/ickb-bot-systemd-update.sh mainnet` only after the same revision has been validated on testnet. -Exit code `2` is an intentional safety stop, including low capital and transaction confirmation timeout after broadcast. Inspect the journal before restarting a service that stopped with code `2`. +Exit code `2` is an intentional safety stop, including low capital and transaction confirmation timeout after broadcast. `RestartPreventExitStatus=2` keeps systemd from relaunching immediately. Before restarting, inspect `launches.ndjson` for the child exit record, `bot.events.ndjson` for the terminal bot event, `bot.stderr.log` for runtime errors, and journald for launcher fallback output: + +```bash +LOG_DIR=/opt/ickb-stack-testnet/log/bot/testnet +jq -c 'select(.type == "launcher.child.exited")' "$LOG_DIR/launches.ndjson" +jq -c 'select(.app == "bot" and (.terminal == true or .type == "bot.decision.skipped" or .type == "bot.transaction.failed" or .type == "bot.iteration.failed"))' "$LOG_DIR/bot.events.ndjson" +sudo journalctl -u ickb-bot-testnet.service -n 200 --no-pager +``` + +The generated units set `LimitCORE=0`, so crash diagnosis should use bot logs, launcher exit records, stderr, journald, and the bundled unit text rather than expecting a core file. + +### Incident Bundles + +Use `scripts/ickb-bot-collect-incident.mjs` before restarting after exit code `2` or any unexpected production behavior. The collector reads only bot production sources: `bot.events.ndjson`, `bot.stderr.log`, `launches.ndjson`, version metadata, and optional systemd status/journal/unit text. It keeps those sources separated and writes a restricted incident directory under the selected bot log directory: + +```text +/bot//incidents// + README.txt + bot.events.ndjson + bot.stderr.log + launches.ndjson + summary.json + version.json + systemd.status.txt # when systemd output is available + systemd.journal.txt # when systemd output is available + systemd.unit.txt # when systemd output is available +``` + +The log root resolves the same way as the launcher: explicit `--log-root`, then runtime `ICKB_BOT_LOG_ROOT`, then `/log`. `--network testnet|mainnet` selects `/bot//`. `--log-dir ` may be used instead of `--network` only when the resolved path stays inside the resolved log root, which is useful for copied logs or a custom contained bot log directory. The collector refuses empty paths, paths outside the resolved log root, symlinked log directories, symlinked incident parents, and symlinked source log files. + +Examples from the deployed checkout: + +```bash +sudo -u ickb-bot-testnet node scripts/ickb-bot-collect-incident.mjs --network testnet --since 2h --until now +sudo -u ickb-bot-mainnet node scripts/ickb-bot-collect-incident.mjs --network mainnet --since 2026-05-25T10:00:00Z --until 2026-05-25T11:00:00Z +sudo -u ickb-bot-testnet node scripts/ickb-bot-collect-incident.mjs --log-root /path/to/ickb-log-root --network testnet --since 30m --until now +sudo -u ickb-bot-testnet ICKB_BOT_LOG_ROOT=/path/to/ickb-log-root node scripts/ickb-bot-collect-incident.mjs --log-dir /path/to/ickb-log-root/bot/testnet --since 30m --until now +``` + +If you do not want systemd status, journal, or unit text in the bundle, add `--no-systemd`. The collector does not include runtime config files or environment dumps because they can contain private keys, credentialed RPC URLs, tokens, passwords, or API keys. Selected source logs and systemd output are bundled as public producer-owned evidence; if a private key or other secret reaches those sources, fix the producer that wrote it before sharing or archiving the bundle. + +Inspect `summary.json` first. It includes selected source files, malformed/undated/out-of-window line counts, first/last timestamps, event counts by type, transaction hashes by outcome, skip/failure reasons, launcher exit codes, systemd capture results, package version, git commit, Node version, and the collector script version. `bot.stderr.log` is raw child stderr, so undated stack-trace lines after an in-window timestamped stderr line are kept with that timestamped line; when stderr has no timestamps at all, the collector includes the last 200 non-empty stderr lines and marks that in `summary.json`. For exit code `2`, review the `launcher.child.exited` record, terminal bot events (`bot.decision.skipped`, `bot.transaction.failed`, `bot.iteration.failed`, or records with `terminal:true`), stderr, and journald before deciding whether the restart is safe. + +The collector writes the incident directory directly and prints a portable compression command instead of assuming `tar`, `gzip`, or `zstd` are present. On a host with `tar` and gzip, run the printed command or equivalently: + +```bash +tar -czf /opt/ickb-stack-testnet/log/bot/testnet/incidents/.tar.gz -C /opt/ickb-stack-testnet/log/bot/testnet/incidents +``` + +Retain incident bundles long enough to cover your operational review and postmortem window, then remove them with the same sensitivity as production logs. A practical default is to keep testnet bundles for 14 days and mainnet bundles for 30 days, matching the rotation examples below unless an active incident review requires longer retention. + +### Log Rotation + +The launcher keeps the three log files open for the lifetime of the service and does not implement a reopen signal. Use `copytruncate` if you want rotation without restarting the bot. This can lose a small write window during copy/truncate, but it preserves continuous systemd supervision. If you require exact handoff instead, restart the service after rotation and treat the restart as an operational event. + +Default-root logrotate example for testnet: + +```text +/opt/ickb-stack-testnet/log/bot/testnet/bot.events.ndjson +/opt/ickb-stack-testnet/log/bot/testnet/bot.stderr.log +/opt/ickb-stack-testnet/log/bot/testnet/launches.ndjson { + daily + rotate 14 + missingok + notifempty + compress + copytruncate + su ickb-bot-testnet ickb-bot-testnet +} +``` + +Default-root logrotate example for mainnet: + +```text +/opt/ickb-stack-mainnet/log/bot/mainnet/bot.events.ndjson +/opt/ickb-stack-mainnet/log/bot/mainnet/bot.stderr.log +/opt/ickb-stack-mainnet/log/bot/mainnet/launches.ndjson { + daily + rotate 30 + missingok + notifempty + compress + copytruncate + su ickb-bot-mainnet ickb-bot-mainnet +} +``` ## Notes diff --git a/scripts/ickb-bot-collect-incident.mjs b/scripts/ickb-bot-collect-incident.mjs new file mode 100644 index 00000000..d4da4b2e --- /dev/null +++ b/scripts/ickb-bot-collect-incident.mjs @@ -0,0 +1,816 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises"; +import { basename, join, parse, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { resolveLauncherPaths } from "./ickb-bot-launcher.mjs"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const supportedNetworks = new Set(["testnet", "mainnet"]); +const sourceFiles = [ + { kind: "botEvents", name: "bot.events.ndjson" }, + { kind: "stderr", name: "bot.stderr.log" }, + { kind: "launches", name: "launches.ndjson" }, +]; +const scriptVersion = 1; +const stderrUndatedLineLimit = 200; +const noFollow = constants.O_NOFOLLOW ?? 0; +const readOnlyNoFollow = constants.O_RDONLY | noFollow; +const writeNewFileFlags = constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | noFollow; + +export function usage() { + return [ + "Usage: node scripts/ickb-bot-collect-incident.mjs [--log-root PATH] (--network testnet|mainnet | --log-dir PATH) --since --until [--no-systemd]", + "Relative times use the current time, for example --since 2h --until now.", + ].join("\n"); +} + +export function parseArgs(argv) { + const args = { includeSystemd: true }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + continue; + } + if (arg === "-h" || arg === "--help") { + return { help: true }; + } + if (arg === "--log-root") { + args.logRoot = requireValue(argv, ++index, arg); + continue; + } + if (arg === "--network") { + args.network = requireValue(argv, ++index, arg); + continue; + } + if (arg === "--log-dir") { + args.logDir = requireValue(argv, ++index, arg); + continue; + } + if (arg === "--since") { + args.since = requireValue(argv, ++index, arg); + continue; + } + if (arg === "--until") { + args.until = requireValue(argv, ++index, arg); + continue; + } + if (arg === "--no-systemd") { + args.includeSystemd = false; + continue; + } + throw new Error(`Unknown option: ${arg}`); + } + + if (args.network !== undefined && !supportedNetworks.has(args.network)) { + throw new Error("Invalid network; expected testnet or mainnet"); + } + if ((args.network === undefined) === (args.logDir === undefined)) { + throw new Error("Specify exactly one of --network or --log-dir"); + } + if (args.since === undefined) { + throw new Error("Missing required --since"); + } + if (args.until === undefined) { + throw new Error("Missing required --until"); + } + + return args; +} + +export function parseTimeBound(value, now) { + if (value === "now") { + return new Date(now.getTime()); + } + + const relativeMatch = /^(\d+)(ms|s|m|h|d)(?:\s+ago)?$/u.exec(value); + if (relativeMatch !== null) { + const amount = Number(relativeMatch[1]); + const unit = relativeMatch[2]; + if (!Number.isSafeInteger(amount)) { + throw new Error(`Invalid time bound: ${value}`); + } + const multipliers = { d: 86_400_000, h: 3_600_000, m: 60_000, ms: 1, s: 1_000 }; + const timestamp = now.getTime() - amount * multipliers[unit]; + if (!Number.isFinite(timestamp)) { + throw new Error(`Invalid time bound: ${value}`); + } + return new Date(timestamp); + } + + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) { + throw new Error(`Invalid time bound: ${value}`); + } + return parsed; +} + +export async function collectIncident({ + argv = process.argv.slice(2), + envLogRoot = process.env.ICKB_BOT_LOG_ROOT, + now = () => new Date(), + root = rootDir, + dependencies = {}, +} = {}) { + const parsed = parseArgs(argv); + if (parsed.help) { + return { help: usage() }; + } + + const createdAt = now(); + const since = parseTimeBound(parsed.since, createdAt); + const until = parseTimeBound(parsed.until, createdAt); + if (since.getTime() > until.getTime()) { + throw new Error("--since must be before or equal to --until"); + } + + const paths = resolveIncidentPaths({ parsed, envLogRoot, root }); + await assertRealDirectory(paths.logRoot, "log root", dependencies); + await assertRealDirectory(paths.logDir, "log directory", dependencies); + + const summary = createSummary({ + createdAt, + logDir: paths.logDir, + logRoot: paths.logRoot, + logRootSource: paths.logRootSource, + network: paths.network, + since, + until, + }); + const outputs = new Map(); + + for (const source of sourceFiles) { + const sourcePath = join(paths.logDir, source.name); + const result = source.kind === "stderr" + ? await filterStderrSource(sourcePath, source.name, since, until, dependencies) + : await filterJsonSource(sourcePath, source.name, since, until, summary, source.kind, dependencies); + if (result === null) { + summary.sources[source.name] = { included: false, path: sourcePath, reason: "missing" }; + continue; + } + + summary.sources[source.name] = { + included: true, + output: source.name, + path: sourcePath, + ...result.stats, + }; + if (source.kind === "stderr") { + summary.stderr.firstTimestamp = result.stats.firstTimestamp; + summary.stderr.lastTimestamp = result.stats.lastTimestamp; + } + summary.sourceFiles.push({ + name: source.name, + output: source.name, + path: sourcePath, + selectedLines: result.stats.selectedLines, + }); + outputs.set(source.name, result.text); + } + + const version = await buildVersionMetadata(root, dependencies); + outputs.set("version.json", `${JSON.stringify(version, null, 2)}\n`); + + if (parsed.includeSystemd) { + const systemd = captureSystemd(paths.network, since, until, dependencies); + summary.systemd = systemd.summary; + for (const [name, text] of systemd.outputs) { + outputs.set(name, text); + } + } else { + summary.systemd = { included: false, reason: "disabled by --no-systemd" }; + } + + const incidentId = buildIncidentId(createdAt, paths.network); + const incidentParent = join(paths.logDir, "incidents"); + const incidentDir = join(incidentParent, incidentId); + summary.incidentId = incidentId; + summary.incidentDir = incidentDir; + summary.compression = { + created: false, + command: `tar -czf ${shellQuote(join(incidentParent, `${incidentId}.tar.gz`))} -C ${shellQuote(incidentParent)} ${shellQuote(incidentId)}`, + reason: "The collector avoids assuming tar/gzip/zstd binaries are present.", + }; + outputs.set("README.txt", incidentReadme(summary)); + outputs.set("summary.json", `${JSON.stringify(summary, null, 2)}\n`); + + await prepareIncidentDirectory(paths.logDir, incidentParent, incidentDir, dependencies); + for (const [name, text] of [...outputs].sort(([left], [right]) => left.localeCompare(right))) { + await writeBundleFile(join(incidentDir, name), text, dependencies); + } + + return { incidentDir, incidentId, summary }; +} + +export async function main(argv, io = {}) { + const stdout = io.stdout ?? process.stdout; + const stderr = io.stderr ?? process.stderr; + try { + const result = await collectIncident({ argv }); + if (result.help !== undefined) { + stdout.write(`${result.help}\n`); + return 0; + } + stdout.write(`Incident bundle directory: ${result.incidentDir}\n`); + stdout.write(`Compression command: ${result.summary.compression.command}\n`); + return 0; + } catch (error) { + stderr.write(`Incident collection failed: ${publicErrorMessage(error)}\n${usage()}\n`); + return 1; + } +} + +function requireValue(argv, index, option) { + const value = argv[index]; + if (value === undefined || value === "" || value.startsWith("--")) { + throw new Error(`Missing value for ${option}`); + } + return value; +} + +function resolveIncidentPaths({ parsed, envLogRoot, root }) { + const logRootSource = parsed.logRoot !== undefined + ? "cli" + : envLogRoot !== undefined + ? "env:ICKB_BOT_LOG_ROOT" + : "default:log"; + const paths = resolveLauncherPaths({ + cliLogRoot: parsed.logRoot, + envLogRoot, + logDir: parsed.logDir, + network: parsed.network, + root, + }); + return { + ...paths, + logRootSource, + network: parsed.network ?? inferNetwork(paths.logDir), + }; +} + +function inferNetwork(logDir) { + const name = basename(logDir); + return supportedNetworks.has(name) ? name : null; +} + +function createSummary({ createdAt, logDir, logRoot, logRootSource, network, since, until }) { + return { + version: 1, + scriptVersion, + createdAt: createdAt.toISOString(), + window: { + since: since.toISOString(), + until: until.toISOString(), + }, + logRoot, + logRootSource, + logDir, + network, + sourceFiles: [], + sources: {}, + botEvents: { + countsByType: {}, + failureReasons: {}, + firstTimestamp: null, + lastTimestamp: null, + skipReasons: {}, + txHashesByOutcome: {}, + }, + launches: { + countsByType: {}, + exitCodes: {}, + firstTimestamp: null, + lastTimestamp: null, + signals: {}, + }, + stderr: { + firstTimestamp: null, + lastTimestamp: null, + }, + }; +} + +async function openSourceHandle(path, label, dependencies) { + let stat; + try { + stat = await (dependencies.lstat ?? lstat)(path); + } catch (error) { + if (error?.code === "ENOENT") { + return null; + } + throw error; + } + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked source log file: ${path}`); + } + if (!stat.isFile()) { + throw new Error(`Source log is not a regular file: ${path}`); + } + + let handle; + try { + handle = await (dependencies.open ?? open)(path, readOnlyNoFollow); + const opened = await handle.stat(); + if (!opened.isFile()) { + throw new Error(`Source log is not a regular file: ${path}`); + } + return handle; + } catch (error) { + await handle?.close().catch(() => undefined); + if (error?.code === "ELOOP") { + throw new Error(`Refusing symlinked source log file: ${path}`); + } + throw new Error(`Unable to read ${label}: ${publicErrorMessage(error)}`); + } +} + +async function processSourceLines(path, label, dependencies, onLine) { + const handle = await openSourceHandle(path, label, dependencies); + if (handle === null) { + return false; + } + + try { + let lineNumber = 0; + let pending = ""; + for await (const chunk of handle.readableWebStream({ type: "bytes" })) { + pending += Buffer.from(chunk).toString("utf8"); + const parts = pending.split("\n"); + pending = parts.pop() ?? ""; + for (const part of parts) { + lineNumber += 1; + onLine(`${part}\n`, lineNumber); + } + } + if (pending !== "") { + onLine(pending, lineNumber + 1); + } + return true; + } finally { + await handle.close().catch(() => undefined); + } +} + +async function filterJsonSource(path, sourceName, since, until, summary, kind, dependencies) { + const selected = []; + const stats = emptySourceStats(); + + const found = await processSourceLines(path, sourceName, dependencies, (line, lineNumber) => { + if (line.trim() === "") { + stats.emptyLines += 1; + return; + } + stats.totalLines += 1; + + let record; + try { + record = JSON.parse(stripLineEnding(line)); + } catch { + stats.malformedLines += 1; + return; + } + + const timestamp = parseRecordTimestamp(record?.timestamp); + if (timestamp === null) { + stats.undatedLines += 1; + return; + } + if (!timestampInWindow(timestamp, since, until)) { + stats.outsideWindowLines += 1; + return; + } + + selected.push(line.endsWith("\n") ? line : `${line}\n`); + stats.selectedLines += 1; + updateStatsTimestamps(stats, timestamp); + if (kind === "botEvents") { + summarizeBotEvent(record, summary, timestamp); + } else { + summarizeLaunch(record, summary, timestamp); + } + }); + if (!found) { + return null; + } + + return { stats, text: selected.join("") }; +} + +async function filterStderrSource(path, sourceName, since, until, dependencies) { + const selected = []; + const stats = emptySourceStats(); + const undatedTail = []; + let selectedSinceLastTimestamp = false; + + const found = await processSourceLines(path, sourceName, dependencies, (line, lineNumber) => { + if (line.trim() === "") { + stats.emptyLines += 1; + return; + } + stats.totalLines += 1; + const timestamp = parseTextTimestamp(line); + if (timestamp === null) { + stats.undatedLines += 1; + rememberUndatedTail(undatedTail, { line }, stderrUndatedLineLimit); + if (selectedSinceLastTimestamp) { + appendSelectedStderrLine(selected, stats, line); + stats.selectedUndatedLines += 1; + } + return; + } + stats.timestampedLines += 1; + selectedSinceLastTimestamp = false; + if (!timestampInWindow(timestamp, since, until)) { + stats.outsideWindowLines += 1; + return; + } + + appendSelectedStderrLine(selected, stats, line); + selectedSinceLastTimestamp = true; + updateStatsTimestamps(stats, timestamp); + }); + if (!found) { + return null; + } + + if (selected.length === 0 && stats.undatedLines > 0 && stats.timestampedLines === 0) { + for (const { line } of undatedTail) { + appendSelectedStderrLine(selected, stats, line); + } + stats.selectedUndatedLines = undatedTail.length; + stats.undatedTailIncluded = true; + stats.undatedTailLimit = stderrUndatedLineLimit; + } + + return { stats, text: selected.join("") }; +} + +function emptySourceStats() { + return { + emptyLines: 0, + firstTimestamp: null, + lastTimestamp: null, + malformedLines: 0, + outsideWindowLines: 0, + selectedLines: 0, + selectedUndatedLines: 0, + timestampedLines: 0, + totalLines: 0, + undatedTailIncluded: false, + undatedTailLimit: null, + undatedLines: 0, + }; +} + +function rememberUndatedTail(tail, entry, limit) { + tail.push(entry); + if (tail.length > limit) { + tail.shift(); + } +} + +function appendSelectedStderrLine(selected, stats, line) { + selected.push(line.endsWith("\n") ? line : `${line}\n`); + stats.selectedLines += 1; +} + +function stripLineEnding(line) { + const withoutNewline = line.endsWith("\n") ? line.slice(0, -1) : line; + return withoutNewline.endsWith("\r") ? withoutNewline.slice(0, -1) : withoutNewline; +} + +function parseRecordTimestamp(timestamp) { + if (typeof timestamp !== "string") { + return null; + } + const parsed = new Date(timestamp); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function parseTextTimestamp(line) { + const match = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})/u.exec(line); + if (match === null) { + return null; + } + const parsed = new Date(match[0]); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function timestampInWindow(timestamp, since, until) { + const value = timestamp.getTime(); + return since.getTime() <= value && value <= until.getTime(); +} + +function updateStatsTimestamps(stats, timestamp) { + const iso = timestamp.toISOString(); + if (stats.firstTimestamp === null || iso < stats.firstTimestamp) { + stats.firstTimestamp = iso; + } + if (stats.lastTimestamp === null || iso > stats.lastTimestamp) { + stats.lastTimestamp = iso; + } +} + +function summarizeBotEvent(record, summary, timestamp) { + if (record?.app !== "bot" || typeof record.type !== "string" || !record.type.startsWith("bot.")) { + return; + } + + increment(summary.botEvents.countsByType, record.type); + updateSummaryTimestamps(summary.botEvents, timestamp); + if (typeof record.outcome === "string" && typeof record.txHash === "string") { + addGroupedUnique(summary.botEvents.txHashesByOutcome, record.outcome, record.txHash); + } + if (record.type === "bot.decision.skipped") { + increment(summary.botEvents.skipReasons, stringReason(record.reason)); + } + if (record.type === "bot.transaction.failed" || record.type === "bot.iteration.failed") { + increment(summary.botEvents.failureReasons, failureReason(record)); + } +} + +function summarizeLaunch(record, summary, timestamp) { + if (record?.app !== "bot-launcher" || typeof record.type !== "string") { + return; + } + + increment(summary.launches.countsByType, record.type); + updateSummaryTimestamps(summary.launches, timestamp); + if (record.status !== undefined && record.status !== null) { + increment(summary.launches.exitCodes, String(record.status)); + } + if (record.signal !== undefined && record.signal !== null) { + increment(summary.launches.signals, String(record.signal)); + } +} + +function updateSummaryTimestamps(summary, timestamp) { + const iso = timestamp.toISOString(); + if (summary.firstTimestamp === null || iso < summary.firstTimestamp) { + summary.firstTimestamp = iso; + } + if (summary.lastTimestamp === null || iso > summary.lastTimestamp) { + summary.lastTimestamp = iso; + } +} + +function increment(counts, key) { + counts[key] = (counts[key] ?? 0) + 1; +} + +function addGroupedUnique(groups, key, value) { + groups[key] ??= []; + if (!groups[key].includes(value)) { + groups[key].push(value); + } +} + +function stringReason(value) { + return typeof value === "string" && value !== "" ? value : ""; +} + +function failureReason(record) { + if (record.type === "bot.iteration.failed" && record.retryBudgetExhausted === true) { + return "retry_budget_exhausted"; + } + if (typeof record.outcome === "string" && record.outcome !== "") { + return record.outcome; + } + if (typeof record.reason === "string" && record.reason !== "") { + return record.reason; + } + if (typeof record.error === "string" && record.error !== "") { + return record.error; + } + if (typeof record.error === "object" && record.error !== null) { + if (typeof record.error.message === "string" && record.error.message !== "") { + return record.error.message; + } + if (typeof record.error.name === "string" && record.error.name !== "") { + return record.error.name; + } + } + return ""; +} + +async function buildVersionMetadata(root, dependencies) { + const [rootPackage, botPackage] = await Promise.all([ + readPackage(join(root, "package.json")), + readPackage(join(root, "apps/bot/package.json")), + ]); + return { + script: { + name: "ickb-bot-collect-incident.mjs", + version: scriptVersion, + }, + nodeVersion: process.version, + package: { + packageManager: rootPackage?.packageManager ?? null, + private: rootPackage?.private === true, + }, + botPackage: botPackage === null + ? null + : { + name: typeof botPackage.name === "string" ? botPackage.name : null, + version: typeof botPackage.version === "string" ? botPackage.version : null, + }, + gitCommit: readGitCommit(root, dependencies), + }; +} + +async function readPackage(path) { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch { + return null; + } +} + +function readGitCommit(root, dependencies) { + const spawn = dependencies.spawnSync ?? spawnSync; + const result = spawn("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8", timeout: 5_000 }); + if (result.status !== 0) { + return null; + } + const commit = result.stdout.trim(); + return commit === "" ? null : commit; +} + +function captureSystemd(network, since, until, dependencies) { + if (!supportedNetworks.has(network)) { + return { + outputs: new Map(), + summary: { included: false, reason: "network unavailable for systemd unit derivation" }, + }; + } + + const unit = `ickb-bot-${network}.service`; + const commands = [ + { args: ["status", unit, "--no-pager", "--lines=0"], file: "systemd.status.txt", name: "systemctl status" }, + { args: ["cat", unit, "--no-pager"], file: "systemd.unit.txt", name: "systemctl cat" }, + { + args: [ + "-u", + unit, + "--since", + since.toISOString(), + "--until", + until.toISOString(), + "-n", + "200", + "--no-pager", + "--output", + "short-iso", + ], + command: "journalctl", + file: "systemd.journal.txt", + note: "last 200 entries inside the requested time window", + name: "journalctl", + }, + ]; + const outputs = new Map(); + const results = []; + const spawn = dependencies.spawnSync ?? spawnSync; + + for (const command of commands) { + const executable = command.command ?? "systemctl"; + const result = spawn(executable, command.args, { + encoding: "utf8", + maxBuffer: 1_000_000, + timeout: 5_000, + }); + const text = [result.stdout, result.stderr].filter(Boolean).join(result.stdout && result.stderr ? "\n" : ""); + results.push({ + command: executable, + args: command.args, + file: command.file, + name: command.name, + note: command.note ?? null, + signal: result.signal ?? null, + status: result.status ?? null, + error: result.error === undefined ? null : publicErrorMessage(result.error), + captured: text !== "", + }); + if (text !== "") { + outputs.set(command.file, text.endsWith("\n") ? text : `${text}\n`); + } + } + + return { + outputs, + summary: { + included: outputs.size > 0, + unit, + results, + }, + }; +} + +function buildIncidentId(createdAt, network) { + const stamp = createdAt.toISOString().replace(/[-:.]/gu, ""); + return `${stamp}-${network ?? "custom"}-${process.pid.toString(36)}-${randomBytes(3).toString("hex")}`; +} + +function shellQuote(value) { + return `'${value.replace(/'/gu, `'"'"'`)}'`; +} + +function incidentReadme(summary) { + return [ + `Incident: ${summary.incidentId}`, + `Window: ${summary.window.since} to ${summary.window.until}`, + `Log directory: ${summary.logDir}`, + "", + "Files are source-separated: bot.events.ndjson, bot.stderr.log, and launches.ndjson are never merged.", + "summary.json contains event counts, transaction outcomes, skip/failure reasons, exit codes, and source inclusion stats.", + "version.json contains collector, package, Node, and git metadata. Config files and environment dumps are intentionally not included.", + "", + `Compression command: ${summary.compression.command}`, + "", + ].join("\n"); +} + +async function prepareIncidentDirectory(logDir, incidentParent, incidentDir, dependencies) { + await assertRealDirectory(logDir, "log directory", dependencies); + await ensureDirectChildDirectory(incidentParent, "incident directory parent", dependencies); + await assertRealDirectory(incidentParent, "incident directory parent", dependencies); + await (dependencies.mkdir ?? mkdir)(incidentDir, { mode: 0o700 }); + await assertRealDirectory(incidentDir, "incident directory", dependencies); +} + +async function ensureDirectChildDirectory(path, label, dependencies) { + try { + await (dependencies.mkdir ?? mkdir)(path, { mode: 0o700 }); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + } + const stat = await (dependencies.lstat ?? lstat)(path); + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked ${label}: ${path}`); + } + if (!stat.isDirectory()) { + throw new Error(`${label} is not a directory: ${path}`); + } +} + +async function assertRealDirectory(path, label, dependencies) { + if (path === "") { + throw new Error(`Empty ${label} path`); + } + await assertNoSymlinkedPathComponents(path, label, dependencies); + const stat = await (dependencies.lstat ?? lstat)(path); + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked ${label}: ${path}`); + } + if (!stat.isDirectory()) { + throw new Error(`${label} is not a directory: ${path}`); + } + const resolved = await (dependencies.realpath ?? realpath)(path); + if (resolved !== path) { + throw new Error(`Resolved ${label} crosses a symlink: ${path}`); + } +} + +async function assertNoSymlinkedPathComponents(path, label, dependencies) { + const parsed = parse(path); + let current = parsed.root; + const parts = relative(parsed.root, path).split(sep).filter(Boolean); + for (const part of parts) { + current = join(current, part); + let stat; + try { + stat = await (dependencies.lstat ?? lstat)(current); + } catch (error) { + if (error?.code === "ENOENT") { + return; + } + throw error; + } + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked ${label} path: ${current}`); + } + } +} + +async function writeBundleFile(path, text, dependencies) { + const handle = await (dependencies.open ?? open)(path, writeNewFileFlags, 0o600); + try { + await handle.writeFile(text, "utf8"); + await handle.chmod(0o600); + } finally { + await handle.close().catch(() => undefined); + } +} + +function publicErrorMessage(error) { + return error instanceof Error ? error.message : "Unknown incident collector error"; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + process.exit(await main(process.argv.slice(2))); +} diff --git a/scripts/ickb-bot-collect-incident.test.mjs b/scripts/ickb-bot-collect-incident.test.mjs new file mode 100644 index 00000000..668f86c8 --- /dev/null +++ b/scripts/ickb-bot-collect-incident.test.mjs @@ -0,0 +1,492 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { collectIncident, parseArgs, parseTimeBound } from "./ickb-bot-collect-incident.mjs"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const collector = join(rootDir, "scripts", "ickb-bot-collect-incident.mjs"); +const canaryPrivateKey = `0x${"42".repeat(32)}`; + +test("parses collector arguments and relative time bounds", () => { + assert.deepEqual(parseArgs([ + "--log-root", + "var/logs", + "--network", + "testnet", + "--since", + "2h", + "--until", + "now", + "--no-systemd", + ]), { + includeSystemd: false, + logRoot: "var/logs", + network: "testnet", + since: "2h", + until: "now", + }); + + const now = new Date("2026-05-25T12:00:00.000Z"); + assert.equal(parseTimeBound("now", now).toISOString(), "2026-05-25T12:00:00.000Z"); + assert.equal(parseTimeBound("2h", now).toISOString(), "2026-05-25T10:00:00.000Z"); + assert.equal(parseTimeBound("2026-05-25T11:00:00Z", now).toISOString(), "2026-05-25T11:00:00.000Z"); + assert.throws(() => parseArgs(["--network", "devnet", "--since", "now", "--until", "now"]), /Invalid network/u); + assert.throws(() => parseArgs(["--network", "testnet", "--log-dir", "log/bot/testnet", "--since", "now", "--until", "now"]), /exactly one/u); + assert.throws(() => parseArgs(["--network", "testnet", "--until", "now"]), /Missing required --since/u); +}); + +test("collects a time-bounded source-separated incident bundle with summary counts", async () => { + const dir = await tempDir(); + try { + const logDir = await writeFixtureLogs(dir, { + events: [ + botEvent("2026-05-25T09:59:59.000Z", "bot.transaction.sent", { outcome: "broadcasted", txHash: txHash("01") }), + botEvent("2026-05-25T10:00:00.000Z", "bot.run.started", { bounded: true }), + "not-json\n", + JSON.stringify({ app: "execution", timestamp: "2026-05-25T10:05:00.000Z", message: "non-bot stdout" }) + "\n", + botEvent("2026-05-25T10:10:00.000Z", "bot.decision.skipped", { reason: "no_actions" }), + botEvent("2026-05-25T10:20:00.000Z", "bot.transaction.sent", { outcome: "broadcasted", txHash: txHash("02") }), + botEvent("2026-05-25T10:21:00.000Z", "bot.transaction.failed", { outcome: "timeout_after_broadcast", txHash: txHash("02") }), + botEvent("2026-05-25T11:00:00.000Z", "bot.iteration.failed", { error: { message: "fetch failed" }, retryable: true, terminal: false }), + botEvent("2026-05-25T10:59:59.500Z", "bot.iteration.failed", { + error: { message: "fetch failed" }, + retryable: true, + terminal: true, + retryableAttempts: 3, + maxRetryableAttempts: 3, + retryBudgetExhausted: true, + }), + botEvent("2026-05-25T11:00:01.000Z", "bot.transaction.committed", { outcome: "committed", txHash: txHash("03") }), + ], + launches: [ + launch("2026-05-25T09:00:00.000Z", "launcher.started", { status: null }), + launch("2026-05-25T10:00:00.000Z", "launcher.started", { status: null }), + launch("2026-05-25T10:30:00.000Z", "launcher.child.exited", { status: 2 }), + "{bad-json\n", + launch("2026-05-25T11:00:01.000Z", "launcher.child.exited", { status: 0 }), + ], + stderr: [ + "2026-05-25T09:00:00.000Z before window\n", + "2026-05-25T10:15:00.000Z runtime warning\n", + " at worker (bot.js:10:1)\n", + "undated warning\n", + "2026-05-25T11:00:01.000Z after window\n", + "outside continuation\n", + ], + }); + + const result = await collectIncident({ + argv: [ + "--log-root", + dir, + "--network", + "testnet", + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T11:00:00.000Z", + "--no-systemd", + ], + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + assert.match(result.incidentDir, /\/incidents\/20260525T120000000Z-testnet-/u); + assert.equal(await mode(result.incidentDir), 0o700); + assert.equal(await mode(join(result.incidentDir, "summary.json")), 0o600); + assert.equal(await readFile(join(result.incidentDir, "bot.stderr.log"), "utf8"), [ + "2026-05-25T10:15:00.000Z runtime warning\n", + " at worker (bot.js:10:1)\n", + "undated warning\n", + ].join("")); + + const eventBundle = await readFile(join(result.incidentDir, "bot.events.ndjson"), "utf8"); + assert.match(eventBundle, /bot\.run\.started/u); + assert.match(eventBundle, /non-bot stdout/u); + assert.doesNotMatch(eventBundle, /09:59:59|11:00:01|not-json/u); + + const summary = JSON.parse(await readFile(join(result.incidentDir, "summary.json"), "utf8")); + assert.equal(summary.logDir, logDir); + assert.deepEqual(summary.botEvents.countsByType, { + "bot.decision.skipped": 1, + "bot.iteration.failed": 2, + "bot.run.started": 1, + "bot.transaction.failed": 1, + "bot.transaction.sent": 1, + }); + assert.deepEqual(summary.botEvents.txHashesByOutcome, { + broadcasted: [txHash("02")], + timeout_after_broadcast: [txHash("02")], + }); + assert.deepEqual(summary.botEvents.skipReasons, { no_actions: 1 }); + assert.deepEqual(summary.botEvents.failureReasons, { "fetch failed": 1, retry_budget_exhausted: 1, timeout_after_broadcast: 1 }); + assert.deepEqual(summary.launches.exitCodes, { 2: 1 }); + assert.equal(summary.sources["bot.events.ndjson"].malformedLines, 1); + assert.equal(summary.sources["bot.events.ndjson"].selectedLines, 7); + assert.equal(summary.sources["bot.stderr.log"].undatedLines, 3); + assert.equal(summary.sources["bot.stderr.log"].selectedUndatedLines, 2); + assert.equal(summary.systemd.included, false); + assert.match(summary.compression.command, /tar -czf/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("includes a bounded tail when stderr has no timestamps", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: Array.from({ length: 205 }, (_, index) => `stack line ${index.toString()}\n`), + }); + + const result = await collectIncident({ + argv: commonArgs(dir), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + const stderr = await readFile(join(result.incidentDir, "bot.stderr.log"), "utf8"); + assert.doesNotMatch(stderr, /^stack line 0$/mu); + assert.match(stderr, /^stack line 5$/mu); + assert.match(stderr, /^stack line 204$/mu); + + const summary = JSON.parse(await readFile(join(result.incidentDir, "summary.json"), "utf8")); + assert.equal(summary.sources["bot.stderr.log"].selectedLines, 200); + assert.equal(summary.sources["bot.stderr.log"].selectedUndatedLines, 200); + assert.equal(summary.sources["bot.stderr.log"].undatedLines, 205); + assert.equal(summary.sources["bot.stderr.log"].undatedTailIncluded, true); + assert.equal(summary.sources["bot.stderr.log"].undatedTailLimit, 200); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("does not use undated stderr tail fallback when timestamped stderr is outside the window", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: [ + "2026-05-25T09:00:00.000Z before window\n", + "outside continuation\n", + ], + }); + + const result = await collectIncident({ + argv: commonArgs(dir), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + assert.equal(await readFile(join(result.incidentDir, "bot.stderr.log"), "utf8"), ""); + const summary = JSON.parse(await readFile(join(result.incidentDir, "summary.json"), "utf8")); + assert.equal(summary.sources["bot.stderr.log"].timestampedLines, 1); + assert.equal(summary.sources["bot.stderr.log"].undatedLines, 1); + assert.equal(summary.sources["bot.stderr.log"].selectedLines, 0); + assert.equal(summary.sources["bot.stderr.log"].undatedTailIncluded, false); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("accepts explicit contained log directory and environment log root", async () => { + const dir = await tempDir(); + try { + const logDir = await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [launch("2026-05-25T10:00:01.000Z", "launcher.child.exited", { status: 0 })], + stderr: ["2026-05-25T10:00:02.000Z ok\n"], + }); + + const result = await collectIncident({ + argv: [ + "--log-dir", + logDir, + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T10:00:02.000Z", + "--no-systemd", + ], + envLogRoot: dir, + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + const summary = JSON.parse(await readFile(join(result.incidentDir, "summary.json"), "utf8")); + assert.equal(summary.logRoot, dir); + assert.equal(summary.logRootSource, "env:ICKB_BOT_LOG_ROOT"); + assert.equal(summary.logDir, logDir); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("refuses log directories outside the resolved log root", async () => { + const dir = await tempDir(); + try { + await assert.rejects(() => collectIncident({ + argv: [ + "--log-root", + join(dir, "root"), + "--log-dir", + join(dir, "outside"), + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T11:00:00.000Z", + "--no-systemd", + ], + root: rootDir, + }), /inside the resolved log root/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("refuses symlinked log roots and log directories", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: [], + }); + const linkedRoot = join(dir, "linked-root"); + await symlink(dir, linkedRoot, "dir"); + await assert.rejects(() => collectIncident({ + argv: commonArgs(linkedRoot), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }), /symlinked log root path/u); + + const linkedLogDir = join(dir, "linked-log-dir"); + await symlink(join(dir, "bot", "testnet"), linkedLogDir, "dir"); + await assert.rejects(() => collectIncident({ + argv: [ + "--log-root", + dir, + "--log-dir", + linkedLogDir, + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T11:00:00.000Z", + "--no-systemd", + ], + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }), /symlinked log directory path/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("refuses symlinked source logs and incident directory parents", async () => { + const dir = await tempDir(); + try { + const logDir = await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: [], + }); + await rm(join(logDir, "bot.events.ndjson")); + await writeFile(join(dir, "target-events"), botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")); + await symlink(join(dir, "target-events"), join(logDir, "bot.events.ndjson")); + + await assert.rejects(() => collectIncident({ + argv: commonArgs(dir), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }), /symlinked source log file/u); + + await rm(join(logDir, "bot.events.ndjson")); + await writeFile(join(logDir, "bot.events.ndjson"), botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")); + await symlink(join(dir, "target-incidents"), join(logDir, "incidents"), "dir"); + + await assert.rejects(() => collectIncident({ + argv: commonArgs(dir), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }), /symlinked incident directory parent/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("includes selected producer log text without masking bundles", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [ + botEvent("2026-05-25T10:00:00.000Z", "bot.run.started", { + error: "public diagnostic", + }), + ], + launches: [], + stderr: [], + }); + + const result = await collectIncident({ + argv: commonArgs(dir), + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + const events = await readFile(join(result.incidentDir, "bot.events.ndjson"), "utf8"); + assert.match(events, /public diagnostic/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("produced bundles do not contain configured canary secrets", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [launch("2026-05-25T10:00:00.000Z", "launcher.started")], + stderr: ["2026-05-25T10:00:00.000Z public diagnostic\n"], + }); + + const result = spawnSync(process.execPath, [collector, ...commonArgs(dir)], { + cwd: rootDir, + encoding: "utf8", + env: { + ...process.env, + ICKB_TESTNET_BOT_PRIVATE_KEY: canaryPrivateKey, + }, + }); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, new RegExp(canaryPrivateKey, "u")); + const incidentDir = /^Incident bundle directory: (.+)$/mu.exec(result.stdout)?.[1]; + assert.ok(incidentDir); + + const produced = [ + "bot.events.ndjson", + "bot.stderr.log", + "launches.ndjson", + "README.txt", + "summary.json", + "version.json", + ]; + const bundleText = (await Promise.all( + produced.map((name) => readFile(join(incidentDir, name), "utf8")), + )).join("\n"); + assert.doesNotMatch(bundleText, new RegExp(canaryPrivateKey, "u")); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("captures systemd metadata into separate files when commands are available", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: [], + }); + const calls = []; + const result = await collectIncident({ + argv: [ + "--log-root", + dir, + "--network", + "testnet", + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T10:01:00.000Z", + ], + dependencies: { + spawnSync(command, args) { + calls.push([command, args]); + return { signal: null, status: 0, stderr: "", stdout: `${command} ${args.join(" ")}\n` }; + }, + }, + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + assert.deepEqual(calls.map(([command]) => command), ["git", "systemctl", "systemctl", "journalctl"]); + assert.deepEqual(calls[1], ["systemctl", ["status", "ickb-bot-testnet.service", "--no-pager", "--lines=0"]]); + assert.match(await readFile(join(result.incidentDir, "systemd.status.txt"), "utf8"), /ickb-bot-testnet\.service/u); + const summary = JSON.parse(await readFile(join(result.incidentDir, "summary.json"), "utf8")); + assert.equal(summary.systemd.included, true); + assert.equal(summary.systemd.unit, "ickb-bot-testnet.service"); + assert.equal(summary.systemd.results.find((entry) => entry.file === "systemd.journal.txt").note, "last 200 entries inside the requested time window"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +async function tempDir() { + return mkdtemp(join(tmpdir(), "ickb-bot-incident-")); +} + +async function writeFixtureLogs(root, { events, launches, stderr }) { + const logDir = join(root, "bot", "testnet"); + await mkdirp(logDir); + await writeFile(join(logDir, "bot.events.ndjson"), events.join(""), { mode: 0o600 }); + await writeFile(join(logDir, "launches.ndjson"), launches.join(""), { mode: 0o600 }); + await writeFile(join(logDir, "bot.stderr.log"), stderr.join(""), { mode: 0o600 }); + return logDir; +} + +async function mkdirp(dir) { + await mkdir(dir, { recursive: true, mode: 0o700 }); +} + +function botEvent(timestamp, type, fields = {}) { + return `${JSON.stringify({ + version: 1, + app: "bot", + chain: "testnet", + runId: "run-fixture", + iterationId: 1, + timestamp, + type, + ...fields, + })}\n`; +} + +function launch(timestamp, type, fields = {}) { + return `${JSON.stringify({ + version: 1, + app: "bot-launcher", + timestamp, + type, + status: null, + signal: null, + ...fields, + })}\n`; +} + +function txHash(byte) { + return `0x${byte.repeat(32)}`; +} + +function commonArgs(logRoot) { + return [ + "--log-root", + resolve(logRoot), + "--network", + "testnet", + "--since", + "2026-05-25T10:00:00.000Z", + "--until", + "2026-05-25T11:00:00.000Z", + "--no-systemd", + ]; +} + +async function mode(path) { + return (await stat(path)).mode & 0o777; +} diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs new file mode 100644 index 00000000..429a2450 --- /dev/null +++ b/scripts/ickb-bot-launcher.mjs @@ -0,0 +1,441 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises"; +import { basename, isAbsolute, join, parse, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const supportedNetworks = new Set(["testnet", "mainnet"]); +const noFollow = constants.O_NOFOLLOW ?? 0; +const logFileFlags = constants.O_APPEND | constants.O_CREAT | constants.O_WRONLY | noFollow; +const signalNames = ["SIGINT", "SIGTERM", "SIGHUP"]; + +export function usage() { + return `Usage: ickb-bot-launcher.mjs [--log-root PATH] (--network testnet|mainnet | --log-dir PATH) -- [args...]\n`; +} + +export function parseArgs(argv) { + if (argv.includes("-h") || argv.includes("--help")) { + return { help: true }; + } + + const separator = argv.indexOf("--"); + if (separator === -1) { + throw new Error("Missing -- before command"); + } + + const options = argv.slice(0, separator); + const command = argv.slice(separator + 1); + let logRoot; + let network; + let logDir; + + for (let index = 0; index < options.length; index += 1) { + const option = options[index]; + if (option === "--log-root") { + logRoot = requireValue(options, index, option); + index += 1; + } else if (option === "--network") { + network = requireValue(options, index, option); + index += 1; + } else if (option === "--log-dir") { + logDir = requireValue(options, index, option); + index += 1; + } else { + throw new Error(`Unknown option: ${option}`); + } + } + + if (command.length === 0 || command[0] === "") { + throw new Error("Missing child command"); + } + if (network !== undefined && !supportedNetworks.has(network)) { + throw new Error("Invalid network; expected testnet or mainnet"); + } + if ((network === undefined) === (logDir === undefined)) { + throw new Error("Specify exactly one of --network or --log-dir"); + } + + return { + command: command[0], + commandArgs: command.slice(1), + logDir, + logRoot, + network, + }; +} + +export function resolveLauncherPaths({ cliLogRoot, envLogRoot, logDir, network, root }) { + const logRoot = resolveConfiguredPath(cliLogRoot ?? envLogRoot ?? "log", root, "log root"); + const resolvedLogDir = logDir === undefined + ? join(logRoot, "bot", network) + : resolveConfiguredPath(logDir, root, "log directory"); + + assertContained(logRoot, resolvedLogDir, "log directory"); + return { logDir: resolvedLogDir, logRoot }; +} + +export async function runBotLauncher({ + argv = process.argv.slice(2), + env = process.env, + now = () => new Date(), + root = rootDir, + spawnProcess = spawn, + stderr = process.stderr, + stdout = process.stdout, +} = {}) { + let parsed; + try { + parsed = parseArgs(argv); + } catch (error) { + stderr.write(`ickb-bot-launcher: ${publicErrorMessage(error)}\n${usage()}`); + return { status: 1 }; + } + + if (parsed.help) { + stdout.write(usage()); + return { status: 0 }; + } + + const startTime = Date.now(); + let sinks; + let child; + let removeSignalHandlers = () => undefined; + + try { + const paths = resolveLauncherPaths({ + cliLogRoot: parsed.logRoot, + envLogRoot: env.ICKB_BOT_LOG_ROOT, + logDir: parsed.logDir, + network: parsed.network, + root, + }); + await prepareLogDirectory(paths.logRoot); + await prepareLogDirectory(paths.logDir); + await proveResolvedPath(paths.logRoot, "log root"); + await proveResolvedPath(paths.logDir, "log directory"); + + sinks = await openLogSinks(paths.logDir); + const packageInfo = await readBotPackageInfo(root); + const childCommand = safeCommandShape(parsed.command, parsed.commandArgs.length); + child = spawnProcess(parsed.command, parsed.commandArgs, { + cwd: root, + env, + stdio: ["inherit", "pipe", "pipe"], + }); + + removeSignalHandlers = forwardSignalsTo(child); + + await sinks.launches.writeLine({ + app: "bot-launcher", + childPid: child.pid ?? null, + command: childCommand, + elapsedMs: 0, + logDir: paths.logDir, + logRoot: paths.logRoot, + network: parsed.network ?? null, + nodeVersion: process.version, + package: packageInfo, + pid: process.pid, + repoRoot: root, + signal: null, + status: null, + timestamp: now().toISOString(), + type: "launcher.started", + version: 1, + }); + + const stdoutCopy = copyBytes(child.stdout, sinks.events, stdout); + const stderrCopy = copyBytes(child.stderr, sinks.stderr, stderr); + const childResult = await waitForChild(child); + const copyResult = await settleCopies(stdoutCopy, stderrCopy); + const elapsedMs = Date.now() - startTime; + + await sinks.launches.writeLine({ + app: "bot-launcher", + childPid: child.pid ?? null, + command: childCommand, + elapsedMs, + logDir: paths.logDir, + logRoot: paths.logRoot, + network: parsed.network ?? null, + nodeVersion: process.version, + package: packageInfo, + pid: process.pid, + signal: childResult.signal, + status: childResult.status, + timestamp: now().toISOString(), + type: copyResult === undefined ? "launcher.child.exited" : "launcher.io.failed", + version: 1, + }); + + await closeSinks(sinks); + sinks = undefined; + removeSignalHandlers(); + + if (copyResult !== undefined) { + stderr.write(`ickb-bot-launcher: ${publicErrorMessage(copyResult)}\n`); + return { status: 1 }; + } + if (childResult.signal !== null) { + return { signal: childResult.signal }; + } + return { status: childResult.status ?? 1 }; + } catch (error) { + removeSignalHandlers(); + if (child !== undefined && child.exitCode === null && !child.killed) { + child.kill("SIGTERM"); + } + if (sinks !== undefined) { + await closeSinks(sinks).catch(() => undefined); + } + stderr.write(`ickb-bot-launcher: ${publicErrorMessage(error)}\n`); + return { status: 1 }; + } +} + +function requireValue(argv, index, option) { + const value = argv[index + 1]; + if (value === undefined || value === "") { + throw new Error(`Missing value for ${option}`); + } + return value; +} + +function resolveConfiguredPath(value, root, label) { + if (value === "") { + throw new Error(`Empty ${label} path`); + } + return isAbsolute(value) ? resolve(value) : resolve(root, value); +} + +function assertContained(root, candidate, label) { + const relationship = relative(root, candidate); + if (relationship === "" || (relationship !== ".." && !relationship.startsWith(`..${sep}`) && !isAbsolute(relationship))) { + return; + } + throw new Error(`${label} must stay inside the resolved log root`); +} + +async function prepareLogDirectory(directory) { + const parsed = parse(directory); + let current = parsed.root; + await assertDirectory(current); + + const parts = relative(parsed.root, directory).split(sep).filter(Boolean); + for (const part of parts) { + current = join(current, part); + try { + await assertDirectory(current); + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + await mkdir(current, { mode: 0o700 }); + await assertDirectory(current); + } + } +} + +async function assertDirectory(path) { + const stat = await lstat(path); + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked log directory path: ${path}`); + } + if (!stat.isDirectory()) { + throw new Error(`Log directory path is not a directory: ${path}`); + } +} + +async function proveResolvedPath(path, label) { + const real = await realpath(path); + if (real !== path) { + throw new Error(`Resolved ${label} crosses a symlink`); + } +} + +async function openLogSinks(logDir) { + return { + events: await openLogSink(join(logDir, "bot.events.ndjson")), + launches: await openLogSink(join(logDir, "launches.ndjson")), + stderr: await openLogSink(join(logDir, "bot.stderr.log")), + }; +} + +async function openLogSink(path) { + try { + const stat = await lstat(path); + if (stat.isSymbolicLink()) { + throw new Error(`Refusing symlinked log file path: ${path}`); + } + if (!stat.isFile()) { + throw new Error(`Log file path is not a regular file: ${path}`); + } + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + } + + const handle = await open(path, logFileFlags, 0o600); + try { + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new Error(`Log file path is not a regular file: ${path}`); + } + await handle.chmod(0o600); + return new LogSink(handle); + } catch (error) { + await handle.close().catch(() => undefined); + throw error; + } +} + +class LogSink { + #pending = Promise.resolve(); + + constructor(handle) { + this.handle = handle; + } + + write(chunk) { + const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); + this.#pending = this.#pending.then(() => this.handle.appendFile(data)); + return this.#pending; + } + + writeLine(record) { + return this.write(`${JSON.stringify(record)}\n`); + } + + async close() { + await this.#pending; + await this.handle.close(); + } +} + +function copyBytes(readable, fileSink, tee) { + if (readable === null) { + return Promise.resolve(); + } + + let pending = Promise.resolve(); + return new Promise((resolvePromise, rejectPromise) => { + readable.on("data", (chunk) => { + readable.pause(); + pending = pending + .then(async () => { + await fileSink.write(chunk); + await writeToStream(tee, chunk); + }) + .then(() => readable.resume()); + }); + readable.once("end", () => pending.then(resolvePromise, rejectPromise)); + readable.once("error", rejectPromise); + }); +} + +function writeToStream(stream, chunk) { + return new Promise((resolvePromise) => { + let settled = false; + const finish = () => { + if (!settled) { + settled = true; + setImmediate(() => { + stream.off("error", finish); + resolvePromise(); + }); + } + }; + + stream.once("error", finish); + try { + stream.write(chunk, finish); + } catch { + finish(); + } + }); +} + +function waitForChild(child) { + return new Promise((resolvePromise) => { + let settled = false; + const settle = (result) => { + if (!settled) { + settled = true; + resolvePromise(result); + } + }; + child.once("error", (error) => settle({ error, signal: null, status: 1 })); + child.once("close", (status, signal) => settle({ signal, status })); + }); +} + +async function settleCopies(...copies) { + const results = await Promise.allSettled(copies); + const failed = results.find((result) => result.status === "rejected"); + return failed?.reason; +} + +function forwardSignalsTo(child) { + const handlers = signalNames.map((signal) => { + const handler = () => { + if (child.exitCode === null && !child.killed) { + child.kill(signal); + } + }; + process.once(signal, handler); + return [signal, handler]; + }); + return () => { + for (const [signal, handler] of handlers) { + process.off(signal, handler); + } + }; +} + +function safeCommandShape(command, argumentCount) { + return { + argumentCount, + arguments: Array.from({ length: argumentCount }, (_, index) => ({ + index, + value: "", + })), + executable: basename(command), + }; +} + +async function readBotPackageInfo(root) { + try { + const text = await readFile(join(root, "apps/bot/package.json"), "utf8"); + const parsed = JSON.parse(text); + return { + name: typeof parsed.name === "string" ? parsed.name : null, + version: typeof parsed.version === "string" ? parsed.version : null, + }; + } catch { + return null; + } +} + +async function closeSinks(sinks) { + await Promise.all([ + sinks.events.close(), + sinks.launches.close(), + sinks.stderr.close(), + ]); +} + +function publicErrorMessage(error) { + return error instanceof Error ? error.message : "Unknown launcher error"; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const result = await runBotLauncher(); + if (result.signal !== undefined) { + process.kill(process.pid, result.signal); + } else { + process.exit(result.status); + } +} diff --git a/scripts/ickb-bot-launcher.test.mjs b/scripts/ickb-bot-launcher.test.mjs new file mode 100644 index 00000000..23f8fb91 --- /dev/null +++ b/scripts/ickb-bot-launcher.test.mjs @@ -0,0 +1,301 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { Writable } from "node:stream"; +import { mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { parseArgs, resolveLauncherPaths, runBotLauncher } from "./ickb-bot-launcher.mjs"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const launcher = join(rootDir, "scripts", "ickb-bot-launcher.mjs"); +const canaryPrivateKey = `0x${"42".repeat(32)}`; + +test("parses launcher arguments", () => { + assert.deepEqual(parseArgs([ + "--log-root", + "var/bot-log", + "--network", + "testnet", + "--", + process.execPath, + "apps/bot/dist/index.js", + ]), { + command: process.execPath, + commandArgs: ["apps/bot/dist/index.js"], + logDir: undefined, + logRoot: "var/bot-log", + network: "testnet", + }); + + assert.throws(() => parseArgs(["--network", "devnet", "--", process.execPath]), /Invalid network/u); + assert.throws(() => parseArgs(["--network", "testnet", "--log-dir", "log/bot/testnet", "--", process.execPath]), /exactly one/u); + assert.throws(() => parseArgs(["--network", "testnet"]), /Missing --/u); +}); + +test("resolves explicit log root before runtime env and keeps log dirs contained", () => { + const root = resolve("/tmp/ickb-stack"); + assert.deepEqual(resolveLauncherPaths({ + cliLogRoot: "cli-log", + envLogRoot: "env-log", + network: "mainnet", + root, + }), { + logDir: join(root, "cli-log", "bot", "mainnet"), + logRoot: join(root, "cli-log"), + }); + + assert.deepEqual(resolveLauncherPaths({ + envLogRoot: "env-log", + network: "testnet", + root, + }), { + logDir: join(root, "env-log", "bot", "testnet"), + logRoot: join(root, "env-log"), + }); + + assert.throws(() => resolveLauncherPaths({ + cliLogRoot: join(root, "log"), + logDir: join(root, "outside"), + root, + }), /inside the resolved log root/u); +}); + +test("writes stdout, stderr, and launch metadata to separate append-only files", async () => { + const dir = await tempDir(); + try { + const first = runLauncher(dir, ["--network", "testnet"], [ + "-e", + "process.stdout.write('event-1\\n'); process.stderr.write('stderr-1\\n');", + ]); + assert.equal(first.status, 0, first.stderr); + assert.equal(first.stdout, "event-1\n"); + assert.equal(first.stderr, "stderr-1\n"); + + const second = runLauncher(dir, ["--network", "testnet"], [ + "-e", + "process.stdout.write('event-2\\n'); process.stderr.write('stderr-2\\n');", + ]); + assert.equal(second.status, 0, second.stderr); + + const logDir = join(dir, "bot", "testnet"); + assert.equal(await readFile(join(logDir, "bot.events.ndjson"), "utf8"), "event-1\nevent-2\n"); + assert.equal(await readFile(join(logDir, "bot.stderr.log"), "utf8"), "stderr-1\nstderr-2\n"); + assert.equal((await stat(join(logDir, "bot.events.ndjson"))).mode & 0o777, 0o600); + assert.equal((await stat(join(logDir, "bot.stderr.log"))).mode & 0o777, 0o600); + assert.equal((await stat(join(logDir, "launches.ndjson"))).mode & 0o777, 0o600); + + const launches = await readLaunches(logDir); + assert.equal(launches.length, 4); + assert.equal(launches[0].type, "launcher.started"); + assert.equal(launches[0].network, "testnet"); + assert.equal(launches[0].status, null); + assert.equal(launches[0].signal, null); + assert.equal(launches[0].elapsedMs, 0); + assert.equal(launches[0].command.executable, basenameOfNode()); + assert.equal(launches[0].command.argumentCount, 2); + assert.deepEqual(launches[0].command.arguments, [ + { index: 0, value: "" }, + { index: 1, value: "" }, + ]); + assert.equal(launches[1].type, "launcher.child.exited"); + assert.equal(launches[1].status, 0); + assert.equal(launches[1].signal, null); + assert.equal(launches[1].logRoot, dir); + assert.equal(launches[1].logDir, logDir); + assert.equal(launches[1].package.name, "@ickb/bot"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("copies stdout and stderr bytes without rewriting", async () => { + const dir = await tempDir(); + try { + const result = runLauncherBuffer(dir, ["--network", "testnet"], [ + "-e", + "process.stdout.write(Buffer.from([0, 255, 10])); process.stderr.write(Buffer.from([1, 254, 10]));", + ]); + assert.equal(result.status, 0, result.stderr.toString("utf8")); + assert.deepEqual(result.stdout, Buffer.from([0, 255, 10])); + assert.deepEqual(result.stderr, Buffer.from([1, 254, 10])); + + const logDir = join(dir, "bot", "testnet"); + assert.deepEqual(await readFile(join(logDir, "bot.events.ndjson")), Buffer.from([0, 255, 10])); + assert.deepEqual(await readFile(join(logDir, "bot.stderr.log")), Buffer.from([1, 254, 10])); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("refuses log directories that escape the resolved root", async () => { + const dir = await tempDir(); + try { + const result = runLauncher(dir, ["--log-dir", join(dir, "..", "escaped")], ["-e", ""]); + assert.equal(result.status, 1); + assert.match(result.stderr, /inside the resolved log root/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("refuses symlinks in the log root, log path parents, and log files", async () => { + const dir = await tempDir(); + try { + const symlinkRoot = join(dir, "root-link"); + await symlink(dir, symlinkRoot, "dir"); + const symlinkRootResult = runLauncher(symlinkRoot, ["--network", "testnet"], ["-e", ""]); + assert.equal(symlinkRootResult.status, 1); + assert.match(symlinkRootResult.stderr, /symlink/u); + + const botParent = join(dir, "bot"); + await symlink(join(dir, "target"), botParent, "dir"); + const symlinkParentResult = runLauncher(dir, ["--network", "testnet"], ["-e", ""]); + assert.equal(symlinkParentResult.status, 1); + assert.match(symlinkParentResult.stderr, /symlink/u); + await rm(botParent, { force: true, recursive: true }); + + const good = runLauncher(dir, ["--network", "testnet"], ["-e", ""]); + assert.equal(good.status, 0, good.stderr); + const eventPath = join(dir, "bot", "testnet", "bot.events.ndjson"); + await rm(eventPath); + await writeFile(join(dir, "target-events"), ""); + await symlink(join(dir, "target-events"), eventPath); + const symlinkFileResult = runLauncher(dir, ["--network", "testnet"], ["-e", ""]); + assert.equal(symlinkFileResult.status, 1); + assert.match(symlinkFileResult.stderr, /symlink/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("preserves child exit code 2 for systemd RestartPreventExitStatus", async () => { + const dir = await tempDir(); + try { + const result = runLauncher(dir, ["--network", "mainnet"], ["-e", "process.exit(2);"]); + assert.equal(result.status, 2); + + const launches = await readLaunches(join(dir, "bot", "mainnet")); + assert.equal(launches.at(-1).status, 2); + assert.equal(launches.at(-1).signal, null); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("tee failures do not override child exit semantics or file logs", async () => { + const dir = await tempDir(); + try { + const failingTee = new Writable({ + write(_chunk, _encoding, callback) { + callback(new Error("journald pipe closed")); + }, + }); + const result = await runBotLauncher({ + argv: [ + "--log-root", + dir, + "--network", + "testnet", + "--", + process.execPath, + "-e", + "process.stdout.write('event\\n'); process.stderr.write('stderr\\n'); process.exit(2);", + ], + root: rootDir, + stderr: failingTee, + stdout: failingTee, + }); + + assert.deepEqual(result, { status: 2 }); + const logDir = join(dir, "bot", "testnet"); + assert.equal(await readFile(join(logDir, "bot.events.ndjson"), "utf8"), "event\n"); + assert.equal(await readFile(join(logDir, "bot.stderr.log"), "utf8"), "stderr\n"); + const launches = await readLaunches(logDir); + assert.equal(launches.at(-1).status, 2); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("preserves child signal termination", async () => { + const dir = await tempDir(); + try { + const result = runLauncher(dir, ["--network", "testnet"], ["-e", "process.kill(process.pid, 'SIGTERM');"]); + assert.equal(result.signal, "SIGTERM"); + + const launches = await readLaunches(join(dir, "bot", "testnet")); + assert.equal(launches.at(-1).status, null); + assert.equal(launches.at(-1).signal, "SIGTERM"); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("launcher metadata does not expose configured canary secrets", async () => { + const dir = await tempDir(); + try { + const result = runLauncher(dir, ["--network", "testnet"], ["-e", "process.exit(0);", canaryPrivateKey], { + ICKB_TESTNET_BOT_PRIVATE_KEY: canaryPrivateKey, + }); + assert.equal(result.status, 0, result.stderr); + + const logDir = join(dir, "bot", "testnet"); + const produced = [ + result.stdout, + result.stderr, + await readFile(join(logDir, "bot.events.ndjson"), "utf8"), + await readFile(join(logDir, "bot.stderr.log"), "utf8"), + await readFile(join(logDir, "launches.ndjson"), "utf8"), + ].join("\n"); + assert.doesNotMatch(produced, new RegExp(canaryPrivateKey, "u")); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +async function tempDir() { + return mkdtemp(join(tmpdir(), "ickb-bot-launcher-")); +} + +function runLauncher(logRoot, launcherArgs, childArgs, extraEnv = {}) { + return spawnSync(process.execPath, [ + launcher, + "--log-root", + logRoot, + ...launcherArgs, + "--", + process.execPath, + ...childArgs, + ], { + cwd: rootDir, + encoding: "utf8", + env: { + ...process.env, + ...extraEnv, + }, + }); +} + +function runLauncherBuffer(logRoot, launcherArgs, childArgs) { + return spawnSync(process.execPath, [ + launcher, + "--log-root", + logRoot, + ...launcherArgs, + "--", + process.execPath, + ...childArgs, + ], { cwd: rootDir }); +} + +async function readLaunches(logDir) { + const text = await readFile(join(logDir, "launches.ndjson"), "utf8"); + return text.trim().split("\n").map((line) => JSON.parse(line)); +} + +function basenameOfNode() { + return process.execPath.split(/[\\/]/u).at(-1); +} diff --git a/scripts/ickb-bot-systemd-credential.sh b/scripts/ickb-bot-systemd-credential.sh index 86d50e4b..5330549e 100755 --- a/scripts/ickb-bot-systemd-credential.sh +++ b/scripts/ickb-bot-systemd-credential.sh @@ -34,7 +34,7 @@ if (config.chain !== expectedChain) fail(); process.stdout.write(JSON.stringify(config)); })(); function fail() { - process.stderr.write("Invalid bot config: expected exact JSON with matching chain, privateKey, rpcUrl, sleepIntervalSeconds, and optional maxIterations. Build @ickb/node-utils before running this helper.\n"); + process.stderr.write("Invalid bot config: expected exact JSON with matching chain, privateKey, optional rpcUrl, sleepIntervalSeconds, optional maxIterations, and optional maxRetryableAttempts. Build @ickb/node-utils before running this helper.\n"); process.exit(1); } ' "${repo_root}" "${expected_chain}" @@ -104,24 +104,33 @@ main() { local rpc_url local sleep_interval local max_iterations + local retryable_prompt + local max_retryable_attempts private_key=$(systemd-ask-password -n "iCKB ${network} bot private key:") - rpc_url=$(systemd-ask-password --echo=yes -n "iCKB ${network} RPC URL:") + rpc_url=$(systemd-ask-password -n "iCKB ${network} RPC URL [empty for CCC default]:") read -r -p "iCKB ${network} bot sleep interval seconds [60]: " sleep_interval sleep_interval=${sleep_interval:-60} read -r -p "iCKB ${network} bot max iterations [empty for unbounded]: " max_iterations - printf '%s\0%s\0%s\0%s\0%s' "${network}" "${private_key}" "${rpc_url}" "${sleep_interval}" "${max_iterations}" | + retryable_prompt="iCKB ${network} bot max retryable attempts [10]: " + read -r -p "${retryable_prompt}" max_retryable_attempts + printf '%s\0%s\0%s\0%s\0%s\0%s' "${network}" "${private_key}" "${rpc_url}" "${sleep_interval}" "${max_iterations}" "${max_retryable_attempts:-10}" | node -e ' const input = require("node:fs").readFileSync(0).toString("utf8").split("\0"); -const [chain, privateKey, rpcUrl, sleepIntervalSeconds, maxIterations] = input; +const [chain, privateKey, rpcUrl, sleepIntervalSeconds, maxIterations, maxRetryableAttempts] = input; const config = { chain, privateKey, - rpcUrl, sleepIntervalSeconds: Number(sleepIntervalSeconds), }; +if (rpcUrl !== "") { + config.rpcUrl = rpcUrl; +} if (maxIterations !== "") { config.maxIterations = Number(maxIterations); } +if (maxRetryableAttempts !== "") { + config.maxRetryableAttempts = Number(maxRetryableAttempts); +} process.stdout.write(JSON.stringify(config)); ' | validate_config "${network}" "${repo_root}" | diff --git a/scripts/ickb-bot-systemd-credential.test.mjs b/scripts/ickb-bot-systemd-credential.test.mjs index 9e8f0e6e..c95fd086 100644 --- a/scripts/ickb-bot-systemd-credential.test.mjs +++ b/scripts/ickb-bot-systemd-credential.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; import { join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; @@ -14,12 +15,23 @@ test("credential helper validation uses the shared runtime parser", () => { privateKey, rpcUrl: "http://127.0.0.1:8114/", sleepIntervalSeconds: 60, + maxRetryableAttempts: 10, }); const valid = validateConfig("testnet", config); assert.equal(valid.status, 0, valid.stderr); assert.equal(valid.stdout, config); + const defaultRpcConfig = JSON.stringify({ + chain: "testnet", + privateKey, + sleepIntervalSeconds: 60, + maxRetryableAttempts: 10, + }); + const validDefaultRpc = validateConfig("testnet", defaultRpcConfig); + assert.equal(validDefaultRpc.status, 0, validDefaultRpc.stderr); + assert.equal(validDefaultRpc.stdout, defaultRpcConfig); + const wrongChain = validateConfig("mainnet", config); assert.equal(wrongChain.status, 1); assert.match(wrongChain.stderr, /Invalid bot config/u); @@ -34,6 +46,19 @@ test("credential helper validation uses the shared runtime parser", () => { assert.doesNotMatch(invalidKey.stderr, /0x11/u); }); +test("credential helper does not echo RPC URL input", async () => { + const text = await readFile(script, "utf8"); + + assert.doesNotMatch(text, /systemd-ask-password --echo=yes/u); +}); + +test("credential helper prompts for retryable-attempt budget", async () => { + const text = await readFile(script, "utf8"); + + assert.match(text, /max retryable attempts/u); + assert.match(text, /maxRetryableAttempts/u); +}); + function validateConfig(network, input) { return spawnSync( "bash", diff --git a/scripts/ickb-bot-systemd-install.sh b/scripts/ickb-bot-systemd-install.sh index b6de7a21..1cf09cb6 100755 --- a/scripts/ickb-bot-systemd-install.sh +++ b/scripts/ickb-bot-systemd-install.sh @@ -3,6 +3,7 @@ set -euo pipefail usage() { printf 'Usage: %s [testnet|mainnet|all]\n' "${0##*/}" >&2 + printf 'Set ICKB_BOT_LOG_ROOT to bake an explicit launcher --log-root into generated units. Relative paths resolve from each deploy directory.\n' >&2 } require_root() { @@ -31,6 +32,90 @@ require_runtime() { } } +resolve_log_root_path() { + local deploy_dir=$1 + local configured_log_root=$2 + + node -e ' +const path = require("node:path"); +const deployDir = process.argv[1]; +const configuredLogRoot = process.argv[2]; +const resolved = path.isAbsolute(configuredLogRoot) + ? path.resolve(configuredLogRoot) + : path.resolve(deployDir, configuredLogRoot); +process.stdout.write(resolved); +' "${deploy_dir}" "${configured_log_root}" +} + +require_systemd_safe_path_arg() { + local value=$1 + + node -e ' +const value = process.argv[1] ?? ""; +const disallowed = new Set([String.fromCharCode(34), String.fromCharCode(39), "\\", "$", "%", ";"]); +if (value === "" || [...value].some((char) => char.charCodeAt(0) <= 32 || disallowed.has(char))) { + process.exit(1); +} +' "${value}" || { + printf 'ICKB_BOT_LOG_ROOT must be a non-empty systemd-safe path without whitespace, quotes, backslashes, semicolons, $, or %%.\n' >&2 + exit 1 + } +} + +safe_install_directory() { + local path=$1 + local mode=$2 + local uid=$3 + local gid=$4 + + node -e ' +const fs = require("node:fs"); +const path = require("node:path"); + +const target = process.argv[1]; +const mode = Number.parseInt(process.argv[2], 8); +const uid = Number(process.argv[3]); +const gid = Number(process.argv[4]); + +if (!path.isAbsolute(target) || !Number.isInteger(mode) || !Number.isInteger(uid) || !Number.isInteger(gid)) { + fail(`Invalid directory install arguments: ${target}`); +} + +const parsed = path.parse(target); +let current = parsed.root; +assertDirectory(current); + +for (const part of path.relative(parsed.root, target).split(path.sep).filter(Boolean)) { + current = path.join(current, part); + try { + assertDirectory(current); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + fs.mkdirSync(current, { mode }); + assertDirectory(current); + } +} + +fs.chmodSync(target, mode); +fs.chownSync(target, uid, gid); + +function assertDirectory(candidate) { + const stat = fs.lstatSync(candidate); + if (stat.isSymbolicLink()) { + fail(`Refusing symlinked directory path: ${candidate}`); + } + if (!stat.isDirectory()) { + fail(`Directory path is not a directory: ${candidate}`); + } +} + +function fail(message) { + process.stderr.write(`${message}\n`); + process.exit(1); +} +' "${path}" "${mode}" "${uid}" "${gid}" +} + install_network() { local network=$1 local user="ickb-bot-${network}" @@ -39,12 +124,28 @@ install_network() { local credential="/etc/ickb/credentials/ickb-bot-${network}-config.cred" local service="ickb-bot-${network}.service" local unit_path="/etc/systemd/system/${service}" + local configured_log_root=${ICKB_BOT_LOG_ROOT:-} + local launcher_log_root_args= + local log_root_path="${deploy_dir}/log" + + if [[ -n ${configured_log_root} ]]; then + require_systemd_safe_path_arg "${configured_log_root}" + log_root_path=$(resolve_log_root_path "${deploy_dir}" "${configured_log_root}") + launcher_log_root_args="--log-root ${log_root_path} " + fi if ! id -u "${user}" >/dev/null 2>&1; then useradd --system --create-home --user-group --shell /usr/sbin/nologin "${user}" fi + local user_id + local group_id + user_id=$(id -u "${user}") + group_id=$(id -g "${user}") - install -d -m 755 -o "${user}" -g "${user}" "${deploy_dir}" + safe_install_directory "${deploy_dir}" 755 "${user_id}" "${group_id}" + safe_install_directory "${log_root_path}" 755 0 0 + safe_install_directory "${log_root_path}/bot" 755 0 0 + safe_install_directory "${log_root_path}/bot/${network}" 700 "${user_id}" "${group_id}" install -d -m 700 /etc/ickb/credentials cat >"${unit_path}" <, build apps/bot, then enable the services.\n' } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/ickb-bot-systemd-install.test.mjs b/scripts/ickb-bot-systemd-install.test.mjs new file mode 100644 index 00000000..292b5177 --- /dev/null +++ b/scripts/ickb-bot-systemd-install.test.mjs @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, stat, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const installScript = join(rootDir, "scripts", "ickb-bot-systemd-install.sh"); + +test("systemd install units run the built bot through the file-log launcher", async () => { + const text = await readFile(installScript, "utf8"); + + assert.match(text, /ExecStart=\/usr\/bin\/node scripts\/ickb-bot-launcher\.mjs .*--network \$\{network\} -- \/usr\/bin\/node apps\/bot\/dist\/index\.js/u); + assert.doesNotMatch(text, /ExecStart=\/usr\/bin\/node apps\/bot\/dist\/index\.js/u); + assert.match(text, /Environment=BOT_CONFIG_FILE=%d\/\$\{credential_name\}/u); + assert.match(text, /LoadCredentialEncrypted=\$\{credential_name\}:\$\{credential\}/u); + assert.match(text, /RestartPreventExitStatus=2/u); + assert.match(text, /LimitCORE=0/u); + assert.match(text, /ProtectSystem=strict/u); + assert.match(text, /ReadWritePaths=\$\{log_root_path\}/u); +}); + +test("systemd install script avoids environment-specific hardcoded log roots", async () => { + const text = await readFile(installScript, "utf8"); + + assert.match(text, /log_root_path="\$\{deploy_dir\}\/log"/u); + assert.doesNotMatch(text, /\/var\/log/u); + assert.doesNotMatch(text, /logs\/live-supervisor/u); +}); + +test("systemd install script resolves configured log roots like the launcher", () => { + const absolute = resolveLogRoot("/opt/ickb-stack-testnet", "/srv/ickb/logs"); + assert.equal(absolute.status, 0, absolute.stderr); + assert.equal(absolute.stdout, "/srv/ickb/logs"); + + const relative = resolveLogRoot("/opt/ickb-stack-testnet", "runtime-log"); + assert.equal(relative.status, 0, relative.stderr); + assert.equal(relative.stdout, "/opt/ickb-stack-testnet/runtime-log"); +}); + +test("systemd install script refuses unsafe log-root unit arguments", () => { + const valid = validatePathArg("/srv/ickb-logs"); + assert.equal(valid.status, 0, valid.stderr); + + for (const value of ["", "/srv/ickb logs", "/srv/ickb;logs", "/srv/ickb%logs", "/srv/ickb$logs", "/srv/ickb\\logs"]) { + const invalid = validatePathArg(value); + assert.equal(invalid.status, 1, value); + assert.match(invalid.stderr, /ICKB_BOT_LOG_ROOT/u); + } +}); + +test("systemd install script creates log dirs without following symlinks", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-install-")); + try { + const uid = String(process.getuid?.() ?? 0); + const gid = String(process.getgid?.() ?? 0); + const createdPath = join(dir, "log", "bot", "testnet"); + const created = safeInstallDirectory(createdPath, "700", uid, gid); + assert.equal(created.status, 0, created.stderr); + assert.equal((await stat(createdPath)).mode & 0o777, 0o700); + + const linkPath = join(dir, "link"); + await symlink(dir, linkPath, "dir"); + const refused = safeInstallDirectory(join(linkPath, "bot"), "755", uid, gid); + assert.equal(refused.status, 1); + assert.match(refused.stderr, /Refusing symlinked directory path/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +function resolveLogRoot(deployDir, logRoot) { + return spawnSync( + "bash", + ["-c", "source \"$1\"; resolve_log_root_path \"$2\" \"$3\"", "bash", installScript, deployDir, logRoot], + { cwd: rootDir, encoding: "utf8" }, + ); +} + +function validatePathArg(value) { + return spawnSync( + "bash", + ["-c", "source \"$1\"; require_systemd_safe_path_arg \"$2\"", "bash", installScript, value], + { cwd: rootDir, encoding: "utf8" }, + ); +} + +function safeInstallDirectory(path, mode, uid, gid) { + return spawnSync( + "bash", + ["-c", "source \"$1\"; safe_install_directory \"$2\" \"$3\" \"$4\" \"$5\"", "bash", installScript, path, mode, uid, gid], + { cwd: rootDir, encoding: "utf8" }, + ); +} diff --git a/scripts/ickb-bot-systemd-update.sh b/scripts/ickb-bot-systemd-update.sh index dc6abc4e..c6c9cc14 100755 --- a/scripts/ickb-bot-systemd-update.sh +++ b/scripts/ickb-bot-systemd-update.sh @@ -61,8 +61,97 @@ require_clean_worktree() { local user_home=$2 local deploy_dir=$3 - run_as_service_user "${user}" "${user_home}" git -C "${deploy_dir}" diff --quiet - run_as_service_user "${user}" "${user_home}" git -C "${deploy_dir}" diff --cached --quiet + if [[ -n $(run_as_service_user "${user}" "${user_home}" git -C "${deploy_dir}" status --porcelain) ]]; then + printf 'Deploy checkout %s has local changes or untracked files; refusing to update.\n' "${deploy_dir}" >&2 + exit 1 + fi +} + +require_launcher_unit() { + local unit_path=$1 + local network=$2 + + if [[ ! -r ${unit_path} ]]; then + printf 'Service unit %s is missing or unreadable. Run scripts/ickb-bot-systemd-install.sh %s first.\n' "${unit_path}" "${network}" >&2 + exit 1 + fi + + local unit_text + unit_text=$(<"${unit_path}") + local credential_name="ickb-bot-${network}-config.json" + local credential="/etc/ickb/credentials/ickb-bot-${network}-config.cred" + local log_root + if ! log_root=$(unit_launcher_log_root "${unit_text}" "${network}"); then + printf 'Service unit %s is not wired for production launcher file logging and core-dump hardening. Run scripts/ickb-bot-systemd-install.sh %s before updating.\n' "${unit_path}" "${network}" >&2 + exit 1 + fi + if ! unit_has_directive "${unit_text}" "Environment" "BOT_CONFIG_FILE=%d/${credential_name}" || + ! unit_has_directive "${unit_text}" "LoadCredentialEncrypted" "${credential_name}:${credential}" || + ! unit_has_directive "${unit_text}" "RestartPreventExitStatus" "2" || + ! unit_has_directive "${unit_text}" "LimitCORE" "0" || + ! unit_has_directive "${unit_text}" "ReadWritePaths" "${log_root}"; then + printf 'Service unit %s is not wired for production launcher file logging and core-dump hardening. Run scripts/ickb-bot-systemd-install.sh %s before updating.\n' "${unit_path}" "${network}" >&2 + exit 1 + fi +} + +unit_has_directive() { + local unit_text=$1 + local key=$2 + local expected=$3 + local line + local in_service=0 + + while IFS= read -r line; do + line=${line%$'\r'} + [[ ${line} =~ ^[[:space:]]*($|#|\;) ]] && continue + if [[ ${line} =~ ^[[:space:]]*\[(.*)\][[:space:]]*$ ]]; then + [[ ${BASH_REMATCH[1]} == Service ]] && in_service=1 || in_service=0 + continue + fi + [[ ${in_service} -eq 1 ]] || continue + if [[ ${line} == "${key}=${expected}" ]]; then + return 0 + fi + done <<<"${unit_text}" + return 1 +} + +unit_launcher_log_root() { + local unit_text=$1 + local network=$2 + local default_log_root="/opt/ickb-stack-${network}/log" + local prefix="ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs " + local suffix="--network ${network} -- /usr/bin/node apps/bot/dist/index.js" + local with_log_root_prefix="${prefix}--log-root " + local suffix_with_separator=" ${suffix}" + local line + local in_service=0 + + while IFS= read -r line; do + line=${line%$'\r'} + [[ ${line} =~ ^[[:space:]]*($|#|\;) ]] && continue + if [[ ${line} =~ ^[[:space:]]*\[(.*)\][[:space:]]*$ ]]; then + [[ ${BASH_REMATCH[1]} == Service ]] && in_service=1 || in_service=0 + continue + fi + [[ ${in_service} -eq 1 ]] || continue + if [[ ${line} == "${prefix}${suffix}" ]]; then + printf '%s\n' "${default_log_root}" + return 0 + fi + if [[ ${line} == "${with_log_root_prefix}"*"${suffix_with_separator}" ]]; then + local rest=${line#"${with_log_root_prefix}"} + local log_root_length=$(( ${#rest} - ${#suffix_with_separator} )) + local log_root=${rest:0:log_root_length} + if [[ -n ${log_root} && ${log_root} == /* && ${log_root} != *[[:space:]]* ]]; then + printf '%s\n' "${log_root}" + return 0 + fi + return 1 + fi + done <<<"${unit_text}" + return 1 } main() { @@ -85,6 +174,7 @@ main() { local user="ickb-bot-${network}" local deploy_dir="/opt/ickb-stack-${network}" local service="ickb-bot-${network}.service" + local unit_path="/etc/systemd/system/${service}" local pnpm_bin pnpm_bin=$(command -v pnpm) local user_home @@ -92,6 +182,7 @@ main() { user_home=$(service_user_home "${user}") run_as_service_user "${user}" "${user_home}" git -C "${deploy_dir}" rev-parse --is-inside-work-tree >/dev/null require_clean_worktree "${user}" "${user_home}" "${deploy_dir}" + require_launcher_unit "${unit_path}" "${network}" run_as_service_user "${user}" "${user_home}" git -C "${deploy_dir}" pull --ff-only run_as_service_user "${user}" "${user_home}" "${pnpm_bin}" -C "${deploy_dir}" bot:install @@ -101,4 +192,6 @@ main() { systemctl --no-pager --full status "${service}" } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/ickb-bot-systemd-update.test.mjs b/scripts/ickb-bot-systemd-update.test.mjs new file mode 100644 index 00000000..c4e37002 --- /dev/null +++ b/scripts/ickb-bot-systemd-update.test.mjs @@ -0,0 +1,221 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const updateScript = join(rootDir, "scripts", "ickb-bot-systemd-update.sh"); + +test("systemd update accepts launcher-wired units", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update accepts launcher-wired units with explicit log roots", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true, logRoot: "/srv/ickb/log" })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update refuses stale direct-exec units", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: false })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /production launcher file logging/u); + assert.match(result.stderr, /ickb-bot-systemd-install\.sh testnet/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update refuses launcher units without core-dump hardening", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true, limitCore: false })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /core-dump hardening/u); + assert.match(result.stderr, /ickb-bot-systemd-install\.sh testnet/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update refuses launcher units with mismatched writable log roots", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true, logRoot: "/srv/ickb/log", readWritePath: "/tmp" })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /production launcher file logging/u); + assert.match(result.stderr, /ickb-bot-systemd-install\.sh testnet/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update refuses malformed launcher log-root arguments", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true, logRoot: "relative-log" })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /production launcher file logging/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update ignores commented launcher directives", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: false, commentedSpoof: true })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /production launcher file logging/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update ignores launcher directives outside the Service section", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: false, inactiveSectionSpoof: true })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 1); + assert.match(result.stderr, /production launcher file logging/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +test("systemd update checks unit wiring before mutating deploy checkout", async () => { + const text = await readFile(updateScript, "utf8"); + const guardIndex = text.indexOf('require_launcher_unit "${unit_path}" "${network}"'); + const pullIndex = text.indexOf('git -C "${deploy_dir}" pull --ff-only'); + + assert.notEqual(guardIndex, -1); + assert.notEqual(pullIndex, -1); + assert.ok(guardIndex < pullIndex); +}); + +test("systemd update refuses untracked files before pulling", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const fakeBin = join(dir, "bin"); + const logPath = join(dir, "git.log"); + await mkdir(fakeBin, { recursive: true }); + await writeFile(join(fakeBin, "runuser"), `#!/usr/bin/env bash +shift 3 +env_args=() +while [[ $# -gt 0 && $1 == *=* ]]; do + env_args+=("$1") + shift +done +exec env "\${env_args[@]}" "$@" +`); + await chmod(join(fakeBin, "runuser"), 0o755); + await writeFile(join(fakeBin, "git"), `#!/usr/bin/env bash +printf '%s\\n' "$*" >> ${JSON.stringify(logPath)} +if [[ $* == *' status --porcelain' ]]; then + printf '?? untracked.txt\\n' +fi +`); + await chmod(join(fakeBin, "git"), 0o755); + + const result = spawnSync( + "bash", + ["-c", "source \"$1\"; PATH=\"$2:$PATH\"; run_as_service_user() { command runuser -u \"$1\" -- env HOME=\"$2\" USER=\"$1\" LOGNAME=\"$1\" SHELL=/bin/bash \"${@:3}\"; }; require_clean_worktree ickb-bot-testnet /home/ickb /deploy", "bash", updateScript, fakeBin], + { cwd: rootDir, encoding: "utf8" }, + ); + + assert.equal(result.status, 1); + assert.match(result.stderr, /local changes or untracked files/u); + assert.doesNotMatch(await readFile(logPath, "utf8"), /pull --ff-only/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + +function requireLauncherUnit(unitPath, network) { + return spawnSync( + "bash", + ["-c", "source \"$1\"; require_launcher_unit \"$2\" \"$3\"", "bash", updateScript, unitPath, network], + { cwd: rootDir, encoding: "utf8" }, + ); +} + +function unitText({ + network, + launcher, + limitCore = true, + commentedSpoof = false, + inactiveSectionSpoof = false, + logRoot, + readWritePath, +}) { + const credentialName = `ickb-bot-${network}-config.json`; + const credential = `/etc/ickb/credentials/ickb-bot-${network}-config.cred`; + const launcherLogRoot = logRoot === undefined ? "" : `--log-root ${logRoot} `; + const execStart = launcher + ? `ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs ${launcherLogRoot}--network ${network} -- /usr/bin/node apps/bot/dist/index.js` + : `ExecStart=/usr/bin/node apps/bot/dist/index.js`; + const writablePath = readWritePath ?? logRoot ?? `/opt/ickb-stack-${network}/log`; + + const comments = commentedSpoof + ? `# ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs --network ${network} -- /usr/bin/node apps/bot/dist/index.js +# ReadWritePaths=/opt/ickb-stack-${network}/log +` + : ""; + const inactive = inactiveSectionSpoof + ? `[Unit] +ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs --network ${network} -- /usr/bin/node apps/bot/dist/index.js +ReadWritePaths=/opt/ickb-stack-${network}/log +[Install] +WantedBy=multi-user.target +` + : ""; + + return `${inactive}[Service] +${comments} +Environment=BOT_CONFIG_FILE=%d/${credentialName} +LoadCredentialEncrypted=${credentialName}:${credential} +${execStart} +RestartPreventExitStatus=2 +${limitCore ? "LimitCORE=0\n" : ""} +ReadWritePaths=${writablePath} +`; +} From 65ec0a4680b3ba32aee7c672e83c6186cf26852f Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 12:36:45 +0000 Subject: [PATCH 2/6] fix(bot): address ops review feedback --- scripts/ickb-bot-collect-incident.mjs | 4 +- scripts/ickb-bot-collect-incident.test.mjs | 57 +++++++++++++++++++++- scripts/ickb-bot-launcher.mjs | 2 +- scripts/ickb-bot-systemd-update.sh | 37 ++++++++++++-- scripts/ickb-bot-systemd-update.test.mjs | 28 ++++++++--- 5 files changed, 114 insertions(+), 14 deletions(-) diff --git a/scripts/ickb-bot-collect-incident.mjs b/scripts/ickb-bot-collect-incident.mjs index d4da4b2e..4c8e1bc0 100644 --- a/scripts/ickb-bot-collect-incident.mjs +++ b/scripts/ickb-bot-collect-incident.mjs @@ -336,8 +336,9 @@ async function processSourceLines(path, label, dependencies, onLine) { try { let lineNumber = 0; let pending = ""; + const decoder = new TextDecoder("utf-8"); for await (const chunk of handle.readableWebStream({ type: "bytes" })) { - pending += Buffer.from(chunk).toString("utf8"); + pending += decoder.decode(chunk, { stream: true }); const parts = pending.split("\n"); pending = parts.pop() ?? ""; for (const part of parts) { @@ -345,6 +346,7 @@ async function processSourceLines(path, label, dependencies, onLine) { onLine(`${part}\n`, lineNumber); } } + pending += decoder.decode(); if (pending !== "") { onLine(pending, lineNumber + 1); } diff --git a/scripts/ickb-bot-collect-incident.test.mjs b/scripts/ickb-bot-collect-incident.test.mjs index 668f86c8..16e121c3 100644 --- a/scripts/ickb-bot-collect-incident.test.mjs +++ b/scripts/ickb-bot-collect-incident.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, open, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import test from "node:test"; @@ -168,6 +168,40 @@ test("includes a bounded tail when stderr has no timestamps", async () => { } }); +test("decodes source log UTF-8 across chunk boundaries", async () => { + const dir = await tempDir(); + try { + const marker = "snowman ☃"; + const logDir = await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started", { marker })], + launches: [], + stderr: [], + }); + const eventPath = join(logDir, "bot.events.ndjson"); + const bytes = Buffer.from(await readFile(eventPath, "utf8")); + const splitAt = bytes.indexOf(Buffer.from("☃")) + 1; + + const result = await collectIncident({ + argv: commonArgs(dir), + dependencies: { + open(path, flags, mode) { + if (path === eventPath && typeof flags === "number") { + return splitReadHandle(path, [bytes.subarray(0, splitAt), bytes.subarray(splitAt)]); + } + return open(path, flags, mode); + }, + }, + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + const events = await readFile(join(result.incidentDir, "bot.events.ndjson"), "utf8"); + assert.match(events, new RegExp(marker, "u")); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + test("does not use undated stderr tail fallback when timestamped stderr is outside the window", async () => { const dir = await tempDir(); try { @@ -490,3 +524,24 @@ function commonArgs(logRoot) { async function mode(path) { return (await stat(path)).mode & 0o777; } + +async function splitReadHandle(path, chunks) { + return { + close() { + return Promise.resolve(); + }, + readableWebStream() { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }); + }, + stat() { + return stat(path); + }, + }; +} diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs index 429a2450..b5190d75 100644 --- a/scripts/ickb-bot-launcher.mjs +++ b/scripts/ickb-bot-launcher.mjs @@ -431,7 +431,7 @@ function publicErrorMessage(error) { return error instanceof Error ? error.message : "Unknown launcher error"; } -if (import.meta.url === `file://${process.argv[1]}`) { +if (fileURLToPath(import.meta.url) === resolve(process.argv[1] ?? "")) { const result = await runBotLauncher(); if (result.signal !== undefined) { process.kill(process.pid, result.signal); diff --git a/scripts/ickb-bot-systemd-update.sh b/scripts/ickb-bot-systemd-update.sh index c6c9cc14..72cf62e8 100755 --- a/scripts/ickb-bot-systemd-update.sh +++ b/scripts/ickb-bot-systemd-update.sh @@ -100,6 +100,7 @@ unit_has_directive() { local key=$2 local expected=$3 local line + local value local in_service=0 while IFS= read -r line; do @@ -110,13 +111,37 @@ unit_has_directive() { continue fi [[ ${in_service} -eq 1 ]] || continue - if [[ ${line} == "${key}=${expected}" ]]; then + if value=$(unit_directive_value "${line}" "${key}") && [[ ${value} == "${expected}" ]]; then return 0 fi done <<<"${unit_text}" return 1 } +unit_directive_value() { + local line=$1 + local expected_key=$2 + if [[ ${line} != *=* ]]; then + return 1 + fi + + local key=${line%%=*} + local value=${line#*=} + key=$(trim_unit_field "${key}") + value=$(trim_unit_field "${value}") + if [[ ${key} != "${expected_key}" ]]; then + return 1 + fi + printf '%s\n' "${value}" +} + +trim_unit_field() { + local value=$1 + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s\n' "${value}" +} + unit_launcher_log_root() { local unit_text=$1 local network=$2 @@ -126,6 +151,7 @@ unit_launcher_log_root() { local with_log_root_prefix="${prefix}--log-root " local suffix_with_separator=" ${suffix}" local line + local exec_start local in_service=0 while IFS= read -r line; do @@ -136,12 +162,15 @@ unit_launcher_log_root() { continue fi [[ ${in_service} -eq 1 ]] || continue - if [[ ${line} == "${prefix}${suffix}" ]]; then + if ! exec_start=$(unit_directive_value "${line}" "ExecStart"); then + continue + fi + if [[ ${exec_start} == "${prefix#ExecStart=}${suffix}" ]]; then printf '%s\n' "${default_log_root}" return 0 fi - if [[ ${line} == "${with_log_root_prefix}"*"${suffix_with_separator}" ]]; then - local rest=${line#"${with_log_root_prefix}"} + if [[ ${exec_start} == "${with_log_root_prefix#ExecStart=}"*"${suffix_with_separator}" ]]; then + local rest=${exec_start#"${with_log_root_prefix#ExecStart=}"} local log_root_length=$(( ${#rest} - ${#suffix_with_separator} )) local log_root=${rest:0:log_root_length} if [[ -n ${log_root} && ${log_root} == /* && ${log_root} != *[[:space:]]* ]]; then diff --git a/scripts/ickb-bot-systemd-update.test.mjs b/scripts/ickb-bot-systemd-update.test.mjs index c4e37002..d0d91e0e 100644 --- a/scripts/ickb-bot-systemd-update.test.mjs +++ b/scripts/ickb-bot-systemd-update.test.mjs @@ -35,6 +35,19 @@ test("systemd update accepts launcher-wired units with explicit log roots", asyn } }); +test("systemd update accepts spaces around service directive separators", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true, separator: " = " })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + test("systemd update refuses stale direct-exec units", async () => { const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); try { @@ -186,13 +199,14 @@ function unitText({ inactiveSectionSpoof = false, logRoot, readWritePath, + separator = "=", }) { const credentialName = `ickb-bot-${network}-config.json`; const credential = `/etc/ickb/credentials/ickb-bot-${network}-config.cred`; const launcherLogRoot = logRoot === undefined ? "" : `--log-root ${logRoot} `; const execStart = launcher - ? `ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs ${launcherLogRoot}--network ${network} -- /usr/bin/node apps/bot/dist/index.js` - : `ExecStart=/usr/bin/node apps/bot/dist/index.js`; + ? `ExecStart${separator}/usr/bin/node scripts/ickb-bot-launcher.mjs ${launcherLogRoot}--network ${network} -- /usr/bin/node apps/bot/dist/index.js` + : `ExecStart${separator}/usr/bin/node apps/bot/dist/index.js`; const writablePath = readWritePath ?? logRoot ?? `/opt/ickb-stack-${network}/log`; const comments = commentedSpoof @@ -211,11 +225,11 @@ WantedBy=multi-user.target return `${inactive}[Service] ${comments} -Environment=BOT_CONFIG_FILE=%d/${credentialName} -LoadCredentialEncrypted=${credentialName}:${credential} +Environment${separator}BOT_CONFIG_FILE=%d/${credentialName} +LoadCredentialEncrypted${separator}${credentialName}:${credential} ${execStart} -RestartPreventExitStatus=2 -${limitCore ? "LimitCORE=0\n" : ""} -ReadWritePaths=${writablePath} +RestartPreventExitStatus${separator}2 +${limitCore ? `LimitCORE${separator}0\n` : ""} +ReadWritePaths${separator}${writablePath} `; } From dfce0a5248799d146e18dea98d426bc1640ee859 Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 13:21:23 +0000 Subject: [PATCH 3/6] fix(bot): harden ops edge cases --- scripts/ickb-bot-collect-incident.mjs | 14 +++++---- scripts/ickb-bot-collect-incident.test.mjs | 27 +++++++++++++++++ scripts/ickb-bot-launcher.mjs | 7 ++++- scripts/ickb-bot-launcher.test.mjs | 34 ++++++++++++++++++++++ scripts/ickb-bot-systemd-update.sh | 4 +-- scripts/ickb-bot-systemd-update.test.mjs | 13 +++++++++ 6 files changed, 91 insertions(+), 8 deletions(-) diff --git a/scripts/ickb-bot-collect-incident.mjs b/scripts/ickb-bot-collect-incident.mjs index 4c8e1bc0..ca0b4d1a 100644 --- a/scripts/ickb-bot-collect-incident.mjs +++ b/scripts/ickb-bot-collect-incident.mjs @@ -632,13 +632,17 @@ async function readPackage(path) { } function readGitCommit(root, dependencies) { - const spawn = dependencies.spawnSync ?? spawnSync; - const result = spawn("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8", timeout: 5_000 }); - if (result.status !== 0) { + try { + const spawn = dependencies.spawnSync ?? spawnSync; + const result = spawn("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8", timeout: 5_000 }); + if (result.error !== undefined || result.status !== 0) { + return null; + } + const commit = result.stdout.trim(); + return commit === "" ? null : commit; + } catch { return null; } - const commit = result.stdout.trim(); - return commit === "" ? null : commit; } function captureSystemd(network, since, until, dependencies) { diff --git a/scripts/ickb-bot-collect-incident.test.mjs b/scripts/ickb-bot-collect-incident.test.mjs index 16e121c3..1cd0a9e0 100644 --- a/scripts/ickb-bot-collect-incident.test.mjs +++ b/scripts/ickb-bot-collect-incident.test.mjs @@ -461,6 +461,33 @@ test("captures systemd metadata into separate files when commands are available" } }); +test("keeps collecting incidents when git metadata lookup fails", async () => { + const dir = await tempDir(); + try { + await writeFixtureLogs(dir, { + events: [botEvent("2026-05-25T10:00:00.000Z", "bot.run.started")], + launches: [], + stderr: [], + }); + + const result = await collectIncident({ + argv: commonArgs(dir), + dependencies: { + spawnSync() { + throw new Error("git unavailable"); + }, + }, + now: () => new Date("2026-05-25T12:00:00.000Z"), + root: rootDir, + }); + + const version = JSON.parse(await readFile(join(result.incidentDir, "version.json"), "utf8")); + assert.equal(version.gitCommit, null); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + async function tempDir() { return mkdtemp(join(tmpdir(), "ickb-bot-incident-")); } diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs index b5190d75..a92738cf 100644 --- a/scripts/ickb-bot-launcher.mjs +++ b/scripts/ickb-bot-launcher.mjs @@ -124,6 +124,7 @@ export async function runBotLauncher({ env, stdio: ["inherit", "pipe", "pipe"], }); + const childResultPromise = waitForChild(child); removeSignalHandlers = forwardSignalsTo(child); @@ -148,7 +149,7 @@ export async function runBotLauncher({ const stdoutCopy = copyBytes(child.stdout, sinks.events, stdout); const stderrCopy = copyBytes(child.stderr, sinks.stderr, stderr); - const childResult = await waitForChild(child); + const childResult = await childResultPromise; const copyResult = await settleCopies(stdoutCopy, stderrCopy); const elapsedMs = Date.now() - startTime; @@ -178,6 +179,10 @@ export async function runBotLauncher({ stderr.write(`ickb-bot-launcher: ${publicErrorMessage(copyResult)}\n`); return { status: 1 }; } + if (childResult.error !== undefined) { + stderr.write(`ickb-bot-launcher: Failed to spawn child process: ${publicErrorMessage(childResult.error)}\n`); + return { status: 1 }; + } if (childResult.signal !== null) { return { signal: childResult.signal }; } diff --git a/scripts/ickb-bot-launcher.test.mjs b/scripts/ickb-bot-launcher.test.mjs index 23f8fb91..f47479db 100644 --- a/scripts/ickb-bot-launcher.test.mjs +++ b/scripts/ickb-bot-launcher.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; import { spawnSync } from "node:child_process"; import { Writable } from "node:stream"; import { mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; @@ -220,6 +221,39 @@ test("tee failures do not override child exit semantics or file logs", async () } }); +test("reports asynchronous child spawn errors", async () => { + const dir = await tempDir(); + try { + let stderr = ""; + const result = await runBotLauncher({ + argv: ["--log-root", dir, "--network", "testnet", "--", "missing-binary"], + root: rootDir, + spawnProcess() { + const child = new EventEmitter(); + child.exitCode = null; + child.killed = false; + child.kill = () => { + child.killed = true; + }; + child.stderr = null; + child.stdout = null; + process.nextTick(() => child.emit("error", new Error("spawn ENOENT"))); + return child; + }, + stderr: { + write(chunk) { + stderr += chunk; + }, + }, + }); + + assert.deepEqual(result, { status: 1 }); + assert.match(stderr, /Failed to spawn child process: spawn ENOENT/u); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + test("preserves child signal termination", async () => { const dir = await tempDir(); try { diff --git a/scripts/ickb-bot-systemd-update.sh b/scripts/ickb-bot-systemd-update.sh index 72cf62e8..814a1c0b 100755 --- a/scripts/ickb-bot-systemd-update.sh +++ b/scripts/ickb-bot-systemd-update.sh @@ -103,7 +103,7 @@ unit_has_directive() { local value local in_service=0 - while IFS= read -r line; do + while IFS= read -r line || [[ -n ${line} ]]; do line=${line%$'\r'} [[ ${line} =~ ^[[:space:]]*($|#|\;) ]] && continue if [[ ${line} =~ ^[[:space:]]*\[(.*)\][[:space:]]*$ ]]; then @@ -154,7 +154,7 @@ unit_launcher_log_root() { local exec_start local in_service=0 - while IFS= read -r line; do + while IFS= read -r line || [[ -n ${line} ]]; do line=${line%$'\r'} [[ ${line} =~ ^[[:space:]]*($|#|\;) ]] && continue if [[ ${line} =~ ^[[:space:]]*\[(.*)\][[:space:]]*$ ]]; then diff --git a/scripts/ickb-bot-systemd-update.test.mjs b/scripts/ickb-bot-systemd-update.test.mjs index d0d91e0e..63a74f79 100644 --- a/scripts/ickb-bot-systemd-update.test.mjs +++ b/scripts/ickb-bot-systemd-update.test.mjs @@ -48,6 +48,19 @@ test("systemd update accepts spaces around service directive separators", async } }); +test("systemd update reads unit files without trailing newlines", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ network: "testnet", launcher: true }).replace(/\n$/u, "")); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + test("systemd update refuses stale direct-exec units", async () => { const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); try { From f28296f07648547bdb6384f780f58c902ba2e0b9 Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 13:49:45 +0000 Subject: [PATCH 4/6] fix(bot): handle launcher write failures --- scripts/ickb-bot-launcher.mjs | 16 ++++++++---- scripts/ickb-bot-launcher.test.mjs | 42 ++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs index a92738cf..a74b1e95 100644 --- a/scripts/ickb-bot-launcher.mjs +++ b/scripts/ickb-bot-launcher.mjs @@ -297,7 +297,7 @@ async function openLogSink(path) { } } -class LogSink { +export class LogSink { #pending = Promise.resolve(); constructor(handle) { @@ -315,12 +315,15 @@ class LogSink { } async close() { - await this.#pending; - await this.handle.close(); + try { + await this.#pending; + } finally { + await this.handle.close(); + } } } -function copyBytes(readable, fileSink, tee) { +export function copyBytes(readable, fileSink, tee) { if (readable === null) { return Promise.resolve(); } @@ -334,7 +337,10 @@ function copyBytes(readable, fileSink, tee) { await fileSink.write(chunk); await writeToStream(tee, chunk); }) - .then(() => readable.resume()); + .then( + () => readable.resume(), + (error) => readable.destroy(error), + ); }); readable.once("end", () => pending.then(resolvePromise, rejectPromise)); readable.once("error", rejectPromise); diff --git a/scripts/ickb-bot-launcher.test.mjs b/scripts/ickb-bot-launcher.test.mjs index f47479db..5ed53312 100644 --- a/scripts/ickb-bot-launcher.test.mjs +++ b/scripts/ickb-bot-launcher.test.mjs @@ -1,14 +1,14 @@ import assert from "node:assert/strict"; import { EventEmitter } from "node:events"; import { spawnSync } from "node:child_process"; -import { Writable } from "node:stream"; +import { PassThrough, Writable } from "node:stream"; import { mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; -import { parseArgs, resolveLauncherPaths, runBotLauncher } from "./ickb-bot-launcher.mjs"; +import { LogSink, copyBytes, parseArgs, resolveLauncherPaths, runBotLauncher } from "./ickb-bot-launcher.mjs"; const rootDir = fileURLToPath(new URL("..", import.meta.url)); const launcher = join(rootDir, "scripts", "ickb-bot-launcher.mjs"); @@ -254,6 +254,44 @@ test("reports asynchronous child spawn errors", async () => { } }); +test("copy failures destroy the readable stream", async () => { + const readable = new PassThrough(); + const failure = new Error("disk full"); + const copy = copyBytes(readable, { + write() { + return Promise.reject(failure); + }, + }, new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + })); + + readable.write("event\n"); + + await assert.rejects(copy, /disk full/u); + assert.equal(readable.destroyed, true); +}); + +test("log sinks close file handles after pending write failures", async () => { + let closed = false; + const sink = new LogSink({ + appendFile() { + return Promise.reject(new Error("disk full")); + }, + close() { + closed = true; + return Promise.resolve(); + }, + }); + + const write = sink.write("event\n"); + + await assert.rejects(write, /disk full/u); + await assert.rejects(sink.close(), /disk full/u); + assert.equal(closed, true); +}); + test("preserves child signal termination", async () => { const dir = await tempDir(); try { From 3cbf8e382e8876b256466f00be31f1981a2e45d6 Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 14:26:15 +0000 Subject: [PATCH 5/6] fix(bot): tighten ops review edge cases --- scripts/ickb-bot-launcher.mjs | 5 +++- scripts/ickb-bot-launcher.test.mjs | 19 ++++++++++++++ scripts/ickb-bot-systemd-update.sh | 32 +++++++++++++++++++++++- scripts/ickb-bot-systemd-update.test.mjs | 26 +++++++++++++++++-- 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs index a74b1e95..a443e364 100644 --- a/scripts/ickb-bot-launcher.mjs +++ b/scripts/ickb-bot-launcher.mjs @@ -339,7 +339,10 @@ export function copyBytes(readable, fileSink, tee) { }) .then( () => readable.resume(), - (error) => readable.destroy(error), + (error) => { + rejectPromise(error); + readable.destroy(error); + }, ); }); readable.once("end", () => pending.then(resolvePromise, rejectPromise)); diff --git a/scripts/ickb-bot-launcher.test.mjs b/scripts/ickb-bot-launcher.test.mjs index 5ed53312..23f97dde 100644 --- a/scripts/ickb-bot-launcher.test.mjs +++ b/scripts/ickb-bot-launcher.test.mjs @@ -273,6 +273,25 @@ test("copy failures destroy the readable stream", async () => { assert.equal(readable.destroyed, true); }); +test("copy failures reject even when stream destroy does not emit error", async () => { + const readable = new PassThrough(); + const failure = new Error("disk full"); + readable.destroy = () => readable; + const copy = copyBytes(readable, { + write() { + return Promise.reject(failure); + }, + }, new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + })); + + readable.write("event\n"); + + await assert.rejects(copy, /disk full/u); +}); + test("log sinks close file handles after pending write failures", async () => { let closed = false; const sink = new LogSink({ diff --git a/scripts/ickb-bot-systemd-update.sh b/scripts/ickb-bot-systemd-update.sh index 814a1c0b..f5c1b41e 100755 --- a/scripts/ickb-bot-systemd-update.sh +++ b/scripts/ickb-bot-systemd-update.sh @@ -111,7 +111,7 @@ unit_has_directive() { continue fi [[ ${in_service} -eq 1 ]] || continue - if value=$(unit_directive_value "${line}" "${key}") && [[ ${value} == "${expected}" ]]; then + if value=$(unit_directive_value "${line}" "${key}") && unit_value_matches "${key}" "${value}" "${expected}"; then return 0 fi done <<<"${unit_text}" @@ -142,6 +142,36 @@ trim_unit_field() { printf '%s\n' "${value}" } +unit_value_matches() { + local key=$1 + local value=$2 + local expected=$3 + + [[ ${value} == "${expected}" ]] && return 0 + case "${key}" in + Environment|ReadWritePaths) unit_value_contains_token "${value}" "${expected}" ;; + *) return 1 ;; + esac +} + +unit_value_contains_token() { + local value=$1 + local expected=$2 + local token + local quoted + local -a tokens + + read -r -a tokens <<<"${value}" + for token in "${tokens[@]}"; do + if [[ ${token} == '"'*'"' && ${#token} -ge 2 ]]; then + quoted=${token#'"'} + token=${quoted%'"'} + fi + [[ ${token} == "${expected}" ]] && return 0 + done + return 1 +} + unit_launcher_log_root() { local unit_text=$1 local network=$2 diff --git a/scripts/ickb-bot-systemd-update.test.mjs b/scripts/ickb-bot-systemd-update.test.mjs index 63a74f79..90704f61 100644 --- a/scripts/ickb-bot-systemd-update.test.mjs +++ b/scripts/ickb-bot-systemd-update.test.mjs @@ -61,6 +61,24 @@ test("systemd update reads unit files without trailing newlines", async () => { } }); +test("systemd update accepts expected values inside multi-value directives", async () => { + const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); + try { + const unitPath = join(dir, "ickb-bot-testnet.service"); + await writeFile(unitPath, unitText({ + network: "testnet", + launcher: true, + extraEnvironment: "NODE_ENV=production", + extraReadWritePath: "/var/tmp", + })); + + const result = requireLauncherUnit(unitPath, "testnet"); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { force: true, recursive: true }); + } +}); + test("systemd update refuses stale direct-exec units", async () => { const dir = await mkdtemp(join(tmpdir(), "ickb-bot-systemd-update-")); try { @@ -212,6 +230,8 @@ function unitText({ inactiveSectionSpoof = false, logRoot, readWritePath, + extraEnvironment, + extraReadWritePath, separator = "=", }) { const credentialName = `ickb-bot-${network}-config.json`; @@ -221,6 +241,8 @@ function unitText({ ? `ExecStart${separator}/usr/bin/node scripts/ickb-bot-launcher.mjs ${launcherLogRoot}--network ${network} -- /usr/bin/node apps/bot/dist/index.js` : `ExecStart${separator}/usr/bin/node apps/bot/dist/index.js`; const writablePath = readWritePath ?? logRoot ?? `/opt/ickb-stack-${network}/log`; + const environmentValue = [`BOT_CONFIG_FILE=%d/${credentialName}`, extraEnvironment].filter(Boolean).join(" "); + const readWritePaths = [writablePath, extraReadWritePath].filter(Boolean).join(" "); const comments = commentedSpoof ? `# ExecStart=/usr/bin/node scripts/ickb-bot-launcher.mjs --network ${network} -- /usr/bin/node apps/bot/dist/index.js @@ -238,11 +260,11 @@ WantedBy=multi-user.target return `${inactive}[Service] ${comments} -Environment${separator}BOT_CONFIG_FILE=%d/${credentialName} +Environment${separator}${environmentValue} LoadCredentialEncrypted${separator}${credentialName}:${credential} ${execStart} RestartPreventExitStatus${separator}2 ${limitCore ? `LimitCORE${separator}0\n` : ""} -ReadWritePaths${separator}${writablePath} +ReadWritePaths${separator}${readWritePaths} `; } From 6928115db29aa92e77a294d585957b8795474c0f Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 27 May 2026 14:38:07 +0000 Subject: [PATCH 6/6] fix(bot): reject ambiguous ops arguments --- scripts/ickb-bot-launcher.mjs | 2 +- scripts/ickb-bot-launcher.test.mjs | 1 + scripts/ickb-bot-systemd-install.sh | 2 +- scripts/ickb-bot-systemd-install.test.mjs | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/ickb-bot-launcher.mjs b/scripts/ickb-bot-launcher.mjs index a443e364..97c4f16d 100644 --- a/scripts/ickb-bot-launcher.mjs +++ b/scripts/ickb-bot-launcher.mjs @@ -202,7 +202,7 @@ export async function runBotLauncher({ function requireValue(argv, index, option) { const value = argv[index + 1]; - if (value === undefined || value === "") { + if (value === undefined || value === "" || value.startsWith("--")) { throw new Error(`Missing value for ${option}`); } return value; diff --git a/scripts/ickb-bot-launcher.test.mjs b/scripts/ickb-bot-launcher.test.mjs index 23f97dde..ee968d30 100644 --- a/scripts/ickb-bot-launcher.test.mjs +++ b/scripts/ickb-bot-launcher.test.mjs @@ -32,6 +32,7 @@ test("parses launcher arguments", () => { }); assert.throws(() => parseArgs(["--network", "devnet", "--", process.execPath]), /Invalid network/u); + assert.throws(() => parseArgs(["--log-root", "--network", "testnet", "--", process.execPath]), /Missing value for --log-root/u); assert.throws(() => parseArgs(["--network", "testnet", "--log-dir", "log/bot/testnet", "--", process.execPath]), /exactly one/u); assert.throws(() => parseArgs(["--network", "testnet"]), /Missing --/u); }); diff --git a/scripts/ickb-bot-systemd-install.sh b/scripts/ickb-bot-systemd-install.sh index 1cf09cb6..d0c20e60 100755 --- a/scripts/ickb-bot-systemd-install.sh +++ b/scripts/ickb-bot-systemd-install.sh @@ -52,7 +52,7 @@ require_systemd_safe_path_arg() { node -e ' const value = process.argv[1] ?? ""; -const disallowed = new Set([String.fromCharCode(34), String.fromCharCode(39), "\\", "$", "%", ";"]); +const disallowed = new Set([String.fromCharCode(34), String.fromCharCode(39), "\\", "$", "%", ";", String.fromCharCode(96)]); if (value === "" || [...value].some((char) => char.charCodeAt(0) <= 32 || disallowed.has(char))) { process.exit(1); } diff --git a/scripts/ickb-bot-systemd-install.test.mjs b/scripts/ickb-bot-systemd-install.test.mjs index 292b5177..9ee07e06 100644 --- a/scripts/ickb-bot-systemd-install.test.mjs +++ b/scripts/ickb-bot-systemd-install.test.mjs @@ -44,7 +44,7 @@ test("systemd install script refuses unsafe log-root unit arguments", () => { const valid = validatePathArg("/srv/ickb-logs"); assert.equal(valid.status, 0, valid.stderr); - for (const value of ["", "/srv/ickb logs", "/srv/ickb;logs", "/srv/ickb%logs", "/srv/ickb$logs", "/srv/ickb\\logs"]) { + for (const value of ["", "/srv/ickb logs", "/srv/ickb;logs", "/srv/ickb%logs", "/srv/ickb$logs", "/srv/ickb\\logs", "/srv/ickb`logs"]) { const invalid = validatePathArg(value); assert.equal(invalid.status, 1, value); assert.match(invalid.stderr, /ICKB_BOT_LOG_ROOT/u);