diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..4d72232 --- /dev/null +++ b/.env.template @@ -0,0 +1,6 @@ +# Optional local native binary overrides for `just test-e2e`. +# Leave commented to infer or privately install the pinned hosts. +# V1 must be OpenCode 1.18.34; V2 must be OpenCode 2.0.15. + +# VIMCODE_E2E_V1_BIN=/absolute/path/to/opencode-v1 +# VIMCODE_E2E_V2_BIN=/absolute/path/to/opencode-v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf3b856..446d947 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,12 +2,23 @@ name: ci on: pull_request: branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.2' - uses: extractions/setup-just@v2 - uses: actions/setup-python@v5 with: @@ -15,3 +26,153 @@ jobs: - run: just install - run: just check - run: just compat-unit + + test-int: + name: Harness integration tests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.2' + - uses: extractions/setup-just@v2 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: just install + - name: Install process inspection tools + run: sudo apt-get update && sudo apt-get install -y procps + - name: Harness unit and integration checks + shell: bash + run: just test-int 2>&1 | tee "$RUNNER_TEMP/test-int.log" + - name: Upload harness failure log + if: failure() + uses: actions/upload-artifact@v4 + with: + name: test-int-failure + path: ${{ runner.temp }}/test-int.log + if-no-files-found: warn + retention-days: 7 + + test-e2e: + name: E2E tests (${{ matrix.host }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - host: v1 + version: '1.18.34' + - host: v2 + version: '2.0.15' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.2' + - uses: extractions/setup-just@v2 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: just install + - name: Install process and archive tools + run: sudo apt-get update && sudo apt-get install -y procps tar curl + - name: Install and verify pinned native host + shell: bash + env: + HOST: ${{ matrix.host }} + VERSION: ${{ matrix.version }} + run: | + prefix="$RUNNER_TEMP/opencode-$HOST" + mkdir -p "$prefix/native" + if [[ "$HOST" == v1 ]]; then + curl --fail --location --retry 3 --output "$prefix/archive.tar.gz" \ + "https://github.com/anomalyco/opencode/releases/download/v$VERSION/opencode-linux-x64-baseline.tar.gz" + echo "24b0d458d21ef548b2752166303defcf7f4945b049fb4876ab78dfaf86d81b27 $prefix/archive.tar.gz" | sha256sum --check + tar -xzf "$prefix/archive.tar.gz" -C "$prefix/native" + else + npm install --prefix "$prefix/native" --no-save --ignore-scripts \ + "@opencode/cli-linux-x64-baseline@$VERSION" + node - "$prefix/native/node_modules/@opencode/cli-linux-x64-baseline/package.json" "$VERSION" <<'NODE' + const fs = require('node:fs'); + const manifest = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); + if (manifest.name !== '@opencode/cli-linux-x64-baseline' || manifest.version !== process.argv[3]) { + throw new Error('Unexpected native host package identity'); + } + NODE + fi + mapfile -t binaries < <(find "$prefix/native" -type f -name opencode -perm /111) + [[ "${#binaries[@]}" == 1 ]] || { echo 'Expected one native host executable'; exit 1; } + binary="$(realpath "${binaries[0]}")" + actual="$("$binary" --version)" + actual="${actual#opencode v}" + [[ "$actual" == "$VERSION" ]] || { echo "Host version mismatch: $actual"; exit 1; } + echo "OPENCODE_E2E_BIN=$binary" >> "$GITHUB_ENV" + - name: Cache-installed plugin E2E + shell: bash + env: + HOST: ${{ matrix.host }} + VERSION: ${{ matrix.version }} + run: | + just test-e2e "$HOST" "$OPENCODE_E2E_BIN" "$VERSION" \ + "$RUNNER_TEMP/vimcode-e2e-$HOST" 2>&1 | tee "$RUNNER_TEMP/terminal-control-e2e-$HOST.log" + - name: Verify no-plugin negative control + shell: bash + env: + HOST: ${{ matrix.host }} + VERSION: ${{ matrix.version }} + run: | + set +e + VIMCODE_E2E_NO_PLUGIN=1 just test-e2e "$HOST" "$OPENCODE_E2E_BIN" "$VERSION" \ + "$RUNNER_TEMP/vimcode-negative-$HOST" 2>&1 | tee "$RUNNER_TEMP/terminal-control-negative-$HOST.log" + status=${PIPESTATUS[0]} + set -e + [[ "$status" == 1 ]] || { echo "Expected negative-control exit 1, got $status"; exit 1; } + node - "$RUNNER_TEMP/vimcode-negative-$HOST/receipt.json" "$HOST" "$VERSION" <<'NODE' + const assert = require('node:assert/strict'); + const receipt = require(process.argv[2]); + assert.equal(receipt.host, process.argv[3]); + assert.equal(receipt.version, process.argv[4]); + assert.equal(receipt.negativeControl, true); + assert.equal(receipt.providersDisabled, true); + assert.deepEqual(receipt.providerObservations, { + 'word-motion': 'Build · No provider selected Connect a provider' + }); + assert.deepEqual(receipt.cleanupErrors, []); + const failures = receipt.checks.filter(check => check.status === 'fail'); + assert.deepEqual(failures.map(check => check.name), ['word-home', 'word-motion']); + assert.deepEqual(failures[0].actual.lines, ['one two three0']); + NODE + - name: Upload synthetic E2E failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: terminal-control-e2e-${{ matrix.host }}-failure + path: | + ${{ runner.temp }}/terminal-control-e2e-${{ matrix.host }}.log + ${{ runner.temp }}/terminal-control-negative-${{ matrix.host }}.log + ${{ runner.temp }}/vimcode-negative-${{ matrix.host }}/receipt.json + ${{ runner.temp }}/vimcode-negative-${{ matrix.host }}/word-motion/*.txt + ${{ runner.temp }}/vimcode-negative-${{ matrix.host }}/word-motion/failure.json + ${{ runner.temp }}/vimcode-negative-${{ matrix.host }}/word-motion/failure.termctrl + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/receipt.json + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/cold-load/*.txt + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/cold-load/failure.json + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/cold-load/failure.termctrl + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/word-motion/*.txt + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/word-motion/failure.json + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/word-motion/failure.termctrl + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/quote-change/*.txt + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/quote-change/failure.json + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/quote-change/failure.termctrl + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/multiline-undo/*.txt + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/multiline-undo/failure.json + ${{ runner.temp }}/vimcode-e2e-${{ matrix.host }}/multiline-undo/failure.termctrl + if-no-files-found: warn + retention-days: 7 diff --git a/.gitignore b/.gitignore index 5200051..1ad8ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ .assets/ package-lock.json bun.lock +.env +.env.* +!.env.template diff --git a/AGENTS.md b/AGENTS.md index 6ddc74d..455bc5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,6 +92,7 @@ test/ child-session-navigation.test.ts (267) #79: isolated intercept and real OpenTUI keymap navigation regressions leader.test.ts (125 lines) Unit tests for leader key matching functions compat/ Optional real-host Python driver and test-only TUI fixture (not packaged) + e2e/ Terminal Control guards: shared harness, *.integration.ts and *.e2e.ts explicit entries (not unit-discovered or packaged) ``` **Data flow:** @@ -164,12 +165,14 @@ Text objects (`iw`/`aw`, the quote/bracket pairs, and the `iq`/`ib` aliases) rou just dev # Launch v1 with the plugin (uses OPENCODE_TUI_CONFIG=dev-tui.json) just dev2 # Launch pinned v2/local source with isolated .dev2/ state bun test # Run characterization tests -just check # Lint + tests (used in GitHub Actions) +just check # Lint + existing fast tests (used in GitHub Actions) +just test-int # On-demand SDK/process integration and parser helpers +just test-e2e # Both pinned PTY hosts (v1 1.18.34, v2 2.0.15); optional v1/v2 selector just compat-unit # Pure checks for the optional host harness just compat v1 /absolute/opencode 1.18.33 /canonical/empty/output # Isolated installed-artifact check ``` -The `dev-tui.json` config is picked up only by `just dev`. Running `opencode` normally in this directory does not load the plugin. `just dev2` runs `scripts/dev2.ts` with the tested v2 package (or an explicit binary), loads local source through the root `tui.ts` shim, and isolates on-disk settings/history under `.dev2/`. The shim/launcher are not distributed; package installs still resolve `exports["./tui"]`. Both dev commands retain arrow navigation and add `ctrl+x j` to open children/picker; v1 uses `h`/`l` to cycle and `k` to return, while v2 uses `h`/`k` and `j`/`l` in the Composer. Visual mode captures the focused editor on entry and re-anchors at the new editor's cursor on the next eligible key after a prompt switch; overlay keys do not change ownership. Horizontal normalization probes `editBuffer.getTextRange()` in host display-cell coordinates and uses lower `editorView` selection calls to retain the renderer's native anchor. The v2 disabled setting is per activation: external storage reconciliation applies on reload, while local `/vim` updates both controller and form guard immediately. `just compat-unit` runs in CI; live-host checks remain optional. +The `dev-tui.json` config is picked up only by `just dev`. Running `opencode` normally in this directory does not load the plugin. `just dev2` runs `scripts/dev2.ts` with the tested v2 package (or an explicit binary), loads local source through the root `tui.ts` shim, and isolates on-disk settings/history under `.dev2/`. The shim/launcher are not distributed; package installs still resolve `exports["./tui"]`. Both dev commands retain arrow navigation and add `ctrl+x j` to open children/picker; v1 uses `h`/`l` to cycle and `k` to return, while v2 uses `h`/`k` and `j`/`l` in the Composer. Visual mode captures the focused editor on entry and re-anchors at the new editor's cursor on the next eligible key after a prompt switch; overlay keys do not change ownership. Horizontal normalization probes `editBuffer.getTextRange()` in host display-cell coordinates and uses lower `editorView` selection calls to retain the renderer's native anchor. The v2 disabled setting is per activation: external storage reconciliation applies on reload, while local `/vim` updates both controller and form guard immediately. `just compat-unit` runs in CI; the Python/tmux live-host checks remain manual. Separate PR jobs run `just test-int` and `just test-e2e` on pinned v1/v2 hosts; harness parser checks are units, driver lifecycle checks are SDK/process integration, and real-host checks are product E2E. These entries use non-discovered `.unit.ts`/`.integration.ts`/`.e2e.ts` suffixes, so ordinary `bun test` and `just check` remain fast and do not launch the guards. `scripts/test-e2e.ts` infers canonical fresh OS-temp outputs, prints receipts, and privately bootstraps exact pins only when no matching PATH/override binary exists. Optional ignored `.env` recognizes only `VIMCODE_E2E_V1_BIN` / `VIMCODE_E2E_V2_BIN` paths (relative to repo root); process values win, CI ignores the file, and unknown/provider keys are never forwarded. Use `bun --env-file=/dev/null run test:e2e`, not bare `bun run`, on Bun 1.3.2 to prevent outer dotenv autoload; synthetic launcher integration tests verify suppression. The advanced `just test-e2e HOST BINARY VERSION OUTPUT` form and CI artifact paths remain supported. See `docs/test-e2e.md` for network/bootstrap limits (local v2 registry access blocked), Linux pending status, and the unchanged private SDK 1.2.1 cleanup boundary. The experimental v2 adapter uses a raw renderer key listener because v2's public keymap does not expose intercepts; see `docs/opencode-v2-poc.md` for configuration and `docs/opencode-v2-strategy.md` for tested alternatives and remaining compatibility gaps. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c54670..3c686bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Version ## [Unreleased] +### Added + +- Added separate PR CI guards via `just test-int` for harness integration and `just test-e2e` for cache-installed OpenCode v1/v2 E2E editing, with exact prompt checks and no-plugin controls; local runs remain on demand and outside the unit suite. +- Added no-argument `just test-e2e` defaults for both pinned hosts, optional single-host selection, fresh temp receipts, private pinned bootstrap, and ignored path-only `.env` binary overrides; existing explicit CI paths remain supported. + ## [0.19.1] — 2026-10-04 ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cee1613..0d7a34d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,11 +6,16 @@ just install # install deps just dev # launch OpenCode v1 with the plugin loaded just dev2 # launch pinned v2 with local source and separate .dev2/ state -just check # run lint + tests +just check # run lint + existing fast tests +just test # existing fast suite +just test-int # on-demand harness integration checks (including launcher) +just test-e2e # both pinned hosts; inferred private binaries/temp outputs ``` Running `opencode` directly in this directory won't load the plugin. `just dev` sets `OPENCODE_TUI_CONFIG=dev-tui.json`. `just dev2` uses npm's package runner (`npx`) for OpenCode 2.0.15 and keeps its on-disk settings/history separate under `.dev2/`. To use an existing v2 binary, run `just dev2 /absolute/path/to/opencode-v2` or set `OPENCODE_V2_BIN`. See [the v2 POC notes](docs/opencode-v2-poc.md#local-development). +`just test-e2e` runs v1@1.18.34 and v2@2.0.15; add `v1` or `v2` to select one. It prints fresh canonical temp receipt paths and may download/install the pinned hosts privately. Optional ignored `.env` path keys `VIMCODE_E2E_V1_BIN` / `VIMCODE_E2E_V2_BIN` use existing binaries; process settings win, CI ignores the file, and unknown keys are not forwarded. The advanced `HOST BINARY VERSION OUTPUT` form remains available. See [the runner settings and validation limits](docs/test-e2e.md) before using overrides (Linux runtime remains pending). + ## Adding a keybinding 1. Add the key check in the matching mode handler under `src/vim/` (`src/vim/normal.ts` for normal mode keys). @@ -39,7 +44,7 @@ Types match commit prefixes: `feat`, `fix`, `refactor`, `chore`, `test`, `docs`. 1. Create a branch: `git checkout -b feat/your-feature` 2. Make changes, run `just check` locally. It must pass with zero errors and zero warnings. 3. Push and open a PR against `main`. -4. CI runs `just check` and `just compat-unit`. Warnings are treated as errors; both checks must pass. Live-host compatibility checks remain manual. +4. CI runs `just check` and `just compat-unit`, plus `just test-int` for harness integration and `just test-e2e` for product E2E on pinned OpenCode v1/v2. Warnings and guard failures fail the build. Locally these guards run only on demand; the existing Python/tmux live-host compatibility harness remains manual. See [the guard commands and limits](docs/test-e2e.md). An owner must configure the new check names as required after their first successful Actions run. 5. PRs are squash-merged. The PR title becomes the commit message on `main`. ## Commit messages diff --git a/README.md b/README.md index 739e016..61b7869 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,8 @@ On v1, vimcode registers a host key intercept. The experimental v2 adapter uses 3. Open issues for bugs or missing keybindings 4. PRs welcome. See [CONTRIBUTING.md](./CONTRIBUTING.md) for dev setup and the release process. +Use `just test` for the fast suite, `just test-int` for harness integration and parser helpers, and **`just test-e2e` with no arguments** for both pinned OpenCode hosts (v1 **1.18.34**, v2 **2.0.15**). Optionally select `just test-e2e v1` or `v2`. The real-PTY guard infers fresh temp outputs, prints receipt paths, and bootstraps missing matching hosts privately (network required); optional ignored `.env` binary paths avoid bootstrap. PR builds run all three suites. The latter two stay outside fast-test discovery and run locally only on demand; see [path settings, advanced overrides, and validation limits](docs/test-e2e.md). + ## License MIT diff --git a/docs/test-e2e.md b/docs/test-e2e.md new file mode 100644 index 0000000..1c0f617 --- /dev/null +++ b/docs/test-e2e.md @@ -0,0 +1,102 @@ +# Terminal Control CI guards + +Terminal Control checks run automatically on pull requests against `main` and through Actions `workflow_dispatch`. Local runs are on demand only. They use Bun 1.3.2 and pinned `@kitlangton/terminal-control` 1.2.1, with its matching native PTY driver. No Vitest, tmux, observer plugin, or model is needed. + +## Suites and commands + +`bun test`, `npm test`, `just test`, and `just check` retain the existing fast suite, including the existing mocked plugin integration tests. They do not discover the new Terminal Control suites. The guard entry files use `.unit.ts`, `.integration.ts`, and `.e2e.ts` suffixes instead of Bun's automatic `.test.ts` discovery; the commands below pass their filenames explicitly. An explicitly requested guard fails on missing setup rather than silently skipping. + +```sh +just test # existing fast suite; no CLI/network startup +just test-int # harness integration, launcher, and parser checks +just test-e2e # BOTH pinned hosts: v1@1.18.34 then v2@2.0.15 +just test-e2e v1 # optionally select just one host (or v2) + +# Advanced CI/troubleshooting: absolute binary, exact PIN, NEW output directory: +just test-e2e v1 /absolute/opencode-v1 1.18.34 /private/tmp/vimcode-v1-new +just test-e2e v2 /absolute/opencode-v2 2.0.15 /private/tmp/vimcode-v2-new + +# Optional no-plugin control: expected exit 1, literal 0 at word-home: +VIMCODE_E2E_NO_PLUGIN=1 just test-e2e v1 +``` + +### Defaults and optional paths + +`scripts/test-e2e.ts` uses a fresh, canonical `os.tmpdir()` directory outside checkout/config ancestry, prints the run directory and each `receipt.json` path **before execution**, and preserves evidence on failure. Selected hosts run sequentially; any failure makes the final exit nonzero without hiding the other host. The v2 host uses `--standalone` and `cli.json`; v1 uses `tui.json`. + +A binary override is checked first; otherwise an `opencode` on PATH is reused only if its bounded `--version` probe exactly matches the pin. If not, the launcher provisions privately in that run directory, never globally or from `latest`. Provisioning needs network access: v1 downloads the official GitHub release and verifies SHA-256; v2 installs the exact native npm package with scripts disabled and checks manifest name/version. Both verify the resulting binary version. Downloads/installers/probes are bounded, use isolated HOME/XDG/cwd and no inherited credentials or npm configuration, and are not a network sandbox. Per-run installs avoid shared partial-cache/concurrency ownership; delete the printed run directory when evidence is no longer needed. + +Automatic bootstrap is verified for macOS arm64 and GNU/Linux x64 native shapes only (Linux uses baseline builds). Other macOS/Linux architectures require a matching native binary override rather than guessed downloads. Windows is unsupported by this guard. + +Optional, ignored repository `.env` settings are **paths only**: + +```dotenv +VIMCODE_E2E_V1_BIN=/absolute/path/to/opencode-v1 +VIMCODE_E2E_V2_BIN=/absolute/path/to/opencode-v2 +``` + +Only these two keys are recognized. Relative paths resolve against the repository root, then canonicalize; there is no shell expansion, version customization, output/cache-root setting, or dotenv dependency. Process environment settings take precedence. Invalid/empty overrides fail only for the selected host, without falling back or preventing another selected host from running; unused host settings are not validated. `CI` set to a nonempty value and fully explicit advanced invocations ignore the local file. Unknown keys/credentials never become runner settings or child environment; never store secrets for this runner. `.env.*` files are ignored by Git but are not read by the launcher. Missing paths/wrong versions fail actionably, never silently switch binaries. + +The equivalent package commands are `bun --env-file=/dev/null run test:int` and `bun --env-file=/dev/null run test:e2e [v1|v2]` (or `npm run test:e2e -- v1`). Bun **1.3.2 accepts but does not actually suppress autoload with `--no-env-file`**; synthetic fixtures verify that `--env-file=/dev/null` does. The guard recipes/scripts and product-test child use the latter, without enabling just-wide dotenv autoload. Use the flagged Bun command so its outer package runner cannot autoload a local file before the launcher enforces precedence/CI exclusion. + +Advanced output parents must exist and outputs must be empty and outside checkout/OpenCode config ancestry. Existing CI arguments and artifact paths are unchanged; the explicit version must still equal the selected host's pin. + +The suites test different boundaries: + +| Checks | Classification | Entry file | +| --- | --- | --- | +| Prompt parsing, borders, provider metadata, and cursor diagnostics | Unit tests of the harness: pure functions and fixed snapshots | `test/e2e/prompt.unit.ts` | +| SDK requests, timeouts, and owned-process cleanup | Harness integration tests: real SDK and OS processes with a synthetic driver | `test/e2e/driver.integration.ts` | +| Launcher selection, pins, paths, dotenv isolation, bounded probes, and aggregate/negative exits | Harness integration tests: synthetic settings and OS child processes, no host downloads | `test/e2e/launcher.integration.ts` | +| Installed plugin behavior in real OpenCode | Product E2E tests: real host, PTY, packaged vimcode, and keyboard input | `test/e2e/opencode.e2e.ts` | + +`just test-int` groups the SDK/process integration checks and supporting parser units under the **Harness integration tests** CI job. The parser units stay outside ordinary product-test discovery as part of the on-demand harness suite. Lifecycle checks cover stalled readiness, launch/shutdown responses, SDK abort, keyboard type/press requests through the real failure/finally path, unresolved cleanup retries, and driver-override rejection. They do not exercise vimcode or OpenCode. + +The four product E2E scenarios run against both host generations: + +- Cold package-cache installation and exact insertion of `cold package loaded`. +- `Escape`, `0`, `w`, `i`, `X` produces exactly `one Xtwo three`. +- `Escape`, `0`, `w`, `ci"`, `new` produces exactly `say "new" now`, including an intermediate empty-quote check. +- Multiline `dG` leaves exactly `alpha` plus an empty line; one `u` restores exactly `alpha`, `bravo`, `charlie`. + +## PR checks and host pins + +`.github/workflows/test.yml` keeps the existing `check` job and adds: + +- **Harness integration tests**, running `just test-int` on Ubuntu 24.04. +- **E2E tests (v1)**, running `just test-e2e` on pinned OpenCode **1.18.34**. +- **E2E tests (v2)**, running `just test-e2e` on pinned OpenCode **2.0.15**. + +The matrix has `fail-fast: false`, so one host failure does not hide the other. Failures remain failing checks; there is no `continue-on-error`. The workflow has read-only repository permissions, no `pull_request_target`, and no provider secrets. Jobs install tools independently and have bounded timeouts. PR updates cancel superseded workflow runs. + +The v1 job downloads the official Linux x64 baseline release asset and verifies its pinned SHA-256 before extraction. The v2 job installs the official `@opencode/cli-linux-x64-baseline@2.0.15` native package with scripts disabled and verifies its manifest. Both find exactly one executable and check its reported version. Hosts install into separate `RUNNER_TEMP` directories, never globally or from `latest`. The baseline x64 builds avoid relying on optional newer CPU instructions. + +Each E2E matrix job also runs a no-plugin negative control. It accepts only exit 1 with the exact recorded `word-home` failure (`one two three0`), the expected scenario failure, matching host/version, observed provider-disabled home state, and no cleanup errors. A missing binary, setup failure, or unrelated failure cannot count as a successful control. This case never sends Enter. The positive multiline case guards Enter with observed Vim cursor transitions. + +After the first successful GitHub run, an owner must make these check names required in branch protection if they should block merge. Workflow configuration alone does not change repository protection. No actual GitHub run or required-status configuration has been performed by this change. + +## Isolation and failure evidence + +Each E2E invocation packs the current source with `npm pack --ignore-scripts`, installs an npm-source tarball spec through the real host cache, and verifies the cached manifest and all source files byte-for-byte against the unpacked artifact. Tests and launchers are not packaged. `receipt.json` records host/binary/version, platform, packaged driver path, tarball SHA-256, per-file hashes, checkpoints, observed provider-disabled state per scenario, and cleanup errors. + +Driver and host receive isolated HOME, all XDG config/data/cache/state directories, a sandbox cwd, and no inherited provider credentials or config overrides. PATH contains only selected Bun/npm/Node tool directories, system directories, and first-priority clipboard stubs. Update checks are disabled. The isolated `opencode.json` sets `enabled_providers: []`, explicitly excluding even free OpenCode Zen models. Before any scenario inputs, the actual TUI must show the strict bordered home prompt with `Build · No provider selected Connect a provider`; only then does the receipt claim `providersDisabled: true` and record that scenario's observation. Provider-free empty prompts support `Ask` or the host's `Ask anything` placeholder. If the observed v1 connection dialog overlays startup, a bounded Escape dismisses it; no Enter or approval is sent to that dialog. No `models` CLI probe is run: v2 can start an unowned service daemon through that route. Clipboard writes are discarded and reads return nothing. Inputs are synthetic; no model submission, permission approval, or real clipboard access is tested. This is environment/config isolation, not an OS/network sandbox: package/cache startup may access the network. + +On failure, Actions uploads only the guard logs, synthetic receipts, scenario checkpoint `.txt` files, and available `failure.txt`, `failure.json`, and `failure.termctrl` evidence, named per host and retained for seven days. Cache, config, home, history, and package trees are not uploaded. Local artifacts stay in the requested output directory. Recordings may contain typed data: do not adapt these guards to real credentials or private prompts and upload the result. + +A nonempty inherited `TERMCTRL_BINARY` is rejected and recorded during setup. The SDK resolves that override in the Bun parent before applying the isolated driver environment; the guard supports only the pinned packaged driver. + +## Pinned private lifecycle boundary + +`test/e2e/driver.ts` uses the SDK 1.2.1 private constructor, `ready`, `child`, and `abort`, because public `make()` hides ownership until readiness and `close()` has no forced-reap fallback. Manifest version, installed JS SHA-256, source shape, and runtime guards fail fast if that boundary changes. Upgrade it only with explicit review and renewed lifecycle probes; do not patch the installed dependency. + +Ownership is retained before waiting for hello. Cooperative stop/shutdown failures remain reported even when forced cleanup succeeds. Unresolved handles survive cleanup retries, block later launches, and never become a claimed successful reap. + +For forced cleanup on macOS/Linux, the adapter freezes the exact owned driver PID before reading `ps` parent/group identities. Each direct PTY child must lead its own process group. It kills those groups before killing the driver, waits for the driver's `close` event, and verifies the groups disappear, including zombies. SDK abort is intercepted before it can lose group identities. There is no process-name matching. This follows pinned native [`Session::terminate`/`Drop`](https://github.com/anomalyco/terminal-control/blob/v1.2.1/src/session.rs) and [`ManagedSession::stop_worker`/`Drop`](https://github.com/anomalyco/terminal-control/blob/v1.2.1/src/driver.rs): killing the driver alone bypasses its destructors. Unexpected ownership or unavailable process inspection is a cleanup failure. External driver SIGKILL/OS failure and descendants escaping their PTY group are outside this test-only guarantee. + +## Oracle and validation limits + +Both pinned hosts expose the same bordered home prompt. Assertions compare every rendered prompt line exactly, including empty lines and internal whitespace, not a substring anywhere on screen. Cases use short ASCII, unwrapped lines without trailing spaces. Terminal padding cannot distinguish trailing buffer spaces; these tests do not establish hidden-buffer state, Unicode/display-cell behavior, selection, other routes, full v2 parity, or all-version compatibility. + +Cursor-only SDK captures may be stale. Word/quote placement is proven by exact editing effects; cursor metadata is diagnostic there. The multiline Enter safety guard requires observed cursor transitions. Captures must be settled and remain unchanged for a 250 ms stability window after each checkpoint starts, with bounded 5-second keyboard sends/editing captures, a 30-second startup observation deadline, and a 10 ms polling cadence. The 240-second outer scenario timeout leaves room for the longest sequence's bounded operations, failure evidence, and finally cleanup; Bun's timeout alone cannot reject a stalled SDK request. This avoids Escape/printable-key coalescing without unconditional per-key sleeps. + +Both pinned hosts (including the provider-free four-scenario run and no-plugin negative controls) and the harness checks (including stalled-send failure/finally/reap probes) were validated on macOS arm64 with Bun 1.3.2. The default no-argument run passed v1 via PATH but v2 automatic npm provisioning was blocked by registry `ENOTCONN` in this environment, so bare/no-override eight-case success is not claimed. A no-argument run with the optional v2 binary path override passed all eight cases; selectors, advanced paths, failure propagation, and the official v1 macOS bootstrap were also checked. Ubuntu workflow syntax and the official Linux package/archive identities were checked, but Linux execution remains pending: the available Docker daemon was not running. The first Actions run must establish that environment. Windows is not supported by this PTY guard; the existing fast suite remains unchanged. diff --git a/justfile b/justfile index 5f61ab6..2627e3f 100644 --- a/justfile +++ b/justfile @@ -25,6 +25,14 @@ check: compat host binary version output: python3 test/compat/run.py --host "{{host}}" --binary "{{binary}}" --expect-version "{{version}}" --output "{{output}}" +# Harness integration checks and parser helpers; local runs are on demand. +test-int: + bun --env-file=/dev/null run test:int + +# Both pinned hosts by default; optionally select one or supply all explicit paths. +test-e2e host="" binary="" version="" output="": + bun --env-file=/dev/null run test:e2e "{{host}}" "{{binary}}" "{{version}}" "{{output}}" + # Pure comparison checks for the optional real-host harness. compat-unit: python3 -B -m unittest discover -s test/compat -p 'test_*.py' diff --git a/package.json b/package.json index 8626605..bfb72b3 100644 --- a/package.json +++ b/package.json @@ -23,12 +23,15 @@ ], "scripts": { "test": "bun test", + "test:int": "bun --env-file=/dev/null test ./test/e2e/prompt.unit.ts ./test/e2e/driver.integration.ts ./test/e2e/launcher.integration.ts", + "test:e2e": "bun --env-file=/dev/null scripts/test-e2e.ts", "typecheck": "tsc --noEmit", "check": "biome check .", "fix": "biome check --write ." }, "devDependencies": { "@biomejs/biome": "2.4.15", + "@kitlangton/terminal-control": "1.2.1", "@opencode-ai/plugin": "^1.15.4", "@opentui/keymap": "0.5.10", "tsx": "^4.22.1" diff --git a/scripts/test-e2e.ts b/scripts/test-e2e.ts new file mode 100644 index 0000000..ad784e8 --- /dev/null +++ b/scripts/test-e2e.ts @@ -0,0 +1,315 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseEnv } from "node:util"; + +export const pins = { v1: "1.18.34", v2: "2.0.15" } as const; +export type Host = keyof typeof pins; +const overrideKeys = { v1: "VIMCODE_E2E_V1_BIN", v2: "VIMCODE_E2E_V2_BIN" } as const; +type Environment = Record; +export type Selection = { hosts: Host[]; explicit?: { binary: string; version: string; output: string } }; +const root = realpathSync(fileURLToPath(new URL("../", import.meta.url))); +const configNames = [ + ".git", + ".opencode", + "opencode.json", + "opencode.jsonc", + "tui.json", + "tui.jsonc", + "cli.json", + "cli.jsonc", +]; + +export function selectHosts(args: string[]): Selection { + const values = [...args]; + while (values.at(-1) === "") values.pop(); + if (!values.length) return { hosts: ["v1", "v2"] }; + const [host, binary, version, output] = values; + if ((host !== "v1" && host !== "v2") || (values.length !== 1 && values.length !== 4)) + throw new Error( + "Use just test-e2e [v1|v2] or just test-e2e HOST /absolute/BINARY PINNED_VERSION /absolute/NEW_OUTPUT", + ); + if (values.length === 1) return { hosts: [host] }; + if (!isAbsolute(binary) || !isAbsolute(output) || version !== pins[host]) + throw new Error(`Explicit ${host} requires absolute binary/output paths and pinned version ${pins[host]}`); + return { hosts: [host], explicit: { binary, version, output } }; +} + +// parseEnv is data-only: neither shell expansion nor unknown keys reach child environments. +export function binarySettings(env: Environment, fileText = ""): Partial> { + const local = env.CI ? {} : parseEnv(fileText); + const settings: Partial> = {}; + for (const host of ["v1", "v2"] as const) { + const key = overrideKeys[host]; + const value = env[key] ?? local[key]; + if (value !== undefined) settings[host] = value; + } + return settings; +} + +export function isolatedToolEnv(directory: string): Record { + for (const name of ["home", "config", "data", "cache", "state", "cwd"]) + mkdirSync(join(directory, name), { recursive: true }); + const tools = [process.execPath, Bun.which("npm"), Bun.which("node")] + .filter((path): path is string => path !== null) + .map((path) => dirname(path)); + return { + PATH: [...new Set([...tools, "/usr/local/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin"])].join(":"), + HOME: join(directory, "home"), + XDG_CONFIG_HOME: join(directory, "config"), + XDG_DATA_HOME: join(directory, "data"), + XDG_CACHE_HOME: join(directory, "cache"), + XDG_STATE_HOME: join(directory, "state"), + OPENCODE_DISABLE_AUTOUPDATE: "1", + }; +} + +function outsideConfig(path: string) { + for (let ancestor = path; ; ancestor = dirname(ancestor)) { + if (configNames.some((name) => existsSync(join(ancestor, name)))) + throw new Error(`E2E output must be outside project/config ancestry: ${ancestor}`); + if (dirname(ancestor) === ancestor) break; + } +} + +export function freshRunRoot(): string { + const temporary = realpathSync(tmpdir()); + outsideConfig(temporary); + return realpathSync(mkdtempSync(join(temporary, "vimcode-e2e-"))); +} + +export function outputPath(path: string): string { + if (!isAbsolute(path)) throw new Error("E2E output must be absolute"); + const canonical = existsSync(path) + ? realpathSync(path) + : join(realpathSync(dirname(path)), path.slice(dirname(path).length + 1)); + outsideConfig(canonical); + if (existsSync(canonical) && (!statSync(canonical).isDirectory() || readdirSync(canonical).length)) + throw new Error(`E2E output must be a new or empty directory: ${canonical}`); + return canonical; +} + +function command(binary: string, args: string[], directory: string, env: Record, timeout = 10_000) { + return execFileSync(binary, args, { + cwd: directory, + env, + encoding: "utf8", + timeout, + killSignal: "SIGKILL", + maxBuffer: 2 * 1024 * 1024, + }); +} + +function versionOf(binary: string, directory: string, env: Record): string { + return command(binary, ["--version"], directory, env) + .trim() + .replace(/^opencode v/, ""); +} + +export function verifiedBinary( + path: string, + host: Host, + directory: string, + env: Record, + base = root, +): string { + // Validate only when this host runs: an unused host override must not block selection. + if (!path.trim() || /[\r\n\0]/.test(path)) throw new Error(`${overrideKeys[host]} must be a nonempty binary path`); + let binary: string; + try { + binary = realpathSync(resolve(base, path)); + if (!statSync(binary).isFile()) throw new Error("not a file"); + } catch { + throw new Error(`${overrideKeys[host]}: binary path does not exist or is not a file: ${path}`); + } + const actual = versionOf(binary, directory, env); + if (actual !== pins[host]) + throw new Error(`Expected OpenCode ${host} ${pins[host]}, got ${JSON.stringify(actual)} at ${binary}`); + return binary; +} + +export function nativeSource( + host: Host, + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, +): { package: string } | { asset: string; sha256: string } { + const target = `${platform}-${arch}`; + if (target !== "darwin-arm64" && target !== "linux-x64") + throw new Error( + `No verified bootstrap for ${target}; set ${overrideKeys[host]} to a native OpenCode ${pins[host]} binary`, + ); + if (host === "v2") return { package: `@opencode/cli-${target === "linux-x64" ? "linux-x64-baseline" : target}` }; + return target === "darwin-arm64" + ? { asset: "opencode-darwin-arm64.zip", sha256: "8522b70f545184b3a8d97c5ca4f814093b2476d72aebfda8c48bcd072ec31d1b" } + : { + asset: "opencode-linux-x64-baseline.tar.gz", + sha256: "24b0d458d21ef548b2752166303defcf7f4945b049fb4876ab78dfaf86d81b27", + }; +} + +function provision(host: Host, directory: string, env: Record): string { + const source = nativeSource(host); + const native = join(directory, "native"); + mkdirSync(native); + if ("package" in source) { + // npm refuses using the same config file for both scopes; neither may read host credentials. + const userConfig = join(directory, "user.npmrc"); + const globalConfig = join(directory, "global.npmrc"); + writeFileSync(userConfig, ""); + writeFileSync(globalConfig, ""); + command( + "npm", + [ + "install", + "--prefix", + native, + "--no-save", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--registry=https://registry.npmjs.org", + `--userconfig=${userConfig}`, + `--globalconfig=${globalConfig}`, + "--fetch-retries=1", + "--fetch-timeout=45000", + `${source.package}@${pins[host]}`, + ], + directory, + env, + 120_000, + ); + const packageRoot = join(native, "node_modules", source.package); + const manifest = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) as { + name: string; + version: string; + }; + if (manifest.name !== source.package || manifest.version !== pins[host]) + throw new Error("Unexpected native host package identity"); + return verifiedBinary(join(packageRoot, "bin/opencode"), host, directory, env); + } + const archive = join(directory, source.asset); + command( + "curl", + [ + "--fail", + "--location", + "--connect-timeout", + "10", + "--max-time", + "90", + "--output", + archive, + `https://github.com/anomalyco/opencode/releases/download/v${pins[host]}/${source.asset}`, + ], + directory, + env, + 100_000, + ); + if (createHash("sha256").update(readFileSync(archive)).digest("hex") !== source.sha256) + throw new Error(`Official ${source.asset} SHA-256 mismatch; refusing extraction`); + const zip = source.asset.endsWith(".zip"); + const contents = command(zip ? "unzip" : "tar", zip ? ["-Z1", archive] : ["-tzf", archive], directory, env).trim(); + if (contents !== "opencode") throw new Error("Expected exactly one native opencode archive entry"); + command(zip ? "unzip" : "tar", zip ? ["-q", archive, "-d", native] : ["-xzf", archive, "-C", native], directory, env); + return verifiedBinary(join(native, "opencode"), host, directory, env); +} + +export function resolveBinary( + host: Host, + override: string | undefined, + directory: string, + env: Record, +): string { + if (override !== undefined) return verifiedBinary(override, host, directory, env); + const installed = Bun.which("opencode"); + if (installed) { + try { + if (versionOf(installed, directory, env) === pins[host]) return realpathSync(installed); + } catch { + // A broken/unmatched PATH host is not an override; bootstrap the exact pin instead. + } + } + process.stdout.write(`Bootstrapping private OpenCode ${host}@${pins[host]} (network required)\n`); + try { + return provision(host, directory, env); + } catch (error) { + throw new Error( + `Could not bootstrap ${host}@${pins[host]}. Set ${overrideKeys[host]} in .env or the process environment to an existing pinned native binary. ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } +} + +export async function runHosts(hosts: Host[], execute: (host: Host) => Promise): Promise { + let status = 0; + for (const host of hosts) if ((await execute(host)) !== 0) status = 1; + return status; +} + +async function main() { + if (!["darwin", "linux"].includes(process.platform)) + throw new Error("Terminal Control E2E requires macOS or GNU/Linux"); + const selection = selectHosts(process.argv.slice(2)); + // CI and fully explicit invocations never read a local file. + const file = join(root, ".env"); + const settings = selection.explicit + ? {} + : binarySettings(process.env, !process.env.CI && existsSync(file) ? readFileSync(file, "utf8") : ""); + const runRoot = freshRunRoot(); + process.stdout.write(`E2E run directory: ${runRoot}\n`); + const outputs = Object.fromEntries( + selection.hosts.map((host) => [host, outputPath(selection.explicit?.output ?? join(runRoot, host))]), + ); + for (const host of selection.hosts) + process.stdout.write(`${host}@${pins[host]} receipt: ${join(outputs[host], "receipt.json")}\n`); + return runHosts(selection.hosts, async (host) => { + try { + const directory = join(runRoot, `${host}-tools`); + const env = isolatedToolEnv(directory); + const binary = resolveBinary(host, selection.explicit?.binary ?? settings[host], directory, env); + process.stdout.write(`${host} binary: ${binary}\n`); + const child = Bun.spawn([process.execPath, "--env-file=/dev/null", "test", "./test/e2e/opencode.e2e.ts"], { + cwd: root, + env: { + ...env, + VIMCODE_E2E_HOST: host, + VIMCODE_E2E_BIN: binary, + VIMCODE_E2E_VERSION: pins[host], + VIMCODE_E2E_OUTPUT: outputs[host], + VIMCODE_E2E_NO_PLUGIN: process.env.VIMCODE_E2E_NO_PLUGIN === "1" ? "1" : "0", + ...(process.env.TERMCTRL_BINARY ? { TERMCTRL_BINARY: process.env.TERMCTRL_BINARY } : {}), + }, + stdout: "inherit", + stderr: "inherit", + stdin: "ignore", + }); + return await child.exited; + } catch (error) { + const message = `${host} launcher failed: ${error instanceof Error ? error.message : String(error)}`; + writeFileSync(join(runRoot, `${host}-launcher-error.txt`), `${message}\n`); + process.stderr.write(`${message}\n`); + return 1; + } + }); +} + +if (import.meta.main) { + try { + process.exitCode = await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/test/e2e/driver.integration.ts b/test/e2e/driver.integration.ts new file mode 100644 index 0000000..eb5ee1a --- /dev/null +++ b/test/e2e/driver.integration.ts @@ -0,0 +1,223 @@ +import { expect, it } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { bounded, closeDriver, ownDriver } from "./driver"; +import { type HostRun, prepareRun, pressKey, scenario, startScenario, stopScenario, typeText } from "./host"; + +if (!["darwin", "linux"].includes(process.platform)) + throw new Error("Harness integration process checks require macOS or Linux"); + +function fixture(mode: "ready-stall" | "shutdown-stall" | "launch-stall" | "abort" | "send-stall") { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "vimcode-driver-probe-")); + const binaryPath = join(directory, "driver"); + writeFileSync( + binaryPath, + `#!${process.execPath} +import { spawn } from "node:child_process"; +import { appendFileSync, writeFileSync } from "node:fs"; +import { createInterface } from "node:readline"; +process.on("SIGTERM", () => {}); +setInterval(() => {}, 1000); +if (${JSON.stringify(mode)} !== "ready-stall") { + console.log(JSON.stringify({ type: "hello", protocolVersion: 2 })); + createInterface({ input: process.stdin }).on("line", line => { + const request = JSON.parse(line); + appendFileSync(${JSON.stringify(join(directory, "requests.txt"))}, request.method + "\\n"); + const result = value => console.log(JSON.stringify({ type: "result", id: request.id, result: value })); + if (${JSON.stringify(mode)} === "send-stall" && request.method !== "launch") { + if (request.method === "capture") result({ reason: "idle", shot: { + text: " ┃\\n ┃ Ask\\n ┃\\n ┃ Build · No provider selected Connect a provider\\n ╹▀▀▀▀▀", + frame: { cursor: null } + }}); + else if (request.method === "recording") result({ bytes: [] }); + else if (request.method === "stop") result(null); + // SEND and shutdown deliberately never respond; the real scenario must unwind/reap. + } + if (request.method === "launch") { + const host = spawn(${JSON.stringify(process.execPath)}, ["-e", "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"], { detached: true, stdio: "ignore" }); + writeFileSync(${JSON.stringify(join(directory, "host.pid"))}, String(host.pid)); + if (${JSON.stringify(mode)} === "abort") console.log("invalid-json"); + else if (${JSON.stringify(mode)} !== "launch-stall") console.log(JSON.stringify({ type: "result", id: request.id, result: null })); + } + }); +} +`, + { mode: 0o755 }, + ); + return { directory, binaryPath }; +} + +function present(pid: number) { + return execFileSync("/bin/ps", ["-axo", "pid="], { encoding: "utf8" }).split(/\s+/).includes(String(pid)); +} + +for (const mode of ["ready-stall", "shutdown-stall", "launch-stall", "abort"] as const) { + it(`owned driver reaps ${mode} and its detached host`, async () => { + const probe = fixture(mode); + const env = Object.fromEntries(Object.keys(process.env).map((key) => [key, undefined])); + Object.assign(env, { HOME: probe.directory, PATH: "/usr/bin:/bin" }); + const owner = await ownDriver({ binaryPath: probe.binaryPath, cwd: probe.directory, env }); + const pid = owner.child.pid; + if (!pid) throw new Error("missing owned driver PID"); + try { + if (mode === "ready-stall") { + await expect(bounded(owner.ready, "probe readiness", 100)).rejects.toThrow("probe readiness exceeded"); + } else { + await bounded(owner.ready, "probe readiness", 2_000); + const launch = owner.terminal.launch({ command: ["unused"] }); + if (mode === "abort") { + await expect(bounded(launch, "probe launch", 2_000)).rejects.toThrow("invalid termctrl driver response"); + } else if (mode === "launch-stall") { + await expect(bounded(launch, "probe launch", 200)).rejects.toThrow("probe launch exceeded"); + } else { + await bounded(launch, "probe launch", 2_000); + } + } + await expect(closeDriver(owner, 100)).rejects.toThrow(/driver (shutdown|reap) exceeded/); + expect(owner.reaped).toBe(true); + expect(present(pid)).toBe(false); + if (mode !== "ready-stall") { + const hostPid = Number(readFileSync(join(probe.directory, "host.pid"), "utf8")); + expect(owner.hostGroups.has(hostPid)).toBe(true); + expect(present(hostPid)).toBe(false); + } + } finally { + if (!owner.reaped) await closeDriver(owner, 1_000).catch(() => {}); + rmSync(probe.directory, { recursive: true, force: true }); + } + }); +} + +for (const send of ["type", "press"] as const) { + it(`scenario captures stalled keyboard ${send} failure and finally reaps its driver/host`, async () => { + const probe = fixture("send-stall"); + const run: HostRun = { + output: probe.directory, + env: { HOME: probe.directory, PATH: "/usr/bin:/bin" }, + receipt: { + host: "v1", + binary: "unused", + version: "unused", + platform: process.platform, + terminalControl: "1.2.1", + driverBinary: probe.binaryPath, + negativeControl: false, + checks: [], + cleanupErrors: [], + }, + }; + let owner: HostRun["driver"]; + try { + await expect( + scenario(run, `send-${send}`, async () => { + owner = run.driver; + if (send === "type") await typeText(run, "synthetic", 100); + else await pressKey(run, "Escape", 100); + throw new Error("stalled send must not complete"); + }), + ).rejects.toThrow("driver shutdown exceeded"); + if (!owner?.child.pid) throw new Error("scenario did not retain driver ownership"); + const hostPid = Number(readFileSync(join(probe.directory, "host.pid"), "utf8")); + const receipt = JSON.parse(readFileSync(join(probe.directory, "receipt.json"), "utf8")) as { + checks: Array<{ name: string; status: string; error: string }>; + cleanupErrors: string[]; + }; + expect(receipt.checks).toEqual([ + { name: `send-${send}`, status: "fail", error: `keyboard ${send} exceeded 100ms` }, + ]); + expect(receipt.cleanupErrors).toEqual(["Error: driver shutdown exceeded 5000ms"]); + expect(readFileSync(join(probe.directory, `send-${send}/failure.txt`), "utf8")).toContain("No provider selected"); + expect(readFileSync(join(probe.directory, `send-${send}/failure.termctrl`)).length).toBe(0); + const requests = readFileSync(join(probe.directory, "requests.txt"), "utf8").trim().split("\n"); + expect(requests).toContain("send"); + expect(requests.slice(-3)).toEqual(["recording", "stop", "shutdown"]); + expect(owner.reaped).toBe(true); + expect(owner.hostGroups.has(hostPid)).toBe(true); + expect(present(owner.child.pid)).toBe(false); + expect(present(hostPid)).toBe(false); + expect(run.driver).toBeUndefined(); + expect(run.session).toBeUndefined(); + } finally { + if (run.driver && !run.driver.reaped) await closeDriver(run.driver, 1_000).catch(() => {}); + rmSync(probe.directory, { recursive: true, force: true }); + } + }, 20_000); +} + +it("retains unresolved ownership across cleanup retries and blocks later launches", async () => { + const probe = fixture("shutdown-stall"); + const env = Object.fromEntries(Object.keys(process.env).map((key) => [key, undefined])); + Object.assign(env, { HOME: probe.directory, PATH: "/usr/bin:/bin" }); + const owner = await ownDriver({ binaryPath: probe.binaryPath, cwd: probe.directory, env }); + const run: HostRun = { + output: probe.directory, + env: {}, + driver: owner, + terminal: owner.terminal, + receipt: { + host: "v1", + binary: "unused", + version: "unused", + platform: process.platform, + terminalControl: "1.2.1", + negativeControl: false, + checks: [], + cleanupErrors: [], + }, + }; + try { + await bounded(owner.ready, "probe readiness", 2_000); + await bounded(owner.terminal.launch({ command: ["unused"] }), "probe launch", 2_000); + owner.discoveryError = new Error("injected unresolved ownership"); + await expect(closeDriver(owner, 100)).rejects.toThrow("injected unresolved ownership"); + await owner.closed; + expect(owner.reaped).toBe(false); + await expect(stopScenario(run)).rejects.toThrow("injected unresolved ownership"); + expect(owner.reaped).toBe(false); + expect(run.driver).toBe(owner); + await expect(startScenario(run, "must-not-launch")).rejects.toThrow("refusing another launch"); + expect(present(Number(readFileSync(join(probe.directory, "host.pid"), "utf8")))).toBe(false); + } finally { + // The injected discovery failure is the only unresolved condition in this fixture. + owner.discoveryError = undefined; + await closeDriver(owner, 1_000).catch(() => {}); + rmSync(probe.directory, { recursive: true, force: true }); + } +}); + +it("rejects inherited TERMCTRL_BINARY before binary execution and records the reason", () => { + const output = mkdtempSync(join(realpathSync(tmpdir()), "vimcode-override-probe-")); + const keys = [ + "TERMCTRL_BINARY", + "VIMCODE_E2E_HOST", + "VIMCODE_E2E_BIN", + "VIMCODE_E2E_VERSION", + "VIMCODE_E2E_OUTPUT", + ] as const; + const before = keys.map((key) => process.env[key]); + Object.assign(process.env, { + TERMCTRL_BINARY: "/not-a-pinned-driver", + VIMCODE_E2E_HOST: "v1", + VIMCODE_E2E_BIN: "/must-not-execute", + VIMCODE_E2E_VERSION: "1.18.34", + VIMCODE_E2E_OUTPUT: output, + }); + try { + expect(() => prepareRun()).toThrow("TERMCTRL_BINARY override is unsupported"); + const receipt = JSON.parse(readFileSync(join(output, "receipt.json"), "utf8")) as { + checks: Array<{ status: string; error: string }>; + driverBinary?: string; + }; + expect(receipt.checks[0].status).toBe("fail"); + expect(receipt.checks[0].error).toContain("unset it"); + expect(receipt.driverBinary).toBeUndefined(); + } finally { + keys.forEach((key, index) => { + if (before[index] === undefined) delete process.env[key]; + else process.env[key] = before[index]; + }); + rmSync(output, { recursive: true, force: true }); + } +}); diff --git a/test/e2e/driver.ts b/test/e2e/driver.ts new file mode 100644 index 0000000..f329cab --- /dev/null +++ b/test/e2e/driver.ts @@ -0,0 +1,159 @@ +import { ChildProcess, execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { type DriverOptions, TerminalControl } from "@kitlangton/terminal-control"; + +export async function bounded(operation: Promise, label: string, timeoutMs = 5_000): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} exceeded ${timeoutMs}ms`)), timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +type PrivateDriver = { child: ChildProcess; ready: Promise; abort: (error: Error) => void }; +export type OwnedDriver = { + terminal: TerminalControl; + ready: Promise; + child: ChildProcess; + closed: Promise; + hostGroups: Set; + frozen: boolean; + reaped: boolean; + discoveryError?: unknown; +}; + +type ProcessRow = { pid: number; parent: number; group: number; state: string }; +function processes(): ProcessRow[] { + return execFileSync("/bin/ps", ["-axo", "pid=,ppid=,pgid=,stat="], { encoding: "utf8", timeout: 1_000 }) + .trim() + .split("\n") + .map((line) => { + const [pid, parent, group, state] = line.trim().split(/\s+/); + return { pid: Number(pid), parent: Number(parent), group: Number(group), state }; + }); +} + +function signal(identity: number, value: NodeJS.Signals) { + try { + process.kill(identity, value); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } +} + +// portable-pty gives each launched host a new session/group (PGID = host PID). +// Freeze the driver before discovery: even a timed-out launch cannot spawn after +// the ownership snapshot. Capture before SDK abort too, which otherwise kills it. +function retainHosts(owner: OwnedDriver) { + const pid = owner.child.pid; + if (!pid || owner.child.exitCode !== null || owner.child.signalCode !== null) return; + signal(pid, "SIGSTOP"); + owner.frozen = true; + const deadline = Date.now() + 1_000; + let rows = processes(); + while (rows.some((row) => row.pid === pid && !row.state.startsWith("T"))) { + if (Date.now() >= deadline) throw new Error(`Owned driver PID ${pid} did not freeze`); + rows = processes(); + } + for (const row of rows) { + if (row.parent !== pid) continue; + if (row.group !== row.pid) throw new Error(`Unexpected owned PTY process group for PID ${row.pid}`); + owner.hostGroups.add(row.group); + } +} + +// SDK 1.2.1 has no public pre-ready ownership or forced-close API. Keep this +// guarded private boundary local to the optional spike; never patch the SDK. +export async function ownDriver(options: DriverOptions): Promise { + if (!["darwin", "linux"].includes(process.platform)) throw new Error("Owned PTY cleanup requires macOS or Linux"); + const entry = fileURLToPath(import.meta.resolve("@kitlangton/terminal-control")); + const manifest = JSON.parse(readFileSync(join(dirname(entry), "../package.json"), "utf8")) as { version: string }; + const constructorSource = TerminalControl.toString(); + const prototype = TerminalControl.prototype as unknown as PrivateDriver; + if ( + manifest.version !== "1.2.1" || + createHash("sha256").update(readFileSync(entry)).digest("hex") !== + "b4476b7d422818b1bd56397a2571db9ffdfe5122799172c572f4dc788edebee1" || + !constructorSource.includes("this.child = spawn(") || + !constructorSource.includes("this.ready = new Promise(") || + typeof prototype.abort !== "function" || + !prototype.abort.toString().includes("this.child.kill()") + ) { + throw new Error("Terminal Control private lifecycle shape changed; expected pinned SDK 1.2.1"); + } + const Constructor = TerminalControl as unknown as new (options: DriverOptions) => TerminalControl; + const terminal = new Constructor(options); + const internals = terminal as unknown as PrivateDriver; + const child = internals.child; + if (!(child instanceof ChildProcess)) throw new Error("Terminal Control private child shape changed"); + const closed = new Promise((resolve) => child.once("close", () => resolve())); + if (!(internals.ready instanceof Promise)) { + child.kill("SIGKILL"); + await bounded(closed, "invalid SDK shape driver reap"); + throw new Error("Terminal Control private ready shape changed"); + } + const owner: OwnedDriver = { + terminal, + ready: internals.ready, + child, + closed, + hostGroups: new Set(), + frozen: false, + reaped: false, + }; + const abort = internals.abort.bind(terminal); + internals.abort = (error) => { + try { + retainHosts(owner); + } catch (discoveryError) { + owner.discoveryError = discoveryError; + } finally { + abort(error); + } + }; + // A rejected hello must be observed even if cleanup begins before awaiting it. + void owner.ready.catch(() => {}); + return owner; +} + +async function confirmReaped(owner: OwnedDriver, timeoutMs: number) { + const deadline = Date.now() + timeoutMs; + while (processes().some((row) => owner.hostGroups.has(row.group))) { + if (Date.now() >= deadline) throw new Error(`forced PTY group reap exceeded ${timeoutMs}ms`); + await Bun.sleep(10); + } + if (owner.discoveryError) throw owner.discoveryError; + owner.reaped = true; +} + +export async function terminateDriver(owner: OwnedDriver, timeoutMs = 5_000) { + retainHosts(owner); + // Kill the PTY groups BEFORE the driver, not just its PID: killing the driver + // alone bypasses Rust Session::drop and can leave an OpenCode host alive. + for (const group of owner.hostGroups) signal(-group, "SIGKILL"); + // SIGKILL also terminates stopped processes; do not resume and reopen a spawn race. + owner.child.kill("SIGKILL"); + await bounded(owner.closed, "forced driver reap", timeoutMs); + await confirmReaped(owner, timeoutMs); +} + +export async function closeDriver(owner: OwnedDriver, timeoutMs = 5_000) { + try { + await bounded(owner.terminal.close(), "driver shutdown", timeoutMs); + await bounded(owner.closed, "driver reap", timeoutMs); + // SDK close becomes a no-op after abort: retries must still verify retained ownership. + await confirmReaped(owner, timeoutMs); + } catch (error) { + await terminateDriver(owner, timeoutMs); + throw error; + } +} diff --git a/test/e2e/host.ts b/test/e2e/host.ts new file mode 100644 index 0000000..7ca4972 --- /dev/null +++ b/test/e2e/host.ts @@ -0,0 +1,403 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + type Key, + resolveTerminalControlBinary, + type ScreenSnapshot, + type Session, + type TerminalControl, +} from "@kitlangton/terminal-control"; +import { bounded, closeDriver, type OwnedDriver, ownDriver } from "./driver"; +import { promptMatches, providerDisabledPrompt, type VisiblePrompt, visiblePrompt } from "./prompt"; + +const root = realpathSync(fileURLToPath(new URL("../../", import.meta.url))); +const configNames = [ + ".git", + ".opencode", + "opencode.json", + "opencode.jsonc", + "tui.json", + "tui.jsonc", + "cli.json", + "cli.jsonc", +]; + +type Check = { name: string; status: "pass" | "fail"; expected?: unknown; actual?: unknown; error?: string }; +type Receipt = { + host: "v1" | "v2"; + binary: string; + version: string; + platform: string; + terminalControl: string; + driverBinary?: string; + negativeControl: boolean; + providersDisabled?: boolean; + providerObservations?: Record; + artifact?: { tarball: string; sha256: string; installed?: string; verifiedFiles?: Record }; + checks: Check[]; + cleanupErrors: string[]; +}; + +export type HostRun = { + output: string; + env: Record; + receipt: Receipt; + terminal?: TerminalControl; + driver?: OwnedDriver; + session?: Session; + scenario?: string; +}; + +export function saveReceipt(run: HostRun) { + writeFileSync(join(run.output, "receipt.json"), `${JSON.stringify(run.receipt, null, 2)}\n`); +} + +export function prepareRun(): HostRun { + if (!["darwin", "linux"].includes(process.platform)) + throw new Error("Terminal Control E2E requires macOS or GNU/Linux"); + const host = process.env.VIMCODE_E2E_HOST; + const binary = process.env.VIMCODE_E2E_BIN; + const version = process.env.VIMCODE_E2E_VERSION; + const outputArg = process.env.VIMCODE_E2E_OUTPUT; + if ( + (host !== "v1" && host !== "v2") || + !binary || + !version || + !outputArg || + !isAbsolute(binary) || + !isAbsolute(outputArg) + ) { + throw new Error("Use just test-e2e v1|v2 /absolute/opencode expected-version /absolute/new-output-directory"); + } + const output = join(realpathSync(dirname(outputArg)), outputArg.slice(dirname(outputArg).length + 1)); + for (let path = output; ; path = dirname(path)) { + if (configNames.some((name) => existsSync(join(path, name)))) + throw new Error(`E2E output must be outside project/config ancestry: ${path}`); + if (dirname(path) === path) break; + } + if (existsSync(output) && readdirSync(output).length) throw new Error(`E2E output must be empty: ${output}`); + mkdirSync(output, { recursive: true }); + const canonicalOutput = realpathSync(output); + for (let path = canonicalOutput; ; path = dirname(path)) { + if (configNames.some((name) => existsSync(join(path, name)))) + throw new Error(`E2E output must be outside project/config ancestry: ${path}`); + if (dirname(path) === path) break; + } + for (const folder of ["home", "config/opencode", "data", "cache", "state", "sandbox", "bin", "build", "reference"]) { + mkdirSync(join(canonicalOutput, folder), { recursive: true }); + } + const toolDirectories = [process.execPath, Bun.which("npm"), Bun.which("node")] + .filter((path): path is string => path !== null) + .map((path) => dirname(path)); + const env = { + PATH: [ + ...new Set([ + join(canonicalOutput, "bin"), + ...toolDirectories, + "/usr/local/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + ]), + ].join(":"), + HOME: join(canonicalOutput, "home"), + XDG_CONFIG_HOME: join(canonicalOutput, "config"), + XDG_DATA_HOME: join(canonicalOutput, "data"), + XDG_CACHE_HOME: join(canonicalOutput, "cache"), + XDG_STATE_HOME: join(canonicalOutput, "state"), + TERM: "xterm-256color", + COLORTERM: "truecolor", + OPENCODE_DISABLE_AUTOUPDATE: "1", + }; + for (const name of ["pbcopy", "wl-copy", "xclip", "xsel", "clip.exe"]) { + writeFileSync(join(canonicalOutput, "bin", name), "#!/bin/sh\ncat >/dev/null\n", { mode: 0o755 }); + } + for (const name of ["pbpaste", "wl-paste"]) { + writeFileSync(join(canonicalOutput, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + } + const run: HostRun = { + output: canonicalOutput, + env, + receipt: { + host, + binary, + version, + platform: `${process.platform}-${process.arch}`, + terminalControl: "1.2.1", + negativeControl: process.env.VIMCODE_E2E_NO_PLUGIN === "1", + checks: [], + cleanupErrors: [], + }, + }; + saveReceipt(run); + try { + if (process.env.TERMCTRL_BINARY) { + throw new Error( + "TERMCTRL_BINARY override is unsupported: unset it to use the pinned Terminal Control 1.2.1 packaged driver", + ); + } + run.receipt.driverBinary = realpathSync(resolveTerminalControlBinary()); + writeFileSync( + join(run.output, "config/opencode/opencode.json"), + JSON.stringify({ autoupdate: false, enabled_providers: [] }), + ); + const actualBinary = realpathSync(binary); + const actualVersion = execFileSync(actualBinary, ["--version"], { env, encoding: "utf8", timeout: 10_000 }) + .trim() + .replace(/^opencode v/, ""); + Object.assign(run.receipt, { binary: actualBinary, version: actualVersion }); + if (actualVersion !== version || !version.startsWith(host === "v1" ? "1." : "2.")) + throw new Error(`Expected OpenCode ${host} ${version}, got ${actualVersion}`); + const packed = JSON.parse( + execFileSync("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", join(run.output, "build")], { + cwd: root, + env, + encoding: "utf8", + timeout: 30_000, + }), + ) as Array<{ filename: string; files: Array<{ path: string }> }>; + if ( + packed.length !== 1 || + packed[0].files.some((file) => file.path.startsWith("test/") || file.path.startsWith("scripts/")) + ) { + throw new Error("Expected one package without test/launcher fixtures"); + } + const tarball = join(run.output, "build", packed[0].filename); + execFileSync("tar", ["-xzf", tarball, "-C", join(run.output, "reference")], { env, timeout: 10_000 }); + run.receipt.artifact = { tarball, sha256: hash(readFileSync(tarball)) }; + const spec = `vimcode@file:${tarball}`; + const options = { updateCheck: false, modeIndicator: "none", experimentalV2Leader: "ctrl+x" }; + const config = + host === "v1" + ? { plugin: run.receipt.negativeControl ? [] : [[spec, options]], keybinds: { leader: "ctrl+x" } } + : { plugins: run.receipt.negativeControl ? [] : [{ package: spec, options }], keybinds: { leader: "ctrl+x" } }; + writeFileSync(join(run.output, `config/opencode/${host === "v1" ? "tui" : "cli"}.json`), JSON.stringify(config)); + saveReceipt(run); + return run; + } catch (error) { + run.receipt.checks.push({ name: "setup", status: "fail", error: String(error) }); + saveReceipt(run); + throw error; + } +} + +function hash(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function verifyInstalledArtifact(run: HostRun) { + if (run.receipt.negativeControl) return; + const cache = join(run.output, "cache/opencode"); + const entries = [...new Bun.Glob("**/node_modules/vimcode/package.json").scanSync({ cwd: cache, absolute: true })]; + if (entries.length !== 1) throw new Error(`Expected exactly one cache-installed vimcode, found ${entries.length}`); + const installed = dirname(entries[0]); + const reference = join(run.output, "reference/package"); + const files = ["package.json", ...sourceFiles(join(reference, "src")).map((path) => `src/${path}`)]; + const verifiedFiles: Record = {}; + for (const relative of files) { + const expected = readFileSync(join(reference, relative)); + if (!expected.equals(readFileSync(join(installed, relative)))) + throw new Error(`Cache artifact mismatch: ${relative}`); + verifiedFiles[relative] = hash(expected); + } + if (!run.receipt.artifact) throw new Error("Package identity missing"); + Object.assign(run.receipt.artifact, { installed, verifiedFiles }); + saveReceipt(run); +} + +function sourceFiles(directory: string, prefix = ""): string[] { + return readdirSync(directory).flatMap((name) => { + const relative = join(prefix, name); + return statSync(join(directory, name)).isDirectory() ? sourceFiles(join(directory, name), relative) : [relative]; + }); +} + +export async function startScenario(run: HostRun, name: string) { + if (run.driver) throw new Error("Previous owned driver has not been reaped; refusing another launch"); + run.scenario = name; + mkdirSync(join(run.output, name)); + // The driver also gets no provider credentials; launch independently rejects inheritance. + const driverEnv = Object.fromEntries(Object.keys(process.env).map((key) => [key, undefined])); + Object.assign(driverEnv, run.env); + run.driver = await ownDriver({ binaryPath: run.receipt.driverBinary, cwd: run.output, env: driverEnv }); + run.terminal = run.driver.terminal; + await bounded(run.driver.ready, "driver startup", 10_000); + run.session = await bounded( + run.terminal.launch({ + command: [ + run.receipt.binary, + ...(run.receipt.host === "v2" ? ["--standalone"] : []), + join(run.output, "sandbox"), + ], + cwd: join(run.output, "sandbox"), + host: "opentui", + viewport: { cols: 110, rows: 34 }, + inheritEnv: false, + env: run.env, + record: "on-failure", + }), + "host launch", + 10_000, + ); + const deadline = Date.now() + 30_000; + const connectionDialog = (snapshot: ScreenSnapshot) => + /^\s+Connect a provider\s+esc\s*$/m.test(snapshot.text) && + /^\s+Search\s*$/m.test(snapshot.text) && + /^\s+Other Custom provider\s*$/m.test(snapshot.text); + const ready = (snapshot: ScreenSnapshot) => { + const prompt = visiblePrompt(snapshot); + return ( + providerDisabledPrompt(snapshot) && + prompt?.lines.length === 1 && + (prompt.lines[0] === "Ask" || prompt.lines[0].startsWith("Ask anything")) + ); + }; + let capture = await waitVisible(run, (snapshot) => connectionDialog(snapshot) || ready(snapshot), 30_000); + // Disabled providers trigger this setup overlay on v1. Never approve or submit it. + if (connectionDialog(capture)) { + await pressKey(run, "Escape"); + capture = await waitVisible( + run, + (snapshot) => !connectionDialog(snapshot) && ready(snapshot), + deadline - Date.now(), + ); + } + if (!ready(capture)) throw new Error("Provider isolation failed: no provider-free home prompt observed"); + run.receipt.providerObservations ??= {}; + run.receipt.providerObservations[name] = "Build · No provider selected Connect a provider"; + run.receipt.providersDisabled = true; + writeFileSync(join(run.output, name, "provider-disabled.txt"), capture.text); + saveReceipt(run); +} + +// Native capture can report idle immediately based on output from BEFORE the last +// input. Observe stability after this call too, so Escape is decoded before 0/w. +async function waitVisible(run: HostRun, predicate: (snapshot: ScreenSnapshot) => boolean, timeoutMs: number) { + const deadline = Date.now() + timeoutMs; + const settleMs = 250; + let previous = ""; + let stableSince = Date.now(); + while (Date.now() < deadline) { + const remaining = deadline - Date.now(); + const snapshot = await bounded( + sessionFor(run).screen.capture({ + allowIncomplete: true, + settleMs, + deadlineMs: Math.min(1_000, remaining), + }), + "visible screen capture", + remaining, + ); + if (snapshot.reason === "exited" || snapshot.reason === "outputclosed") + throw new Error(`OpenCode capture ended with ${snapshot.reason}`); + const current = JSON.stringify({ text: snapshot.text, cursor: snapshot.frame.cursor }); + const matches = predicate(snapshot); + if (current !== previous || snapshot.reason !== "idle" || !matches) stableSince = Date.now(); + previous = current; + if (matches && snapshot.reason === "idle" && Date.now() - stableSince >= settleMs) return snapshot; + // Polling cadence, not a substitute for an observed settled frame. + await Bun.sleep(Math.min(10, Math.max(0, deadline - Date.now()))); + } + throw new Error("Timed out waiting for a settled visible prompt"); +} + +export function sessionFor(run: HostRun): Session { + if (!run.session) throw new Error("Host session missing"); + return run.session; +} + +export async function typeText(run: HostRun, text: string, timeoutMs = 5_000) { + await bounded(sessionFor(run).keyboard.type(text), "keyboard type", timeoutMs); +} + +export async function pressKey(run: HostRun, key: Key, timeoutMs = 5_000) { + await bounded(sessionFor(run).keyboard.press(key), "keyboard press", timeoutMs); +} + +export async function checkpoint(run: HostRun, name: string, lines: string[], cursor?: { x: number; y: number }) { + let actual: VisiblePrompt | null = null; + try { + const capture = await waitVisible( + run, + (snapshot) => { + actual = visiblePrompt(snapshot); + return promptMatches(actual, lines, cursor); + }, + 5_000, + ); + run.receipt.checks.push({ name, status: "pass", expected: { lines, cursor }, actual }); + writeFileSync(join(run.output, run.scenario ?? "", `${name}.txt`), capture.text); + saveReceipt(run); + } catch (error) { + const message = `Checkpoint ${name}: expected ${JSON.stringify({ lines, cursor })}, last visible prompt ${JSON.stringify(actual)}`; + run.receipt.checks.push({ name, status: "fail", expected: { lines, cursor }, actual, error: message }); + saveReceipt(run); + throw new Error(message, { cause: error }); + } +} + +export async function saveFailure(run: HostRun, error: unknown) { + run.receipt.checks.push({ + name: run.scenario ?? "setup", + status: "fail", + error: error instanceof Error ? error.message : String(error), + }); + if (run.session) { + try { + const capture: ScreenSnapshot = await bounded( + run.session.screen.capture({ allowIncomplete: true, settleMs: 0, deadlineMs: 0 }), + "failure capture", + ); + const directory = join(run.output, run.scenario ?? ""); + writeFileSync(join(directory, "failure.txt"), capture.text); + writeFileSync(join(directory, "failure.json"), JSON.stringify(capture, null, 2)); + await bounded(run.session.saveRecording(join(directory, "failure.termctrl")), "failure recording"); + } catch (captureError) { + run.receipt.checks.push({ name: "failure-evidence", status: "fail", error: String(captureError) }); + } + } + saveReceipt(run); +} + +export async function stopScenario(run: HostRun) { + const errors: string[] = []; + if (run.session) { + try { + await bounded(run.session.stop(), "host stop"); + } catch (error) { + errors.push(String(error)); + } + } + if (run.driver) { + try { + await closeDriver(run.driver); + } catch (error) { + errors.push(String(error)); + } + // Never forget a live handle if the forced reap itself failed. + if (run.driver.reaped) { + run.session = undefined; + run.terminal = undefined; + run.driver = undefined; + } + } + run.receipt.cleanupErrors.push(...errors); + saveReceipt(run); + if (errors.length) throw new Error(`Owned E2E session cleanup failed: ${errors.join("; ")}`); +} + +export async function scenario(run: HostRun, name: string, test: () => Promise) { + try { + await startScenario(run, name); + await test(); + } catch (error) { + await saveFailure(run, error); + throw error; + } finally { + await stopScenario(run); + } +} diff --git a/test/e2e/launcher.integration.ts b/test/e2e/launcher.integration.ts new file mode 100644 index 0000000..22c46ea --- /dev/null +++ b/test/e2e/launcher.integration.ts @@ -0,0 +1,210 @@ +import { expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + binarySettings, + freshRunRoot, + isolatedToolEnv, + nativeSource, + outputPath, + pins, + resolveBinary, + runHosts, + selectHosts, + verifiedBinary, +} from "../../scripts/test-e2e"; + +function fixture() { + return mkdtempSync(join(realpathSync(tmpdir()), "vimcode-launcher-probe-")); +} + +it("defaults to both exact pins, supports one host and existing four-argument CI calls", () => { + expect(pins).toEqual({ v1: "1.18.34", v2: "2.0.15" }); + expect(selectHosts([])).toEqual({ hosts: ["v1", "v2"] }); + expect(selectHosts(["", "", "", ""])).toEqual({ hosts: ["v1", "v2"] }); + for (const host of ["v1", "v2"] as const) { + expect(selectHosts([host, "", "", ""])).toEqual({ hosts: [host] }); + expect(selectHosts([host, "/native/opencode", pins[host], "/outside/output"])).toEqual({ + hosts: [host], + explicit: { binary: "/native/opencode", version: pins[host], output: "/outside/output" }, + }); + } +}); + +it("rejects bad selectors, incomplete explicit arguments, relative CI paths and unpinned versions", () => { + for (const args of [ + ["v3"], + ["v1", "/binary"], + ["", "/binary", pins.v1, "/output"], + ["v2", "/binary", pins.v2, "/output", "extra"], + ]) + expect(() => selectHosts(args)).toThrow("Use just test-e2e"); + for (const args of [ + ["v1", "relative", pins.v1, "/output"], + ["v2", "/binary", "latest", "/output"], + ]) + expect(() => selectHosts(args)).toThrow("requires absolute binary/output paths and pinned version"); +}); + +it("parses synthetic dotenv as data, with process precedence, only path keys, and CI file exclusion", () => { + const text = `# synthetic, never a user's file +export VIMCODE_E2E_V1_BIN="./host one" # comment +VIMCODE_E2E_V2_BIN='./$(touch must-not-run)' +SYNTHETIC_PROVIDER_SECRET=do-not-forward +OPENCODE_CONFIG_CONTENT=do-not-forward +VIMCODE_E2E_VERSION=latest +UNKNOWN_KEY=do-not-forward +`; + expect(binarySettings({}, text)).toEqual({ v1: "./host one", v2: "./$(touch must-not-run)" }); + expect(binarySettings({ VIMCODE_E2E_V1_BIN: "/process/host" }, text)).toEqual({ + v1: "/process/host", + v2: "./$(touch must-not-run)", + }); + expect(binarySettings({ CI: "true" }, text)).toEqual({}); + expect(binarySettings({ CI: "1", VIMCODE_E2E_V2_BIN: "/ci/host" }, text)).toEqual({ v2: "/ci/host" }); + expect(binarySettings({ VIMCODE_E2E_V1_BIN: "" }, text)).toEqual({ v1: "", v2: "./$(touch must-not-run)" }); + expect(binarySettings({}, 'VIMCODE_E2E_V1_BIN="line\\nnext"')).toEqual({ v1: "line\nnext" }); +}); + +it("Bun --env-file=/dev/null really excludes synthetic local secrets from the launched child", () => { + const directory = fixture(); + try { + writeFileSync(join(directory, ".env"), "SYNTHETIC_PROVIDER_SECRET=do-not-forward\nVIMCODE_E2E_V1_BIN=./host\n"); + const env = isolatedToolEnv(join(directory, "tools")); + const probe = + "process.stdout.write(JSON.stringify({secret:process.env.SYNTHETIC_PROVIDER_SECRET,bin:process.env.VIMCODE_E2E_V1_BIN}))"; + const autoload = Bun.spawnSync([process.execPath, "-e", probe], { cwd: directory, env, stdout: "pipe" }); + expect(autoload.exitCode).toBe(0); + expect(JSON.parse(new TextDecoder().decode(autoload.stdout))).toEqual({ secret: "do-not-forward", bin: "./host" }); + const isolated = Bun.spawnSync([process.execPath, "--env-file=/dev/null", "-e", probe], { + cwd: directory, + env, + stdout: "pipe", + }); + expect(isolated.exitCode).toBe(0); + expect(JSON.parse(new TextDecoder().decode(isolated.stdout))).toEqual({}); + expect(env).not.toHaveProperty("SYNTHETIC_PROVIDER_SECRET"); + expect(env).not.toHaveProperty("VIMCODE_E2E_V1_BIN"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +it("infers unique canonical OS-temp outputs and rejects nonempty/config ancestors", () => { + const first = freshRunRoot(); + const second = freshRunRoot(); + try { + expect(first).not.toBe(second); + expect(first.startsWith(join(realpathSync(tmpdir()), "vimcode-e2e-"))).toBe(true); + expect(outputPath(join(first, "v1"))).toBe(join(first, "v1")); + writeFileSync(join(first, "evidence.txt"), "preserved"); + expect(() => outputPath(first)).toThrow("new or empty"); + mkdirSync(join(second, "project")); + mkdirSync(join(second, "project/.git")); + expect(() => outputPath(join(second, "project/new"))).toThrow("outside project/config ancestry"); + symlinkSync(join(second, "project"), join(first, "linked")); + expect(() => outputPath(join(first, "linked/new"))).toThrow("outside project/config ancestry"); + } finally { + rmSync(first, { recursive: true, force: true }); + rmSync(second, { recursive: true, force: true }); + } +}); + +it("binary overrides resolve relative to the repository base, canonicalize, and reject missing/wrong versions", () => { + const directory = fixture(); + try { + const env = isolatedToolEnv(join(directory, "tools")); + const binary = join(directory, "native"); + writeFileSync(binary, "#!/bin/sh\nprintf 'opencode v1.18.34\\n'\n", { mode: 0o755 }); + symlinkSync(binary, join(directory, "link")); + expect(verifiedBinary("./link", "v1", directory, env, directory)).toBe(binary); + expect(() => verifiedBinary("./missing", "v1", directory, env, directory)).toThrow("binary path does not exist"); + expect(() => verifiedBinary(binary, "v2", directory, env)).toThrow("Expected OpenCode v2 2.0.15"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +it("invalid selected-host settings fail without blocking another host or an unrelated selector", async () => { + const directory = fixture(); + try { + const env = isolatedToolEnv(join(directory, "tools")); + const binary = join(directory, "v2-native"); + writeFileSync(binary, "#!/bin/sh\nprintf 'opencode v2.0.15\\n'\n", { mode: 0o755 }); + for (const invalid of ["", " ", "line\nnext"]) { + const settings = binarySettings({ VIMCODE_E2E_V1_BIN: invalid, VIMCODE_E2E_V2_BIN: binary }); + const completed: string[] = []; + const execute = async (host: "v1" | "v2") => { + try { + resolveBinary(host, settings[host], directory, env); + completed.push(host); + return 0; + } catch (error) { + expect(String(error)).toContain("VIMCODE_E2E_V1_BIN must be a nonempty binary path"); + return 1; + } + }; + expect(await runHosts(selectHosts([]).hosts, execute)).toBe(1); + expect(completed).toEqual(["v2"]); + completed.length = 0; + expect(await runHosts(selectHosts(["v2"]).hosts, execute)).toBe(0); + expect(completed).toEqual(["v2"]); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +it("version probes are bounded even when an override stalls", () => { + const directory = fixture(); + try { + const env = isolatedToolEnv(join(directory, "tools")); + const binary = join(directory, "stalled"); + writeFileSync(binary, `#!${process.execPath}\nprocess.on("SIGTERM", () => {});\nsetInterval(() => {}, 1000);\n`, { + mode: 0o755, + }); + const start = Date.now(); + expect(() => verifiedBinary(binary, "v1", directory, env)).toThrow("ETIMEDOUT"); + expect(Date.now() - start).toBeLessThan(15_000); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}, 20_000); + +it("only bootstraps verified native shapes and gives unsupported platforms an actionable override", () => { + expect(nativeSource("v1", "darwin", "arm64")).toEqual({ + asset: "opencode-darwin-arm64.zip", + sha256: "8522b70f545184b3a8d97c5ca4f814093b2476d72aebfda8c48bcd072ec31d1b", + }); + expect(nativeSource("v1", "linux", "x64")).toEqual({ + asset: "opencode-linux-x64-baseline.tar.gz", + sha256: "24b0d458d21ef548b2752166303defcf7f4945b049fb4876ab78dfaf86d81b27", + }); + expect(nativeSource("v2", "darwin", "arm64")).toEqual({ package: "@opencode/cli-darwin-arm64" }); + expect(nativeSource("v2", "linux", "x64")).toEqual({ package: "@opencode/cli-linux-x64-baseline" }); + expect(() => nativeSource("v2", "linux", "arm64")).toThrow("set VIMCODE_E2E_V2_BIN"); + expect(() => nativeSource("v1", "win32", "x64")).toThrow("No verified bootstrap"); +}); + +it("both-host execution is sequential, continues after failure, and never turns negative exit 1 green", async () => { + const visited: string[] = []; + expect( + await runHosts(["v1", "v2"], async (host) => { + visited.push(host); + const child = Bun.spawn( + [process.execPath, "--env-file=/dev/null", "-e", `process.exit(${host === "v1" ? 1 : 0})`], + { + env: {}, + stdout: "ignore", + stderr: "ignore", + }, + ); + return child.exited; + }), + ).toBe(1); + expect(visited).toEqual(["v1", "v2"]); + expect(await runHosts(["v1", "v2"], async () => 0)).toBe(0); + expect(await runHosts(["v2"], async () => 1)).toBe(1); + expect(await runHosts(["v1", "v2"], async (host) => (host === "v2" ? 1 : 0))).toBe(1); +}); diff --git a/test/e2e/opencode.e2e.ts b/test/e2e/opencode.e2e.ts new file mode 100644 index 0000000..b5616c9 --- /dev/null +++ b/test/e2e/opencode.e2e.ts @@ -0,0 +1,102 @@ +import { beforeAll, describe, it } from "bun:test"; +import { checkpoint, type HostRun, prepareRun, pressKey, scenario, typeText, verifyInstalledArtifact } from "./host"; + +const negativeControl = process.env.VIMCODE_E2E_NO_PLUGIN === "1"; +// Leave time for bounded sends/captures and finally cleanup before Bun's outer timeout. +const scenarioTimeoutMs = 240_000; + +describe("Terminal Control / cache-installed OpenCode", () => { + let run: HostRun; + beforeAll(() => { + run = prepareRun(); + }, 60_000); + + it.skipIf(negativeControl)( + "cold tarball load and text insertion", + async () => { + await scenario(run, "cold-load", async () => { + verifyInstalledArtifact(run); + await typeText(run, "cold package loaded"); + await checkpoint(run, "cold-insert", ["cold package loaded"]); + }); + }, + scenarioTimeoutMs, + ); + + it( + "Escape / 0 / w / i / X edits exactly at the next word", + async () => { + await scenario(run, "word-motion", async () => { + await typeText(run, "one two three"); + await checkpoint(run, "word-insert", ["one two three"]); + await pressKey(run, "Escape"); + await checkpoint(run, "word-escape", ["one two three"]); + await typeText(run, "0"); + await checkpoint(run, "word-home", ["one two three"]); + await typeText(run, "w"); + await checkpoint(run, "word-next", ["one two three"]); + await typeText(run, "i"); + await checkpoint(run, "word-insert-mode", ["one two three"]); + await typeText(run, "X"); + // Cursor-only SDK frames can be stale; the editing effect proves 0/w placement. + await checkpoint(run, "word-result", ["one Xtwo three"]); + }); + }, + scenarioTimeoutMs, + ); + + it.skipIf(negativeControl)( + "ci quote replaces only the quoted contents", + async () => { + await scenario(run, "quote-change", async () => { + await typeText(run, 'say "old words" now'); + await checkpoint(run, "quote-insert", ['say "old words" now']); + await pressKey(run, "Escape"); + await checkpoint(run, "quote-escape", ['say "old words" now']); + await typeText(run, "0"); + await checkpoint(run, "quote-home", ['say "old words" now']); + await typeText(run, "w"); + await checkpoint(run, "quote-next", ['say "old words" now']); + await typeText(run, 'ci"'); + await checkpoint(run, "quote-empty", ['say "" now']); + await typeText(run, "new"); + await checkpoint(run, "quote-result", ['say "new" now']); + }); + }, + scenarioTimeoutMs, + ); + + it.skipIf(negativeControl)( + "multiline dG and one u restore the entire prompt", + async () => { + await scenario(run, "multiline-undo", async () => { + await typeText(run, "alpha"); + await checkpoint(run, "line-one", ["alpha"]); + // Providers are disabled; also prove Vim handling is live before Enter. + await pressKey(run, "Escape"); + await checkpoint(run, "newline-guard", ["alpha"], { x: 4, y: 0 }); + await typeText(run, "A"); + await checkpoint(run, "newline-insert-mode", ["alpha"], { x: 5, y: 0 }); + await pressKey(run, "Enter"); + await checkpoint(run, "newline-one", ["alpha", ""], { x: 0, y: 1 }); + await typeText(run, "bravo"); + await checkpoint(run, "line-two", ["alpha", "bravo"]); + await pressKey(run, "Enter"); + await checkpoint(run, "newline-two", ["alpha", "bravo", ""], { x: 0, y: 2 }); + await typeText(run, "charlie"); + await checkpoint(run, "line-three", ["alpha", "bravo", "charlie"]); + await pressKey(run, "Escape"); + await checkpoint(run, "multiline-escape", ["alpha", "bravo", "charlie"], { x: 6, y: 2 }); + await typeText(run, "gg"); + await checkpoint(run, "multiline-home", ["alpha", "bravo", "charlie"], { x: 0, y: 0 }); + await typeText(run, "j"); + await checkpoint(run, "multiline-down", ["alpha", "bravo", "charlie"], { x: 0, y: 1 }); + await typeText(run, "dG"); + await checkpoint(run, "multiline-delete", ["alpha", ""], { x: 0, y: 1 }); + await typeText(run, "u"); + await checkpoint(run, "multiline-restored", ["alpha", "bravo", "charlie"], { x: 0, y: 1 }); + }); + }, + scenarioTimeoutMs, + ); +}); diff --git a/test/e2e/prompt.ts b/test/e2e/prompt.ts new file mode 100644 index 0000000..da3b2f5 --- /dev/null +++ b/test/e2e/prompt.ts @@ -0,0 +1,58 @@ +import type { ScreenSnapshot } from "@kitlangton/terminal-control"; + +export type VisiblePrompt = { + lines: string[]; + origin: { x: number; y: number }; + cursor: { x: number; y: number; style: string } | null; +}; + +// Both pinned hosts use this bordered home prompt; arbitrary terminal text is not an oracle. +// Terminal padding cannot distinguish trailing buffer spaces, so cases use short ASCII lines. +export function visiblePrompt(snapshot: ScreenSnapshot): VisiblePrompt | null { + const rows = snapshot.text.split("\n"); + const metadata = rows.flatMap((row, y) => { + const match = /^( *)┃ {2}(?:Build|Plan) · /.exec(row); + return match ? [{ y, x: match[1].length }] : []; + }); + if (metadata.length !== 1) return null; + const { x, y } = metadata[0]; + const border = `${" ".repeat(x)}┃`; + if (!rows[y + 1]?.startsWith(`${" ".repeat(x)}╹▀`)) return null; + let top = y; + while (top > 0 && rows[top - 1].startsWith(border)) top--; + if (y - top < 3) return null; + const content = (row: string) => row.slice(x + 3).trimEnd(); + if (content(rows[top]) !== "" || content(rows[y - 1]) !== "") return null; + const lines = rows.slice(top + 1, y - 1).map(content); + const origin = { x: x + 3, y: top + 1 }; + const cursor = snapshot.frame.cursor; + const cursorInPrompt = cursor && cursor.x >= origin.x && cursor.y >= origin.y && cursor.y < y - 1; + return { + lines, + origin, + cursor: cursorInPrompt ? { x: cursor.x - origin.x, y: cursor.y - origin.y, style: cursor.style } : null, + }; +} + +export function providerDisabledPrompt(snapshot: ScreenSnapshot): boolean { + const prompt = visiblePrompt(snapshot); + if (!prompt) return false; + const metadataRow = prompt.origin.y + prompt.lines.length + 1; + return ( + snapshot.text.split("\n")[metadataRow].slice(prompt.origin.x).trimEnd() === + "Build · No provider selected Connect a provider" + ); +} + +export function promptMatches( + prompt: VisiblePrompt | null, + lines: string[], + cursor?: { x: number; y: number }, +): boolean { + return ( + prompt !== null && + prompt.lines.length === lines.length && + prompt.lines.every((line, index) => line === lines[index]) && + (!cursor || (prompt.cursor?.x === cursor.x && prompt.cursor.y === cursor.y)) + ); +} diff --git a/test/e2e/prompt.unit.ts b/test/e2e/prompt.unit.ts new file mode 100644 index 0000000..478f76b --- /dev/null +++ b/test/e2e/prompt.unit.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "bun:test"; +import type { ScreenSnapshot } from "@kitlangton/terminal-control"; +import { promptMatches, providerDisabledPrompt, visiblePrompt } from "./prompt"; + +function screen(lines: string[], outside = "one Xtwo three"): ScreenSnapshot { + return { + reason: "idle", + text: [outside, " ┃", ...lines.map((line) => ` ┃ ${line}`), " ┃", " ┃ Build · Test model", " ╹▀▀▀▀▀"].join( + "\n", + ), + frame: { + version: 2, + cols: 110, + rows: 34, + foreground: { r: 255, g: 255, b: 255 }, + background: { r: 0, g: 0, b: 0 }, + cursor: { x: 5, y: 2, style: "block", blinking: false, color: { r: 255, g: 255, b: 255 } }, + cells: [], + }, + }; +} + +describe("pinned-host home prompt screen oracle", () => { + it("ignores matching text outside the prompt and rejects a matching suffix", () => { + expect(promptMatches(visiblePrompt(screen(["wrong"])), ["one Xtwo three"])).toBe(false); + expect(promptMatches(visiblePrompt(screen(["extra one Xtwo three"])), ["one Xtwo three"])).toBe(false); + }); + + it("compares every line, including empty lines and internal whitespace", () => { + const prompt = visiblePrompt(screen(["alpha beta", "", "gamma"])); + expect(promptMatches(prompt, ["alpha beta", "", "gamma"])).toBe(true); + expect(promptMatches(prompt, ["alpha beta", "", "gamma"])).toBe(false); + expect(promptMatches(prompt, ["alpha beta", "gamma"])).toBe(false); + }); + + it("requires provider-free metadata inside the strict bordered prompt for both empty placeholders", () => { + for (const placeholder of ["Ask", "Ask anything..."]) { + const snapshot = screen([placeholder]); + const disabled = { + ...snapshot, + text: snapshot.text.replace("Build · Test model", "Build · No provider selected Connect a provider"), + }; + expect(promptMatches(visiblePrompt(disabled), [placeholder])).toBe(true); + expect(providerDisabledPrompt(disabled)).toBe(true); + expect(providerDisabledPrompt(snapshot)).toBe(false); + expect( + providerDisabledPrompt({ + ...snapshot, + text: `Build · No provider selected Connect a provider\n${snapshot.text}`, + }), + ).toBe(false); + expect(providerDisabledPrompt({ ...disabled, text: disabled.text.replace("╹", "x") })).toBe(false); + expect(providerDisabledPrompt({ ...disabled, text: `${disabled.text}\n ┃ Build · Duplicate` })).toBe(false); + } + }); + + it("normalizes a visible cursor relative to the prompt", () => { + const prompt = visiblePrompt(screen(["abc"])); + expect(promptMatches(prompt, ["abc"], { x: 0, y: 0 })).toBe(true); + expect(promptMatches(prompt, ["abc"], { x: 1, y: 0 })).toBe(false); + }); + + it("rejects missing or ambiguous borders", () => { + const snapshot = screen(["abc"]); + expect(visiblePrompt({ ...snapshot, text: snapshot.text.replace("╹", "x") })).toBeNull(); + expect(visiblePrompt({ ...snapshot, text: `${snapshot.text}\n ┃ Build · Duplicate` })).toBeNull(); + }); + + it("keeps exact visible text authoritative when cursor metadata is missing or outside the editor", () => { + const snapshot = screen(["abc"]); + if (!snapshot.frame.cursor) throw new Error("fixture cursor missing"); + for (const cursor of [null, { ...snapshot.frame.cursor, y: 0 }]) { + const prompt = visiblePrompt({ ...snapshot, frame: { ...snapshot.frame, cursor } }); + expect(promptMatches(prompt, ["abc"])).toBe(true); + expect(prompt?.cursor).toBeNull(); + expect(promptMatches(prompt, ["abc"], { x: 0, y: 0 })).toBe(false); + } + }); +});