From 1427d9ecf3da75190931c351edca2a7347514351 Mon Sep 17 00:00:00 2001 From: Paulo Date: Sun, 6 Sep 2026 18:52:06 +0200 Subject: [PATCH] Add the acceptance kit for the secrets path The first acceptance kit tested the model call only, and that is how GitHub was missed. This kit covers what an agent does, from inside the box over SSH: the placeholders in a plain session, claude -p, gh on a private repository, git clone and a push of a throwaway branch, the placeholder sent straight to Anthropic, a wrong placeholder, the issuer failing, a restart of the exchange, and a clean delete. One run covers one provider, with an issuer-backed anthropic secret and an issuer-backed github secret. The anthropic value lives 70 seconds in the run, so the run sees a refresh and an outage. The github value lives an hour and carries the counts of one fetch per secret. The dummy issuer answers from files the operator writes, behind a bearer from the environment, and takes two control paths: a fetch count per service and a fail switch. A host carries a one hour lease, so a run that dies leaves no permanent host. Git ignores the token files. The kit lives in api-tests/acceptance, run by hand against a deployment like the black-box suite, and not in CI, since it needs a subscription token and an installation token. The runbook says what to set and what passes. --- AGENTS.md | 2 +- api-tests/acceptance/.gitignore | 3 + api-tests/acceptance/README.md | 102 +++++++++ api-tests/acceptance/check.py | 364 ++++++++++++++++++++++++++++++++ api-tests/acceptance/issuer.py | 71 +++++++ docs/deploy.md | 4 + 6 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 api-tests/acceptance/.gitignore create mode 100644 api-tests/acceptance/README.md create mode 100644 api-tests/acceptance/check.py create mode 100644 api-tests/acceptance/issuer.py diff --git a/AGENTS.md b/AGENTS.md index 85137bb..d1a5ae0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ src/ conftest.py # Test env defaults and database reset fixture alembic/ # Database migrations deploy/proxy/ # The secrets proxy: a mitmproxy addon, mounted into the official image -api-tests/ # Playwright black-box API tests +api-tests/ # Playwright black-box API tests, and the acceptance kit for the secrets path docs/ # Architecture, networking, deploy, add-a-provider Dockerfile # Single image: API + cron commands + migrations ``` diff --git a/api-tests/acceptance/.gitignore b/api-tests/acceptance/.gitignore new file mode 100644 index 0000000..3b50e3c --- /dev/null +++ b/api-tests/acceptance/.gitignore @@ -0,0 +1,3 @@ +# Token files. Never in the repository. +*.json +*.token diff --git a/api-tests/acceptance/README.md b/api-tests/acceptance/README.md new file mode 100644 index 0000000..e1566bd --- /dev/null +++ b/api-tests/acceptance/README.md @@ -0,0 +1,102 @@ +# Acceptance from inside the box + +Black-box checks of the secrets path, run by hand against a deployment. One +run covers one provider. The check creates a host with an issuer-backed +`anthropic` secret and an issuer-backed `github` secret. Then it works from +inside the box over SSH, and deletes the host at the end. + +Not in CI. A run needs a subscription token, an installation token for a +private repository, and a box image with `claude`, `git`, and `gh`. + +## What passes + +- `POST /hosts` answers 201 and echoes no secret. The host becomes active. +- The issuer is asked once for the github secret, and not before its first + use. On docker-sbx the API asks at provisioning, and the exchange asks again + when it first sees the host. +- The anthropic secret lives 70 seconds in this run, so the exchange asks for + it again and again. The outage step needs that. +- A plain session sees both placeholders. +- `claude -p` answers `ok`. +- `gh api` and `gh pr list` work on the private repository. +- `git clone`, a push of a throwaway branch, and its delete work. +- The placeholder sent straight to Anthropic, with the proxy bypassed, is + refused with 401. +- A wrong placeholder is refused: with 403 by the exchange, or with 401 by + Anthropic on docker-sbx, where nothing swaps it. +- While the issuer fails, the last value keeps the box working, and the + exchange kept asking the issuer in the meantime. +- A restart of the exchange costs one fetch of the github secret, and + `claude -p` and `gh` answer again. +- On docker, a restarted box comes back with the same git setup. +- On docker-sbx, every secret is scoped to the sandbox, and the value files + are 0600 beside the workspaces. Teardown leaves no secret in + `sbx secret ls`, no value file, and no workspace. +- `DELETE /hosts` answers 204. + +## The issuer + +`issuer.py` stands in for the service that mints tokens. It answers +`GET /mint/` with the JSON in `.json` next to it, behind the +bearer in `ISSUER_BEARER`. Write the token files yourself, with mode 0600, +and delete them when the run ends. Git ignores them. + +```bash +umask 077 +printf '{"value": "%s"}' "$SUBSCRIPTION_TOKEN" > api-tests/acceptance/anthropic.json +printf '{"value": "%s"}' "$INSTALLATION_TOKEN" > api-tests/acceptance/github.json +ISSUER_BEARER= python3 api-tests/acceptance/issuer.py 8791 +``` + +An `expires_at` in a file is optional. Without it the secret's refresh +interval sets the lifetime. The API takes issuer URLs over HTTPS only, so put +the stub behind a TLS front that the exchange trusts. Caddy with `tls +internal` and `reverse_proxy 127.0.0.1:8791` does it on one host. + +The check drives the stub through two control paths, behind the same bearer. +`GET /fetches` counts the mint requests per service. `POST /fail` with +`{"failing": true}` makes every mint answer 500 until `{"failing": false}`. + +## Run + +```bash +SERVICE_URL=http://localhost:8780 \ +SERVICE_TOKEN= \ +PROVIDER=docker \ +ISSUER_URL=https://localhost:8443 \ +ISSUER_BEARER= \ +ISSUER_CONTROL_URL=http://127.0.0.1:8791 \ +GITHUB_REPO=/ \ +RESTART_EXCHANGE='docker compose restart exchange' \ +python3 api-tests/acceptance/check.py +``` + +- `PROVIDER` is `docker`, `docker-sbx`, or `exe`. +- `ISSUER_URL` is the issuer as the exchange reaches it. The check appends + `/mint/anthropic` and `/mint/github`. +- `ISSUER_CONTROL_URL` is the issuer as the check reaches it, for the two + control paths. It defaults to `ISSUER_URL`. +- `RESTART_EXCHANGE` is a shell command that restarts the exchange process. +- `HOST_IMAGE` names the box image. `images/local` has `git` and `gh`. Add + Claude Code on top of it for this run. +- `SSH_KEY` is the account key for a provider whose boxes take it, such as + exe. +- `SBX_WORKSPACE_ROOT` turns on the docker-sbx checks. Run the check on the + sbx host then, where `sbx` and the workspace root are. +- `HOST_ACTIVE_TIMEOUT` is the wait for an active host, in seconds. The + default is 600. `POST /hosts` provisions before it answers, so the request + waits that long too. + +A box image for docker with Claude Code: + +```Dockerfile +FROM drukbox/sandbox:local +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g @anthropic-ai/claude-code \ + && rm -rf /var/lib/apt/lists/* +``` + +The check prints one line per step and a summary. It exits 1 when a step +fails. It deletes its host in every case. The host carries a one hour lease +in case the run dies. The throwaway branch is named after the host. diff --git a/api-tests/acceptance/check.py b/api-tests/acceptance/check.py new file mode 100644 index 0000000..a7121b9 --- /dev/null +++ b/api-tests/acceptance/check.py @@ -0,0 +1,364 @@ +"""Acceptance from inside the box, one provider per run. README.md says what to set.""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + + +def get_setting(name: str) -> str: + return os.environ.get(name) or sys.exit(f"set {name}") + + +SERVICE_URL = get_setting("SERVICE_URL").rstrip("/") +SERVICE_TOKEN = get_setting("SERVICE_TOKEN") +PROVIDER = get_setting("PROVIDER") +ISSUER_URL = get_setting("ISSUER_URL").rstrip("/") +ISSUER_BEARER = get_setting("ISSUER_BEARER") +ISSUER_CONTROL_URL = os.environ.get("ISSUER_CONTROL_URL", ISSUER_URL).rstrip("/") +GITHUB_REPO = get_setting("GITHUB_REPO") +RESTART_EXCHANGE = get_setting("RESTART_EXCHANGE") +HOST_ACTIVE_TIMEOUT = int(os.environ.get("HOST_ACTIVE_TIMEOUT", "600")) +HOST_IMAGE = os.environ.get("HOST_IMAGE", "") +SSH_KEY = get_setting("SSH_KEY") if PROVIDER == "exe" else "" +SBX_WORKSPACE_ROOT = os.environ.get("SBX_WORKSPACE_ROOT", "") +PROVIDER_NEEDS_VALUE = PROVIDER == "docker-sbx" + +# 70 seconds: a refresh and an outage in one run. An hour: one fetch, one more at a restart. +LIFETIMES = {"anthropic": "70s", "github": "1h"} +GIT = "GIT_TERMINAL_PROMPT=0 git -c user.name=drukbox -c user.email=drukbox@example.invalid" +PROMPT = "Reply with exactly the word ok and nothing else." +MODELS_URL = "https://api.anthropic.com/v1/models?limit=1" +results: list[tuple[str, bool, str]] = [] + + +def call_api( + method: str, path: str, body: dict | None = None, timeout: int = 600 +) -> tuple[int, dict]: + request = urllib.request.Request( + f"{SERVICE_URL}{path}", + method=method, + data=json.dumps(body).encode() if body else None, + headers={"Authorization": f"Bearer {SERVICE_TOKEN}", "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, (json.load(response) if response.status != 204 else {}) + except urllib.error.HTTPError as error: + print( + f" {method} {path} answered {error.code}: " + f"{error.read().decode(errors='replace')[:300]}" + ) + return error.code, {} + + +def call_issuer(path: str, body: dict | None = None) -> dict: + request = urllib.request.Request( + f"{ISSUER_CONTROL_URL}{path}", + method="POST" if body else "GET", + data=json.dumps(body).encode() if body else None, + headers={"Authorization": f"Bearer {ISSUER_BEARER}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def count_fetches(service: str) -> int: + return call_issuer("/fetches")["services"].get(service, 0) - at_start.get(service, 0) + + +def check(name: str, ok: bool, detail: str = "") -> None: + results.append((name, ok, detail)) + print(f" [{'ok' if ok else 'FAIL'}] {name} {detail}") + + +def run_on_host(command: str) -> str: + done = subprocess.run( + command, shell=True, capture_output=True, text=True, errors="replace", timeout=600 + ) + return (done.stdout + done.stderr).strip() + + +def run_in_box(host: dict, script: str) -> str: + with tempfile.TemporaryDirectory() as directory: + key = f"{directory}/key" + known = f"{directory}/known_hosts" + with open(key, "w") as file: + os.fchmod(file.fileno(), 0o600) + file.write(host["private_key"] or pathlib.Path(SSH_KEY).read_text()) + with open(known, "w") as file: + file.write(host["known_hosts"]) + command = [ + "ssh", + "-i", + key, + "-o", + f"UserKnownHostsFile={known}", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=60", + "-p", + str(host["external_ssh_port"]), + f"{host['ssh_username']}@{host['external_ssh_host']}", + script, + ] + done = subprocess.run( + command, capture_output=True, text=True, errors="replace", timeout=600 + ) + return done.stdout.strip() + ( + f"\n[stderr] {done.stderr.strip()[-300:]}" if done.returncode else "" + ) + + +def call_claude(host: dict) -> str: + script = ( + f"export ANTHROPIC_API_KEY=; timeout 120 claude -p {json.dumps(PROMPT)}" + " --output-format text &1 | tail -1" + ) + return run_in_box(host, script) + + +def get_status(host: dict, curl: str) -> str: + return run_in_box( + host, + f"curl -s -m 25 -o /dev/null -w '%{{http_code}}' {curl} -H 'anthropic-version: 2023-06-01'", + ) + + +def make_entry(service: str) -> dict: + return { + "issuer": { + "url": f"{ISSUER_URL}/mint/{service}", + "headers": {"Authorization": f"Bearer {ISSUER_BEARER}"}, + "refresh": LIFETIMES[service], + } + } + + +print(f"== {PROVIDER}") +at_start = call_issuer("/fetches")["services"] +# A lease, so a run that dies leaves no permanent host. +expires_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + 3600)) +body = { + "provider": PROVIDER, + "expires_at": expires_at, + "secrets": {"anthropic": make_entry("anthropic"), "github": make_entry("github")}, +} +if HOST_IMAGE: + body["image"] = HOST_IMAGE +status, host = call_api("POST", "/hosts", body, timeout=HOST_ACTIVE_TIMEOUT) +check( + "POST /hosts with two issuer-backed secrets answers 201, and echoes no secret", + status == 201 and "secrets" not in host, + str(status), +) +if not host: + sys.exit(1) +name = host["name"] +branch = f"drukbox-acceptance/{name}" +try: + deadline = time.time() + HOST_ACTIVE_TIMEOUT + while host["status"] not in ("active", "error") and time.time() < deadline: + time.sleep(5) + host = {**host, **call_api("GET", f"/hosts/{host['id']}")[1]} + check( + "the host becomes active", + host["status"] == "active", + f"{name} {host['status']} {host.get('last_error') or ''}", + ) + if PROVIDER_NEEDS_VALUE: + time.sleep(8) + check( + "the issuer was asked for each secret at provisioning, and again at first sight", + count_fetches("github") == 2 and count_fetches("anthropic") >= 2, + f"github={count_fetches('github')} anthropic={count_fetches('anthropic')}", + ) + else: + check( + "the issuer was not asked at boot", + count_fetches("github") + count_fetches("anthropic") == 0, + ) + + env = run_in_box( + host, 'echo "$ANTHROPIC_AUTH_TOKEN" | cut -c1-12; echo "$GH_TOKEN" | cut -c1-12' + ) + check( + "a plain session sees both placeholders", env.count("drk.") == 2, env.replace("\n", " | ") + ) + tools = run_in_box(host, "command -v claude git gh | wc -l") + check("claude, git, and gh are in the box", tools == "3", tools[-40:]) + + if SBX_WORKSPACE_ROOT: + listing = run_on_host(f"sbx secret ls --sandbox {name}") + check( + "sbx holds the github service secret and a custom secret for api.anthropic.com", + "github" in listing and "api.anthropic.com" in listing, + listing.replace("\n", " | ")[-120:], + ) + rows = [ + re.split(r"\s{2,}", line.strip()) for line in run_on_host("sbx secret ls").splitlines() + ] + check( + "no sbx secret is global", + not [row for row in rows if len(row) >= 4 and row[0] == "global"], + ) + modes = { + service: stat.S_IMODE(os.stat(f"{SBX_WORKSPACE_ROOT}/secrets/{name}/{service}").st_mode) + for service in ("anthropic", "github") + } + check( + "the value files are 0600 beside the workspaces", + all(mode == 0o600 for mode in modes.values()), + " ".join(f"{service}={mode:o}" for service, mode in modes.items()), + ) + + answer = call_claude(host) + check("claude -p answers ok", answer.endswith("ok"), answer[-60:]) + check( + "gh api reaches the private repository", + run_in_box(host, f"gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1") + == GITHUB_REPO, + ) + check( + "gh pr list works", + run_in_box(host, f"gh pr list -R {GITHUB_REPO} --limit 3 >/dev/null 2>&1 && echo listed") + == "listed", + ) + check( + "the issuer was asked once for the github secret", + count_fetches("github") == (2 if PROVIDER_NEEDS_VALUE else 1), + f"github={count_fetches('github')}", + ) + + clone = run_in_box( + host, + f"rm -rf /tmp/repo && {GIT} clone -q https://github.com/{GITHUB_REPO}.git /tmp/repo 2>&1" + " && echo cloned", + ) + check("git clone of the private repository", clone.endswith("cloned"), clone[-100:]) + push = run_in_box( + host, + f"cd /tmp/repo && {GIT} checkout -q -b {branch} && date > acceptance.txt" + f" && {GIT} add acceptance.txt && {GIT} commit -q -m acceptance" + f" && {GIT} push -q origin {branch} 2>&1 && echo pushed", + ) + check("git push of a throwaway branch", push.endswith("pushed"), push[-100:]) + delete = run_in_box( + host, f"cd /tmp/repo && {GIT} push -q origin --delete {branch} 2>&1 && echo deleted" + ) + check("the throwaway branch is deleted", delete.endswith("deleted"), delete[-100:]) + + direct = get_status( + host, f"--noproxy '*' {MODELS_URL} -H \"Authorization: Bearer $ANTHROPIC_AUTH_TOKEN\"" + ) + check("the placeholder sent straight to Anthropic is refused with 401", direct == "401", direct) + wrong = get_status( + host, f'{MODELS_URL} -H "Authorization: Bearer ${{ANTHROPIC_AUTH_TOKEN%??}}xx"' + ) + check( + "a wrong placeholder is refused" + + ("" if PROVIDER_NEEDS_VALUE else " with 403 by the exchange"), + wrong == ("401" if PROVIDER_NEEDS_VALUE else "403"), + wrong, + ) + + # A fresh value first, then the issuer fails while that value nears its end. + answer = call_claude(host) + asked = count_fetches("anthropic") + call_issuer("/fail", {"failing": True}) + try: + time.sleep(20) + answer = call_claude(host) + time.sleep(2) + check( + "while the issuer fails, the last value keeps the box working", + answer.endswith("ok") and count_fetches("anthropic") > asked, + f"{answer[-40:]} failed attempts={count_fetches('anthropic') - asked}", + ) + finally: + call_issuer("/fail", {"failing": False}) + + asked = count_fetches("github") + restarted = subprocess.run( + RESTART_EXCHANGE, shell=True, capture_output=True, text=True, timeout=300 + ) + check( + "the exchange restarts", + restarted.returncode == 0, + (restarted.stdout + restarted.stderr).strip()[-100:], + ) + time.sleep(10) + answer = call_claude(host) + check("after the restart claude -p answers ok again", answer.endswith("ok"), answer[-60:]) + check( + "after the restart gh still works", + run_in_box(host, f"gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1") + == GITHUB_REPO, + ) + check( + "the restart cost one fetch of the github secret", + count_fetches("github") - asked == 1, + f"github={count_fetches('github') - asked}", + ) + + if PROVIDER == "docker" and shutil.which("docker"): + # The entrypoint runs again at a restart: sshd comes back, the git setup stays single. + restarted = subprocess.run( + ["docker", "restart", name], capture_output=True, text=True, timeout=120 + ) + time.sleep(3) + port = ( + subprocess.run(["docker", "port", name, "22"], capture_output=True, text=True) + .stdout.strip() + .rsplit(":", 1)[-1] + ) + host = { + **host, + "external_ssh_port": int(port), + "known_hosts": host["known_hosts"].replace( + f"]:{host['external_ssh_port']} ", f"]:{port} " + ), + } + helpers = run_in_box( + host, + "git config --system --get-all credential.https://github.com.helper | wc -l;" + f" gh api repos/{GITHUB_REPO} --jq .full_name 2>&1 | tail -1", + ) + check( + "a restarted box comes back with the same git setup and gh still works", + restarted.returncode == 0 and helpers.startswith("2") and GITHUB_REPO in helpers, + helpers.replace("\n", " | ")[-80:], + ) +finally: + status, _ = call_api("DELETE", f"/hosts/{host['id']}") + check("DELETE /hosts answers 204", status == 204, str(status)) + if SBX_WORKSPACE_ROOT: + check( + "teardown leaves no secret in sbx secret ls", + "No secrets found" in run_on_host(f"sbx secret ls --sandbox {name}"), + ) + check( + "teardown leaves no value file and no workspace", + not os.path.exists(f"{SBX_WORKSPACE_ROOT}/secrets/{name}") + and not os.path.exists(f"{SBX_WORKSPACE_ROOT}/{name}"), + ) + +failed = [check_name for check_name, ok, _ in results if not ok] +print( + f"== {PROVIDER}: {len(results) - len(failed)}/{len(results)} passed" + + (f", failed: {', '.join(failed)}" if failed else "") +) +sys.exit(1 if failed else 0) diff --git a/api-tests/acceptance/issuer.py b/api-tests/acceptance/issuer.py new file mode 100644 index 0000000..2df3d23 --- /dev/null +++ b/api-tests/acceptance/issuer.py @@ -0,0 +1,71 @@ +"""A dummy issuer: GET /mint/ answers the JSON in .json from DIRECTORY. + +usage: ISSUER_BEARER= issuer.py PORT [DIRECTORY] + +Control, behind the same bearer: GET /fetches counts the mint requests per +service. POST /fail with {"failing": true} makes every mint answer 500 until +{"failing": false}. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +PORT = int(sys.argv[1]) +BEARER = os.environ.get("ISSUER_BEARER") or sys.exit("set ISSUER_BEARER") +DIRECTORY = sys.argv[2] if len(sys.argv) > 2 else os.path.dirname(os.path.abspath(__file__)) +state: dict = {"fetches": 0, "services": {}, "failing": False} + + +class Issuer(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if not self.is_authorized(): + return self.answer(403) + if self.path == "/fetches": + return self.answer(200, {"fetches": state["fetches"], "services": state["services"]}) + if match := re.fullmatch(r"/mint/([a-z0-9-]+)", self.path): + return self.mint(match.group(1)) + return self.answer(404) + + def do_POST(self) -> None: + if not self.is_authorized(): + return self.answer(403) + if self.path != "/fail": + return self.answer(404) + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0")) or b"{}")) + state["failing"] = bool(body.get("failing")) + print(f"failing={state['failing']}", flush=True) + return self.answer(200, {"failing": state["failing"]}) + + def mint(self, service: str) -> None: + state["fetches"] += 1 + state["services"][service] = state["services"].get(service, 0) + 1 + print(f"fetch #{state['fetches']} {service} failing={state['failing']}", flush=True) + if state["failing"]: + return self.answer(500) + try: + with open(os.path.join(DIRECTORY, f"{service}.json")) as file: + return self.answer(200, json.load(file)) + except FileNotFoundError: + print(f"no {service}.json in {DIRECTORY}", flush=True) + return self.answer(404) + + def is_authorized(self) -> bool: + return self.headers.get("Authorization") == f"Bearer {BEARER}" + + def answer(self, status: int, body: dict | None = None) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + if body: + self.wfile.write(json.dumps(body).encode()) + + def log_message(self, *args: object) -> None: + pass + + +HTTPServer(("127.0.0.1", PORT), Issuer).serve_forever() diff --git a/docs/deploy.md b/docs/deploy.md index 941f8e9..724076a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -406,6 +406,10 @@ infrastructure only): SERVICE_URL=http://localhost:8780 SERVICE_TOKEN=... npm --prefix api-tests test ``` +The secrets path has its own check, run by hand from inside a box. It runs +`claude -p`, `git push`, and `gh` through one provider's secrets, with a dummy +issuer. `api-tests/acceptance/README.md` says what to set and what passes. + ## Configuration reference Core, required: