diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a90f82..93d595b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,17 +141,15 @@ jobs: run: | cargo build --release --locked -p pyo-envd --target x86_64-unknown-linux-musl cargo build --locked -p pyo-sandbox - # Pyo's built-in template `base`, from this branch's Dockerfile. - - name: Build the base template image - run: | - docker build -f deploy/images/sandbox-base.Dockerfile -t localhost:5000/pyo/sandbox-base:e2e deploy/images - docker push localhost:5000/pyo/sandbox-base:e2e - docker image ls localhost:5000/pyo/sandbox-base:e2e + # Pyo's built-in template `base`, from the images.json Pyo ships with + # (upsetdev/pyo-sandbox-images), pulled from ghcr.io. - name: pyo-sandbox e2e env: PYO_SANDBOX_E2E: "1" - PYO_SANDBOX_E2E_BASE_IMAGE: 127.0.0.1:5000/pyo/sandbox-base:e2e - run: cargo test --locked -p pyo-sandbox --test e2e --test template_e2e -- --nocapture + run: | + PYO_SANDBOX_E2E_BASE_IMAGE=$(jq -r .images.base.ref crates/pyo-api/assets/sandbox-images/images.json) + export PYO_SANDBOX_E2E_BASE_IMAGE + cargo test --locked -p pyo-sandbox --test e2e --test template_e2e -- --nocapture - name: pyo-api sandbox e2e env: PYO_SANDBOX_E2E: "1" @@ -235,42 +233,6 @@ jobs: sudo nft list ruleset | head -80 || true ip netns list || true - # The base sandbox template's image for both architectures, built - # without pushing, when its Dockerfile changes. The release publishes it. - sandbox-base: - name: Base sandbox template image - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - name: Did the Dockerfile change - id: changed - env: - PR_BASE: ${{ github.event.pull_request.base.sha }} - BEFORE: ${{ github.event.before }} - run: | - base=${PR_BASE:-$BEFORE} - if [ -z "$base" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null || - git diff --name-only "$base" HEAD | grep -qx deploy/images/sandbox-base.Dockerfile; then - echo "build=true" >> "$GITHUB_OUTPUT" - else - echo "Its Dockerfile didn't change." - fi - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 - if: steps.changed.outputs.build == 'true' - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - if: steps.changed.outputs.build == 'true' - - name: Build - if: steps.changed.outputs.build == 'true' - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: deploy/images - file: deploy/images/sandbox-base.Dockerfile - platforms: linux/amd64,linux/arm64 - push: false - waf: name: WAF runs-on: ubuntu-24.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5115726..ef86eef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: strategy: fail-fast: true matrix: - image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static, sandbox-base] + image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static] # Pyo's Postgres, one image per version, from one Dockerfile. # One repository, tagged -: CloudNativePG reads # the version from the tag. @@ -111,9 +111,6 @@ jobs: # (deploy/images/moco), one Dockerfile with a target each. - { image: mysql-operator, file: moco/moco, context: deploy/images/moco, target: operator } - { image: mysql-backup, file: moco/moco, context: deploy/images/moco, target: backup } - # Pyo Sandbox's built-in template `base`. Sandbox servers run on - # both architectures, so it's built for both (arm64 under QEMU). - - { image: sandbox-base, context: deploy/images, platforms: "linux/amd64,linux/arm64" } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 @@ -209,7 +206,7 @@ jobs: tagged() { docker buildx imagetools inspect "$IMAGES/$1:$2$VERSION" --format '{{json .Manifest}}' | jq -r .digest; } bun run scripts/release-manifest.ts --version "$VERSION" --images "$IMAGES" \ --pyo "$(digest pyo)" --agent "$(digest agent)" --builder "$(digest builder)" --flows "$(digest flows)" --waf "$(digest waf)" --geoip "$(digest geoip)" \ - --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" --sandbox-base "$(digest sandbox-base)" \ + --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" \ --mysql-operator "$(digest mysql-operator)" --mysql-backup "$(digest mysql-backup)" \ --postgres-16 "$(tagged postgres 16-)" --postgres-17 "$(tagged postgres 17-)" --postgres-18 "$(tagged postgres 18-)" \ --agent-access "$(docker run --rm "$IMAGES/agent:$VERSION@$(digest agent)" access-hash)" \ diff --git a/Cargo.lock b/Cargo.lock index 4697e3e..5d770c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pyo-agent" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base64 0.23.1", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "pyo-api" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "argon2 0.6.0", @@ -3413,7 +3413,7 @@ dependencies = [ [[package]] name = "pyo-builder" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base64 0.23.1", @@ -3432,7 +3432,7 @@ dependencies = [ [[package]] name = "pyo-cli" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "clap", @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "pyo-db-controller" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base64 0.23.1", @@ -3474,7 +3474,7 @@ dependencies = [ [[package]] name = "pyo-envd" -version = "0.4.1" +version = "0.4.2" dependencies = [ "libc", "pyo-proto", @@ -3483,7 +3483,7 @@ dependencies = [ [[package]] name = "pyo-flows" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "libc", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "pyo-kv" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "axum", @@ -3514,7 +3514,7 @@ dependencies = [ [[package]] name = "pyo-proto" -version = "0.4.1" +version = "0.4.2" dependencies = [ "base64 0.23.1", "regex", @@ -3528,7 +3528,7 @@ dependencies = [ [[package]] name = "pyo-resp" -version = "0.4.1" +version = "0.4.2" dependencies = [ "futures", "socket2", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "pyo-sandbox" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "axum", @@ -3567,7 +3567,7 @@ dependencies = [ [[package]] name = "pyo-static" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 835adc5..cd5ef6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/pyo-proto", "crates/pyo-api", "crates/pyo-agent", "crates/pyo-builder", "crates/pyo-cli", "crates/pyo-flows", "crates/pyo-resp", "crates/pyo-kv", "crates/pyo-db-controller", "crates/pyo-static", "crates/pyo-envd", "crates/pyo-sandbox"] [workspace.package] -version = "0.4.1" +version = "0.4.2" edition = "2024" license = "Apache-2.0" publish = false diff --git a/RELEASING.md b/RELEASING.md index 84b8e9e..07a7510 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -19,7 +19,7 @@ A version is one number for everything: `version` in the root `Cargo.toml`. `.github/workflows/release.yml` then checks that the tag matches `Cargo.toml`, runs the tests (the WAF's Go tests in `waf/` too), publishes signed images -to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup,sandbox-base}` +to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup}` (all from `deploy/images/`, at the same version. The Rust binaries are compiled once, by the `binaries` job, and the Rust images' jobs take them from that run's build cache), @@ -53,13 +53,26 @@ crates/pyo-sandbox/scripts/bundle.sh --arch x86_64 \ `sandbox.sh --bundle dist/pyo-sandbox-linux-x86_64.tar.gz` installs such a build, with `--bundle-sha256 ` or, unchecked, `--dev`. -`sandbox-base` is Pyo Sandbox's built-in template `base` -(`deploy/images/sandbox-base.Dockerfile`), built for amd64 and arm64. -`release.json` pins it as `images.sandboxBase`, and Pyo seeds the template -from it. Sandbox servers pull it without logging in, so the release checks -that every image can be pulled anonymously. The first time a new image is -published, ghcr.io may make its package private: make it public in the -package's settings, then run the failed job again. +Installations and sandbox servers pull every image without logging in, +so the release checks that each can be pulled anonymously. The first +time a new image is published, ghcr.io may make its package private: +make it public in the package's settings, then run the failed job again. + +Pyo Sandbox's built-in templates `base` and `desktop` aren't built here. +They come from [upsetdev/pyo-sandbox-images](https://github.com/upsetdev/pyo-sandbox-images), +whose signed `images.json` Pyo reads itself every hour. Pyo ships the +`images.json` of when it was built, in +`crates/pyo-api/assets/sandbox-images/` with its signature (a test checks +it), for installs that can't reach GitHub. `release.json` names those +images as `images.sandboxBase` and `images.sandboxDesktop`, as Pyo 0.4 +reads them there. Before a release, take the latest: + +```sh +cd crates/pyo-api/assets/sandbox-images +curl -fsSLO https://github.com/upsetdev/pyo-sandbox-images/releases/latest/download/images.json +curl -fsSLO https://github.com/upsetdev/pyo-sandbox-images/releases/latest/download/images.json.sigstore.json +cargo test -p pyo-api sandbox::images +``` `release.json` is signed with cosign keyless as `release.json.sigstore.json` (the job needs `id-token: diff --git a/crates/pyo-agent/Cargo.toml b/crates/pyo-agent/Cargo.toml index f8deb0a..5d4ee7f 100644 --- a/crates/pyo-agent/Cargo.toml +++ b/crates/pyo-agent/Cargo.toml @@ -7,7 +7,7 @@ publish.workspace = true [dependencies] anyhow = "1.0.104" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } futures = "0.3.34" http = "1.5.0" k8s-openapi = { version = "0.28.0", features = ["latest"] } diff --git a/crates/pyo-api/Cargo.toml b/crates/pyo-api/Cargo.toml index 2256875..51ef554 100644 --- a/crates/pyo-api/Cargo.toml +++ b/crates/pyo-api/Cargo.toml @@ -12,7 +12,7 @@ axum = { version = "0.8.9", features = ["ws"] } axum-extra = { version = "0.12.6", features = ["cookie", "typed-header"] } base64 = "0.23.1" cookie = "0.18" -pyo-proto = { version = "0.4.1", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } +pyo-proto = { version = "0.4.2", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } hex = "0.4.3" hmac = "0.13.0" jiff = "0.2.38" diff --git a/crates/pyo-api/assets/connect.sh b/crates/pyo-api/assets/connect.sh index 4148386..8f693fb 100644 --- a/crates/pyo-api/assets/connect.sh +++ b/crates/pyo-api/assets/connect.sh @@ -11,6 +11,7 @@ # --tailscale-key joins your tailnet first (a single-use key from Pyo) # --registry Pyo's registry door, for images Pyo builds # --registry-token this cluster's login there (pull only) +# --version the Pyo release, for pyo.dev to run its script (ignored here) # # Or only joins this machine to the tailnet (Pyo's own machine does this # once Tailscale is set up in Pyo): --tailnet-only, with TS_AUTHKEY. @@ -49,11 +50,16 @@ become_root() { exit 1 } # <<< pyo:become-root +# Before anything else is defined: piped in, the root shell reads on +# from here, without what came before. +# --uninstall uses kubectl as you with KUBECONFIG set, or without k3s here. +case " $* " in *" --uninstall "*) [ -n "${KUBECONFIG:-}" ] || ! command -v k3s >/dev/null || become_root "$@" ;; *) become_root "$@" ;; esac # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) # A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), the same line comes -# every 20 seconds. progress_start MESSAGE, then progress_tick as often as -# it likes, then progress_end. +# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line +# comes once (not again for a step with the same line), then with its time +# every 20 seconds while it runs. progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end. progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -64,8 +70,12 @@ progress_tick() { if [ -t 1 ]; then printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) elif [ "$t" -ge "$PROGRESS_NEXT" ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) - PROGRESS_NEXT=$((t + 20)) + if [ "$PROGRESS_NEXT" -gt 0 ]; then + printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then + printf '%s\n' "$PROGRESS_MSG" + fi + PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } @@ -114,8 +124,6 @@ quietly() { return "$rc" } # <<< pyo:progress -# --uninstall uses kubectl as you with KUBECONFIG set, or without k3s here. -case " $* " in *" --uninstall "*) [ -n "${KUBECONFIG:-}" ] || ! command -v k3s >/dev/null || become_root "$@" ;; *) become_root "$@" ;; esac set -euo pipefail export DEBIAN_FRONTEND=noninteractive @@ -135,6 +143,8 @@ while [ $# -gt 0 ]; do --dry-run) DRY_RUN=1; shift ;; # install.sh --uninstall on a k3s Pyo didn't install: Pyo itself goes too. --with-pyo) WITH_PYO=1; shift ;; + # The release pyo.dev's wrapper ran this script from. Nothing to do here. + --version) shift 2 || shift ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac done diff --git a/crates/pyo-api/assets/join.sh b/crates/pyo-api/assets/join.sh index 010d875..1bc0229 100644 --- a/crates/pyo-api/assets/join.sh +++ b/crates/pyo-api/assets/join.sh @@ -15,6 +15,7 @@ # --builder a build-only server: only Pyo's builder runs on it # --registry Pyo's registry door, for images Pyo builds # --registry-token the cluster's login there (pull only) +# --version the Pyo release, for pyo.dev to run its script (ignored here) # >>> pyo:become-root (the same in connect.sh and join.sh) # Root, through sudo when it isn't: the script runs again as root, with # the same arguments and Tailscale key. Run from a file, sudo runs that @@ -39,11 +40,15 @@ become_root() { exit 1 } # <<< pyo:become-root +# Before anything else is defined: piped in, the root shell reads on +# from here, without what came before. +become_root "$@" # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) # A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), the same line comes -# every 20 seconds. progress_start MESSAGE, then progress_tick as often as -# it likes, then progress_end. +# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line +# comes once (not again for a step with the same line), then with its time +# every 20 seconds while it runs. progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end. progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -54,8 +59,12 @@ progress_tick() { if [ -t 1 ]; then printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) elif [ "$t" -ge "$PROGRESS_NEXT" ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) - PROGRESS_NEXT=$((t + 20)) + if [ "$PROGRESS_NEXT" -gt 0 ]; then + printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then + printf '%s\n' "$PROGRESS_MSG" + fi + PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } @@ -104,7 +113,6 @@ quietly() { return "$rc" } # <<< pyo:progress -become_root "$@" set -euo pipefail export DEBIAN_FRONTEND=noninteractive @@ -119,6 +127,8 @@ while [ $# -gt 0 ]; do --builder) BUILDER=1; shift ;; --registry) REGISTRY="$2"; shift 2 ;; --registry-token) REGISTRY_TOKEN="$2"; shift 2 ;; + # The release pyo.dev's wrapper ran this script from. Nothing to do here. + --version) shift 2 || shift ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac done diff --git a/crates/pyo-api/assets/sandbox-images/images.json b/crates/pyo-api/assets/sandbox-images/images.json new file mode 100644 index 0000000..d953c25 --- /dev/null +++ b/crates/pyo-api/assets/sandbox-images/images.json @@ -0,0 +1,28 @@ +{ + "contract": 1, + "ubuntu": "24.04", + "date": "20261116", + "commit": "89639c4a0c6b24b3626824de45bf5968cd459d59", + "images": { + "base": { + "repository": "ghcr.io/upsetdev/pyo-sandbox-images/base", + "digest": "sha256:8aec579b223e818cb6e44146658ea4164212b9ea3250acf025329dfcdbaec520", + "ref": "ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:8aec579b223e818cb6e44146658ea4164212b9ea3250acf025329dfcdbaec520", + "tags": { + "24.04": "sha256:8aec579b223e818cb6e44146658ea4164212b9ea3250acf025329dfcdbaec520", + "24.04-20261116": "sha256:8aec579b223e818cb6e44146658ea4164212b9ea3250acf025329dfcdbaec520", + "89639c4a0c6b24b3626824de45bf5968cd459d59": "sha256:8aec579b223e818cb6e44146658ea4164212b9ea3250acf025329dfcdbaec520" + } + }, + "desktop": { + "repository": "ghcr.io/upsetdev/pyo-sandbox-images/desktop", + "digest": "sha256:f86cd4701ae7f89978fa22ca9421776a90ad0312df431b95c5159c791dfb9335", + "ref": "ghcr.io/upsetdev/pyo-sandbox-images/desktop@sha256:f86cd4701ae7f89978fa22ca9421776a90ad0312df431b95c5159c791dfb9335", + "tags": { + "24.04": "sha256:f86cd4701ae7f89978fa22ca9421776a90ad0312df431b95c5159c791dfb9335", + "24.04-20261116": "sha256:f86cd4701ae7f89978fa22ca9421776a90ad0312df431b95c5159c791dfb9335", + "89639c4a0c6b24b3626824de45bf5968cd459d59": "sha256:f86cd4701ae7f89978fa22ca9421776a90ad0312df431b95c5159c791dfb9335" + } + } + } +} diff --git a/crates/pyo-api/assets/sandbox-images/images.json.sigstore.json b/crates/pyo-api/assets/sandbox-images/images.json.sigstore.json new file mode 100644 index 0000000..4e5cbc4 --- /dev/null +++ b/crates/pyo-api/assets/sandbox-images/images.json.sigstore.json @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHHDCCBqGgAwIBAgIUN/HP6d7ZP1rbNU78fCp6M4PVrREwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMTE2MTkwOTM5WhcNMjYxMTE2MTkxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEMoWz+9QimD1Ws1knKnSJL4aqAVlvDVQKpJJWyLA3zyzb9smVg0gDbTsVZqe7GRfZeaj9vmNqu/jTmerWkPO3vqOCBcAwggW8MA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBqBgNVHREBAf8EYDBehlxodHRwczovL2dpdGh1Yi5jb20vdXBzZXRkZXYvcHlvLXNhbmRib3gtaW1hZ2VzLy5naXRodWIvd29ya2Zsb3dzL3JlbGVhc2UueW1sQHJlZnMvaGVhZHMvbWFpbjA5BgorBgEEAYO/MAEBBCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMBIGCisGAQQBg78wAQIEBHB1c2gwNgYKKwYBBAGDvzABAwQoODk2MzljNGEwYzZiMjRiMzYyNjgyNGRlNDViZjU5NjhjZDQ1OWQ1OTAVBgorBgEEAYO/MAEEBAdSZWxlYXNlMCkGCisGAQQBg78wAQUEG3Vwc2V0ZGV2L3B5by1zYW5kYm94LWltYWdlczAdBgorBgEEAYO/MAEGBA9yZWZzL2hlYWRzL21haW4wOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMGwGCisGAQQBg78wAQkEXgxcaHR0cHM6Ly9naXRodWIuY29tL3Vwc2V0ZGV2L3B5by1zYW5kYm94LWltYWdlcy8uZ2l0aHViL3dvcmtmbG93cy9yZWxlYXNlLnltbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDCg4OTYzOWM0YTBjNmIyNGIzNjI2ODI0ZGU0NWJmNTk2OGNkNDU5ZDU5MB0GCisGAQQBg78wAQsEDwwNZ2l0aHViLWhvc3RlZDA+BgorBgEEAYO/MAEMBDAMLmh0dHBzOi8vZ2l0aHViLmNvbS91cHNldGRldi9weW8tc2FuZGJveC1pbWFnZXMwOAYKKwYBBAGDvzABDQQqDCg4OTYzOWM0YTBjNmIyNGIzNjI2ODI0ZGU0NWJmNTk2OGNkNDU5ZDU5MB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTQ2MDM3Mjg3NDArBgorBgEEAYO/MAEQBB0MG2h0dHBzOi8vZ2l0aHViLmNvbS91cHNldGRldjAYBgorBgEEAYO/MAERBAoMCDU5NzQ2Njg2MGwGCisGAQQBg78wARIEXgxcaHR0cHM6Ly9naXRodWIuY29tL3Vwc2V0ZGV2L3B5by1zYW5kYm94LWltYWdlcy8uZ2l0aHViL3dvcmtmbG93cy9yZWxlYXNlLnltbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABEwQqDCg4OTYzOWM0YTBjNmIyNGIzNjI2ODI0ZGU0NWJmNTk2OGNkNDU5ZDU5MBQGCisGAQQBg78wARQEBgwEcHVzaDBiBgorBgEEAYO/MAEVBFQMUmh0dHBzOi8vZ2l0aHViLmNvbS91cHNldGRldi9weW8tc2FuZGJveC1pbWFnZXMvYWN0aW9ucy9ydW5zLzQzOTM0ODgwODMyL2F0dGVtcHRzLzEwFgYKKwYBBAGDvzABFgQIDAZwdWJsaWMwWAYKKwYBBAGDvzABGARKDEhyZXBvOnVwc2V0ZGV2QDU5NzQ2Njg2L3B5by1zYW5kYm94LWltYWdlc0AxNDYwMzcyODc0OnJlZjpyZWZzL2hlYWRzL21haW4wgYkGCisGAQQB1nkCBAIEewR5AHcAdQDdPTBqxscRMmMZHhyZZzcCokpeuN48rf+HinKALynujgAAAaHlw9kKAAAEAwBGMEQCIG2bRVNHbLYxrYaVmYulGmoctRL+jDFAof9W3LLaiUN4AiB7SNCbDAKunK1NiwSslBxSfsh5CZFf6LTkeKhQhxiK2TAKBggqhkjOPQQDAwNpADBmAjEA4d/jHmAFy3yTsPtwDZkzWbHmiUSYznDWsoPjR5jDeu9Wvc4y9mi3wsC2LeImPyuSAjEA2mj0DynHfsqoj20sEI9LPPhPZLmhCBVDc4pkQoh0ctMeOlw5LaLsuG9mK4AhhmyA"}, "tlogEntries":[{"logIndex":"3932569354", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1794856180", "inclusionPromise":{"signedEntryTimestamp":"MEQCIGLaUWfgyoaZEO3tCrCuiNA5wJwOXrWUJf2sn8ZarWCdAiAeM00ycCX1p1R6k69lsGx4nVeX7YRj9RADB3/ZMGP3Dg=="}, "inclusionProof":{"logIndex":"3810665092", "rootHash":"S3lhwh6s6zCZXHWtyBEgaPCrU0vUqDe1etkST5MT86w=", "treeSize":"3810665123", "hashes":["vXWIH9B23XtDUzmz8uB1CFfLqTl9hW36OiHC1o3fNI4=", "3jJiPiQ/EsguaMsb/l7NQCjSCDxydJNMrmzhiPRZG3o=", "0/kGE6Ykv4kEN6Tp62rbzYZwPRGdiHV/aFVXp+S5eWs=", "F+LIJE0F5AOsIbTcN+Iba5/8Q6SReuAh/QJeCbLR+0Y=", "d2zAiQbC5VL4U02vQk9D1BzppRhXrjCqDX3KACA1NNA=", "CW0NCFfaY1vP3l4oN50FPkfjUDnbbpagwOCZTgYvms8=", "ch3SCTvMeHFDsDb+wA5d1FUmwYKcAeFrewOvzPBvrdw=", "U2fO2XmusNP7BAZhiFxO0FmHhdRoV759rKTYkgRE72c=", "Y1ic7+hYB75m8XCr63IR9FTNlbbJ7cSCr2rnQG4A5F0=", "+u0T2dYnVuAy03jUlJrJmUc5sptI4b1yG2Le76YNf8o=", "K3VqSm7izSE0SvDgE8B3JVP/MGFY9Nf+N55wwGuvr8M=", "0+cBBYqtr4Po0VGL5C9PobOQC2fSBFwsfDTrvX3+a0w=", "FXFT4Kl8hEvzGmPG13WRa+BbPZcWJmwfdjVmkXCnVAU=", "k0+Z1T6dI0Tgluh54oO/hnxtuoW22MqFIjVTPYeUJGU=", "1DaDKzrCNNBhmSdY5vFjz+NpAmxmYfYEgBCdPLrMP1A=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3810665123\nS3lhwh6s6zCZXHWtyBEgaPCrU0vUqDe1etkST5MT86w=\n\n— rekor.sigstore.dev wNI9ajBFAiAnFpRFZFQWeS6WS4qLNs9t6CfTx4zddRnfazX6DsN+NwIhAJa8JKrpuP+sey/O9Epfurqsdd3DDLZL59yk3Ioc8haB\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5N2Q3MTNkMGM3MWM2MmZiNWJmOWNiOGQxMjdlNzk1ZTdhMTM2OTM3Y2ViYmYyNGY3ZmMwZTVmMDM1ZGRjOWMyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJQVdPK1IwR0lwSmRMTm55akhlTXRqWHdaYm03MllDWUJJYlhqUTBnUEM4NUFpRUFpbk1NRForNWpCNW0rNWtFYkRCTVBzUGRteFc1aGVPMWtHOUtkdFVCb09BPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaElSRU5EUW5GSFowRjNTVUpCWjBsVlRpOUlVRFprTjFwUU1YSmlUbFUzT0daRGNEWk5ORkJXY2xKRmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFVUlRKTlZHdDNUMVJOTlZkb1kwNU5hbGw0VFZSRk1rMVVhM2hQVkUwMVYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZOYjFkNkt6bFJhVzFFTVZkek1XdHVTMjVUU2t3MFlYRkJWbXgyUkZaUlMzQktTbGNLZVV4Qk0zcDVlbUk1YzIxV1p6Qm5SR0pVYzFaYWNXVTNSMUptV21WaGFqbDJiVTV4ZFM5cVZHMWxjbGRyVUU4emRuRlBRMEpqUVhkbloxYzRUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuRkNaMDVXU0ZKRlFrRm1PRVZaUkVKbGFHeDRiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmtXRUo2V2xoU2ExcFlXWFpqU0d4MlRGaE9hR0p0VW1saU0yZDBZVmN4YUZveVZucE1lVFZ1WVZoU2IyUlhTWFprTWpsNVlUSmFjd3BpTTJSNlRETktiR0pIVm1oak1sVjFaVmN4YzFGSVNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOVUpuYjNKQ1owVkZRVmxQTDAxQlJVSkNRM1J2Q21SSVVuZGplbTkyVEROU2RtRXlWblZNYlVacVpFZHNkbUp1VFhWYU1td3dZVWhXYVdSWVRteGpiVTUyWW01U2JHSnVVWFZaTWpsMFRVSkpSME5wYzBjS1FWRlJRbWMzT0hkQlVVbEZRa2hDTVdNeVozZE9aMWxMUzNkWlFrSkJSMFIyZWtGQ1FYZFJiMDlFYXpKTmVteHFUa2RGZDFsNldtbE5hbEpwVFhwWmVRcE9hbWQ1VGtkU2JFNUVWbWxhYWxVMVRtcG9hbHBFVVRGUFYxRXhUMVJCVmtKbmIzSkNaMFZGUVZsUEwwMUJSVVZDUVdSVFdsZDRiRmxZVG14TlEydEhDa05wYzBkQlVWRkNaemM0ZDBGUlZVVkhNMVozWXpKV01GcEhWakpNTTBJMVlua3hlbGxYTld0WmJUazBURmRzZEZsWFpHeGpla0ZrUW1kdmNrSm5SVVVLUVZsUEwwMUJSVWRDUVRsNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBkMWxMUzNkWlFrSkJSMFIyZWtGQ1EwRlJkRVJEZEc5a1NGSjNZM3B2ZGdwTU0xSjJZVEpXZFV4dFJtcGtSMngyWW01TmRWb3liREJoU0ZacFpGaE9iR050VG5aaWJsSnNZbTVSZFZreU9YUk5SM2RIUTJselIwRlJVVUpuTnpoM0NrRlJhMFZZWjNoallVaFNNR05JVFRaTWVUbHVZVmhTYjJSWFNYVlpNamwwVEROV2QyTXlWakJhUjFZeVRETkNOV0o1TVhwWlZ6VnJXVzA1TkV4WGJIUUtXVmRrYkdONU9IVmFNbXd3WVVoV2FVd3paSFpqYlhSdFlrYzVNMk41T1hsYVYzaHNXVmhPYkV4dWJIUmlSVUo1V2xkYWVrd3lhR3haVjFKNlRESXhhQXBoVnpSM1QwRlpTMHQzV1VKQ1FVZEVkbnBCUWtOblVYRkVRMmMwVDFSWmVrOVhUVEJaVkVKcVRtMUplVTVIU1hwT2Fra3lUMFJKTUZwSFZUQk9WMHB0Q2s1VWF6SlBSMDVyVGtSVk5WcEVWVFZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZ6UlVSM2QwNWFNbXd3WVVoV2FVeFhhSFpqTTFKc1drUkJLMEpuYjNJS1FtZEZSVUZaVHk5TlFVVk5Ra1JCVFV4dGFEQmtTRUo2VDJrNGRsb3liREJoU0ZacFRHMU9kbUpUT1RGalNFNXNaRWRTYkdScE9YZGxWemgwWXpKR2RRcGFSMHAyWlVNeGNHSlhSbTVhV0UxM1QwRlpTMHQzV1VKQ1FVZEVkbnBCUWtSUlVYRkVRMmMwVDFSWmVrOVhUVEJaVkVKcVRtMUplVTVIU1hwT2Fra3lDazlFU1RCYVIxVXdUbGRLYlU1VWF6SlBSMDVyVGtSVk5WcEVWVFZOUWpoSFEybHpSMEZSVVVKbk56aDNRVkUwUlVWUmQxQmpiVlp0WTNrNWIxcFhSbXNLWTNrNWRGbFhiSFZOUW05SFEybHpSMEZSVVVKbk56aDNRVkU0UlVSQmQwdE5WRkV5VFVSTk0wMXFaek5PUkVGeVFtZHZja0puUlVWQldVOHZUVUZGVVFwQ1FqQk5SekpvTUdSSVFucFBhVGgyV2pKc01HRklWbWxNYlU1MllsTTVNV05JVG14a1IxSnNaR3BCV1VKbmIzSkNaMFZGUVZsUEwwMUJSVkpDUVc5TkNrTkVWVFZPZWxFeVRtcG5NazFIZDBkRGFYTkhRVkZSUW1jM09IZEJVa2xGV0dkNFkyRklVakJqU0UwMlRIazVibUZZVW05a1YwbDFXVEk1ZEV3elZuY0tZekpXTUZwSFZqSk1NMEkxWW5reGVsbFhOV3RaYlRrMFRGZHNkRmxYWkd4amVUaDFXakpzTUdGSVZtbE1NMlIyWTIxMGJXSkhPVE5qZVRsNVdsZDRiQXBaV0U1c1RHNXNkR0pGUW5sYVYxcDZUREpvYkZsWFVucE1NakZvWVZjMGQwOUJXVXRMZDFsQ1FrRkhSSFo2UVVKRmQxRnhSRU5uTkU5VVdYcFBWMDB3Q2xsVVFtcE9iVWw1VGtkSmVrNXFTVEpQUkVrd1drZFZNRTVYU20xT1ZHc3lUMGRPYTA1RVZUVmFSRlUxVFVKUlIwTnBjMGRCVVZGQ1p6YzRkMEZTVVVVS1FtZDNSV05JVm5waFJFSnBRbWR2Y2tKblJVVkJXVTh2VFVGRlZrSkdVVTFWYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemt4WTBoT2JBcGtSMUpzWkdrNWQyVlhPSFJqTWtaMVdrZEtkbVZETVhCaVYwWnVXbGhOZGxsWFRqQmhWemwxWTNrNWVXUlhOWHBNZWxGNlQxUk5NRTlFWjNkUFJFMTVDa3d5UmpCa1IxWjBZMGhTZWt4NlJYZEdaMWxMUzNkWlFrSkJSMFIyZWtGQ1JtZFJTVVJCV25ka1YwcHpZVmROZDFkQldVdExkMWxDUWtGSFJIWjZRVUlLUjBGU1MwUkZhSGxhV0VKMlQyNVdkMk15VmpCYVIxWXlVVVJWTlU1NlVUSk9hbWN5VEROQ05XSjVNWHBaVnpWcldXMDVORXhYYkhSWlYyUnNZekJCZUFwT1JGbDNUWHBqZVU5RVl6QlBia3BzV21wd2VWcFhXbnBNTW1oc1dWZFNla3d5TVdoaFZ6UjNaMWxyUjBOcGMwZEJVVkZDTVc1clEwSkJTVVZsZDFJMUNrRklZMEZrVVVSa1VGUkNjWGh6WTFKTmJVMWFTR2g1V2xwNlkwTnZhM0JsZFU0ME9ISm1LMGhwYmt0QlRIbHVkV3BuUVVGQllVaHNkemxyUzBGQlFVVUtRWGRDUjAxRlVVTkpSekppVWxaT1NHSk1XWGh5V1dGV2JWbDFiRWR0YjJOMFVrd3Jha1JHUVc5bU9WY3pURXhoYVZWT05FRnBRamRUVGtOaVJFRkxkUXB1U3pGT2FYZFRjMnhDZUZObWMyZzFRMXBHWmpaTVZHdGxTMmhSYUhocFN6SlVRVXRDWjJkeGFHdHFUMUJSVVVSQmQwNXdRVVJDYlVGcVJVRTBaQzlxQ2todFFVWjVNM2xVYzFCMGQwUmFhM3BYWWtodGFWVlRXWHB1UkZkemIxQnFValZxUkdWMU9WZDJZelI1T1cxcE0zZHpRekpNWlVsdFVIbDFVMEZxUlVFS01tMXFNRVI1YmtobWMzRnZhakl3YzBWSk9VeFFVR2hRV2t4dGFFTkNWa1JqTkhCclVXOW9NR04wVFdWUGJIYzFUR0ZNYzNWSE9XMUxORUZvYUcxNVFRb3RMUzB0TFVWT1JDQkRSVkpVU1VaSlEwRlVSUzB0TFMwdENnPT0ifX19fQ=="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgPyx66DV5YlT6ZuCLuRijdk574Kf0FHpFMG7fRpo8XnMCFQDQVHC4cSf6eqWgoccig8jdBOaPqhgPMjAyNjExMTYxOTA5NDBaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTExNjE5MDk0MFowLwYJKoZIhvcNAQkEMSIEICxvpQ7yeEcV6dubfkBe5dh0B3W4k9qVk8iBstKFYiIjMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQCfpT8FWSYbND5q+nmFbtTVOUVhNkVgQeSDYiLpFlOsAsu2prPNXB3zG6ytA0rb4+sCMQCu9HbPnPH/h1L4Q+RQzqwfniAteerhQTsGGWYHIjkoShQ/SHYkg1WzPBT6AEflhRI="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"l9cT0MccYvtb+cuNEn55XnoTaTfOu/JPf8Dl8DXdycI="}, "signature":"MEUCIAWO+R0GIpJdLNnyjHeMtjXwZbm72YCYBIbXjQ0gPC85AiEAinMMDZ+5jB5m+5kEbDBMPsPdmxW5heO1kG9KdtUBoOA="}} \ No newline at end of file diff --git a/crates/pyo-api/migrations/0016_sandbox_lifetimes.sql b/crates/pyo-api/migrations/0016_sandbox_lifetimes.sql new file mode 100644 index 0000000..360fe39 --- /dev/null +++ b/crates/pyo-api/migrations/0016_sandbox_lifetimes.sql @@ -0,0 +1,5 @@ +-- Sandboxes may live up to a year, or run until they're deleted. A team's +-- longest lifetime NULL: no limit. A sandbox's expires_at NULL: it never +-- ends on its own. +ALTER TABLE pyo.sandbox_limits ALTER COLUMN max_ttl_secs DROP NOT NULL; +ALTER TABLE pyo.sandboxes ALTER COLUMN expires_at DROP NOT NULL; diff --git a/crates/pyo-api/migrations/0017_sandbox_names.sql b/crates/pyo-api/migrations/0017_sandbox_names.sql new file mode 100644 index 0000000..b750dc1 --- /dev/null +++ b/crates/pyo-api/migrations/0017_sandbox_names.sql @@ -0,0 +1,6 @@ +-- Sandboxes' names (sandbox/names.rs), for people to tell them apart and +-- find them. Not unique: the id addresses a sandbox, so a rename breaks +-- nothing. Pyo names every sandbox it makes, with a short one like +-- quiet-river when none is given. NULL on sandboxes from before this: +-- they show the name made from their id, the same every time. +ALTER TABLE pyo.sandboxes ADD COLUMN name text; diff --git a/crates/pyo-api/src/agent.rs b/crates/pyo-api/src/agent.rs index 7db49da..05bfe85 100644 --- a/crates/pyo-api/src/agent.rs +++ b/crates/pyo-api/src/agent.rs @@ -1,11 +1,13 @@ //! Endpoints the in-cluster agent dials out to: heartbeats with what it sees, //! a long-poll for commands, and reports on commands and rollouts. +use crate::error::Json; +use crate::error::Query; use std::time::Duration; use axum::{ - Json, Router, - extract::{FromRequestParts, Path, Query, State}, + Router, + extract::{FromRequestParts, Path, State}, http::{StatusCode, request::Parts}, routing::{get, post}, }; diff --git a/crates/pyo-api/src/auth.rs b/crates/pyo-api/src/auth.rs index dbea0ed..4894c54 100644 --- a/crates/pyo-api/src/auth.rs +++ b/crates/pyo-api/src/auth.rs @@ -4,12 +4,13 @@ //! sha256. Every browser request that changes something must also carry //! `x-pyo-csrf: 1`, which a cross-site form or image can't add. +use crate::error::Json; use argon2::{ Argon2, password_hash::{PasswordHasher, PasswordVerifier, phc::PasswordHash}, }; use axum::{ - Json, Router, + Router, extract::{FromRequestParts, State}, http::{HeaderMap, StatusCode, request::Parts}, routing::{get, post}, @@ -606,7 +607,7 @@ struct ConfigQuery { async fn config( State(state): State, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let mut conn = state.db.acquire().await?; let invite = match q.invite.as_deref() { diff --git a/crates/pyo-api/src/builders.rs b/crates/pyo-api/src/builders.rs index 42c8882..d11f2ff 100644 --- a/crates/pyo-api/src/builders.rs +++ b/crates/pyo-api/src/builders.rs @@ -5,11 +5,13 @@ //! build-only server that joined it. It dials out here with its own token: heartbeats, a long-poll for build jobs, build logs and the //! result. See deploy.rs for what happens to a finished build. +use crate::error::Json; +use crate::error::Query; use std::time::Duration; use axum::{ - Json, Router, - extract::{FromRequestParts, Path, Query, State}, + Router, + extract::{FromRequestParts, Path, State}, http::{StatusCode, request::Parts}, routing::{get, post}, }; diff --git a/crates/pyo-api/src/cli.rs b/crates/pyo-api/src/cli.rs index c53c51f..cccb8f2 100644 --- a/crates/pyo-api/src/cli.rs +++ b/crates/pyo-api/src/cli.rs @@ -8,7 +8,8 @@ //! good for a year unless `pyo login --expires` says otherwise (`pyo login` //! again makes a new one). Only hashes are stored; codes last 5 minutes and work once. -use axum::{Json, Router, extract::State, routing::post}; +use crate::error::Json; +use axum::{Router, extract::State, routing::post}; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/cloud/bootstrap.rs b/crates/pyo-api/src/cloud/bootstrap.rs index fe83a6f..2677233 100644 --- a/crates/pyo-api/src/cloud/bootstrap.rs +++ b/crates/pyo-api/src/cloud/bootstrap.rs @@ -21,13 +21,14 @@ //! //! Wrong codes are rate-limited. The server then shows as Joining. +use crate::error::Query; use std::net::{IpAddr, SocketAddr}; use std::time::Duration; use axum::{ Router, body::Bytes, - extract::{ConnectInfo, Query, State}, + extract::{ConnectInfo, State}, http::{StatusCode, header}, response::{IntoResponse, Response}, routing::post, diff --git a/crates/pyo-api/src/cloud/mod.rs b/crates/pyo-api/src/cloud/mod.rs index 0c56248..a6be384 100644 --- a/crates/pyo-api/src/cloud/mod.rs +++ b/crates/pyo-api/src/cloud/mod.rs @@ -28,14 +28,13 @@ pub mod bootstrap; pub mod hetzner; +use crate::error::Json; +use crate::error::Query; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use axum::{ - Json, - extract::{Query, State}, -}; +use axum::extract::State; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sqlx::PgPool; diff --git a/crates/pyo-api/src/cloudflare.rs b/crates/pyo-api/src/cloudflare.rs index e18b80a..de9e01a 100644 --- a/crates/pyo-api/src/cloudflare.rs +++ b/crates/pyo-api/src/cloudflare.rs @@ -20,13 +20,11 @@ //! - A proxied record suggests the service's "Behind Cloudflare" firewall //! switch (the dashboard's DNS record line), until it's on. +use crate::error::Json; +use crate::error::Query; use std::{collections::HashMap, time::Duration}; -use axum::{ - Json, - extract::{Query, State}, - http::StatusCode, -}; +use axum::{extract::State, http::StatusCode}; use reqwest::{Method, Url}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/compose.rs b/crates/pyo-api/src/compose.rs index 9c2691a..2994443 100644 --- a/crates/pyo-api/src/compose.rs +++ b/crates/pyo-api/src/compose.rs @@ -1653,8 +1653,8 @@ pub struct RepoInput { pub async fn preview( axum::extract::State(s): axum::extract::State, crate::auth::SessionUser(user): crate::auth::SessionUser, - axum::Json(b): axum::Json, -) -> crate::error::ApiResult> { + crate::error::Json(b): crate::error::Json, +) -> crate::error::ApiResult> { use crate::error::ApiError; let mut tx = s.db.begin().await?; crate::ops::require_role(&mut tx, &user, &b.team_id, crate::ops::Role::Viewer).await?; @@ -1769,7 +1769,7 @@ pub async fn preview( domain_base, }) .map_err(ApiError::bad_request)?; - Ok(axum::Json(plan)) + Ok(crate::error::Json(plan)) } #[cfg(test)] diff --git a/crates/pyo-api/src/cronruns.rs b/crates/pyo-api/src/cronruns.rs index c86355c..dee2018 100644 --- a/crates/pyo-api/src/cronruns.rs +++ b/crates/pyo-api/src/cronruns.rs @@ -2,10 +2,9 @@ //! Kubernetes holds on to. A run that fails says so in the activity feed, //! once, which is what notifications follow. -use axum::{ - Json, - extract::{Query, State}, -}; +use crate::error::Json; +use crate::error::Query; +use axum::extract::State; use pyo_proto::CronRunReport; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/database_analytics.rs b/crates/pyo-api/src/database_analytics.rs index b14864f..acb00c0 100644 --- a/crates/pyo-api/src/database_analytics.rs +++ b/crates/pyo-api/src/database_analytics.rs @@ -6,12 +6,11 @@ //! they become series for the dashboard. Team members can read it, like //! service metrics, and it's an MCP tool like them. +use crate::error::Json; +use crate::error::Query; use std::collections::BTreeMap; -use axum::{ - Json, - extract::{Path, Query, State}, -}; +use axum::extract::{Path, State}; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/database_backups.rs b/crates/pyo-api/src/database_backups.rs index a6b5d47..4dbc29e 100644 --- a/crates/pyo-api/src/database_backups.rs +++ b/crates/pyo-api/src/database_backups.rs @@ -3,9 +3,10 @@ //! (agent backup.rs) that every backup then runs as. The admin login is used //! for that and never stored. +use crate::error::Json; use std::time::Duration; -use axum::{Json, extract::State}; +use axum::extract::State; use pyo_proto::{BackupSpec, Command, CommandResult, DbConnection}; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/database_imports.rs b/crates/pyo-api/src/database_imports.rs index 978d4d6..7938441 100644 --- a/crates/pyo-api/src/database_imports.rs +++ b/crates/pyo-api/src/database_imports.rs @@ -31,12 +31,14 @@ //! unless an import is using it; an upload never sent, once its URL //! expired. +use crate::error::Json; +use crate::error::Query; use std::time::Duration; use axum::{ - Json, Router, + Router, body::Body, - extract::{DefaultBodyLimit, Path, Query, State}, + extract::{DefaultBodyLimit, Path, State}, http::{HeaderMap, StatusCode}, response::Response, routing::{get, post, put}, diff --git a/crates/pyo-api/src/database_logs.rs b/crates/pyo-api/src/database_logs.rs index 7c095b6..8a6e1b7 100644 --- a/crates/pyo-api/src/database_logs.rs +++ b/crates/pyo-api/src/database_logs.rs @@ -6,12 +6,11 @@ //! `reported`) and the team's activity on it. Team members only, like //! service logs; not an MCP tool for now. +use crate::error::Json; +use crate::error::Query; use std::time::Duration; -use axum::{ - Json, - extract::{Path, Query, State}, -}; +use axum::extract::{Path, State}; use pyo_proto::{Command, CommandResult, now_ms}; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/databases.rs b/crates/pyo-api/src/databases.rs index 3a6fd47..2e3051d 100644 --- a/crates/pyo-api/src/databases.rs +++ b/crates/pyo-api/src/databases.rs @@ -16,10 +16,9 @@ //! `revealDatabasePassword` (owners and admins). //! - **Status:** the controller's, as each heartbeat reports it. -use axum::{ - Json, - extract::{Path, Query, State}, -}; +use crate::error::Json; +use crate::error::Query; +use axum::extract::{Path, State}; use pyo_proto::{Command, DatabaseReport, DatabaseSpec, EnvVar, NamedSecret}; use serde::Deserialize; use serde_json::{Value, json}; diff --git a/crates/pyo-api/src/error.rs b/crates/pyo-api/src/error.rs index 241ebb0..cbda71e 100644 --- a/crates/pyo-api/src/error.rs +++ b/crates/pyo-api/src/error.rs @@ -1,5 +1,4 @@ use axum::{ - Json, http::StatusCode, response::{IntoResponse, Response}, }; @@ -71,7 +70,7 @@ impl IntoResponse for ApiError { fn into_response(self) -> Response { let mut r = ( self.status, - Json(ErrorBody { + axum::Json(ErrorBody { error: self.message, needs_confirm: self.needs_confirm, }), @@ -114,3 +113,224 @@ impl From for ApiError { pub async fn not_found() -> ApiError { ApiError::new(StatusCode::NOT_FOUND, "No such endpoint.") } + +/// Axum's `Query`, with a missing or wrong parameter as a plain 400 the +/// dashboard can show, instead of axum's own text. +#[derive(Debug, Clone, Copy, Default)] +pub struct Query(pub T); + +impl axum::extract::FromRequestParts for Query +where + T: serde::de::DeserializeOwned, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + state: &S, + ) -> Result { + match axum::extract::Query::::from_request_parts(parts, state).await { + Ok(axum::extract::Query(v)) => Ok(Query(v)), + Err(e) => Err(ApiError::bad_request(query_message(&e.body_text()))), + } + } +} + +/// axum's "Failed to deserialize query string: missing field `team`", in +/// plain words. +fn query_message(text: &str) -> String { + let quoted = |after: &str| { + let rest = &text[text.find(after)? + after.len()..]; + rest.split('`').next().map(str::to_owned) + }; + if let Some(name) = quoted("missing field `") { + return format!("Add the {name} parameter."); + } + if let Some(value) = quoted("unknown variant `") { + return format!("\"{value}\" isn't a value this parameter takes."); + } + "A parameter in the address isn't valid.".to_owned() +} + +/// Axum's `Json`, with a body it can't read as a plain 400 the dashboard +/// can show, instead of axum's own text and 422. As a response, the same. +#[derive(Debug, Clone, Copy, Default)] +pub struct Json(pub T); + +impl axum::extract::FromRequest for Json +where + T: serde::de::DeserializeOwned, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request(req: axum::extract::Request, state: &S) -> Result { + match axum::Json::::from_request(req, state).await { + Ok(axum::Json(v)) => Ok(Json(v)), + Err(e) => Err(json_rejection(&e)), + } + } +} + +/// An optional body: none without a JSON Content-Type, as axum's. +impl axum::extract::OptionalFromRequest for Json +where + T: serde::de::DeserializeOwned, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request( + req: axum::extract::Request, + state: &S, + ) -> Result, ApiError> { + match as axum::extract::OptionalFromRequest>::from_request(req, state) + .await + { + Ok(v) => Ok(v.map(|axum::Json(v)| Json(v))), + Err(e) => Err(json_rejection(&e)), + } + } +} + +fn json_rejection(e: &axum::extract::rejection::JsonRejection) -> ApiError { + let status = match e.status() { + StatusCode::UNPROCESSABLE_ENTITY => StatusCode::BAD_REQUEST, + s => s, + }; + ApiError::new(status, json_message(&e.body_text())) +} + +impl IntoResponse for Json { + fn into_response(self) -> Response { + axum::Json(self.0).into_response() + } +} + +/// axum's "Failed to deserialize the JSON body into the target type: +/// missing field `template` at line 1 column 2", in plain words. +fn json_message(text: &str) -> String { + let quoted = |after: &str| { + let rest = &text[text.find(after)? + after.len()..]; + rest.split('`').next().map(str::to_owned) + }; + if let Some(name) = quoted("missing field `") { + return format!("Add the {name} field."); + } + if let Some(name) = quoted("unknown field `") { + return format!("There's no field named {name}."); + } + if let Some(value) = quoted("unknown variant `") { + return format!("\"{value}\" isn't a value it takes."); + } + if text.contains("Content-Type") { + return "Send the body as JSON, with Content-Type: application/json.".to_owned(); + } + if text.starts_with("Failed to parse") || text.contains("EOF while parsing") { + return "The body isn't valid JSON.".to_owned(); + } + // "…target type: cpus: invalid type: …": the field it's about. + if let Some(rest) = text.split_once("target type: ").map(|(_, r)| r) + && let Some((path, _)) = rest.split_once(": ") + && !path.contains(' ') + && path != "." + { + return format!("The {path} field isn't valid."); + } + if text.contains("length limit") { + return "The body is too large.".to_owned(); + } + "The body isn't valid.".to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_errors_in_plain_words() { + assert_eq!( + query_message("Failed to deserialize query string: missing field `team`"), + "Add the team parameter." + ); + assert_eq!( + query_message( + "Failed to deserialize query string: unknown variant `x`, expected `a` or `b`" + ), + "\"x\" isn't a value this parameter takes." + ); + assert_eq!( + query_message("Failed to deserialize query string: invalid digit found in string"), + "A parameter in the address isn't valid." + ); + } + + #[test] + fn body_errors_in_plain_words() { + let t = "Failed to deserialize the JSON body into the target type: "; + assert_eq!( + json_message(&format!("{t}missing field `template` at line 1 column 2")), + "Add the template field." + ); + assert_eq!( + json_message(&format!( + "{t}unknown field `x`, expected one of `a`, `b` at line 1 column 4" + )), + "There's no field named x." + ); + assert_eq!( + json_message(&format!( + "{t}cpus: invalid type: string \"a\", expected u32 at line 1 column 11" + )), + "The cpus field isn't valid." + ); + assert_eq!( + json_message( + "Failed to parse the request body as JSON: EOF while parsing an object at line 1 column 1" + ), + "The body isn't valid JSON." + ); + assert_eq!( + json_message("Expected request with `Content-Type: application/json`"), + "Send the body as JSON, with Content-Type: application/json." + ); + } + + #[tokio::test] + async fn a_body_it_cant_read_is_a_plain_400() { + #[derive(serde::Deserialize)] + #[allow(dead_code)] + struct B { + template: String, + } + let req = axum::http::Request::post("/x") + .header("content-type", "application/json") + .body(axum::body::Body::from("{}")) + .unwrap(); + let e = as axum::extract::FromRequest<()>>::from_request(req, &()) + .await + .err() + .unwrap(); + assert_eq!(e.status, StatusCode::BAD_REQUEST); + assert_eq!(e.message, "Add the template field."); + } + + #[tokio::test] + async fn a_missing_parameter_is_a_plain_400() { + #[derive(serde::Deserialize)] + #[allow(dead_code)] + struct Q { + team: String, + } + let req = axum::http::Request::get("/x?other=1").body(()).unwrap(); + let (mut parts, ()) = req.into_parts(); + let e = + as axum::extract::FromRequestParts<()>>::from_request_parts(&mut parts, &()) + .await + .err() + .unwrap(); + assert_eq!(e.status, StatusCode::BAD_REQUEST); + assert_eq!(e.message, "Add the team parameter."); + } +} diff --git a/crates/pyo-api/src/git/mod.rs b/crates/pyo-api/src/git/mod.rs index 7688ff5..522d439 100644 --- a/crates/pyo-api/src/git/mod.rs +++ b/crates/pyo-api/src/git/mod.rs @@ -30,12 +30,14 @@ pub mod gitlab; pub mod remote; +use crate::error::Json; +use crate::error::Query; use std::time::Duration; use axum::{ - Json, Router, + Router, body::Bytes, - extract::{Path, Query, State}, + extract::{Path, State}, http::{HeaderMap, StatusCode}, response::IntoResponse, routing::{get, post}, diff --git a/crates/pyo-api/src/github.rs b/crates/pyo-api/src/github.rs index ed8ccd7..1dea1d9 100644 --- a/crates/pyo-api/src/github.rs +++ b/crates/pyo-api/src/github.rs @@ -12,6 +12,8 @@ //! - Each deploy reports a commit status on GitHub (pending, success, //! failure) linking to the deployment. +use crate::error::Json; +use crate::error::Query; use std::{ collections::HashMap, sync::OnceLock, @@ -19,9 +21,9 @@ use std::{ }; use axum::{ - Json, Router, + Router, body::Bytes, - extract::{Query, State}, + extract::State, http::{HeaderMap, StatusCode}, response::{IntoResponse, Redirect}, routing::{get, post}, diff --git a/crates/pyo-api/src/hooks.rs b/crates/pyo-api/src/hooks.rs index 751fd9f..67a8536 100644 --- a/crates/pyo-api/src/hooks.rs +++ b/crates/pyo-api/src/hooks.rs @@ -6,8 +6,9 @@ //! The token is the only credential, so it's long and random; the dashboard //! reveals it only to developers of the team. +use crate::error::Json; use axum::{ - Json, Router, + Router, extract::{Path, State}, http::StatusCode, routing::post, diff --git a/crates/pyo-api/src/images.rs b/crates/pyo-api/src/images.rs index 9458096..d829d48 100644 --- a/crates/pyo-api/src/images.rs +++ b/crates/pyo-api/src/images.rs @@ -3,12 +3,11 @@ //! them. Images Pyo built live in its registry, which also gives their size. use crate::error::ApiResult; +use crate::error::Json; +use crate::error::Query; use crate::ops::{Role, require_role}; use crate::{AppState, auth::SessionUser, deploy}; -use axum::{ - Json, - extract::{Query, State}, -}; +use axum::extract::State; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::collections::BTreeMap; diff --git a/crates/pyo-api/src/install.rs b/crates/pyo-api/src/install.rs index e543bbe..b3f1e08 100644 --- a/crates/pyo-api/src/install.rs +++ b/crates/pyo-api/src/install.rs @@ -3,11 +3,8 @@ //! The token is the cluster's agent credential; it's only valid for the //! cluster it was issued to and can be rotated from the dashboard. -use axum::{ - extract::{Query, State}, - http::header, - response::IntoResponse, -}; +use crate::error::Query; +use axum::{extract::State, http::header, response::IntoResponse}; use serde::Deserialize; use crate::{ @@ -768,10 +765,10 @@ pub struct ThisMachineQuery { pub async fn this_machine( State(s): State, Query(q): Query, -) -> ApiResult> { +) -> ApiResult> { use serde_json::json; let key = bootstrap_key(&q.key)?; - Ok(axum::Json(match hand_over(&s.db, &key).await? { + Ok(crate::error::Json(match hand_over(&s.db, &key).await? { Handover::Connected => json!({ "state": "connected" }), Handover::Waiting => json!({ "state": "waiting" }), Handover::Connect { cluster_id, token } => { @@ -807,10 +804,12 @@ fn bootstrap_key(given: &str) -> ApiResult { pub async fn first_sign_in( State(s): State, Query(q): Query, -) -> ApiResult> { +) -> ApiResult> { bootstrap_key(&q.key)?; let pending = first_admin_pending(&s.db).await?; - Ok(axum::Json(serde_json::json!({ "pending": pending }))) + Ok(crate::error::Json( + serde_json::json!({ "pending": pending }), + )) } #[derive(Deserialize)] @@ -830,7 +829,7 @@ pub struct ResetLogin { pub async fn reset_login( State(s): State, axum::Form(f): axum::Form, -) -> ApiResult> { +) -> ApiResult> { bootstrap_key(&f.key)?; let email = f.email.trim().to_lowercase(); let mut conn = s.db.acquire().await?; @@ -864,7 +863,7 @@ pub async fn reset_login( }; let token = crate::recovery::new_link(&mut conn, &id).await?; tracing::info!(user = %id, "sign-in link made on the host"); - Ok(axum::Json(serde_json::json!({ + Ok(crate::error::Json(serde_json::json!({ "email": email, "path": crate::recovery::link_path(&token), "minutes": crate::recovery::LINK_MINUTES, @@ -902,12 +901,12 @@ pub struct ThisMachineTailnet { pub async fn this_machine_tailnet( State(s): State, axum::Form(f): axum::Form, -) -> ApiResult> { +) -> ApiResult> { let key = std::env::var("PYO_BOOTSTRAP_KEY").unwrap_or_default(); if key.len() < 32 || !crate::util::constant_time_eq(key.as_bytes(), f.key.as_bytes()) { return Err(ApiError::not_found("Install key")); } - Ok(axum::Json( + Ok(crate::error::Json( crate::tailscale::host_report(&s, &f.ip, &f.name, &f.k3s, &f.error).await?, )) } diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index 15ef499..e057afb 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -63,10 +63,9 @@ fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String { /// its public IP: that address may not be reachable from outside (a server /// behind NAT, like Starlink's). So: /// 1. `PYO_SCRIPTS_URL`, when set; -/// 2. else pyo.dev. Its sandbox.sh runs the script of the release -/// `--version` names. Its join.sh and connect.sh run the latest -/// release's script, and join.sh's `--k3s` keeps the server on the -/// cluster's k3s. +/// 2. else pyo.dev. Its sandbox.sh, join.sh and connect.sh run the script +/// of the release `--version` names ([`pinned`]), and join.sh's `--k3s` +/// keeps the server on the cluster's k3s. /// /// A dev build's version may not be a published release, so pyo.dev has no /// sandbox.sh for it and its wrapper fails to download one: dev builds set @@ -266,10 +265,24 @@ fn quoted(args: &[(&str, Option)]) -> String { pub fn command(base: &str, j: &Join) -> String { format!( "curl -fsSL {base}/join.sh | bash -s -- {}", - quoted(&args(j)) + quoted(&pinned(base, args(j))) ) } +/// A script's arguments for where it's fetched from (`base`). pyo.dev's +/// join.sh and connect.sh run the script of the release `--version` names, +/// else the latest one's: fetched from anywhere but a Pyo (whose own are +/// its release's), they get this Pyo's release. The scripts ignore it. +pub fn pinned( + base: &str, + mut args: Vec<(&'static str, Option)>, +) -> Vec<(&'static str, Option)> { + if !base.trim_end_matches('/').ends_with("/install") { + args.push(("--version", Some(env!("CARGO_PKG_VERSION").to_owned()))); + } + args +} + /// Pyo on the machine it runs on: where "This machine"'s installer /// reaches it. PYO_LOCAL_URL, else the API's host port. pub fn local_api_url() -> String { @@ -292,7 +305,10 @@ pub fn connect_command( ) -> String { format!( "curl -fsSL {base}/connect.sh | bash -s -- {}", - quoted(&connect_args(api, token, tailscale_key, registry)) + quoted(&pinned( + base, + connect_args(api, token, tailscale_key, registry) + )) ) } @@ -321,7 +337,10 @@ pub fn connect_args( /// with Pyo's: connect.sh run on the cluster's first server, which reads /// Pyo's address and the token from the agent already there. pub fn refresh_command(base: &str) -> String { - format!("curl -fsSL {base}/connect.sh | bash") + match quoted(&pinned(base, Vec::new())) { + a if a.is_empty() => format!("curl -fsSL {base}/connect.sh | bash"), + a => format!("curl -fsSL {base}/connect.sh | bash -s -- {a}"), + } } /// GET /install/join.sh @@ -357,10 +376,62 @@ pub async fn serve_sandbox() -> impl axum::response::IntoResponse { ) } +/// install.sh's `host_units`: the scripts and timers on the machine Pyo +/// runs on. The host's tailnet timer runs it whenever it changes, so an +/// update from the dashboard brings them too. +static HOST_UNITS: std::sync::LazyLock = std::sync::LazyLock::new(|| { + let install = include_str!("../../../deploy/install.sh"); + let mut lines = install.lines().skip_while(|l| *l != "host_units() {"); + let mut body = String::new(); + // Its end is the first "}" line outside the scripts it writes (their + // own functions end the same way). + let mut heredoc: Option = None; + for line in lines.by_ref() { + body.push_str(line); + body.push('\n'); + match &heredoc { + Some(end) if line == end => heredoc = None, + Some(_) => {} + None if line == "}" => break, + None => { + if let Some((_, rest)) = line.split_once("<<'") { + heredoc = rest.split_once('\'').map(|(d, _)| d.to_owned()); + } + } + } + } + assert!(body.ends_with("\n}\n"), "install.sh's host_units() ends"); + format!("#!/usr/bin/env bash\nset -euo pipefail\n{body}host_units\n") +}); + +/// GET /install/host-units.sh +pub async fn serve_host_units() -> impl axum::response::IntoResponse { + ( + [( + axum::http::header::CONTENT_TYPE, + "text/x-shellscript; charset=utf-8", + )], + HOST_UNITS.as_str(), + ) +} + #[cfg(test)] mod tests { use super::*; + /// pyo.dev's wrappers run this release's scripts. + const PIN: &str = concat!(" --version '", env!("CARGO_PKG_VERSION"), "'"); + + #[test] + fn host_units_script_is_install_sh_s() { + let s = HOST_UNITS.as_str(); + assert!(s.starts_with("#!/usr/bin/env bash\nset -euo pipefail\nhost_units() {\n")); + assert!(s.ends_with("\n}\nhost_units\n")); + assert!(s.contains("/usr/local/bin/pyo-reset-login")); + assert!(s.contains("http://127.0.0.1:8000/install/host-units.sh")); + assert!(s.contains("systemctl enable --now pyo-connect.timer pyo-tailnet.timer")); + } + /// connect.sh, join.sh and install.sh carry the AppArmor workaround's /// profile: it must be the asset's, character for character. #[test] @@ -696,10 +767,12 @@ mod tests { ); assert_eq!( c, - "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server '100.64.0.1' --token 'abc123.0123456789abcdef' --name 'sg-1' --k3s 'v1.36.5+k3s1'" + format!( + "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server '100.64.0.1' --token 'abc123.0123456789abcdef' --name 'sg-1' --k3s 'v1.36.5+k3s1'{PIN}" + ) ); let c = command( - "https://x", + "https://x/install", &Join { server_ip: "1", token: "t", @@ -724,11 +797,18 @@ mod tests { None, Some(("100.64.0.1:5005", "def")) ), - "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'http://100.64.0.1:8000' --token 'abc' --registry '100.64.0.1:5005' --registry-token 'def'" + format!( + "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'http://100.64.0.1:8000' --token 'abc' --registry '100.64.0.1:5005' --registry-token 'def'{PIN}" + ) ); assert_eq!( refresh_command("https://pyo.dev"), - "curl -fsSL https://pyo.dev/connect.sh | bash" + format!("curl -fsSL https://pyo.dev/connect.sh | bash -s --{PIN}") + ); + // Pyo's own scripts are its release's. + assert_eq!( + refresh_command("https://pyo.example/install"), + "curl -fsSL https://pyo.example/install/connect.sh | bash" ); } @@ -880,7 +960,9 @@ mod tests { registry: None, }, ), - "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'" + format!( + "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'{PIN}" + ) ); assert_eq!( connect_command( @@ -890,7 +972,9 @@ mod tests { Some("tskey-auth-1"), Some(("pyo-registry.tail1234.ts.net", "def")) ), - "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'" + format!( + "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'{PIN}" + ) ); // A mirror the operator set. assert!( @@ -1043,13 +1127,11 @@ if apply_agent; then echo "ok after $applies"; else echo "failed after $applies" let (code, out, err) = apply_agent(&version_json("37", "37"), 3); assert_eq!(code, 0, "{err}"); assert!(out.trim().ends_with("ok after 4"), "{out}"); - // What it's doing, then one progress line with the time (every 20 - // seconds off a terminal), and no Kubernetes errors or tries. + // What it's doing, then its progress line once and with the time + // every 20 seconds off a terminal, and no Kubernetes errors or tries. assert!(out.starts_with("Installing the Pyo agent…")); - assert!( - out.contains("Still installing the Pyo agent… 0:00"), - "{out}" - ); + assert!(out.contains("\nStill installing the Pyo agent…\n"), "{out}"); + assert!(!out.contains("agent… 0:00"), "{out}"); assert!( out.contains("Still installing the Pyo agent… 0:20"), "{out}" diff --git a/crates/pyo-api/src/lib.rs b/crates/pyo-api/src/lib.rs index 67cdf9e..970d6e9 100644 --- a/crates/pyo-api/src/lib.rs +++ b/crates/pyo-api/src/lib.rs @@ -347,8 +347,10 @@ pub async fn run(ext: Extensions) -> anyhow::Result<()> { if role.web() { tokio::spawn(bg::listen(state.clone())); tokio::spawn(geoip::keeper(state.clone())); - tokio::spawn(tailscale::addresses_follower(state.db.clone())); } + // Every process: the worker sends sandbox servers images through the + // registry door's tailnet address (sandbox/templates.rs). + tokio::spawn(tailscale::addresses_follower(state.db.clone())); if role.worker() { // Each loop runs on one worker at a time (leader lock); jobs on any. tokio::spawn(deploy::sweeper(state.clone())); @@ -367,6 +369,7 @@ pub async fn run(ext: Extensions) -> anyhow::Result<()> { tokio::spawn(cdn::keeper(state.clone())); tokio::spawn(sandbox::runner::runner(state.clone())); tokio::spawn(sandbox::base::seed(state.clone())); + tokio::spawn(sandbox::images::runner(state.clone())); for task in tasks { tokio::spawn(task(state.clone())); } @@ -453,6 +456,7 @@ pub async fn run(ext: Extensions) -> anyhow::Result<()> { .route("/install/join.sh", get(join::serve)) .route("/install/connect.sh", get(join::serve_connect)) .route("/install/sandbox.sh", get(join::serve_sandbox)) + .route("/install/host-units.sh", get(join::serve_host_units)) .fallback_service(tower_http::compression::Compression::new(spa)) .layer(axum::middleware::from_fn_with_state( state.config.cookie_secure, diff --git a/crates/pyo-api/src/mcp.rs b/crates/pyo-api/src/mcp.rs index dba3850..97c1600 100644 --- a/crates/pyo-api/src/mcp.rs +++ b/crates/pyo-api/src/mcp.rs @@ -13,8 +13,10 @@ //! same handlers and `/ops` code with the same checks. Only the tools the //! connection's access allows are listed. +use crate::error::Json; +use crate::error::Query; use axum::{ - Json, Router, + Router, extract::State, http::StatusCode, response::{IntoResponse, Response}, @@ -178,6 +180,7 @@ const COVERED: &[(&str, &str)] = &[ ("getServiceLogs", "get_service_logs"), ("createSandbox", "create_sandbox"), ("pauseSandbox", "pause_sandbox"), + ("renameSandbox", "rename_sandbox"), ("openSandboxPort", "open_port"), ("closeSandboxPort", "close_port"), ("deleteSandbox", "delete_sandbox"), @@ -244,6 +247,7 @@ const CURATED_RUNS: &[(&str, &str)] = &[ ("write_file", "createSandboxToken"), ("open_port", "openSandboxPort"), ("close_port", "closeSandboxPort"), + ("rename_sandbox", "renameSandbox"), ("pause_sandbox", "pauseSandbox"), ("delete_sandbox", "deleteSandbox"), ]; @@ -595,7 +599,7 @@ fn json_of(r: ApiResult>) -> Result /// An endpoint's tool: the endpoint's own handler, as this user. async fn run_endpoint(s: &AppState, user: &User, id: &str, args: &Value) -> Result { - use axum::extract::{Path, Query}; + use axum::extract::Path; let st = || State(s.clone()); let me = || SessionUser(user.clone()); let path_id = || arg(args, "id").map(|i| Path(i.to_owned())); @@ -677,6 +681,9 @@ async fn run_endpoint(s: &AppState, user: &User, id: &str, args: &Value) -> Resu "getCloudOptions" => { json_of(crate::cloud::options(st(), me(), Query(args_as(args)?)).await) } + "getTailnetLock" => { + json_of(crate::tailscale::lock(st(), me(), Query(args_as(args)?)).await) + } "listSandboxes" => { json_of(crate::sandbox::api::list(st(), me(), Query(args_as(args)?)).await) } diff --git a/crates/pyo-api/src/oauth.rs b/crates/pyo-api/src/oauth.rs index 5956861..4e5a549 100644 --- a/crates/pyo-api/src/oauth.rs +++ b/crates/pyo-api/src/oauth.rs @@ -11,9 +11,11 @@ //! //! API keys (Profile → API keys) stay the way scripts call the REST API. +use crate::error::Json; +use crate::error::Query; use axum::{ - Form, Json, Router, - extract::{FromRequestParts, Query, State}, + Form, Router, + extract::{FromRequestParts, State}, http::{HeaderMap, StatusCode, header, request::Parts}, response::{IntoResponse, Response}, routing::{get, post}, diff --git a/crates/pyo-api/src/openapi.rs b/crates/pyo-api/src/openapi.rs index 7c700be..ed0b3a8 100644 --- a/crates/pyo-api/src/openapi.rs +++ b/crates/pyo-api/src/openapi.rs @@ -69,6 +69,7 @@ use crate::{auth::KeyAccess, error::ErrorBody, ops::Op}; crate::git::build_file, crate::compose::preview, crate::cloud::options, + crate::tailscale::lock, crate::platform_import::preview, crate::platform_import::apply, crate::platform_import::get_run, @@ -76,6 +77,7 @@ use crate::{auth::KeyAccess, error::ErrorBody, ops::Op}; crate::sandbox::api::list, crate::sandbox::api::create, crate::sandbox::api::get_one, + crate::sandbox::api::rename, crate::sandbox::api::delete, crate::sandbox::api::pause, crate::sandbox::api::resume, @@ -462,6 +464,7 @@ fn common_responses(spec: &mut Spec) { &mut item.get, &mut item.post, &mut item.put, + &mut item.patch, &mut item.delete, ] .into_iter() diff --git a/crates/pyo-api/src/ops.rs b/crates/pyo-api/src/ops.rs index 1364901..584141e 100644 --- a/crates/pyo-api/src/ops.rs +++ b/crates/pyo-api/src/ops.rs @@ -5,10 +5,11 @@ //! The dashboard makes ids itself so it can update optimistically; they're //! validated here and the database decides uniqueness. +use crate::error::Json; use std::time::Duration; use axum::{ - Json, Router, + Router, extract::{Path, State}, routing::{get, post}, }; @@ -104,6 +105,7 @@ pub fn routes() -> Router { .route("/database-backups", post(crate::database_backups::add)) .route("/compose/preview", post(crate::compose::preview)) .route("/cloud/options", get(crate::cloud::options)) + .route("/tailnet/lock", get(crate::tailscale::lock)) } /// Everything you can see @@ -2061,7 +2063,7 @@ pub(crate) enum Op { #[serde(default)] usage: Option, }, - /// Set a team's sandbox limits: how many sandboxes it may have at once, their CPUs, memory and disk together, and the longest lifetime. Only the installation's admin can do this. + /// Set a team's sandbox limits: how many sandboxes it may have at once, their CPUs, memory and disk together, and the longest lifetime (up to 365 days, or null for no limit). Only the installation's admin can do this. SetSandboxLimits { team_id: String, limits: crate::sandbox::limits::Limits, @@ -2076,7 +2078,7 @@ pub(crate) enum Op { CreateApiToken { id: String, name: String, - /// "read", "deploy" or "full" (the default). + /// "read", "deploy", "sandboxes" or "full" (the default). access: Option, /// Only these teams; all of yours when omitted. team_ids: Option>, @@ -7030,7 +7032,10 @@ async fn apply( // It's on the tailnet before it asks: no key in its details. let material = crate::cloud::bootstrap::Material::new( format!("{}/join.sh", parts.base), - &crate::join::args(&parts.join(&name, builder, false)), + &crate::join::pinned( + &parts.base, + crate::join::args(&parts.join(&name, builder, false)), + ), ); if builder { let cluster_name: String = @@ -7136,13 +7141,17 @@ async fn apply( let (api, tailscale_key, registry) = reach(tx, s, &id, "Pyo: connect a cluster").await?; // It's on the tailnet before it asks: no key in its details. + let base = crate::join::scripts_url(Some(&api)); let material = crate::cloud::bootstrap::Material::new( - format!("{}/connect.sh", crate::join::scripts_url(Some(&api))), - &crate::join::connect_args( - &api, - &token, - None, - registry.as_ref().map(|(r, t)| (r.as_str(), t.as_str())), + format!("{base}/connect.sh"), + &crate::join::pinned( + &base, + crate::join::connect_args( + &api, + &token, + None, + registry.as_ref().map(|(r, t)| (r.as_str(), t.as_str())), + ), ), ); let made = make_cloud_server( @@ -7924,7 +7933,7 @@ pub(crate) async fn observability_settings( pub(crate) async fn metrics( State(s): State, SessionUser(user): SessionUser, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let mut tx = s.db.begin().await?; require_role(&mut tx, &user, &q.team, Role::Viewer).await?; @@ -7994,7 +8003,7 @@ pub(crate) struct AccessLogQuery { pub(crate) async fn access_logs( State(s): State, SessionUser(user): SessionUser, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let hosts = team_hosts( &s, @@ -8070,7 +8079,7 @@ async fn team_hosts( pub(crate) async fn observability( State(s): State, SessionUser(user): SessionUser, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let hosts = team_hosts(&s, &user, &q.team, q.project, q.site, q.service).await?; let range = match q.range.as_deref() { @@ -8107,7 +8116,7 @@ pub(crate) async fn connections( State(s): State, SessionUser(user): SessionUser, axum::extract::Path(id): axum::extract::Path, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let mut tx = s.db.begin().await?; let team_id = team_of(&mut tx, "services", &id).await?; @@ -8150,7 +8159,7 @@ pub(crate) struct BlockedQuery { pub(crate) async fn firewall_blocked( State(s): State, SessionUser(user): SessionUser, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let mut tx = s.db.begin().await?; let team_id = team_of(&mut tx, "services", &q.service).await?; @@ -8185,7 +8194,7 @@ pub(crate) async fn firewall_blocked( pub(crate) async fn waf_hits( State(s): State, SessionUser(user): SessionUser, - axum::extract::Query(q): axum::extract::Query, + crate::error::Query(q): crate::error::Query, ) -> ApiResult> { let mut tx = s.db.begin().await?; let team_id = team_of(&mut tx, "services", &q.service).await?; diff --git a/crates/pyo-api/src/passkeys.rs b/crates/pyo-api/src/passkeys.rs index b953c27..2d153ae 100644 --- a/crates/pyo-api/src/passkeys.rs +++ b/crates/pyo-api/src/passkeys.rs @@ -19,8 +19,9 @@ //! password, so removing passkeys never locks anyone out, and needs only the //! session. +use crate::error::Json; use axum::{ - Json, Router, + Router, extract::{Path, State}, http::{HeaderMap, StatusCode, header}, routing::{patch, post}, diff --git a/crates/pyo-api/src/platform_import/mod.rs b/crates/pyo-api/src/platform_import/mod.rs index 6274c52..4e40b05 100644 --- a/crates/pyo-api/src/platform_import/mod.rs +++ b/crates/pyo-api/src/platform_import/mod.rs @@ -35,12 +35,14 @@ mod links; mod railway; mod vercel; +use crate::error::Json; +use crate::error::Query; use std::collections::{BTreeMap, HashSet}; use std::time::Duration; use axum::{ - Json, Router, - extract::{Path, Query, State}, + Router, + extract::{Path, State}, http::StatusCode, routing::{get, post}, }; diff --git a/crates/pyo-api/src/recovery.rs b/crates/pyo-api/src/recovery.rs index c66ec37..02a4b4e 100644 --- a/crates/pyo-api/src/recovery.rs +++ b/crates/pyo-api/src/recovery.rs @@ -17,9 +17,11 @@ //! Using it sets the new password, ends the account's sessions and signs //! this browser in. +use crate::error::Json; +use crate::error::Query; use axum::{ - Json, Router, - extract::{Query, State}, + Router, + extract::State, http::{HeaderMap, StatusCode}, routing::{get, post}, }; diff --git a/crates/pyo-api/src/registrydoor.rs b/crates/pyo-api/src/registrydoor.rs index 1203469..f599c5b 100644 --- a/crates/pyo-api/src/registrydoor.rs +++ b/crates/pyo-api/src/registrydoor.rs @@ -10,7 +10,8 @@ //! builds (`pyo/services/`), pull its team's images; //! - a sandbox server's registry token (username `pyo-sandbox`, signed by //! the sandbox key for an hour): pull its team's images and builds, to -//! make a sandbox template from one. +//! make a sandbox template from one. An installation's template gets one +//! for Pyo's own images only (`pyo/`, like a dev build's desktop). //! //! It listens beside the API (`PYO_REGISTRY_DOOR_ADDR`, :5006, in the //! cluster). Users reach it over HTTPS at the registry's tailnet name @@ -97,7 +98,7 @@ pub fn router(state: AppState) -> Router { fn errors(status: StatusCode, code: &str, message: &str) -> Response { let mut r = ( status, - axum::Json(json!({ "errors": [{ "code": code, "message": message }] })), + crate::error::Json(json!({ "errors": [{ "code": code, "message": message }] })), ) .into_response(); r.headers_mut().insert( @@ -147,7 +148,7 @@ pub(crate) enum Machine { /// A builder: its team's builds too. Builder, /// A sandbox server making a template: pull only, its team's images - /// and builds. + /// and builds, or for an installation's template Pyo's own. Sandbox, } @@ -174,11 +175,19 @@ pub(crate) fn machine_allowed( .strip_prefix("pyo/") .is_some_and(|n| !n.is_empty() && !n.contains('/')) { - return read && who == Machine::Cluster; + // And an installation's sandbox template (no team: an empty slug). + return read && (who == Machine::Cluster || (who == Machine::Sandbox && slug.is_empty())); } read && parts(name).is_some_and(|(t, _, _)| t == slug) } +/// The door's 403 for an image that isn't the team's: what to do instead. +pub(crate) fn not_theirs(slug: &str) -> String { + format!( + "Not one of this team's images. Push it as {slug}// first. The Registry page shows how." + ) +} + /// What a request asks for: the repository it names (None for /v2/ itself) /// and whether it writes. #[derive(Debug, PartialEq)] @@ -405,6 +414,10 @@ async fn machine_door( Err(_) => None, }; match team { + // An installation's template: Pyo's own images only. + Some(t) if t == crate::sandbox::keys::PYO_IMAGES_ONLY => { + Ok(Some((String::new(), String::new()))) + } Some(t) => { sqlx::query_as("SELECT id, slug FROM pyo.teams WHERE id = $1") .bind(t) @@ -456,11 +469,7 @@ async fn machine_door( &slug, service_team.as_deref() == Some(team_id.as_str()), ) { - return errors( - StatusCode::FORBIDDEN, - "DENIED", - "Not one of this team's images.", - ); + return errors(StatusCode::FORBIDDEN, "DENIED", ¬_theirs(&slug)); } // A team's own repository: only one it pushed first. if self::parts(&name).is_some() { @@ -796,6 +805,29 @@ mod tests { false )); assert!(!machine_allowed(Sandbox, &get, "pyo/agent", "acme", false)); + // An installation's template (no team) pulls Pyo's own, and only those. + assert!(machine_allowed( + Sandbox, + &get, + "pyo/sandbox-desktop", + "", + false + )); + assert!(!machine_allowed( + Sandbox, + &put, + "pyo/sandbox-desktop", + "", + false + )); + assert!(!machine_allowed(Sandbox, &get, "acme/shop/web", "", false)); + assert!(!machine_allowed( + Sandbox, + &get, + "pyo/services/svc_a", + "", + false + )); assert!(!machine_allowed( Sandbox, &put, diff --git a/crates/pyo-api/src/sandbox/api.rs b/crates/pyo-api/src/sandbox/api.rs index c909421..53d50d3 100644 --- a/crates/pyo-api/src/sandbox/api.rs +++ b/crates/pyo-api/src/sandbox/api.rs @@ -3,9 +3,11 @@ //! talk to the sandbox server with the connection's token: exec, terminals, //! files and ports never go through Pyo. +use crate::error::Json; +use crate::error::Query; use axum::{ - Json, Router, - extract::{Path, Query, State}, + Router, + extract::{Path, State}, http::StatusCode, routing::{get, post}, }; @@ -22,7 +24,7 @@ use crate::{ pub fn routes() -> Router { Router::new() .route("/sandboxes", get(list).post(create)) - .route("/sandboxes/{id}", get(get_one).delete(delete)) + .route("/sandboxes/{id}", get(get_one).patch(rename).delete(delete)) .route("/sandboxes/{id}/pause", post(pause)) .route("/sandboxes/{id}/resume", post(resume)) .route("/sandboxes/{id}/extend", post(extend)) @@ -59,6 +61,9 @@ pub struct ListQuery { /// "live" (the default: not stopped or failed) or "all" (the latest 500). #[serde(default)] state: Option, + /// Only sandboxes whose name or id has this text, ignoring case. + #[serde(default)] + q: Option, } #[derive(Debug, Deserialize, utoipa::IntoParams)] @@ -80,7 +85,7 @@ async fn team_read(s: &AppState, user: &crate::auth::User, team: &str) -> ApiRes #[utoipa::path(get, path = "/sandboxes", operation_id = "listSandboxes", tag = "Sandboxes", params(ListQuery), responses( - (status = 200, description = "`sandboxes`, newest first, each with `id`, `teamId`, `projectId`, `serverId`, `server`, `template`, `templateId`, `state` (\"creating\", \"running\", \"pausing\", \"paused\", \"resuming\", \"stopping\", \"stopped\" or \"failed\"), `runtime` (\"firecracker\" or \"gvisor\"), `cpus`, `memoryMib`, `diskMib`, `network`, `isolation`, `labels`, `env` (names only), `url` (its server's daemon), `edgeUrl` (its server's edge proxy, or null), `ports` (each `port`, `public` and `url`), `error`, `createdBy`, `createdAt`, `startedAt`, `expiresAt`, `pausedAt` and `endedAt`. Also `limits` (the team's) and `inUse` (what its sandboxes take now).", body = Object), + (status = 200, description = "`sandboxes`, newest first, each with `id`, `name`, `teamId`, `projectId`, `serverId`, `server`, `template`, `templateId`, `state` (\"creating\", \"running\", \"pausing\", \"paused\", \"resuming\", \"stopping\", \"stopped\" or \"failed\"), `preparing` (true while it waits for its server to make its template), `runtime` (\"firecracker\" or \"gvisor\"), `cpus`, `memoryMib`, `diskMib`, `network`, `isolation`, `labels`, `env` (names only), `url` (its server's daemon), `edgeUrl` (its server's edge proxy, or null), `ports` (each `port`, `public` and `url`), `error`, `createdBy`, `createdAt`, `startedAt`, `expiresAt` (null when it runs until it's deleted), `pausedAt` and `endedAt`. Also `limits` (the team's, `maxTtlSecs` null when there's no limit) and `inUse` (what its sandboxes take now).", body = Object), (status = 404, description = "No such team, or you aren't in it.", body = ErrorBody), ), extensions(("x-pyo-access" = json!("read"))))] @@ -100,7 +105,7 @@ pub(crate) async fn list( let limits = super::limits::load(&mut c, &q.team).await?; let in_use = super::limits::in_use(&mut c, &q.team).await?; Ok(Json(json!({ - "sandboxes": super::sandbox_views(&s.db, &teams, None, live).await?, + "sandboxes": super::sandbox_views_matching(&s.db, &teams, None, live, q.q.as_deref()).await?, "limits": limits, "inUse": in_use, }))) @@ -133,16 +138,21 @@ pub(crate) async fn get_one( /// `connection`: the server's address and a token, valid an hour, for exec, /// terminals, files and ports there. Ask for a new token with /// `createSandboxToken`. `teamId` may be left out when the API key (or -/// you) can use only one team. +/// you) can use only one team. `server` runs it on that server only. +/// +/// A server that doesn't have Pyo's template `base` (or `desktop`) yet +/// makes it first: the first sandbox from it there takes a minute or so +/// longer, and later ones start at once. #[utoipa::path(post, path = "/sandboxes", operation_id = "createSandbox", tag = "Sandboxes", request_body(content = CreateSandbox), responses( (status = 201, description = "`sandbox` (as in `listSandboxes`) and `connection`: `url`, `sandboxId`, `token` and `expiresAt`.", body = Object), (status = 400, description = "A size, lifetime or other value Pyo can't accept, or no `teamId` when you can use more than one team.", body = ErrorBody), (status = 403, description = "Your role, the API key's access or the team's sandbox limits don't allow it.", body = ErrorBody), - (status = 404, description = "No such team, template or project.", body = ErrorBody), + (status = 404, description = "No such team, template, project or server.", body = ErrorBody), (status = 409, description = "No sandbox server can run it: none yet, none with the template, or none with KVM for `isolation: vm`.", body = ErrorBody), - (status = 503, description = "Every sandbox server is full or down. Try again soon.", body = ErrorBody), + (status = 502, description = "The server couldn't make `base` or `desktop`. The error says why.", body = ErrorBody), + (status = 503, description = "Every sandbox server is full or down, the server asked for has no room, or `base` or `desktop` is still being made there. Try again soon.", body = ErrorBody), ), extensions(("x-pyo-access" = json!("sandboxes"))))] pub(crate) async fn create( @@ -156,6 +166,36 @@ pub(crate) async fn create( )) } +#[derive(Debug, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct Rename { + /// The new name: up to 63 letters, digits, spaces, dashes, underscores + /// and dots. It needn't be unique. + name: String, +} + +/// Rename a sandbox +/// +/// Gives the sandbox another name. Its id stays the same, so scripts that +/// use it keep working. Works on ended sandboxes too. +#[utoipa::path(patch, path = "/sandboxes/{id}", operation_id = "renameSandbox", tag = "Sandboxes", + params(("id" = String, Path, description = "The sandbox's id.")), + request_body(content = Rename), + responses( + (status = 200, description = "The sandbox, with its new `name`.", body = Object), + (status = 400, description = "A name Pyo can't accept. The error says why.", body = ErrorBody), + (status = 404, description = "No such sandbox.", body = ErrorBody), + ), + extensions(("x-pyo-access" = json!("sandboxes"))))] +pub(crate) async fn rename( + State(s): State, + SessionUser(user): SessionUser, + Path(id): Path, + Json(b): Json, +) -> ApiResult> { + Ok(Json(super::rename(&s, &user, &id, &b.name).await?)) +} + /// Delete a sandbox /// /// Ends the sandbox and throws away its files. When its server can't be @@ -217,20 +257,21 @@ pub(crate) async fn resume( #[derive(Debug, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct Extend { - /// The sandbox now ends this long from now, like "30m" or "1h", or - /// seconds. Within the team's longest lifetime. + /// The sandbox now ends this long from now, like "30m", "1h" or "7d", + /// or seconds. "none": it runs until it's deleted. Within the team's + /// longest lifetime. #[schema(value_type = String)] ttl: Value, } /// Extend a sandbox /// -/// Sets when the sandbox ends: `ttl` from now. +/// Sets when the sandbox ends: `ttl` from now, or never with `"none"`. #[utoipa::path(post, path = "/sandboxes/{id}/extend", operation_id = "extendSandbox", tag = "Sandboxes", params(("id" = String, Path, description = "The sandbox's id.")), request_body(content = Extend), responses( - (status = 200, description = "The sandbox, with its new `expiresAt`.", body = Object), + (status = 200, description = "The sandbox, with its new `expiresAt` (null when it runs until it's deleted).", body = Object), (status = 403, description = "Longer than the team's sandboxes may live.", body = ErrorBody), (status = 404, description = "No such sandbox.", body = ErrorBody), ), @@ -491,7 +532,7 @@ pub(crate) async fn create_dns_record( #[utoipa::path(get, path = "/sandbox-templates", operation_id = "listSandboxTemplates", tag = "Sandboxes", params(TeamQuery), responses( - (status = 200, description = "`templates`, each with `id`, `teamId` (null for the installation's), `name`, `source` (`kind` and `image`), `state` (\"pending\", \"building\", \"ready\" or \"failed\", from its servers), `servers`, `error` and `createdAt`. Each of `servers` is a sandbox server that may use it, with `id`, `name`, `state` (\"pending\", \"building\", \"ready\" or \"failed\"), `phase` while building (\"queued\", \"pulling\", \"unpacking\" or \"snapshotting\"), `pulled` and `imageBytes`, `pullMs`, `unpackMs` and `snapshotMs`, `bytes` (disk it takes there), `snapshots` and `error`.", body = Object), + (status = 200, description = "`templates`, each with `id`, `teamId` (null for the installation's), `name`, `source` (`kind` and `image`), `state` (\"pending\", \"building\", \"ready\" or \"failed\", from its servers, and \"unavailable\" for `base` or `desktop` while Pyo has no image for it), `builtin`, `servers`, `error` and `createdAt`. Pyo's own `base` and `desktop` come first, are always listed and are ready to use: servers make them on first use, so their `servers` are those that have them. Each of the other templates' `servers` is a sandbox server that may use it, with `id`, `name`, `state` (\"pending\", \"building\", \"ready\" or \"failed\"), `phase` while building (\"queued\", \"pulling\", \"unpacking\" or \"snapshotting\"), `pulled` and `unpacked` of `imageBytes` (asked of the server as the list is read, while it builds), `pullMs`, `unpackMs` and `snapshotMs`, `bytes` (disk it takes there), `snapshots` and `error`.", body = Object), (status = 404, description = "No such team, or you aren't in it.", body = ErrorBody), ), extensions(("x-pyo-access" = json!("read"))))] @@ -503,7 +544,7 @@ pub(crate) async fn list_templates( team_read(&s, &user, &q.team).await?; let teams = [q.team]; Ok(Json( - json!({ "templates": super::template_views(&s.db, &teams, None).await? }), + json!({ "templates": super::template_views_live(&s, &teams, None).await? }), )) } diff --git a/crates/pyo-api/src/sandbox/base.rs b/crates/pyo-api/src/sandbox/base.rs index 70c39fc..a1e0637 100644 --- a/crates/pyo-api/src/sandbox/base.rs +++ b/crates/pyo-api/src/sandbox/base.rs @@ -1,50 +1,130 @@ -//! Pyo's built-in sandbox template `base`: Debian 13 with the common tools -//! (deploy/images/sandbox-base.Dockerfile). It's the installation's, so -//! every team has it, and a sandbox created without a template starts -//! from it. +//! Pyo's built-in sandbox templates ([`ALL`]), from upsetdev/pyo-sandbox-images +//! ([`super::images`]): +//! - `base`: Ubuntu with the tools coding agents expect. A sandbox created +//! without a template starts from it. +//! - `desktop`: `base` with a desktop, noVNC on port 6080 and a browser with +//! CDP on port 9222, started on first use. Its sandboxes get 2 CPUs and +//! 2 GiB unless the create says otherwise. //! -//! - Seeding: when a worker starts, and in every sandbox round, Pyo makes -//! sure `base` exists and points at its release's image -//! (`PYO_SANDBOX_BASE_IMAGE`, pinned by digest in release.json). It's -//! idempotent. -//! - On update: a new release has a new image, so `base` gets a new -//! revision (the digest's start). Servers make it beside the old one in -//! the background, and [`super::templates`] moves new sandboxes over -//! once it's ready and deletes the old one once no sandbox uses it. -//! An older Pyo still running during the update never moves it back. -//! - Dev builds have no image of their own: they use the latest release -//! Pyo found when it checked for updates, or wait for one. -//! - Nobody deletes it, and no other template may be named `base`. - -use std::sync::atomic::{AtomicBool, Ordering}; +//! They're the installation's, so every team has them. For each: +//! - Its image, in this order: its env var (`PYO_SANDBOX_BASE_REF`, +//! `PYO_SANDBOX_DESKTOP_REF`) when that's pinned by digest, for dev +//! builds and pinning. Else the newest of the images.json Pyo last read +//! and the one it shipped with ([`super::images::current`]). +//! - Seeding: when a worker starts, in every sandbox round and when Pyo +//! reads a newer images.json, Pyo makes sure it exists and points at that +//! image. It's idempotent. It's ready to use from the start, with or +//! without servers. Lists show it even before it's added ([`listed`]). +//! - On first use: servers don't make it when they join. The first sandbox +//! that needs it on a server has the server make it, then starts +//! ([`super::firstuse`]). Later ones there start from that copy. +//! - On a new image: it gets a new revision (the digest's start). Servers +//! that have an older one make it beside it in the background, and +//! [`super::templates`] moves new sandboxes over once it's ready and +//! deletes the old one once no sandbox uses it. Servers that never used it +//! stay as they are. It only moves forward: an older Pyo still running +//! during an update, or an older build of the images, never moves it back +//! ([`decide`]). +//! - Nobody deletes it, and no other template may take its name. + +use std::sync::atomic::{AtomicU8, Ordering}; use pyo_proto::{parse_version, pinned_image}; use serde_json::{Value, json}; +use super::images::Images; use crate::AppState; -/// Its name, which sandboxes ask for. -pub const NAME: &str = "base"; -/// Its id, the same on every installation. -pub const ID: &str = "stp_base"; -/// What it runs, shown beside its image. -pub const OS: &str = "Debian 13"; +/// One of Pyo's built-in templates. +#[derive(Debug)] +pub struct Builtin { + /// Its name, which sandboxes ask for, and its image's in images.json. + pub name: &'static str, + /// Its id, the same on every installation. + pub id: &'static str, + /// What it runs, when its image comes from its env var. + pub os: &'static str, + /// The env var that overrides its image. + pub env: &'static str, + /// The size its sandboxes get when the create gives none. + pub cpus: u32, + pub memory_mib: u32, +} + +pub const BASE: Builtin = Builtin { + name: "base", + id: "stp_base", + os: "Ubuntu 24.04", + env: pyo_proto::SANDBOX_BASE_ENV, + cpus: pyo_proto::sandbox::DEFAULT_CPUS, + memory_mib: pyo_proto::sandbox::DEFAULT_MEMORY_MIB, +}; + +pub const DESKTOP: Builtin = Builtin { + name: "desktop", + id: "stp_desktop", + os: "Ubuntu 24.04", + env: pyo_proto::SANDBOX_DESKTOP_ENV, + // The browser and the screen need room. + cpus: 2, + memory_mib: 2048, +}; + +/// Every built-in template, in the order lists show them. +pub const ALL: [&Builtin; 2] = [&BASE, &DESKTOP]; + +/// `base`'s name, which a create without a template asks for. +pub const NAME: &str = BASE.name; + +/// The built-in template named `name`. +pub fn by_name(name: &str) -> Option<&'static Builtin> { + ALL.into_iter().find(|b| b.name == name) +} + +/// The built-in template with id `id`. +pub fn by_id(id: &str) -> Option<&'static Builtin> { + ALL.into_iter().find(|b| b.id == id) +} + +/// Whether servers make template `id` on first use instead of when they +/// join: only the built-in ones. +pub fn on_demand(id: &str) -> bool { + by_id(id).is_some() +} -/// The image to seed `base` from, and the release it belongs to. -#[derive(Debug, Clone, PartialEq, Eq)] +/// The image to seed a built-in template from. +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Wanted { pub image: String, + /// The Pyo that seeds it. pub release: String, + /// What it runs, like "Ubuntu 24.04". + pub os: String, + /// The images' build date (YYYYMMDD) when they come from images.json. + pub build: Option, } -/// This release's image, from `PYO_SANDBOX_BASE_IMAGE`. None for a build -/// that isn't a release (scripts/pyo-up.sh leaves it empty). -pub fn from_env() -> Option { - let image = std::env::var("PYO_SANDBOX_BASE_IMAGE").ok()?; +/// `b`'s image from its env var, when that's pinned by digest. None +/// otherwise (scripts/pyo-up.sh leaves it empty unless it built one). +pub fn from_env(b: &Builtin) -> Option { + let image = std::env::var(b.env).ok()?; let image = image.trim(); pinned_image(image).then(|| Wanted { image: image.to_owned(), release: crate::updates::current().to_owned(), + os: b.os.to_owned(), + build: None, + }) +} + +/// `b`'s image in images.json. +pub fn from_images(b: &Builtin, i: &Images) -> Option { + let image = i.image(b.name)?; + pinned_image(image).then(|| Wanted { + image: image.to_owned(), + release: crate::updates::current().to_owned(), + os: i.os(), + build: Some(i.date.clone()), }) } @@ -57,11 +137,55 @@ pub fn revision(image: &str) -> Option { /// The template's source for an image. pub fn source(w: &Wanted) -> Value { - json!({ "kind": "image", "image": w.image, "os": OS, "release": w.release }) + let mut s = json!({ "kind": "image", "image": w.image, "os": w.os, "release": w.release }); + if let Some(build) = &w.build { + s["build"] = json!(build); + } + s +} + +/// Why `b` can't be used yet, when Pyo has no image for it. +pub fn unavailable(b: &Builtin) -> String { + format!( + "Pyo has no image for {}. Set {} to an image pinned by digest, or update Pyo.", + b.name, b.env + ) +} + +/// `b` as a list shows it before it's added: as it will be, from the image +/// it would have. Unavailable, with why, when there's none. +pub fn view(b: &Builtin, want: Option<&Wanted>) -> Value { + let (source, state, error) = match want { + Some(w) => (source(w), "ready", None), + None => ( + json!({ "kind": "image", "os": b.os }), + "unavailable", + Some(unavailable(b)), + ), + }; + json!({ + "id": b.id, "teamId": null, "name": b.name, "source": source, "state": state, + "builtin": true, "revision": want.and_then(|w| revision(&w.image)), + "snapshots": {}, "error": error, "createdBy": "Pyo", + "createdAt": null, "updatedAt": null, "servers": [], + }) +} + +/// Whether `b` is in the database yet. +pub async fn added(db: &sqlx::PgPool, b: &Builtin) -> sqlx::Result { + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pyo.sandbox_templates WHERE id = $1)") + .bind(b.id) + .fetch_one(db) + .await +} + +/// [`view`] with the image Pyo would add it with now. +pub async fn listed(db: &sqlx::PgPool, b: &Builtin) -> Value { + view(b, wanted(db, b).await.as_ref()) } -/// What seeding does, given the template's source now (None: there's no -/// `base` yet). +/// What seeding does, given the template's source now (None: it isn't +/// there yet). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Seed { Insert, @@ -70,6 +194,8 @@ pub enum Seed { Keep, } +/// Forward only: a newer Pyo's choice stays (an older Pyo still runs +/// during an update), and so does a newer build of the images. pub fn decide(have: Option<&Value>, want: &Wanted) -> Seed { let Some(src) = have else { return Seed::Insert; @@ -77,54 +203,78 @@ pub fn decide(have: Option<&Value>, want: &Wanted) -> Seed { if src["image"].as_str() == Some(want.image.as_str()) { return Seed::Keep; } - // A newer Pyo already moved it: this one is on its way out. let theirs = src["release"].as_str().and_then(parse_version); - match (theirs, parse_version(&want.release)) { - (Some(a), Some(b)) if a > b => Seed::Keep, - _ => Seed::Move, + if let (Some(a), Some(b)) = (theirs, parse_version(&want.release)) + && a > b + { + return Seed::Keep; + } + if let (Some(a), Some(b)) = (src["build"].as_str(), want.build.as_deref()) + && a > b + { + return Seed::Keep; } + Seed::Move +} + +/// Which image `b` takes: `env`'s (its env var, pinned) over `images`'s. +pub fn choose(b: &Builtin, env: Option, images: Option<&Images>) -> Option { + env.or_else(|| from_images(b, images?)) } -/// The image `base` should have now: this release's, else (a dev build) -/// the latest release found by the update check. -async fn wanted(s: &AppState) -> Option { - if let Some(w) = from_env() { +/// The image `b` should have now: its env var's, else images.json's. +pub async fn wanted(db: &sqlx::PgPool, b: &Builtin) -> Option { + if let Some(w) = from_env(b) { return Some(w); } - let found = crate::updates::checked(&s.db).await.ok()?.found?; - let image = found.release.images.sandbox_base; - let pinned = pinned_image(&image); - pinned.then_some(Wanted { - image, - release: found.release.version, - }) + choose(b, None, super::images::current(db).await.as_ref()) } -/// Said once per process, so a dev build doesn't repeat it every round. -static SAID_WAITING: AtomicBool = AtomicBool::new(false); +/// Which templates' missing image was said (a bit each, by place in +/// [`ALL`]), once per process, so it isn't repeated every round. +static SAID_MISSING: AtomicU8 = AtomicU8::new(0); -/// Makes sure `base` exists and has the image it should. Tells the servers -/// at once when that changed. -pub async fn ensure(s: &AppState) -> anyhow::Result<()> { - let Some(want) = wanted(s).await else { - if !SAID_WAITING.swap(true, Ordering::Relaxed) { - tracing::info!( - "sandbox template base: this build has no image for it, and no release was found yet, so it's added later" - ); +/// Makes sure `b` exists and has the image it should. Tells the servers at +/// once when that changed. False when there's no image for it. +pub async fn ensure(s: &AppState, b: &'static Builtin) -> anyhow::Result { + let Some(want) = wanted(&s.db, b).await else { + let bit = 1 << ALL.iter().position(|x| x.id == b.id).unwrap_or(0); + if SAID_MISSING.fetch_or(bit, Ordering::Relaxed) & bit == 0 { + tracing::warn!("sandbox template {}: {}", b.name, unavailable(b)); } - return Ok(()); + return Ok(false); }; - seed_from(s, &want).await.map(|_| ()) + seed_from(s, b, &want).await.map(|_| true) } -/// Seeds `base` from `want`, and says what it did. -pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result { +/// [`ensure`] for every built-in template. True once they all have an +/// image. The first error, after trying them all. +pub async fn ensure_all(s: &AppState) -> anyhow::Result { + let mut all = true; + let mut first = None; + for b in ALL { + match ensure(s, b).await { + Ok(had) => all &= had, + Err(e) => { + all = false; + first.get_or_insert(e); + } + } + } + match first { + Some(e) => Err(e), + None => Ok(all), + } +} + +/// Seeds `b` from `want`, and says what it did. +pub async fn seed_from(s: &AppState, b: &Builtin, want: &Wanted) -> anyhow::Result { let want = want.clone(); let rev = revision(&want.image); let mut tx = s.db.begin().await?; let have: Option = sqlx::query_scalar("SELECT source FROM pyo.sandbox_templates WHERE id = $1 FOR UPDATE") - .bind(ID) + .bind(b.id) .fetch_optional(&mut *tx) .await?; let seed = decide(have.as_ref(), &want); @@ -133,11 +283,11 @@ pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result { Seed::Insert => { let res = sqlx::query( "INSERT INTO pyo.sandbox_templates (id, team_id, name, source, state, created_by, builtin, revision) - VALUES ($1, NULL, $2, $3, 'pending', 'Pyo', true, $4) + VALUES ($1, NULL, $2, $3, 'ready', 'Pyo', true, $4) ON CONFLICT (id) DO NOTHING", ) - .bind(ID) - .bind(NAME) + .bind(b.id) + .bind(b.name) .bind(source(&want)) .bind(&rev) .execute(&mut *tx) @@ -145,13 +295,14 @@ pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result { if let Err(e) = res { if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23505") { tracing::warn!( - "sandbox template base: the installation has another template named base, so Pyo's isn't added" + "sandbox template {0}: the installation has another template named {0}, so Pyo's isn't added", + b.name ); return Ok(Seed::Keep); } return Err(e.into()); } - tracing::info!(image = %want.image, "sandbox template base added"); + tracing::info!(image = %want.image, "sandbox template {} added", b.name); } Seed::Move => { sqlx::query( @@ -159,26 +310,33 @@ pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result { SET source = $2, revision = $3, builtin = true, error = NULL, updated_at = now() WHERE id = $1", ) - .bind(ID) + .bind(b.id) .bind(source(&want)) .bind(&rev) .execute(&mut *tx) .await?; - tracing::info!(image = %want.image, "sandbox template base moved to a new image"); + tracing::info!(image = %want.image, "sandbox template {} moved to a new image", b.name); } } tx.commit().await?; - tokio::spawn(super::templates::push(s.clone(), ID.to_owned())); + tokio::spawn(super::templates::push(s.clone(), b.id.to_owned())); Ok(seed) } -/// On a worker's start: [`ensure`], retried while the database is busy. +/// On a worker's start: reads the latest images.json once, then +/// [`ensure_all`], retried while the database is busy. Sandbox rounds and +/// [`super::images::runner`] keep them in step after that. pub async fn seed(s: AppState) { - for attempt in 1..=10u64 { - match ensure(&s).await { - Ok(()) => return, + if let Err(e) = super::images::refresh(&s.db).await { + tracing::warn!(error = %format!("{e:#}"), "sandbox images: couldn't keep the latest"); + } + let mut attempt = 0u64; + while attempt < 10 && !s.stopping() { + match ensure_all(&s).await { + Ok(_) => return, Err(e) => { - tracing::warn!(attempt, error = %format!("{e:#}"), "sandbox template base: couldn't add it, retrying"); + attempt += 1; + tracing::warn!(attempt, error = %format!("{e:#}"), "sandbox templates: couldn't add Pyo's, retrying"); tokio::time::sleep(std::time::Duration::from_secs(15 * attempt)).await; } } @@ -189,24 +347,40 @@ pub async fn seed(s: AppState) { mod tests { use super::*; - const A: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - const B: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.2@sha256:cccccccccccc0000000000000000000000000000000000000000000000000000"; + const A: &str = "ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + const B: &str = "ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:cccccccccccc0000000000000000000000000000000000000000000000000000"; fn want(image: &str, release: &str) -> Wanted { Wanted { image: image.into(), release: release.into(), + os: "Ubuntu 24.04".into(), + build: None, } } + fn built(image: &str, release: &str, build: &str) -> Wanted { + Wanted { + build: Some(build.into()), + ..want(image, release) + } + } + + fn images() -> &'static Images { + super::super::images::shipped().unwrap() + } + #[test] fn seeding_is_idempotent() { let w = want(A, "0.4.1"); assert_eq!(decide(None, &w), Seed::Insert); let src = source(&w); assert_eq!(decide(Some(&src), &w), Seed::Keep, "the same image again"); - assert_eq!(src["os"], "Debian 13"); + assert_eq!(src["os"], "Ubuntu 24.04"); assert_eq!(src["kind"], "image"); + assert!(src.get("build").is_none()); + let src = source(&built(A, "0.4.1", "20261116")); + assert_eq!(src["build"], "20261116"); } #[test] @@ -228,12 +402,113 @@ mod tests { ); } + #[test] + fn a_newer_build_of_the_images_moves_it_forward_only() { + let old = built(A, "0.5.0", "20261116"); + let new = built(B, "0.5.0", "20261123"); + assert_eq!(decide(Some(&source(&old)), &new), Seed::Move); + // A replica that hasn't read the newer build yet leaves it. + assert_eq!(decide(Some(&source(&new)), &old), Seed::Keep); + // A rebuild the same day moves it. + assert_eq!( + decide(Some(&source(&old)), &built(B, "0.5.0", "20261116")), + Seed::Move + ); + // A newer Pyo with an older build (it couldn't reach GitHub) leaves + // it too. + assert_eq!( + decide(Some(&source(&new)), &built(A, "0.5.1", "20261116")), + Seed::Keep + ); + // An older Pyo (0.4.x, Debian from release.json) never moves it + // back, and the images move it from there. + let debian = json!({ "kind": "image", "os": "Debian 13", "release": "0.4.1", + "image": "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }); + assert_eq!(decide(Some(&debian), &new), Seed::Move); + let older_pyo = Wanted { + image: debian["image"].as_str().unwrap().into(), + release: "0.4.1".into(), + os: "Debian 13".into(), + build: None, + }; + assert_eq!(decide(Some(&source(&new)), &older_pyo), Seed::Keep); + // A pinned image (env var) of the same Pyo moves it, either way. + assert_eq!(decide(Some(&source(&new)), &want(A, "0.5.0")), Seed::Move); + } + + #[test] + fn the_env_var_wins_over_images_json() { + let i = images(); + let from_json = choose(&BASE, None, Some(i)).unwrap(); + assert_eq!(from_json.image, i.image("base").unwrap()); + assert_eq!(from_json.os, i.os()); + assert_eq!(from_json.build.as_deref(), Some(i.date.as_str())); + assert_eq!( + choose(&DESKTOP, None, Some(i)).unwrap().image, + i.image("desktop").unwrap() + ); + let pinned = want(A, "0.5.0"); + assert_eq!( + choose(&BASE, Some(pinned.clone()), Some(i)), + Some(pinned.clone()) + ); + assert_eq!(choose(&BASE, Some(pinned.clone()), None), Some(pinned)); + assert_eq!(choose(&BASE, None, None), None); + } + + #[test] + fn listed_before_it_is_added() { + let v = view(&BASE, Some(&want(A, "0.4.1"))); + assert_eq!( + (v["id"].as_str(), v["name"].as_str(), v["state"].as_str()), + (Some(BASE.id), Some(NAME), Some("ready")) + ); + assert_eq!(v["source"]["image"], A); + assert_eq!(v["revision"], "aaaaaaaaaaaa"); + assert_eq!(v["builtin"], true); + assert_eq!(v["servers"], json!([])); + // No image anywhere: listed, and why it can't be used yet. + let v = view(&BASE, None); + assert_eq!(v["state"], "unavailable"); + assert_eq!( + v["error"], + "Pyo has no image for base. Set PYO_SANDBOX_BASE_REF to an image pinned by digest, or update Pyo." + ); + assert_eq!(v["source"]["os"], BASE.os); + assert!(!unavailable(&BASE).contains(';')); + // desktop the same way, by its own name. + let v = view(&DESKTOP, None); + assert_eq!( + (v["id"].as_str(), v["name"].as_str()), + (Some("stp_desktop"), Some("desktop")) + ); + assert!(v["error"].as_str().unwrap().contains("image for desktop")); + } + + #[test] + fn the_builtins() { + assert_eq!(ALL.map(|b| b.name), ["base", "desktop"]); + assert!(on_demand("stp_base") && on_demand("stp_desktop")); + assert!(!on_demand("stp_other")); + assert_eq!(by_name("desktop").map(|b| b.id), Some("stp_desktop")); + assert!(by_name("node-22").is_none()); + // desktop's sandboxes get room for the browser. + const { assert!(DESKTOP.cpus >= 2 && DESKTOP.memory_mib >= 2048) }; + for b in ALL { + assert!(pyo_proto::sandbox::is_template_name(b.name)); + assert!( + !b.env.ends_with("_IMAGE"), + "older Pyos refuse new *_IMAGE vars" + ); + } + } + #[test] fn revisions_come_from_the_digest() { assert_eq!(revision(A).as_deref(), Some("aaaaaaaaaaaa")); assert_eq!(revision(B).as_deref(), Some("cccccccccccc")); assert_eq!(revision("debian:13"), None); - let name = super::super::templates::server_name(ID, revision(A).as_deref()); + let name = super::super::templates::server_name(BASE.id, revision(A).as_deref()); assert_eq!(name, "stp_base.aaaaaaaaaaaa"); assert!(pyo_proto::sandbox::is_template_name(&name)); } @@ -241,19 +516,20 @@ mod tests { #[test] fn only_a_pinned_image_from_the_env_counts() { // SAFETY: tests in this module don't read it concurrently. - unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", "") }; - assert_eq!(from_env(), None, "a dev build (pyo-up.sh)"); + unsafe { std::env::set_var("PYO_SANDBOX_BASE_REF", "") }; + assert_eq!(from_env(&BASE), None, "a dev build (pyo-up.sh)"); unsafe { std::env::set_var( - "PYO_SANDBOX_BASE_IMAGE", + "PYO_SANDBOX_BASE_REF", "127.0.0.1:5005/pyo/sandbox-base:dev", ) }; - assert_eq!(from_env(), None, "not pinned"); - unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", A) }; - assert_eq!(from_env(), Some(want(A, crate::updates::current()))); - unsafe { std::env::remove_var("PYO_SANDBOX_BASE_IMAGE") }; - assert_eq!(from_env(), None); + assert_eq!(from_env(&BASE), None, "not pinned"); + unsafe { std::env::set_var("PYO_SANDBOX_BASE_REF", A) }; + assert_eq!(from_env(&BASE), Some(want(A, crate::updates::current()))); + assert_eq!(from_env(&DESKTOP), None, "each has its own"); + unsafe { std::env::remove_var("PYO_SANDBOX_BASE_REF") }; + assert_eq!(from_env(&BASE), None); } /// Against a real Postgres (DATABASE_URL, a throwaway one, in a database @@ -283,19 +559,23 @@ mod tests { sqlx::query_as::<_, (String, String, Value, Option, bool, Option)>( "SELECT name, state, source, revision, builtin, team_id FROM pyo.sandbox_templates WHERE id = $1", ) - .bind(ID) + .bind(BASE.id) .fetch_one(&db) .await .unwrap() }; let old = want(A, "0.4.1"); - assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Insert); - assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep, "idempotent"); + assert_eq!(seed_from(&s, &BASE, &old).await.unwrap(), Seed::Insert); + assert_eq!( + seed_from(&s, &BASE, &old).await.unwrap(), + Seed::Keep, + "idempotent" + ); let (n, state, src, rev, builtin, team) = row().await; assert_eq!( (n.as_str(), state.as_str(), rev.as_deref(), builtin, team), - ("base", "pending", Some("aaaaaaaaaaaa"), true, None) + ("base", "ready", Some("aaaaaaaaaaaa"), true, None) ); assert_eq!(src["image"], A); let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pyo.sandbox_templates") @@ -310,12 +590,33 @@ mod tests { .unwrap(); assert_eq!(views[0]["name"], "base"); assert_eq!(views[0]["builtin"], true); - assert_eq!(views[0]["source"]["os"], "Debian 13"); + assert_eq!(views[0]["source"]["os"], "Ubuntu 24.04"); + // desktop right after, listed before it's added. A team's template + // named before both still comes after them. + sqlx::raw_sql( + "INSERT INTO pyo.teams (id, name, slug) VALUES ('tm_any', 'any', 'any'); + INSERT INTO pyo.sandbox_templates (id, team_id, name, source, state, created_by) + VALUES ('stp_alpine', 'tm_any', 'alpine', '{\"kind\": \"image\", \"image\": \"alpine\"}', 'ready', 'x')", + ) + .execute(&db) + .await + .unwrap(); + let views = super::super::template_views(&db, &["tm_any".into()], None) + .await + .unwrap(); + let names: Vec<&str> = views.iter().filter_map(|v| v["name"].as_str()).collect(); + assert_eq!(names, ["base", "desktop", "alpine"]); + assert_eq!(views[1]["id"], DESKTOP.id); + assert_eq!(views[1]["builtin"], true); + sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = 'stp_alpine'") + .execute(&db) + .await + .unwrap(); // A new release moves it. The old Pyo, still running, doesn't move it back. let new = want(B, "0.4.2"); - assert_eq!(seed_from(&s, &new).await.unwrap(), Seed::Move); - assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep); + assert_eq!(seed_from(&s, &BASE, &new).await.unwrap(), Seed::Move); + assert_eq!(seed_from(&s, &BASE, &old).await.unwrap(), Seed::Keep); let (_, _, src, rev, ..) = row().await; assert_eq!( (src["image"].as_str(), rev.as_deref()), @@ -332,7 +633,7 @@ mod tests { must_set_up: false, key: None, }; - let e = super::super::delete_template(&s, &user, ID) + let e = super::super::delete_template(&s, &user, BASE.id) .await .unwrap_err(); assert_eq!(e.status, axum::http::StatusCode::CONFLICT, "{}", e.message); diff --git a/crates/pyo-api/src/sandbox/e2e.rs b/crates/pyo-api/src/sandbox/e2e.rs index 893f8fc..1497aeb 100644 --- a/crates/pyo-api/src/sandbox/e2e.rs +++ b/crates/pyo-api/src/sandbox/e2e.rs @@ -249,6 +249,8 @@ async fn pyo_drives_a_real_daemon() { ); let candidate = scheduler::Candidate { id: server.into(), + name: "e2e".into(), + team_id: Some(team.into()), enabled: true, ready: true, report_age_secs: 0, @@ -437,7 +439,7 @@ async fn the_control_plane_drives_a_real_daemon() { &mut c, &team, &super::limits::Limits { - max_ttl_secs: 600, + max_ttl_secs: Some(600), ..Default::default() }, ) @@ -446,6 +448,8 @@ async fn the_control_plane_drives_a_real_daemon() { drop(c); let e = super::create(&s, &user, req("1h")).await.unwrap_err(); assert_eq!(e.status, axum::http::StatusCode::FORBIDDEN, "{}", e.message); + let e = super::create(&s, &user, req("none")).await.unwrap_err(); + assert_eq!(e.status, axum::http::StatusCode::FORBIDDEN, "{}", e.message); let out = super::create(&s, &user, req("5m")).await.unwrap(); let id = out["sandbox"]["id"].as_str().unwrap().to_owned(); @@ -605,10 +609,11 @@ async fn the_control_plane_drives_a_real_daemon() { } // ── The built-in template `base`: the default, and how a release moves it ── - // Two small public images stand in for two releases' sandbox-base. + // Two small public images stand in for two releases' base, of Pyos + // newer than this one, so the rounds below leave them as they are. use super::base; sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1") - .bind(base::ID) + .bind(base::BASE.id) .execute(&db) .await .unwrap(); @@ -616,25 +621,107 @@ async fn the_control_plane_drives_a_real_daemon() { image: "alpine:3.21@sha256:bc21487101ae47172509130ee1092cc273a831968d64b617776d1f69c81e42ab" .into(), - release: "0.0.1".into(), + release: "99.0.1".into(), + ..Default::default() }; let new = base::Wanted { image: "alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f" .into(), - release: "0.0.2".into(), + release: "99.0.2".into(), + ..Default::default() }; let rev = |w: &base::Wanted| { - super::templates::server_name(base::ID, base::revision(&w.image).as_deref()) + super::templates::server_name(base::BASE.id, base::revision(&w.image).as_deref()) }; let (old_name, new_name) = (rev(&old), rev(&new)); - assert_eq!(base::seed_from(&s, &old).await.unwrap(), base::Seed::Insert); - wait_for(&s, &team, &server, &old_name, false).await; - // No template: base. + // Listed, ready to use, before Pyo adds it. + let listed = super::template_views(&db, std::slice::from_ref(&team), None) + .await + .unwrap(); + assert_eq!(listed[0]["name"], "base", "{listed:?}"); + assert_eq!(listed[0]["servers"], json!([])); + assert_eq!( + base::seed_from(&s, &base::BASE, &old).await.unwrap(), + base::Seed::Insert + ); + // The server doesn't make it ahead of use. + for _ in 0..3 { + super::runner::round(&s).await.unwrap(); + } + let srv = &super::server_views(&db, std::slice::from_ref(&team), Some(&server)) + .await + .unwrap()[0]; + assert!( + srv["templates"] + .as_array() + .unwrap() + .iter() + .all(|t| !t["name"].as_str().unwrap_or("").starts_with("stp_base")), + "{srv}" + ); + let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::BASE.id)) + .await + .unwrap(); + assert_eq!(views[0]["state"], "ready", "ready to use: {}", views[0]); + assert_eq!(views[0]["servers"], json!([]), "no server has it yet"); + // No template: base. Two at once on a server without it: the server + // makes it once, both wait for it, then both start. let no_template: super::CreateSandbox = serde_json::from_value(json!({ "teamId": team, "network": "none", "ttl": "5m" })).unwrap(); - let first = super::create(&s, &user, no_template.clone()).await.unwrap(); + let t0 = std::time::Instant::now(); + let a = tokio::spawn({ + let (s, user, req) = (s.clone(), user.clone(), no_template.clone()); + async move { super::create(&s, &user, req).await } + }); + let b = tokio::spawn({ + let (s, user, req) = (s.clone(), user.clone(), no_template.clone()); + async move { super::create(&s, &user, req).await } + }); + // Meanwhile the sandboxes say they're waiting for base. + let mut saw_preparing = false; + while !(a.is_finished() && b.is_finished()) { + let list = super::sandbox_views(&db, std::slice::from_ref(&team), None, true) + .await + .unwrap(); + saw_preparing |= list.iter().any(|x| x["preparing"] == true); + tokio::time::sleep(Duration::from_millis(100)).await; + } + let first = a.await.unwrap().unwrap(); + let other = b.await.unwrap().unwrap(); + println!("e2e base on first use: {} ms", t0.elapsed().as_millis()); + assert!(saw_preparing); assert_eq!(first["sandbox"]["template"], "base", "{first}"); + assert_eq!(first["sandbox"]["preparing"], false, "{first}"); + assert_eq!(other["sandbox"]["state"], "running", "{other}"); + let builds: Vec = Daemon::new(&address) + .report(&keys::admin_token(&key, &server)) + .await + .unwrap() + .templates + .into_iter() + .filter(|t| t.name.starts_with("stp_base")) + .map(|t| json!({ "name": t.name, "startedAt": t.build.map(|b| b.started_at) })) + .collect(); + assert_eq!(builds.len(), 1, "{builds:?}"); + let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::BASE.id)) + .await + .unwrap(); + assert_eq!(views[0]["servers"][0]["id"], json!(server), "{}", views[0]); + super::delete(&s, &user, other["sandbox"]["id"].as_str().unwrap()) + .await + .unwrap(); + // On the server asked for, by name. One the team doesn't have is refused. + let mut on_e2e = no_template.clone(); + on_e2e.server = Some("e2e".into()); + let pinned = super::create(&s, &user, on_e2e.clone()).await.unwrap(); + assert_eq!(pinned["sandbox"]["serverId"], json!(server), "{pinned}"); + super::delete(&s, &user, pinned["sandbox"]["id"].as_str().unwrap()) + .await + .unwrap(); + on_e2e.server = Some("nowhere".into()); + let e = super::create(&s, &user, on_e2e).await.unwrap_err(); + assert_eq!(e.status, axum::http::StatusCode::NOT_FOUND, "{}", e.message); let first_id = first["sandbox"]["id"].as_str().unwrap().to_owned(); let on: String = sqlx::query_scalar("SELECT spec->>'template' FROM pyo.sandboxes WHERE id = $1") @@ -654,9 +741,12 @@ async fn the_control_plane_drives_a_real_daemon() { assert!(r.stdout.starts_with("3.21"), "{r:?}"); // A new release: made beside the old one, new sandboxes move over. - assert_eq!(base::seed_from(&s, &new).await.unwrap(), base::Seed::Move); assert_eq!( - base::seed_from(&s, &old).await.unwrap(), + base::seed_from(&s, &base::BASE, &new).await.unwrap(), + base::Seed::Move + ); + assert_eq!( + base::seed_from(&s, &base::BASE, &old).await.unwrap(), base::Seed::Keep, "the old Pyo doesn't move it back" ); @@ -686,18 +776,18 @@ async fn the_control_plane_drives_a_real_daemon() { assert!(r.stdout.starts_with("3.21"), "{r:?}"); super::delete(&s, &user, &first_id).await.unwrap(); wait_for(&s, &team, &server, &old_name, true).await; - let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::ID)) + let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::BASE.id)) .await .unwrap(); assert_eq!(views[0]["state"], "ready", "{}", views[0]); println!("e2e base: {}", views[0]["servers"]); super::delete(&s, &user, &second_id).await.unwrap(); sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1") - .bind(base::ID) + .bind(base::BASE.id) .execute(&db) .await .unwrap(); - super::templates::forget(s.clone(), base::ID.into()).await; + super::templates::forget(s.clone(), base::BASE.id.into()).await; sqlx::query("DELETE FROM pyo.teams WHERE id = $1") .bind(&team) diff --git a/crates/pyo-api/src/sandbox/edgecert/tests.rs b/crates/pyo-api/src/sandbox/edgecert/tests.rs index c295c08..c190092 100644 --- a/crates/pyo-api/src/sandbox/edgecert/tests.rs +++ b/crates/pyo-api/src/sandbox/edgecert/tests.rs @@ -3,10 +3,11 @@ //! wildcard record against the same fake. Nothing here reaches Let's //! Encrypt or Cloudflare. +use crate::error::Json; use std::sync::{Arc, Mutex}; use axum::{ - Json, Router, + Router, extract::{Path, State}, http::{HeaderMap, HeaderValue, StatusCode}, response::{IntoResponse, Response}, diff --git a/crates/pyo-api/src/sandbox/firstuse.rs b/crates/pyo-api/src/sandbox/firstuse.rs new file mode 100644 index 0000000..05e31f4 --- /dev/null +++ b/crates/pyo-api/src/sandbox/firstuse.rs @@ -0,0 +1,364 @@ +//! A template made on first use (Pyo's `base`, see [`super::base`]): the +//! first sandbox that needs it on a server has the server make it, waits +//! until it's ready, then starts. Later ones there start from that copy. +//! +//! - The create waits up to [`LIMIT`], under what HTTP clients wait for an +//! answer, so `Sandbox.create()` just takes longer the first time. Past +//! that the server keeps making it, and the create says to try again. +//! - Two creates at once share one build: they're placed on the server +//! already making it ([`super::scheduler::place`]), the server makes a +//! template it has only once, and a failed copy is cleared and made +//! again by one create at a time (a lock per server and template). +//! - What the server says is saved in its report at once, so the +//! templates page and the scheduler see the build before the next round. + +use std::future::Future; +use std::time::{Duration, Instant}; + +use pyo_proto::sandbox::TemplateState; +use pyo_proto::sandbox::token::SigningKey; +use pyo_proto::sandbox::{DaemonCommand, DaemonError, DaemonResult, ErrorCode, TemplateReport}; +use sqlx::PgPool; + +use super::{daemon::Daemon, keys, templates}; +use crate::error::ApiError; + +/// How long a create waits for its server to make the template. +pub const LIMIT: Duration = Duration::from_secs(240); +/// How often it asks the server how far it got. +const EVERY: Duration = Duration::from_secs(1); + +/// A server, as the first use sees it. +pub(crate) trait Host { + type Guard: Send; + /// One create at a time clears a failed copy and starts the build. + fn lock(&self) -> impl Future> + Send; + /// Starts making the template unless the server has it (ready, + /// building or failed), and says how far it got. + fn prepare(&self) -> impl Future> + Send; + /// Deletes the server's copy. + fn delete(&self) -> impl Future> + Send; +} + +/// Why the template isn't ready on the server. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotReady { + /// The build failed, and why. + Failed(String), + /// Still being made after the limit. + Slow, + /// The server didn't answer, or refused. + Server(DaemonError), +} + +impl NotReady { + /// As the create answers it. + pub fn error(self, template: &str, server: &str) -> ApiError { + match self { + NotReady::Failed(why) => ApiError::new( + axum::http::StatusCode::BAD_GATEWAY, + format!( + "{server} couldn't get {template} ready: {}", + why.trim_end_matches('.') + ) + ".", + ), + NotReady::Slow => ApiError::new( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + format!("{template} is still getting ready on {server}. Try again in a minute."), + ), + NotReady::Server(e) => super::daemon_error(e), + } + } +} + +impl From for NotReady { + fn from(e: DaemonError) -> Self { + NotReady::Server(e) + } +} + +/// Has `host` make the template unless it has it, then waits until it's +/// ready, up to `limit`, asking every `every`. +pub(crate) async fn get_ready( + host: &H, + limit: Duration, + every: Duration, +) -> Result<(), NotReady> { + let deadline = Instant::now() + limit; + let mut r = { + let _one_at_a_time = host + .lock() + .await + .map_err(|e| NotReady::Server(DaemonError::new(ErrorCode::Internal, e.message)))?; + let r = host.prepare().await?; + if r.state == TemplateState::Failed { + // Tried before and failed: try again. + host.delete().await?; + host.prepare().await? + } else { + r + } + }; + loop { + match r.state { + TemplateState::Ready => return Ok(()), + TemplateState::Failed => { + let why = r.build.and_then(|b| b.error); + return Err(NotReady::Failed( + why.unwrap_or_else(|| "it failed to build".into()), + )); + } + TemplateState::Building => {} + } + if Instant::now() >= deadline { + return Err(NotReady::Slow); + } + tokio::time::sleep(every).await; + r = host.prepare().await?; + } +} + +/// A sandbox server over the tailnet, for one template. +struct Server<'a> { + db: &'a PgPool, + id: &'a str, + daemon: Daemon, + token: String, + prepare: DaemonCommand, + name: String, +} + +impl Server<'_> { + async fn save(&self, r: Option<&TemplateReport>) { + templates::save_report(self.db, self.id, &self.name, r).await; + } +} + +impl Host for Server<'_> { + type Guard = sqlx::Transaction<'static, sqlx::Postgres>; + + async fn lock(&self) -> Result { + let mut tx = self.db.begin().await?; + sqlx::query( + "SELECT pg_advisory_xact_lock(hashtext('pyo-sandbox-template:' || $1 || ':' || $2))", + ) + .bind(self.id) + .bind(&self.name) + .execute(&mut *tx) + .await?; + Ok(tx) + } + + async fn prepare(&self) -> Result { + match self.daemon.command(&self.token, &self.prepare).await? { + DaemonResult::Template { template } => { + self.save(Some(&template)).await; + Ok(template) + } + other => Err(DaemonError::new( + ErrorCode::Internal, + format!("The sandbox server answered with something else: {other:?}"), + )), + } + } + + async fn delete(&self) -> Result<(), DaemonError> { + let cmd = DaemonCommand::DeleteTemplate { + template: self.name.clone(), + }; + self.daemon.command(&self.token, &cmd).await?; + self.save(None).await; + Ok(()) + } +} + +/// Has server `server_id` (at `address`) make template `template_id` +/// under `name` (its current revision) and waits until it's ready there. +pub async fn on_server( + db: &PgPool, + key: &SigningKey, + server_id: &str, + address: &str, + template_id: &str, + name: &str, +) -> Result<(), NotReady> { + let t = sqlx::query_as::<_, templates::Template>( + "SELECT id, team_id, source, revision FROM pyo.sandbox_templates WHERE id = $1", + ) + .bind(template_id) + .fetch_optional(db) + .await + .map_err(|e| NotReady::Server(DaemonError::new(ErrorCode::Internal, e.to_string())))? + .ok_or_else(|| NotReady::Failed("it was deleted".into()))?; + // Pyo's registry has no tailnet address yet (it just started): the + // create says to try again in a minute. + let prepare = templates::prepare_command(key, &t).ok_or(NotReady::Slow)?; + let host = Server { + db, + id: server_id, + daemon: Daemon::new(address), + token: keys::admin_token(key, server_id), + prepare, + name: name.to_owned(), + }; + tracing::info!(server = %server_id, template = %name, "a sandbox needs a template its server doesn't have yet: making it"); + get_ready(&host, LIMIT, EVERY).await +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo_proto::sandbox::TemplateBuild; + use std::sync::Arc; + use std::sync::Mutex as StdMutex; + + /// A server that makes a template in `polls` asks, or fails to. + #[derive(Default)] + struct Fake { + /// The template's state on the server, and asks since its build + /// started. None: it has none. + state: StdMutex>, + builds: StdMutex, + deletes: StdMutex, + polls: u32, + fails: Option, + lock: Arc>, + } + + impl Fake { + fn report(&self, state: TemplateState) -> TemplateReport { + TemplateReport { + name: "stp_base.aaaa".into(), + state, + build: Some(TemplateBuild { + error: (state == TemplateState::Failed) + .then(|| self.fails.clone().unwrap_or_default()), + ..Default::default() + }), + ..Default::default() + } + } + } + + impl Host for Fake { + type Guard = tokio::sync::OwnedMutexGuard<()>; + + async fn lock(&self) -> Result { + Ok(self.lock.clone().lock_owned().await) + } + + async fn prepare(&self) -> Result { + // A real server answers over the network: others get a turn. + tokio::task::yield_now().await; + let mut st = self.state.lock().unwrap(); + let next = match *st { + None => { + *self.builds.lock().unwrap() += 1; + (TemplateState::Building, 0) + } + Some((TemplateState::Building, n)) if n + 1 >= self.polls => match self.fails { + Some(_) => (TemplateState::Failed, n + 1), + None => (TemplateState::Ready, n + 1), + }, + Some((TemplateState::Building, n)) => (TemplateState::Building, n + 1), + Some(done) => done, + }; + *st = Some(next); + Ok(self.report(next.0)) + } + + async fn delete(&self) -> Result<(), DaemonError> { + tokio::task::yield_now().await; + *self.deletes.lock().unwrap() += 1; + *self.state.lock().unwrap() = None; + Ok(()) + } + } + + const QUICK: Duration = Duration::from_millis(1); + + #[tokio::test] + async fn makes_it_then_waits_until_ready() { + let f = Fake { + polls: 3, + ..Default::default() + }; + get_ready(&f, LIMIT, QUICK).await.unwrap(); + assert_eq!(*f.builds.lock().unwrap(), 1); + // The next create finds it ready, with no build. + get_ready(&f, LIMIT, QUICK).await.unwrap(); + assert_eq!(*f.builds.lock().unwrap(), 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_creates_at_once_start_one_build() { + let f = Arc::new(Fake { + polls: 5, + ..Default::default() + }); + let (a, b) = tokio::join!(get_ready(&*f, LIMIT, QUICK), get_ready(&*f, LIMIT, QUICK)); + a.unwrap(); + b.unwrap(); + assert_eq!(*f.builds.lock().unwrap(), 1); + // Many at once, after a failed build: one clears it and starts + // again, the rest wait on that build. + let f = Arc::new(Fake { + polls: 3, + state: StdMutex::new(Some((TemplateState::Failed, 0))), + ..Default::default() + }); + let all = futures::future::join_all((0..8).map(|_| { + let f = f.clone(); + tokio::spawn(async move { get_ready(&*f, LIMIT, QUICK).await }) + })) + .await; + for r in all { + r.unwrap().unwrap(); + } + assert_eq!( + (*f.builds.lock().unwrap(), *f.deletes.lock().unwrap()), + (1, 1) + ); + } + + #[tokio::test] + async fn a_failed_build_says_why_and_is_tried_again_next_time() { + let f = Fake { + polls: 2, + fails: Some("no space left on device".into()), + ..Default::default() + }; + let e = get_ready(&f, LIMIT, QUICK).await.unwrap_err(); + assert_eq!(e, NotReady::Failed("no space left on device".into())); + let api = e.error("base", "sandbox-1"); + assert_eq!( + api.message, + "sandbox-1 couldn't get base ready: no space left on device." + ); + assert_eq!(api.status, axum::http::StatusCode::BAD_GATEWAY); + get_ready(&f, LIMIT, QUICK).await.unwrap_err(); + assert_eq!( + (*f.builds.lock().unwrap(), *f.deletes.lock().unwrap()), + (2, 1) + ); + } + + #[tokio::test] + async fn past_the_limit_it_says_to_try_again() { + let f = Fake { + polls: u32::MAX, + ..Default::default() + }; + let e = get_ready(&f, Duration::from_millis(20), QUICK) + .await + .unwrap_err(); + assert_eq!(e, NotReady::Slow); + let api = e.error("base", "sandbox-1"); + assert_eq!( + api.message, + "base is still getting ready on sandbox-1. Try again in a minute." + ); + assert_eq!(api.status, axum::http::StatusCode::SERVICE_UNAVAILABLE); + assert!(!api.message.contains(';')); + } +} diff --git a/crates/pyo-api/src/sandbox/images.rs b/crates/pyo-api/src/sandbox/images.rs new file mode 100644 index 0000000..b9508c5 --- /dev/null +++ b/crates/pyo-api/src/sandbox/images.rs @@ -0,0 +1,406 @@ +//! Where the images of Pyo's built-in sandbox templates ([`super::base`]) +//! come from: upsetdev/pyo-sandbox-images. Every build there publishes +//! `base` and `desktop` (Ubuntu) and a release whose images.json names +//! each by digest, with the image contract they're made for, signed by +//! that repo's release workflow on main. +//! +//! - Pyo reads the latest release's images.json when a worker starts and +//! every hour after. It uses it only when the signature checks out +//! (sigstore.rs) and its contract is one Pyo runs +//! ([`pyo_proto::sandbox::CONTRACTS`]). It keeps it in pyo.settings, so +//! every replica takes the same one. +//! - A newer build there gives `base` and `desktop` a new revision. It +//! never goes back to an older build (by its date). +//! - Pyo ships the images.json of when it was made +//! (assets/sandbox-images/), so a fresh install has images before it +//! reaches GitHub, and one that never does still has some. +//! - `PYO_UPDATE_CHECK=off` stops the reading too, as it stops every check +//! of GitHub. Pyo then uses the images.json it shipped with. +//! - `PYO_SANDBOX_BASE_REF` and `PYO_SANDBOX_DESKTOP_REF` override both, +//! for dev builds and pinning ([`super::base::from_env`]). + +use std::collections::BTreeMap; +use std::sync::LazyLock; +use std::time::Duration; + +use pyo_proto::pinned_image; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::PgPool; + +use crate::AppState; + +/// Where the latest release's files are. +pub const LATEST: &str = "https://github.com/upsetdev/pyo-sandbox-images/releases/latest/download"; +/// How often Pyo looks for a newer build. +const EVERY: Duration = Duration::from_secs(3600); +/// Its key in pyo.settings. +const KEY: &str = "sandbox_images"; +/// images.json is small, and so is its signature. +const MAX: usize = 1 << 20; + +/// What Pyo shipped with: the images.json of when it was made, and its +/// signature (a test checks it). +const SHIPPED_JSON: &[u8] = include_bytes!("../../assets/sandbox-images/images.json"); + +/// One image in images.json. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Image { + pub repository: String, + pub digest: String, + /// `repository@digest`. + #[serde(rename = "ref")] + pub reference: String, + /// Each tag's digest when it was built. + #[serde(default)] + pub tags: BTreeMap, +} + +/// images.json. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Images { + /// The image contract they're made for. + pub contract: u32, + /// Ubuntu's version, like "24.04". + pub ubuntu: String, + /// The day it was built, YYYYMMDD. + pub date: String, + /// The commit it was built from. + pub commit: String, + pub images: BTreeMap, +} + +impl Images { + /// Reads and checks images.json. Its signature is [`verified`]'s. + pub fn parse(json: &[u8]) -> anyhow::Result { + let i: Images = serde_json::from_slice(json) + .map_err(|e| anyhow::anyhow!("images.json can't be read: {e}"))?; + i.check()?; + Ok(i) + } + + fn check(&self) -> anyhow::Result<()> { + anyhow::ensure!( + pyo_proto::sandbox::CONTRACTS.contains(&self.contract), + "images.json is for image contract {}, which this Pyo doesn't run. A newer Pyo does.", + self.contract + ); + anyhow::ensure!( + self.date.len() == 8 && self.date.bytes().all(|b| b.is_ascii_digit()), + "images.json has no build date" + ); + for b in super::base::ALL { + let image = self + .images + .get(b.name) + .ok_or_else(|| anyhow::anyhow!("images.json has no {} image", b.name))?; + anyhow::ensure!( + pinned_image(&image.reference) + && image.reference == format!("{}@{}", image.repository, image.digest), + "images.json's {} image isn't pinned by digest: {}", + b.name, + image.reference + ); + } + Ok(()) + } + + /// The image of template `name`, pinned by digest. + pub fn image(&self, name: &str) -> Option<&str> { + self.images.get(name).map(|i| i.reference.as_str()) + } + + /// What the images run, like "Ubuntu 24.04". + pub fn os(&self) -> String { + format!("Ubuntu {}", self.ubuntu) + } +} + +/// The images.json Pyo shipped with. None only if it's broken, which a +/// test rules out. +pub fn shipped() -> Option<&'static Images> { + static SHIPPED: LazyLock> = LazyLock::new(|| { + match Images::parse(SHIPPED_JSON) { + Ok(i) => Some(i), + Err(e) => { + tracing::error!(error = %format!("{e:#}"), "the sandbox images Pyo shipped with can't be used"); + None + } + } + }); + SHIPPED.as_ref() +} + +/// images.json, once its signature checks out: signed by +/// pyo-sandbox-images' release workflow on main. +pub fn verified(json: &[u8], bundle: &str) -> anyhow::Result { + crate::sigstore::verify(json, bundle, &crate::sigstore::SANDBOX_IMAGES_SIGNER).map_err( + |e| { + anyhow::anyhow!( + "images.json isn't signed by pyo-sandbox-images' release workflow: {e:#}" + ) + }, + )?; + Images::parse(json) +} + +/// What Pyo keeps of the last read. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Stored { + /// The newest images.json read so far that checked out. + pub images: Option, + pub checked_at: Option, + /// Why the last read failed. + pub error: Option, +} + +impl Stored { + /// Takes in a read: a newer images.json replaces the one kept, an older + /// one doesn't, and a failed read keeps it and says why. True when the + /// images changed. + pub fn take(&mut self, read: anyhow::Result, now_ms: i64) -> bool { + self.checked_at = Some(now_ms); + match read { + Ok(new) => { + self.error = None; + let newer = match &self.images { + None => true, + Some(have) => new.date >= have.date && &new != have, + }; + if newer { + self.images = Some(new); + } + newer + } + Err(e) => { + self.error = Some(format!("{e:#}")); + false + } + } + } +} + +/// The newer of two by build date, `a` when they're as new. +pub fn newest<'a>(a: Option<&'a Images>, b: Option<&'a Images>) -> Option<&'a Images> { + match (a, b) { + (Some(a), Some(b)) if b.date > a.date => Some(b), + (Some(a), _) => Some(a), + (None, b) => b, + } +} + +pub async fn stored(db: &PgPool) -> sqlx::Result { + let v: Option = sqlx::query_scalar("SELECT value FROM pyo.settings WHERE key = $1") + .bind(KEY) + .fetch_optional(db) + .await?; + Ok(v.and_then(|v| serde_json::from_value::(v).ok()) + .map(|mut s| { + // Kept by a Pyo that ran another contract: not for this one. + if s.images.as_ref().is_some_and(|i| i.check().is_err()) { + s.images = None; + } + s + }) + .unwrap_or_default()) +} + +/// The images Pyo uses now: the newest of the last one read and the one it +/// shipped with. +pub async fn current(db: &PgPool) -> Option { + let kept = stored(db).await.ok().and_then(|s| s.images); + newest(kept.as_ref(), shipped()).cloned() +} + +/// The latest release's images.json from GitHub, signature checked. +pub async fn fetch() -> anyhow::Result { + let http = crate::updates::client()?; + let get = |name: &'static str| { + let http = http.clone(); + async move { + let res = http.get(format!("{LATEST}/{name}")).send().await?; + anyhow::ensure!( + res.status().is_success(), + "GitHub answered {} for {name}", + res.status() + ); + crate::updates::body_limited(res, MAX).await + } + }; + let json = get("images.json").await?; + let bundle = get("images.json.sigstore.json").await?; + verified(&json, &String::from_utf8_lossy(&bundle)) +} + +/// Reads the latest images.json and keeps it when it's newer. True when +/// the images changed. Nothing is read with `PYO_UPDATE_CHECK=off`. +pub async fn refresh(db: &PgPool) -> anyhow::Result { + if crate::updates::checks_off_by_env() { + return Ok(false); + } + let read = fetch().await; + if let Err(e) = &read { + tracing::warn!(error = %format!("{e:#}"), "sandbox images: couldn't read the latest images.json, keeping the ones Pyo has"); + } + let mut s = stored(db).await?; + let changed = s.take(read, pyo_proto::now_ms()); + if changed && let Some(i) = &s.images { + tracing::info!(date = %i.date, commit = %i.commit, "sandbox images: a newer build"); + } + sqlx::query( + "INSERT INTO pyo.settings (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = excluded.value", + ) + .bind(KEY) + .bind(serde_json::to_value(&s)?) + .execute(db) + .await?; + Ok(changed) +} + +/// On a worker: reads images.json every hour, and moves the built-in +/// templates when there's a newer build. [`super::base::seed`] reads it +/// once at start. +pub async fn runner(s: AppState) { + loop { + tokio::time::sleep(EVERY).await; + if s.stopping() { + return; + } + match refresh(&s.db).await { + Ok(true) => { + if let Err(e) = super::base::ensure_all(&s).await { + tracing::warn!(error = %format!("{e:#}"), "sandbox templates: couldn't move Pyo's to the newer images"); + } + } + Ok(false) => {} + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "sandbox images: couldn't keep the latest") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const BUNDLE: &str = include_str!("../../assets/sandbox-images/images.json.sigstore.json"); + + fn with(date: &str, digest: char) -> Images { + let mut i = Images::parse(SHIPPED_JSON).unwrap(); + i.date = date.into(); + for img in i.images.values_mut() { + img.digest = format!("sha256:{}", digest.to_string().repeat(64)); + img.reference = format!("{}@{}", img.repository, img.digest); + } + i + } + + #[test] + fn what_pyo_shipped_with_is_signed_and_usable() { + let i = verified(SHIPPED_JSON, BUNDLE).unwrap(); + assert_eq!(Some(&i), shipped()); + assert_eq!(i.contract, 1); + assert!(i.os().starts_with("Ubuntu "), "{}", i.os()); + for b in super::super::base::ALL { + let image = i.image(b.name).unwrap(); + assert!(pinned_image(image), "{image}"); + assert!( + image.starts_with(&format!("ghcr.io/upsetdev/pyo-sandbox-images/{}@", b.name)), + "{image}" + ); + } + } + + #[test] + fn a_signature_that_doesnt_check_out_is_refused() { + // Another file under the same signature. + let mut other = SHIPPED_JSON.to_vec(); + let at = other.iter().position(|&b| b == b'7').unwrap(); + other[at] = b'8'; + let e = format!("{:#}", verified(&other, BUNDLE).unwrap_err()); + assert!(e.contains("isn't signed by pyo-sandbox-images"), "{e}"); + // No signature, or not a bundle. + assert!(verified(SHIPPED_JSON, "").is_err()); + assert!(verified(SHIPPED_JSON, "{}").is_err()); + // Signed by Pyo's own release workflow instead: not this one. + assert!( + crate::sigstore::verify(SHIPPED_JSON, BUNDLE, &crate::sigstore::RELEASE_SIGNER) + .is_err() + ); + } + + #[test] + fn only_contracts_pyo_runs() { + let mut v: Value = serde_json::from_slice(SHIPPED_JSON).unwrap(); + v["contract"] = 2.into(); + let e = format!("{:#}", Images::parse(v.to_string().as_bytes()).unwrap_err()); + assert_eq!( + e, + "images.json is for image contract 2, which this Pyo doesn't run. A newer Pyo does." + ); + assert!(!e.contains(';')); + } + + #[test] + fn every_image_pinned() { + let mut v: Value = serde_json::from_slice(SHIPPED_JSON).unwrap(); + v["images"]["desktop"]["ref"] = "ghcr.io/upsetdev/pyo-sandbox-images/desktop:24.04".into(); + let e = format!("{:#}", Images::parse(v.to_string().as_bytes()).unwrap_err()); + assert!(e.contains("desktop image isn't pinned"), "{e}"); + let mut v: Value = serde_json::from_slice(SHIPPED_JSON).unwrap(); + v["images"].as_object_mut().unwrap().remove("base"); + let e = format!("{:#}", Images::parse(v.to_string().as_bytes()).unwrap_err()); + assert_eq!(e, "images.json has no base image"); + let mut v: Value = serde_json::from_slice(SHIPPED_JSON).unwrap(); + v["date"] = "2026-11-16".into(); + assert!(Images::parse(v.to_string().as_bytes()).is_err()); + assert!(Images::parse(b"").is_err()); + } + + #[test] + fn a_read_moves_forward_only() { + let mut s = Stored::default(); + let (a, b) = (with("20261116", 'a'), with("20261123", 'b')); + assert!(s.take(Ok(a.clone()), 1), "the first"); + assert!(!s.take(Ok(a.clone()), 2), "the same again"); + assert!(s.take(Ok(b.clone()), 3), "a newer build"); + assert!(!s.take(Ok(a.clone()), 4), "never back to an older one"); + assert_eq!(s.images.as_ref(), Some(&b)); + // A rebuild the same day replaces it. + let b2 = with("20261123", 'c'); + assert!(s.take(Ok(b2.clone()), 5)); + // A failed read keeps what it had, and says why. + assert!(!s.take(Err(anyhow::anyhow!("GitHub answered 503")), 6)); + assert_eq!(s.images.as_ref(), Some(&b2)); + assert_eq!(s.error.as_deref(), Some("GitHub answered 503")); + assert_eq!(s.checked_at, Some(6)); + assert!(!s.take(Ok(b2.clone()), 7)); + assert_eq!(s.error, None, "cleared by the next read"); + // It survives a round trip through pyo.settings. + let back: Stored = serde_json::from_value(serde_json::to_value(&s).unwrap()).unwrap(); + assert_eq!(back, s); + } + + #[test] + fn the_newest_wins() { + let (old, new) = (with("20261116", 'a'), with("20261201", 'b')); + assert_eq!(newest(Some(&old), Some(&new)), Some(&new)); + assert_eq!(newest(Some(&new), Some(&old)), Some(&new)); + // As new: the one read wins over the one shipped. + let same_day = with("20261116", 'c'); + assert_eq!(newest(Some(&same_day), Some(&old)), Some(&same_day)); + assert_eq!(newest(None, Some(&old)), Some(&old)); + assert_eq!(newest(Some(&old), None), Some(&old)); + assert_eq!(newest(None, None), None); + } + + /// The real thing, from GitHub. + #[tokio::test] + #[ignore = "reads GitHub"] + async fn reads_the_latest_images_json() { + let i = fetch().await.unwrap(); + assert!(i.date >= shipped().unwrap().date, "{i:?}"); + } +} diff --git a/crates/pyo-api/src/sandbox/join.rs b/crates/pyo-api/src/sandbox/join.rs index 95aeb2e..e1ae203 100644 --- a/crates/pyo-api/src/sandbox/join.rs +++ b/crates/pyo-api/src/sandbox/join.rs @@ -18,12 +18,13 @@ //! A server that can run neither Firecracker nor gVisor is refused, with //! the detection's reason. +use crate::error::Query; use std::net::{IpAddr, SocketAddr}; use std::time::Duration; use axum::{ Router, - extract::{ConnectInfo, Query, State}, + extract::{ConnectInfo, State}, http::{StatusCode, header}, response::{IntoResponse, Response}, routing::post, diff --git a/crates/pyo-api/src/sandbox/keys.rs b/crates/pyo-api/src/sandbox/keys.rs index f31e1e3..eba502c 100644 --- a/crates/pyo-api/src/sandbox/keys.rs +++ b/crates/pyo-api/src/sandbox/keys.rs @@ -154,9 +154,13 @@ pub fn admin_token(key: &SigningKey, server: &str) -> String { .token } +/// The team a registry token names for an installation's template: it may +/// pull Pyo's own images (`pyo/`) and no team's. +pub const PYO_IMAGES_ONLY: &str = "pyo"; + /// A sandbox server's login at Pyo's registry door (password, with the /// username [`crate::registrydoor::SANDBOX_USER`]) to pull a template's -/// image of `team`. +/// image of `team` (or, with [`PYO_IMAGES_ONLY`], one of Pyo's own). pub fn registry_token(key: &SigningKey, template: &str, team: &str) -> String { sign( key, @@ -311,6 +315,17 @@ mod tests { check_registry_token(&keys, ®istry_token(&k, "stp_one", "")), None ); + // An installation's template: Pyo's own images, which the door + // tells apart from a team's. + assert_eq!( + check_registry_token(&keys, ®istry_token(&k, "stp_desktop", PYO_IMAGES_ONLY)) + .as_deref(), + Some(PYO_IMAGES_ONLY) + ); + assert!( + !PYO_IMAGES_ONLY.contains('_'), + "never a team's id (prefix_random)" + ); } /// Against a throwaway Postgres: `DATABASE_URL=… cargo test -p pyo-api diff --git a/crates/pyo-api/src/sandbox/limits.rs b/crates/pyo-api/src/sandbox/limits.rs index 4906551..0a7ffb8 100644 --- a/crates/pyo-api/src/sandbox/limits.rs +++ b/crates/pyo-api/src/sandbox/limits.rs @@ -22,8 +22,11 @@ pub struct Limits { pub max_memory_mib: i32, /// Disk of those sandboxes together, in MiB. pub max_disk_mib: i32, - /// The longest lifetime one sandbox may have, in seconds. - pub max_ttl_secs: i32, + /// The longest lifetime one sandbox may have, in seconds, up to 365 + /// days. Null: no limit, and sandboxes may run until they're deleted. + #[serde(deserialize_with = "Option::deserialize")] + #[schema(required = true)] + pub max_ttl_secs: Option, } impl Default for Limits { @@ -33,7 +36,7 @@ impl Default for Limits { max_cpus: 16, max_memory_mib: 16 * 1024, max_disk_mib: 100 * 1024, - max_ttl_secs: 24 * 3600, + max_ttl_secs: Some(24 * 3600), } } } @@ -46,15 +49,18 @@ impl Limits { ("CPUs", self.max_cpus), ("Memory", self.max_memory_mib), ("Disk", self.max_disk_mib), - ("Lifetime", self.max_ttl_secs), + ("Lifetime", self.max_ttl_secs.unwrap_or(0)), ]; for (name, v) in fields { if v < 0 { return Err(format!("{name} can't be below 0.")); } } - if self.max_ttl_secs as u64 > pyo_proto::sandbox::MAX_TTL_SECS { - return Err("The lifetime can be 24 hours at most.".into()); + if self + .max_ttl_secs + .is_some_and(|t| t as u64 > pyo_proto::sandbox::MAX_TTL_SECS) + { + return Err("The lifetime can be 365 days at most.".into()); } Ok(()) } @@ -102,12 +108,16 @@ pub fn check(limits: &Limits, used: &InUse, spec: &SandboxSpec) -> Result<(), St check_ttl(l, spec.ttl_secs) } -/// Whether a lifetime is within the team's longest. -pub fn check_ttl(limits: &Limits, ttl_secs: u64) -> Result<(), String> { - if ttl_secs > limits.max_ttl_secs as u64 { +/// Whether a lifetime is within the team's longest. `None` is no +/// lifetime, which only a team without a limit may have. +pub fn check_ttl(limits: &Limits, ttl_secs: Option) -> Result<(), String> { + let Some(max) = limits.max_ttl_secs.map(|m| m.max(0) as u64) else { + return Ok(()); + }; + if ttl_secs.is_none_or(|t| t > max) { return Err(format!( "Your team's sandboxes can live {} at most.", - super::format_ttl(limits.max_ttl_secs as u64) + super::format_ttl(max) )); } Ok(()) @@ -163,6 +173,10 @@ mod tests { use super::*; fn spec(cpus: u32, memory_mib: u32, ttl_secs: u64) -> SandboxSpec { + spec_for(cpus, memory_mib, Some(ttl_secs)) + } + + fn spec_for(cpus: u32, memory_mib: u32, ttl_secs: Option) -> SandboxSpec { let mut s = SandboxSpec::new("sbx_abcdefghijklmnop", "tm_one", "node-22"); s.cpus = cpus; s.memory_mib = memory_mib; @@ -185,7 +199,7 @@ mod tests { max_cpus: 4, max_memory_mib: 2048, max_disk_mib: 4096, - max_ttl_secs: 3600, + max_ttl_secs: Some(3600), }; let used = InUse { sandboxes: 1, @@ -224,7 +238,11 @@ mod tests { check(&l, &used, &spec(1, 128, 3601)).unwrap_err(), "Your team's sandboxes can live 1h at most." ); - check_ttl(&l, 3600).unwrap(); + check_ttl(&l, Some(3600)).unwrap(); + assert_eq!( + check(&l, &used, &spec_for(1, 128, None)).unwrap_err(), + "Your team's sandboxes can live 1h at most." + ); } #[test] @@ -243,10 +261,73 @@ mod tests { ..Limits::default() }; assert!(l.validate().is_err()); + let year = pyo_proto::sandbox::MAX_TTL_SECS as i32; + for ok in [Some(2 * 24 * 3600), Some(year), None] { + Limits { + max_ttl_secs: ok, + ..Limits::default() + } + .validate() + .unwrap(); + } + let l = Limits { + max_ttl_secs: Some(year + 1), + ..Limits::default() + }; + assert!(l.validate().unwrap_err().contains("365 days")); + let l = Limits { + max_ttl_secs: Some(-1), + ..Limits::default() + }; + assert!(l.validate().is_err()); + } + + #[test] + fn new_teams_keep_a_day_at_most() { + let l = Limits::default(); + assert_eq!(l.max_ttl_secs, Some(86400)); + check_ttl(&l, Some(86400)).unwrap(); + assert!(check_ttl(&l, Some(86401)).is_err()); + assert_eq!( + check_ttl(&l, None).unwrap_err(), + "Your team's sandboxes can live 1d at most." + ); + } + + #[test] + fn a_team_without_a_limit_takes_any_lifetime() { let l = Limits { - max_ttl_secs: 2 * 24 * 3600, + max_ttl_secs: None, + ..Limits::default() + }; + check_ttl(&l, None).unwrap(); + check_ttl(&l, Some(pyo_proto::sandbox::MAX_TTL_SECS)).unwrap(); + check(&l, &InUse::default(), &spec_for(1, 128, None)).unwrap(); + let year = Limits { + max_ttl_secs: Some(pyo_proto::sandbox::MAX_TTL_SECS as i32), ..Limits::default() }; - assert!(l.validate().unwrap_err().contains("24 hours")); + check_ttl(&year, Some(pyo_proto::sandbox::MAX_TTL_SECS)).unwrap(); + assert!(check_ttl(&year, None).is_err()); + } + + #[test] + fn no_limit_is_written_as_null() { + let l: Limits = serde_json::from_value(serde_json::json!({ + "maxRunning": 1, "maxCpus": 1, "maxMemoryMib": 1, "maxDiskMib": 1, "maxTtlSecs": null + })) + .unwrap(); + assert_eq!(l.max_ttl_secs, None); + assert_eq!( + serde_json::to_value(l).unwrap()["maxTtlSecs"], + serde_json::Value::Null + ); + // Left out isn't no limit: it's a mistake. + assert!( + serde_json::from_value::(serde_json::json!({ + "maxRunning": 1, "maxCpus": 1, "maxMemoryMib": 1, "maxDiskMib": 1 + })) + .is_err() + ); } } diff --git a/crates/pyo-api/src/sandbox/live.rs b/crates/pyo-api/src/sandbox/live.rs new file mode 100644 index 0000000..92886fe --- /dev/null +++ b/crates/pyo-api/src/sandbox/live.rs @@ -0,0 +1,271 @@ +//! How far a template got on the servers making it, asked of them as the +//! templates list is read. The runner reads every server's report each +//! [`super::runner::TICK`], too slow for a progress bar. While a server +//! builds a template, the list asks it with `Template`, which answers from +//! memory, and lays the answer over what the last report said. Nothing is +//! saved: the next round saves the report as before. +//! +//! - Answers are kept for [`FRESH`], so many pages open at once ask each +//! server once a second at most. +//! - A server that doesn't answer in [`WAIT`], or a daemon too old for the +//! command, keeps what its last report said, and isn't asked again for +//! [`BACKOFF`]. + +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use pyo_proto::sandbox::{DaemonCommand, DaemonResult, TemplateReport, TemplateState}; +use serde_json::{Value, json}; + +use super::{daemon::Daemon, keys}; +use crate::AppState; + +const FRESH: Duration = Duration::from_millis(900); +const WAIT: Duration = Duration::from_millis(1500); +const BACKOFF: Duration = Duration::from_secs(15); + +/// Server id and template key: when asked, and what it said. +type Cache = HashMap<(String, String), (Instant, Option)>; +static CACHE: LazyLock> = LazyLock::new(Default::default); + +/// Server ids and template keys still building in `rows`. +fn building(rows: &[Value]) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = rows + .iter() + .flat_map(|r| r["servers"].as_array().into_iter().flatten()) + .filter(|s| s["state"] == "building") + .filter_map(|s| Some((s["id"].as_str()?.to_owned(), s["key"].as_str()?.to_owned()))) + .collect(); + out.sort(); + out.dedup(); + out +} + +/// Lays what the servers say now over `rows`. +pub async fn overlay(s: &AppState, rows: &mut [Value]) { + let wanted = building(rows); + if wanted.is_empty() { + return; + } + let now = Instant::now(); + let stale: Vec<(String, String)> = { + let cache = CACHE.lock().unwrap_or_else(|p| p.into_inner()); + wanted + .iter() + .filter(|k| match cache.get(*k) { + Some((at, Some(_))) => now.duration_since(*at) >= FRESH, + Some((at, None)) => now.duration_since(*at) >= BACKOFF, + None => true, + }) + .cloned() + .collect() + }; + if !stale.is_empty() { + ask(s, stale).await; + } + let cache = CACHE.lock().unwrap_or_else(|p| p.into_inner()); + let answers: HashMap<&(String, String), &TemplateReport> = wanted + .iter() + .filter_map(|k| Some((k, cache.get(k)?.1.as_ref()?))) + .collect(); + for r in rows.iter_mut() { + for v in r["servers"].as_array_mut().into_iter().flatten() { + let k = ( + v["id"].as_str().unwrap_or_default().to_owned(), + v["key"].as_str().unwrap_or_default().to_owned(), + ); + if let Some(t) = answers.get(&k) { + merge(v, t); + } + } + roll_up(r); + } +} + +/// Asks each server how far its templates got, and keeps the answers. +async fn ask(s: &AppState, keys_: Vec<(String, String)>) { + let ids: Vec = keys_.iter().map(|(id, _)| id.clone()).collect(); + let addresses: HashMap = match sqlx::query_as::<_, (String, String)>( + "SELECT id, address FROM pyo.sandbox_servers WHERE id = ANY($1) AND address IS NOT NULL", + ) + .bind(&ids) + .fetch_all(&s.db) + .await + { + Ok(v) => v.into_iter().collect(), + Err(e) => { + tracing::debug!(error = %e, "sandbox templates: couldn't find the servers to ask"); + return; + } + }; + let Ok(key) = keys::signing_key(&s.db, &s.sealer).await else { + return; + }; + let answers: Vec<((String, String), Option)> = futures::stream::iter(keys_) + .map(|(id, name)| { + let key = key.clone(); + let address = addresses.get(&id).cloned(); + async move { + let answer = match address { + Some(a) => { + let token = keys::admin_token(&key, &id); + let cmd = DaemonCommand::Template { + template: name.clone(), + }; + match tokio::time::timeout(WAIT, Daemon::new(&a).command(&token, &cmd)) + .await + { + Ok(Ok(DaemonResult::Template { template })) => Some(template), + _ => None, + } + } + None => None, + }; + ((id, name), answer) + } + }) + .buffer_unordered(8) + .collect() + .await; + let now = Instant::now(); + let mut cache = CACHE.lock().unwrap_or_else(|p| p.into_inner()); + // Answers long past use go. + cache.retain(|_, (at, _)| now.duration_since(*at) < BACKOFF * 4); + for (k, a) in answers { + cache.insert(k, (now, a)); + } +} + +fn state_name(s: TemplateState) -> &'static str { + match s { + TemplateState::Building => "building", + TemplateState::Ready => "ready", + TemplateState::Failed => "failed", + } +} + +/// One server's view of a template, as the server says it now. +fn merge(v: &mut Value, t: &TemplateReport) { + v["state"] = json!(state_name(t.state)); + if t.state == TemplateState::Ready { + v["bytes"] = json!(t.bytes); + v["snapshots"] = json!(t.snapshots); + } + if let Some(b) = &t.build { + v["phase"] = json!(b.phase); + v["pulled"] = json!(b.pulled); + v["unpacked"] = json!(b.unpacked); + v["imageBytes"] = json!(b.image_bytes); + v["pullMs"] = json!(b.pull_ms); + v["unpackMs"] = json!(b.unpack_ms); + v["snapshotMs"] = json!(b.snapshot_ms); + v["error"] = json!(b.error); + } +} + +/// A template's state from its servers' as they are now, until the next +/// round saves it: ready once one has it, building while one makes it. +/// Pyo's own and failed ones stay as they are. +fn roll_up(r: &mut Value) { + if r["builtin"] == true || !(r["state"] == "pending" || r["state"] == "building") { + return; + } + let servers = r["servers"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default(); + let any = |st: &str| servers.iter().any(|s| s["state"] == st); + if any("ready") { + r["state"] = json!("ready"); + r["error"] = Value::Null; + } else if any("building") { + r["state"] = json!("building"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo_proto::sandbox::TemplateBuild; + + fn row(state: &str, servers: Value) -> Value { + json!({ "id": "stp_a", "builtin": false, "state": state, "error": null, "servers": servers }) + } + + #[test] + fn only_building_servers_are_asked() { + let rows = vec![ + row( + "building", + json!([ + { "id": "sbs_1", "key": "stp_a", "state": "building" }, + { "id": "sbs_2", "key": "stp_a", "state": "ready" }, + { "id": "sbs_3", "key": null, "state": "pending" }, + ]), + ), + row( + "building", + json!([{ "id": "sbs_1", "key": "stp_a", "state": "building" }]), + ), + ]; + assert_eq!(building(&rows), vec![("sbs_1".into(), "stp_a".into())]); + } + + #[test] + fn an_answer_replaces_the_last_report() { + let mut v = json!({ "id": "sbs_1", "state": "building", "phase": "pulling", "pulled": 10, "imageBytes": 100 }); + let t = TemplateReport { + name: "stp_a".into(), + state: TemplateState::Building, + build: Some(TemplateBuild { + phase: Some("unpacking".into()), + pulled: 100, + unpacked: 40, + image_bytes: 100, + ..Default::default() + }), + ..Default::default() + }; + merge(&mut v, &t); + assert_eq!(v["phase"], "unpacking"); + assert_eq!(v["unpacked"], 40); + assert_eq!(v["pulled"], 100); + + let done = TemplateReport { + state: TemplateState::Ready, + bytes: 5, + snapshots: vec!["k".into()], + build: Some(TemplateBuild::default()), + ..t + }; + merge(&mut v, &done); + assert_eq!(v["state"], "ready"); + assert_eq!(v["phase"], Value::Null); + assert_eq!(v["bytes"], 5); + } + + #[test] + fn the_template_follows_its_servers() { + let mut r = row( + "pending", + json!([{ "state": "building" }, { "state": "pending" }]), + ); + roll_up(&mut r); + assert_eq!(r["state"], "building"); + let mut r = row( + "building", + json!([{ "state": "building" }, { "state": "ready" }]), + ); + roll_up(&mut r); + assert_eq!(r["state"], "ready"); + let mut r = row("failed", json!([{ "state": "ready" }])); + roll_up(&mut r); + assert_eq!(r["state"], "failed"); + let mut r = row("ready", json!([{ "state": "building" }])); + r["builtin"] = json!(true); + roll_up(&mut r); + assert_eq!(r["state"], "ready"); + } +} diff --git a/crates/pyo-api/src/sandbox/mcp.rs b/crates/pyo-api/src/sandbox/mcp.rs index 95c7390..8943707 100644 --- a/crates/pyo-api/src/sandbox/mcp.rs +++ b/crates/pyo-api/src/sandbox/mcp.rs @@ -9,7 +9,7 @@ use super::Need; use crate::{AppState, auth::User, error::ApiError}; /// The tools, in the order they're listed. -pub const TOOLS: [&str; 9] = [ +pub const TOOLS: [&str; 10] = [ "create_sandbox", "run_code", "exec", @@ -17,6 +17,7 @@ pub const TOOLS: [&str; 9] = [ "write_file", "open_port", "close_port", + "rename_sandbox", "pause_sandbox", "delete_sandbox", ]; @@ -39,10 +40,12 @@ pub fn defs() -> Vec { vec![ json!({ "name": "create_sandbox", "title": "Create sandbox", "description": "Create an isolated machine (a sandbox) on the team's sandbox servers, to run code, commands and files in. It ends after its lifetime (5 minutes when omitted). Returns its id.", "inputSchema": schema(json!({ "team": team, - "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them). base when omitted: Debian 13 with git, Python, uv, Node.js and pnpm" }, + "name": { "type": "string", "description": "A name to tell it apart, like checkout tests. Need not be unique. A short one like quiet-river when omitted" }, + "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them). base when omitted: Ubuntu 24.04 with git, Python, uv, Node.js, pnpm, Bun, Deno, the GitHub and Docker CLIs, Claude Code, Codex and opencode. desktop adds a desktop and a browser: drive it over CDP at 127.0.0.1:9222 inside the sandbox, and people see the screen in the dashboard. A server makes base or desktop the first time it's used there, which takes a minute or so" }, + "server": { "type": "string", "description": "A sandbox server's name or id to run it on. Pyo picks one when omitted" }, "cpus": { "type": "integer", "minimum": 1, "maximum": 32 }, "memory_mib": { "type": "integer", "minimum": 128 }, - "ttl": { "type": "string", "description": "How long it lives, like 30m or 1h (24h at most)" }, + "ttl": { "type": "string", "description": "How long it lives, like 30m, 1h or 7d (365d at most, and within the team's limit). none: it runs until it's deleted, when the team allows it" }, "env": { "type": "object", "additionalProperties": { "type": "string" } }, "network": { "type": "string", "enum": ["internet", "none"] }, }), &[]), "annotations": write }), @@ -76,6 +79,10 @@ pub fn defs() -> Vec { "sandbox_id": sandbox, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, }), &["sandbox_id", "port"]), "annotations": { "readOnlyHint": false, "destructiveHint": true, "idempotentHint": true } }), + json!({ "name": "rename_sandbox", "title": "Rename sandbox", "description": "Give a sandbox another name. Its id stays the same.", "inputSchema": schema(json!({ + "sandbox_id": sandbox, + "name": { "type": "string", "description": "Up to 63 letters, digits, spaces, dashes, underscores and dots" }, + }), &["sandbox_id", "name"]), "annotations": { "readOnlyHint": false, "destructiveHint": false, "idempotentHint": true } }), json!({ "name": "pause_sandbox", "title": "Pause sandbox", "description": "Pause a sandbox: its memory and files are kept, and it resumes where it was.", "inputSchema": schema(json!({ "sandbox_id": sandbox }), &["sandbox_id"]), "annotations": { "readOnlyHint": false, "destructiveHint": false, "idempotentHint": true } }), json!({ "name": "delete_sandbox", "title": "Delete sandbox", "description": "End a sandbox and throw away its files.", "inputSchema": schema(json!({ "sandbox_id": sandbox }), &["sandbox_id"]), "annotations": { "readOnlyHint": false, "destructiveHint": true, "idempotentHint": true } }), ] @@ -151,7 +158,9 @@ pub async fn run(s: &AppState, user: &User, name: &str, args: &Value) -> Result< let team = team_id(s, user, args).await?; let mut req = json!({ "teamId": team }); for (from, to) in [ + ("name", "name"), ("template", "template"), + ("server", "server"), ("cpus", "cpus"), ("memory_mib", "memoryMib"), ("ttl", "ttl"), @@ -166,7 +175,7 @@ pub async fn run(s: &AppState, user: &User, name: &str, args: &Value) -> Result< let out = super::create(s, user, req).await.map_err(err)?; let x = &out["sandbox"]; Ok( - json!({ "id": x["id"], "template": x["template"], "state": x["state"], "runtime": x["runtime"], "expiresAt": x["expiresAt"] }), + json!({ "id": x["id"], "name": x["name"], "template": x["template"], "state": x["state"], "runtime": x["runtime"], "expiresAt": x["expiresAt"] }), ) } "run_code" => { @@ -251,6 +260,9 @@ pub async fn run(s: &AppState, user: &User, name: &str, args: &Value) -> Result< .await .map_err(err) } + "rename_sandbox" => super::rename(s, user, arg(args, "sandbox_id")?, arg(args, "name")?) + .await + .map_err(err), "pause_sandbox" => super::pause(s, user, arg(args, "sandbox_id")?) .await .map_err(err), diff --git a/crates/pyo-api/src/sandbox/mod.rs b/crates/pyo-api/src/sandbox/mod.rs index 49b0f90..4321872 100644 --- a/crates/pyo-api/src/sandbox/mod.rs +++ b/crates/pyo-api/src/sandbox/mod.rs @@ -11,6 +11,7 @@ //! - [`templates`]: which servers make which templates, and their state. //! [`base`]: Pyo's built-in template, kept in step with the release. //! - [`limits`]: per-team caps, checked on create. +//! - [`names`]: sandboxes' names, made up when left out. //! - [`daemon`]: Pyo's calls to a daemon over the tailnet. //! - [`edgecert`]: the edge proxies' wildcard certificates and DNS records. //! - [`runner`]: reads each server's report, keeps the records in step, @@ -28,10 +29,14 @@ pub mod daemon; #[cfg(test)] mod e2e; pub mod edgecert; +pub mod firstuse; +pub mod images; pub mod join; pub mod keys; pub mod limits; +mod live; pub mod mcp; +pub mod names; pub mod runner; pub mod scheduler; pub mod templates; @@ -52,10 +57,18 @@ use crate::{ ops::{Role, require_role}, }; -/// "1h", "1h30m", "90s". +/// "7d", "1h", "1h30m", "90s". pub fn format_ttl(secs: u64) -> String { - let (h, m, s) = (secs / 3600, secs % 3600 / 60, secs % 60); + let (d, h, m, s) = ( + secs / 86400, + secs % 86400 / 3600, + secs % 3600 / 60, + secs % 60, + ); let mut out = String::new(); + if d > 0 { + out.push_str(&format!("{d}d")); + } if h > 0 { out.push_str(&format!("{h}h")); } @@ -225,7 +238,7 @@ pub async fn save_status(db: &mut PgConnection, st: &SandboxStatus) -> sqlx::Res .bind(runtime_name(st.runtime)) .bind(&st.error) .bind(st.started_at.map(|t| t as f64)) - .bind(st.expires_at as f64) + .bind(st.expires_at.map(|t| t as f64)) .bind(json!(st.ports)) .execute(db) .await?; @@ -260,12 +273,30 @@ pub async fn sandbox_views( id: Option<&str>, live_only: bool, ) -> sqlx::Result> { + sandbox_views_matching(db, team_ids, id, live_only, None).await +} + +/// [`sandbox_views`], only those whose name or id has `q` in it. +pub async fn sandbox_views_matching( + db: &PgPool, + team_ids: &[String], + id: Option<&str>, + live_only: bool, + q: Option<&str>, +) -> sqlx::Result> { + let q = q + .map(str::trim) + .filter(|q| !q.is_empty()) + .map(str::to_lowercase); sqlx::query_scalar( "SELECT jsonb_build_object( - 'id', x.id, 'teamId', x.team_id, 'projectId', x.project_id, + 'id', x.id, 'name', x.name, 'teamId', x.team_id, 'projectId', x.project_id, 'serverId', x.server_id, 'server', sv.name, 'template', coalesce(t.name, x.spec->>'template'), 'templateId', x.template_id, 'state', x.state, 'runtime', x.runtime, + 'preparing', x.state = 'creating' AND sv.id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(coalesce(sv.templates, '[]')) e + WHERE e->>'name' = x.spec->>'template' AND coalesce(e->>'state', 'ready') = 'ready'), 'cpus', x.spec->'cpus', 'memoryMib', x.spec->'memoryMib', 'diskMib', x.spec->'diskMib', 'network', x.spec->'network', 'isolation', x.spec->'isolation', 'labels', coalesce(x.spec->'labels', '{}'), @@ -280,14 +311,32 @@ pub async fn sandbox_views( LEFT JOIN pyo.sandbox_templates t ON t.id = x.template_id WHERE x.team_id = ANY($1) AND ($2::text IS NULL OR x.id = $2) AND (NOT $3 OR x.state NOT IN ('stopped', 'failed')) + AND ($4::text IS NULL OR x.name IS NULL + OR strpos(lower(x.name), $4) > 0 OR strpos(lower(x.id), $4) > 0) ORDER BY x.created_at DESC LIMIT 500", ) .bind(team_ids) .bind(id) .bind(live_only) + .bind(&q) .fetch_all(db) .await .map(|mut rows: Vec| { + // A sandbox from before names has the one made from its id. + for r in &mut rows { + if r["name"].is_null() { + r["name"] = json!(names::generated(r["id"].as_str().unwrap_or_default())); + } + } + if let Some(q) = &q { + rows.retain(|r| { + names::matches( + r["name"].as_str().unwrap_or_default(), + r["id"].as_str().unwrap_or_default(), + q, + ) + }); + } // The server's tailnet address as its daemon's URL, and each open // port's URL on its edge proxy. for r in &mut rows { @@ -355,39 +404,106 @@ pub async fn server_views( Ok(rows) } +/// The templates as the last reports have them, for tests. +#[cfg(test)] pub async fn template_views( db: &PgPool, team_ids: &[String], id: Option<&str>, ) -> sqlx::Result> { - sqlx::query_scalar( + let mut rows = template_rows(db, team_ids, id).await?; + finish_template_views(&mut rows); + Ok(rows) +} + +/// The templates a team can use, with how far each server got asked of +/// the servers still making one, for pages that follow it live. +pub async fn template_views_live( + s: &AppState, + team_ids: &[String], + id: Option<&str>, +) -> sqlx::Result> { + let mut rows = template_rows(&s.db, team_ids, id).await?; + live::overlay(s, &mut rows).await; + finish_template_views(&mut rows); + Ok(rows) +} + +/// The template views as Pyo has them, each server's with the `key` the +/// server keeps the template under. +async fn template_rows( + db: &PgPool, + team_ids: &[String], + id: Option<&str>, +) -> sqlx::Result> { + let mut rows: Vec = sqlx::query_scalar( "SELECT jsonb_build_object( 'id', t.id, 'teamId', t.team_id, 'name', t.name, 'source', t.source, 'state', t.state, 'builtin', t.builtin, 'revision', t.revision, 'snapshots', t.snapshots, 'error', t.error, 'createdBy', t.created_by, 'createdAt', pyo.ms(t.created_at), 'updatedAt', pyo.ms(t.updated_at), 'servers', (SELECT coalesce(jsonb_agg(jsonb_build_object( - 'id', s.id, 'name', s.name, + 'id', s.id, 'name', s.name, 'key', m.e->>'name', 'state', CASE WHEN m.e IS NULL THEN 'pending' ELSE coalesce(m.e->>'state', 'ready') END, 'phase', m.e->'build'->'phase', 'error', m.e->'build'->'error', - 'pulled', m.e->'build'->'pulled', 'imageBytes', m.e->'build'->'imageBytes', + 'pulled', m.e->'build'->'pulled', 'unpacked', m.e->'build'->'unpacked', + 'imageBytes', m.e->'build'->'imageBytes', 'pullMs', m.e->'build'->'pullMs', 'unpackMs', m.e->'build'->'unpackMs', 'snapshotMs', m.e->'build'->'snapshotMs', 'bytes', m.e->'bytes', 'snapshots', coalesce(m.e->'snapshots', '[]')) ORDER BY s.name), '[]') FROM pyo.sandbox_servers s + -- Its current revision there, else an older one. LEFT JOIN LATERAL (SELECT x AS e FROM jsonb_array_elements(s.templates) x - WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') LIMIT 1) m ON true + WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') + OR (t.revision IS NOT NULL AND starts_with(x->>'name', t.id || '.')) + ORDER BY x->>'name' = t.id || coalesce('.' || t.revision, '') DESC LIMIT 1) m ON true WHERE (s.team_id IS NULL OR s.team_id = t.team_id OR t.team_id IS NULL) AND (s.team_id IS NULL OR s.team_id = ANY($1)) - AND (m.e IS NOT NULL OR s.state NOT IN ('joining', 'refused')))) + AND (m.e IS NOT NULL OR s.state NOT IN ('joining', 'refused')) + -- One made on first use: the servers that have it. + AND (m.e IS NOT NULL OR t.id <> ALL($3)))) FROM pyo.sandbox_templates t WHERE (t.team_id IS NULL OR t.team_id = ANY($1)) AND ($2::text IS NULL OR t.id = $2) ORDER BY t.team_id NULLS FIRST, t.name", ) .bind(team_ids) .bind(id) + .bind(base::ALL.map(|b| b.id).as_slice()) .fetch_all(db) - .await + .await?; + // Pyo's own are listed before they're added, as they will be. + for b in base::ALL { + if id.is_none_or(|i| i == b.id) && !rows.iter().any(|r| r["id"] == b.id) { + rows.push(base::listed(db, b).await); + } + } + // Pyo's own first, in their order. + let place = |r: &Value| { + base::ALL + .iter() + .position(|b| r["id"] == b.id && r["builtin"] == true) + .unwrap_or(base::ALL.len()) + }; + rows.sort_by_key(place); + Ok(rows) +} + +/// Build errors in plain words, and without the servers' `key`. +fn finish_template_views(rows: &mut [Value]) { + let plain = |e: &mut Value| { + if let Some(raw) = e.as_str() { + *e = json!(templates::plain_build_error(raw)); + } + }; + for r in rows { + plain(&mut r["error"]); + for s in r["servers"].as_array_mut().into_iter().flatten() { + plain(&mut s["error"]); + if let Some(o) = s.as_object_mut() { + o.remove("key"); + } + } + } } // ── Create ───────────────────────────────────────────────────────────────── @@ -401,22 +517,36 @@ pub struct CreateSandbox { /// (or user) has exactly one team. #[serde(default)] pub team_id: Option, + /// A name to tell it apart, like "checkout tests": up to 63 letters, + /// digits, spaces, dashes, underscores and dots. It needn't be unique + /// and never addresses the sandbox: its id does. A short one like + /// "quiet-river" when omitted. + #[serde(default)] + pub name: Option, /// A sandbox template's name (or id): the team's own, else the - /// installation's. "base" (Pyo's built-in Debian 13 template) when - /// omitted. + /// installation's. "base" (Pyo's built-in Ubuntu 24.04 template) when + /// omitted. "desktop" is Pyo's built-in desktop with a browser: CDP on + /// port 9222 and the screen (noVNC) on port 6080. A server that doesn't + /// have Pyo's template yet makes it first, so the first sandbox from it + /// there takes longer. #[serde(default)] pub template: Option, - /// 1 when omitted. + /// The sandbox server to run it on, by id or name: the team's own or + /// the installation's. Pyo picks one when omitted. + #[serde(default)] + pub server: Option, + /// 1 when omitted (2 for "desktop"). #[serde(default)] pub cpus: Option, - /// 512 when omitted. + /// 512 when omitted (2048 for "desktop"). #[serde(default)] pub memory_mib: Option, /// The writable disk, sparse. 2048 when omitted. #[serde(default)] pub disk_mib: Option, - /// How long it lives, like "30m" or "1h", or seconds. 5 minutes when - /// omitted. + /// How long it lives, like "30m", "1h" or "7d", or seconds, up to 365 + /// days and the team's longest lifetime. "none": it runs until it's + /// deleted, when the team has no limit. 5 minutes when omitted. #[serde(default)] #[schema(value_type = Option)] pub ttl: Option, @@ -443,21 +573,21 @@ pub struct CreateSandbox { pub project_id: Option, } -/// A lifetime given as "30m", "1h" or seconds. -pub fn parse_ttl_value(v: Option<&Value>) -> ApiResult { - let bad = - || ApiError::bad_request("The lifetime is like \"30m\", \"1h\" or a number of seconds."); +/// A lifetime given as "30m", "1h", "7d" or seconds, or "none": `None`, +/// the sandbox runs until it's deleted. 5 minutes when left out. +pub fn parse_ttl_value(v: Option<&Value>) -> ApiResult> { + let bad = || { + ApiError::bad_request( + "The lifetime is like \"30m\", \"1h\", \"7d\", a number of seconds or \"none\".", + ) + }; let secs = match v { - None | Some(Value::Null) => proto::DEFAULT_TTL_SECS, - Some(Value::Number(n)) => n.as_u64().ok_or_else(bad)?, - Some(Value::String(s)) => proto::parse_ttl(s).ok_or_else(bad)?, + None | Some(Value::Null) => Some(proto::DEFAULT_TTL_SECS), + Some(Value::Number(n)) => Some(n.as_u64().ok_or_else(bad)?), + Some(Value::String(s)) => proto::parse_lifetime(s).ok_or_else(bad)?, Some(_) => return Err(bad()), }; - if !(1..=proto::MAX_TTL_SECS).contains(&secs) { - return Err(ApiError::bad_request( - "The lifetime must be 1 second to 24 hours.", - )); - } + proto::check_ttl(secs).map_err(ApiError::bad_request)?; Ok(secs) } @@ -493,10 +623,8 @@ async fn template_for( .fetch_optional(tx) .await?; let Some((id, name, state, revision)) = row else { - return Err(if name == base::NAME { - ApiError::conflict( - "The base sandbox template isn't ready yet. Try again soon, or pass a template.", - ) + return Err(if let Some(b) = base::by_name(name) { + ApiError::conflict(base::unavailable(b)) } else { ApiError::not_found("Sandbox template") }); @@ -521,6 +649,8 @@ pub async fn candidates( #[derive(sqlx::FromRow)] struct Row { id: String, + name: String, + team_id: Option, address: Option, state: String, enabled: bool, @@ -531,7 +661,7 @@ pub async fn candidates( pending_mib: i64, } let rows: Vec = sqlx::query_as( - "SELECT s.id, s.address, s.state, s.enabled, s.capability, s.capacity, s.templates, + "SELECT s.id, s.name, s.team_id, s.address, s.state, s.enabled, s.capability, s.capacity, s.templates, extract(epoch FROM now() - s.last_report_at)::bigint AS age, coalesce((SELECT sum((x.spec->>'memoryMib')::bigint) FROM pyo.sandboxes x WHERE x.server_id = s.id AND x.state NOT IN ('stopped', 'failed', 'paused') @@ -547,6 +677,8 @@ pub async fn candidates( .map(|r| { let c = scheduler::Candidate { id: r.id, + name: r.name, + team_id: r.team_id, enabled: r.enabled, ready: r.state == "ready" && r.address.is_some(), report_age_secs: r.age.unwrap_or(i64::MAX), @@ -629,7 +761,19 @@ pub(crate) fn only_team(teams: &[String], field: &str) -> Result /// asks the server. Returns the sandbox and how to reach it. pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult { let ttl_secs = parse_ttl_value(req.ttl.as_ref())?; + let name = match req.name.as_deref().map(str::trim).filter(|n| !n.is_empty()) { + Some(n) => Some(names::check(n).map_err(ApiError::bad_request)?), + None => None, + }; let team_id = team_for_create(s, user, req.team_id.as_deref()).await?; + let asked = template_asked(req.template.as_deref()); + // Pyo's own are listed before they're added: added on first use. + if let Some(b) = base::by_name(asked) + && !base::added(&s.db, b).await? + && let Err(e) = base::ensure(s, b).await + { + tracing::warn!(error = %format!("{e:#}"), "sandbox template {}: couldn't add it", b.name); + } let mut tx = s.db.begin().await?; require(&mut tx, user, &team_id, Need::Use).await?; // One create at a time per team, so two can't both fit the last slot. @@ -637,7 +781,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< .bind(&team_id) .execute(&mut *tx) .await?; - let template = template_for(&mut tx, &team_id, template_asked(req.template.as_deref())).await?; + let template = template_for(&mut tx, &team_id, asked).await?; if let Some(p) = &req.project_id { let ok: bool = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM pyo.projects WHERE id = $1 AND team_id = $2)", @@ -651,8 +795,14 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< } } let mut spec = SandboxSpec::new(new_sandbox_id(), &team_id, &template.server_name); - spec.cpus = req.cpus.unwrap_or(proto::DEFAULT_CPUS); - spec.memory_mib = req.memory_mib.unwrap_or(proto::DEFAULT_MEMORY_MIB); + // Pyo's own templates may want more room by default (desktop). + let builtin = base::by_id(&template.id); + spec.cpus = req + .cpus + .unwrap_or(builtin.map_or(proto::DEFAULT_CPUS, |b| b.cpus)); + spec.memory_mib = req + .memory_mib + .unwrap_or(builtin.map_or(proto::DEFAULT_MEMORY_MIB, |b| b.memory_mib)); spec.disk_mib = req.disk_mib.unwrap_or(proto::DEFAULT_DISK_MIB); spec.ttl_secs = ttl_secs; spec.pause_when_idle_secs = req.pause_when_idle_secs; @@ -666,8 +816,21 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< limits::check(&lim, &used, &spec).map_err(ApiError::forbidden)?; let servers = candidates(&mut tx, &team_id).await?; let list: Vec = servers.iter().map(|(c, _)| c.clone()).collect(); - let (chosen, server_name) = - scheduler::pick_revision(&list, &spec, &template.id).map_err(|n| { + let on_demand = base::on_demand(&template.id); + let placed = match req + .server + .as_deref() + .map(str::trim) + .filter(|v| !v.is_empty()) + { + Some(asked) => scheduler::place_on(&list, &team_id, asked, &spec, &template.id, on_demand) + .map_err(|(status, message)| { + ApiError::new( + status, + message.replace(&template.server_name, &template.name), + ) + })?, + None => scheduler::place(&list, &spec, &template.id, on_demand).map_err(|n| { let n = match n { scheduler::NoServer::NoTemplate(_) => { scheduler::NoServer::NoTemplate(template.name.clone()) @@ -675,23 +838,27 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< n => n, }; ApiError::new(n.status(), n.message()) - })?; + })?, + }; // The server may still start it from an older revision of the template. - spec.template = server_name; + spec.template = placed.template.clone(); + let build = placed.build; + let (server_id, server_label) = (placed.server.id.clone(), placed.server.name.clone()); let address = servers .iter() - .find(|(c, _)| c.id == chosen.id) + .find(|(c, _)| c.id == server_id) .map(|(_, a)| a.clone()) .unwrap_or_default(); - let server_id = chosen.id.clone(); // Kept without env values: they may be secrets. let mut kept = spec.clone(); for v in kept.env.values_mut() { v.clear(); } + // While its server makes the template, its lifetime hasn't started. + let wait = if build { firstuse::LIMIT.as_secs() } else { 0 }; sqlx::query( - "INSERT INTO pyo.sandboxes (id, team_id, project_id, server_id, template_id, spec, state, created_by, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, 'creating', $7, now() + make_interval(secs => $8))", + "INSERT INTO pyo.sandboxes (id, team_id, project_id, server_id, template_id, spec, state, created_by, expires_at, name) + VALUES ($1, $2, $3, $4, $5, $6, 'creating', $7, now() + make_interval(secs => $8), $9)", ) .bind(&spec.id) .bind(&team_id) @@ -700,12 +867,42 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< .bind(&template.id) .bind(json!(kept)) .bind(&user.name) - .bind(ttl_secs as f64) + .bind(ttl_secs.map(|t| (t + wait) as f64)) + .bind(name.unwrap_or_else(|| names::generated(&spec.id))) .execute(&mut *tx) .await?; tx.commit().await?; let key = keys::signing_key(&s.db, &s.sealer).await?; + if build { + let ready = firstuse::on_server( + &s.db, + &key, + &server_id, + &address, + &template.id, + &spec.template, + ) + .await; + if let Err(e) = ready { + let e = e.error(&template.name, &server_label); + let mut c = s.db.acquire().await?; + end(&mut c, &spec.id, "failed", Some(&e.message)).await?; + return Err(e); + } + // Deleted while it waited. + let state: Option = + sqlx::query_scalar("SELECT state FROM pyo.sandboxes WHERE id = $1") + .bind(&spec.id) + .fetch_optional(&s.db) + .await?; + if state.as_deref() != Some("creating") { + return Err(ApiError::conflict(format!( + "The sandbox {} was deleted before it started.", + spec.id + ))); + } + } let token = keys::create_token(&key, &spec.id, &team_id, &server_id); let d = daemon::Daemon::new(&address); let st = match d @@ -863,6 +1060,19 @@ pub async fn delete(s: &AppState, user: &User, id: &str) -> ApiResult { view(s, &f).await } +/// Gives a sandbox another name. Its id stays, so nothing that uses it +/// breaks. Also once it has ended. +pub async fn rename(s: &AppState, user: &User, id: &str, name: &str) -> ApiResult { + let name = names::check(name).map_err(ApiError::bad_request)?; + let f = find(s, user, id, Need::Use).await?; + sqlx::query("UPDATE pyo.sandboxes SET name = $2 WHERE id = $1") + .bind(id) + .bind(&name) + .execute(&s.db) + .await?; + view(s, &f).await +} + pub async fn pause(s: &AppState, user: &User, id: &str) -> ApiResult { lifecycle(s, user, id, |id| DaemonCommand::Pause { id }).await } @@ -871,7 +1081,8 @@ pub async fn resume(s: &AppState, user: &User, id: &str) -> ApiResult { lifecycle(s, user, id, |id| DaemonCommand::Resume { id }).await } -/// The sandbox ends `ttl` from now instead, within the team's longest. +/// The sandbox ends `ttl` from now instead, or never ("none"), within the +/// team's longest. pub async fn extend(s: &AppState, user: &User, id: &str, ttl: Option<&Value>) -> ApiResult { let ttl_secs = parse_ttl_value(ttl)?; let f = find(s, user, id, Need::Use).await?; @@ -1429,10 +1640,11 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) -> "A template name is lowercase letters, digits, dashes, dots and underscores, like node-22.", )); } - if req.name == base::NAME { - return Err(ApiError::conflict( - "base is Pyo's built-in sandbox template. Pick another name.", - )); + if let Some(b) = base::by_name(&req.name) { + return Err(ApiError::conflict(format!( + "{} is Pyo's built-in sandbox template. Pick another name.", + b.name + ))); } let image = req.image.trim(); if image.is_empty() || image.len() > 512 || image.contains(char::is_whitespace) { @@ -1461,7 +1673,7 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) -> tx.commit().await?; tokio::spawn(templates::push(s.clone(), id.clone())); let teams: Vec = req.team_id.iter().cloned().collect(); - Ok(template_views(&s.db, &teams, Some(&id)) + Ok(template_views_live(s, &teams, Some(&id)) .await? .into_iter() .next() @@ -1517,10 +1729,29 @@ mod tests { assert_eq!(format_ttl(5400), "1h30m"); assert_eq!(format_ttl(90), "1m30s"); assert_eq!(format_ttl(0), "0s"); - assert_eq!(parse_ttl_value(None).unwrap(), proto::DEFAULT_TTL_SECS); - assert_eq!(parse_ttl_value(Some(&json!("1h"))).unwrap(), 3600); - assert_eq!(parse_ttl_value(Some(&json!(90))).unwrap(), 90); - assert!(parse_ttl_value(Some(&json!("2d"))).is_err()); + assert_eq!(format_ttl(86400), "1d"); + assert_eq!(format_ttl(36 * 3600), "1d12h"); + assert_eq!(format_ttl(proto::MAX_TTL_SECS), "365d"); + assert_eq!( + parse_ttl_value(None).unwrap(), + Some(proto::DEFAULT_TTL_SECS) + ); + assert_eq!(parse_ttl_value(Some(&json!("1h"))).unwrap(), Some(3600)); + assert_eq!(parse_ttl_value(Some(&json!(90))).unwrap(), Some(90)); + assert_eq!( + parse_ttl_value(Some(&json!("2d"))).unwrap(), + Some(2 * 86400) + ); + assert_eq!( + parse_ttl_value(Some(&json!("365d"))).unwrap(), + Some(proto::MAX_TTL_SECS) + ); + assert_eq!( + parse_ttl_value(Some(&json!(proto::MAX_TTL_SECS))).unwrap(), + Some(proto::MAX_TTL_SECS) + ); + assert_eq!(parse_ttl_value(Some(&json!("none"))).unwrap(), None); + assert!(parse_ttl_value(Some(&json!("366d"))).is_err()); assert!(parse_ttl_value(Some(&json!(0))).is_err()); assert!(parse_ttl_value(Some(&json!("soon"))).is_err()); assert!(parse_ttl_value(Some(&json!(true))).is_err()); @@ -1608,6 +1839,9 @@ mod tests { .unwrap(); assert_eq!(r.network, Some(NetworkMode::None)); assert_eq!(r.isolation, Some(Isolation::Vm)); + assert_eq!(r.name, None); + let r: CreateSandbox = serde_json::from_value(json!({ "name": "checkout tests" })).unwrap(); + assert_eq!(r.name.as_deref(), Some("checkout tests")); assert!( serde_json::from_value::( json!({ "teamId": "t", "template": "x", "nope": 1 }) @@ -1616,6 +1850,123 @@ mod tests { ); } + #[tokio::test] + #[ignore = "needs DATABASE_URL (a throwaway Postgres)"] + async fn a_rename_keeps_the_id_and_search_finds_names() { + let db = PgPool::connect(&std::env::var("DATABASE_URL").unwrap()) + .await + .unwrap(); + crate::migrate::run(&db).await.unwrap(); + let s = e2e::test_state(db.clone()); + let run = crate::ids::random_string(8); + let team = format!("tm_names{run}"); + let user = User { + id: format!("usr_names{run}"), + name: "Names".into(), + email: format!("{run}@names.test"), + timezone: "UTC".into(), + is_admin: false, + must_set_up: false, + key: None, + }; + sqlx::query( + "INSERT INTO pyo.users (id, email, name, password_hash) VALUES ($1, $2, 'Names', '')", + ) + .bind(&user.id) + .bind(&user.email) + .execute(&db) + .await + .unwrap(); + sqlx::query("INSERT INTO pyo.teams (id, name, slug) VALUES ($1, 'Names', $1)") + .bind(&team) + .execute(&db) + .await + .unwrap(); + sqlx::query("INSERT INTO pyo.members (id, team_id, user_id, email, role, status) VALUES ($1, $2, $3, $4, 'developer', 'active')") + .bind(format!("mem_names{run}")) + .bind(&team) + .bind(&user.id) + .bind(&user.email) + .execute(&db) + .await + .unwrap(); + // One from before names (NULL), one named. + let (old, named) = (format!("sbx_old{run}"), format!("sbx_named{run}")); + for (id, name) in [(&old, None), (&named, Some("Checkout tests"))] { + sqlx::query("INSERT INTO pyo.sandboxes (id, team_id, spec, state, created_by, expires_at, ended_at, name) VALUES ($1, $2, '{}', 'stopped', 'test', now(), now(), $3)") + .bind(id) + .bind(&team) + .bind(name) + .execute(&db) + .await + .unwrap(); + } + let teams = [team.clone()]; + let all = sandbox_views(&db, &teams, None, false).await.unwrap(); + let name_of = |id: &str| { + all.iter().find(|x| x["id"] == id).unwrap()["name"] + .as_str() + .unwrap() + .to_owned() + }; + assert_eq!(name_of(&old), names::generated(&old)); + assert_eq!(name_of(&named), "Checkout tests"); + + let found = |q: &str| { + let db = db.clone(); + let teams = teams.clone(); + let q = q.to_owned(); + async move { + sandbox_views_matching(&db, &teams, None, false, Some(&q)) + .await + .unwrap() + .into_iter() + .map(|x| x["id"].as_str().unwrap().to_owned()) + .collect::>() + } + }; + assert_eq!(found("CHECKOUT").await, std::slice::from_ref(&named)); + assert_eq!( + found(&names::generated(&old)).await, + std::slice::from_ref(&old), + "a made-up name is found too" + ); + assert_eq!( + found(&format!("old{run}")).await, + std::slice::from_ref(&old), + "and the id" + ); + assert!(found("nothing like it").await.is_empty()); + + // A rename changes the name and keeps the id, also once it ended. + let v = rename(&s, &user, &old, " nightly build ").await.unwrap(); + assert_eq!( + (v["id"].as_str(), v["name"].as_str()), + (Some(old.as_str()), Some("nightly build")) + ); + assert_eq!(found("nightly").await, std::slice::from_ref(&old)); + let e = rename(&s, &user, &old, "a/b").await.unwrap_err(); + assert_eq!(e.status, StatusCode::BAD_REQUEST); + let e = rename(&s, &user, &old, " ").await.unwrap_err(); + assert_eq!(e.status, StatusCode::BAD_REQUEST); + let e = rename(&s, &user, "sbx_nope", "x").await.unwrap_err(); + assert_eq!(e.status, StatusCode::NOT_FOUND); + + // Viewers can't. + sqlx::query("UPDATE pyo.members SET role = 'viewer' WHERE team_id = $1") + .bind(&team) + .execute(&db) + .await + .unwrap(); + let e = rename(&s, &user, &old, "mine").await.unwrap_err(); + assert_eq!(e.status, StatusCode::FORBIDDEN); + sqlx::query("DELETE FROM pyo.teams WHERE id = $1") + .bind(&team) + .execute(&db) + .await + .unwrap(); + } + #[test] fn create_without_a_template_uses_base() { let r: CreateSandbox = serde_json::from_value(json!({})).unwrap(); diff --git a/crates/pyo-api/src/sandbox/names.rs b/crates/pyo-api/src/sandbox/names.rs new file mode 100644 index 0000000..aa41228 --- /dev/null +++ b/crates/pyo-api/src/sandbox/names.rs @@ -0,0 +1,127 @@ +//! Sandbox names: for people to tell sandboxes apart and find them. They +//! aren't unique and never address a sandbox: the id does, so a rename +//! breaks nothing. A sandbox made without one gets a short name like +//! `quiet-river`, picked from its id, so a sandbox from before names (whose +//! name is NULL) shows the same name every time. + +const ADJECTIVES: [&str; 64] = [ + "amber", "bold", "brave", "bright", "brisk", "calm", "clear", "clever", "cosy", "crisp", + "curious", "daring", "eager", "early", "fair", "fancy", "fast", "fresh", "friendly", "gentle", + "glad", "golden", "good", "grand", "green", "happy", "honest", "jolly", "keen", "kind", + "lively", "lucky", "merry", "mighty", "misty", "neat", "nimble", "noble", "patient", "plain", + "polite", "proud", "quick", "quiet", "rapid", "ready", "rosy", "shiny", "silent", "silver", + "simple", "sleepy", "smooth", "snowy", "steady", "sunny", "swift", "tidy", "tiny", "warm", + "wild", "wise", "witty", "young", +]; + +const NOUNS: [&str; 64] = [ + "acorn", "badger", "beach", "bear", "birch", "bird", "breeze", "brook", "canyon", "cedar", + "cloud", "comet", "coral", "creek", "dawn", "deer", "dune", "eagle", "falcon", "fern", "field", + "finch", "forest", "fox", "garden", "glade", "harbor", "hawk", "hill", "island", "lake", + "leaf", "maple", "meadow", "moon", "moss", "mountain", "oak", "ocean", "otter", "owl", "panda", + "pebble", "pine", "planet", "pond", "rain", "raven", "reef", "river", "robin", "sea", "shore", + "sky", "spark", "star", "stone", "storm", "sun", "tiger", "valley", "wave", "willow", "wolf", +]; + +/// The longest name, in characters. +pub const MAX_LEN: usize = 63; + +/// The name a sandbox gets when it's made without one, from its id. The +/// same id always gives the same name. +pub fn generated(id: &str) -> String { + // FNV-1a: stable across releases, unlike std's hasher. + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in id.bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + // Then mixed (splitmix64's finish), so ids a character apart spread too. + h ^= h >> 30; + h = h.wrapping_mul(0xbf58_476d_1ce4_e5b9); + h ^= h >> 27; + h = h.wrapping_mul(0x94d0_49bb_1331_11eb); + h ^= h >> 31; + let adjective = ADJECTIVES[(h % 64) as usize]; + let noun = NOUNS[((h >> 32) % 64) as usize]; + format!("{adjective}-{noun}") +} + +/// A name someone gave, trimmed, or why it can't be one. +pub fn check(name: &str) -> Result { + let name = name.trim(); + if name.is_empty() { + return Err("Give the sandbox a name.".into()); + } + if name.chars().count() > MAX_LEN { + return Err(format!("A sandbox name is {MAX_LEN} characters at most.")); + } + if !name + .chars() + .all(|c| c.is_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.')) + { + return Err( + "A sandbox name can have letters, digits, spaces, dashes, underscores and dots.".into(), + ); + } + Ok(name.to_owned()) +} + +/// Whether a sandbox's name or id has `q` in it, ignoring case. +pub fn matches(name: &str, id: &str, q: &str) -> bool { + let q = q.to_lowercase(); + name.to_lowercase().contains(&q) || id.to_lowercase().contains(&q) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_names_are_short_plain_and_stable() { + let a = generated("sbx_0123456789abcdef"); + assert_eq!( + a, + generated("sbx_0123456789abcdef"), + "the same id, the same name" + ); + let (adj, noun) = a.split_once('-').unwrap(); + assert!(ADJECTIVES.contains(&adj) && NOUNS.contains(&noun), "{a}"); + assert!(check(&a).is_ok()); + // Different ids spread over the words. + let names: std::collections::HashSet = + (0..200).map(|i| generated(&format!("sbx_{i}"))).collect(); + assert!(names.len() > 150, "{} names", names.len()); + } + + #[test] + fn word_lists_are_plain() { + for w in ADJECTIVES.iter().chain(NOUNS.iter()) { + assert!(w.chars().all(|c| c.is_ascii_lowercase()), "{w}"); + } + let a: std::collections::HashSet<_> = ADJECTIVES.iter().collect(); + let n: std::collections::HashSet<_> = NOUNS.iter().collect(); + assert_eq!((a.len(), n.len()), (64, 64), "no word twice"); + } + + #[test] + fn names_people_give() { + assert_eq!(check(" my sandbox ").unwrap(), "my sandbox"); + assert_eq!(check("job-42_v1.2").unwrap(), "job-42_v1.2"); + assert_eq!(check("café").unwrap(), "café"); + assert_eq!(check(&"a".repeat(63)).unwrap().len(), 63); + assert!(check(&"a".repeat(64)).is_err()); + assert!(check("").is_err()); + assert!(check(" ").is_err()); + for bad in ["a/b", "a;b", "", "tab\there", "line\nbreak", "a@b"] { + let e = check(bad).unwrap_err(); + assert!(!e.contains(';'), "{e}"); + } + } + + #[test] + fn search_takes_the_name_or_the_id() { + assert!(matches("Quiet River", "sbx_abc", "river")); + assert!(matches("quiet-river", "sbx_ABC", "abc")); + assert!(!matches("quiet-river", "sbx_abc", "lake")); + } +} diff --git a/crates/pyo-api/src/sandbox/runner.rs b/crates/pyo-api/src/sandbox/runner.rs index fe6538f..1383012 100644 --- a/crates/pyo-api/src/sandbox/runner.rs +++ b/crates/pyo-api/src/sandbox/runner.rs @@ -21,8 +21,9 @@ pub const TICK: Duration = Duration::from_secs(15); pub const UNREACHABLE_SECS: i64 = 60; /// A sandbox past its lifetime by this much is ended by Pyo. const GRACE_SECS: i64 = 30; -/// A sandbox still creating after this long failed. -const CREATE_TIMEOUT_SECS: i64 = 120; +/// A sandbox still creating after this long failed. Its create may first +/// wait for its server to make `base` ([`super::firstuse::LIMIT`]). +const CREATE_TIMEOUT_SECS: i64 = super::firstuse::LIMIT.as_secs() as i64 + 120; pub async fn runner(s: AppState) { let mut tick = tokio::time::interval(TICK); @@ -67,9 +68,9 @@ struct Server { /// One round: reports, then lifetimes. pub async fn round(s: &AppState) -> anyhow::Result<()> { let _ = super::join::prune(&s.db).await; - // A dev build's `base` comes once an update check finds a release. - if let Err(e) = super::base::ensure(s).await { - tracing::warn!(error = %format!("{e:#}"), "sandboxes: couldn't keep the base template"); + // Pyo's own templates, at the images it has now. + if let Err(e) = super::base::ensure_all(s).await { + tracing::warn!(error = %format!("{e:#}"), "sandboxes: couldn't keep Pyo's templates"); } let servers: Vec = sqlx::query_as( "SELECT id, team_id, address, state FROM pyo.sandbox_servers WHERE address IS NOT NULL AND state <> 'refused'", @@ -261,7 +262,7 @@ async fn sweep(s: &AppState) -> anyhow::Result<()> { end(&mut c, &id, "failed", Some("It didn't start in time.")).await?; } let due: Vec<(String, Option, Option, bool)> = sqlx::query_as( - "SELECT x.id, x.server_id, sv.address, x.expires_at < now() - make_interval(secs => $1) FROM pyo.sandboxes x LEFT JOIN pyo.sandbox_servers sv ON sv.id = x.server_id + "SELECT x.id, x.server_id, sv.address, coalesce(x.expires_at < now() - make_interval(secs => $1), false) FROM pyo.sandboxes x LEFT JOIN pyo.sandbox_servers sv ON sv.id = x.server_id WHERE x.state NOT IN ('stopped', 'failed') AND (x.state = 'stopping' OR x.expires_at < now() - make_interval(secs => $1)) LIMIT 200", diff --git a/crates/pyo-api/src/sandbox/scheduler.rs b/crates/pyo-api/src/sandbox/scheduler.rs index 605648b..15cc1a7 100644 --- a/crates/pyo-api/src/sandbox/scheduler.rs +++ b/crates/pyo-api/src/sandbox/scheduler.rs @@ -10,6 +10,10 @@ //! 2. a snapshot of the template at this size, so it restores in //! milliseconds instead of booting; //! 3. the most headroom left. +//! +//! A template made on first use (Pyo's `base`) may also go to a server +//! that doesn't have it yet, which makes it first ([`place`]). One already +//! making it comes first, so two creates share one build. use pyo_proto::sandbox::{ CapabilityReport, CapacityReport, Isolation, NetworkMode, Runtime, SandboxSpec, TemplateReport, @@ -23,6 +27,9 @@ pub const FRESH_SECS: i64 = 90; #[derive(Debug, Clone)] pub struct Candidate { pub id: String, + pub name: String, + /// None for the installation's, which every team may use. + pub team_id: Option, pub enabled: bool, pub ready: bool, /// Seconds since its last report. @@ -77,6 +84,29 @@ impl NoServer { } } + /// Why the one server asked for (`server`, by name) can't take a + /// sandbox of `memory_mib`. + pub fn message_on(&self, server: &str, memory_mib: u32) -> String { + match self { + NoServer::None | NoServer::NoneUp => { + format!("{server} can't take sandboxes right now. Pick another server.") + } + NoServer::NoVm => { + format!("{server} has no KVM, so isolation \"vm\" can't run there.") + } + NoServer::NoNetwork => { + format!("{server} has no networking. Pick another server, or set network to none.") + } + NoServer::NoTemplate(t) => { + format!("{server} doesn't have the template {t} yet. Pick another server.") + } + NoServer::Full => format!( + "{server} has no room for {}. Pick another server or a smaller size.", + size(memory_mib) + ), + } + } + /// 503 for what passes on its own, 409 for what needs a change. pub fn status(&self) -> axum::http::StatusCode { match self { @@ -86,9 +116,21 @@ impl NoServer { } } -/// The best server for `spec`, whose template servers keep as -/// `spec.template`. -pub fn pick<'a>(servers: &'a [Candidate], spec: &SandboxSpec) -> Result<&'a Candidate, NoServer> { +/// "512 MB", "2 GB", "1.5 GB". +fn size(mib: u32) -> String { + if mib >= 1024 { + let gb = (mib as f64 / 1024.0 * 10.0).round() / 10.0; + format!("{gb} GB") + } else { + format!("{mib} MB") + } +} + +/// The servers up and able to run `spec`, whatever their templates. +fn usable<'a>( + servers: &'a [Candidate], + spec: &SandboxSpec, +) -> Result, NoServer> { if servers.is_empty() { return Err(NoServer::None); } @@ -122,7 +164,13 @@ pub fn pick<'a>(servers: &'a [Candidate], spec: &SandboxSpec) -> Result<&'a Cand if networked.is_empty() { return Err(NoServer::NoNetwork); } - let with_template: Vec<&Candidate> = networked + Ok(networked) +} + +/// The best server for `spec`, whose template servers keep as +/// `spec.template`. +pub fn pick<'a>(servers: &'a [Candidate], spec: &SandboxSpec) -> Result<&'a Candidate, NoServer> { + let with_template: Vec<&Candidate> = usable(servers, spec)? .into_iter() .filter(|c| { c.templates @@ -191,6 +239,89 @@ pub fn pick_revision<'a>( Err(why) } +/// Where a new sandbox goes, and what it starts from there. +#[derive(Debug, Clone)] +pub struct Placement<'a> { + pub server: &'a Candidate, + /// The name the server keeps the template under. + pub template: String, + /// The server makes the template first: it doesn't have it ready. + pub build: bool, +} + +/// [`pick_revision`], and for a template made on first use (`on_demand`) +/// that no server has ready: a server with room, which makes it first. +/// One already making it comes first, then Firecracker, then the most +/// headroom. +pub fn place<'a>( + servers: &'a [Candidate], + spec: &SandboxSpec, + id: &str, + on_demand: bool, +) -> Result, NoServer> { + let why = match pick_revision(servers, spec, id) { + Ok((server, template)) => { + return Ok(Placement { + server, + template, + build: false, + }); + } + Err(e) => e, + }; + if !on_demand || !matches!(why, NoServer::NoTemplate(_)) { + return Err(why); + } + let need = (spec.memory_mib as u64) << 20; + let building = |c: &Candidate| { + c.templates + .iter() + .any(|t| t.name == spec.template && t.state == TemplateState::Building) + }; + usable(servers, spec)? + .into_iter() + .filter(|c| c.headroom() >= need) + .max_by_key(|c| { + ( + building(c), + c.capability.runtime == Runtime::Firecracker, + c.headroom(), + std::cmp::Reverse(c.id.clone()), + ) + }) + .map(|server| Placement { + server, + template: spec.template.clone(), + build: true, + }) + .ok_or(NoServer::Full) +} + +/// [`place`] on one server the create asked for, by its id or name: one of +/// `team`'s or the installation's. The error says why it can't, plainly. +pub fn place_on<'a>( + servers: &'a [Candidate], + team: &str, + asked: &str, + spec: &SandboxSpec, + id: &str, + on_demand: bool, +) -> Result, (axum::http::StatusCode, String)> { + let asked = asked.trim(); + let server = servers + .iter() + .filter(|c| c.team_id.as_deref().is_none_or(|t| t == team)) + .find(|c| c.id == asked || c.name == asked) + .ok_or_else(|| { + ( + axum::http::StatusCode::NOT_FOUND, + format!("No sandbox server {asked} in this team."), + ) + })?; + place(std::slice::from_ref(server), spec, id, on_demand) + .map_err(|n| (n.status(), n.message_on(&server.name, spec.memory_mib))) +} + #[cfg(test)] mod tests { use super::*; @@ -201,6 +332,8 @@ mod tests { fn server(id: &str, runtime: Runtime, free_gib: u64) -> Candidate { Candidate { id: id.into(), + name: format!("{id}-name"), + team_id: None, enabled: true, ready: true, report_age_secs: 5, @@ -376,6 +509,167 @@ mod tests { ); } + #[test] + fn base_goes_to_a_server_without_it_which_makes_it_first() { + let mut s = spec(); + s.template = "stp_base.new".into(); + let servers = vec![ + server("small", Runtime::Firecracker, 2), + server("big", Runtime::Firecracker, 8), + server("gv", Runtime::Gvisor, 30), + ]; + // A team's template isn't made on first use. + assert_eq!( + place(&servers, &s, "stp_base", false).map(|p| p.server.id.clone()), + Err(NoServer::NoTemplate("stp_base.new".into())) + ); + let p = place(&servers, &s, "stp_base", true).unwrap(); + assert_eq!( + (p.server.id.as_str(), p.template.as_str(), p.build), + ("big", "stp_base.new", true), + "Firecracker, then room" + ); + // A server that has it ready wins, with no build. + let mut has = server("has", Runtime::Firecracker, 2); + has.templates[0].name = "stp_base.new".into(); + let mut with = servers.clone(); + with.push(has); + let p = place(&with, &s, "stp_base", true).unwrap(); + assert_eq!((p.server.id.as_str(), p.build), ("has", false)); + // So does one with an older revision, while the new one is made. + let mut old = server("old", Runtime::Firecracker, 2); + old.templates[0].name = "stp_base.old".into(); + let mut with = servers.clone(); + with.push(old); + let p = place(&with, &s, "stp_base", true).unwrap(); + assert_eq!( + (p.server.id.as_str(), p.template.as_str(), p.build), + ("old", "stp_base.old", false) + ); + // No room anywhere. + let mut full = servers.clone(); + for c in &mut full { + c.capacity.memory_available = 0; + } + assert_eq!( + place(&full, &s, "stp_base", true).map(|p| p.server.id.clone()), + Err(NoServer::Full) + ); + // The usual reasons still count. + let mut vm = s.clone(); + vm.isolation = Isolation::Vm; + assert_eq!( + place(&[server("gv", Runtime::Gvisor, 30)], &vm, "stp_base", true) + .map(|p| p.server.id.clone()), + Err(NoServer::NoVm) + ); + assert_eq!( + place(&[], &s, "stp_base", true).map(|p| p.server.id.clone()), + Err(NoServer::None) + ); + } + + #[test] + fn two_creates_share_one_build() { + let mut s = spec(); + s.template = "stp_base.new".into(); + // The first create set "small" making it. The second goes there + // too, though "big" has more room. + let mut small = server("small", Runtime::Firecracker, 3); + small.templates[0] = TemplateReport { + name: "stp_base.new".into(), + state: TemplateState::Building, + ..Default::default() + }; + small.pending_bytes = 1 << 30; + let mut servers = vec![small, server("big", Runtime::Firecracker, 8)]; + let p = place(&servers, &s, "stp_base", true).unwrap(); + assert_eq!((p.server.id.as_str(), p.build), ("small", true)); + // Unless it has no room left: then another server makes it. + servers[0].pending_bytes = 3 << 30; + let p = place(&servers, &s, "stp_base", true).unwrap(); + assert_eq!((p.server.id.as_str(), p.build), ("big", true)); + // A failed build is tried again. + let mut failed = server("failed", Runtime::Firecracker, 8); + failed.templates[0] = TemplateReport { + name: "stp_base.new".into(), + state: TemplateState::Failed, + ..Default::default() + }; + let p = place(std::slice::from_ref(&failed), &s, "stp_base", true).unwrap(); + assert_eq!((p.server.id.as_str(), p.build), ("failed", true)); + } + + #[test] + fn a_server_asked_for_takes_it_or_says_why() { + use axum::http::StatusCode; + let mut s = spec(); + let mut other_team = server("theirs", Runtime::Firecracker, 30); + other_team.team_id = Some("tm_two".into()); + let mut ours = server("ours", Runtime::Firecracker, 2); + ours.team_id = Some("tm_one".into()); + let mut joining = server("joining", Runtime::Firecracker, 30); + joining.ready = false; + let installation = server("shared", Runtime::Gvisor, 30); + let servers = vec![other_team, ours, joining, installation]; + let on = |asked: &str, s: &SandboxSpec, on_demand: bool| { + place_on(&servers, "tm_one", asked, s, "stp_node", on_demand) + .map(|p| (p.server.id.clone(), p.build)) + }; + // By id or name, the team's own or the installation's, even when + // another has more room. + assert_eq!(on("ours", &s, false), Ok(("ours".into(), false))); + assert_eq!(on("ours-name", &s, false), Ok(("ours".into(), false))); + assert_eq!(on(" shared ", &s, false), Ok(("shared".into(), false))); + // Another team's is as good as missing. + assert_eq!( + on("theirs", &s, false), + Err(( + StatusCode::NOT_FOUND, + "No sandbox server theirs in this team.".into() + )) + ); + // Not ready. + assert_eq!( + on("joining-name", &s, false), + Err(( + StatusCode::SERVICE_UNAVAILABLE, + "joining-name can't take sandboxes right now. Pick another server.".into() + )) + ); + // No room. + s.memory_mib = 3072; + assert_eq!( + on("ours", &s, false), + Err(( + StatusCode::SERVICE_UNAVAILABLE, + "ours-name has no room for 3 GB. Pick another server or a smaller size.".into() + )) + ); + s.memory_mib = 1024; + // No KVM for "vm". + let mut vm = s.clone(); + vm.isolation = Isolation::Vm; + assert_eq!( + on("shared", &vm, false).unwrap_err().1, + "shared-name has no KVM, so isolation \"vm\" can't run there." + ); + // A server without base makes it first. A team template it lacks + // is refused. + let mut base = s.clone(); + base.template = "stp_base.new".into(); + let p = place_on(&servers, "tm_one", "ours", &base, "stp_base", true).unwrap(); + assert_eq!((p.server.id.as_str(), p.build), ("ours", true)); + assert_eq!( + place_on(&servers, "tm_one", "ours", &base, "stp_base", false) + .unwrap_err() + .1, + "ours-name doesn't have the template stp_base.new yet. Pick another server." + ); + assert_eq!(size(512), "512 MB"); + assert_eq!(size(1536), "1.5 GB"); + } + #[test] fn internet_needs_a_server_with_networking() { let mut off = server("off", Runtime::Firecracker, 8); diff --git a/crates/pyo-api/src/sandbox/templates.rs b/crates/pyo-api/src/sandbox/templates.rs index 561f4d1..d4d7566 100644 --- a/crates/pyo-api/src/sandbox/templates.rs +++ b/crates/pyo-api/src/sandbox/templates.rs @@ -5,7 +5,8 @@ //! //! - On create, and in every runner round for a server that doesn't have //! one yet (a new server, or one that was down), Pyo sends -//! `PrepareTemplate`. +//! `PrepareTemplate`. Pyo's `base` is the exception: a server makes it +//! the first time a sandbox needs it there ([`super::firstuse`]). //! - An image in Pyo's own registry is pulled over the tailnet from the //! registry door, with a registry token for the template's team that //! lasts an hour. Any other image is pulled from its registry as is. @@ -30,7 +31,8 @@ use std::collections::HashSet; use pyo_proto::sandbox::token::SigningKey; use pyo_proto::sandbox::{ - DaemonCommand, ErrorCode, RegistryAuth, Runtime, ServerReport, TemplateReport, TemplateState, + DaemonCommand, DaemonResult, ErrorCode, RegistryAuth, Runtime, ServerReport, TemplateReport, + TemplateState, }; use serde_json::Value; use sqlx::PgPool; @@ -96,16 +98,62 @@ pub fn eligible(server_team: Option<&str>, template_team: Option<&str>) -> bool } /// The repository path of an image in Pyo's registry, by any of the -/// addresses it's known by. None for any other registry. -pub fn in_pyos_registry<'a>(image: &'a str, doors: &[String]) -> Option<&'a str> { - doors +/// addresses it's known by: its doors, and its names on Pyo's own machine +/// (`127.0.0.1:5005`, `localhost:5005`, in the cluster). None for any other +/// registry. +pub fn in_pyos_registry(image: &str, doors: &[String]) -> Option { + if let Some(path) = doors .iter() .filter(|d| !d.is_empty()) .find_map(|d| image.strip_prefix(d.as_str())?.strip_prefix('/')) + { + return Some(path.to_owned()); + } + let pulled = crate::deploy::pyo_registry_image(image)?.ok()?; + pulled + .strip_prefix(crate::deploy::registry().pull.as_str())? + .strip_prefix('/') + .map(str::to_owned) +} + +/// Whether a server makes template `id` before a sandbox needs it: every +/// template but one made on first use ([`super::base`]), which only once +/// the server has some revision of it. +pub fn made_ahead(id: &str, reports: &[TemplateReport]) -> bool { + !super::base::on_demand(id) || reports.iter().any(|r| is_revision_of(&r.name, id)) } -/// The command that has a server prepare `t`. -pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand { +/// What a server answers a failed copy of one of Pyo's own templates +/// with: it's deleted after this long, and made again the next round. +pub const RETRY_FAILED_MS: u64 = 60_000; + +/// Failed copies of Pyo's own templates' current revisions that a server +/// should delete now, so it makes them again. Sandboxes keep starting +/// from an older revision meanwhile, so nothing else would retry them. +/// A team's failed template stays failed until someone makes it again. +pub fn retry_failed(report: &ServerReport, templates: &[Template], now_ms: u64) -> Vec { + templates + .iter() + .filter(|t| super::base::on_demand(&t.id)) + .map(Template::server_name) + .filter(|name| { + report.templates.iter().any(|r| { + &r.name == name + && r.state == TemplateState::Failed + && r.build + .as_ref() + .and_then(|b| b.finished_at) + .is_some_and(|at| now_ms.saturating_sub(at) >= RETRY_FAILED_MS) + }) + }) + .collect() +} + +/// The command that has a server prepare `t`. None while an image in +/// Pyo's own registry can't be reached: Pyo knows no tailnet address for +/// its registry door yet (right after it starts). A server can't pull +/// `127.0.0.1:5005`, so it would only fail. +pub fn prepare_command(key: &SigningKey, t: &Template) -> Option { let image = t.source["image"].as_str().unwrap_or_default().trim(); let push = crate::deploy::registry_push(); let doors: Vec = [crate::registrydoor::address(), Some(push.clone())] @@ -113,24 +161,31 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand { .flatten() .collect(); match (in_pyos_registry(image, &doors), push.is_empty()) { - (Some(path), false) => DaemonCommand::PrepareTemplate { + (Some(_), true) => None, + (Some(path), false) => Some(DaemonCommand::PrepareTemplate { template: t.server_name(), // The door on the tailnet, as other machines pull from it. image: format!("{push}/{path}"), plain_http: true, - auth: t.team_id.as_ref().map(|team| RegistryAuth { + // An installation's template (Pyo's own, like a dev build's + // desktop) may pull only Pyo's own images. + auth: Some(RegistryAuth { username: crate::registrydoor::SANDBOX_USER.into(), - password: keys::registry_token(key, &t.id, team), + password: keys::registry_token( + key, + &t.id, + t.team_id.as_deref().unwrap_or(keys::PYO_IMAGES_ONLY), + ), }), max_mib: None, - }, - _ => DaemonCommand::PrepareTemplate { + }), + (None, _) => Some(DaemonCommand::PrepareTemplate { template: t.server_name(), image: image.to_owned(), plain_http: false, auth: None, max_mib: None, - }, + }), } } @@ -163,12 +218,14 @@ pub fn plan( .collect(); let has: HashSet<&str> = report.templates.iter().map(|t| t.name.as_str()).collect(); // Firecracker servers boot and snapshot a template. gVisor servers make - // the same disk and start sandboxes cold from it. + // the same disk and start sandboxes cold from it. One made on first use + // only once the server has used it: then a new revision too. let mut prepare: Vec = if report.capability.runtime != Runtime::None { - wanted - .iter() - .filter(|id| !has.contains(*id)) - .map(|s| s.to_string()) + mine.iter() + .zip(&names) + .filter(|(_, n)| !has.contains(n.as_str())) + .filter(|(t, _)| made_ahead(&t.id, &report.templates)) + .map(|(_, n)| n.clone()) .collect() } else { Vec::new() @@ -203,7 +260,8 @@ pub async fn sync_server( report: &ServerReport, ) -> sqlx::Result<()> { let templates = all(db).await?; - let (prepare, stale) = plan(server_team, report, &templates); + let (prepare, mut stale) = plan(server_team, report, &templates); + stale.extend(retry_failed(report, &templates, pyo_proto::now_ms() as u64)); if prepare.is_empty() && stale.is_empty() { return Ok(()); } @@ -213,7 +271,11 @@ pub async fn sync_server( let Some(t) = templates.iter().find(|t| t.server_name() == id) else { continue; }; - match d.command(&token, &prepare_command(key, t)).await { + let Some(cmd) = prepare_command(key, t) else { + tracing::debug!(server = %server_id, template = %id, "Pyo's registry door has no tailnet address yet: next round"); + continue; + }; + match d.command(&token, &cmd).await { Ok(_) => { tracing::info!(server = %server_id, template = %id, "asked the sandbox server to prepare a template") } @@ -258,8 +320,8 @@ pub async fn push(s: AppState, template_id: String) { else { return anyhow::Ok(()); }; - let servers: Vec<(String, Option, String)> = sqlx::query_as( - "SELECT id, team_id, address FROM pyo.sandbox_servers + let servers: Vec<(String, Option, String, Value)> = sqlx::query_as( + "SELECT id, team_id, address, templates FROM pyo.sandbox_servers WHERE state = 'ready' AND enabled AND address IS NOT NULL AND capability->>'runtime' IN ('firecracker', 'gvisor')", ) @@ -272,16 +334,26 @@ pub async fn push(s: AppState, template_id: String) { let key = keys::signing_key(&s.db, &s.sealer) .await .map_err(|e| anyhow::anyhow!(e.message))?; - for (id, team, address) in servers { - if !eligible(team.as_deref(), t.team_id.as_deref()) { + for (id, team, address, reports) in servers { + let reports: Vec = serde_json::from_value(reports).unwrap_or_default(); + if !eligible(team.as_deref(), t.team_id.as_deref()) || !made_ahead(&t.id, &reports) { continue; } + // The runner sends it once Pyo's registry door has an address. + let Some(cmd) = prepare_command(&key, &t) else { + continue; + }; let token = keys::admin_token(&key, &id); - if let Err(e) = Daemon::new(&address) - .command(&token, &prepare_command(&key, &t)) - .await - { - tracing::warn!(server = %id, template = %t.id, error = %e, "the sandbox server didn't take a template"); + match Daemon::new(&address).command(&token, &cmd).await { + // Saved at once, so the list shows it building before + // the next round. + Ok(DaemonResult::Template { template }) => { + save_report(&s.db, &id, &t.server_name(), Some(&template)).await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(server = %id, template = %t.id, error = %e, "the sandbox server didn't take a template"); + } } } anyhow::Ok(()) @@ -292,6 +364,26 @@ pub async fn push(s: AppState, template_id: String) { } } +/// Keeps what a server said of a template in its report, in place of +/// what it said of it before. None: it has none. +pub async fn save_report(db: &PgPool, server_id: &str, name: &str, r: Option<&TemplateReport>) { + let res = sqlx::query( + "UPDATE pyo.sandbox_servers + SET templates = coalesce((SELECT jsonb_agg(x) FROM jsonb_array_elements(templates) x + WHERE x->>'name' <> $2), '[]'::jsonb) + || coalesce($3::jsonb, '[]'::jsonb) + WHERE id = $1", + ) + .bind(server_id) + .bind(name) + .bind(r.map(|r| serde_json::json!([r]))) + .execute(db) + .await; + if let Err(e) = res { + tracing::warn!(server = %server_id, template = %name, error = %e, "couldn't save how far the template got"); + } +} + /// Tells the servers that have a deleted template to delete it, every /// revision of it. The runner catches the ones that don't answer now. pub async fn forget(s: AppState, template_id: String) { @@ -349,6 +441,44 @@ pub fn roll_up(reports: &[TemplateReport], missing: usize) -> (&'static str, Opt ("pending", None) } +/// A server's build error, as the dashboard shows it: the common ones in +/// plain words, the others as they are. +pub fn plain_build_error(raw: &str) -> String { + let lower = raw.to_ascii_lowercase(); + let has = |s: &str| lower.contains(s); + // TLS that got something else back: a web server, not a registry. + if has("corrupt message") || has("invalidcontenttype") || has("wrong version number") { + return "That address isn't a registry.".into(); + } + if has("certificate") && has("tls with") { + return "The registry's certificate isn't valid.".into(); + } + if let Some(rest) = raw.split_once("The registry answered ").map(|(_, r)| r) { + let status = rest.split_whitespace().next().unwrap_or_default(); + // Pyo's registry door says what to do itself. + if let Some((_, door)) = rest.split_once("Not one of this team's images.") { + let door = door.trim(); + return if door.is_empty() { + "Push this image to Pyo's registry with your team's login first.".into() + } else { + format!("Not one of this team's images. {door}") + }; + } + match status { + "401" | "403" => return "The registry refused the login for this image.".into(), + "404" => return "That image isn't in the registry. Check its name and tag.".into(), + _ => {} + } + } + if has("dns error") || has("failed to lookup") || has("name or service not known") { + return "That registry's address doesn't resolve.".into(); + } + if has("connecting to") && (has("timed out") || has("refused")) { + return "The server couldn't reach that registry.".into(); + } + raw.to_owned() +} + /// Rolls every template's state up from the servers' latest reports, by /// its current revision. While a new revision is made, a template whose /// older one is ready on a server stays ready: new sandboxes start from it. @@ -377,6 +507,8 @@ pub async fn update_states(db: &PgPool) -> sqlx::Result<()> { let (new, why) = match roll_up(&reports, missing.max(0) as usize) { ("ready", why) => ("ready", why), _ if older_ready => ("ready", None), + // Made on first use: ready to use on any server. + _ if super::base::on_demand(&id) => ("ready", None), r => r, }; if new != state || why != error { @@ -437,6 +569,42 @@ mod tests { } } + #[test] + fn a_failed_new_revision_of_pyos_own_is_made_again() { + let mut desktop = t("stp_desktop", None); + desktop.revision = Some("bbbb".into()); + let templates = [desktop, t("stp_team", Some("tm_a"))]; + let failed = |name: &str, at: u64| { + let mut r = made(name, TemplateState::Failed, Some("connection refused")); + r.build.as_mut().unwrap().finished_at = Some(at); + r + }; + let r = report( + Runtime::Firecracker, + vec![ + made("stp_desktop.aaaa", TemplateState::Ready, None), + failed("stp_desktop.bbbb", 1_000), + failed("stp_team", 1_000), + ], + ); + // Not at once: a minute after it failed. + assert!(retry_failed(&r, &templates, 1_000 + RETRY_FAILED_MS - 1).is_empty()); + assert_eq!( + retry_failed(&r, &templates, 1_000 + RETRY_FAILED_MS), + vec!["stp_desktop.bbbb"], + "a team's failed template stays failed" + ); + // Once deleted, the next round makes it again beside the old one. + let r = report( + Runtime::Firecracker, + vec![made("stp_desktop.aaaa", TemplateState::Ready, None)], + ); + assert_eq!( + plan(None, &r, &templates).0, + vec!["stp_desktop.bbbb", "stp_team"] + ); + } + #[test] fn which_servers_take_which_templates() { assert!(eligible(None, Some("tm_a")), "the installation's server"); @@ -540,12 +708,43 @@ mod tests { ); } + #[test] + fn base_is_made_on_first_use_then_kept_up() { + let mut base = t("stp_base", None); + base.revision = Some("bbbb".into()); + let templates = [base, t("stp_a", None)]; + // A new server makes the team's templates, not base. + let (prepare, stale) = plan(None, &report(Runtime::Firecracker, vec![]), &templates); + assert_eq!(prepare, vec!["stp_a"]); + assert!(stale.is_empty()); + // Once a sandbox made it there, it's kept, not deleted. + let r = report( + Runtime::Firecracker, + vec![ + made("stp_a", TemplateState::Ready, None), + made("stp_base.bbbb", TemplateState::Ready, None), + ], + ); + assert_eq!(plan(None, &r, &templates), (vec![], vec![])); + // A new release: a server that used it makes the new revision. + let r = report( + Runtime::Firecracker, + vec![made("stp_base.aaaa", TemplateState::Ready, None)], + ); + let (prepare, stale) = plan(None, &r, &templates); + assert_eq!(prepare, vec!["stp_a", "stp_base.bbbb"]); + assert!(stale.is_empty()); + assert!(!made_ahead("stp_base", &[])); + assert!(made_ahead("stp_a", &[])); + } + #[test] fn a_revision_is_prepared_under_its_name() { let key = SigningKey::generate("sbk_t"); let mut base = t("stp_base", None); base.revision = Some("bbbb".into()); - let DaemonCommand::PrepareTemplate { template, .. } = prepare_command(&key, &base) else { + let Some(DaemonCommand::PrepareTemplate { template, .. }) = prepare_command(&key, &base) + else { panic!("not a prepare"); }; assert_eq!(template, "stp_base.bbbb"); @@ -579,14 +778,25 @@ mod tests { "registry.tail1.ts.net".to_string(), "100.64.0.1:5006".to_string(), ]; + let path = |p: &str| Some(p.to_owned()); assert_eq!( in_pyos_registry("registry.tail1.ts.net/acme/app/api:1", &doors), - Some("acme/app/api:1") + path("acme/app/api:1") ); assert_eq!( in_pyos_registry("100.64.0.1:5006/acme/app/api@sha256:abc", &doors), - Some("acme/app/api@sha256:abc") + path("acme/app/api@sha256:abc") + ); + // Its names on Pyo's own machine, which a server can't reach. + assert_eq!( + in_pyos_registry("127.0.0.1:5005/frans/sandbox-base:1", &doors), + path("frans/sandbox-base:1") + ); + assert_eq!( + in_pyos_registry("localhost:5005/frans/sandbox-base", &doors), + path("frans/sandbox-base") ); + assert_eq!(in_pyos_registry("localhost:8000/x", &doors), None); assert_eq!(in_pyos_registry("python:3.13", &doors), None); assert_eq!( in_pyos_registry("registry.tail1.ts.net.evil.com/x", &doors), @@ -595,16 +805,53 @@ mod tests { assert_eq!(in_pyos_registry("ghcr.io/o/r", &[String::new()]), None); } + #[test] + fn build_errors_in_plain_words() { + let p = plain_build_error; + assert_eq!( + p( + "TLS with asus.blue-pancake.ts.net: received corrupt message of type InvalidContentType" + ), + "That address isn't a registry." + ); + assert_eq!( + p( + "The registry answered 403 Forbidden for the manifest of frans/sandbox-base: Not one of this team's images." + ), + "Push this image to Pyo's registry with your team's login first." + ); + assert_eq!( + p( + "The registry answered 403 Forbidden for the manifest of frans/sandbox-base: Not one of this team's images. Push it as acme// first." + ), + "Not one of this team's images. Push it as acme// first." + ); + assert_eq!( + p( + "The registry answered 404 Not Found for the manifest of library/nope: manifest unknown" + ), + "That image isn't in the registry. Check its name and tag." + ); + assert_eq!( + p("connecting to 10.0.0.9:443: Connection refused (os error 111)"), + "The server couldn't reach that registry." + ); + assert_eq!( + p("The image has no linux/arm64 variant."), + "The image has no linux/arm64 variant." + ); + } + #[test] fn a_public_image_goes_as_is() { let key = SigningKey::generate("sbk_t"); - let DaemonCommand::PrepareTemplate { + let Some(DaemonCommand::PrepareTemplate { template, image, plain_http, auth, .. - } = prepare_command(&key, &t("stp_a", Some("tm_a"))) + }) = prepare_command(&key, &t("stp_a", Some("tm_a"))) else { panic!("not a prepare"); }; diff --git a/crates/pyo-api/src/sigstore.rs b/crates/pyo-api/src/sigstore.rs index 070a4de..9179d54 100644 --- a/crates/pyo-api/src/sigstore.rs +++ b/crates/pyo-api/src/sigstore.rs @@ -32,15 +32,26 @@ const FULCIO_ROOT: &[u8] = include_bytes!("../assets/sigstore/fulcio-root.der"); const REKOR_KEY: &[u8] = include_bytes!("../assets/sigstore/rekor.der"); /// Who must have signed: the certificate's SAN (a URI or email) starting -/// with `identity_prefix`, issued for a token from `issuer`. +/// with `identity_prefix` (or equal to it when `exact`), issued for a +/// token from `issuer`. pub struct Signer<'a> { pub identity_prefix: &'a str, + pub exact: bool, pub issuer: &'a str, } /// Pyo's release workflow, at a version tag. pub const RELEASE_SIGNER: Signer<'static> = Signer { identity_prefix: "https://github.com/upsetdev/pyo/.github/workflows/release.yml@refs/tags/v", + exact: false, + issuer: "https://token.actions.githubusercontent.com", +}; + +/// upsetdev/pyo-sandbox-images' release workflow, on main: what signs its +/// images.json (sandbox/images.rs). +pub const SANDBOX_IMAGES_SIGNER: Signer<'static> = Signer { + identity_prefix: "https://github.com/upsetdev/pyo-sandbox-images/.github/workflows/release.yml@refs/heads/main", + exact: true, issuer: "https://token.actions.githubusercontent.com", }; @@ -156,10 +167,15 @@ pub fn verify(blob: &[u8], bundle: &str, signer: &Signer) -> anyhow::Result<()> }) .unwrap_or_default(); ensure!( - names.iter().any(|n| n.starts_with(signer.identity_prefix)), - "signed by {}, not {}…", + names.iter().any(|n| if signer.exact { + n == signer.identity_prefix + } else { + n.starts_with(signer.identity_prefix) + }), + "signed by {}, not {}{}", names.join(", "), - signer.identity_prefix + signer.identity_prefix, + if signer.exact { "" } else { "…" } ); let issuer = leaf .extensions() @@ -294,6 +310,7 @@ mod tests { /// cosign's own release (v3.1.4), signed keyless by its release account. const COSIGN: Signer<'static> = Signer { identity_prefix: "keyless@projectsigstore.iam.gserviceaccount.com", + exact: true, issuer: "https://accounts.google.com", }; @@ -324,6 +341,23 @@ mod tests { .is_err(), "another issuer" ); + let start = Signer { + identity_prefix: "keyless@projectsigstore", + ..COSIGN + }; + assert!(verify(BLOB, BUNDLE, &start).is_err(), "exact: not a prefix"); + assert!( + verify( + BLOB, + BUNDLE, + &Signer { + exact: false, + ..start + } + ) + .is_ok(), + "a prefix" + ); // Rekor's promise, changed. let mut b: Value = serde_json::from_str(BUNDLE).unwrap(); let t = &mut b["verificationMaterial"]["tlogEntries"][0]; diff --git a/crates/pyo-api/src/sites.rs b/crates/pyo-api/src/sites.rs index ec1bff4..9a9b70a 100644 --- a/crates/pyo-api/src/sites.rs +++ b/crates/pyo-api/src/sites.rs @@ -14,10 +14,11 @@ //! - The newest [`KEEP`] folders of each service stay for rollbacks; older //! ones, and those of deleted services, are removed by [`sweeper`]. +use crate::error::Json; use std::time::Duration; use axum::{ - Json, Router, + Router, body::Body, extract::{FromRequestParts, Path, State}, http::{HeaderMap, StatusCode, request::Parts}, diff --git a/crates/pyo-api/src/tailscale.rs b/crates/pyo-api/src/tailscale.rs index e6a4019..503ecff 100644 --- a/crates/pyo-api/src/tailscale.rs +++ b/crates/pyo-api/src/tailscale.rs @@ -508,6 +508,87 @@ pub struct Device { pub lock_error: String, } +/// Pyo's machines (tag:pyo) that Tailnet Lock keeps out until a signing +/// device signs their key: their hostname, and the command that signs it. +pub fn lock_waiting(devices: &[Device]) -> Vec<(String, String)> { + devices + .iter() + .filter(|d| { + !d.lock_error.is_empty() + && d.node_key.starts_with("nodekey:") + && d.tags.iter().any(|t| t == TAG) + }) + .map(|d| { + ( + d.hostname.clone(), + format!("tailscale lock sign {}", d.node_key), + ) + }) + .collect() +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct LockQuery { + /// The team adding the server (slug or id). + team: String, +} + +/// Machines waiting for a Tailnet Lock signature +/// +/// With Tailnet Lock on, a new server stays off the tailnet until a +/// signing device signs its key. These are the ones Pyo's commands added +/// that wait, with the command to run on a signing device. Read from +/// Tailscale at most every 15 seconds. Empty without Tailscale set up. +#[utoipa::path(get, path = "/tailnet/lock", operation_id = "getTailnetLock", tag = "Servers", + params(LockQuery), + responses( + (status = 200, description = "`waiting`: each with `name` (the machine's hostname) and `signCommand`.", body = Object), + (status = 403, description = "Only owners and admins add servers.", body = crate::error::ErrorBody), + (status = 404, description = "No such team, or you aren't in it.", body = crate::error::ErrorBody), + ), + extensions(("x-pyo-access" = json!("read"))))] +pub async fn lock( + axum::extract::State(s): axum::extract::State, + crate::auth::SessionUser(user): crate::auth::SessionUser, + crate::error::Query(q): crate::error::Query, +) -> ApiResult> { + let mut tx = s.db.begin().await?; + crate::ops::require_role(&mut tx, &user, &q.team, crate::ops::Role::Admin).await?; + drop(tx); + /// When the list was read, and what it said. + type Seen = Option<(Instant, Vec<(String, String)>)>; + static SEEN: Mutex = Mutex::new(None); + let fresh = |seen: &Seen| { + seen.as_ref() + .filter(|(at, _)| at.elapsed() < Duration::from_secs(15)) + .map(|(_, w)| w.clone()) + }; + let cached = fresh(&SEEN.lock().unwrap_or_else(|e| e.into_inner())); + let waiting = match cached { + Some(w) => w, + None => { + let w = match saved_client(&s).await? { + Some((_, c)) => match c.device_details().await { + Ok(d) => lock_waiting(&d), + Err(e) => { + tracing::warn!(error = %e, "tailnet lock: couldn't read the tailnet"); + Vec::new() + } + }, + None => Vec::new(), + }; + *SEEN.lock().unwrap_or_else(|e| e.into_inner()) = Some((Instant::now(), w.clone())); + w + } + }; + let waiting: Vec = waiting + .into_iter() + .map(|(name, sign)| json!({ "name": name, "signCommand": sign })) + .collect(); + Ok(crate::error::Json(json!({ "waiting": waiting }))) +} + fn strings(v: &Value) -> Vec { v.as_array() .map(|a| { @@ -647,7 +728,7 @@ pub async fn runner(state: AppState) { } } -/// Every web process: Pyo's HTTPS names as the runner last saved them. +/// Every process: Pyo's addresses as the runner last saved them. pub async fn addresses_follower(db: PgPool) { loop { if let Ok(Some(v)) = @@ -1442,6 +1523,34 @@ mod tests { assert_eq!(*writes.lock().unwrap(), ["PUT svc:blog", "DELETE svc:mine"]); } + #[test] + fn machines_waiting_for_a_signature() { + let d = |host: &str, tag: &str, key: &str, err: &str| Device { + node_id: "n".into(), + hostname: host.into(), + tags: vec![tag.into()], + node_key: key.into(), + lock_error: err.into(), + }; + let waiting = lock_waiting(&[ + d( + "sbx-1", + TAG, + "nodekey:ab12", + "tailnet lock: node key not signed", + ), + d("laptop", "tag:other", "nodekey:cd34", "not signed"), + d("web-1", TAG, "nodekey:ef56", ""), + ]); + assert_eq!( + waiting, + [( + "sbx-1".to_owned(), + "tailscale lock sign nodekey:ab12".to_owned() + )] + ); + } + #[test] fn names() { assert_eq!( diff --git a/crates/pyo-api/src/terminal.rs b/crates/pyo-api/src/terminal.rs index a4a901b..bfa4517 100644 --- a/crates/pyo-api/src/terminal.rs +++ b/crates/pyo-api/src/terminal.rs @@ -6,12 +6,13 @@ //! on another one, which forwards its socket to the replica holding the //! browser (`relay`, its pod address). +use crate::error::Query; use std::{collections::HashMap, sync::Arc, time::Duration}; use axum::{ Router, extract::{ - Path, Query, State, + Path, State, ws::{Message, WebSocket, WebSocketUpgrade}, }, http::{HeaderMap, header}, diff --git a/crates/pyo-api/src/traffic.rs b/crates/pyo-api/src/traffic.rs index 28a9905..d16da11 100644 --- a/crates/pyo-api/src/traffic.rs +++ b/crates/pyo-api/src/traffic.rs @@ -7,16 +7,14 @@ //! it's answered by the gateway on the server it reached; from there it goes //! to a replica on that server, else one in that server's region, else any. +use crate::error::Json; use std::{ collections::{BTreeMap, HashMap}, sync::{LazyLock, Mutex}, time::{Duration, Instant}, }; -use axum::{ - Json, - extract::{Path, State}, -}; +use axum::extract::{Path, State}; use serde_json::{Value, json}; use crate::{ diff --git a/crates/pyo-api/src/updates.rs b/crates/pyo-api/src/updates.rs index 929831b..595e733 100644 --- a/crates/pyo-api/src/updates.rs +++ b/crates/pyo-api/src/updates.rs @@ -296,7 +296,7 @@ fn source() -> String { mirror().unwrap_or_else(|| format!("https://api.github.com/repos/{REPO}")) } -fn client() -> anyhow::Result { +pub(crate) fn client() -> anyhow::Result { let mirror = std::env::var("PYO_UPDATE_SOURCE") .ok() .and_then(|s| reqwest::Url::parse(&s).ok()) @@ -323,7 +323,10 @@ fn client() -> anyhow::Result { .build()?) } -async fn body_limited(mut res: reqwest::Response, max: usize) -> anyhow::Result> { +pub(crate) async fn body_limited( + mut res: reqwest::Response, + max: usize, +) -> anyhow::Result> { let mut out = vec![]; while let Some(chunk) = res.chunk().await? { out.extend_from_slice(&chunk); diff --git a/crates/pyo-api/src/volumes.rs b/crates/pyo-api/src/volumes.rs index 3f01b87..534ea85 100644 --- a/crates/pyo-api/src/volumes.rs +++ b/crates/pyo-api/src/volumes.rs @@ -732,7 +732,7 @@ pub(crate) async fn snapshots( axum::extract::State(s): axum::extract::State, SessionUser(user): SessionUser, axum::extract::Path(id): axum::extract::Path, -) -> ApiResult> { +) -> ApiResult> { let mut tx = s.db.begin().await?; let team_id = team_of(&mut tx, "volumes", &id).await?; require_role(&mut tx, &user, &team_id, Role::Developer).await?; @@ -765,7 +765,7 @@ pub(crate) async fn snapshots( .data .and_then(|d| serde_json::from_value(d["snapshots"].clone()).ok()) .unwrap_or_default(); - Ok(axum::Json(json!({ "snapshots": list }))) + Ok(crate::error::Json(json!({ "snapshots": list }))) } /// Wait for a command the API waits on (a short Job on the cluster): its @@ -828,7 +828,7 @@ pub(crate) async fn delete_snapshot( axum::extract::State(s): axum::extract::State, SessionUser(user): SessionUser, axum::extract::Path((id, snapshot_id)): axum::extract::Path<(String, String)>, -) -> ApiResult> { +) -> ApiResult> { if user .key .as_ref() @@ -888,7 +888,9 @@ pub(crate) async fn delete_snapshot( ) .await?; tx.commit().await?; - Ok(axum::Json(json!({ "ok": true, "message": r.message }))) + Ok(crate::error::Json( + json!({ "ok": true, "message": r.message }), + )) } /// Why a volume on storage class `class` (as its cluster reports it) can't diff --git a/crates/pyo-builder/Cargo.toml b/crates/pyo-builder/Cargo.toml index 5ea440f..e50e9b5 100644 --- a/crates/pyo-builder/Cargo.toml +++ b/crates/pyo-builder/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0.104" base64 = "0.23.1" futures = "0.3.34" hex = "0.4.3" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] } serde = "1.0.229" serde_json = "1.0.154" diff --git a/crates/pyo-cli/Cargo.toml b/crates/pyo-cli/Cargo.toml index a7c3dd5..f7f154d 100644 --- a/crates/pyo-cli/Cargo.toml +++ b/crates/pyo-cli/Cargo.toml @@ -24,4 +24,4 @@ sha2 = "0.11.1" tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "net", "io-util", "io-std", "time", "sync", "fs"] } tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } toml = "0.9" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } diff --git a/crates/pyo-cli/src/api.rs b/crates/pyo-cli/src/api.rs index 3b9d7a6..9f9bb7b 100644 --- a/crates/pyo-cli/src/api.rs +++ b/crates/pyo-cli/src/api.rs @@ -47,12 +47,10 @@ impl Api { .await .with_context(|| format!("Can't reach Pyo at {}", self.base))?; let status = res.status(); - let body: Value = res.json().await.unwrap_or(Value::Null); + let text = res.text().await.unwrap_or_default(); + let body: Value = serde_json::from_str(&text).unwrap_or(Value::Null); if !status.is_success() { - let msg = body["error"] - .as_str() - .map(str::to_owned) - .unwrap_or_else(|| format!("Request failed ({status})")); + let msg = error_message(status, &body, &text); if status == reqwest::StatusCode::UNAUTHORIZED { bail!("{msg}\nSign in again: pyo login"); } @@ -81,6 +79,11 @@ impl Api { .await } + pub async fn patch(&self, path: &str, body: &Value) -> Result { + self.send(self.http.patch(format!("{}{path}", self.base)).json(body)) + .await + } + pub async fn delete(&self, path: &str) -> Result { self.send(self.http.delete(format!("{}{path}", self.base))) .await @@ -107,6 +110,28 @@ impl Api { } } +/// What a failed request says: the JSON `error`, else the body's text +/// (an older Pyo answers some bad requests in plain text), else the status. +fn error_message(status: reqwest::StatusCode, body: &Value, text: &str) -> String { + let mut msg = match body["error"].as_str() { + Some(e) => e.to_owned(), + None => { + let t = text.trim(); + if t.is_empty() || t.starts_with('<') { + format!("Request failed ({status})") + } else { + t.chars().take(300).collect() + } + } + }; + // Pyo answers a request it can't read with a 400. A 422 is an older + // Pyo's, for something this CLI sends that it doesn't know. + if status == reqwest::StatusCode::UNPROCESSABLE_ENTITY { + msg.push_str("\nThis Pyo may be older than the CLI. Update it or use an older CLI."); + } + msg +} + /// The dashboard's snapshot, with the lookups the CLI needs. pub struct State(pub Value); @@ -212,3 +237,38 @@ pub fn now_ms() -> i64 { .map(|d| d.as_millis() as i64) .unwrap_or(0) } + +#[cfg(test)] +mod error_tests { + use super::*; + use reqwest::StatusCode; + + #[test] + fn failed_requests_say_why() { + let m = |s, body: &str| { + error_message(s, &serde_json::from_str(body).unwrap_or(Value::Null), body) + }; + assert_eq!( + m( + StatusCode::BAD_REQUEST, + r#"{"error":"Add the team parameter."}"# + ), + "Add the team parameter." + ); + assert_eq!( + m( + StatusCode::UNPROCESSABLE_ENTITY, + "Failed to deserialize the JSON body into the target type: missing field `template`\n" + ), + "Failed to deserialize the JSON body into the target type: missing field `template`\nThis Pyo may be older than the CLI. Update it or use an older CLI." + ); + assert_eq!( + m(StatusCode::BAD_GATEWAY, ""), + "Request failed (502 Bad Gateway)" + ); + assert_eq!( + m(StatusCode::BAD_GATEWAY, ""), + "Request failed (502 Bad Gateway)" + ); + } +} diff --git a/crates/pyo-cli/src/main.rs b/crates/pyo-cli/src/main.rs index 5ff6c2a..5b7d495 100644 --- a/crates/pyo-cli/src/main.rs +++ b/crates/pyo-cli/src/main.rs @@ -262,6 +262,9 @@ enum Command { #[command(subcommand)] command: ServerCommand, }, + /// The CLI's version, like --version. + #[command(disable_version_flag = true)] + Version, /// Update the CLI to the latest release. /// /// Once a day, the CLI looks for a new release and says when one is @@ -996,6 +999,10 @@ async fn run(mut cli: Cli) -> anyhow::Result<()> { Command::Sandbox { command, team } => { sandbox::run(&config::client(url, token)?, team.as_deref(), command).await } + Command::Version => { + println!("pyo {}", env!("CARGO_PKG_VERSION")); + Ok(()) + } Command::Upgrade { version } => upgrade::run(version).await, Command::Hetzner { command, team } => { let api = config::client(url, token)?; diff --git a/crates/pyo-cli/src/sandbox.rs b/crates/pyo-cli/src/sandbox.rs index a3ee79d..94a9986 100644 --- a/crates/pyo-cli/src/sandbox.rs +++ b/crates/pyo-cli/src/sandbox.rs @@ -22,18 +22,30 @@ pub enum Command { /// Make a sandbox from a sandbox template and print its id. Create { /// The sandbox template, like node-22 (see pyo sandbox templates ls). - /// Default base: Debian 13 with git, Python, uv, Node.js and pnpm. + /// Default base: Ubuntu 24.04 with git, Python, uv, Node.js, pnpm, + /// Bun, Deno, the GitHub and Docker CLIs, Claude Code, Codex and + /// opencode. desktop adds a desktop and a browser: CDP on port 9222, + /// the screen on port 6080. A server makes these the first time + /// they're used there. template: Option, - /// CPUs. Default 1. + /// A name to tell it apart. Pyo makes up a short one when left out. + #[arg(long)] + name: Option, + /// The sandbox server to run it on, by name or id. Pyo picks one + /// when left out. + #[arg(long)] + server: Option, + /// CPUs. Default 1 (2 for desktop). #[arg(long)] cpus: Option, - /// Memory in MiB. Default 512. + /// Memory in MiB. Default 512 (2048 for desktop). #[arg(long)] memory: Option, /// Writable disk in MiB. Default 2048. #[arg(long)] disk: Option, - /// How long it lives, like 30m or 1h. Default 5m. + /// How long it lives, like 30m, 1h or 7d, up to 365d. "none" keeps it + /// until you delete it. Default 5m. #[arg(long)] ttl: Option, /// Pause after this many seconds without use. @@ -75,6 +87,13 @@ pub enum Command { #[arg(long)] json: bool, }, + /// Give a sandbox another name. Its id stays the same. + Rename { + /// The sandbox's id, sbx_…. + id: String, + /// The new name. + name: String, + }, /// Delete a sandbox and its files. Rm { /// The sandbox's id, sbx_…. @@ -94,7 +113,7 @@ pub enum Command { Extend { /// The sandbox's id, sbx_…. id: String, - /// Like 30m or 1h. + /// Like 30m, 1h or 7d. "none" keeps it until you delete it. ttl: String, }, /// A new token for the sandbox's server, valid an hour. @@ -248,6 +267,8 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> { match cmd { Command::Create { template, + name, + server, cpus, memory, disk, @@ -264,6 +285,8 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> { let body = create_body(CreateArgs { team: &team, template: template.as_deref(), + name: name.as_deref(), + server: server.as_deref(), cpus, memory, disk, @@ -305,6 +328,19 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> { } Ok(()) } + Command::Rename { id, name } => { + let r = api + .patch( + &format!("/api/v1/sandboxes/{}", seg(&id)), + &json!({ "name": name }), + ) + .await?; + ui::ok(&format!( + "{id} is now {}.", + r["name"].as_str().unwrap_or(&name) + )); + Ok(()) + } Command::Rm { id } => { let r = api .delete(&format!("/api/v1/sandboxes/{}", seg(&id))) @@ -324,10 +360,10 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> { &json!({ "ttl": ttl }), ) .await?; - ui::ok(&format!( - "{id} ends in {}.", - left(r["expiresAt"].as_i64(), crate::api::now_ms()) - )); + ui::ok(&match r["expiresAt"].as_i64() { + Some(at) => format!("{id} ends in {}.", left(Some(at), crate::api::now_ms())), + None => format!("{id} runs until you delete it."), + }); Ok(()) } Command::Token { id, json } => { @@ -411,6 +447,8 @@ async fn lifecycle(api: &Api, id: &str, what: &str, done: &str) -> Result<()> { pub struct CreateArgs<'a> { pub team: &'a str, pub template: Option<&'a str>, + pub name: Option<&'a str>, + pub server: Option<&'a str>, pub cpus: Option, pub memory: Option, pub disk: Option, @@ -432,6 +470,8 @@ pub fn create_body(a: CreateArgs) -> Result { } }; set("template", json!(a.template)); + set("name", json!(a.name)); + set("server", json!(a.server)); set("cpus", json!(a.cpus)); set("memoryMib", json!(a.memory)); set("diskMib", json!(a.disk)); @@ -460,8 +500,11 @@ fn pairs(items: &[String], flag: &str) -> Result> { } /// "45m", "2h 10m", or "ended". +/// Time left until `at`. None: the sandbox has no lifetime. fn left(at: Option, now: i64) -> String { - let Some(at) = at else { return "-".into() }; + let Some(at) = at else { + return "never".into(); + }; let s = (at - now) / 1000; match s { ..=0 => "ended".into(), @@ -490,13 +533,14 @@ fn print_sandboxes(list: &Value) { return; } let now = crate::api::now_ms(); - let rows: Vec<[String; 6]> = list + let rows: Vec<[String; 7]> = list .iter() .map(|x| { let s = |k: &str| x[k].as_str().unwrap_or("").to_owned(); let ended = matches!(x["state"].as_str(), Some("stopped" | "failed")); [ s("id"), + s("name"), s("template"), s("state"), size(x), @@ -510,7 +554,7 @@ fn print_sandboxes(list: &Value) { }) .collect(); table( - &["ID", "TEMPLATE", "STATE", "SIZE", "LEFT", "CREATED"], + &["ID", "NAME", "TEMPLATE", "STATE", "SIZE", "LEFT", "CREATED"], &rows, ); } @@ -555,10 +599,18 @@ fn print_sandbox(x: &Value) { .collect(); let rows = [ ("id", s("id")), + ("name", s("name")), ("state", s("state")), ("template", s("template")), ("size", size(x)), - ("ends in", left(x["expiresAt"].as_i64(), now)), + ( + "ends in", + if matches!(x["state"].as_str(), Some("stopped" | "failed")) { + "-".into() + } else { + left(x["expiresAt"].as_i64(), now) + }, + ), ("server", s("server")), ("runtime", s("runtime")), ("network", s("network")), @@ -775,14 +827,15 @@ async fn servers(api: &Api, team: Option<&str>, cmd: ServersCommand) -> Result<( } /// A template's image as the list shows it. Pyo's built-in one says what -/// it runs and leaves out its version: Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base. +/// it runs and leaves out its version: Ubuntu 24.04 · ghcr.io/upsetdev/pyo-sandbox-images/base. fn template_image(t: &Value) -> String { - let image = t["source"]["image"].as_str().unwrap_or("-"); - match t["source"]["os"].as_str() { - Some(os) if t["builtin"] == true => { + let image = t["source"]["image"].as_str(); + match (t["source"]["os"].as_str(), image) { + (Some(os), Some(image)) if t["builtin"] == true => { format!("{os} · {}", pyo_proto::image_repo(image)) } - _ => image.to_owned(), + (Some(os), None) if t["builtin"] == true => os.to_owned(), + _ => image.unwrap_or("-").to_owned(), } } @@ -1113,6 +1166,8 @@ mod tests { let b = create_body(CreateArgs { team: "tm_1", template: Some("node-22"), + name: Some("checkout tests"), + server: Some("sandbox-1"), cpus: Some(2), memory: None, disk: None, @@ -1130,6 +1185,8 @@ mod tests { json!({ "teamId": "tm_1", "template": "node-22", + "name": "checkout tests", + "server": "sandbox-1", "cpus": 2, "ttl": "1h", "network": "none", @@ -1143,6 +1200,8 @@ mod tests { let b = create_body(CreateArgs { team: "tm_1", template: None, + name: None, + server: None, cpus: None, memory: None, disk: None, @@ -1160,10 +1219,10 @@ mod tests { #[test] fn the_builtin_template_shows_what_it_runs() { - let base = json!({ "builtin": true, "source": { "image": "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:ab", "os": "Debian 13" } }); + let base = json!({ "builtin": true, "source": { "image": "ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:ab", "os": "Ubuntu 24.04" } }); assert_eq!( template_image(&base), - "Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base" + "Ubuntu 24.04 · ghcr.io/upsetdev/pyo-sandbox-images/base" ); let own = json!({ "builtin": false, "source": { "image": "node:22" } }); assert_eq!(template_image(&own), "node:22"); @@ -1175,6 +1234,8 @@ mod tests { let e = create_body(CreateArgs { team: "t", template: Some("x"), + name: None, + server: None, cpus: None, memory: None, disk: None, @@ -1242,7 +1303,7 @@ mod tests { assert_eq!(left(Some(now + 130 * 60_000), now), "2h 10m"); assert_eq!(left(Some(now + 3 * 86_400_000), now), "3d"); assert_eq!(left(Some(now - 1), now), "ended"); - assert_eq!(left(None, now), "-"); + assert_eq!(left(None, now), "never"); } #[test] diff --git a/crates/pyo-envd/Cargo.toml b/crates/pyo-envd/Cargo.toml index b4c8450..d7d0fb8 100644 --- a/crates/pyo-envd/Cargo.toml +++ b/crates/pyo-envd/Cargo.toml @@ -12,5 +12,5 @@ publish.workspace = true [dependencies] libc = "0.2.191" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } serde_json.workspace = true diff --git a/crates/pyo-envd/src/main.rs b/crates/pyo-envd/src/main.rs index c20a676..03b3823 100644 --- a/crates/pyo-envd/src/main.rs +++ b/crates/pyo-envd/src/main.rs @@ -15,16 +15,26 @@ //! `unix:` or `none` //! - `PYO_ENVD_DIAL`: `unix:` or `vsock::`. Dial mode, for //! gVisor. PID 1 then leaves the mounts to runsc. +//! +//! A template's image may have an executable [`START`]. PID 1 runs it once, +//! after the server starts, in a session of its own: the template's own +//! background programs (Pyo's `desktop` starts its listener this way). +//! Templates don't run an image's CMD. On Firecracker the template is +//! snapshotted after boot, so what it started is already running in every +//! sandbox restored from it. mod init; mod listen; mod net; mod server; -use std::process::Command; +use std::os::unix::process::CommandExt; +use std::process::{Command, Stdio}; use std::time::Duration; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +/// What a template runs at boot, when its image has it. +pub const START: &str = "/etc/pyo/start"; fn main() { let arg = std::env::args().nth(1); @@ -42,6 +52,7 @@ fn pid1(mode: listen::Mode) -> ! { init::setup(); } let mut server = spawn_server(); + run_start(); loop { let mut status = 0; // SAFETY: plain waitpid on any child. @@ -63,6 +74,33 @@ fn pid1(mode: listen::Mode) -> ! { } } +/// Runs the template's [`START`], when there is one. PID 1 reaps it and +/// what it leaves behind. +fn run_start() { + let Ok(meta) = std::fs::metadata(START) else { + return; + }; + use std::os::unix::fs::PermissionsExt; + if !meta.is_file() || meta.permissions().mode() & 0o111 == 0 { + eprintln!("pyo-envd: {START} isn't an executable file, so it doesn't run"); + return; + } + let mut cmd = Command::new(START); + cmd.stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + // SAFETY: setsid is async-signal-safe. + unsafe { + cmd.pre_exec(|| { + libc::setsid(); + Ok(()) + }); + } + if let Err(e) = cmd.spawn() { + eprintln!("pyo-envd: can't run {START}: {e}"); + } +} + fn spawn_server() -> Option { match Command::new("/proc/self/exe").arg("serve").spawn() { Ok(c) => Some(c), diff --git a/crates/pyo-flows/Cargo.toml b/crates/pyo-flows/Cargo.toml index a8964f1..b7494b5 100644 --- a/crates/pyo-flows/Cargo.toml +++ b/crates/pyo-flows/Cargo.toml @@ -8,7 +8,7 @@ publish.workspace = true [dependencies] anyhow = "1.0.104" libc = "0.2.191" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls"] } serde_json = "1.0.154" tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "time", "sync"] } diff --git a/crates/pyo-proto/src/lib.rs b/crates/pyo-proto/src/lib.rs index a67436e..e3aa8d9 100644 --- a/crates/pyo-proto/src/lib.rs +++ b/crates/pyo-proto/src/lib.rs @@ -1137,18 +1137,21 @@ pub struct ReleaseImages { pub postgres17: String, #[serde(default)] pub postgres18: String, - /// Pyo Sandbox's built-in template `base`, multi-arch - /// (deploy/images/sandbox-base.Dockerfile). Pyo seeds the template - /// from it; nothing runs it on a cluster. + /// Pyo Sandbox's built-in template `base` this release ships with, + /// from upsetdev/pyo-sandbox-images. Pyo itself takes the latest from + /// that repo's images.json. Pyo 0.4 reads it here, so releases keep it. #[serde(default)] pub sandbox_base: String, + /// The same for the built-in template `desktop`. + #[serde(default)] + pub sandbox_desktop: String, } impl ReleaseImages { /// Every image, by its key in release.json. A release pins each image /// scripts/release-manifest.ts writes; adding one there means adding /// it here (a test compares the two). - pub fn all(&self) -> [(&'static str, &str); 15] { + pub fn all(&self) -> [(&'static str, &str); 16] { [ ("pyo", &self.pyo), ("agent", &self.agent), @@ -1165,6 +1168,7 @@ impl ReleaseImages { ("postgres17", &self.postgres17), ("postgres18", &self.postgres18), ("sandboxBase", &self.sandbox_base), + ("sandboxDesktop", &self.sandbox_desktop), ] } } @@ -1264,6 +1268,22 @@ impl Release { )); } } + for (env, key, pinned) in [ + (SANDBOX_BASE_ENV, "sandboxBase", &self.images.sandbox_base), + ( + SANDBOX_DESKTOP_ENV, + "sandboxDesktop", + &self.images.sandbox_desktop, + ), + ] { + let mut refs = vec![]; + collect_env(o, env, &mut refs); + for r in refs { + if !r.is_empty() && &r != pinned { + out.push(format!("{env} is {r}, not the release's {key} image.")); + } + } + } } if let Some(m) = &self.min_from && parse_version(m).is_none() @@ -1296,8 +1316,39 @@ pub fn image_repo(image: &str) -> &str { } } +/// The env var that overrides the sandbox template `base`'s image, when +/// it's pinned by digest. Releases leave it empty. When a release sets it, +/// it must be the release's `sandboxBase`. +pub const SANDBOX_BASE_ENV: &str = "PYO_SANDBOX_BASE_REF"; +/// The same for the sandbox template `desktop` (the release's +/// `sandboxDesktop`). +pub const SANDBOX_DESKTOP_ENV: &str = "PYO_SANDBOX_DESKTOP_REF"; + +/// The values of env vars named `name` in a Kubernetes object. +pub fn collect_env(v: &serde_json::Value, name: &str, out: &mut Vec) { + match v { + serde_json::Value::Object(m) => { + if m.get("name").and_then(|n| n.as_str()) == Some(name) + && let Some(serde_json::Value::String(value)) = m.get("value") + { + out.push(value.clone()); + } + m.values().for_each(|x| collect_env(x, name, out)); + } + serde_json::Value::Array(a) => a.iter().for_each(|x| collect_env(x, name, out)), + _ => {} + } +} + /// Every `image` value in a Kubernetes object, and image references in its /// containers' env (Pyo passes its agent and builder images that way). +/// +/// Older Pyos run this same rule on a new release before they update to +/// it, with only the images they knew. So an env var whose name ends in +/// `_IMAGE` may only hold one of the images every supported Pyo knows +/// (the test `manifests_pass_older_pyos_checks` keeps this). A new image +/// goes in a var named otherwise, like [`SANDBOX_BASE_ENV`], and +/// [`Release::problems`] checks it by name. pub fn collect_images(v: &serde_json::Value, out: &mut Vec) { match v { serde_json::Value::Object(m) => { @@ -2569,7 +2620,8 @@ mod release_tests { postgres16: img("postgres-16"), postgres17: img("postgres-17"), postgres18: img("postgres-18"), - sandbox_base: img("sandbox-base"), + sandbox_base: format!("ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:{D}"), + sandbox_desktop: format!("ghcr.io/upsetdev/pyo-sandbox-images/desktop@sha256:{D}"), }, manifests: ReleaseManifests { platform: vec![ @@ -2578,7 +2630,7 @@ mod release_tests { pyo: vec![ json!({ "kind": "Deployment", "spec": { "template": { "spec": { "containers": [{ "name": "pyo", "image": img("pyo"), - "env": [{ "name": "PYO_AGENT_IMAGE", "value": img("agent") }, { "name": "PYO_BUILDER_IMAGE", "value": img("builder") }], + "env": [{ "name": "PYO_AGENT_IMAGE", "value": img("agent") }, { "name": "PYO_BUILDER_IMAGE", "value": img("builder") }, { "name": SANDBOX_BASE_ENV, "value": "" }, { "name": SANDBOX_DESKTOP_ENV, "value": "" }], }] } } } }), ], }, @@ -2678,13 +2730,12 @@ mod release_tests { // Each of its images is a field here, with the same value (an // image added since, pyo-static's, is empty). let mut images = serde_json::to_value(&r.images).unwrap(); - images - .as_object_mut() - .unwrap() - .retain(|k, v| !(matches!(k.as_str(), "static" | "sandboxBase") && v == "")); + images.as_object_mut().unwrap().retain(|k, v| { + !(matches!(k.as_str(), "static" | "sandboxBase" | "sandboxDesktop") && v == "") + }); assert_eq!(images, raw["images"]); for (name, image) in r.images.all() { - if matches!(name, "static" | "sandboxBase") && image.is_empty() { + if matches!(name, "static" | "sandboxBase" | "sandboxDesktop") && image.is_empty() { continue; } assert!(pinned_image(image), "{name}: {image}"); @@ -2725,6 +2776,14 @@ mod release_tests { .filter_map(|l| l.split_once(':')) .map(|(_, v)| v.trim().trim_end_matches(',').trim_matches('"').to_owned()) .collect(); + // The sandbox templates' images, from images.json. + for key in ["sandboxBase", "sandboxDesktop"] { + assert!( + script.contains(&format!("pinned.{key} = sandboxRef(")), + "release-manifest.ts sets {key}" + ); + keys.push(key.into()); + } let majors = script .split_once("for (const major of [") .and_then(|(_, rest)| rest.split_once(']')) @@ -2792,6 +2851,121 @@ mod release_tests { assert!(bad.problems().iter().any(|p| p.contains("isn't one of"))); } + #[test] + fn the_sandbox_refs_are_empty_or_the_pinned_ones() { + // Releases leave them empty: Pyo takes images.json's. + assert!( + release().problems().is_empty(), + "{:?}", + release().problems() + ); + let env = |r: &mut Release, i: usize, v: String| { + r.manifests.pyo[0]["spec"]["template"]["spec"]["containers"][0]["env"][i]["value"] = + json!(v); + }; + // When one is set, it's the release's. + let mut r = release(); + let base = r.images.sandbox_base.clone(); + env(&mut r, 2, base); + assert!(r.problems().is_empty(), "{:?}", r.problems()); + env( + &mut r, + 2, + format!("ghcr.io/upsetdev/pyo/sandbox-base:0.2.1@sha256:{D}"), + ); + assert!( + r.problems() + .iter() + .any(|p| p.starts_with("PYO_SANDBOX_BASE_REF is")), + "{:?}", + r.problems() + ); + let mut r = release(); + env( + &mut r, + 3, + format!("ghcr.io/upsetdev/pyo/sandbox-desktop:0.2.1@sha256:{D}"), + ); + assert!( + r.problems() + .iter() + .any(|p| p.starts_with("PYO_SANDBOX_DESKTOP_REF is")), + "{:?}", + r.problems() + ); + // The images live in another repository than Pyo's own, and must + // be pinned all the same: Pyo 0.4 checks that too. + let mut r = release(); + r.images.sandbox_base = "ghcr.io/upsetdev/pyo-sandbox-images/base:24.04".into(); + assert!( + r.problems() + .iter() + .any(|p| p.contains("sandboxBase image isn't pinned")), + "{:?}", + r.problems() + ); + } + + /// Before updating, an older Pyo checks the new release with its own + /// rule: every `image` and every env var named `*_IMAGE` that's one of + /// Pyo's must be one of the release's images it knows. 0.4.0 knew these. + /// A new image has to go in an env var named otherwise. + #[test] + fn manifests_pass_older_pyos_checks() { + const KNOWN_TO_0_4_0: &[&str] = &[ + "pyo", + "agent", + "builder", + "flows", + "waf", + "geoip", + "db-controller", + "kv", + "static", + "mysql-operator", + "mysql-backup", + "postgres", + ]; + let docs: Vec = [ + include_str!("../../../deploy/k8s/platform.yaml"), + include_str!("../../../deploy/k8s/pyo.yaml"), + ] + .join("\n---\n") + .split("\n---") + .filter_map(|d| serde_norway::from_str::(d).ok()) + .filter(|d| d.is_object()) + .collect(); + assert!(docs.len() > 5, "the manifests parse"); + let mut images = vec![]; + for d in &docs { + collect_images(d, &mut images); + } + let ours: Vec<&String> = images + .iter() + .filter(|i| i.starts_with("__IMAGES__/")) + .collect(); + assert!(ours.len() > 5, "{images:?}"); + for i in ours { + let name = i["__IMAGES__/".len()..].split(':').next().unwrap(); + assert!( + KNOWN_TO_0_4_0.contains(&name), + "{i}: Pyo 0.4.0 would refuse a release that has it. Name its env var otherwise." + ); + } + // The sandbox templates' overrides keep their names, which releases + // leave empty (scripts/release-manifest.ts). + let mut refs = vec![]; + for d in &docs { + collect_env(d, SANDBOX_BASE_ENV, &mut refs); + } + assert_eq!(refs, vec!["__SANDBOX_BASE_REF__".to_string()]); + let mut refs = vec![]; + for d in &docs { + collect_env(d, SANDBOX_DESKTOP_ENV, &mut refs); + } + assert_eq!(refs, vec!["__SANDBOX_DESKTOP_REF__".to_string()]); + } + #[test] fn min_from() { let mut r = release(); diff --git a/crates/pyo-proto/src/sandbox/mod.rs b/crates/pyo-proto/src/sandbox/mod.rs index 0156128..656c0ae 100644 --- a/crates/pyo-proto/src/sandbox/mod.rs +++ b/crates/pyo-proto/src/sandbox/mod.rs @@ -43,13 +43,45 @@ pub const DEFAULT_DISK_MIB: u32 = 2048; pub const MIN_DISK_MIB: u32 = 64; pub const MAX_DISK_MIB: u32 = 256 * 1024; pub const DEFAULT_TTL_SECS: u64 = 5 * 60; -pub const MAX_TTL_SECS: u64 = 24 * 60 * 60; +/// The longest lifetime a sandbox can have. A sandbox may also have none +/// (`ttl_secs` of `None`): it runs until it's deleted. +pub const MAX_TTL_SECS: u64 = 365 * 24 * 60 * 60; +/// How "no lifetime" is written where a lifetime is asked for. +pub const NO_TTL: &str = "none"; pub const MAX_ENV: usize = 128; pub const MAX_LABELS: usize = 32; /// The most a template's files may take, unpacked, unless Pyo says /// otherwise when it asks a server to prepare one. pub const DEFAULT_TEMPLATE_MAX_MIB: u32 = 10 * 1024; +// ── Image contract ───────────────────────────────────────────────────────── + +/// The label an image made for Pyo sandboxes carries: the contract it was +/// made for (upsetdev/pyo-sandbox-images). Images without it are anyone's +/// and are taken as they are. +pub const CONTRACT_LABEL: &str = "dev.pyo.sandbox.contract"; + +/// The image contracts this Pyo runs. Contract 1: pyo-envd runs commands as +/// root in /root with its own PATH, and runs /etc/pyo/start once at boot +/// when it's there. +pub const CONTRACTS: &[u32] = &[1]; + +/// Why an image whose [`CONTRACT_LABEL`] is `label` can't be used, or None +/// when it can. +pub fn contract_refusal(label: &str) -> Option { + let label = label.trim(); + match label.parse::() { + Ok(n) if CONTRACTS.contains(&n) => None, + Ok(n) => Some(format!( + "The image is made for Pyo sandbox contract {n}, and this server runs contract {}. Update Pyo and the server first.", + CONTRACTS.iter().max().copied().unwrap_or(1) + )), + Err(_) => Some(format!( + "The image's {CONTRACT_LABEL} label is \"{label}\", which isn't a contract number." + )), + } +} + // ── Runtime detection ────────────────────────────────────────────────────── /// Boot args of every sandbox VM, the detection test boot included. They @@ -256,9 +288,10 @@ pub struct SandboxSpec { pub memory_mib: u32, #[serde(default = "default_disk")] pub disk_mib: u32, - /// The sandbox is deleted this long after it starts. + /// The sandbox is deleted this long after it starts. `None` (null): + /// it runs until it's deleted. #[serde(default = "default_ttl")] - pub ttl_secs: u64, + pub ttl_secs: Option, /// Pause after this long without a command or connection. Pause and /// resume come in v0.4.x, so the daemon doesn't act on it yet. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -282,8 +315,8 @@ fn default_memory() -> u32 { fn default_disk() -> u32 { DEFAULT_DISK_MIB } -fn default_ttl() -> u64 { - DEFAULT_TTL_SECS +fn default_ttl() -> Option { + Some(DEFAULT_TTL_SECS) } impl SandboxSpec { @@ -300,7 +333,7 @@ impl SandboxSpec { cpus: DEFAULT_CPUS, memory_mib: DEFAULT_MEMORY_MIB, disk_mib: DEFAULT_DISK_MIB, - ttl_secs: DEFAULT_TTL_SECS, + ttl_secs: Some(DEFAULT_TTL_SECS), pause_when_idle_secs: None, env: BTreeMap::new(), network: NetworkMode::default(), @@ -338,9 +371,7 @@ impl SandboxSpec { MAX_DISK_MIB / 1024 )); } - if !(1..=MAX_TTL_SECS).contains(&self.ttl_secs) { - return Err("The lifetime must be 1 second to 24 hours.".into()); - } + check_ttl(self.ttl_secs)?; if self.env.len() > MAX_ENV { return Err(format!( "A sandbox takes at most {MAX_ENV} environment variables." @@ -408,8 +439,34 @@ pub fn is_label_key(s: &str) -> bool { .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'/')) } -/// Parses a lifetime like `90s`, `30m`, `1h` or `1h30m`. A bare number is -/// seconds. +/// Whether a lifetime is one a sandbox can have. `None` is no lifetime. +/// The message is fit to show the user. +pub fn check_ttl(ttl_secs: Option) -> Result<(), String> { + match ttl_secs { + Some(t) if !(1..=MAX_TTL_SECS).contains(&t) => { + Err("The lifetime must be 1 second to 365 days.".into()) + } + _ => Ok(()), + } +} + +/// When a sandbox with this lifetime ends, from `now` (epoch +/// milliseconds). `None`: never. +pub fn expiry(now: u64, ttl_secs: Option) -> Option { + ttl_secs.map(|t| now.saturating_add(t.saturating_mul(1000))) +} + +/// Parses a lifetime like [`parse_ttl`], or [`NO_TTL`] for none: +/// `Some(None)`. +pub fn parse_lifetime(s: &str) -> Option> { + if s.trim().eq_ignore_ascii_case(NO_TTL) { + return Some(None); + } + parse_ttl(s).map(Some) +} + +/// Parses a lifetime like `90s`, `30m`, `1h`, `1h30m` or `7d`. A bare +/// number is seconds. pub fn parse_ttl(s: &str) -> Option { let s = s.trim(); if s.is_empty() { @@ -482,7 +539,9 @@ pub struct SandboxStatus { pub created_at: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub started_at: Option, - pub expires_at: u64, + /// When its lifetime ends. `None`: it runs until it's deleted. + #[serde(default)] + pub expires_at: Option, /// From the start of the VM process to envd's ready signal, or to the /// first answer after a restore. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -622,9 +681,13 @@ pub struct TemplateBuild { /// While building: "queued", "pulling", "unpacking" or "snapshotting". #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, - /// Bytes pulled so far and in all, while pulling. + /// Bytes pulled so far, of `image_bytes` in all. #[serde(default)] pub pulled: u64, + /// Bytes of the pulled layers unpacked so far, of `image_bytes` in + /// all. Daemons before it leave it out. + #[serde(default)] + pub unpacked: u64, /// The image's compressed layers. #[serde(default)] pub image_bytes: u64, @@ -728,10 +791,11 @@ pub enum DaemonCommand { Resume { id: String, }, - /// Moves the end of the lifetime to `ttl_secs` from now. + /// Moves the end of the lifetime to `ttl_secs` from now. `None`: the + /// sandbox runs until it's deleted. SetTtl { id: String, - ttl_secs: u64, + ttl_secs: Option, }, Get { id: String, @@ -793,6 +857,12 @@ pub enum DaemonCommand { DeleteTemplate { template: String, }, + /// How far a template got, without the rest of the report: cheap + /// enough to ask every second while it's made. Daemons before it + /// refuse it, and Pyo falls back to the report. + Template { + template: String, + }, } /// A private key in a command: never printed. @@ -836,7 +906,8 @@ impl DaemonCommand { | DaemonCommand::SnapshotTemplate { .. } | DaemonCommand::SetEdgeCert { .. } | DaemonCommand::PrepareTemplate { .. } - | DaemonCommand::DeleteTemplate { .. } => None, + | DaemonCommand::DeleteTemplate { .. } + | DaemonCommand::Template { .. } => None, } } } @@ -983,6 +1054,22 @@ pub struct ExecResult { mod tests { use super::*; + #[test] + fn image_contracts() { + assert_eq!(contract_refusal("1"), None); + assert_eq!(contract_refusal(" 1\n"), None); + let two = contract_refusal("2").unwrap(); + assert!( + two.contains("contract 2") && two.contains("runs contract 1"), + "{two}" + ); + let bad = contract_refusal("one").unwrap(); + assert!(bad.contains("isn't a contract number"), "{bad}"); + for m in [two, bad] { + assert!(!m.contains(';'), "{m}"); + } + } + fn spec() -> SandboxSpec { SandboxSpec::new("sbx_abcdefghijklmnop", "team1", "node-22") } @@ -995,8 +1082,14 @@ mod tests { .unwrap(); assert_eq!(s.cpus, DEFAULT_CPUS); assert_eq!(s.memory_mib, DEFAULT_MEMORY_MIB); - assert_eq!(s.ttl_secs, DEFAULT_TTL_SECS); + assert_eq!(s.ttl_secs, Some(DEFAULT_TTL_SECS)); assert_eq!(s.network, NetworkMode::Internet); + let s: SandboxSpec = serde_json::from_str( + r#"{"id":"sbx_abcdefghijklmnop","team":"t","template":"node-22","ttlSecs":null}"#, + ) + .unwrap(); + assert_eq!(s.ttl_secs, None, "null is no lifetime"); + s.validate().unwrap(); assert_eq!(s.isolation, Isolation::Auto); s.validate().unwrap(); } @@ -1026,8 +1119,8 @@ mod tests { |s| s.cpus = MAX_CPUS + 1, |s| s.memory_mib = 64, |s| s.disk_mib = 1, - |s| s.ttl_secs = 0, - |s| s.ttl_secs = MAX_TTL_SECS + 1, + |s| s.ttl_secs = Some(0), + |s| s.ttl_secs = Some(MAX_TTL_SECS + 1), |s| { s.env.insert("1BAD".into(), "x".into()); }, @@ -1067,6 +1160,26 @@ mod tests { assert!(!is_template_name("")); } + #[test] + fn long_and_endless_lifetimes() { + assert_eq!(check_ttl(None), Ok(())); + assert_eq!(check_ttl(Some(1)), Ok(())); + assert_eq!(check_ttl(Some(MAX_TTL_SECS)), Ok(())); + assert!(check_ttl(Some(0)).is_err()); + assert!(check_ttl(Some(MAX_TTL_SECS + 1)).is_err()); + assert_eq!(parse_ttl("365d"), Some(MAX_TTL_SECS)); + assert_eq!(parse_lifetime("none"), Some(None)); + assert_eq!(parse_lifetime(" None "), Some(None)); + assert_eq!(parse_lifetime("30d"), Some(Some(30 * 86400))); + assert_eq!(parse_lifetime("never"), None); + assert_eq!(expiry(1_000, None), None); + assert_eq!( + expiry(1_000, Some(MAX_TTL_SECS)), + Some(1_000 + MAX_TTL_SECS * 1000) + ); + assert_eq!(expiry(u64::MAX, Some(10)), Some(u64::MAX)); + } + #[test] fn ttl_parsing() { assert_eq!(parse_ttl("90"), Some(90)); @@ -1138,7 +1251,11 @@ mod tests { }, DaemonCommand::SetTtl { id: "sbx_abcdefghijklmnop".into(), - ttl_secs: 60, + ttl_secs: Some(60), + }, + DaemonCommand::SetTtl { + id: "sbx_abcdefghijklmnop".into(), + ttl_secs: None, }, DaemonCommand::List, DaemonCommand::SnapshotTemplate { @@ -1161,6 +1278,9 @@ mod tests { DaemonCommand::DeleteTemplate { template: "stp_abc".into(), }, + DaemonCommand::Template { + template: "stp_abc".into(), + }, ]; for c in cmds { let j = serde_json::to_string(&c).unwrap(); diff --git a/crates/pyo-proto/src/system.rs b/crates/pyo-proto/src/system.rs index 3ff540e..496e16f 100644 --- a/crates/pyo-proto/src/system.rs +++ b/crates/pyo-proto/src/system.rs @@ -654,6 +654,10 @@ pub fn envoy_proxy(geoip_image: &str, public: bool) -> Value { "type": "Kubernetes", "kubernetes": { "envoyDaemonSet": { + // New before old, like the WAF: a server's new proxy + // is ready before its old one stops. Envoy has no + // host ports, so two fit on one server. + "strategy": { "type": "RollingUpdate", "rollingUpdate": { "maxSurge": 1, "maxUnavailable": 0 } }, "pod": { "affinity": gateway_affinity(), "tolerations": [{ "key": crate::GPU_RESOURCE, "operator": "Exists", "effect": "NoSchedule" }], @@ -674,6 +678,10 @@ pub fn envoy_proxy(geoip_image: &str, public: bool) -> Value { "envoyService": envoy_service(public), }, }, + // A stopping proxy drains its connections, then exits. Envoy's + // PreStop waits for the shutdown manager, which closes that wait + // at 10 s (the EOF it failed with), so the drain ends before. + "shutdown": { "drainTimeout": "6s", "minDrainDuration": "3s" }, // The firewall's country rules place visitors with this. "geoIP": { "provider": { "type": "MaxMind", "maxMind": { "countryDbSource": { "local": { "path": GEOIP_PATH } } } } }, "telemetry": { "accessLog": { "settings": [{ @@ -1481,6 +1489,22 @@ mod tests { } } + /// An update never leaves a server without a proxy: the new one starts + /// first, and the old one drains within the shutdown manager's 10 s. + #[test] + fn proxies_update_new_before_old() { + let p = envoy_proxy("g", true); + let ds = &p["spec"]["provider"]["kubernetes"]["envoyDaemonSet"]; + assert_eq!(ds["strategy"]["rollingUpdate"]["maxSurge"], 1); + assert_eq!(ds["strategy"]["rollingUpdate"]["maxUnavailable"], 0); + let secs = |v: &serde_json::Value| -> u64 { + v.as_str().unwrap().trim_end_matches('s').parse().unwrap() + }; + let shutdown = &p["spec"]["shutdown"]; + assert!(secs(&shutdown["minDrainDuration"]) < secs(&shutdown["drainTimeout"])); + assert!(secs(&shutdown["drainTimeout"]) + 2 < 10); + } + /// The tailnet listeners: their own ports, tailnet sources only. Public /// clients reach the same proxies, so the policy on the Gateway refuses /// them, whatever Host or SNI they send: it matches on the client's diff --git a/crates/pyo-sandbox/Cargo.toml b/crates/pyo-sandbox/Cargo.toml index 65fa1c7..8eb89ae 100644 --- a/crates/pyo-sandbox/Cargo.toml +++ b/crates/pyo-sandbox/Cargo.toml @@ -16,7 +16,7 @@ http-body-util = "0.1.5" hyper = { version = "1.12.0", features = ["client", "http1", "server"] } hyper-util = { version = "0.1.21", features = ["tokio"] } libc = "0.2.191" -pyo-proto = { version = "0.4.1", path = "../pyo-proto", features = ["sandbox-token"] } +pyo-proto = { version = "0.4.2", path = "../pyo-proto", features = ["sandbox-token"] } serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/crates/pyo-sandbox/src/api.rs b/crates/pyo-sandbox/src/api.rs index eb4ffc9..51ea506 100644 --- a/crates/pyo-sandbox/src/api.rs +++ b/crates/pyo-sandbox/src/api.rs @@ -17,6 +17,16 @@ //! | `POST /v1/sandboxes/{id}/ports`, `DELETE .../ports/{port}` | ports | //! | `GET PUT DELETE .../files`, `GET .../files/list`, `.../files/stat`, `POST .../files/mkdir` | files | //! | `GET /v1/sandboxes/{id}/connect` (WebSocket) | any scope, then each request is checked | +//! | `ANY /v1/sandboxes/{id}/web/{port}/...` | ports, for that sandbox only | +//! +//! `.../web/{port}/` serves a web page on a port of the sandbox to the +//! browser that opens it, over the tailnet, like the dashboard's Open +//! desktop (noVNC on 6080). The port needn't be open on the edge. The +//! token comes once as `?pyo_token=`, which sets an HttpOnly cookie for +//! that sandbox's pages (SameSite=Lax, so other sites can't use it) and +//! redirects, then in the cookie, or in `Authorization`. The rest of the +//! path goes to the port, WebSocket upgrades too. Pages there must use +//! relative links. //! //! The WebSocket speaks the envd wire protocol, one frame per binary //! message. The first message is envd's `Hello`. The daemon checks every @@ -52,7 +62,7 @@ use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; use axum::extract::{DefaultBodyLimit, Path, Query, State}; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Response}; -use axum::routing::{get, post}; +use axum::routing::{any, get, post}; use futures::{SinkExt, StreamExt}; use pyo_proto::sandbox::token::{self, Claims, Expect, LEEWAY_SECS, TokenError, VerifyingKey}; use pyo_proto::sandbox::wire::{self, ErrorKind, Frame, Kind, Request, Scope}; @@ -125,6 +135,9 @@ pub fn router(state: Arc) -> Router { .route("/v1/sandboxes/{id}/files/stat", get(stat_file)) .route("/v1/sandboxes/{id}/files/mkdir", post(mkdir)) .route("/v1/sandboxes/{id}/connect", get(connect)) + .route("/v1/sandboxes/{id}/web/{port}", any(web)) + .route("/v1/sandboxes/{id}/web/{port}/", any(web)) + .route("/v1/sandboxes/{id}/web/{port}/{*path}", any(web)) .layer(axum::middleware::from_fn(cors)) .with_state(state) } @@ -300,7 +313,7 @@ pub async fn run_command( DaemonCommand::SetTtl { id, ttl_secs } => { st.authorize(token, Some(&id), Scope::Lifecycle)?; DaemonResult::Sandbox { - sandbox: mgr.set_ttl(&id, ttl_secs)?, + sandbox: mgr.set_ttl(&id, ttl_secs).await?, } } DaemonCommand::OpenPort { id, port, public } => { @@ -380,6 +393,17 @@ pub async fn run_command( mgr.delete_template(&template).await?; DaemonResult::TemplateDeleted { template } } + DaemonCommand::Template { template } => { + st.authorize(token, None, Scope::Admin)?; + DaemonResult::Template { + template: mgr.template_progress(&template).ok_or_else(|| { + fail( + ErrorCode::NotFound, + format!("The sandbox template {template} isn't on this server."), + ) + })?, + } + } }) } @@ -509,10 +533,93 @@ async fn close_port( )) } +/// A port's web pages through the daemon: see the module's docs. +async fn web( + State(st): S, + Path(p): Path>, + mut req: axum::extract::Request, +) -> Response { + let fail = |status: StatusCode, msg: &str| crate::edge::text(status, msg).into_response(); + let id = p.get("id").cloned().unwrap_or_default(); + let Some(port) = p.get("port").and_then(|v| v.parse::().ok()) else { + return fail(StatusCode::NOT_FOUND, "That isn't a port."); + }; + let prefix = format!("/v1/sandboxes/{id}/web/{port}/"); + let Some((token, from_query)) = crate::edge::take_token(&mut req) else { + return fail(StatusCode::UNAUTHORIZED, "This needs a sandbox token."); + }; + // Only a token for this sandbox, never a server-wide one. + if !matches!(st.authorize(Some(&token), Some(&id), Scope::Ports), Ok(c) if c.sbx == id) { + return fail( + StatusCode::UNAUTHORIZED, + "The sandbox token isn't valid here. Open it again from Pyo.", + ); + } + let path = req.uri().path().to_owned(); + let rest = path.strip_prefix(&prefix); + if from_query || rest.is_none() { + // Into a cookie for this sandbox's pages, out of the address bar. + // Its pages are under the prefix, so it ends in a slash. + let query = req + .uri() + .query() + .map(|q| format!("?{q}")) + .unwrap_or_default(); + let to = format!("{prefix}{}{query}", rest.unwrap_or_default()); + let cookie = from_query.then(|| { + format!( + "{}={token}; Path=/v1/sandboxes/{id}/web/; HttpOnly; SameSite=Lax", + crate::edge::TOKEN_COOKIE + ) + }); + return crate::edge::redirect(StatusCode::SEE_OTHER, &to, cookie.as_deref()) + .into_response(); + } + match st.mgr.status(&id).map(|s| s.state) { + Ok(pyo_proto::sandbox::SandboxState::Running) => {} + Ok(pyo_proto::sandbox::SandboxState::Paused) => { + return fail(StatusCode::SERVICE_UNAVAILABLE, "The sandbox is paused."); + } + Ok(_) => { + return fail( + StatusCode::SERVICE_UNAVAILABLE, + "The sandbox isn't running.", + ); + } + Err(_) => { + return fail( + StatusCode::NOT_FOUND, + "There is no such sandbox on this server.", + ); + } + } + // The page sees itself at the root of the port, as from inside. + let pq = format!( + "/{}{}", + rest.unwrap_or_default(), + req.uri() + .query() + .map(|q| format!("?{q}")) + .unwrap_or_default() + ); + match pq.parse() { + Ok(uri) => *req.uri_mut() = uri, + Err(_) => return fail(StatusCode::BAD_REQUEST, "That address isn't valid."), + } + if let Ok(v) = header::HeaderValue::from_str(&format!("localhost:{port}")) { + req.headers_mut().insert(header::HOST, v); + } + crate::edge::proxy(&st, req, &id, port) + .await + .into_response() +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct TtlBody { - ttl_secs: u64, + /// Null: the sandbox runs until it's deleted. + #[serde(deserialize_with = "Option::deserialize")] + ttl_secs: Option, } async fn set_ttl( diff --git a/crates/pyo-sandbox/src/edge.rs b/crates/pyo-sandbox/src/edge.rs index 4a30dc5..750c6ed 100644 --- a/crates/pyo-sandbox/src/edge.rs +++ b/crates/pyo-sandbox/src/edge.rs @@ -457,60 +457,71 @@ impl Edge { for (k, v) in forwarded(req.headers(), peer, tls, &host) { req.headers_mut().insert(k, v); } - self.proxy(req, &id, port).await + proxy(&self.app, req, &id, port).await } +} - async fn proxy(&self, mut req: Request, id: &str, port: u16) -> Response { - let stream = match self.app.mgr.envd(id).await { - Ok((_, conn)) => conn.port_stream(port).await.ok(), - Err(_) => None, - }; - let Some(stream) = stream else { - return text( - StatusCode::BAD_GATEWAY, - &format!("Nothing answers on port {port} in the sandbox."), - ); - }; - let closed = || { - text( - StatusCode::BAD_GATEWAY, - "The sandbox port closed the connection.", - ) - }; - let Ok((mut send, conn)) = - hyper::client::conn::http1::handshake(TokioIo::new(stream)).await - else { - return closed(); - }; +/// Passes `req` to port `port` in sandbox `id` over envd, WebSocket and +/// other upgrades too, and its answer back. The edge serves ports this way, +/// and the daemon API too (`.../web/{port}/`, see `api`). +pub(crate) async fn proxy( + app: &AppState, + mut req: Request, + id: &str, + port: u16, +) -> Response +where + B: hyper::body::Body + Send + 'static, + B::Error: Into>, +{ + let stream = match app.mgr.envd(id).await { + Ok((_, conn)) => conn.port_stream(port).await.ok(), + Err(_) => None, + }; + let Some(stream) = stream else { + return text( + StatusCode::BAD_GATEWAY, + &format!("Nothing answers on port {port} in the sandbox."), + ); + }; + let closed = || { + text( + StatusCode::BAD_GATEWAY, + "The sandbox port closed the connection.", + ) + }; + let Ok((mut send, conn)) = + hyper::client::conn::http1::handshake::<_, B>(TokioIo::new(stream)).await + else { + return closed(); + }; + tokio::spawn(async move { + let _ = conn.with_upgrades().await; + }); + let client_upgrade = req + .headers() + .contains_key(header::UPGRADE) + .then(|| hyper::upgrade::on(&mut req)); + let Ok(mut res) = send.send_request(req).await else { + return closed(); + }; + if res.status() == StatusCode::SWITCHING_PROTOCOLS + && let Some(client) = client_upgrade + { + let guest = hyper::upgrade::on(&mut res); tokio::spawn(async move { - let _ = conn.with_upgrades().await; + if let (Ok(c), Ok(g)) = (client.await, guest.await) { + let _ = + tokio::io::copy_bidirectional(&mut TokioIo::new(c), &mut TokioIo::new(g)).await; + } }); - let client_upgrade = req - .headers() - .contains_key(header::UPGRADE) - .then(|| hyper::upgrade::on(&mut req)); - let Ok(mut res) = send.send_request(req).await else { - return closed(); - }; - if res.status() == StatusCode::SWITCHING_PROTOCOLS - && let Some(client) = client_upgrade - { - let guest = hyper::upgrade::on(&mut res); - tokio::spawn(async move { - if let (Ok(c), Ok(g)) = (client.await, guest.await) { - let _ = - tokio::io::copy_bidirectional(&mut TokioIo::new(c), &mut TokioIo::new(g)) - .await; - } - }); - } - res.map(|b| b.boxed()) } + res.map(|b| b.boxed()) } /// Takes the sandbox token out of the request, so the guest never sees it. /// Returns it and whether it came in the query. -fn take_token(req: &mut Request) -> Option<(String, bool)> { +pub(crate) fn take_token(req: &mut Request) -> Option<(String, bool)> { let mut found = None; let h = req.headers_mut(); if let Some(t) = h @@ -620,7 +631,7 @@ fn forwarded( v } -fn text(status: StatusCode, msg: &str) -> Response { +pub(crate) fn text(status: StatusCode, msg: &str) -> Response { let mut r = Response::new( Full::new(Bytes::from(format!("{msg}\n"))) .map_err(|e| match e {}) @@ -636,7 +647,7 @@ fn text(status: StatusCode, msg: &str) -> Response { r } -fn redirect(status: StatusCode, to: &str, cookie: Option<&str>) -> Response { +pub(crate) fn redirect(status: StatusCode, to: &str, cookie: Option<&str>) -> Response { let mut r = text(status, "Redirecting."); if let Ok(v) = HeaderValue::from_str(to) { r.headers_mut().insert(header::LOCATION, v); diff --git a/crates/pyo-sandbox/src/manager.rs b/crates/pyo-sandbox/src/manager.rs index 0b21a51..0cc0c8d 100644 --- a/crates/pyo-sandbox/src/manager.rs +++ b/crates/pyo-sandbox/src/manager.rs @@ -14,8 +14,8 @@ use std::time::Duration; use pyo_proto::sandbox::{ CapabilityReport, CapacityReport, DaemonError, ErrorCode, ExecRequest, ExecResult, Isolation, - MAX_TTL_SECS, NetworkMode, OpenPort, Runtime, SANDBOX_PROTOCOL, SandboxSpec, SandboxState, - SandboxStatus, ServerReport, TemplateReport, TemplateState, + NetworkMode, OpenPort, Runtime, SANDBOX_PROTOCOL, SandboxSpec, SandboxState, SandboxStatus, + ServerReport, TemplateReport, TemplateState, }; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; @@ -130,6 +130,8 @@ pub struct Manager { pub(crate) builds: StdMutex>>, /// One template build at a time. pub(crate) build_gate: tokio::sync::Semaphore, + /// Starting a build: two asks at once start one. + pub(crate) starting: StdMutex<()>, } impl Manager { @@ -150,6 +152,7 @@ impl Manager { edge_tls: StdMutex::new(None), builds: StdMutex::new(HashMap::new()), build_gate: tokio::sync::Semaphore::new(1), + starting: StdMutex::new(()), } } @@ -408,7 +411,7 @@ impl Manager { labels: spec.labels.clone(), created_at: now, started_at: None, - expires_at: now + spec.ttl_secs * 1000, + expires_at: pyo_proto::sandbox::expiry(now, spec.ttl_secs), start_ms: None, from_snapshot, memory_used: None, @@ -488,7 +491,7 @@ impl Manager { let now = crate::now_ms(); s.state = SandboxState::Running; s.started_at = Some(now); - s.expires_at = now + spec.ttl_secs * 1000; + s.expires_at = pyo_proto::sandbox::expiry(now, spec.ttl_secs); s.start_ms = Some(took.as_millis() as u32); }); self.persist(&sbx, Some(pid)); @@ -908,15 +911,18 @@ impl Manager { Ok(st) } - pub fn set_ttl(&self, id: &str, ttl_secs: u64) -> Result { - if !(1..=MAX_TTL_SECS).contains(&ttl_secs) { - return Err(err( - ErrorCode::InvalidSpec, - "The lifetime must be 1 second to 24 hours.", - )); - } + /// Ends the sandbox `ttl_secs` from now, or never. Saved to its record, + /// so a restarted daemon keeps it. + pub async fn set_ttl(&self, id: &str, ttl_secs: Option) -> Result { + pyo_proto::sandbox::check_ttl(ttl_secs).map_err(|m| err(ErrorCode::InvalidSpec, m))?; let sbx = self.find(id)?; - let st = sbx.update(|s| s.expires_at = crate::now_ms() + ttl_secs * 1000); + let vm = sbx.vm.lock().await; + if sbx.status().state.is_final() { + return Err(err(ErrorCode::Conflict, "The sandbox has ended.")); + } + let st = + sbx.update(|s| s.expires_at = pyo_proto::sandbox::expiry(crate::now_ms(), ttl_secs)); + self.persist(&sbx, vm.as_ref().map(Instance::pid)); Ok(st) } @@ -960,7 +966,7 @@ impl Manager { let now = crate::now_ms(); for sbx in self.all() { let st = sbx.status(); - if now >= st.expires_at { + if st.expires_at.is_some_and(|t| now >= t) { tracing::info!(id = %st.id, "sandbox lifetime is over"); let _ = self.delete(&st.id).await; continue; @@ -1112,3 +1118,98 @@ impl Manager { self.layout.root.clone() } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A manager over `root`, with a paused sandbox left there by a + /// previous daemon. A paused sandbox has no process, so it's adopted + /// without a VM. + async fn with_paused(root: &Path, id: &str) -> Manager { + let cfg: Config = serde_json::from_value(serde_json::json!({ + "serverId": "srv1", + "listen": "127.0.0.1:0", + "firecracker": "/nonexistent/firecracker", + "kernel": "/nonexistent/vmlinux", + "envd": "/nonexistent/pyo-envd", + "keys": [], + "root": root, + })) + .unwrap(); + let layout = Layout::new(root, false); + let fc = Firecracker { + firecracker: "/nonexistent/firecracker".into(), + kernel: "/nonexistent/vmlinux".into(), + jailer: None, + cpu_template: None, + serial: false, + }; + let capability = CapabilityReport { + runtime: Runtime::Firecracker, + ..Default::default() + }; + let dir = layout.vm_dir(id); + if !dir.join("sandbox.json").exists() { + std::fs::create_dir_all(&dir).unwrap(); + let mut spec = SandboxSpec::new(id, "tm_one", "node-22"); + spec.ttl_secs = Some(60); + let status = SandboxStatus { + id: id.into(), + team: "tm_one".into(), + template: "node-22".into(), + state: SandboxState::Paused, + runtime: Runtime::Firecracker, + cpus: 1, + memory_mib: 512, + disk_mib: 2048, + network: NetworkMode::None, + labels: Default::default(), + created_at: 1, + started_at: Some(1), + expires_at: pyo_proto::sandbox::expiry(crate::now_ms(), spec.ttl_secs), + start_ms: None, + from_snapshot: false, + memory_used: None, + error: None, + ports: Vec::new(), + }; + let rec = Record { + spec, + status, + pid: None, + lease: None, + }; + std::fs::write(dir.join("sandbox.json"), serde_json::to_vec(&rec).unwrap()).unwrap(); + } + let mgr = Manager::new(cfg, layout, fc, capability); + assert_eq!(mgr.adopt().await, 1); + mgr + } + + #[tokio::test] + async fn a_new_lifetime_outlives_a_daemon_restart() { + let root = std::env::temp_dir().join(format!("pyo-sbx-ttl-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let id = "sbx_abcdefghijklmnop"; + + // A year, then no lifetime at all: each is kept across restarts. + let mgr = with_paused(&root, id).await; + let year = pyo_proto::sandbox::MAX_TTL_SECS; + let st = mgr.set_ttl(id, Some(year)).await.unwrap(); + let want = st.expires_at.unwrap(); + assert!(want > crate::now_ms() + (year - 60) * 1000); + let mgr = with_paused(&root, id).await; + assert_eq!(mgr.status(id).unwrap().expires_at, Some(want)); + + assert_eq!(mgr.set_ttl(id, None).await.unwrap().expires_at, None); + let mgr = with_paused(&root, id).await; + assert_eq!(mgr.status(id).unwrap().expires_at, None); + mgr.sweep().await; + assert!(mgr.find(id).is_ok(), "no lifetime: the sweeper keeps it"); + + assert!(mgr.set_ttl(id, Some(0)).await.is_err()); + assert!(mgr.set_ttl(id, Some(year + 1)).await.is_err()); + let _ = std::fs::remove_dir_all(&root); + } +} diff --git a/crates/pyo-sandbox/src/oci.rs b/crates/pyo-sandbox/src/oci.rs index 627da25..368de2e 100644 --- a/crates/pyo-sandbox/src/oci.rs +++ b/crates/pyo-sandbox/src/oci.rs @@ -154,6 +154,8 @@ pub struct Image { /// The platform manifest's digest. pub digest: String, pub layers: Vec, + /// The image's config blob, when the manifest names one. + pub config: Option, } #[derive(Deserialize)] @@ -165,6 +167,27 @@ struct Manifest { manifests: Option>, #[serde(default)] layers: Option>, + #[serde(default)] + config: Option, +} + +/// The part of an image's config Pyo reads: its labels. +#[derive(Deserialize, Default)] +struct ImageConfig { + #[serde(default)] + config: Option, +} + +#[derive(Deserialize, Default)] +struct ImageConfigInner { + #[serde(default, rename = "Labels")] + labels: Option>, +} + +/// An image config's labels (none when it has none). +pub fn config_labels(body: &[u8]) -> anyhow::Result> { + let c: ImageConfig = serde_json::from_slice(body).context("reading the image config")?; + Ok(c.config.and_then(|c| c.labels).unwrap_or_default()) } #[derive(Deserialize)] @@ -265,7 +288,40 @@ impl Client { bail!("The image has a foreign layer ({}).", l.media_type); } } - Ok(Image { digest, layers }) + Ok(Image { + digest, + layers, + config: m.config, + }) + } + + /// The image's labels, from its config blob (a few KB), checked + /// against its digest. None when the manifest names no config. + pub async fn labels( + &mut self, + img: &Image, + ) -> anyhow::Result> { + let Some(c) = &img.config else { + return Ok(Default::default()); + }; + if c.size as usize > MAX_SMALL_BODY { + bail!("The image config is too big."); + } + let res = self + .get(&self.url(&format!("blobs/{}", c.digest)), None) + .await?; + let status = res.status(); + let body = read_small(res).await?; + if !status.is_success() { + bail!( + "The registry answered {status} for the image config: {}", + error_text(&body) + ); + } + if format!("sha256:{}", hex(&Sha256::digest(&body))) != c.digest { + bail!("The image config doesn't match its digest."); + } + config_labels(&body) } /// A manifest's body and digest. A digest reference is checked. @@ -712,6 +768,30 @@ mod tests { } } + #[test] + fn labels_from_the_config() { + let l = config_labels( + br#"{"architecture":"amd64","config":{"Env":["PATH=/bin"],"Labels":{"dev.pyo.sandbox.contract":"1","org.opencontainers.image.version":"24.04"}}}"#, + ) + .unwrap(); + assert_eq!( + l.get("dev.pyo.sandbox.contract").map(String::as_str), + Some("1") + ); + // No labels, or no config at all. + assert!( + config_labels(br#"{"config":{"Labels":null}}"#) + .unwrap() + .is_empty() + ); + assert!( + config_labels(br#"{"architecture":"arm64"}"#) + .unwrap() + .is_empty() + ); + assert!(config_labels(b"not json").is_err()); + } + #[test] fn challenges() { let (s, p) = parse_challenge( diff --git a/crates/pyo-sandbox/src/templates.rs b/crates/pyo-sandbox/src/templates.rs index 7e925be..543aa87 100644 --- a/crates/pyo-sandbox/src/templates.rs +++ b/crates/pyo-sandbox/src/templates.rs @@ -59,6 +59,8 @@ pub struct Record { pub struct Build { record: StdMutex, pulled: AtomicU64, + /// Compressed layer bytes unpacked so far. + unpacked: Arc, cancel: Arc, } @@ -70,6 +72,7 @@ impl Build { .unwrap_or_else(|p| p.into_inner()) .clone(); b.pulled = self.pulled.load(Ordering::Relaxed); + b.unpacked = self.unpacked.load(Ordering::Relaxed); b } @@ -78,6 +81,20 @@ impl Build { } } +/// Counts the bytes read through it, for the unpack's progress. +struct Counted { + inner: R, + n: Arc, +} + +impl std::io::Read for Counted { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let k = self.inner.read(buf)?; + self.n.fetch_add(k as u64, Ordering::Relaxed); + Ok(k) + } +} + /// Where a template's files are. struct Paths { dir: PathBuf, @@ -148,18 +165,33 @@ impl Manager { /// One template as the report shows it, or None when there's none by /// that name. pub fn template_report(&self, name: &str) -> Option { + self.template_report_with(name, true) + } + + /// How far a template got, for Pyo to ask every second while it's + /// made: a build in progress leaves out its size, which takes a walk + /// through every file it unpacked so far. + pub fn template_progress(&self, name: &str) -> Option { + self.template_report_with(name, false) + } + + fn template_report_with(&self, name: &str, building_bytes: bool) -> Option { let p = self.template_paths(name); let snapshots = storage::dir_names(&p.dir.join("snapshots")); - let bytes = disk_bytes(&p.dir, &mut Default::default()); if let Some(b) = self.build_of(name) { return Some(TemplateReport { name: name.into(), snapshots, state: TemplateState::Building, - bytes, + bytes: if building_bytes { + disk_bytes(&p.dir, &mut Default::default()) + } else { + 0 + }, build: Some(b.snapshot()), }); } + let bytes = disk_bytes(&p.dir, &mut Default::default()); if let Some(r) = read_record(&p.dir) { return Some(TemplateReport { name: name.into(), @@ -216,6 +248,9 @@ impl Manager { "This server can't run sandboxes, so it makes no templates.", )); } + // Two asks at once (Pyo's round and a sandbox's first use) start + // one build: the second finds the first's. + let _starting = self.starting.lock().unwrap_or_else(|p| p.into_inner()); if let Some(r) = self.template_report(name) { return Ok(r); } @@ -237,6 +272,7 @@ impl Manager { let build = Arc::new(Build { record: StdMutex::new(record), pulled: AtomicU64::new(0), + unpacked: Arc::new(AtomicU64::new(0)), cancel: Arc::new(AtomicBool::new(false)), }); self.builds @@ -339,6 +375,15 @@ impl Manager { let t0 = Instant::now(); let mut client = oci::Client::new(image, plain_http, auth); let img = client.resolve().await?; + // An image made for Pyo says which contract it was made for. One + // this server doesn't run would start but not work. + let labels = client.labels(&img).await?; + if let Some(why) = labels + .get(pyo_proto::sandbox::CONTRACT_LABEL) + .and_then(|l| pyo_proto::sandbox::contract_refusal(l)) + { + bail!(why); + } let total: u64 = img.layers.iter().map(|l| l.size).sum(); b.set(|r| { r.digest = Some(img.digest.clone()); @@ -383,14 +428,23 @@ impl Manager { let out = p.build().join("rootfs.ext4"); let envd = self.cfg.envd.clone(); let cancel = b.cancel.clone(); + let unpacked = b.unpacked.clone(); let out2 = out.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let mut u = Unpacker::new(&staging, max_bytes, cancel)?; + let mut done = 0; for (i, (media, f)) in files.iter().enumerate() { - let file = std::fs::File::open(f)?; + // A layer may end before its last bytes are read: each + // layer counts in full once applied. + done += std::fs::metadata(f)?.len(); + let file = Counted { + inner: std::fs::File::open(f)?, + n: unpacked.clone(), + }; let r = rootfs::decompress(media, std::io::BufReader::new(file))?; u.apply(r) .with_context(|| format!("unpacking layer {}", i + 1))?; + unpacked.store(done, Ordering::Relaxed); let _ = std::fs::remove_file(f); } u.add_file("sbin/pyo-envd", &envd, 0o755)?; @@ -508,3 +562,24 @@ impl Manager { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + #[test] + fn counted_reads_add_up() { + let n = Arc::new(AtomicU64::new(5)); + let mut r = Counted { + inner: &b"hello world"[..], + n: n.clone(), + }; + let mut buf = [0u8; 4]; + r.read_exact(&mut buf).unwrap(); + assert_eq!(n.load(Ordering::Relaxed), 9); + let mut rest = Vec::new(); + r.read_to_end(&mut rest).unwrap(); + assert_eq!(n.load(Ordering::Relaxed), 16); + } +} diff --git a/crates/pyo-sandbox/tests/e2e.rs b/crates/pyo-sandbox/tests/e2e.rs index 2d187ee..9223d88 100644 --- a/crates/pyo-sandbox/tests/e2e.rs +++ b/crates/pyo-sandbox/tests/e2e.rs @@ -368,6 +368,50 @@ async fn edge_proxy(state: &Arc, keys: &Keys, run: &Path, id: &str) { let admin = format!("Authorization: Bearer {}", keys.admin()); assert_eq!(curl(&url(3000, true), &cert, &["-H", &admin]).await.0, 401); + // The same page through the daemon's API (the dashboard's Open + // desktop), with this sandbox's ports token only. + let web = format!("/v1/sandboxes/{id}/web/3000/"); + let (s, b) = call(&app, "GET", &web, Some(&ports), Body::empty()).await; + assert_eq!( + (s.as_u16(), String::from_utf8_lossy(&b).trim()), + (200, "hello-edge") + ); + assert_eq!(call(&app, "GET", &web, None, Body::empty()).await.0, 401); + let admin = keys.admin(); + assert_eq!( + call(&app, "GET", &web, Some(&admin), Body::empty()).await.0, + 401 + ); + let res = app + .clone() + .oneshot( + HttpRequest::builder() + .uri(format!("{web}?a=1&pyo_token={ports}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), 303); + assert_eq!(res.headers()["location"], format!("{web}?a=1")); + let set = res.headers()["set-cookie"].to_str().unwrap(); + assert!( + set.contains(&format!("Path=/v1/sandboxes/{id}/web/")) && set.contains("HttpOnly"), + "{set}" + ); + let res = app + .clone() + .oneshot( + HttpRequest::builder() + .uri(&web) + .header("cookie", format!("pyo_token={ports}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), 200); + // Nothing listening, closing, and names that aren't sandbox ports. open(3999, true).await; assert_eq!(curl(&url(3999, true), &cert, &[]).await.0, 502); @@ -429,6 +473,45 @@ async fn edge_proxy(state: &Arc, keys: &Keys, run: &Path, id: &str) { echo.extend_from_slice(&buf[..n]); } assert_eq!(echo, b"ping through the edge\n"); + + // And through the daemon's API (noVNC's WebSocket on Open desktop). + let api = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let api_port = api.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = axum::serve(api, app.clone()).await; + }); + let mut c = tokio::net::TcpStream::connect(("127.0.0.1", api_port)) + .await + .unwrap(); + let req = format!( + "GET /v1/sandboxes/{id}/web/4000/ HTTP/1.1\r\nHost: daemon\r\nCookie: pyo_token={ports}\r\nConnection: Upgrade\r\nUpgrade: echo\r\n\r\n" + ); + c.write_all(req.as_bytes()).await.unwrap(); + let mut got = Vec::new(); + while !got.windows(4).any(|w| w == b"\r\n\r\n") { + let n = tokio::time::timeout(Duration::from_secs(5), c.read(&mut buf)) + .await + .unwrap() + .unwrap(); + assert!(n > 0, "closed: {}", String::from_utf8_lossy(&got)); + got.extend_from_slice(&buf[..n]); + } + assert!( + got.starts_with(b"HTTP/1.1 101"), + "{}", + String::from_utf8_lossy(&got) + ); + c.write_all(b"ping through the daemon\n").await.unwrap(); + let mut echo = Vec::new(); + while !echo.ends_with(b"\n") { + let n = tokio::time::timeout(Duration::from_secs(5), c.read(&mut buf)) + .await + .unwrap() + .unwrap(); + assert!(n > 0); + echo.extend_from_slice(&buf[..n]); + } + assert_eq!(echo, b"ping through the daemon\n"); } /// A self-signed certificate for `*.sbx.test`, as (cert, key) PEM. diff --git a/crates/pyo-sandbox/tests/gvisor_e2e.rs b/crates/pyo-sandbox/tests/gvisor_e2e.rs index 0451864..b19f2c9 100644 --- a/crates/pyo-sandbox/tests/gvisor_e2e.rs +++ b/crates/pyo-sandbox/tests/gvisor_e2e.rs @@ -441,7 +441,7 @@ async fn gvisor_end_to_end() { assert!(connects(&mgr, &paused_c.id, "1.1.1.1", 53).await); // ── The lifetime ends a sandbox ── - mgr.set_ttl(&paused_c.id, 1).unwrap(); + mgr.set_ttl(&paused_c.id, Some(1)).await.unwrap(); tokio::time::sleep(Duration::from_millis(1500)).await; mgr.sweep().await; assert!(mgr.find(&paused_c.id).is_err(), "its lifetime is over"); diff --git a/crates/pyo-sandbox/tests/template_e2e.rs b/crates/pyo-sandbox/tests/template_e2e.rs index 64e0eac..d48a876 100644 --- a/crates/pyo-sandbox/tests/template_e2e.rs +++ b/crates/pyo-sandbox/tests/template_e2e.rs @@ -304,6 +304,8 @@ async fn makes_a_template_from_a_public_image() { note("total_ms", (b.finished_at.unwrap() - b.started_at).into()); note("snapshots", serde_json::json!(done.snapshots)); assert_eq!(b.pulled, b.image_bytes); + // Every layer counts in full once unpacked. + assert_eq!(b.unpacked, b.image_bytes); assert!( done.has_snapshot(NetworkMode::None, 1, 512, 2048), "{done:?}" @@ -422,9 +424,9 @@ fn t0(t: &TemplateReport) -> u64 { pyo_sandbox::now_ms().saturating_sub(t.build.as_ref().map(|b| b.started_at).unwrap_or(0)) } -/// Pyo's built-in template `base` (deploy/images/sandbox-base.Dockerfile), -/// from the image named by `PYO_SANDBOX_E2E_BASE_IMAGE`: CI builds it and -/// pushes it to a registry on 127.0.0.1. Skipped without it. +/// Pyo's built-in template `base` (upsetdev/pyo-sandbox-images), from the +/// image named by `PYO_SANDBOX_E2E_BASE_IMAGE`: CI takes the one Pyo ships +/// with, from ghcr.io. Skipped without it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn makes_the_base_template() { if !enabled() { @@ -479,13 +481,13 @@ async fn makes_the_base_template() { assert!(s.from_snapshot); for (cmd, want) in [ ("git --version", "git version 2."), - ("python3 --version", "Python 3.13"), + ("python3 --version", "Python 3.12"), ("uv --version", "uv 0."), - ("node --version", "v26."), + ("node --version", "v24."), ("pnpm --version", "12."), ("npm --version", ""), ("gcc --version", "gcc"), - ("head -1 /etc/os-release", "Debian GNU/Linux 13"), + ("head -1 /etc/os-release", "Ubuntu 24.04"), ] { let out = mgr.exec(&spec.id, &shell(cmd)).await.unwrap(); println!("base e2e {cmd}: {}", out.stdout.trim()); diff --git a/crates/pyo-static/Cargo.toml b/crates/pyo-static/Cargo.toml index b68440d..aa8b545 100644 --- a/crates/pyo-static/Cargo.toml +++ b/crates/pyo-static/Cargo.toml @@ -11,7 +11,7 @@ anyhow.workspace = true axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json"] } futures.workspace = true hex = "0.4.3" -pyo-proto = { version = "0.4.1", path = "../pyo-proto" } +pyo-proto = { version = "0.4.2", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] } serde.workspace = true serde_json.workspace = true diff --git a/dash/scripts/sandbox-templates.test.ts b/dash/scripts/sandbox-templates.test.ts index ca6cb58..f62c6cf 100644 --- a/dash/scripts/sandbox-templates.test.ts +++ b/dash/scripts/sandbox-templates.test.ts @@ -1,21 +1,50 @@ -// Pyo's built-in sandbox template in the dashboard: how its image reads, -// and that new sandboxes start on it. +// Pyo's built-in sandbox templates in the dashboard: how its image reads, +// that new sandboxes start on it, and the servers a new sandbox can go to. // bun test scripts/sandbox-templates.test.ts import { expect, test } from "bun:test"; -import { defaultTemplate, imageRepo, templateImage } from "../src/lib/pyo/sandboxTemplates"; +import { + defaultTemplate, + desktopUrl, + fieldsSwapped, + freeLabel, + gettingReady, + imageParts, + imageRepo, + minSize, + overallPercent, + pyosFirst, + serverProgress, + serverRefusal, + shortImage, + templateImage, + templateOs, + templateProgress, + templateStatus, + usable, +} from "../src/lib/pyo/sandboxTemplates"; + +test("the new template form notices a swapped name and image", () => { + expect(fieldsSwapped("frans/sandbox-base", "sandbox-base")).toBe(true); + expect(fieldsSwapped("node:22", "node")).toBe(true); + expect(fieldsSwapped("node-22", "node:22")).toBe(false); + expect(fieldsSwapped("base", "frans/sandbox-base")).toBe(false); + expect(fieldsSwapped("node:22", "")).toBe(false); +}); const base = { name: "base", state: "ready", builtin: true, source: { - image: "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - os: "Debian 13", + image: + "ghcr.io/upsetdev/pyo-sandbox-images/base@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + os: "Ubuntu 24.04", }, }; test("base reads as what it runs and where it comes from", () => { - expect(templateImage(base)).toBe("Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base"); + expect(templateImage(base)).toBe("Ubuntu 24.04 · ghcr.io/upsetdev/pyo-sandbox-images/base"); + expect(templateImage({ ...base, source: { os: "Ubuntu 24.04" } })).toBe("Ubuntu 24.04"); expect(templateImage({ source: { image: "node:22" } })).toBe("node:22"); expect(templateImage({ source: null })).toBe(""); }); @@ -26,11 +55,139 @@ test("images lose their tag and digest, not a registry's port", () => { expect(imageRepo("127.0.0.1:5005/pyo/x:dev")).toBe("127.0.0.1:5005/pyo/x"); }); -test("new sandboxes start on base once it's ready", () => { +test("new sandboxes start on base, which servers make on first use", () => { const node = { name: "node-22", state: "ready" }; expect(defaultTemplate([node, base])).toBe("base"); - expect(defaultTemplate([node, { ...base, state: "building" }])).toBe("node-22"); - expect(defaultTemplate([{ ...base, state: "building" }])).toBe(""); - // A team's own template named base isn't Pyo's. + // Not on any server yet, or being made: still the default. + expect(defaultTemplate([node, { ...base, state: "pending" }])).toBe("base"); + expect(defaultTemplate([{ ...base, state: "building" }])).toBe("base"); + // Pyo has no image for it yet. + const none = { ...base, state: "unavailable" }; + expect(usable(none)).toBe(false); + expect(defaultTemplate([node, none])).toBe("node-22"); + expect(defaultTemplate([none])).toBe(""); + // A team's template waits until it's built. One named base isn't Pyo's. + expect(usable({ name: "node-22", state: "building" })).toBe(false); expect(defaultTemplate([node, { name: "base", state: "ready" }])).toBe("node-22"); }); + +test("base's status says it's ready to use", () => { + expect(templateStatus(base)).toBe("Ready to use"); + expect(templateStatus({ ...base, state: "pending" })).toBe("Ready to use"); + expect(templateStatus({ ...base, state: "unavailable" })).toBe("Not available yet"); + expect(templateStatus({ name: "node-22", state: "pending" })).toBe("Pending"); + expect(gettingReady("base", "sandbox-1")).toBe("Getting base ready on sandbox-1…"); + expect(gettingReady("base", null)).toBe("Getting base ready…"); +}); + +test("a server is offered with its room, or why it can't take the sandbox", () => { + const GB = 1024 ** 3; + const ok = { + state: "ready", + enabled: true, + capability: { runtime: "firecracker", network: true }, + capacity: { memoryAvailable: 7.2 * GB, memoryReserved: GB }, + }; + expect(serverRefusal(ok, 2048, "internet")).toBe(""); + expect(freeLabel(6.2 * GB)).toBe("6.2 GB free"); + expect(freeLabel(512 * 1024 ** 2)).toBe("512 MB free"); + expect(serverRefusal(ok, 8192, "internet")).toBe("No room"); + expect(serverRefusal({ ...ok, state: "joining" }, 512, "none")).toBe("Setting up"); + expect(serverRefusal({ ...ok, state: "unreachable" }, 512, "none")).toBe("Not reachable"); + expect(serverRefusal({ ...ok, enabled: false }, 512, "none")).toBe("Turned off"); + const offline = { ...ok, capability: { runtime: "gvisor", network: false } }; + expect(serverRefusal(offline, 512, "internet")).toBe("No networking"); + expect(serverRefusal(offline, 512, "none")).toBe(""); +}); + +const desktop = { + name: "desktop", + state: "ready", + builtin: true, + source: { image: "ghcr.io/upsetdev/pyo-sandbox-images/desktop@sha256:ab", os: "Ubuntu 24.04" }, +}; + +test("desktop comes right after base, made on first use too", () => { + const node = { name: "alpine", state: "ready" }; + expect(pyosFirst([node, desktop, base]).map((t) => t.name)).toEqual(["base", "desktop", "alpine"]); + expect(defaultTemplate([desktop, base])).toBe("base"); + expect(usable({ ...desktop, state: "pending" })).toBe(true); + expect(usable({ ...desktop, state: "unavailable" })).toBe(false); + expect(templateStatus({ ...desktop, state: "pending" })).toBe("Ready to use"); + // One named desktop isn't Pyo's. + expect(usable({ name: "desktop", state: "building" })).toBe(false); + expect(templateImage(desktop)).toBe("Ubuntu 24.04 · ghcr.io/upsetdev/pyo-sandbox-images/desktop"); +}); + +test("the template tooltip says what it runs and a short image", () => { + expect(templateOs(base)).toBe("Ubuntu 24.04"); + expect(templateOs({ name: "base", builtin: true })).toBe("Ubuntu 24.04"); + expect(templateOs({ name: "node-22", source: { image: "node:22" } })).toBe(""); + const digest = `sha256:ad5c5976${"0".repeat(56)}`; + expect(shortImage(`ghcr.io/upsetdev/pyo-sandbox-images/desktop@${digest}`)).toBe( + "ghcr.io/upsetdev/pyo-sandbox-images/desktop@sha256:ad5c59760000…", + ); + expect(shortImage("node:22")).toBe("node:22"); + expect(imageParts("ghcr.io/a/b:1@sha256:ab")).toEqual(["ghcr.io/", "a/", "b:", "1@", "sha256:", "ab"]); + expect(imageParts("node")).toEqual(["node"]); + expect(imageParts("")).toEqual([]); +}); + +test("the desktop needs room, and opens through the daemon", () => { + expect(minSize(desktop)).toEqual({ cpus: 2, memoryMib: 2048 }); + expect(minSize(base)).toBe(null); + expect(minSize(undefined)).toBe(null); + const url = new URL(desktopUrl("http://100.64.0.2:7070/", "sbx_abc", "pyosbx1.x.y")); + expect(url.origin + url.pathname).toBe("http://100.64.0.2:7070/v1/sandboxes/sbx_abc/web/6080/vnc.html"); + expect(url.searchParams.get("pyo_token")).toBe("pyosbx1.x.y"); + expect(url.searchParams.get("autoconnect")).toBe("1"); + expect(url.searchParams.has("app")).toBe(false); + const chromium = new URL(desktopUrl("http://100.64.0.2:7070", "sbx_abc", "t", "chromium")); + expect(chromium.searchParams.get("app")).toBe("chromium"); + // noVNC finds its WebSocket beside the page. + expect(url.searchParams.has("path")).toBe(false); +}); + +test("a server making a template says how far it got", () => { + const b = (phase: string | null, more = {}) => ({ state: "building", phase, imageBytes: 1000, ...more }); + expect(serverProgress({ state: "ready" })).toBe(null); + expect(serverProgress(b("queued"))).toEqual({ label: "Waiting", percent: null }); + expect(serverProgress(b("pulling", { pulled: 0 }))).toEqual({ label: "Pulling 0%", percent: 0 }); + expect(serverProgress(b("pulling", { pulled: 425 }))).toEqual({ label: "Pulling 42%", percent: 42 }); + // Never 100 while it still runs. + expect(serverProgress(b("pulling", { pulled: 1000 }))?.percent).toBe(99); + // Before the manifest says the size: no end known. + expect(serverProgress(b("pulling", { imageBytes: 0 }))).toEqual({ label: "Pulling", percent: null }); + expect(serverProgress(b("unpacking", { unpacked: 800 }))).toEqual({ label: "Unpacking 80%", percent: 80 }); + // A server too old to count what it unpacked. + expect(serverProgress(b("unpacking", { unpacked: 0 }))).toEqual({ label: "Unpacking", percent: null }); + expect(serverProgress(b("snapshotting"))).toEqual({ label: "Starting", percent: null }); + expect(serverProgress(b(null))).toEqual({ label: "Starting", percent: null }); +}); + +test("a template shows its slowest server", () => { + const ready = { state: "ready" }; + const pulling = { state: "building", phase: "pulling", pulled: 300, imageBytes: 1000 }; + const pulling2 = { ...pulling, pulled: 700 }; + const unpacking = { state: "building", phase: "unpacking", unpacked: 100, imageBytes: 1000 }; + expect(templateProgress({ servers: [ready] })).toBe(null); + expect(templateProgress({ servers: [] })).toBe(null); + expect(templateProgress({ servers: [ready, unpacking] })).toEqual({ label: "Unpacking 10%", percent: 10, servers: 1 }); + expect(templateProgress({ servers: [unpacking, pulling2, pulling] })).toEqual({ + label: "Pulling 30%", + percent: 30, + servers: 3, + }); + expect(templateProgress({ servers: [{ state: "building", phase: "queued" }, pulling] })?.label).toBe("Waiting"); +}); + +test("the create button counts up over every step", () => { + const b = (phase: string, more = {}) => ({ state: "building", phase, imageBytes: 1000, ...more }); + expect(overallPercent({ state: "ready" })).toBe(null); + expect(overallPercent(b("queued"))).toBe(0); + expect(overallPercent(b("pulling", { pulled: 500 }))).toBe(30); + expect(overallPercent(b("unpacking", { unpacked: 500 }))).toBe(75); + expect(overallPercent(b("unpacking", { unpacked: 0 }))).toBe(60); + expect(overallPercent(b("snapshotting"))).toBe(90); + expect(gettingReady("desktop", "sandbox-1", 42)).toBe("Getting desktop ready on sandbox-1… 42%"); +}); diff --git a/dash/src/components/pyo/ClusterChecklist.tsx b/dash/src/components/pyo/ClusterChecklist.tsx index 3f72729..0535293 100644 --- a/dash/src/components/pyo/ClusterChecklist.tsx +++ b/dash/src/components/pyo/ClusterChecklist.tsx @@ -2,6 +2,7 @@ import { plural } from "~/lib/pyo/format"; import { serversOf, usePyo } from "~/lib/pyo/store"; import { CodeText } from "./common"; import { SetupChecklist, type SetupCheck } from "./SetupChecklist"; +import { TailnetLockWait } from "./TailnetLock"; /** What the agent has reported so far, ticked off as it comes in. */ export function ClusterChecklist({ clusterId }: { clusterId: string }) { @@ -61,11 +62,15 @@ export function ClusterChecklist({ clusterId }: { clusterId: string }) { ]; return ( - - i >= 2 ? { ...ch, missing: connected && !ch.done } : ch, - )} - /> + <> + + i >= 2 ? { ...ch, missing: connected && !ch.done } : ch, + )} + /> + {/* Its hostname isn't known: any of Pyo's machines that waits. */} + {connected ? null : } + ); } diff --git a/dash/src/components/pyo/Hetzner.tsx b/dash/src/components/pyo/Hetzner.tsx index 4884e45..15ea6a1 100644 --- a/dash/src/components/pyo/Hetzner.tsx +++ b/dash/src/components/pyo/Hetzner.tsx @@ -12,7 +12,7 @@ import { ConfirmDialog } from "~/components/ui/Dialog"; import { Field, Input, Select, Textarea } from "~/components/ui/Field"; import { Callout } from "~/components/ui/Panel"; import { StatusNote } from "~/components/ui/StatusNote"; -import { CopyButton } from "~/components/ui/Table"; +import { TailnetLockSign } from "~/components/pyo/TailnetLock"; import { site } from "~/config/site"; import { fetchCloudOptions } from "~/lib/pyo/backend"; import { api, usePyo, type PyoState } from "~/lib/pyo/store"; @@ -443,19 +443,7 @@ export function CloudProgress({ server }: { server: CloudServer }) { ))} - {server.signCommand ? ( - - - Your tailnet has Tailnet Lock on. On a signing device, run: - - - {server.signCommand} - - - - - - ) : null} + {server.signCommand ? : null} {server.error ?

{server.error}

: null} ); diff --git a/dash/src/components/pyo/TailnetLock.tsx b/dash/src/components/pyo/TailnetLock.tsx new file mode 100644 index 0000000..0567189 --- /dev/null +++ b/dash/src/components/pyo/TailnetLock.tsx @@ -0,0 +1,48 @@ +import { Callout } from "~/components/ui/Panel"; +import { CopyButton } from "~/components/ui/Table"; +import { fetchTailnetLock } from "~/lib/pyo/backend"; +import { useQuery } from "~/lib/pyo/query"; +import { roleIn, usePyo } from "~/lib/pyo/store"; + +/** The command a signing device runs to let a new machine on the tailnet. */ +export function TailnetLockSign({ command, name }: { command: string; name?: string }) { + return ( + + + {name ? `${name} is waiting for a Tailnet Lock signature.` : "Waiting for a Tailnet Lock signature."} On a + signing device, run: + + {command} + + + + + ); +} + +/** + * While a new machine joins: the sign command when Tailnet Lock keeps it + * off the tailnet. `name` is its hostname. Without one (a new cluster, + * whose hostname Pyo doesn't know), every machine of Pyo's that waits. + */ +export function TailnetLockWait({ name }: { name?: string }) { + const s = usePyo(); + const team = s.currentTeamId; + const role = roleIn(s); + const allowed = role === "owner" || role === "admin"; + const q = useQuery( + `tailnet-lock:${team}:${allowed}`, + () => (allowed ? fetchTailnetLock(team) : Promise.resolve({ waiting: [] })), + { refreshMs: 10_000 }, + ); + if (q.status !== "success") return null; + const waiting = q.data.waiting.filter((w) => !name || w.name.toLowerCase() === name.toLowerCase()); + if (!waiting.length) return null; + return ( +
+ {waiting.map((w) => ( + + ))} +
+ ); +} diff --git a/dash/src/components/pyo/sandbox/Lifetime.tsx b/dash/src/components/pyo/sandbox/Lifetime.tsx new file mode 100644 index 0000000..bc652cf --- /dev/null +++ b/dash/src/components/pyo/sandbox/Lifetime.tsx @@ -0,0 +1,80 @@ +import { Input } from "~/components/ui/Field"; +import { Select } from "~/components/ui/Select"; +import { LIFETIME_UNITS, MAX_LIFETIME_SECS, type LifetimeUnit } from "~/lib/pyo/sandbox"; + +// A lifetime typed in: a whole number and a unit. The new sandbox form and +// the team's sandbox limits use it. + +export type CustomLifetime = { amount: string; unit: LifetimeUnit }; + +const unitSecs = (u: LifetimeUnit) => LIFETIME_UNITS.find((x) => x.value === u)!.secs; + +/** Its seconds, or null when the amount isn't a whole number above 0. */ +export function customSecs(c: CustomLifetime) { + const n = Number(c.amount); + if (c.amount.trim() === "" || !Number.isInteger(n) || n < 1) return null; + return n * unitSecs(c.unit); +} + +/** How Pyo reads it, like "3d". */ +export const customTtl = (c: CustomLifetime) => `${Number(c.amount)}${c.unit}`; + +/** Seconds as the largest unit they divide into. */ +export function toCustom(secs: number, units: readonly LifetimeUnit[]): CustomLifetime { + const unit = units.find((u) => secs % unitSecs(u) === 0) ?? units[units.length - 1]; + return { amount: String(Math.max(1, Math.round(secs / unitSecs(unit)))), unit }; +} + +/** Why it can't be used, or null. */ +export function customError(c: CustomLifetime) { + const secs = customSecs(c); + if (secs === null) return "Enter a whole number."; + if (secs > MAX_LIFETIME_SECS) return "365 days at most."; + return null; +} + +export function CustomLifetimeInput({ + value, + onChange, + units, + id, + ...aria +}: { + value: CustomLifetime; + onChange: (v: CustomLifetime) => void; + units: readonly LifetimeUnit[]; + id?: string; + "aria-label"?: string; + "aria-invalid"?: boolean; + "aria-describedby"?: string; +}) { + return ( +
+
+ onChange({ ...value, amount: e.target.value })} + /> +
+
+ +
+
+ ); +} diff --git a/dash/src/components/pyo/sandbox/SandboxLimits.tsx b/dash/src/components/pyo/sandbox/SandboxLimits.tsx index 4e2d834..da897ee 100644 --- a/dash/src/components/pyo/sandbox/SandboxLimits.tsx +++ b/dash/src/components/pyo/sandbox/SandboxLimits.tsx @@ -2,23 +2,49 @@ import { useEffect, useState } from "react"; import { Button } from "~/components/ui/Button"; import { Field, Input } from "~/components/ui/Field"; import { Panel } from "~/components/ui/Panel"; +import { Select } from "~/components/ui/Select"; import { op } from "~/lib/pyo/backend"; import type { SandboxLimits } from "~/lib/pyo/sandbox"; import { toast } from "~/lib/pyo/toast"; import { useSandboxes } from "./common"; +import { CustomLifetimeInput, customError, customSecs, toCustom, type CustomLifetime } from "./Lifetime"; // A team's sandbox limits. Only the installation's admin sets them -// (setSandboxLimits). Memory and disk show in GB, the lifetime in hours. +// (setSandboxLimits). Memory and disk show in GB. The longest lifetime is a +// preset, a custom one up to 365 days, or no limit. -type Form = { running: string; cpus: string; memory: string; disk: string; hours: string }; +const MAX_LIFETIMES = [ + { value: "3600", label: "1 hour" }, + { value: "86400", label: "1 day" }, + { value: "604800", label: "7 days" }, + { value: "2592000", label: "30 days" }, + { value: "31536000", label: "365 days" }, +] as const; -const toForm = (l: SandboxLimits): Form => ({ - running: String(l.maxRunning), - cpus: String(l.maxCpus), - memory: String(+(l.maxMemoryMib / 1024).toFixed(1)), - disk: String(+(l.maxDiskMib / 1024).toFixed(1)), - hours: String(+(l.maxTtlSecs / 3600).toFixed(2)), -}); +const UNITS = ["m", "h", "d"] as const; + +type Form = { + running: string; + cpus: string; + memory: string; + disk: string; + /** A preset's seconds, "custom" or "none". */ + ttl: string; + custom: CustomLifetime; +}; + +const toForm = (l: SandboxLimits): Form => { + const t = l.maxTtlSecs; + const preset = t !== null && MAX_LIFETIMES.some((x) => x.value === String(t)); + return { + running: String(l.maxRunning), + cpus: String(l.maxCpus), + memory: String(+(l.maxMemoryMib / 1024).toFixed(1)), + disk: String(+(l.maxDiskMib / 1024).toFixed(1)), + ttl: t === null ? "none" : preset ? String(t) : "custom", + custom: t === null || preset ? { amount: "2", unit: "d" } : toCustom(t, UNITS), + }; +}; export function SandboxLimitsPanel({ teamId }: { teamId: string }) { const q = useSandboxes(teamId); @@ -30,24 +56,26 @@ export function SandboxLimitsPanel({ teamId }: { teamId: string }) { }, [saved, f]); if (!f || !saved) return null; const num = (v: string) => Number(v); - const bad = Object.values(f).some((v) => v.trim() === "" || !Number.isFinite(num(v)) || num(v) < 0); - const tooLong = num(f.hours) > 24; + const numbers = [f.running, f.cpus, f.memory, f.disk]; + const bad = numbers.some((v) => v.trim() === "" || !Number.isFinite(num(v)) || num(v) < 0); + const ttlError = f.ttl === "custom" ? customError(f.custom) : null; + const maxTtlSecs = f.ttl === "none" ? null : f.ttl === "custom" ? customSecs(f.custom) : num(f.ttl); const dirty = JSON.stringify(f) !== JSON.stringify(toForm(saved)); - const set = (k: keyof Form) => (e: { target: { value: string } }) => setF({ ...f, [k]: e.target.value }); + const set = (k: "running" | "cpus" | "memory" | "disk") => (e: { target: { value: string } }) => + setF({ ...f, [k]: e.target.value }); const save = async () => { setBusy(true); try { - await op("setSandboxLimits", { - teamId, - limits: { - maxRunning: Math.round(num(f.running)), - maxCpus: Math.round(num(f.cpus)), - maxMemoryMib: Math.round(num(f.memory) * 1024), - maxDiskMib: Math.round(num(f.disk) * 1024), - maxTtlSecs: Math.round(num(f.hours) * 3600), - }, - }); + const limits: SandboxLimits = { + maxRunning: Math.round(num(f.running)), + maxCpus: Math.round(num(f.cpus)), + maxMemoryMib: Math.round(num(f.memory) * 1024), + maxDiskMib: Math.round(num(f.disk) * 1024), + maxTtlSecs, + }; + await op("setSandboxLimits", { teamId, limits }); toast({ title: "Sandbox limits saved" }); + setF(toForm(limits)); q.retry(); } catch (e) { toast({ tone: "error", title: (e as Error).message }); @@ -55,7 +83,7 @@ export function SandboxLimitsPanel({ teamId }: { teamId: string }) { setBusy(false); } }; - const field = (k: keyof Form, label: string, hint?: string) => ( + const field = (k: "running" | "cpus" | "memory" | "disk", label: string, hint?: string) => ( {(p) => } @@ -67,12 +95,36 @@ export function SandboxLimitsPanel({ teamId }: { teamId: string }) { {field("cpus", "CPUs")} {field("memory", "Memory (GB)")} {field("disk", "Disk (GB)")} - - {(p) => } + + {(p) => ( +
+ + {f.ttl === "custom" ? ( + setF({ ...f, custom })} + units={UNITS} + /> + ) : null} +
+ )}
-
diff --git a/dash/src/components/pyo/sandbox/TemplateMark.tsx b/dash/src/components/pyo/sandbox/TemplateMark.tsx new file mode 100644 index 0000000..e655309 --- /dev/null +++ b/dash/src/components/pyo/sandbox/TemplateMark.tsx @@ -0,0 +1,136 @@ +import { CubeIcon, MonitorIcon } from "@phosphor-icons/react"; +import { Fragment } from "react"; +import { + siAlmalinux, + siAlpinelinux, + siArchlinux, + siBun, + siCentos, + siDebian, + siDeno, + siDocker, + siFedora, + siGo, + siNodedotjs, + siOpenjdk, + siPhp, + siPython, + siRockylinux, + siRuby, + siRust, + siUbuntu, + type SimpleIcon, +} from "simple-icons"; +import { BrandMark } from "~/components/pyo/common"; +import { + BASE_TEMPLATE, + DESKTOP_TEMPLATE, + imageParts, + isBase, + isDesktop, + type Sandbox, + type SandboxTemplate, + shortImage, + templateOs, +} from "~/lib/pyo/sandbox"; + +// A sandbox template's logo, by what its image runs: languages first, since +// node:22 is Debian underneath but reads as Node. Pyo's base is Ubuntu, its +// desktop a screen. + +type TemplateLike = Pick & + Partial>; + +const LOGOS: [RegExp, SimpleIcon][] = [ + [/\bnode(js)?\b/, siNodedotjs], + [/\bpython\b/, siPython], + [/\bbun\b/, siBun], + [/\bdeno\b/, siDeno], + [/\bgolang\b|(^|\/)go(:|$)/, siGo], + [/\brust\b/, siRust], + [/\bruby\b/, siRuby], + [/\bphp\b/, siPhp], + [/\b(openjdk|java|temurin)\b/, siOpenjdk], + [/\bubuntu\b/, siUbuntu], + [/\bdebian\b/, siDebian], + [/\balpine\b/, siAlpinelinux], + [/\bfedora\b/, siFedora], + [/\barch(linux)?\b/, siArchlinux], + [/\bcentos\b/, siCentos], + [/\brocky(linux)?\b/, siRockylinux], + [/\balma(linux)?\b/, siAlmalinux], + [/\bdocker\b/, siDocker], +]; + +export function templateLogo(t: TemplateLike) { + if (isBase(t)) return siUbuntu; + const text = `${t.source?.image ?? ""} ${t.source?.os ?? ""} ${t.name}`.toLowerCase(); + return LOGOS.find(([re]) => re.test(text))?.[1]; +} + +/** A sandbox's template, from the team's templates when it's there. */ +export function templateOf(x: Pick, templates: SandboxTemplate[]): TemplateLike { + return ( + templates.find((t) => t.id === x.templateId) ?? + templates.find((t) => t.name === x.template) ?? { + name: x.template, + builtin: x.template === BASE_TEMPLATE || x.template === DESKTOP_TEMPLATE, + } + ); +} + +const SIZES = { + sm: { box: "h-6 w-6 rounded-md", icon: 13 }, + md: { box: "h-8 w-8 rounded-lg", icon: 16 }, + lg: { box: "h-9 w-9 rounded-lg", icon: 18 }, +}; + +/** The template's logo in a tile, or a cube when nothing matches. */ +export function TemplateMark({ + template, + size = "md", + className = "", +}: { + template: TemplateLike; + size?: keyof typeof SIZES; + className?: string; +}) { + const logo = isDesktop(template) ? undefined : templateLogo(template); + const { box, icon } = SIZES[size]; + return ( + + {isDesktop(template) ? ( + + ) : logo ? ( + + ) : ( + + )} + + ); +} + +/** A template on hover: its name, what it runs and its image. A long image + * wraps at its "/", ":" and "@", its digest cut short. */ +export function TemplateTip({ template }: { template: TemplateLike }) { + const os = templateOs(template); + const image = template.source?.image; + return ( + + {template.name} + {os ? {os} : null} + {image ? ( + + {imageParts(shortImage(image)).map((part, i) => ( + + {part} + + + ))} + + ) : null} + + ); +} diff --git a/dash/src/components/pyo/sandbox/common.tsx b/dash/src/components/pyo/sandbox/common.tsx index 2bb14f4..10f2013 100644 --- a/dash/src/components/pyo/sandbox/common.tsx +++ b/dash/src/components/pyo/sandbox/common.tsx @@ -1,6 +1,7 @@ import { CheckCircleIcon, CircleNotchIcon } from "@phosphor-icons/react"; import { useEffect, useState } from "react"; import { NeedsTailscale } from "~/components/pyo/Tailscale"; +import { TailnetLockWait } from "~/components/pyo/TailnetLock"; import { AppLink } from "~/components/ui/AppLink"; import { LiveCommand } from "~/components/pyo/LiveCommand"; import { Badge, Dot } from "~/components/ui/Badge"; @@ -13,6 +14,7 @@ import { RUNTIME_LABEL, SANDBOX_STATE, SERVER_STATE, + gettingReady, sandboxApi, type Sandbox, type SandboxServer, @@ -32,12 +34,14 @@ export function useNow(ms = 15_000) { return now; } -export function SandboxStatus({ state }: { state: SandboxState }) { +export function SandboxStatus({ state, sandbox }: { state: SandboxState; sandbox?: Sandbox }) { const st = SANDBOX_STATE[state]; + const label = + state === "creating" && sandbox?.preparing ? gettingReady(sandbox.template, sandbox.server) : st.label; return ( - {st.label} + {label} ); } @@ -174,6 +178,7 @@ export function AddSandboxServer({ onDone }: { onDone?: () => void }) { )} + {joined || expired ? null : } {onDone ? (
+ ) : null} + {ended(x) ? null : x.state === "paused" ? ( @@ -137,9 +154,11 @@ function SandboxPage() { Pause )} - + {ended(x) ? null : ( + + )} ) : undefined } @@ -183,7 +202,7 @@ function SandboxPage() { setDeleting(false)} - title={`Delete ${x.id}?`} + title={`Delete ${x.name}?`} description="Its files are gone for good." confirmLabel="Delete sandbox" onConfirm={() => { @@ -196,16 +215,93 @@ function SandboxPage() { ); } +/** The sandbox's name, edited in place. Enter or leaving the field saves, + * Escape puts it back. Its id stays. */ +function NameField({ x, onDone }: { x: Sandbox; onDone: () => void }) { + const [editing, setEditing] = useState(false); + const [name, setName] = useState(x.name); + const [shown, setShown] = useState(x.name); + const [busy, setBusy] = useState(false); + // Follows renames from elsewhere while not editing. + if (!editing && !busy && shown !== x.name) setShown(x.name); + const done = async (keep: boolean) => { + const n = name.trim(); + setEditing(false); + if (!keep || !n || n === shown) return; + setShown(n); + setBusy(true); + try { + await sandboxApi.rename(x.id, n); + onDone(); + } catch (e) { + setShown(x.name); + toast({ tone: "error", title: (e as Error).message }); + } finally { + setBusy(false); + } + }; + if (editing) + return ( + // The hidden copy sizes the field to its text, so it fits like the name. + + + + {name || " "} + + e.currentTarget.select()} + onChange={(e) => setName(e.target.value)} + onBlur={() => void done(true)} + onKeyDown={(e) => { + if (e.key === "Enter") e.currentTarget.blur(); + if (e.key === "Escape") { + setName(shown); + void done(false); + } + }} + className="col-start-1 row-start-1 w-full min-w-0 bg-transparent p-0 font-semibold tracking-tight text-foreground outline-none" + /> + + + ); + return ( + + ); +} + function Overview({ x, canUse, onChange }: { x: Sandbox; canUse: boolean; onChange: () => void }) { const s = usePyo(); const now = useNow(1000); const server = s.sandboxServers.find((v) => v.id === x.serverId); const project = x.projectId ? projectsIn(s).find((p) => p.id === x.projectId) : undefined; - const left = timeLeft(x.expiresAt, now); - const extend = async (ttl: string) => { + const list = useSandboxes(x.teamId); + const limits = list.status === "success" ? list.data.limits : null; + const left = x.expiresAt === null ? "No limit" : timeLeft(x.expiresAt, now); + const extend = async (ttl: string, label: string) => { try { await sandboxApi.extend(x.id, ttl); - toast({ title: `Ends in ${ttl} from now` }); + toast({ title: ttl === NO_LIFETIME ? "It runs until you delete it" : `Ends ${label} from now` }); onChange(); } catch (e) { toast({ tone: "error", title: (e as Error).message }); @@ -217,7 +313,7 @@ function Overview({ x, canUse, onChange }: { x: Sandbox; canUse: boolean; onChan ], + ["Status", ], [ "Time left", ended(x) ? ( @@ -229,10 +325,16 @@ function Overview({ x, canUse, onChange }: { x: Sandbox; canUse: boolean; onChan ({ - label: `${l.label} from now`, - onSelect: () => void extend(l.value), - }))} + items={[ + ...LIFETIMES.map((l) => ({ + label: `${l.label} from now`, + disabled: !lifetimeAllowed(l.secs, limits), + onSelect: () => void extend(l.value, l.label), + })), + ...(limits?.maxTtlSecs === null + ? [{ label: "No limit", onSelect: () => void extend(NO_LIFETIME, "") }] + : []), + ]} trigger={(t) => (
diff --git a/dash/src/routes/_app/$team/sandboxes/templates.tsx b/dash/src/routes/_app/$team/sandboxes/templates.tsx index fe53da4..f8878b7 100644 --- a/dash/src/routes/_app/$team/sandboxes/templates.tsx +++ b/dash/src/routes/_app/$team/sandboxes/templates.tsx @@ -1,17 +1,27 @@ import { DotsThreeIcon, SquaresFourIcon, TrashIcon } from "@phosphor-icons/react"; -import { createFileRoute } from "@tanstack/react-router"; +import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { useState } from "react"; +import { TemplateMark, TemplateTip } from "~/components/pyo/sandbox/TemplateMark"; import { Dot } from "~/components/ui/Badge"; import { Button } from "~/components/ui/Button"; import { ConfirmDialog } from "~/components/ui/Dialog"; import { Field, Input } from "~/components/ui/Field"; import { Menu } from "~/components/ui/Menu"; -import { EmptyState, ErrorState, Page, PageHeader, Panel } from "~/components/ui/Panel"; +import { EmptyState, ErrorState, Page, PageHeader, Panel, Progress } from "~/components/ui/Panel"; import { Table, Td, Th, Tr } from "~/components/ui/Table"; -import { megabytes, timeAgo } from "~/lib/pyo/format"; +import { Tooltip } from "~/components/ui/Tooltip"; +import { currentSlug } from "~/lib/pyo/paths"; import { useQuery } from "~/lib/pyo/query"; -import { sandboxApi, templateImage, type SandboxTemplate, type SandboxTemplateServer } from "~/lib/pyo/sandbox"; -import { roleIn, usePyo } from "~/lib/pyo/store"; +import { + fieldsSwapped, + pyosFirst, + sandboxApi, + templateProgress, + templateStatus, + usable, + type SandboxTemplate, +} from "~/lib/pyo/sandbox"; +import { roleIn, sandboxServersIn, usePyo } from "~/lib/pyo/store"; import { toast } from "~/lib/pyo/toast"; import { seo } from "~/lib/seo"; @@ -29,39 +39,35 @@ const STATE: Record, string> = { - queued: "Queued", - pulling: "Pulling", - unpacking: "Unpacking", - snapshotting: "Booting", -}; - -/** How far one server got with a template. */ -function serverLabel(x: SandboxTemplateServer) { - if (x.state === "building" && x.phase) { - const pct = x.phase === "pulling" && x.imageBytes ? ` ${Math.floor(((x.pulled ?? 0) / x.imageBytes) * 100)}%` : ""; - return PHASE[x.phase] + pct; - } - if (x.state === "ready" && x.bytes) return `Ready · ${megabytes(x.bytes / 1024 ** 2)}`; - return STATE[x.state]?.label ?? x.state; -} - function TemplatesPage() { const s = usePyo(); + const navigate = useNavigate(); const team = s.currentTeamId; - const q = useQuery(`sandbox-templates:${team}`, () => sandboxApi.templates(team), { refreshMs: 10_000 }); + // Fast while a server makes one, so its bar moves. + const [fast, setFast] = useState(false); + const q = useQuery(`sandbox-templates:${team}`, () => sandboxApi.templates(team), { refreshMs: fast ? 1500 : 10_000 }); + const preparing = q.status === "success" && q.data.templates.some((t) => templateProgress(t)); + if (preparing !== fast) setFast(preparing); const canCreate = roleIn(s) !== "viewer"; + // New sandboxes need a server that runs them. + const canUse = canCreate && sandboxServersIn(s).some((x) => x.state === "ready"); const [name, setName] = useState(""); const [image, setImage] = useState(""); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); const [removing, setRemoving] = useState(null); const canRemove = (t: SandboxTemplate) => !t.builtin && (t.teamId ? canCreate : !!s.user.isAdmin); - const nameError = name && !NAME.test(name) ? "Lowercase letters, digits, dashes, dots and underscores." : ""; + const swapped = fieldsSwapped(name, image); + const nameError = swapped + ? "This looks like the image. Swap the name and image." + : name && !NAME.test(name) + ? "Lowercase letters, digits, dashes, dots and underscores." + : ""; const create = async () => { if (!name || !image.trim() || nameError || busy) return; @@ -80,9 +86,9 @@ function TemplatesPage() { } }; - const templates = q.status === "success" ? q.data.templates : []; + const templates = q.status === "success" ? pyosFirst(q.data.templates) : []; return ( - + {q.status === "error" ? ( @@ -90,55 +96,62 @@ function TemplatesPage() { ) : templates.length ? ( - +
- - - - - - + + {templates.map((t) => ( - - - - - ))} @@ -164,7 +177,24 @@ function TemplatesPage() { setName(e.target.value.toLowerCase())} /> )} - + { + setName(image.trim().toLowerCase()); + setImage(name); + }} + > + Swap them + + ) : undefined + } + > {(p) => setImage(e.target.value)} />}
NameImageStatusServersAdded + TemplateStatus + Actions +
- {t.name} - {t.teamId ? null : Installation} - {templateImage(t) || "—"} - - - {STATE[t.state]?.label ?? t.state} - + }> + + + {t.name} + + - {t.servers.length - ? t.servers.map((x) => ( -
- - {x.name} - {serverLabel(x)} -
- )) - : —} +
+ {timeAgo(t.createdAt)} - {canRemove(t) ? ( - , danger: true, onSelect: () => setRemoving(t) }]} - trigger={(p) => ( - - )} - /> - ) : null} + +
+ {canUse && usable(t) ? ( + + ) : null} + {/* The menu keeps its slot, so Use lines up on every row. */} +
+ {canRemove(t) ? ( + , danger: true, onSelect: () => setRemoving(t) }]} + trigger={(p) => ( + + )} + /> + ) : null} +
+