diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac7ea84..8a90f82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,10 @@ jobs: --health-interval 2s --health-timeout 5s --health-retries 30 + # Where this branch's base template image goes, for the daemon to pull. + registry: + image: registry:3.0.0 + ports: ["5000:5000"] env: PYO_SANDBOX_ENVD: ${{ github.workspace }}/target/x86_64-unknown-linux-musl/release/pyo-envd steps: @@ -137,9 +141,16 @@ jobs: run: | cargo build --release --locked -p pyo-envd --target x86_64-unknown-linux-musl cargo build --locked -p pyo-sandbox + # Pyo's built-in template `base`, from this branch's Dockerfile. + - name: Build the base template image + run: | + docker build -f deploy/images/sandbox-base.Dockerfile -t localhost:5000/pyo/sandbox-base:e2e deploy/images + docker push localhost:5000/pyo/sandbox-base:e2e + docker image ls localhost:5000/pyo/sandbox-base:e2e - name: pyo-sandbox e2e env: PYO_SANDBOX_E2E: "1" + PYO_SANDBOX_E2E_BASE_IMAGE: 127.0.0.1:5000/pyo/sandbox-base:e2e run: cargo test --locked -p pyo-sandbox --test e2e --test template_e2e -- --nocapture - name: pyo-api sandbox e2e env: @@ -164,6 +175,102 @@ jobs: sudo nft list ruleset | head -80 || true ls -la .cache/sandbox-e2e || true + # Pyo Sandbox on gVisor, end to end: runsc as root, as on a server + # without KVM (Hetzner Cloud). Firecracker is left out, so detection + # falls back to gVisor although the runner has KVM. It mounts disks and + # changes the host's network, which the runner is for. + gvisor: + name: Sandbox on gVisor + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + PYO_SANDBOX_ENVD: ${{ github.workspace }}/target/x86_64-unknown-linux-musl/release/pyo-envd + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Packages + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends busybox-static e2fsprogs nftables musl-tools zstd + # runsc, the release pinned in gvisor.env, checked against its sha512. + - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + id: runsc + with: + path: .cache/sandbox-gvisor + key: sandbox-gvisor-${{ hashFiles('crates/pyo-sandbox/scripts/gvisor.env', 'crates/pyo-sandbox/scripts/gvisor-assets.sh') }} + - name: runsc + run: | + set -euo pipefail + crates/pyo-sandbox/scripts/gvisor-assets.sh | tee -a "$GITHUB_ENV" + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: steps.runsc.outputs.cache-hit != 'true' + with: + path: .cache/sandbox-gvisor + key: ${{ steps.runsc.outputs.cache-primary-key }} + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 + with: + toolchain: 1.99.1 + targets: x86_64-unknown-linux-musl + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + key: sandbox-gvisor + - name: Build pyo-envd (musl) + env: + CC_x86_64_unknown_linux_musl: musl-gcc + run: cargo build --release --locked -p pyo-envd --target x86_64-unknown-linux-musl + # As root. cargo isn't on root's PATH, so the runner user builds the + # test binary and sudo -E runs only it. + - name: gVisor e2e (root) + env: + PYO_SANDBOX_GVISOR_E2E: "1" + run: | + set -euo pipefail + bin=$(cargo test --locked -p pyo-sandbox --test gvisor_e2e --no-run --message-format=json \ + | jq -r 'select(.reason == "compiler-artifact" and .target.name == "gvisor_e2e" and .executable != null) | .executable') + [ -x "$bin" ] || { echo "::error::No gvisor_e2e test binary."; exit 1; } + sudo -E "$bin" --nocapture + - name: What went wrong + if: failure() + run: | + sudo find /tmp -name runsc.log -path '*pyo-gvisor-e2e*' -exec tail -n 40 {} \; || true + sudo nft list ruleset | head -80 || true + ip netns list || true + + # The base sandbox template's image for both architectures, built + # without pushing, when its Dockerfile changes. The release publishes it. + sandbox-base: + name: Base sandbox template image + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Did the Dockerfile change + id: changed + env: + PR_BASE: ${{ github.event.pull_request.base.sha }} + BEFORE: ${{ github.event.before }} + run: | + base=${PR_BASE:-$BEFORE} + if [ -z "$base" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null || + git diff --name-only "$base" HEAD | grep -qx deploy/images/sandbox-base.Dockerfile; then + echo "build=true" >> "$GITHUB_OUTPUT" + else + echo "Its Dockerfile didn't change." + fi + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + if: steps.changed.outputs.build == 'true' + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + if: steps.changed.outputs.build == 'true' + - name: Build + if: steps.changed.outputs.build == 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: deploy/images + file: deploy/images/sandbox-base.Dockerfile + platforms: linux/amd64,linux/arm64 + push: false + waf: name: WAF runs-on: ubuntu-24.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86a5903..5115726 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: strategy: fail-fast: true matrix: - image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static] + image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static, sandbox-base] # Pyo's Postgres, one image per version, from one Dockerfile. # One repository, tagged -: CloudNativePG reads # the version from the tag. @@ -111,8 +111,13 @@ jobs: # (deploy/images/moco), one Dockerfile with a target each. - { image: mysql-operator, file: moco/moco, context: deploy/images/moco, target: operator } - { image: mysql-backup, file: moco/moco, context: deploy/images/moco, target: backup } + # Pyo Sandbox's built-in template `base`. Sandbox servers run on + # both architectures, so it's built for both (arm64 under QEMU). + - { image: sandbox-base, context: deploy/images, platforms: "linux/amd64,linux/arm64" } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + if: contains(matrix.platforms, 'arm64') - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: @@ -128,7 +133,7 @@ jobs: file: deploy/images/${{ matrix.file || matrix.image }}.Dockerfile build-args: ${{ matrix.args }} target: ${{ matrix.target }} - platforms: linux/amd64 + platforms: ${{ matrix.platforms || 'linux/amd64' }} cache-from: ${{ matrix.rust && format('type=gha,scope=rust-{0}', github.run_id) || '' }} push: true tags: ${{ env.IMAGES }}/${{ matrix.repo || matrix.image }}:${{ matrix.tag }}${{ needs.verify.outputs.version }} @@ -138,6 +143,22 @@ jobs: env: IMAGE: ${{ env.IMAGES }}/${{ matrix.repo || matrix.image }}@${{ steps.build.outputs.digest }} run: cosign sign --yes "$IMAGE" + # Installations and sandbox servers pull without logging in. A new + # package on ghcr.io may start private: make it public in its + # package settings, then run the job again. + - name: Anyone can pull it + env: + REPO: upsetdev/pyo/${{ matrix.repo || matrix.image }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + token=$(curl -fsS "https://ghcr.io/token?scope=repository:$REPO:pull" | jq -r .token) + code=$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json' \ + "https://ghcr.io/v2/$REPO/manifests/$DIGEST") + if [ "$code" != 200 ]; then + echo "::error::ghcr.io/$REPO can't be pulled without logging in (HTTP $code). Make the package public in its settings on GitHub, then run this job again." + exit 1 + fi release: name: GitHub release @@ -176,6 +197,7 @@ jobs: cd sandbox for a in x86_64 aarch64; do sha256sum -c "pyo-sandbox-linux-$a.tar.gz.sha256" + sha256sum -c "pyo-sandbox-gvisor-linux-$a.tar.gz.sha256" done # What Pyo needs to update itself from the dashboard: the images # pinned by the digests that were just signed, and the manifests. @@ -187,13 +209,15 @@ jobs: tagged() { docker buildx imagetools inspect "$IMAGES/$1:$2$VERSION" --format '{{json .Manifest}}' | jq -r .digest; } bun run scripts/release-manifest.ts --version "$VERSION" --images "$IMAGES" \ --pyo "$(digest pyo)" --agent "$(digest agent)" --builder "$(digest builder)" --flows "$(digest flows)" --waf "$(digest waf)" --geoip "$(digest geoip)" \ - --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" \ + --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" --sandbox-base "$(digest sandbox-base)" \ --mysql-operator "$(digest mysql-operator)" --mysql-backup "$(digest mysql-backup)" \ --postgres-16 "$(tagged postgres 16-)" --postgres-17 "$(tagged postgres 17-)" --postgres-18 "$(tagged postgres 18-)" \ --agent-access "$(docker run --rm "$IMAGES/agent:$VERSION@$(digest agent)" access-hash)" \ --sandbox-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256)" \ - --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" > release.json - jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, migrations: (.migrations | length)}' release.json + --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" \ + --sandbox-gvisor-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256)" \ + --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" > release.json + jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, migrations: (.migrations | length)}' release.json # The release's own upgrader reads it before it's signed: an image # it doesn't accept, or an agentAccess that isn't its agent's roles, # fails the release here, not on installations. @@ -276,3 +300,7 @@ jobs: sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256 sandbox/pyo-sandbox-linux-aarch64.tar.gz sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256 + sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz + sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256 + sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz + sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256 diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 34203a1..cd1bc58 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -3,7 +3,8 @@ name: Sandbox # A sandbox server's files for each architecture, as # pyo-sandbox-linux-.tar.gz with its .sha256: the daemon and # pyo-envd (static, musl), Firecracker and its jailer, and Pyo's guest -# kernel. deploy/sandbox.sh installs them. The Release workflow runs it and +# kernel. Beside it, pyo-sandbox-gvisor-linux-.tar.gz: runsc for +# servers without KVM. deploy/sandbox.sh installs them. The Release workflow runs it and # puts the tarballs in the GitHub release. Run it by hand (Actions › # Sandbox › Run workflow) to try the builds on a branch: it publishes # nothing, the tarballs stay in the run's artifacts. @@ -97,6 +98,9 @@ jobs: check/pyo-envd --version check/firecracker --version check/jailer --version + mkdir check-gvisor + tar -xzf "out/pyo-sandbox-gvisor-linux-$ARCH.tar.gz" -C check-gvisor + check-gvisor/runsc --version file check/pyo-sandbox check/pyo-envd | tee /dev/stderr | grep -c 'static' | grep -qx 2 - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb0ed64..63abb2c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,8 @@ Read the script before you run it. It changes the machine: To undo it, run `/usr/local/bin/k3s-uninstall.sh`. +If you skip this step, `scripts/pyo-up.sh` asks to run it for you. + **2. Build and deploy (as your normal user):** ```bash diff --git a/Cargo.lock b/Cargo.lock index bc2af04..4697e3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pyo-agent" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base64 0.23.1", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "pyo-api" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "argon2 0.6.0", @@ -3413,7 +3413,7 @@ dependencies = [ [[package]] name = "pyo-builder" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base64 0.23.1", @@ -3432,7 +3432,7 @@ dependencies = [ [[package]] name = "pyo-cli" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "clap", @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "pyo-db-controller" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base64 0.23.1", @@ -3474,7 +3474,7 @@ dependencies = [ [[package]] name = "pyo-envd" -version = "0.4.0" +version = "0.4.1" dependencies = [ "libc", "pyo-proto", @@ -3483,7 +3483,7 @@ dependencies = [ [[package]] name = "pyo-flows" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "libc", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "pyo-kv" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "axum", @@ -3514,7 +3514,7 @@ dependencies = [ [[package]] name = "pyo-proto" -version = "0.4.0" +version = "0.4.1" dependencies = [ "base64 0.23.1", "regex", @@ -3528,7 +3528,7 @@ dependencies = [ [[package]] name = "pyo-resp" -version = "0.4.0" +version = "0.4.1" dependencies = [ "futures", "socket2", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "pyo-sandbox" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "axum", @@ -3567,7 +3567,7 @@ dependencies = [ [[package]] name = "pyo-static" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 66aabff..835adc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/pyo-proto", "crates/pyo-api", "crates/pyo-agent", "crates/pyo-builder", "crates/pyo-cli", "crates/pyo-flows", "crates/pyo-resp", "crates/pyo-kv", "crates/pyo-db-controller", "crates/pyo-static", "crates/pyo-envd", "crates/pyo-sandbox"] [workspace.package] -version = "0.4.0" +version = "0.4.1" edition = "2024" license = "Apache-2.0" publish = false diff --git a/RELEASING.md b/RELEASING.md index c1d24ab..84b8e9e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -19,7 +19,7 @@ A version is one number for everything: `version` in the root `Cargo.toml`. `.github/workflows/release.yml` then checks that the tag matches `Cargo.toml`, runs the tests (the WAF's Go tests in `waf/` too), publishes signed images -to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup}` +to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup,sandbox-base}` (all from `deploy/images/`, at the same version. The Rust binaries are compiled once, by the `binaries` job, and the Rust images' jobs take them from that run's build cache), @@ -53,6 +53,14 @@ crates/pyo-sandbox/scripts/bundle.sh --arch x86_64 \ `sandbox.sh --bundle dist/pyo-sandbox-linux-x86_64.tar.gz` installs such a build, with `--bundle-sha256 ` or, unchecked, `--dev`. +`sandbox-base` is Pyo Sandbox's built-in template `base` +(`deploy/images/sandbox-base.Dockerfile`), built for amd64 and arm64. +`release.json` pins it as `images.sandboxBase`, and Pyo seeds the template +from it. Sandbox servers pull it without logging in, so the release checks +that every image can be pulled anonymously. The first time a new image is +published, ghcr.io may make its package private: make it public in the +package's settings, then run the failed job again. + `release.json` is signed with cosign keyless as `release.json.sigstore.json` (the job needs `id-token: write`, and `.github/release-signing-config.json` pins Sigstore's public diff --git a/crates/pyo-agent/Cargo.toml b/crates/pyo-agent/Cargo.toml index c02f93d..f8deb0a 100644 --- a/crates/pyo-agent/Cargo.toml +++ b/crates/pyo-agent/Cargo.toml @@ -7,7 +7,7 @@ publish.workspace = true [dependencies] anyhow = "1.0.104" -pyo-proto = { version = "0.4.0", path = "../pyo-proto" } +pyo-proto = { version = "0.4.1", path = "../pyo-proto" } futures = "0.3.34" http = "1.5.0" k8s-openapi = { version = "0.28.0", features = ["latest"] } diff --git a/crates/pyo-api/Cargo.toml b/crates/pyo-api/Cargo.toml index 7f28303..2256875 100644 --- a/crates/pyo-api/Cargo.toml +++ b/crates/pyo-api/Cargo.toml @@ -12,7 +12,7 @@ axum = { version = "0.8.9", features = ["ws"] } axum-extra = { version = "0.12.6", features = ["cookie", "typed-header"] } base64 = "0.23.1" cookie = "0.18" -pyo-proto = { version = "0.4.0", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } +pyo-proto = { version = "0.4.1", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } hex = "0.4.3" hmac = "0.13.0" jiff = "0.2.38" diff --git a/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql b/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql new file mode 100644 index 0000000..8b6854e --- /dev/null +++ b/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql @@ -0,0 +1,13 @@ +-- pyo:contract since=v0.4.0 the role check only widens and cluster_id only stops being required, so every row a release writes still fits +-- Sandbox servers Pyo makes at a cloud provider (cloud/mod.rs): the cloud +-- server points at the sandbox server it becomes, and so does its +-- bootstrap code, which then opens sandbox.sh's details instead of a +-- cluster's. Kept as plain text, so a removed sandbox server's cloud server +-- still knows which label to check before Pyo deletes it at the provider. +ALTER TABLE pyo.cloud_servers ADD COLUMN sandbox_server_id text; +CREATE INDEX cloud_servers_sandbox ON pyo.cloud_servers (sandbox_server_id) WHERE sandbox_server_id IS NOT NULL; + +ALTER TABLE pyo.cloud_bootstrap ADD COLUMN sandbox_server_id text; +ALTER TABLE pyo.cloud_bootstrap ALTER COLUMN cluster_id DROP NOT NULL; +ALTER TABLE pyo.cloud_bootstrap DROP CONSTRAINT cloud_bootstrap_role_check; +ALTER TABLE pyo.cloud_bootstrap ADD CONSTRAINT cloud_bootstrap_role_check CHECK (role IN ('join', 'connect', 'sandbox')); diff --git a/crates/pyo-api/migrations/0015_sandbox_base_template.sql b/crates/pyo-api/migrations/0015_sandbox_base_template.sql new file mode 100644 index 0000000..643d70e --- /dev/null +++ b/crates/pyo-api/migrations/0015_sandbox_base_template.sql @@ -0,0 +1,7 @@ +-- Pyo's own sandbox templates (the built-in `base`, sandbox/base.rs): Pyo +-- keeps them in step with its release, and nobody deletes them. A +-- revision names the image a template is made from now: servers keep it +-- as `.`, so a new image is made beside the old one, which +-- running and paused sandboxes keep using. NULL: the template's id alone. +ALTER TABLE pyo.sandbox_templates ADD COLUMN builtin boolean NOT NULL DEFAULT false; +ALTER TABLE pyo.sandbox_templates ADD COLUMN revision text; diff --git a/crates/pyo-api/src/cloud/boot.sh b/crates/pyo-api/src/cloud/boot.sh index 51292bd..fbdcedd 100644 --- a/crates/pyo-api/src/cloud/boot.sh +++ b/crates/pyo-api/src/cloud/boot.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash -# Made by Pyo: joins this server to a cluster over your tailnet. +# Made by Pyo: joins this server to Pyo over your tailnet. # # boot.sh PYO SERVER CLUSTER ROLE NAME # # 1. Joins your tailnet with a single-use Tailscale key. # 2. Trades a single-use code for this server's join details at Pyo, over # the tailnet (PYO: Pyo's tailnet door). -# 3. Runs join.sh, or connect.sh for a new cluster (ROLE), with them. +# 3. Runs join.sh, connect.sh for a new cluster, or sandbox.sh for a +# sandbox server (ROLE), with them. For a sandbox server CLUSTER is the +# sandbox server's ID. # # The key and the code are the only secrets in this server's user data. # They're in /run/pyo-boot (root only, in memory) and deleted once used. @@ -64,7 +66,7 @@ done < <(printf '%s\n' "$body") unset out body # <<< parse case "$ROLE:$script" in - join:http*/join.sh | connect:http*/connect.sh) ;; + join:http*/join.sh | connect:http*/connect.sh | sandbox:http*/sandbox.sh) ;; *) fail "Pyo's answer has no $ROLE script." ;; esac diff --git a/crates/pyo-api/src/cloud/bootstrap.rs b/crates/pyo-api/src/cloud/bootstrap.rs index e8e9863..fe83a6f 100644 --- a/crates/pyo-api/src/cloud/bootstrap.rs +++ b/crates/pyo-api/src/cloud/bootstrap.rs @@ -3,9 +3,11 @@ //! The user data holds two one-time values ([`super::user_data`]): a //! single-use Tailscale key and a single-use code made here ([`new_code`]). //! The code is random, kept only as its hash, expires after [`CODE_TTL`], -//! and is bound to the cloud server, its cluster and its role (join.sh or -//! connect.sh). Its join details wait here sealed until then -//! ([`keep`]). +//! and is bound to the cloud server, its cluster (or the sandbox server it +//! becomes) and its role (join.sh, connect.sh or sandbox.sh). Its join +//! details wait here sealed until then ([`keep`]). A sandbox server's +//! details get a fresh sandbox join code when they're handed over +//! (`sandbox::join`), so its 15 minutes start then. //! //! On its first boot the server joins the tailnet with the key, then asks //! Pyo's tailnet door (pyo-forward's :5006, which admits tailnet peers @@ -68,6 +70,8 @@ pub enum Role { Join, /// connect.sh: a new cluster's first server. Connect, + /// sandbox.sh: a sandbox server. + Sandbox, } impl Role { @@ -75,6 +79,7 @@ impl Role { match self { Role::Join => "join", Role::Connect => "connect", + Role::Sandbox => "sandbox", } } } @@ -121,23 +126,29 @@ pub fn new_code() -> String { /// Keeps a code (its hash) and the details it opens, sealed, for /// [`CODE_TTL`]. After the cloud server's row, in the same transaction. +/// `owner` is its cluster, or the sandbox server it becomes. pub async fn keep( tx: &mut Transaction<'_, Postgres>, sealer: &Sealer, code: &str, cloud_server_id: &str, - cluster_id: &str, + owner: &str, role: Role, material: &Material, ) -> ApiResult<()> { let sealed = sealer.seal(&material.render()?)?; + let (cluster, sandbox) = match role { + Role::Sandbox => (None, Some(owner)), + Role::Join | Role::Connect => (Some(owner), None), + }; sqlx::query( - "INSERT INTO pyo.cloud_bootstrap (code_hash, cloud_server_id, cluster_id, role, material_sealed, expires_at) - VALUES ($1, $2, $3, $4, $5, now() + make_interval(secs => $6))", + "INSERT INTO pyo.cloud_bootstrap (code_hash, cloud_server_id, cluster_id, sandbox_server_id, role, material_sealed, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, now() + make_interval(secs => $7))", ) .bind(sha256_hex(code)) .bind(cloud_server_id) - .bind(cluster_id) + .bind(cluster) + .bind(sandbox) .bind(role.as_str()) .bind(sealed) .bind(CODE_TTL.as_secs_f64()) @@ -236,6 +247,7 @@ impl Refusal { pub struct Ask { /// Its cloud server ID ("cs_…"). pub server: String, + /// Its cluster, or the sandbox server it becomes ("sbs_…"). pub cluster: String, pub role: Role, } @@ -285,7 +297,8 @@ async fn claim(db: &PgPool, code: &str) -> sqlx::Result UPDATE pyo.cloud_bootstrap c SET used_at = now(), material_sealed = NULL FROM prev WHERE c.code_hash = prev.code_hash - RETURNING c.cloud_server_id, c.cluster_id, c.role, c.expires_at <= now() AS expired, + RETURNING c.cloud_server_id, coalesce(c.cluster_id, c.sandbox_server_id, '') AS cluster_id, + c.role, c.expires_at <= now() AS expired, prev.material_sealed, (SELECT state FROM pyo.cloud_servers WHERE id = c.cloud_server_id) AS state", ) @@ -369,7 +382,7 @@ pub async fn answer( return text(StatusCode::FORBIDDEN, r.message()); } }; - let material = match sealer.open(&sealed) { + let mut material = match sealer.open(&sealed) { Ok(m) => m, Err(e) => { tracing::warn!(server = %ask.server, error = %e, "cloud server setup: couldn't open its details"); @@ -379,6 +392,20 @@ pub async fn answer( ); } }; + // A sandbox server's join code is made now, so sandbox.sh has its full + // 15 minutes from here. + if ask.role == Role::Sandbox { + match sandbox_code(db, &ask.cluster).await { + Ok(code) => material.push_str(&format!("arg=--code\narg={code}\n")), + Err(e) => { + tracing::warn!(server = %ask.server, error = %e.message, "cloud server setup: couldn't make a sandbox join code"); + return text( + StatusCode::SERVICE_UNAVAILABLE, + "Pyo couldn't make this server's join code. Delete it and make a new one.", + ); + } + } + } if let Err(e) = sqlx::query( "UPDATE pyo.cloud_servers SET state = 'joining', error = NULL, updated_at = now() WHERE id = $1 AND state IN ('creating', 'booting')", ) @@ -392,8 +419,17 @@ pub async fn answer( text(StatusCode::OK, material) } -/// POST /cloud/bootstrap?server=…&cluster=…&role=join|connect on Pyo's -/// tailnet door, with the code as the body. +/// A new sandbox join code for a sandbox server Pyo made. +async fn sandbox_code(db: &PgPool, sandbox_server: &str) -> ApiResult { + let code = crate::sandbox::join::new_code(); + let mut tx = db.begin().await?; + crate::sandbox::join::keep(&mut tx, sandbox_server, &code).await?; + tx.commit().await?; + Ok(code) +} + +/// POST /cloud/bootstrap?server=…&cluster=…&role=join|connect|sandbox on +/// Pyo's tailnet door, with the code as the body. pub async fn handle( State(s): State, ConnectInfo(peer): ConnectInfo, @@ -720,6 +756,101 @@ mod tests { let (st, _) = answer(&db, &sealer, tailnet, &a, &code).await; assert_eq!(st, StatusCode::OK); + // A sandbox server: its details come with a fresh sandbox join code, + // made as they're handed over, which sandbox.sh then redeems. + let sbs = format!("sbs_boot{run}"); + sqlx::query("INSERT INTO pyo.sandbox_servers (id, team_id, name, state, created_by) VALUES ($1, $2, $1, 'joining', 'test')") + .bind(&sbs) + .bind(&team) + .execute(&db) + .await + .unwrap(); + let seven = format!("cs_seven{run}"); + sqlx::query( + "INSERT INTO pyo.cloud_servers (id, team_id, provider, sandbox_server_id, name, location, server_type, monthly, remote_id, state, created_by) + VALUES ($1, $2, 'hetzner', $3, $1, 'fsn1', 'cx22', 4.29, '7', 'booting', 'test')", + ) + .bind(&seven) + .bind(&team) + .bind(&sbs) + .execute(&db) + .await + .unwrap(); + let sandbox_material = Material::new( + "https://pyo.tail1234.ts.net/install/sandbox.sh".into(), + &[ + ("--server", Some(sbs.clone())), + ("--version", Some("0.4.0".into())), + ], + ); + let code = new_code(); + let mut tx = db.begin().await.unwrap(); + keep( + &mut tx, + &sealer, + &code, + &seven, + &sbs, + Role::Sandbox, + &sandbox_material, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + let (st, body) = answer( + &db, + &sealer, + tailnet, + &ask(&seven, &cluster, Role::Sandbox), + &code, + ) + .await; + assert_eq!( + (st, body.as_str()), + (StatusCode::FORBIDDEN, "This code is for another server."), + "a cluster's ID isn't the sandbox server's" + ); + let code = new_code(); + let mut tx = db.begin().await.unwrap(); + keep( + &mut tx, + &sealer, + &code, + &seven, + &sbs, + Role::Sandbox, + &sandbox_material, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + let (st, body) = answer( + &db, + &sealer, + tailnet, + &ask(&seven, &sbs, Role::Sandbox), + &code, + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + let base = sandbox_material.render().unwrap(); + assert!(body.starts_with(&base), "{body}"); + let join_code = body[base.len()..] + .strip_prefix("arg=--code\narg=") + .and_then(|r| r.strip_suffix('\n')) + .unwrap_or_else(|| panic!("no join code in {body}")); + assert_eq!(join_code.len(), 64); + let live: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pyo.sandbox_join_codes WHERE server_id = $1 AND used_at IS NULL AND code_hash = $2", + ) + .bind(&sbs) + .bind(sha256_hex(join_code)) + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(live, 1, "sandbox.sh can redeem it"); + assert_eq!(state(&seven).await, "joining"); + // Expired details go at once, their rows a day later. prune(&db).await.unwrap(); let left: Option = sqlx::query_scalar( diff --git a/crates/pyo-api/src/cloud/mod.rs b/crates/pyo-api/src/cloud/mod.rs index 3dd89d4..0c56248 100644 --- a/crates/pyo-api/src/cloud/mod.rs +++ b/crates/pyo-api/src/cloud/mod.rs @@ -1,5 +1,6 @@ //! Servers Pyo makes at a cloud provider and joins to a cluster over the -//! tailnet (Add server › Hetzner, Add cluster › Hetzner). +//! tailnet (Add server › Hetzner, Add cluster › Hetzner), or sets up as a +//! sandbox server (Sandboxes › Add server › Cloud provider). //! //! - **The connection** is a team's (Integrations › Hetzner, team admins): //! clusters belong to a team, and the servers it makes are billed to the @@ -11,9 +12,10 @@ //! single-use Tailscale key and a single-use code ([`user_data`]). On its //! first boot it joins the tailnet, trades the code for its join details //! over the tailnet ([`bootstrap`]), and runs join.sh (or connect.sh for -//! a new cluster) with them. The server carries this installation's label -//! (`pyo=`, `pyo-cluster=`) and Pyo's firewall: the -//! web from anywhere and Tailscale's port, nothing else. +//! a new cluster) with them, or sandbox.sh for a sandbox server. The +//! server carries this installation's label (`pyo=`, and +//! `pyo-cluster=` or `pyo-sandbox-server=`) and Pyo's +//! firewall: the web from anywhere and Tailscale's port, nothing else. //! - **Following it** ([`tick`]): Creating → Booting → Joining → Ready, //! with the Tailnet Lock sign command when its key waits for a signature. //! After [`TIMEOUT`] it stops waiting and says why. @@ -50,6 +52,8 @@ use crate::{ pub const LABEL: &str = "pyo"; /// The cluster a server was made for. pub const CLUSTER_LABEL: &str = "pyo-cluster"; +/// The sandbox server a server was made as. +pub const SANDBOX_LABEL: &str = "pyo-sandbox-server"; /// How long Pyo waits for a new server to be ready. pub const TIMEOUT: Duration = Duration::from_secs(15 * 60); /// The providers there are. @@ -235,19 +239,44 @@ pub trait Provider { async fn firewall(&self, install: &str) -> Result; } +/// What a server is made for: a cluster, or a sandbox server. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Owner<'a> { + Cluster(&'a str), + Sandbox(&'a str), +} + +impl<'a> Owner<'a> { + /// Its label at the provider. + pub fn label(self) -> (&'static str, &'a str) { + match self { + Owner::Cluster(id) => (CLUSTER_LABEL, id), + Owner::Sandbox(id) => (SANDBOX_LABEL, id), + } + } + + pub fn id(self) -> &'a str { + self.label().1 + } +} + /// The labels a new server carries. -pub fn labels(install: &str, cluster_id: &str) -> BTreeMap { +pub fn labels(install: &str, owner: Owner) -> BTreeMap { + let (key, value) = owner.label(); BTreeMap::from([ (LABEL.to_owned(), install.to_owned()), - (CLUSTER_LABEL.to_owned(), cluster_id.to_owned()), + (key.to_owned(), value.to_owned()), ]) } /// Whether Pyo may delete a server: it carries this installation's label, -/// and the cluster's when one is given. -pub fn ours(r: &Remote, install: &str, cluster_id: Option<&str>) -> bool { +/// and its cluster's or sandbox server's when one is given. +pub fn ours(r: &Remote, install: &str, owner: Option) -> bool { r.labels.get(LABEL).map(String::as_str) == Some(install) - && cluster_id.is_none_or(|c| r.labels.get(CLUSTER_LABEL).map(String::as_str) == Some(c)) + && owner.is_none_or(|o| { + let (key, value) = o.label(); + r.labels.get(key).map(String::as_str) == Some(value) + }) } /// Deletes a server, only when it carries this installation's label @@ -256,12 +285,12 @@ pub async fn delete( p: &P, id: &str, install: &str, - cluster_id: Option<&str>, + owner: Option>, ) -> Result<(), CloudError> { let Some(r) = p.server(id).await? else { return Ok(()); }; - if !ours(&r, install, cluster_id) { + if !ours(&r, install, owner) { return Err(CloudError::NotOurs); } p.delete_unchecked(id).await @@ -306,6 +335,7 @@ pub struct Boot<'a> { pub pyo: &'a str, /// Its cloud server ID. pub server_id: &'a str, + /// Its cluster, or the sandbox server it becomes. pub cluster_id: &'a str, pub role: bootstrap::Role, /// Its name in the cluster and on the tailnet. @@ -346,7 +376,7 @@ pub fn user_data(b: &Boot, ssh_key: Option<&str>) -> ApiResult { )); } let mut out = String::from( - "#cloud-config\n# Made by Pyo: joins this server to a cluster over your tailnet.\nssh_pwauth: false\n", + "#cloud-config\n# Made by Pyo: joins this server to Pyo over your tailnet.\nssh_pwauth: false\n", ); if let Some(k) = ssh_key { // JSON's quoting is YAML's double-quoted string. @@ -549,6 +579,7 @@ struct Row { id: String, team_id: String, cluster_id: Option, + sandbox_server_id: Option, name: String, remote_id: String, state: String, @@ -568,6 +599,10 @@ pub struct Seen { pub in_cluster: Option, /// Tailnet Lock waits for its key to be signed: the command. pub sign: Option, + /// It becomes a sandbox server, not a cluster's. + pub sandbox: bool, + /// The sandbox server can't run sandboxes: why. + pub refused: Option, } /// The next state of a server being made, and why it failed, if it did. @@ -578,6 +613,9 @@ pub fn next_state(seen: &Seen, age: Duration) -> (&'static str, Option) let Some(remote) = seen.remote.as_deref() else { return ("failed", Some("It was deleted at Hetzner.".into())); }; + if let Some(why) = &seen.refused { + return ("failed", Some(format!("It can't run sandboxes. {why}"))); + } let state = if seen.in_cluster.is_some() || seen.on_tailnet { "joining" } else if matches!(remote, "running" | "starting") { @@ -596,6 +634,9 @@ pub fn next_state(seen: &Seen, age: Duration) -> (&'static str, Option) _ if seen.sign.is_some() => { "Its key wasn't signed in 15 minutes. Sign it, or delete the server." } + _ if seen.sandbox => { + "It joined your tailnet but didn't become a sandbox server in 15 minutes. Read /var/log/pyo-join.log on it." + } _ => { "It joined your tailnet but not the cluster in 15 minutes. Read /var/log/pyo-join.log on it." } @@ -621,7 +662,7 @@ async fn tailnet_devices(s: &AppState) -> Option> pub async fn tick(s: &AppState) -> ApiResult<()> { bootstrap::prune(&s.db).await?; let rows: Vec = sqlx::query_as( - "SELECT id, team_id, cluster_id, name, remote_id, state, + "SELECT id, team_id, cluster_id, sandbox_server_id, name, remote_id, state, extract(epoch FROM now() - created_at)::float8 AS age_secs, sign_command, first_server FROM pyo.cloud_servers WHERE state IN ('creating', 'booting', 'joining', 'removing') ORDER BY created_at", @@ -691,8 +732,9 @@ async fn follow( devices: Option<&[crate::tailscale::Device]>, ) -> ApiResult<()> { let remote = p.server(&r.remote_id).await?; - let in_cluster: Option = match &r.cluster_id { - Some(c) => sqlx::query_scalar( + let mut refused = None; + let in_cluster: Option = match (&r.cluster_id, &r.sandbox_server_id) { + (Some(c), _) => sqlx::query_scalar( "SELECT report->>'status' = 'ready' FROM pyo.servers WHERE cluster_id = $1 AND name = $2", ) .bind(c) @@ -700,7 +742,26 @@ async fn follow( .fetch_optional(&s.db) .await? .map(|x: Option| x.unwrap_or(false)), - None => None, + // A sandbox server counts once sandbox.sh redeemed its code, and is + // ready once its daemon reports. + (None, Some(sbs)) => { + let row: Option<(String, bool, Option)> = sqlx::query_as( + "SELECT state, joined_at IS NOT NULL, error FROM pyo.sandbox_servers WHERE id = $1", + ) + .bind(sbs) + .fetch_optional(&s.db) + .await?; + match row { + Some((state, _, _)) if state == "ready" => Some(true), + Some((state, _, error)) if state == "refused" => { + refused = Some(error.unwrap_or_default()); + Some(false) + } + Some((_, true, _)) => Some(false), + _ => None, + } + } + (None, None) => None, }; let mine: Vec<&crate::tailscale::Device> = devices .map(|d| device_of(d, &r.name).collect()) @@ -714,6 +775,8 @@ async fn follow( on_tailnet: !mine.is_empty(), in_cluster, sign: sign.clone(), + sandbox: r.sandbox_server_id.is_some(), + refused, }; let (mut state, error) = next_state(&seen, Duration::from_secs_f64(r.age_secs.max(0.0))); // It traded its code over the tailnet: Joining, even when the tailnet's @@ -836,7 +899,12 @@ async fn finish_removal( tracing::warn!(server = %r.name, error = %e, "couldn't take the server off the tailnet"); } } - match delete(p, &r.remote_id, install, r.cluster_id.as_deref()).await { + let owner = match (&r.cluster_id, &r.sandbox_server_id) { + (Some(c), _) => Some(Owner::Cluster(c)), + (None, Some(sbs)) => Some(Owner::Sandbox(sbs)), + (None, None) => None, + }; + match delete(p, &r.remote_id, install, owner).await { Ok(()) => { sqlx::query( "UPDATE pyo.cloud_servers SET state = 'deleted', error = NULL, sign_command = NULL, deleted_at = now(), updated_at = now() WHERE id = $1", @@ -872,7 +940,8 @@ pub async fn snapshot(db: &PgPool, team_ids: &[String]) -> ApiResult<(Value, Val .fetch_all(db) .await?; let servers: Vec = sqlx::query_scalar( - "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'provider', provider, 'clusterId', cluster_id, 'name', name, + "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'provider', provider, 'clusterId', cluster_id, + 'sandboxServerId', sandbox_server_id, 'name', name, 'firstServer', first_server, 'builder', builder, 'location', location, 'serverType', server_type, 'monthly', monthly, 'currency', currency, 'ipv4', ipv4, 'state', state, 'error', error, 'signCommand', sign_command, 'createdBy', created_by, 'createdAt', pyo.ms(created_at), @@ -898,7 +967,7 @@ mod tests { location: "fsn1", server_type: ty, user_data, - labels: labels(INSTALL, cluster), + labels: labels(INSTALL, Owner::Cluster(cluster)), firewall: None, } } @@ -968,10 +1037,14 @@ mod tests { assert_eq!(h.server(&r.id).await.unwrap().unwrap().id, r.id); // The same name again: Hetzner's names are unique in a project. assert_eq!(h.create(&c).await.unwrap_err(), CloudError::NameTaken); - delete(&h, &r.id, INSTALL, Some("cl_one")).await.unwrap(); + delete(&h, &r.id, INSTALL, Some(Owner::Cluster("cl_one"))) + .await + .unwrap(); assert_eq!(h.server(&r.id).await.unwrap(), None); // Gone already is fine. - delete(&h, &r.id, INSTALL, Some("cl_one")).await.unwrap(); + delete(&h, &r.id, INSTALL, Some(Owner::Cluster("cl_one"))) + .await + .unwrap(); // Ids are numbers: anything else is never sent. assert_eq!(h.server("1/../../firewalls").await.unwrap(), None); } @@ -985,13 +1058,35 @@ mod tests { json!({ "pyo": "ffffffffffffffffffffffffffffffff", "pyo-cluster": "cl_one" }), ); let other_cluster = m.add_server(json!({ "pyo": INSTALL, "pyo-cluster": "cl_two" })); - for id in [unlabelled, other_pyo, other_cluster] { + let sandbox = m.add_server(json!({ "pyo": INSTALL, "pyo-sandbox-server": "sbs_one" })); + for id in [unlabelled, other_pyo, other_cluster, sandbox] { assert_eq!( - delete(&h, &id.to_string(), INSTALL, Some("cl_one")).await, + delete(&h, &id.to_string(), INSTALL, Some(Owner::Cluster("cl_one"))).await, Err(CloudError::NotOurs) ); } - assert_eq!(m.0.lock().unwrap().servers.len(), 3); + // A sandbox server only by its own label. + assert_eq!( + delete( + &h, + &other_cluster.to_string(), + INSTALL, + Some(Owner::Sandbox("sbs_one")) + ) + .await, + Err(CloudError::NotOurs) + ); + assert_eq!( + delete( + &h, + &sandbox.to_string(), + INSTALL, + Some(Owner::Sandbox("sbs_two")) + ) + .await, + Err(CloudError::NotOurs) + ); + assert_eq!(m.0.lock().unwrap().servers.len(), 4); assert!( !m.0.lock() .unwrap() @@ -1004,6 +1099,14 @@ mod tests { delete(&h, &other_cluster.to_string(), INSTALL, None) .await .unwrap(); + delete( + &h, + &sandbox.to_string(), + INSTALL, + Some(Owner::Sandbox("sbs_one")), + ) + .await + .unwrap(); assert_eq!(m.0.lock().unwrap().servers.len(), 2); } @@ -1243,6 +1346,8 @@ mod tests { on_tailnet, in_cluster, sign: sign.map(str::to_owned), + sandbox: false, + refused: None, }; let soon = Duration::from_secs(60); let late = TIMEOUT + Duration::from_secs(1); @@ -1282,5 +1387,22 @@ mod tests { let (s, why) = next_state(&seen(None, false, None, None), soon); assert_eq!(s, "failed"); assert!(why.unwrap().contains("deleted at Hetzner")); + + // A sandbox server: refused at once, and its own words when late. + let sbx = |in_cluster, refused: Option<&str>| Seen { + sandbox: true, + refused: refused.map(str::to_owned), + ..seen(Some("running"), true, in_cluster, None) + }; + assert_eq!(next_state(&sbx(Some(true), None), soon), ("ready", None)); + let (s, why) = next_state(&sbx(Some(false), Some("runsc isn't installed.")), soon); + assert_eq!(s, "failed"); + assert_eq!( + why.unwrap(), + "It can't run sandboxes. runsc isn't installed." + ); + let (s, why) = next_state(&sbx(Some(false), None), late); + assert_eq!(s, "failed"); + assert!(why.unwrap().contains("didn't become a sandbox server")); } } diff --git a/crates/pyo-api/src/github.rs b/crates/pyo-api/src/github.rs index e5c5d25..ed8ccd7 100644 --- a/crates/pyo-api/src/github.rs +++ b/crates/pyo-api/src/github.rs @@ -210,7 +210,7 @@ async fn created( webhook, }) = take(&q.state).await else { - return Ok(Redirect::to("/settings?github=expired")); + return Ok(Redirect::to("/github?github=expired")); }; if user_id != user.id || !user.is_admin { return Err(ApiError::forbidden( @@ -225,7 +225,7 @@ async fn created( .map_err(anyhow::Error::from)?; if !res.status().is_success() { tracing::warn!(status = %res.status(), "GitHub app conversion failed"); - return Ok(Redirect::to("/settings?github=failed")); + return Ok(Redirect::to("/github?github=app-failed")); } let v: Value = res.json().await.map_err(anyhow::Error::from)?; let seal = |k: &str| s.sealer.seal(v[k].as_str().unwrap_or("")); @@ -246,7 +246,8 @@ async fn created( .bind(serde_json::to_value(&app).map_err(anyhow::Error::from)?) .execute(&s.db) .await?; - Ok(Redirect::to("/settings?github=created")) + // Back to GitHub in Settings, which says how it went. + Ok(Redirect::to("/github?github=created")) } // ── Installations (teams) ──────────────────────────────────────────────────── @@ -304,6 +305,7 @@ async fn installed( .fetch_optional(&s.db) .await?; let Some(t) = linked else { + // The dashboard opens the current team's GitHub card (/?github=). return Ok(Redirect::to("/?github=expired")); }; let mut tx = s.db.begin().await?; diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index 705c578..15ef499 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -26,6 +26,9 @@ pub const PUBLISHED_SCRIPTS_URL: &str = "https://pyo.dev"; /// 3. Pyo on the tailnet ([`crate::web::tailnet_api_url`]), then its HTTPS /// dashboard; /// 4. pyo.dev, when Pyo is only on its own machine or in its cluster. +/// +/// Only for machines already on the tailnet, or told Pyo's public URL. A +/// command with a Tailscale key uses [`scripts_url_before_tailnet`]. pub fn scripts_url(reach: Option<&str>) -> String { let configured = std::env::var("PYO_SCRIPTS_URL").ok(); let tailnet = crate::web::tailnet_api_url().map(|u| friendly(&u)); @@ -40,8 +43,8 @@ pub fn scripts_url(reach: Option<&str>) -> String { } fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String { - if let Some(u) = configured.map(str::trim).filter(|u| !u.is_empty()) { - return u.trim_end_matches('/').to_owned(); + if let Some(u) = setting(configured) { + return u; } match pyo .iter() @@ -54,6 +57,38 @@ fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String { } } +/// Where a command that gives the machine a Tailscale key (`--tailscale-key`) +/// fetches its script. The machine joins the tailnet only once the script +/// runs, so the script can't come from Pyo on the tailnet. Nor from Pyo on +/// its public IP: that address may not be reachable from outside (a server +/// behind NAT, like Starlink's). So: +/// 1. `PYO_SCRIPTS_URL`, when set; +/// 2. else pyo.dev. Its sandbox.sh runs the script of the release +/// `--version` names. Its join.sh and connect.sh run the latest +/// release's script, and join.sh's `--k3s` keeps the server on the +/// cluster's k3s. +/// +/// A dev build's version may not be a published release, so pyo.dev has no +/// sandbox.sh for it and its wrapper fails to download one: dev builds set +/// `PYO_SCRIPTS_URL` (to a mirror, or a Pyo the new machine can reach). +/// +/// Commands for machines already on the tailnet use [`scripts_url`]. +pub fn scripts_url_before_tailnet() -> String { + pick_scripts_url_before_tailnet(std::env::var("PYO_SCRIPTS_URL").ok().as_deref()) +} + +pub(crate) fn pick_scripts_url_before_tailnet(configured: Option<&str>) -> String { + setting(configured).unwrap_or_else(|| PUBLISHED_SCRIPTS_URL.to_owned()) +} + +/// `PYO_SCRIPTS_URL`, when it's set to something. +fn setting(configured: Option<&str>) -> Option { + configured + .map(str::trim) + .filter(|u| !u.is_empty()) + .map(|u| u.trim_end_matches('/').to_owned()) +} + /// The host of an http(s) URL, lowercase, without brackets or port. fn url_host(url: &str) -> Option { let rest = url @@ -813,6 +848,63 @@ mod tests { assert_eq!(pick_scripts_url(None, &[None, None]), PUBLISHED_SCRIPTS_URL); } + #[test] + fn a_machine_off_the_tailnet_fetches_scripts_from_the_setting_or_pyo_dev() { + let pick = pick_scripts_url_before_tailnet; + assert_eq!( + pick(Some(" https://mirror.example.com/ ")), + "https://mirror.example.com" + ); + // Unset or blank: pyo.dev, never Pyo itself (on the tailnet, or on + // a public IP that may sit behind NAT). + for unset in [None, Some(""), Some(" ")] { + assert_eq!(pick(unset), PUBLISHED_SCRIPTS_URL); + } + } + + #[test] + fn commands_with_a_tailscale_key_fetch_their_script_off_the_tailnet() { + let base = pick_scripts_url_before_tailnet(None); + // join.sh: pyo.dev runs the latest release's join.sh, and --k3s keeps + // the server on the cluster's k3s. + assert_eq!( + command( + &base, + &Join { + server_ip: "server-1.tail1234.ts.net", + token: "t", + server_name: "server-2", + k3s_version: "v1.37.2+k3s1", + tailscale_key: Some("tskey-auth-1"), + builder: false, + registry: None, + }, + ), + "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'" + ); + assert_eq!( + connect_command( + &base, + "https://pyo.tail1234.ts.net", + "abc", + Some("tskey-auth-1"), + Some(("pyo-registry.tail1234.ts.net", "def")) + ), + "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'" + ); + // A mirror the operator set. + assert!( + connect_command( + &pick_scripts_url_before_tailnet(Some("https://mirror.example.com")), + "https://pyo.tail1234.ts.net", + "abc", + Some("k"), + None + ) + .starts_with("curl -fsSL https://mirror.example.com/connect.sh | ") + ); + } + #[test] fn plain_http_only_to_private_addresses() { for ok in [ diff --git a/crates/pyo-api/src/lib.rs b/crates/pyo-api/src/lib.rs index b422d15..67cdf9e 100644 --- a/crates/pyo-api/src/lib.rs +++ b/crates/pyo-api/src/lib.rs @@ -366,6 +366,7 @@ pub async fn run(ext: Extensions) -> anyhow::Result<()> { tokio::spawn(tailscale::runner(state.clone())); tokio::spawn(cdn::keeper(state.clone())); tokio::spawn(sandbox::runner::runner(state.clone())); + tokio::spawn(sandbox::base::seed(state.clone())); for task in tasks { tokio::spawn(task(state.clone())); } diff --git a/crates/pyo-api/src/mcp.rs b/crates/pyo-api/src/mcp.rs index 03d05f0..dba3850 100644 --- a/crates/pyo-api/src/mcp.rs +++ b/crates/pyo-api/src/mcp.rs @@ -688,9 +688,12 @@ async fn run_endpoint(s: &AppState, user: &User, id: &str, args: &Value) -> Resu "listSandboxServers" => { json_of(crate::sandbox::api::list_servers(st(), me(), Query(args_as(args)?)).await) } - "removeSandboxServer" => { - json_of(crate::sandbox::api::remove_server(st(), me(), path_id()?).await) + "createCloudSandboxServer" => { + json_of(crate::sandbox::api::add_cloud_server(st(), me(), Json(args_as(args)?)).await) } + "removeSandboxServer" => json_of( + crate::sandbox::api::remove_server(st(), me(), path_id()?, Query(args_as(args)?)).await, + ), "listSandboxTemplates" => { json_of(crate::sandbox::api::list_templates(st(), me(), Query(args_as(args)?)).await) } diff --git a/crates/pyo-api/src/openapi.rs b/crates/pyo-api/src/openapi.rs index cf9b654..7c700be 100644 --- a/crates/pyo-api/src/openapi.rs +++ b/crates/pyo-api/src/openapi.rs @@ -85,6 +85,7 @@ use crate::{auth::KeyAccess, error::ErrorBody, ops::Op}; crate::sandbox::api::close_port, crate::sandbox::api::list_servers, crate::sandbox::api::add_server, + crate::sandbox::api::add_cloud_server, crate::sandbox::api::remove_server, crate::sandbox::api::issue_certificate, crate::sandbox::api::create_dns_record, diff --git a/crates/pyo-api/src/ops.rs b/crates/pyo-api/src/ops.rs index 9536807..1364901 100644 --- a/crates/pyo-api/src/ops.rs +++ b/crates/pyo-api/src/ops.rs @@ -330,9 +330,16 @@ pub async fn team_of( /// Top-level paths the dashboard owns can't be team slugs. // "pyo": Pyo registry keeps Pyo's own builds under pyo/. -const RESERVED_SLUGS: [&str; 18] = [ +// "backups" to "updates": the installation's settings pages +// (SETTINGS_PAGES in dash/src/lib/pyo/paths.ts). +const RESERVED_SLUGS: [&str; 23] = [ "profile", "settings", + "backups", + "email", + "github", + "observability", + "updates", "oauth", "api-keys", "mcp", @@ -742,7 +749,7 @@ async fn reach( let api = crate::web::tailnet_api_url().ok_or_else(|| { ApiError::conflict("Pyo's server isn't on your tailnet yet. Try again in a minute.") })?; - let key = crate::tailscale::join_key(s, what).await?; + let key = crate::tailscale::join_key(s, what, "this server").await?; let door = crate::join::friendly(&deploy::registry_push()); let registry = if door.is_empty() { None @@ -817,9 +824,11 @@ async fn connect_command( )); } let (api, key, registry) = reach(tx, s, cluster_id, "Pyo: connect a cluster").await?; + // The machine joins the tailnet in connect.sh (--tailscale-key), so it + // fetches the script off the tailnet. Ok(( crate::join::connect_command( - &crate::join::scripts_url(Some(&api)), + &crate::join::scripts_url_before_tailnet(), &api, token, Some(&key), @@ -832,7 +841,8 @@ async fn connect_command( /// What a new server of a cluster joins with (a k3s join token the agent /// makes, valid an hour): join.sh's details and a fresh Tailscale key. struct JoinParts { - /// Where join.sh is. + /// Where join.sh is for a server already on the tailnet. A command + /// with the Tailscale key uses [`crate::join::scripts_url_before_tailnet`]. base: String, server_ip: String, token: String, @@ -873,9 +883,12 @@ async fn join_command( wake: &mut Vec, ) -> ApiResult { let p = join_parts(tx, s, team_id, cluster_id, server_name, wake).await?; - let command = crate::join::command(&p.base, &p.join(server_name, builder, true)); + // The server joins the tailnet in join.sh (--tailscale-key), so it + // fetches the script off the tailnet. + let base = crate::join::scripts_url_before_tailnet(); + let command = crate::join::command(&base, &p.join(server_name, builder, true)); Ok( - json!({ "ok": true, "command": command, "scriptUrl": format!("{}/join.sh", p.base), "expiresAt": now_ms() + p.ttl as i64 * 1000, "tailscale": true }), + json!({ "ok": true, "command": command, "scriptUrl": format!("{base}/join.sh"), "expiresAt": now_ms() + p.ttl as i64 * 1000, "tailscale": true }), ) } @@ -975,7 +988,7 @@ fn cloud_ssh_key(k: Option) -> ApiResult> { } /// A price as people read it: "€4.29". -fn money(currency: &str, amount: f64) -> String { +pub(crate) fn money(currency: &str, amount: f64) -> String { match currency { "EUR" => format!("€{amount:.2}"), "USD" => format!("${amount:.2}"), @@ -1006,31 +1019,32 @@ async fn cloud_name_free( } /// A server to make at the provider. -struct CloudNew<'a> { - id: &'a str, - team_id: &'a str, - cluster_id: &'a str, - name: &'a str, - location: &'a str, - server_type: &'a str, - monthly: f64, - currency: &'a str, +pub(crate) struct CloudNew<'a> { + pub id: &'a str, + pub team_id: &'a str, + /// Its cluster, or the sandbox server it becomes. + pub owner: crate::cloud::Owner<'a>, + pub name: &'a str, + pub location: &'a str, + pub server_type: &'a str, + pub monthly: f64, + pub currency: &'a str, /// The cluster's first server (connect.sh). - first: bool, - builder: bool, + pub first: bool, + pub builder: bool, /// Single-use, for its user data. - tailscale_key: &'a str, - /// What its code opens: join.sh's or connect.sh's details. - material: crate::cloud::bootstrap::Material, - ssh_key: Option<&'a str>, - by: &'a str, + pub tailscale_key: &'a str, + /// What its code opens: join.sh's, connect.sh's or sandbox.sh's details. + pub material: crate::cloud::bootstrap::Material, + pub ssh_key: Option<&'a str>, + pub by: &'a str, } /// Makes the server at the provider, labelled as this installation's and /// behind its firewall, with user data that holds only its Tailscale key /// and a single-use code (cloud/bootstrap.rs), and keeps it to follow /// (cloud/mod.rs `tick`). The last step before the operation commits. -async fn make_cloud_server( +pub(crate) async fn make_cloud_server( s: &AppState, p: &P, n: CloudNew<'_>, @@ -1043,10 +1057,15 @@ async fn make_cloud_server( "Pyo's server isn't on your tailnet yet. Try again in a minute.", )); } - let role = if n.first { - bootstrap::Role::Connect - } else { - bootstrap::Role::Join + use crate::cloud::Owner; + let role = match n.owner { + Owner::Sandbox(_) => bootstrap::Role::Sandbox, + Owner::Cluster(_) if n.first => bootstrap::Role::Connect, + Owner::Cluster(_) => bootstrap::Role::Join, + }; + let (cluster_id, sandbox_server_id) = match n.owner { + Owner::Cluster(c) => (Some(c), None), + Owner::Sandbox(x) => (None, Some(x)), }; // Checked before anything is made: its details are kept sealed. n.material.render()?; @@ -1061,7 +1080,7 @@ async fn make_cloud_server( &crate::cloud::Boot { pyo: &format!("http://{door}"), server_id: n.id, - cluster_id: n.cluster_id, + cluster_id: n.owner.id(), role, name: n.name, tailscale_key: n.tailscale_key, @@ -1077,7 +1096,7 @@ async fn make_cloud_server( location: n.location, server_type: n.server_type, user_data: &user_data, - labels: crate::cloud::labels(&install, n.cluster_id), + labels: crate::cloud::labels(&install, n.owner), firewall: Some(&firewall), }) .await?; @@ -1087,12 +1106,13 @@ async fn make_cloud_server( "cloud server created" ); sqlx::query( - "INSERT INTO pyo.cloud_servers (id, team_id, provider, cluster_id, name, first_server, builder, location, server_type, monthly, currency, remote_id, ipv4, state, created_by) - VALUES ($1, $2, 'hetzner', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 'creating', $13)", + "INSERT INTO pyo.cloud_servers (id, team_id, provider, cluster_id, sandbox_server_id, name, first_server, builder, location, server_type, monthly, currency, remote_id, ipv4, state, created_by) + VALUES ($1, $2, 'hetzner', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, 'creating', $14)", ) .bind(n.id) .bind(n.team_id) - .bind(n.cluster_id) + .bind(cluster_id) + .bind(sandbox_server_id) .bind(n.name) .bind(n.first) .bind(n.builder) @@ -1105,10 +1125,10 @@ async fn make_cloud_server( .bind(n.by) .execute(&mut **tx) .await?; - bootstrap::keep(tx, &s.sealer, &code, n.id, n.cluster_id, role, &n.material).await?; + bootstrap::keep(tx, &s.sealer, &code, n.id, n.owner.id(), role, &n.material).await?; Ok(json!({ - "ok": true, "id": n.id, "clusterId": n.cluster_id, "name": n.name, "state": "creating", - "monthly": n.monthly, "currency": n.currency, + "ok": true, "id": n.id, "clusterId": cluster_id, "sandboxServerId": sandbox_server_id, + "name": n.name, "state": "creating", "monthly": n.monthly, "currency": n.currency, })) } @@ -1123,16 +1143,17 @@ async fn delete_cloud_server( confirm: bool, wake: &mut Vec, ) -> ApiResult { - let (team_id, cluster_id, name, first, state, monthly, currency): ( + let (team_id, cluster_id, sandbox_server_id, name, first, state, monthly, currency): ( String, Option, + Option, String, bool, String, f64, String, ) = sqlx::query_as( - "SELECT team_id, cluster_id, name, first_server, state, monthly, currency FROM pyo.cloud_servers WHERE id = $1 AND state <> 'deleted'", + "SELECT team_id, cluster_id, sandbox_server_id, name, first_server, state, monthly, currency FROM pyo.cloud_servers WHERE id = $1 AND state <> 'deleted'", ) .bind(id) .fetch_optional(&mut **tx) @@ -1185,6 +1206,11 @@ async fn delete_cloud_server( remove_node(tx, user, &server_id, !online, wake).await?; } } + // A sandbox server goes with it: its sandboxes end with the server. + if let Some(sbs) = &sandbox_server_id { + crate::sandbox::forget_server(tx, sbs, "Its sandbox server was deleted at Hetzner.") + .await?; + } sqlx::query("UPDATE pyo.cloud_servers SET state = 'removing', error = NULL, sign_command = NULL, updated_at = now() WHERE id = $1") .bind(id) .execute(&mut **tx) @@ -7032,7 +7058,7 @@ async fn apply( CloudNew { id: &id, team_id: &team_id, - cluster_id: &cluster_id, + owner: crate::cloud::Owner::Cluster(&cluster_id), name: &name, location: &location, server_type: &server_type, @@ -7125,7 +7151,7 @@ async fn apply( CloudNew { id: &server_id, team_id: &team_id, - cluster_id: &id, + owner: crate::cloud::Owner::Cluster(&id), name: &server, location: &location, server_type: &server_type, diff --git a/crates/pyo-api/src/sandbox/api.rs b/crates/pyo-api/src/sandbox/api.rs index 52b8637..c909421 100644 --- a/crates/pyo-api/src/sandbox/api.rs +++ b/crates/pyo-api/src/sandbox/api.rs @@ -12,7 +12,7 @@ use axum::{ use serde::Deserialize; use serde_json::{Value, json}; -use super::{AddServer, Connection, CreateSandbox, CreateTemplate, Need}; +use super::{AddCloudServer, AddServer, Connection, CreateSandbox, CreateTemplate, Need}; use crate::{ AppState, auth::SessionUser, @@ -33,6 +33,7 @@ pub fn routes() -> Router { axum::routing::delete(close_port), ) .route("/sandbox-servers", get(list_servers).post(add_server)) + .route("/sandbox-servers/cloud", post(add_cloud_server)) .route( "/sandbox-servers/{id}", axum::routing::delete(remove_server), @@ -328,7 +329,7 @@ pub(crate) async fn token( #[utoipa::path(get, path = "/sandbox-servers", operation_id = "listSandboxServers", tag = "Sandboxes", params(TeamQuery), responses( - (status = 200, description = "`servers`, each with `id`, `teamId` (null for the installation's), `name`, `address`, `state` (\"joining\", \"ready\", \"unreachable\" or \"refused\"), `enabled`, `capability` (`runtime`, `kvm`, `bootMs`, `arch`, `cpus`, `memory`, `disk` and the checks), `summary` (one line about it), `capacity`, `templates`, `daemonVersion`, `lastReportAt`, `error`, `sandboxes` (how many run there), `createdAt`, `edgeDomain` and, with an edge domain, `edge`: its `domain`, `https` (whether it serves HTTPS), `cert` (`state`: \"none\", \"issuing\", \"ready\" or \"failed\", `notAfter`, `issuedAt`, `error`, `served`), `cloudflare` (the connected zone that covers it: `teamId`, `zone`, `canEdit`, or null) and `dns` (`name`, the `records` to point it at the server, and what Pyo `made`).", body = Object), + (status = 200, description = "`servers`, each with `id`, `teamId` (null for the installation's), `name`, `address`, `state` (\"joining\", \"ready\", \"unreachable\" or \"refused\"), `enabled`, `runtime` (\"firecracker\" or \"gvisor\", what runs its sandboxes, null until it reports), `capability` (`runtime`, `kvm`, `bootMs`, `arch`, `cpus`, `memory`, `disk`, `gvisorVersion` and the checks), `summary` (one line about it), `capacity`, `templates`, `daemonVersion`, `lastReportAt`, `error`, `sandboxes` (how many run there), `createdAt`, `edgeDomain` and, with an edge domain, `edge`: its `domain`, `https` (whether it serves HTTPS), `cert` (`state`: \"none\", \"issuing\", \"ready\" or \"failed\", `notAfter`, `issuedAt`, `error`, `served`), `cloudflare` (the connected zone that covers it: `teamId`, `zone`, `canEdit`, or null) and `dns` (`name`, the `records` to point it at the server, and what Pyo `made`).", body = Object), (status = 404, description = "No such team, or you aren't in it.", body = ErrorBody), ), extensions(("x-pyo-access" = json!("read"))))] @@ -367,23 +368,64 @@ pub(crate) async fn add_server( Ok(Json(super::add_server(&s, &user, req).await?)) } +/// Buy a sandbox server at a cloud provider +/// +/// Makes a server in the team's Hetzner project and sets it up as a sandbox +/// server, the way Pyo makes a cluster's servers there. Read the locations, +/// server types and prices with getCloudOptions first. On its first boot it +/// joins your tailnet and runs sandbox.sh with a one-time code. Hetzner +/// Cloud servers have no KVM, so it runs sandboxes on gVisor. Follow it in +/// the server's `cloud` (creating, booting, joining, ready) and `state`. +/// Without `confirm` it answers 409 with the monthly price. Confirm that +/// with the user, then send it again with `confirm: true`. Team owners and +/// admins. Needs Tailscale and the team's Hetzner connection. +#[utoipa::path(post, path = "/sandbox-servers/cloud", operation_id = "createCloudSandboxServer", tag = "Sandboxes", + request_body(content = AddCloudServer), + responses( + (status = 200, description = "`server` (as listSandboxServers shows it) and `cloudServer` (`id`, `name`, `state`, `monthly`, `currency`).", body = Object), + (status = 400, description = "A bad name, location or server type, or Tailscale or Hetzner isn't set up.", body = ErrorBody), + (status = 409, description = "`needsConfirm` with the monthly price, a name in use, a type Hetzner can't make now, or Pyo isn't on the tailnet yet.", body = ErrorBody), + (status = 502, description = "Hetzner failed.", body = ErrorBody), + ), + extensions(("x-pyo-access" = json!("full"))))] +pub(crate) async fn add_cloud_server( + State(s): State, + SessionUser(user): SessionUser, + Json(req): Json, +) -> ApiResult> { + Ok(Json(super::add_cloud_server(&s, &user, req).await?)) +} + +#[derive(Debug, Default, Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct RemoveQuery { + /// Delete a server Pyo made at a cloud provider: without it the answer + /// is 409 with the monthly cost that stops. + #[serde(default)] + confirm: bool, +} + /// Remove a sandbox server /// /// Deletes its sandboxes where it still answers, and forgets it. To take the -/// daemon off the server, run `sandbox.sh --uninstall` there. +/// daemon off the server, run `sandbox.sh --uninstall` there. A server Pyo +/// made at a cloud provider leaves your tailnet and is deleted there, which +/// takes `confirm`. #[utoipa::path(delete, path = "/sandbox-servers/{id}", operation_id = "removeSandboxServer", tag = "Sandboxes", - params(("id" = String, Path, description = "The sandbox server's id, `sbs_…`.")), + params(("id" = String, Path, description = "The sandbox server's id, `sbs_…`."), RemoveQuery), responses( - (status = 200, description = "`ok`, and `stopped`: how many sandboxes ended.", body = Object), + (status = 200, description = "`ok`, `stopped` (how many sandboxes ended) and `deleting` (whether it's being deleted at the provider).", body = Object), (status = 404, description = "No such sandbox server.", body = ErrorBody), + (status = 409, description = "`needsConfirm`: it's deleted at the provider, with the monthly cost that stops.", body = ErrorBody), ), extensions(("x-pyo-access" = json!("full"))))] pub(crate) async fn remove_server( State(s): State, SessionUser(user): SessionUser, Path(id): Path, + Query(q): Query, ) -> ApiResult> { - Ok(Json(super::remove_server(&s, &user, &id).await?)) + Ok(Json(super::remove_server(&s, &user, &id, q.confirm).await?)) } /// Get a sandbox server's certificate @@ -468,8 +510,9 @@ pub(crate) async fn list_templates( /// Create a sandbox template /// /// A template made from an OCI image, like "node:22", or an image in Pyo's -/// registry. Every sandbox server that may use it pulls the image, makes a -/// root disk from it, boots it once and snapshots it. The template is +/// registry. Every sandbox server that may use it pulls the image and makes +/// a root disk from it. A Firecracker server boots it once and snapshots +/// it, and a gVisor server starts sandboxes from the disk. The template is /// ready once one server has it. #[utoipa::path(post, path = "/sandbox-templates", operation_id = "createSandboxTemplate", tag = "Sandboxes", request_body(content = CreateTemplate), diff --git a/crates/pyo-api/src/sandbox/base.rs b/crates/pyo-api/src/sandbox/base.rs new file mode 100644 index 0000000..70c39fc --- /dev/null +++ b/crates/pyo-api/src/sandbox/base.rs @@ -0,0 +1,356 @@ +//! Pyo's built-in sandbox template `base`: Debian 13 with the common tools +//! (deploy/images/sandbox-base.Dockerfile). It's the installation's, so +//! every team has it, and a sandbox created without a template starts +//! from it. +//! +//! - Seeding: when a worker starts, and in every sandbox round, Pyo makes +//! sure `base` exists and points at its release's image +//! (`PYO_SANDBOX_BASE_IMAGE`, pinned by digest in release.json). It's +//! idempotent. +//! - On update: a new release has a new image, so `base` gets a new +//! revision (the digest's start). Servers make it beside the old one in +//! the background, and [`super::templates`] moves new sandboxes over +//! once it's ready and deletes the old one once no sandbox uses it. +//! An older Pyo still running during the update never moves it back. +//! - Dev builds have no image of their own: they use the latest release +//! Pyo found when it checked for updates, or wait for one. +//! - Nobody deletes it, and no other template may be named `base`. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use pyo_proto::{parse_version, pinned_image}; +use serde_json::{Value, json}; + +use crate::AppState; + +/// Its name, which sandboxes ask for. +pub const NAME: &str = "base"; +/// Its id, the same on every installation. +pub const ID: &str = "stp_base"; +/// What it runs, shown beside its image. +pub const OS: &str = "Debian 13"; + +/// The image to seed `base` from, and the release it belongs to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Wanted { + pub image: String, + pub release: String, +} + +/// This release's image, from `PYO_SANDBOX_BASE_IMAGE`. None for a build +/// that isn't a release (scripts/pyo-up.sh leaves it empty). +pub fn from_env() -> Option { + let image = std::env::var("PYO_SANDBOX_BASE_IMAGE").ok()?; + let image = image.trim(); + pinned_image(image).then(|| Wanted { + image: image.to_owned(), + release: crate::updates::current().to_owned(), + }) +} + +/// The revision servers keep an image under: the first 12 hex digits of +/// its digest. +pub fn revision(image: &str) -> Option { + let (_, digest) = image.split_once("@sha256:")?; + (digest.len() >= 12).then(|| digest[..12].to_owned()) +} + +/// The template's source for an image. +pub fn source(w: &Wanted) -> Value { + json!({ "kind": "image", "image": w.image, "os": OS, "release": w.release }) +} + +/// What seeding does, given the template's source now (None: there's no +/// `base` yet). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Seed { + Insert, + /// A new image: a new revision. + Move, + Keep, +} + +pub fn decide(have: Option<&Value>, want: &Wanted) -> Seed { + let Some(src) = have else { + return Seed::Insert; + }; + if src["image"].as_str() == Some(want.image.as_str()) { + return Seed::Keep; + } + // A newer Pyo already moved it: this one is on its way out. + let theirs = src["release"].as_str().and_then(parse_version); + match (theirs, parse_version(&want.release)) { + (Some(a), Some(b)) if a > b => Seed::Keep, + _ => Seed::Move, + } +} + +/// The image `base` should have now: this release's, else (a dev build) +/// the latest release found by the update check. +async fn wanted(s: &AppState) -> Option { + if let Some(w) = from_env() { + return Some(w); + } + let found = crate::updates::checked(&s.db).await.ok()?.found?; + let image = found.release.images.sandbox_base; + let pinned = pinned_image(&image); + pinned.then_some(Wanted { + image, + release: found.release.version, + }) +} + +/// Said once per process, so a dev build doesn't repeat it every round. +static SAID_WAITING: AtomicBool = AtomicBool::new(false); + +/// Makes sure `base` exists and has the image it should. Tells the servers +/// at once when that changed. +pub async fn ensure(s: &AppState) -> anyhow::Result<()> { + let Some(want) = wanted(s).await else { + if !SAID_WAITING.swap(true, Ordering::Relaxed) { + tracing::info!( + "sandbox template base: this build has no image for it, and no release was found yet, so it's added later" + ); + } + return Ok(()); + }; + seed_from(s, &want).await.map(|_| ()) +} + +/// Seeds `base` from `want`, and says what it did. +pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result { + let want = want.clone(); + let rev = revision(&want.image); + let mut tx = s.db.begin().await?; + let have: Option = + sqlx::query_scalar("SELECT source FROM pyo.sandbox_templates WHERE id = $1 FOR UPDATE") + .bind(ID) + .fetch_optional(&mut *tx) + .await?; + let seed = decide(have.as_ref(), &want); + match seed { + Seed::Keep => return Ok(Seed::Keep), + Seed::Insert => { + let res = sqlx::query( + "INSERT INTO pyo.sandbox_templates (id, team_id, name, source, state, created_by, builtin, revision) + VALUES ($1, NULL, $2, $3, 'pending', 'Pyo', true, $4) + ON CONFLICT (id) DO NOTHING", + ) + .bind(ID) + .bind(NAME) + .bind(source(&want)) + .bind(&rev) + .execute(&mut *tx) + .await; + if let Err(e) = res { + if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23505") { + tracing::warn!( + "sandbox template base: the installation has another template named base, so Pyo's isn't added" + ); + return Ok(Seed::Keep); + } + return Err(e.into()); + } + tracing::info!(image = %want.image, "sandbox template base added"); + } + Seed::Move => { + sqlx::query( + "UPDATE pyo.sandbox_templates + SET source = $2, revision = $3, builtin = true, error = NULL, updated_at = now() + WHERE id = $1", + ) + .bind(ID) + .bind(source(&want)) + .bind(&rev) + .execute(&mut *tx) + .await?; + tracing::info!(image = %want.image, "sandbox template base moved to a new image"); + } + } + tx.commit().await?; + tokio::spawn(super::templates::push(s.clone(), ID.to_owned())); + Ok(seed) +} + +/// On a worker's start: [`ensure`], retried while the database is busy. +pub async fn seed(s: AppState) { + for attempt in 1..=10u64 { + match ensure(&s).await { + Ok(()) => return, + Err(e) => { + tracing::warn!(attempt, error = %format!("{e:#}"), "sandbox template base: couldn't add it, retrying"); + tokio::time::sleep(std::time::Duration::from_secs(15 * attempt)).await; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + const B: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.2@sha256:cccccccccccc0000000000000000000000000000000000000000000000000000"; + + fn want(image: &str, release: &str) -> Wanted { + Wanted { + image: image.into(), + release: release.into(), + } + } + + #[test] + fn seeding_is_idempotent() { + let w = want(A, "0.4.1"); + assert_eq!(decide(None, &w), Seed::Insert); + let src = source(&w); + assert_eq!(decide(Some(&src), &w), Seed::Keep, "the same image again"); + assert_eq!(src["os"], "Debian 13"); + assert_eq!(src["kind"], "image"); + } + + #[test] + fn a_new_release_moves_it_forward_only() { + let old = source(&want(A, "0.4.1")); + let new = want(B, "0.4.2"); + assert_eq!(decide(Some(&old), &new), Seed::Move); + // The older Pyo, still running during the update, leaves it. + assert_eq!(decide(Some(&source(&new)), &want(A, "0.4.1")), Seed::Keep); + // A rebuilt image of the same release moves it too. + assert_eq!(decide(Some(&old), &want(B, "0.4.1")), Seed::Move); + // One without a release (made by hand) is taken over. + assert_eq!( + decide( + Some(&json!({ "kind": "image", "image": "debian:13" })), + &new + ), + Seed::Move + ); + } + + #[test] + fn revisions_come_from_the_digest() { + assert_eq!(revision(A).as_deref(), Some("aaaaaaaaaaaa")); + assert_eq!(revision(B).as_deref(), Some("cccccccccccc")); + assert_eq!(revision("debian:13"), None); + let name = super::super::templates::server_name(ID, revision(A).as_deref()); + assert_eq!(name, "stp_base.aaaaaaaaaaaa"); + assert!(pyo_proto::sandbox::is_template_name(&name)); + } + + #[test] + fn only_a_pinned_image_from_the_env_counts() { + // SAFETY: tests in this module don't read it concurrently. + unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", "") }; + assert_eq!(from_env(), None, "a dev build (pyo-up.sh)"); + unsafe { + std::env::set_var( + "PYO_SANDBOX_BASE_IMAGE", + "127.0.0.1:5005/pyo/sandbox-base:dev", + ) + }; + assert_eq!(from_env(), None, "not pinned"); + unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", A) }; + assert_eq!(from_env(), Some(want(A, crate::updates::current()))); + unsafe { std::env::remove_var("PYO_SANDBOX_BASE_IMAGE") }; + assert_eq!(from_env(), None); + } + + /// Against a real Postgres (DATABASE_URL, a throwaway one, in a database + /// of its own): seeding adds `base` once, moves it to a new release's + /// image, never back, and nobody deletes it or takes its name. + #[tokio::test] + #[ignore = "needs DATABASE_URL (a throwaway Postgres)"] + async fn seeds_base_in_the_database() { + use sqlx::PgPool; + use sqlx::postgres::PgConnectOptions; + let opts: PgConnectOptions = std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + let admin = PgPool::connect_with(opts.clone()).await.unwrap(); + let name = format!("pyo_base_{}", std::process::id()); + for q in [ + format!("DROP DATABASE IF EXISTS {name}"), + format!("CREATE DATABASE {name}"), + ] { + sqlx::raw_sql(sqlx::AssertSqlSafe(q)) + .execute(&admin) + .await + .unwrap(); + } + let db = PgPool::connect_with(opts.database(&name)).await.unwrap(); + crate::migrate::run(&db).await.unwrap(); + let s = super::super::e2e::test_state(db.clone()); + let row = || async { + sqlx::query_as::<_, (String, String, Value, Option, bool, Option)>( + "SELECT name, state, source, revision, builtin, team_id FROM pyo.sandbox_templates WHERE id = $1", + ) + .bind(ID) + .fetch_one(&db) + .await + .unwrap() + }; + + let old = want(A, "0.4.1"); + assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Insert); + assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep, "idempotent"); + let (n, state, src, rev, builtin, team) = row().await; + assert_eq!( + (n.as_str(), state.as_str(), rev.as_deref(), builtin, team), + ("base", "pending", Some("aaaaaaaaaaaa"), true, None) + ); + assert_eq!(src["image"], A); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pyo.sandbox_templates") + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(count, 1); + + // Every team sees it, as the installation's, at its current revision. + let views = super::super::template_views(&db, &["tm_any".into()], None) + .await + .unwrap(); + assert_eq!(views[0]["name"], "base"); + assert_eq!(views[0]["builtin"], true); + assert_eq!(views[0]["source"]["os"], "Debian 13"); + + // A new release moves it. The old Pyo, still running, doesn't move it back. + let new = want(B, "0.4.2"); + assert_eq!(seed_from(&s, &new).await.unwrap(), Seed::Move); + assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep); + let (_, _, src, rev, ..) = row().await; + assert_eq!( + (src["image"].as_str(), rev.as_deref()), + (Some(B), Some("cccccccccccc")) + ); + + // Nobody deletes it, and no team takes its name. + let user = crate::auth::User { + id: "usr_base".into(), + name: "Admin".into(), + email: "admin@base.test".into(), + timezone: "UTC".into(), + is_admin: true, + must_set_up: false, + key: None, + }; + let e = super::super::delete_template(&s, &user, ID) + .await + .unwrap_err(); + assert_eq!(e.status, axum::http::StatusCode::CONFLICT, "{}", e.message); + let e = super::super::create_template( + &s, + &user, + serde_json::from_value(json!({ "name": "base", "image": "debian:13" })).unwrap(), + ) + .await + .unwrap_err(); + assert_eq!(e.status, axum::http::StatusCode::CONFLICT, "{}", e.message); + + db.close().await; + sqlx::raw_sql(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .unwrap(); + } +} diff --git a/crates/pyo-api/src/sandbox/e2e.rs b/crates/pyo-api/src/sandbox/e2e.rs index 65affe6..893f8fc 100644 --- a/crates/pyo-api/src/sandbox/e2e.rs +++ b/crates/pyo-api/src/sandbox/e2e.rs @@ -604,6 +604,101 @@ async fn the_control_plane_drives_a_real_daemon() { tokio::time::sleep(Duration::from_millis(500)).await; } + // ── The built-in template `base`: the default, and how a release moves it ── + // Two small public images stand in for two releases' sandbox-base. + use super::base; + sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1") + .bind(base::ID) + .execute(&db) + .await + .unwrap(); + let old = base::Wanted { + image: + "alpine:3.21@sha256:bc21487101ae47172509130ee1092cc273a831968d64b617776d1f69c81e42ab" + .into(), + release: "0.0.1".into(), + }; + let new = base::Wanted { + image: + "alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f" + .into(), + release: "0.0.2".into(), + }; + let rev = |w: &base::Wanted| { + super::templates::server_name(base::ID, base::revision(&w.image).as_deref()) + }; + let (old_name, new_name) = (rev(&old), rev(&new)); + assert_eq!(base::seed_from(&s, &old).await.unwrap(), base::Seed::Insert); + wait_for(&s, &team, &server, &old_name, false).await; + // No template: base. + let no_template: super::CreateSandbox = + serde_json::from_value(json!({ "teamId": team, "network": "none", "ttl": "5m" })).unwrap(); + let first = super::create(&s, &user, no_template.clone()).await.unwrap(); + assert_eq!(first["sandbox"]["template"], "base", "{first}"); + let first_id = first["sandbox"]["id"].as_str().unwrap().to_owned(); + let on: String = + sqlx::query_scalar("SELECT spec->>'template' FROM pyo.sandboxes WHERE id = $1") + .bind(&first_id) + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(on, old_name); + let r = Daemon::new(&address) + .exec( + first["connection"]["token"].as_str().unwrap(), + &first_id, + &exec_req("cat /etc/alpine-release"), + ) + .await + .unwrap(); + assert!(r.stdout.starts_with("3.21"), "{r:?}"); + + // A new release: made beside the old one, new sandboxes move over. + assert_eq!(base::seed_from(&s, &new).await.unwrap(), base::Seed::Move); + assert_eq!( + base::seed_from(&s, &old).await.unwrap(), + base::Seed::Keep, + "the old Pyo doesn't move it back" + ); + wait_for(&s, &team, &server, &new_name, false).await; + let second = super::create(&s, &user, no_template).await.unwrap(); + let second_id = second["sandbox"]["id"].as_str().unwrap().to_owned(); + let r = Daemon::new(&address) + .exec( + second["connection"]["token"].as_str().unwrap(), + &second_id, + &exec_req("cat /etc/alpine-release"), + ) + .await + .unwrap(); + assert!(r.stdout.starts_with("3.22"), "{r:?}"); + // The running sandbox keeps the old disk, which goes once it ends. + super::runner::round(&s).await.unwrap(); + assert!(ready_on(&db, &team, &server).await.contains(&old_name)); + let r = Daemon::new(&address) + .exec( + first["connection"]["token"].as_str().unwrap(), + &first_id, + &exec_req("cat /etc/alpine-release"), + ) + .await + .unwrap(); + assert!(r.stdout.starts_with("3.21"), "{r:?}"); + super::delete(&s, &user, &first_id).await.unwrap(); + wait_for(&s, &team, &server, &old_name, true).await; + let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::ID)) + .await + .unwrap(); + assert_eq!(views[0]["state"], "ready", "{}", views[0]); + println!("e2e base: {}", views[0]["servers"]); + super::delete(&s, &user, &second_id).await.unwrap(); + sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1") + .bind(base::ID) + .execute(&db) + .await + .unwrap(); + super::templates::forget(s.clone(), base::ID.into()).await; + sqlx::query("DELETE FROM pyo.teams WHERE id = $1") .bind(&team) .execute(&db) @@ -617,7 +712,38 @@ async fn the_control_plane_drives_a_real_daemon() { keys::reset_cache(); } -fn test_state(db: PgPool) -> crate::AppState { +/// The templates a server reports ready, by the names it keeps them under. +async fn ready_on(db: &PgPool, team: &str, server: &str) -> Vec { + let srv = &super::server_views(db, &[team.to_owned()], Some(server)) + .await + .unwrap()[0]; + srv["templates"] + .as_array() + .unwrap() + .iter() + .filter(|t| t["state"] == "ready" || t["state"].is_null()) + .filter_map(|t| t["name"].as_str().map(str::to_owned)) + .collect() +} + +/// Runs the runner until the server has `name` ready, or (`gone`) no longer has it. +async fn wait_for(s: &crate::AppState, team: &str, server: &str, name: &str, gone: bool) { + let t0 = std::time::Instant::now(); + loop { + super::runner::round(s).await.unwrap(); + let names = ready_on(&s.db, team, server).await; + if names.iter().any(|n| n == name) != gone { + return; + } + assert!( + t0.elapsed() < Duration::from_secs(300), + "{name} gone={gone}: {names:?}" + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +pub(super) fn test_state(db: PgPool) -> crate::AppState { let config = crate::Config { agent_api_url: String::new(), agent_image: String::new(), diff --git a/crates/pyo-api/src/sandbox/join.rs b/crates/pyo-api/src/sandbox/join.rs index ec37b8b..95aeb2e 100644 --- a/crates/pyo-api/src/sandbox/join.rs +++ b/crates/pyo-api/src/sandbox/join.rs @@ -221,6 +221,8 @@ pub fn daemon_config( "jailer": format!("{LIB_DIR}/jailer"), "kernel": format!("{LIB_DIR}/vmlinux"), "envd": format!("{LIB_DIR}/pyo-envd"), + // sandbox.sh puts gVisor's runsc there on servers without KVM. + "runsc": format!("{LIB_DIR}/runsc"), "keys": keys, "jailUid": jail_uid, "jailGid": jail_gid, @@ -382,7 +384,8 @@ fn q(s: &str) -> String { /// The command that makes a machine a sandbox server. pub struct Command<'a> { - /// Where scripts come from ([`crate::join::scripts_url`]). + /// Where scripts come from, off the tailnet the server isn't on yet + /// ([`crate::join::scripts_url_before_tailnet`]). pub base: &'a str, /// Pyo's tailnet door, "pyo.tail1234.ts.net:5006". pub door: &'a str, @@ -468,6 +471,7 @@ mod tests { assert_eq!(v["listen"], "100.64.0.7:7070"); assert_eq!(v["keys"][0]["publicKey"], "AAAA"); assert_eq!(v["jailUid"], 998); + assert_eq!(v["runsc"], "/usr/local/lib/pyo-sandbox/runsc"); assert_eq!( daemon_config("s", "fd7a:115c:a1e0::9", 1, 1, &keys)["listen"], "[fd7a:115c:a1e0::9]:7070" @@ -476,8 +480,10 @@ mod tests { #[test] fn the_command_quotes_everything() { + // The server isn't on the tailnet yet: pyo.dev. + let base = crate::join::pick_scripts_url_before_tailnet(None); let c = Command { - base: "https://pyo.tail1234.ts.net/install", + base: &base, door: "pyo.tail1234.ts.net:5006", server_id: "sbs_one", name: "it's sbx", @@ -488,7 +494,7 @@ mod tests { }; assert_eq!( c.render(), - "curl -fsSL https://pyo.tail1234.ts.net/install/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'" + "curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'" ); let c = Command { edge_domain: Some("sbx.example.com"), @@ -502,6 +508,38 @@ mod tests { ); } + #[test] + fn a_new_server_fetches_sandbox_sh_off_the_tailnet() { + let render = |base: &str| { + Command { + base, + door: "pyo.tail1234.ts.net:5006", + server_id: "sbs_one", + name: "sbx", + code: "abc", + tailscale_key: "tskey-auth-1", + version: "0.4.0", + edge_domain: None, + } + .render() + }; + let pick = crate::join::pick_scripts_url_before_tailnet; + // PYO_SCRIPTS_URL wins. + assert!( + render(&pick(Some("https://mirror.example.com/"))) + .starts_with("curl -fsSL https://mirror.example.com/sandbox.sh | bash -s -- ") + ); + // Else pyo.dev, which runs the sandbox.sh of the --version release. + for unset in [None, Some(" ")] { + let c = render(&pick(unset)); + assert!( + c.starts_with("curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- ") + && c.contains(" --version '0.4.0'"), + "{c}" + ); + } + } + #[test] fn refusals_are_plain() { for r in [ diff --git a/crates/pyo-api/src/sandbox/mcp.rs b/crates/pyo-api/src/sandbox/mcp.rs index f164d5e..95c7390 100644 --- a/crates/pyo-api/src/sandbox/mcp.rs +++ b/crates/pyo-api/src/sandbox/mcp.rs @@ -39,13 +39,13 @@ pub fn defs() -> Vec { vec![ json!({ "name": "create_sandbox", "title": "Create sandbox", "description": "Create an isolated machine (a sandbox) on the team's sandbox servers, to run code, commands and files in. It ends after its lifetime (5 minutes when omitted). Returns its id.", "inputSchema": schema(json!({ "team": team, - "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them)" }, + "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them). base when omitted: Debian 13 with git, Python, uv, Node.js and pnpm" }, "cpus": { "type": "integer", "minimum": 1, "maximum": 32 }, "memory_mib": { "type": "integer", "minimum": 128 }, "ttl": { "type": "string", "description": "How long it lives, like 30m or 1h (24h at most)" }, "env": { "type": "object", "additionalProperties": { "type": "string" } }, "network": { "type": "string", "enum": ["internet", "none"] }, - }), &["template"]), "annotations": write }), + }), &[]), "annotations": write }), json!({ "name": "run_code", "title": "Run code", "description": "Run a piece of code in a sandbox and wait for it: its output and exit code.", "inputSchema": schema(json!({ "sandbox_id": sandbox, "code": { "type": "string" }, @@ -149,8 +149,9 @@ pub async fn run(s: &AppState, user: &User, name: &str, args: &Value) -> Result< match name { "create_sandbox" => { let team = team_id(s, user, args).await?; - let mut req = json!({ "teamId": team, "template": arg(args, "template")? }); + let mut req = json!({ "teamId": team }); for (from, to) in [ + ("template", "template"), ("cpus", "cpus"), ("memory_mib", "memoryMib"), ("ttl", "ttl"), diff --git a/crates/pyo-api/src/sandbox/mod.rs b/crates/pyo-api/src/sandbox/mod.rs index 1011d8e..49b0f90 100644 --- a/crates/pyo-api/src/sandbox/mod.rs +++ b/crates/pyo-api/src/sandbox/mod.rs @@ -5,8 +5,11 @@ //! //! - [`keys`]: the installation's signing key and the tokens. //! - [`join`]: a sandbox server's one-time join code and sandbox.sh. +//! [`add_cloud_server`] makes one at a cloud provider instead, which runs +//! sandbox.sh from its first boot (cloud/bootstrap.rs). //! - [`scheduler`]: which server a new sandbox runs on. //! - [`templates`]: which servers make which templates, and their state. +//! [`base`]: Pyo's built-in template, kept in step with the release. //! - [`limits`]: per-team caps, checked on create. //! - [`daemon`]: Pyo's calls to a daemon over the tailnet. //! - [`edgecert`]: the edge proxies' wildcard certificates and DNS records. @@ -20,6 +23,7 @@ //! sandbox server exists ([`available`]). pub mod api; +pub mod base; pub mod daemon; #[cfg(test)] mod e2e; @@ -325,7 +329,13 @@ pub async fn server_views( 'daemonVersion', s.daemon_version, 'lastReportAt', pyo.ms(s.last_report_at), 'error', s.error, 'createdBy', s.created_by, 'createdAt', pyo.ms(s.created_at), 'joinedAt', pyo.ms(s.joined_at), 'edgeDomain', s.edge_domain, - 'sandboxes', (SELECT count(*) FROM pyo.sandboxes x WHERE x.server_id = s.id AND x.state NOT IN ('stopped', 'failed'))) + 'sandboxes', (SELECT count(*) FROM pyo.sandboxes x WHERE x.server_id = s.id AND x.state NOT IN ('stopped', 'failed')), + 'cloud', (SELECT jsonb_build_object('id', c.id, 'provider', c.provider, 'location', c.location, + 'serverType', c.server_type, 'monthly', c.monthly, 'currency', c.currency, 'ipv4', c.ipv4, + 'state', c.state, 'error', c.error, 'signCommand', c.sign_command, + 'createdAt', pyo.ms(c.created_at), 'deadlineAt', pyo.ms(c.created_at + interval '15 minutes')) + FROM pyo.cloud_servers c WHERE c.sandbox_server_id = s.id AND c.state <> 'deleted' + ORDER BY c.created_at DESC LIMIT 1)) FROM pyo.sandbox_servers s WHERE (s.team_id IS NULL OR s.team_id = ANY($1)) AND ($2::text IS NULL OR s.id = $2) ORDER BY s.team_id NULLS FIRST, s.name", @@ -335,10 +345,11 @@ pub async fn server_views( .fetch_all(db) .await?; for r in &mut rows { - let summary = serde_json::from_value::(r["capability"].clone()) - .ok() - .map(|c| c.summary()); - r["summary"] = json!(summary); + let cap = serde_json::from_value::(r["capability"].clone()).ok(); + r["summary"] = json!(cap.as_ref().map(|c| c.summary())); + // What runs its sandboxes: "firecracker" or "gvisor", null until it + // reports (or when it can run neither). + r["runtime"] = json!(cap.and_then(|c| runtime_name(c.runtime))); } edgecert::attach(db, team_ids, &mut rows).await?; Ok(rows) @@ -352,6 +363,7 @@ pub async fn template_views( sqlx::query_scalar( "SELECT jsonb_build_object( 'id', t.id, 'teamId', t.team_id, 'name', t.name, 'source', t.source, 'state', t.state, + 'builtin', t.builtin, 'revision', t.revision, 'snapshots', t.snapshots, 'error', t.error, 'createdBy', t.created_by, 'createdAt', pyo.ms(t.created_at), 'updatedAt', pyo.ms(t.updated_at), 'servers', (SELECT coalesce(jsonb_agg(jsonb_build_object( @@ -364,7 +376,7 @@ pub async fn template_views( 'bytes', m.e->'bytes', 'snapshots', coalesce(m.e->'snapshots', '[]')) ORDER BY s.name), '[]') FROM pyo.sandbox_servers s LEFT JOIN LATERAL (SELECT x AS e FROM jsonb_array_elements(s.templates) x - WHERE x->>'name' = t.id LIMIT 1) m ON true + WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') LIMIT 1) m ON true WHERE (s.team_id IS NULL OR s.team_id = t.team_id OR t.team_id IS NULL) AND (s.team_id IS NULL OR s.team_id = ANY($1)) AND (m.e IS NOT NULL OR s.state NOT IN ('joining', 'refused')))) @@ -390,8 +402,10 @@ pub struct CreateSandbox { #[serde(default)] pub team_id: Option, /// A sandbox template's name (or id): the team's own, else the - /// installation's. - pub template: String, + /// installation's. "base" (Pyo's built-in Debian 13 template) when + /// omitted. + #[serde(default)] + pub template: Option, /// 1 when omitted. #[serde(default)] pub cpus: Option, @@ -447,14 +461,30 @@ pub fn parse_ttl_value(v: Option<&Value>) -> ApiResult { Ok(secs) } +/// The template a create asks for: the one named, else [`base::NAME`]. +pub fn template_asked(template: Option<&str>) -> &str { + template + .map(str::trim) + .filter(|t| !t.is_empty()) + .unwrap_or(base::NAME) +} + +/// A template as a create finds it. +struct AskedTemplate { + id: String, + name: String, + /// The name its servers keep it under (see [`templates::server_name`]). + server_name: String, +} + /// The template a name stands for: the team's, else the installation's. async fn template_for( tx: &mut PgConnection, team_id: &str, name: &str, -) -> ApiResult<(String, String, String)> { - let row: Option<(String, String, String)> = sqlx::query_as( - "SELECT id, name, state FROM pyo.sandbox_templates +) -> ApiResult { + let row: Option<(String, String, String, Option)> = sqlx::query_as( + "SELECT id, name, state, revision FROM pyo.sandbox_templates WHERE (team_id = $1 OR team_id IS NULL) AND (name = $2 OR id = $2) ORDER BY team_id NULLS LAST LIMIT 1", ) @@ -462,13 +492,25 @@ async fn template_for( .bind(name) .fetch_optional(tx) .await?; - let (id, name, state) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?; + let Some((id, name, state, revision)) = row else { + return Err(if name == base::NAME { + ApiError::conflict( + "The base sandbox template isn't ready yet. Try again soon, or pass a template.", + ) + } else { + ApiError::not_found("Sandbox template") + }); + }; if state == "failed" { return Err(ApiError::conflict(format!( "The sandbox template {name} failed to build. Make it again." ))); } - Ok((id, name, state)) + Ok(AskedTemplate { + server_name: templates::server_name(&id, revision.as_deref()), + id, + name, + }) } /// The servers a team's sandbox can go to, with their addresses. @@ -595,8 +637,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< .bind(&team_id) .execute(&mut *tx) .await?; - let (template_id, template_name, _) = - template_for(&mut tx, &team_id, req.template.trim()).await?; + let template = template_for(&mut tx, &team_id, template_asked(req.template.as_deref())).await?; if let Some(p) = &req.project_id { let ok: bool = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM pyo.projects WHERE id = $1 AND team_id = $2)", @@ -609,7 +650,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< return Err(ApiError::not_found("Project")); } } - let mut spec = SandboxSpec::new(new_sandbox_id(), &team_id, &template_id); + let mut spec = SandboxSpec::new(new_sandbox_id(), &team_id, &template.server_name); spec.cpus = req.cpus.unwrap_or(proto::DEFAULT_CPUS); spec.memory_mib = req.memory_mib.unwrap_or(proto::DEFAULT_MEMORY_MIB); spec.disk_mib = req.disk_mib.unwrap_or(proto::DEFAULT_DISK_MIB); @@ -625,15 +666,18 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< limits::check(&lim, &used, &spec).map_err(ApiError::forbidden)?; let servers = candidates(&mut tx, &team_id).await?; let list: Vec = servers.iter().map(|(c, _)| c.clone()).collect(); - let chosen = scheduler::pick(&list, &spec).map_err(|n| { - let n = match n { - scheduler::NoServer::NoTemplate(_) => { - scheduler::NoServer::NoTemplate(template_name.clone()) - } - n => n, - }; - ApiError::new(n.status(), n.message()) - })?; + let (chosen, server_name) = + scheduler::pick_revision(&list, &spec, &template.id).map_err(|n| { + let n = match n { + scheduler::NoServer::NoTemplate(_) => { + scheduler::NoServer::NoTemplate(template.name.clone()) + } + n => n, + }; + ApiError::new(n.status(), n.message()) + })?; + // The server may still start it from an older revision of the template. + spec.template = server_name; let address = servers .iter() .find(|(c, _)| c.id == chosen.id) @@ -653,7 +697,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult< .bind(&team_id) .bind(&req.project_id) .bind(&server_id) - .bind(&template_id) + .bind(&template.id) .bind(json!(kept)) .bind(&user.name) .bind(ttl_secs as f64) @@ -998,8 +1042,7 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult< } crate::cloud::needs_tailscale(&s.db).await?; let door = crate::deploy::registry_push(); - let api = crate::web::tailnet_api_url(); - let (false, Some(api)) = (door.is_empty(), api) else { + let (false, Some(_)) = (door.is_empty(), crate::web::tailnet_api_url()) else { return Err(ApiError::conflict( "Pyo's server isn't on your tailnet yet. Try again in a minute.", )); @@ -1046,8 +1089,12 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult< }; let code = join::new_code(); join::keep(&mut tx, &id, &code).await?; - let ts_key = - crate::tailscale::join_key(s, &format!("Pyo: sandbox server {}", req.name)).await?; + let ts_key = crate::tailscale::join_key( + s, + &format!("Pyo: sandbox server {}", req.name), + "a sandbox server", + ) + .await?; if let Some(t) = &req.team_id { crate::ops::log_activity( &mut tx, @@ -1062,7 +1109,9 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult< .await?; } tx.commit().await?; - let base = crate::join::scripts_url(Some(&crate::join::friendly(&api))); + // The server joins the tailnet in sandbox.sh, so it fetches the script + // off the tailnet. + let base = crate::join::scripts_url_before_tailnet(); let door = crate::join::friendly(&door); let command = join::Command { base: &base, @@ -1087,10 +1136,36 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult< })) } +/// Ends a sandbox server's sandboxes in Pyo's records and forgets the +/// server. Its daemon isn't told: see [`remove_server`]. The number of +/// sandboxes that ended. +pub async fn forget_server( + tx: &mut Transaction<'_, Postgres>, + id: &str, + why: &str, +) -> sqlx::Result { + let live: Vec = sqlx::query_scalar( + "SELECT id FROM pyo.sandboxes WHERE server_id = $1 AND state NOT IN ('stopped', 'failed')", + ) + .bind(id) + .fetch_all(&mut **tx) + .await?; + for x in &live { + end(tx, x, "stopped", Some(why)).await?; + } + sqlx::query("DELETE FROM pyo.sandbox_servers WHERE id = $1") + .bind(id) + .execute(&mut **tx) + .await?; + Ok(live.len()) +} + /// Removes a sandbox server: its sandboxes are deleted where it still /// answers, and Pyo forgets it. The daemon stays installed on the server -/// until `sandbox.sh --uninstall` runs there. -pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult { +/// until `sandbox.sh --uninstall` runs there. One Pyo made at a cloud +/// provider is deleted there too, which takes `confirm` (409 with the +/// monthly cost that stops until then). +pub async fn remove_server(s: &AppState, user: &User, id: &str, confirm: bool) -> ApiResult { let row: Option<(Option, Option, String)> = sqlx::query_as("SELECT team_id, address, name FROM pyo.sandbox_servers WHERE id = $1") .bind(id) @@ -1102,6 +1177,21 @@ pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult require(&mut tx, user, t, Need::Manage).await?, None => require_install_admin(user)?, } + let cloud: Option<(String, f64, String)> = sqlx::query_as( + "SELECT id, monthly, currency FROM pyo.cloud_servers + WHERE sandbox_server_id = $1 AND state NOT IN ('deleted', 'removing') ORDER BY created_at DESC LIMIT 1", + ) + .bind(id) + .fetch_optional(&mut *tx) + .await?; + if let Some((_, monthly, currency)) = &cloud + && !confirm + { + return Err(ApiError::needs_confirm(format!( + "Deletes {name} at Hetzner. {} a month stops.", + crate::ops::money(currency, *monthly) + ))); + } let live: Vec = sqlx::query_scalar( "SELECT id FROM pyo.sandboxes WHERE server_id = $1 AND state NOT IN ('stopped', 'failed')", ) @@ -1123,34 +1213,197 @@ pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult, + /// A location's `name` from getCloudOptions ("fsn1"). + pub location: String, + /// A server type's `name` from getCloudOptions ("cx22"). + pub server_type: String, + /// A domain whose wildcard points at the server, as for + /// `addSandboxServer`. + #[serde(default)] + pub edge_domain: Option, + /// Your OpenSSH public key, to sign in to it over your tailnet. + #[serde(default)] + pub ssh_key: Option, + /// Without it the answer is 409 with the monthly price. + #[serde(default)] + pub confirm: bool, +} + +/// Makes a sandbox server at a cloud provider: its row (joining), and the +/// server, which joins the tailnet on its first boot and runs sandbox.sh +/// with a one-time code. Cloud servers have no KVM, so it runs gVisor. +pub async fn add_cloud_server(s: &AppState, user: &User, req: AddCloudServer) -> ApiResult { + crate::util::require_dns_label(&req.name, "Server name")?; + let edge = edge_domain(req.edge_domain.as_deref())?; + if !matches!(req.provider.as_deref(), None | Some("hetzner")) { + return Err(ApiError::bad_request("The provider is \"hetzner\".")); + } + let ssh_key = match req + .ssh_key + .as_deref() + .map(str::trim) + .filter(|k| !k.is_empty()) + { + Some(k) if crate::cloud::valid_ssh_key(k) => Some(k.to_owned()), + Some(_) => { + return Err(ApiError::bad_request( + "Paste an OpenSSH public key on one line, like the contents of ~/.ssh/id_ed25519.pub.", + )); + } + None => None, + }; + let mut tx = s.db.begin().await?; + require(&mut tx, user, &req.team_id, Need::Manage).await?; + crate::cloud::needs_tailscale(&s.db).await?; + let door = crate::deploy::registry_push(); + let api = crate::web::tailnet_api_url(); + let (false, Some(api)) = (door.is_empty(), api) else { + return Err(ApiError::conflict( + "Pyo's server isn't on your tailnet yet. Try again in a minute.", + )); + }; + if !s.sealer.enabled() { + return Err(ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "This installation has no PYO_SECRET_KEY, so it can't sign sandbox tokens.", + )); + } + let taken: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pyo.sandbox_servers WHERE team_id = $1 AND name = $2)", + ) + .bind(&req.team_id) + .bind(&req.name) + .fetch_one(&mut *tx) + .await?; + if taken { + return Err(ApiError::conflict( + "A sandbox server has that name already.", + )); + } + let p = crate::cloud::provider(&s.db, &req.team_id).await?; + let catalog = crate::cloud::catalog(&s.db, &req.team_id).await?; + let monthly = catalog.pick(&req.location, &req.server_type)?; + let price = crate::ops::money(&catalog.currency, monthly); + if !req.confirm { + return Err(ApiError::needs_confirm(format!( + "{} costs {price} a month at Hetzner ({} in {}, before VAT).", + req.name, req.server_type, req.location + ))); + } + let id = crate::ids::new_id("sbs"); + sqlx::query("INSERT INTO pyo.sandbox_servers (id, team_id, name, state, created_by, edge_domain) VALUES ($1, $2, $3, 'joining', $4, $5)") + .bind(&id) + .bind(&req.team_id) + .bind(&req.name) + .bind(&user.name) + .bind(&edge) + .execute(&mut *tx) + .await?; + let ts_key = crate::tailscale::join_key( + s, + &format!("Pyo: sandbox server {}", req.name), + "a sandbox server", + ) + .await?; + // It's on the tailnet before it asks. Its join code comes with the + // details, made when it asks (cloud/bootstrap.rs). + let base = crate::join::scripts_url(Some(&crate::join::friendly(&api))); + let door = crate::join::friendly(&door); + let mut args: Vec<(&str, Option)> = vec![ + ("--pyo", Some(door)), + ("--server", Some(id.clone())), + ("--name", Some(req.name.clone())), + ("--version", Some(env!("CARGO_PKG_VERSION").into())), + ]; + if let Some(d) = &edge { + args.push(("--edge-domain", Some(d.clone()))); + } + let material = crate::cloud::bootstrap::Material::new(format!("{base}/sandbox.sh"), &args); + let cloud_id = crate::ids::new_id("cs"); + let made = crate::ops::make_cloud_server( + s, + &p, + crate::ops::CloudNew { + id: &cloud_id, + team_id: &req.team_id, + owner: crate::cloud::Owner::Sandbox(&id), + name: &req.name, + location: &req.location, + server_type: &req.server_type, + monthly, + currency: &catalog.currency, + first: false, + builder: false, + tailscale_key: &ts_key, + material, + ssh_key: ssh_key.as_deref(), + by: &user.name, + }, + &mut tx, + ) + .await?; + crate::ops::log_activity( + &mut tx, + &req.team_id, + &user.name, + "sandbox.server", + "info", + &format!( + "Creating the sandbox server {} at Hetzner ({}, {price} a month)", + req.name, req.server_type + ), + json!({ "kind": "sandboxServer", "id": id, "label": req.name }), + json!({ "cloudServerId": cloud_id }), + ) + .await?; + tx.commit().await?; + let server = server_views(&s.db, std::slice::from_ref(&req.team_id), Some(&id)) + .await? + .into_iter() + .next(); + Ok(json!({ "server": server, "cloudServer": made })) } #[derive(Debug, Clone, Deserialize, utoipa::ToSchema)] @@ -1176,6 +1429,11 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) -> "A template name is lowercase letters, digits, dashes, dots and underscores, like node-22.", )); } + if req.name == base::NAME { + return Err(ApiError::conflict( + "base is Pyo's built-in sandbox template. Pick another name.", + )); + } let image = req.image.trim(); if image.is_empty() || image.len() > 512 || image.contains(char::is_whitespace) { return Err(ApiError::bad_request("Give the image, like node:22.")); @@ -1213,12 +1471,17 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) -> /// Deletes a sandbox template, and has the servers delete their copies. /// Refused while sandboxes run from it. pub async fn delete_template(s: &AppState, user: &User, id: &str) -> ApiResult { - let row: Option<(Option, String)> = - sqlx::query_as("SELECT team_id, name FROM pyo.sandbox_templates WHERE id = $1") + let row: Option<(Option, String, bool)> = + sqlx::query_as("SELECT team_id, name, builtin FROM pyo.sandbox_templates WHERE id = $1") .bind(id) .fetch_optional(&s.db) .await?; - let (team_id, name) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?; + let (team_id, name, builtin) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?; + if builtin { + return Err(ApiError::conflict(format!( + "{name} is Pyo's built-in sandbox template, so it can't be deleted." + ))); + } let mut tx = s.db.begin().await?; match &team_id { Some(t) => require(&mut tx, user, t, Need::Use).await?, @@ -1352,4 +1615,13 @@ mod tests { .is_err() ); } + + #[test] + fn create_without_a_template_uses_base() { + let r: CreateSandbox = serde_json::from_value(json!({})).unwrap(); + assert_eq!(r.template, None); + assert_eq!(template_asked(r.template.as_deref()), "base"); + assert_eq!(template_asked(Some(" ")), "base"); + assert_eq!(template_asked(Some(" node-22 ")), "node-22"); + } } diff --git a/crates/pyo-api/src/sandbox/runner.rs b/crates/pyo-api/src/sandbox/runner.rs index 66dddb4..fe6538f 100644 --- a/crates/pyo-api/src/sandbox/runner.rs +++ b/crates/pyo-api/src/sandbox/runner.rs @@ -67,6 +67,10 @@ struct Server { /// One round: reports, then lifetimes. pub async fn round(s: &AppState) -> anyhow::Result<()> { let _ = super::join::prune(&s.db).await; + // A dev build's `base` comes once an update check finds a release. + if let Err(e) = super::base::ensure(s).await { + tracing::warn!(error = %format!("{e:#}"), "sandboxes: couldn't keep the base template"); + } let servers: Vec = sqlx::query_as( "SELECT id, team_id, address, state FROM pyo.sandbox_servers WHERE address IS NOT NULL AND state <> 'refused'", ) diff --git a/crates/pyo-api/src/sandbox/scheduler.rs b/crates/pyo-api/src/sandbox/scheduler.rs index 78417c5..605648b 100644 --- a/crates/pyo-api/src/sandbox/scheduler.rs +++ b/crates/pyo-api/src/sandbox/scheduler.rs @@ -155,6 +155,42 @@ pub fn pick<'a>(servers: &'a [Candidate], spec: &SandboxSpec) -> Result<&'a Cand .ok_or(NoServer::Full) } +/// [`pick`] for a template that may have revisions (template `id`, kept +/// as `spec.template` now): a server with the current revision ready, +/// else one that still has an older one ready, while the new one is made. +/// Returns the server and the name of the revision it starts from. +pub fn pick_revision<'a>( + servers: &'a [Candidate], + spec: &SandboxSpec, + id: &str, +) -> Result<(&'a Candidate, String), NoServer> { + let first = match pick(servers, spec) { + Ok(c) => return Ok((c, spec.template.clone())), + Err(e) => e, + }; + if !matches!(first, NoServer::NoTemplate(_)) { + return Err(first); + } + let mut older: Vec<&str> = servers + .iter() + .flat_map(|c| super::templates::older_ready(&c.templates, id, &spec.template)) + .collect(); + older.sort_unstable(); + older.dedup(); + let mut why = first; + for name in older { + let mut s = spec.clone(); + s.template = name.to_owned(); + match pick(servers, &s) { + Ok(c) => return Ok((c, s.template)), + // Say why the older one can't run it either (full, say). + Err(e) if !matches!(e, NoServer::NoTemplate(_)) => why = e, + Err(_) => {} + } + } + Err(why) +} + #[cfg(test)] mod tests { use super::*; @@ -301,6 +337,45 @@ mod tests { assert_eq!(pick_id(&[building, ready], &spec()).unwrap(), "ready"); } + #[test] + fn new_sandboxes_move_to_a_new_revision_once_it_is_ready() { + let named = |id: &str, name: &str, state: TemplateState| { + let mut c = server(id, Runtime::Firecracker, 8); + c.templates[0].name = name.into(); + c.templates[0].state = state; + c + }; + let mut s = spec(); + s.template = "stp_base.new".into(); + // Only the old one is ready: it's used. + let mut a = named("a", "stp_base.old", TemplateState::Ready); + a.templates.push(TemplateReport { + name: "stp_base.new".into(), + state: TemplateState::Building, + ..Default::default() + }); + let (c, name) = pick_revision(std::slice::from_ref(&a), &s, "stp_base").unwrap(); + assert_eq!((c.id.as_str(), name.as_str()), ("a", "stp_base.old")); + // Once a server has the new one ready, it's preferred. + let b = named("b", "stp_base.new", TemplateState::Ready); + let both = [a.clone(), b]; + let (c, name) = pick_revision(&both, &s, "stp_base").unwrap(); + assert_eq!((c.id.as_str(), name.as_str()), ("b", "stp_base.new")); + // Another template's revisions don't count. + let other = named("c", "stp_basement.x", TemplateState::Ready); + assert_eq!( + pick_revision(&[other], &s, "stp_base").map(|(c, _)| c.id.clone()), + Err(NoServer::NoTemplate("stp_base.new".into())) + ); + // The old one's server is full: that's the reason given. + let mut full = named("a", "stp_base.old", TemplateState::Ready); + full.capacity.memory_available = 0; + assert_eq!( + pick_revision(&[full], &s, "stp_base").map(|(c, _)| c.id.clone()), + Err(NoServer::Full) + ); + } + #[test] fn internet_needs_a_server_with_networking() { let mut off = server("off", Runtime::Firecracker, 8); diff --git a/crates/pyo-api/src/sandbox/templates.rs b/crates/pyo-api/src/sandbox/templates.rs index 20faea8..561f4d1 100644 --- a/crates/pyo-api/src/sandbox/templates.rs +++ b/crates/pyo-api/src/sandbox/templates.rs @@ -14,6 +14,11 @@ //! could make it failed. //! - A template Pyo no longer has is deleted from the servers that made //! it. One put on a server by hand is never touched. +//! - A template with a revision (Pyo's built-in `base`, see [`super::base`]) +//! is kept on the servers as `.`. A new image is a new +//! revision: servers make it beside the old one, new sandboxes move to it +//! once it's ready there, and the old one is deleted once no sandbox on +//! that server uses it (running and paused ones keep their disk). //! //! TODO(sandbox): templates from a repo or a Dockerfile, through Pyo's //! builder. The builder runs as a deployment's build job today (deploy.rs @@ -39,6 +44,46 @@ pub struct Template { pub id: String, pub team_id: Option, pub source: Value, + pub revision: Option, +} + +impl Template { + /// The name servers keep it under. + pub fn server_name(&self) -> String { + server_name(&self.id, self.revision.as_deref()) + } +} + +/// The name servers keep a template under: its id, and its revision when +/// it has one. +pub fn server_name(id: &str, revision: Option<&str>) -> String { + match revision { + Some(r) => format!("{id}.{r}"), + None => id.to_owned(), + } +} + +/// Whether `name`, as a server keeps it, is a revision of the template `id` +/// (the current one or an older one). +pub fn is_revision_of(name: &str, id: &str) -> bool { + name == id + || name + .strip_prefix(id) + .is_some_and(|rest| rest.starts_with('.')) +} + +/// Revisions of template `id` a server has ready, other than `current`. +pub fn older_ready<'a>( + reports: &'a [TemplateReport], + id: &'a str, + current: &'a str, +) -> impl Iterator { + reports + .iter() + .filter(move |r| { + r.name != current && is_revision_of(&r.name, id) && r.state == TemplateState::Ready + }) + .map(|r| r.name.as_str()) } /// Whether a server may use a template: an installation's server takes @@ -69,7 +114,7 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand { .collect(); match (in_pyos_registry(image, &doors), push.is_empty()) { (Some(path), false) => DaemonCommand::PrepareTemplate { - template: t.id.clone(), + template: t.server_name(), // The door on the tailnet, as other machines pull from it. image: format!("{push}/{path}"), plain_http: true, @@ -80,7 +125,7 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand { max_mib: None, }, _ => DaemonCommand::PrepareTemplate { - template: t.id.clone(), + template: t.server_name(), image: image.to_owned(), plain_http: false, auth: None, @@ -90,19 +135,36 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand { } /// What to tell a server after its report: templates to prepare, and -/// templates it made that Pyo no longer has (or that aren't for it). +/// templates it made that Pyo no longer has (or that aren't for it). Both +/// by the name the server keeps them under. An older revision stays until +/// the current one is ready there, so new sandboxes have one to start from. pub fn plan( server_team: Option<&str>, report: &ServerReport, templates: &[Template], ) -> (Vec, Vec) { - let wanted: HashSet<&str> = templates + let mine: Vec<&Template> = templates .iter() .filter(|t| eligible(server_team, t.team_id.as_deref())) - .map(|t| t.id.as_str()) + .collect(); + let names: Vec = mine.iter().map(|t| t.server_name()).collect(); + let wanted: HashSet<&str> = names.iter().map(String::as_str).collect(); + // Templates whose current revision isn't ready here yet. + let waiting: Vec<&str> = mine + .iter() + .zip(&names) + .filter(|(_, n)| { + !report + .templates + .iter() + .any(|r| &r.name == *n && r.state == TemplateState::Ready) + }) + .map(|(t, _)| t.id.as_str()) .collect(); let has: HashSet<&str> = report.templates.iter().map(|t| t.name.as_str()).collect(); - let mut prepare: Vec = if report.capability.runtime == Runtime::Firecracker { + // Firecracker servers boot and snapshot a template. gVisor servers make + // the same disk and start sandboxes cold from it. + let mut prepare: Vec = if report.capability.runtime != Runtime::None { wanted .iter() .filter(|id| !has.contains(*id)) @@ -117,6 +179,7 @@ pub fn plan( .iter() // Only what the server made for Pyo: one put there by hand stays. .filter(|t| t.build.is_some() && !wanted.contains(t.name.as_str())) + .filter(|t| !waiting.iter().any(|id| is_revision_of(&t.name, id))) .map(|t| t.name.clone()) .collect(); stale.sort(); @@ -124,7 +187,7 @@ pub fn plan( } pub async fn all(db: &PgPool) -> sqlx::Result> { - sqlx::query_as("SELECT id, team_id, source FROM pyo.sandbox_templates") + sqlx::query_as("SELECT id, team_id, source, revision FROM pyo.sandbox_templates") .fetch_all(db) .await } @@ -147,7 +210,7 @@ pub async fn sync_server( let d = Daemon::new(address); let token = keys::admin_token(key, server_id); for id in prepare { - let Some(t) = templates.iter().find(|t| t.id == id) else { + let Some(t) = templates.iter().find(|t| t.server_name() == id) else { continue; }; match d.command(&token, &prepare_command(key, t)).await { @@ -187,7 +250,7 @@ pub async fn sync_server( pub async fn push(s: AppState, template_id: String) { let res = async { let Some(t) = sqlx::query_as::<_, Template>( - "SELECT id, team_id, source FROM pyo.sandbox_templates WHERE id = $1", + "SELECT id, team_id, source, revision FROM pyo.sandbox_templates WHERE id = $1", ) .bind(&template_id) .fetch_optional(&s.db) @@ -198,10 +261,14 @@ pub async fn push(s: AppState, template_id: String) { let servers: Vec<(String, Option, String)> = sqlx::query_as( "SELECT id, team_id, address FROM pyo.sandbox_servers WHERE state = 'ready' AND enabled AND address IS NOT NULL - AND capability->>'runtime' = 'firecracker'", + AND capability->>'runtime' IN ('firecracker', 'gvisor')", ) .fetch_all(&s.db) .await?; + // No server yet: each makes it once it joins. + if servers.is_empty() { + return anyhow::Ok(()); + } let key = keys::signing_key(&s.db, &s.sealer) .await .map_err(|e| anyhow::anyhow!(e.message))?; @@ -225,14 +292,12 @@ pub async fn push(s: AppState, template_id: String) { } } -/// Tells the servers that have a deleted template to delete it. The runner -/// catches the ones that don't answer now. +/// Tells the servers that have a deleted template to delete it, every +/// revision of it. The runner catches the ones that don't answer now. pub async fn forget(s: AppState, template_id: String) { - let servers: Vec<(String, String)> = match sqlx::query_as( - "SELECT id, address FROM pyo.sandbox_servers - WHERE address IS NOT NULL AND templates @> jsonb_build_array(jsonb_build_object('name', $1::text))", + let servers: Vec<(String, String, Value)> = match sqlx::query_as( + "SELECT id, address, templates FROM pyo.sandbox_servers WHERE address IS NOT NULL", ) - .bind(&template_id) .fetch_all(&s.db) .await { @@ -245,16 +310,22 @@ pub async fn forget(s: AppState, template_id: String) { let Ok(key) = keys::signing_key(&s.db, &s.sealer).await else { return; }; - for (id, address) in servers { + for (id, address, templates) in servers { + let reports: Vec = serde_json::from_value(templates).unwrap_or_default(); let token = keys::admin_token(&key, &id); - let _ = Daemon::new(&address) - .command( - &token, - &DaemonCommand::DeleteTemplate { - template: template_id.clone(), - }, - ) - .await; + for r in reports + .iter() + .filter(|r| is_revision_of(&r.name, &template_id)) + { + let _ = Daemon::new(&address) + .command( + &token, + &DaemonCommand::DeleteTemplate { + template: r.name.clone(), + }, + ) + .await; + } } } @@ -278,25 +349,36 @@ pub fn roll_up(reports: &[TemplateReport], missing: usize) -> (&'static str, Opt ("pending", None) } -/// Rolls every template's state up from the servers' latest reports. +/// Rolls every template's state up from the servers' latest reports, by +/// its current revision. While a new revision is made, a template whose +/// older one is ready on a server stays ready: new sandboxes start from it. pub async fn update_states(db: &PgPool) -> sqlx::Result<()> { - let rows: Vec<(String, String, Option, Value, i64)> = sqlx::query_as( + let rows: Vec<(String, String, Option, Value, i64, bool)> = sqlx::query_as( "SELECT t.id, t.state, t.error, coalesce(jsonb_agg(m.e) FILTER (WHERE m.e IS NOT NULL), '[]'), count(s.id) FILTER (WHERE m.e IS NULL AND s.state = 'ready' AND s.enabled - AND s.capability->>'runtime' = 'firecracker') + AND s.capability->>'runtime' IN ('firecracker', 'gvisor')), + coalesce(bool_or(o.ready), false) FROM pyo.sandbox_templates t LEFT JOIN pyo.sandbox_servers s ON s.team_id IS NULL OR t.team_id IS NULL OR s.team_id = t.team_id LEFT JOIN LATERAL (SELECT x AS e FROM jsonb_array_elements(s.templates) x - WHERE x->>'name' = t.id LIMIT 1) m ON true + WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') LIMIT 1) m ON true + LEFT JOIN LATERAL (SELECT true AS ready FROM jsonb_array_elements(s.templates) x + WHERE t.revision IS NOT NULL AND starts_with(x->>'name', t.id || '.') + AND x->>'name' <> t.id || '.' || t.revision + AND coalesce(x->>'state', 'ready') = 'ready' LIMIT 1) o ON true GROUP BY t.id", ) .fetch_all(db) .await?; - for (id, state, error, reports, missing) in rows { + for (id, state, error, reports, missing, older_ready) in rows { let reports: Vec = serde_json::from_value(reports).unwrap_or_default(); - let (new, why) = roll_up(&reports, missing.max(0) as usize); + let (new, why) = match roll_up(&reports, missing.max(0) as usize) { + ("ready", why) => ("ready", why), + _ if older_ready => ("ready", None), + r => r, + }; if new != state || why != error { sqlx::query( "UPDATE pyo.sandbox_templates SET state = $2, error = $3, updated_at = now() WHERE id = $1", @@ -323,6 +405,7 @@ mod tests { id: id.into(), team_id: team.map(str::to_owned), source: json!({ "kind": "image", "image": "alpine" }), + revision: None, } } @@ -396,11 +479,76 @@ mod tests { "the installation's server takes all" ); let (prepare, stale) = plan(Some("tm_a"), &report(Runtime::Gvisor, vec![]), &templates); + assert_eq!( + prepare, + vec!["stp_a", "stp_i"], + "gVisor servers make them too" + ); + assert!(stale.is_empty()); + let (prepare, _) = plan(Some("tm_a"), &report(Runtime::None, vec![]), &templates); assert!( prepare.is_empty(), - "gVisor servers can't make templates yet" + "a server that runs nothing makes nothing" ); - assert!(stale.is_empty()); + } + + #[test] + fn revisions_are_made_beside_the_old_one() { + let mut base = t("stp_base", None); + base.revision = Some("bbbb".into()); + assert_eq!(base.server_name(), "stp_base.bbbb"); + assert!(is_revision_of("stp_base.aaaa", "stp_base")); + assert!(is_revision_of("stp_base", "stp_base")); + assert!(!is_revision_of("stp_basement", "stp_base")); + let templates = [base]; + // The new revision is made, the old one kept until it's ready. + let r = report( + Runtime::Firecracker, + vec![made("stp_base.aaaa", TemplateState::Ready, None)], + ); + let (prepare, stale) = plan(None, &r, &templates); + assert_eq!(prepare, vec!["stp_base.bbbb"]); + assert!(stale.is_empty(), "kept while the new one isn't ready"); + let r = report( + Runtime::Firecracker, + vec![ + made("stp_base.aaaa", TemplateState::Ready, None), + made("stp_base.bbbb", TemplateState::Building, None), + ], + ); + let (prepare, stale) = plan(None, &r, &templates); + assert!(prepare.is_empty() && stale.is_empty()); + assert_eq!( + older_ready(&r.templates, "stp_base", "stp_base.bbbb").collect::>(), + vec!["stp_base.aaaa"], + "new sandboxes still start from the old one" + ); + // Once the new one is ready, the old one goes. + let r = report( + Runtime::Firecracker, + vec![ + made("stp_base.aaaa", TemplateState::Ready, None), + made("stp_base.bbbb", TemplateState::Ready, None), + ], + ); + let (prepare, stale) = plan(None, &r, &templates); + assert!(prepare.is_empty()); + assert_eq!(stale, vec!["stp_base.aaaa"]); + assert_eq!( + older_ready(&r.templates, "stp_base", "stp_base.bbbb").count(), + 1 + ); + } + + #[test] + fn a_revision_is_prepared_under_its_name() { + let key = SigningKey::generate("sbk_t"); + let mut base = t("stp_base", None); + base.revision = Some("bbbb".into()); + let DaemonCommand::PrepareTemplate { template, .. } = prepare_command(&key, &base) else { + panic!("not a prepare"); + }; + assert_eq!(template, "stp_base.bbbb"); } #[test] diff --git a/crates/pyo-api/src/tailscale.rs b/crates/pyo-api/src/tailscale.rs index 8cc7448..e6a4019 100644 --- a/crates/pyo-api/src/tailscale.rs +++ b/crates/pyo-api/src/tailscale.rs @@ -461,7 +461,7 @@ impl Client { let body = json!({ "capabilities": { "devices": { "create": { "reusable": false, "ephemeral": false, "preauthorized": true, "tags": [TAG] } } }, "expirySeconds": 3600, - "description": description, + "description": key_description(description), }); let v = self .call(reqwest::Method::POST, "/tailnet/-/keys", Some(&body)) @@ -483,6 +483,19 @@ impl Client { } } +/// What Tailscale takes as a key's description: letters, digits, spaces, +/// dashes and underscores, at most 50 characters. "Pyo: join web-1" +/// becomes "Pyo join web-1". +fn key_description(s: &str) -> String { + let kept: String = s + .chars() + .filter(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_')) + .collect(); + let mut out = kept.split_whitespace().collect::>().join(" "); + out.truncate(50); + out.trim_end().to_owned() +} + /// A machine on the tailnet ([`Client::device_details`]). #[derive(Debug, Clone, PartialEq)] pub struct Device { @@ -1008,17 +1021,23 @@ pub async fn wanted_for( /// Tailscale made no auth key: the details go to the log, the user gets one /// plain sentence and where to look. -fn no_auth_key(e: anyhow::Error) -> ApiError { - tracing::warn!(error = %format!("{e:#}"), "Tailscale made no auth key"); - ApiError::bad_request("Tailscale made no auth key. Check the Tailscale setup in Integrations.") +/// `server` names what is being added, e.g. "a sandbox server". +fn no_auth_key(server: &'static str) -> impl FnOnce(anyhow::Error) -> ApiError { + move |e| { + tracing::warn!(error = %format!("{e:#}"), "Tailscale made no auth key"); + ApiError::bad_request(format!("Set up Tailscale in Integrations to add {server}.")) + } } /// A join command's auth key: single-use, preauthorized, tag:pyo, an hour. -pub async fn join_key(s: &AppState, what: &str) -> ApiResult { +pub async fn join_key(s: &AppState, what: &str, server: &'static str) -> ApiResult { let Some((_, c)) = saved_client(s).await? else { return Err(ApiError::bad_request(NEEDED)); }; - c.auth_key(what).await.map(|(_, k)| k).map_err(no_auth_key) + c.auth_key(what) + .await + .map(|(_, k)| k) + .map_err(no_auth_key(server)) } /// Whether a tailnet domain someone typed can be had: Pyo's reserved names @@ -1335,13 +1354,26 @@ pub async fn host_report( let (_, key) = c .auth_key("Pyo: its own server") .await - .map_err(no_auth_key)?; + .map_err(no_auth_key("this server"))?; tracing::info!("this machine: joining the tailnet"); Ok(json!({ "state": "join", "authKey": key })) } #[cfg(test)] mod tests { + #[test] + fn key_descriptions_are_what_tailscale_takes() { + use super::key_description; + assert_eq!(key_description("Pyo: join web-1"), "Pyo join web-1"); + assert_eq!( + key_description("Pyo: sandbox server sbx_1"), + "Pyo sandbox server sbx_1" + ); + assert_eq!(key_description("Pyo: its own server"), "Pyo its own server"); + let long = key_description(&format!("Pyo: join {}", "a".repeat(80))); + assert_eq!(long.len(), 50); + } + use super::*; /// A mock Tailscale API: a Service ("docker", someone else's) that the diff --git a/crates/pyo-builder/Cargo.toml b/crates/pyo-builder/Cargo.toml index 632e48c..5ea440f 100644 --- a/crates/pyo-builder/Cargo.toml +++ b/crates/pyo-builder/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0.104" base64 = "0.23.1" futures = "0.3.34" hex = "0.4.3" -pyo-proto = { version = "0.4.0", path = "../pyo-proto" } +pyo-proto = { version = "0.4.1", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] } serde = "1.0.229" serde_json = "1.0.154" diff --git a/crates/pyo-cli/Cargo.toml b/crates/pyo-cli/Cargo.toml index dd9e7e2..a7c3dd5 100644 --- a/crates/pyo-cli/Cargo.toml +++ b/crates/pyo-cli/Cargo.toml @@ -24,4 +24,4 @@ sha2 = "0.11.1" tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "net", "io-util", "io-std", "time", "sync", "fs"] } tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } toml = "0.9" -pyo-proto = { version = "0.4.0", path = "../pyo-proto" } +pyo-proto = { version = "0.4.1", path = "../pyo-proto" } diff --git a/crates/pyo-cli/src/sandbox.rs b/crates/pyo-cli/src/sandbox.rs index 62622e5..a3ee79d 100644 --- a/crates/pyo-cli/src/sandbox.rs +++ b/crates/pyo-cli/src/sandbox.rs @@ -22,7 +22,8 @@ pub enum Command { /// Make a sandbox from a sandbox template and print its id. Create { /// The sandbox template, like node-22 (see pyo sandbox templates ls). - template: String, + /// Default base: Debian 13 with git, Python, uv, Node.js and pnpm. + template: Option, /// CPUs. Default 1. #[arg(long)] cpus: Option, @@ -186,6 +187,30 @@ pub enum ServersCommand { #[arg(long, value_name = "DOMAIN")] edge_domain: Option, }, + /// Buy a sandbox server at Hetzner. Pyo sets it up over your tailnet, + /// and it runs sandboxes on gVisor. Owners and admins. + Create { + /// Its name, lowercase letters, digits and dashes. + name: String, + /// A location, e.g. fsn1 (see pyo server types). + #[arg(long)] + location: String, + /// A server type, e.g. cx22 (see pyo server types). + #[arg(long = "type")] + server_type: String, + /// Serve sandbox ports at -.DOMAIN. Point *.DOMAIN at the server. + #[arg(long, value_name = "DOMAIN")] + edge_domain: Option, + /// Your SSH public key file, to sign in over your tailnet. + #[arg(long, value_name = "FILE")] + ssh_key: Option, + /// Don't ask to confirm the price. + #[arg(long, short = 'y')] + yes: bool, + /// Start it and exit, without following it. + #[arg(long)] + detach: bool, + }, /// The sandbox servers the team can use. Ls { /// Print it as JSON. @@ -238,7 +263,7 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> { let team = team_id(api, team).await?; let body = create_body(CreateArgs { team: &team, - template: &template, + template: template.as_deref(), cpus, memory, disk, @@ -385,7 +410,7 @@ async fn lifecycle(api: &Api, id: &str, what: &str, done: &str) -> Result<()> { pub struct CreateArgs<'a> { pub team: &'a str, - pub template: &'a str, + pub template: Option<&'a str>, pub cpus: Option, pub memory: Option, pub disk: Option, @@ -400,12 +425,13 @@ pub struct CreateArgs<'a> { /// The createSandbox body. Left-out values take the API's defaults. pub fn create_body(a: CreateArgs) -> Result { - let mut b = json!({ "teamId": a.team, "template": a.template }); + let mut b = json!({ "teamId": a.team }); let mut set = |k: &str, v: Value| { if !v.is_null() { b[k] = v; } }; + set("template", json!(a.template)); set("cpus", json!(a.cpus)); set("memoryMib", json!(a.memory)); set("diskMib", json!(a.disk)); @@ -460,7 +486,7 @@ fn size(v: &Value) -> String { fn print_sandboxes(list: &Value) { let list = list.as_array().map(Vec::as_slice).unwrap_or(&[]); if list.is_empty() { - println!("No sandboxes. Make one: pyo sandbox create