diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ac7ea84..8a90f82 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -83,6 +83,10 @@ jobs:
--health-interval 2s
--health-timeout 5s
--health-retries 30
+ # Where this branch's base template image goes, for the daemon to pull.
+ registry:
+ image: registry:3.0.0
+ ports: ["5000:5000"]
env:
PYO_SANDBOX_ENVD: ${{ github.workspace }}/target/x86_64-unknown-linux-musl/release/pyo-envd
steps:
@@ -137,9 +141,16 @@ jobs:
run: |
cargo build --release --locked -p pyo-envd --target x86_64-unknown-linux-musl
cargo build --locked -p pyo-sandbox
+ # Pyo's built-in template `base`, from this branch's Dockerfile.
+ - name: Build the base template image
+ run: |
+ docker build -f deploy/images/sandbox-base.Dockerfile -t localhost:5000/pyo/sandbox-base:e2e deploy/images
+ docker push localhost:5000/pyo/sandbox-base:e2e
+ docker image ls localhost:5000/pyo/sandbox-base:e2e
- name: pyo-sandbox e2e
env:
PYO_SANDBOX_E2E: "1"
+ PYO_SANDBOX_E2E_BASE_IMAGE: 127.0.0.1:5000/pyo/sandbox-base:e2e
run: cargo test --locked -p pyo-sandbox --test e2e --test template_e2e -- --nocapture
- name: pyo-api sandbox e2e
env:
@@ -164,6 +175,102 @@ jobs:
sudo nft list ruleset | head -80 || true
ls -la .cache/sandbox-e2e || true
+ # Pyo Sandbox on gVisor, end to end: runsc as root, as on a server
+ # without KVM (Hetzner Cloud). Firecracker is left out, so detection
+ # falls back to gVisor although the runner has KVM. It mounts disks and
+ # changes the host's network, which the runner is for.
+ gvisor:
+ name: Sandbox on gVisor
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ env:
+ PYO_SANDBOX_ENVD: ${{ github.workspace }}/target/x86_64-unknown-linux-musl/release/pyo-envd
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Packages
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends busybox-static e2fsprogs nftables musl-tools zstd
+ # runsc, the release pinned in gvisor.env, checked against its sha512.
+ - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ id: runsc
+ with:
+ path: .cache/sandbox-gvisor
+ key: sandbox-gvisor-${{ hashFiles('crates/pyo-sandbox/scripts/gvisor.env', 'crates/pyo-sandbox/scripts/gvisor-assets.sh') }}
+ - name: runsc
+ run: |
+ set -euo pipefail
+ crates/pyo-sandbox/scripts/gvisor-assets.sh | tee -a "$GITHUB_ENV"
+ - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ if: steps.runsc.outputs.cache-hit != 'true'
+ with:
+ path: .cache/sandbox-gvisor
+ key: ${{ steps.runsc.outputs.cache-primary-key }}
+ - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
+ with:
+ toolchain: 1.99.1
+ targets: x86_64-unknown-linux-musl
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+ with:
+ key: sandbox-gvisor
+ - name: Build pyo-envd (musl)
+ env:
+ CC_x86_64_unknown_linux_musl: musl-gcc
+ run: cargo build --release --locked -p pyo-envd --target x86_64-unknown-linux-musl
+ # As root. cargo isn't on root's PATH, so the runner user builds the
+ # test binary and sudo -E runs only it.
+ - name: gVisor e2e (root)
+ env:
+ PYO_SANDBOX_GVISOR_E2E: "1"
+ run: |
+ set -euo pipefail
+ bin=$(cargo test --locked -p pyo-sandbox --test gvisor_e2e --no-run --message-format=json \
+ | jq -r 'select(.reason == "compiler-artifact" and .target.name == "gvisor_e2e" and .executable != null) | .executable')
+ [ -x "$bin" ] || { echo "::error::No gvisor_e2e test binary."; exit 1; }
+ sudo -E "$bin" --nocapture
+ - name: What went wrong
+ if: failure()
+ run: |
+ sudo find /tmp -name runsc.log -path '*pyo-gvisor-e2e*' -exec tail -n 40 {} \; || true
+ sudo nft list ruleset | head -80 || true
+ ip netns list || true
+
+ # The base sandbox template's image for both architectures, built
+ # without pushing, when its Dockerfile changes. The release publishes it.
+ sandbox-base:
+ name: Base sandbox template image
+ runs-on: ubuntu-24.04
+ timeout-minutes: 45
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ fetch-depth: 0
+ - name: Did the Dockerfile change
+ id: changed
+ env:
+ PR_BASE: ${{ github.event.pull_request.base.sha }}
+ BEFORE: ${{ github.event.before }}
+ run: |
+ base=${PR_BASE:-$BEFORE}
+ if [ -z "$base" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null ||
+ git diff --name-only "$base" HEAD | grep -qx deploy/images/sandbox-base.Dockerfile; then
+ echo "build=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "Its Dockerfile didn't change."
+ fi
+ - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+ if: steps.changed.outputs.build == 'true'
+ - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+ if: steps.changed.outputs.build == 'true'
+ - name: Build
+ if: steps.changed.outputs.build == 'true'
+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+ with:
+ context: deploy/images
+ file: deploy/images/sandbox-base.Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: false
+
waf:
name: WAF
runs-on: ubuntu-24.04
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 86a5903..5115726 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -91,7 +91,7 @@ jobs:
strategy:
fail-fast: true
matrix:
- image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static]
+ image: [pyo, agent, builder, flows, waf, geoip, db-controller, kv, static, sandbox-base]
# Pyo's Postgres, one image per version, from one Dockerfile.
# One repository, tagged -: CloudNativePG reads
# the version from the tag.
@@ -111,8 +111,13 @@ jobs:
# (deploy/images/moco), one Dockerfile with a target each.
- { image: mysql-operator, file: moco/moco, context: deploy/images/moco, target: operator }
- { image: mysql-backup, file: moco/moco, context: deploy/images/moco, target: backup }
+ # Pyo Sandbox's built-in template `base`. Sandbox servers run on
+ # both architectures, so it's built for both (arm64 under QEMU).
+ - { image: sandbox-base, context: deploy/images, platforms: "linux/amd64,linux/arm64" }
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+ if: contains(matrix.platforms, 'arm64')
- uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
with:
@@ -128,7 +133,7 @@ jobs:
file: deploy/images/${{ matrix.file || matrix.image }}.Dockerfile
build-args: ${{ matrix.args }}
target: ${{ matrix.target }}
- platforms: linux/amd64
+ platforms: ${{ matrix.platforms || 'linux/amd64' }}
cache-from: ${{ matrix.rust && format('type=gha,scope=rust-{0}', github.run_id) || '' }}
push: true
tags: ${{ env.IMAGES }}/${{ matrix.repo || matrix.image }}:${{ matrix.tag }}${{ needs.verify.outputs.version }}
@@ -138,6 +143,22 @@ jobs:
env:
IMAGE: ${{ env.IMAGES }}/${{ matrix.repo || matrix.image }}@${{ steps.build.outputs.digest }}
run: cosign sign --yes "$IMAGE"
+ # Installations and sandbox servers pull without logging in. A new
+ # package on ghcr.io may start private: make it public in its
+ # package settings, then run the job again.
+ - name: Anyone can pull it
+ env:
+ REPO: upsetdev/pyo/${{ matrix.repo || matrix.image }}
+ DIGEST: ${{ steps.build.outputs.digest }}
+ run: |
+ token=$(curl -fsS "https://ghcr.io/token?scope=repository:$REPO:pull" | jq -r .token)
+ code=$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" \
+ -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json' \
+ "https://ghcr.io/v2/$REPO/manifests/$DIGEST")
+ if [ "$code" != 200 ]; then
+ echo "::error::ghcr.io/$REPO can't be pulled without logging in (HTTP $code). Make the package public in its settings on GitHub, then run this job again."
+ exit 1
+ fi
release:
name: GitHub release
@@ -176,6 +197,7 @@ jobs:
cd sandbox
for a in x86_64 aarch64; do
sha256sum -c "pyo-sandbox-linux-$a.tar.gz.sha256"
+ sha256sum -c "pyo-sandbox-gvisor-linux-$a.tar.gz.sha256"
done
# What Pyo needs to update itself from the dashboard: the images
# pinned by the digests that were just signed, and the manifests.
@@ -187,13 +209,15 @@ jobs:
tagged() { docker buildx imagetools inspect "$IMAGES/$1:$2$VERSION" --format '{{json .Manifest}}' | jq -r .digest; }
bun run scripts/release-manifest.ts --version "$VERSION" --images "$IMAGES" \
--pyo "$(digest pyo)" --agent "$(digest agent)" --builder "$(digest builder)" --flows "$(digest flows)" --waf "$(digest waf)" --geoip "$(digest geoip)" \
- --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" \
+ --db-controller "$(digest db-controller)" --kv "$(digest kv)" --static "$(digest static)" --sandbox-base "$(digest sandbox-base)" \
--mysql-operator "$(digest mysql-operator)" --mysql-backup "$(digest mysql-backup)" \
--postgres-16 "$(tagged postgres 16-)" --postgres-17 "$(tagged postgres 17-)" --postgres-18 "$(tagged postgres 18-)" \
--agent-access "$(docker run --rm "$IMAGES/agent:$VERSION@$(digest agent)" access-hash)" \
--sandbox-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256)" \
- --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" > release.json
- jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, migrations: (.migrations | length)}' release.json
+ --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" \
+ --sandbox-gvisor-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256)" \
+ --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" > release.json
+ jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, migrations: (.migrations | length)}' release.json
# The release's own upgrader reads it before it's signed: an image
# it doesn't accept, or an agentAccess that isn't its agent's roles,
# fails the release here, not on installations.
@@ -276,3 +300,7 @@ jobs:
sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256
sandbox/pyo-sandbox-linux-aarch64.tar.gz
sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256
+ sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz
+ sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256
+ sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz
+ sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256
diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
index 34203a1..cd1bc58 100644
--- a/.github/workflows/sandbox.yml
+++ b/.github/workflows/sandbox.yml
@@ -3,7 +3,8 @@ name: Sandbox
# A sandbox server's files for each architecture, as
# pyo-sandbox-linux-.tar.gz with its .sha256: the daemon and
# pyo-envd (static, musl), Firecracker and its jailer, and Pyo's guest
-# kernel. deploy/sandbox.sh installs them. The Release workflow runs it and
+# kernel. Beside it, pyo-sandbox-gvisor-linux-.tar.gz: runsc for
+# servers without KVM. deploy/sandbox.sh installs them. The Release workflow runs it and
# puts the tarballs in the GitHub release. Run it by hand (Actions ›
# Sandbox › Run workflow) to try the builds on a branch: it publishes
# nothing, the tarballs stay in the run's artifacts.
@@ -97,6 +98,9 @@ jobs:
check/pyo-envd --version
check/firecracker --version
check/jailer --version
+ mkdir check-gvisor
+ tar -xzf "out/pyo-sandbox-gvisor-linux-$ARCH.tar.gz" -C check-gvisor
+ check-gvisor/runsc --version
file check/pyo-sandbox check/pyo-envd | tee /dev/stderr | grep -c 'static' | grep -qx 2
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cb0ed64..63abb2c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -50,6 +50,8 @@ Read the script before you run it. It changes the machine:
To undo it, run `/usr/local/bin/k3s-uninstall.sh`.
+If you skip this step, `scripts/pyo-up.sh` asks to run it for you.
+
**2. Build and deploy (as your normal user):**
```bash
diff --git a/Cargo.lock b/Cargo.lock
index bc2af04..4697e3e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3345,7 +3345,7 @@ dependencies = [
[[package]]
name = "pyo-agent"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"base64 0.23.1",
@@ -3366,7 +3366,7 @@ dependencies = [
[[package]]
name = "pyo-api"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"argon2 0.6.0",
@@ -3413,7 +3413,7 @@ dependencies = [
[[package]]
name = "pyo-builder"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"base64 0.23.1",
@@ -3432,7 +3432,7 @@ dependencies = [
[[package]]
name = "pyo-cli"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"clap",
@@ -3452,7 +3452,7 @@ dependencies = [
[[package]]
name = "pyo-db-controller"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"base64 0.23.1",
@@ -3474,7 +3474,7 @@ dependencies = [
[[package]]
name = "pyo-envd"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"libc",
"pyo-proto",
@@ -3483,7 +3483,7 @@ dependencies = [
[[package]]
name = "pyo-flows"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"libc",
@@ -3497,7 +3497,7 @@ dependencies = [
[[package]]
name = "pyo-kv"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"axum",
@@ -3514,7 +3514,7 @@ dependencies = [
[[package]]
name = "pyo-proto"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"base64 0.23.1",
"regex",
@@ -3528,7 +3528,7 @@ dependencies = [
[[package]]
name = "pyo-resp"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"futures",
"socket2",
@@ -3537,7 +3537,7 @@ dependencies = [
[[package]]
name = "pyo-sandbox"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"axum",
@@ -3567,7 +3567,7 @@ dependencies = [
[[package]]
name = "pyo-static"
-version = "0.4.0"
+version = "0.4.1"
dependencies = [
"anyhow",
"axum",
diff --git a/Cargo.toml b/Cargo.toml
index 66aabff..835adc5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,7 @@ resolver = "3"
members = ["crates/pyo-proto", "crates/pyo-api", "crates/pyo-agent", "crates/pyo-builder", "crates/pyo-cli", "crates/pyo-flows", "crates/pyo-resp", "crates/pyo-kv", "crates/pyo-db-controller", "crates/pyo-static", "crates/pyo-envd", "crates/pyo-sandbox"]
[workspace.package]
-version = "0.4.0"
+version = "0.4.1"
edition = "2024"
license = "Apache-2.0"
publish = false
diff --git a/RELEASING.md b/RELEASING.md
index c1d24ab..84b8e9e 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -19,7 +19,7 @@ A version is one number for everything: `version` in the root `Cargo.toml`.
`.github/workflows/release.yml` then checks that the tag matches `Cargo.toml`,
runs the tests (the WAF's Go tests in `waf/` too), publishes signed images
-to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup}`
+to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,postgres,mysql-operator,mysql-backup,sandbox-base}`
(all from `deploy/images/`, at the same version. The Rust binaries are
compiled once, by the `binaries` job, and the Rust images' jobs take them
from that run's build cache),
@@ -53,6 +53,14 @@ crates/pyo-sandbox/scripts/bundle.sh --arch x86_64 \
`sandbox.sh --bundle dist/pyo-sandbox-linux-x86_64.tar.gz` installs such a
build, with `--bundle-sha256 ` or, unchecked, `--dev`.
+`sandbox-base` is Pyo Sandbox's built-in template `base`
+(`deploy/images/sandbox-base.Dockerfile`), built for amd64 and arm64.
+`release.json` pins it as `images.sandboxBase`, and Pyo seeds the template
+from it. Sandbox servers pull it without logging in, so the release checks
+that every image can be pulled anonymously. The first time a new image is
+published, ghcr.io may make its package private: make it public in the
+package's settings, then run the failed job again.
+
`release.json` is signed with
cosign keyless as `release.json.sigstore.json` (the job needs `id-token:
write`, and `.github/release-signing-config.json` pins Sigstore's public
diff --git a/crates/pyo-agent/Cargo.toml b/crates/pyo-agent/Cargo.toml
index c02f93d..f8deb0a 100644
--- a/crates/pyo-agent/Cargo.toml
+++ b/crates/pyo-agent/Cargo.toml
@@ -7,7 +7,7 @@ publish.workspace = true
[dependencies]
anyhow = "1.0.104"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
futures = "0.3.34"
http = "1.5.0"
k8s-openapi = { version = "0.28.0", features = ["latest"] }
diff --git a/crates/pyo-api/Cargo.toml b/crates/pyo-api/Cargo.toml
index 7f28303..2256875 100644
--- a/crates/pyo-api/Cargo.toml
+++ b/crates/pyo-api/Cargo.toml
@@ -12,7 +12,7 @@ axum = { version = "0.8.9", features = ["ws"] }
axum-extra = { version = "0.12.6", features = ["cookie", "typed-header"] }
base64 = "0.23.1"
cookie = "0.18"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto", features = ["openapi", "sandbox-token"] }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto", features = ["openapi", "sandbox-token"] }
hex = "0.4.3"
hmac = "0.13.0"
jiff = "0.2.38"
diff --git a/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql b/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql
new file mode 100644
index 0000000..8b6854e
--- /dev/null
+++ b/crates/pyo-api/migrations/0014_cloud_sandbox_servers.sql
@@ -0,0 +1,13 @@
+-- pyo:contract since=v0.4.0 the role check only widens and cluster_id only stops being required, so every row a release writes still fits
+-- Sandbox servers Pyo makes at a cloud provider (cloud/mod.rs): the cloud
+-- server points at the sandbox server it becomes, and so does its
+-- bootstrap code, which then opens sandbox.sh's details instead of a
+-- cluster's. Kept as plain text, so a removed sandbox server's cloud server
+-- still knows which label to check before Pyo deletes it at the provider.
+ALTER TABLE pyo.cloud_servers ADD COLUMN sandbox_server_id text;
+CREATE INDEX cloud_servers_sandbox ON pyo.cloud_servers (sandbox_server_id) WHERE sandbox_server_id IS NOT NULL;
+
+ALTER TABLE pyo.cloud_bootstrap ADD COLUMN sandbox_server_id text;
+ALTER TABLE pyo.cloud_bootstrap ALTER COLUMN cluster_id DROP NOT NULL;
+ALTER TABLE pyo.cloud_bootstrap DROP CONSTRAINT cloud_bootstrap_role_check;
+ALTER TABLE pyo.cloud_bootstrap ADD CONSTRAINT cloud_bootstrap_role_check CHECK (role IN ('join', 'connect', 'sandbox'));
diff --git a/crates/pyo-api/migrations/0015_sandbox_base_template.sql b/crates/pyo-api/migrations/0015_sandbox_base_template.sql
new file mode 100644
index 0000000..643d70e
--- /dev/null
+++ b/crates/pyo-api/migrations/0015_sandbox_base_template.sql
@@ -0,0 +1,7 @@
+-- Pyo's own sandbox templates (the built-in `base`, sandbox/base.rs): Pyo
+-- keeps them in step with its release, and nobody deletes them. A
+-- revision names the image a template is made from now: servers keep it
+-- as `.`, so a new image is made beside the old one, which
+-- running and paused sandboxes keep using. NULL: the template's id alone.
+ALTER TABLE pyo.sandbox_templates ADD COLUMN builtin boolean NOT NULL DEFAULT false;
+ALTER TABLE pyo.sandbox_templates ADD COLUMN revision text;
diff --git a/crates/pyo-api/src/cloud/boot.sh b/crates/pyo-api/src/cloud/boot.sh
index 51292bd..fbdcedd 100644
--- a/crates/pyo-api/src/cloud/boot.sh
+++ b/crates/pyo-api/src/cloud/boot.sh
@@ -1,12 +1,14 @@
#!/usr/bin/env bash
-# Made by Pyo: joins this server to a cluster over your tailnet.
+# Made by Pyo: joins this server to Pyo over your tailnet.
#
# boot.sh PYO SERVER CLUSTER ROLE NAME
#
# 1. Joins your tailnet with a single-use Tailscale key.
# 2. Trades a single-use code for this server's join details at Pyo, over
# the tailnet (PYO: Pyo's tailnet door).
-# 3. Runs join.sh, or connect.sh for a new cluster (ROLE), with them.
+# 3. Runs join.sh, connect.sh for a new cluster, or sandbox.sh for a
+# sandbox server (ROLE), with them. For a sandbox server CLUSTER is the
+# sandbox server's ID.
#
# The key and the code are the only secrets in this server's user data.
# They're in /run/pyo-boot (root only, in memory) and deleted once used.
@@ -64,7 +66,7 @@ done < <(printf '%s\n' "$body")
unset out body
# <<< parse
case "$ROLE:$script" in
- join:http*/join.sh | connect:http*/connect.sh) ;;
+ join:http*/join.sh | connect:http*/connect.sh | sandbox:http*/sandbox.sh) ;;
*) fail "Pyo's answer has no $ROLE script." ;;
esac
diff --git a/crates/pyo-api/src/cloud/bootstrap.rs b/crates/pyo-api/src/cloud/bootstrap.rs
index e8e9863..fe83a6f 100644
--- a/crates/pyo-api/src/cloud/bootstrap.rs
+++ b/crates/pyo-api/src/cloud/bootstrap.rs
@@ -3,9 +3,11 @@
//! The user data holds two one-time values ([`super::user_data`]): a
//! single-use Tailscale key and a single-use code made here ([`new_code`]).
//! The code is random, kept only as its hash, expires after [`CODE_TTL`],
-//! and is bound to the cloud server, its cluster and its role (join.sh or
-//! connect.sh). Its join details wait here sealed until then
-//! ([`keep`]).
+//! and is bound to the cloud server, its cluster (or the sandbox server it
+//! becomes) and its role (join.sh, connect.sh or sandbox.sh). Its join
+//! details wait here sealed until then ([`keep`]). A sandbox server's
+//! details get a fresh sandbox join code when they're handed over
+//! (`sandbox::join`), so its 15 minutes start then.
//!
//! On its first boot the server joins the tailnet with the key, then asks
//! Pyo's tailnet door (pyo-forward's :5006, which admits tailnet peers
@@ -68,6 +70,8 @@ pub enum Role {
Join,
/// connect.sh: a new cluster's first server.
Connect,
+ /// sandbox.sh: a sandbox server.
+ Sandbox,
}
impl Role {
@@ -75,6 +79,7 @@ impl Role {
match self {
Role::Join => "join",
Role::Connect => "connect",
+ Role::Sandbox => "sandbox",
}
}
}
@@ -121,23 +126,29 @@ pub fn new_code() -> String {
/// Keeps a code (its hash) and the details it opens, sealed, for
/// [`CODE_TTL`]. After the cloud server's row, in the same transaction.
+/// `owner` is its cluster, or the sandbox server it becomes.
pub async fn keep(
tx: &mut Transaction<'_, Postgres>,
sealer: &Sealer,
code: &str,
cloud_server_id: &str,
- cluster_id: &str,
+ owner: &str,
role: Role,
material: &Material,
) -> ApiResult<()> {
let sealed = sealer.seal(&material.render()?)?;
+ let (cluster, sandbox) = match role {
+ Role::Sandbox => (None, Some(owner)),
+ Role::Join | Role::Connect => (Some(owner), None),
+ };
sqlx::query(
- "INSERT INTO pyo.cloud_bootstrap (code_hash, cloud_server_id, cluster_id, role, material_sealed, expires_at)
- VALUES ($1, $2, $3, $4, $5, now() + make_interval(secs => $6))",
+ "INSERT INTO pyo.cloud_bootstrap (code_hash, cloud_server_id, cluster_id, sandbox_server_id, role, material_sealed, expires_at)
+ VALUES ($1, $2, $3, $4, $5, $6, now() + make_interval(secs => $7))",
)
.bind(sha256_hex(code))
.bind(cloud_server_id)
- .bind(cluster_id)
+ .bind(cluster)
+ .bind(sandbox)
.bind(role.as_str())
.bind(sealed)
.bind(CODE_TTL.as_secs_f64())
@@ -236,6 +247,7 @@ impl Refusal {
pub struct Ask {
/// Its cloud server ID ("cs_…").
pub server: String,
+ /// Its cluster, or the sandbox server it becomes ("sbs_…").
pub cluster: String,
pub role: Role,
}
@@ -285,7 +297,8 @@ async fn claim(db: &PgPool, code: &str) -> sqlx::Result
UPDATE pyo.cloud_bootstrap c SET used_at = now(), material_sealed = NULL
FROM prev
WHERE c.code_hash = prev.code_hash
- RETURNING c.cloud_server_id, c.cluster_id, c.role, c.expires_at <= now() AS expired,
+ RETURNING c.cloud_server_id, coalesce(c.cluster_id, c.sandbox_server_id, '') AS cluster_id,
+ c.role, c.expires_at <= now() AS expired,
prev.material_sealed,
(SELECT state FROM pyo.cloud_servers WHERE id = c.cloud_server_id) AS state",
)
@@ -369,7 +382,7 @@ pub async fn answer(
return text(StatusCode::FORBIDDEN, r.message());
}
};
- let material = match sealer.open(&sealed) {
+ let mut material = match sealer.open(&sealed) {
Ok(m) => m,
Err(e) => {
tracing::warn!(server = %ask.server, error = %e, "cloud server setup: couldn't open its details");
@@ -379,6 +392,20 @@ pub async fn answer(
);
}
};
+ // A sandbox server's join code is made now, so sandbox.sh has its full
+ // 15 minutes from here.
+ if ask.role == Role::Sandbox {
+ match sandbox_code(db, &ask.cluster).await {
+ Ok(code) => material.push_str(&format!("arg=--code\narg={code}\n")),
+ Err(e) => {
+ tracing::warn!(server = %ask.server, error = %e.message, "cloud server setup: couldn't make a sandbox join code");
+ return text(
+ StatusCode::SERVICE_UNAVAILABLE,
+ "Pyo couldn't make this server's join code. Delete it and make a new one.",
+ );
+ }
+ }
+ }
if let Err(e) = sqlx::query(
"UPDATE pyo.cloud_servers SET state = 'joining', error = NULL, updated_at = now() WHERE id = $1 AND state IN ('creating', 'booting')",
)
@@ -392,8 +419,17 @@ pub async fn answer(
text(StatusCode::OK, material)
}
-/// POST /cloud/bootstrap?server=…&cluster=…&role=join|connect on Pyo's
-/// tailnet door, with the code as the body.
+/// A new sandbox join code for a sandbox server Pyo made.
+async fn sandbox_code(db: &PgPool, sandbox_server: &str) -> ApiResult {
+ let code = crate::sandbox::join::new_code();
+ let mut tx = db.begin().await?;
+ crate::sandbox::join::keep(&mut tx, sandbox_server, &code).await?;
+ tx.commit().await?;
+ Ok(code)
+}
+
+/// POST /cloud/bootstrap?server=…&cluster=…&role=join|connect|sandbox on
+/// Pyo's tailnet door, with the code as the body.
pub async fn handle(
State(s): State,
ConnectInfo(peer): ConnectInfo,
@@ -720,6 +756,101 @@ mod tests {
let (st, _) = answer(&db, &sealer, tailnet, &a, &code).await;
assert_eq!(st, StatusCode::OK);
+ // A sandbox server: its details come with a fresh sandbox join code,
+ // made as they're handed over, which sandbox.sh then redeems.
+ let sbs = format!("sbs_boot{run}");
+ sqlx::query("INSERT INTO pyo.sandbox_servers (id, team_id, name, state, created_by) VALUES ($1, $2, $1, 'joining', 'test')")
+ .bind(&sbs)
+ .bind(&team)
+ .execute(&db)
+ .await
+ .unwrap();
+ let seven = format!("cs_seven{run}");
+ sqlx::query(
+ "INSERT INTO pyo.cloud_servers (id, team_id, provider, sandbox_server_id, name, location, server_type, monthly, remote_id, state, created_by)
+ VALUES ($1, $2, 'hetzner', $3, $1, 'fsn1', 'cx22', 4.29, '7', 'booting', 'test')",
+ )
+ .bind(&seven)
+ .bind(&team)
+ .bind(&sbs)
+ .execute(&db)
+ .await
+ .unwrap();
+ let sandbox_material = Material::new(
+ "https://pyo.tail1234.ts.net/install/sandbox.sh".into(),
+ &[
+ ("--server", Some(sbs.clone())),
+ ("--version", Some("0.4.0".into())),
+ ],
+ );
+ let code = new_code();
+ let mut tx = db.begin().await.unwrap();
+ keep(
+ &mut tx,
+ &sealer,
+ &code,
+ &seven,
+ &sbs,
+ Role::Sandbox,
+ &sandbox_material,
+ )
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ let (st, body) = answer(
+ &db,
+ &sealer,
+ tailnet,
+ &ask(&seven, &cluster, Role::Sandbox),
+ &code,
+ )
+ .await;
+ assert_eq!(
+ (st, body.as_str()),
+ (StatusCode::FORBIDDEN, "This code is for another server."),
+ "a cluster's ID isn't the sandbox server's"
+ );
+ let code = new_code();
+ let mut tx = db.begin().await.unwrap();
+ keep(
+ &mut tx,
+ &sealer,
+ &code,
+ &seven,
+ &sbs,
+ Role::Sandbox,
+ &sandbox_material,
+ )
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ let (st, body) = answer(
+ &db,
+ &sealer,
+ tailnet,
+ &ask(&seven, &sbs, Role::Sandbox),
+ &code,
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+ let base = sandbox_material.render().unwrap();
+ assert!(body.starts_with(&base), "{body}");
+ let join_code = body[base.len()..]
+ .strip_prefix("arg=--code\narg=")
+ .and_then(|r| r.strip_suffix('\n'))
+ .unwrap_or_else(|| panic!("no join code in {body}"));
+ assert_eq!(join_code.len(), 64);
+ let live: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM pyo.sandbox_join_codes WHERE server_id = $1 AND used_at IS NULL AND code_hash = $2",
+ )
+ .bind(&sbs)
+ .bind(sha256_hex(join_code))
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert_eq!(live, 1, "sandbox.sh can redeem it");
+ assert_eq!(state(&seven).await, "joining");
+
// Expired details go at once, their rows a day later.
prune(&db).await.unwrap();
let left: Option = sqlx::query_scalar(
diff --git a/crates/pyo-api/src/cloud/mod.rs b/crates/pyo-api/src/cloud/mod.rs
index 3dd89d4..0c56248 100644
--- a/crates/pyo-api/src/cloud/mod.rs
+++ b/crates/pyo-api/src/cloud/mod.rs
@@ -1,5 +1,6 @@
//! Servers Pyo makes at a cloud provider and joins to a cluster over the
-//! tailnet (Add server › Hetzner, Add cluster › Hetzner).
+//! tailnet (Add server › Hetzner, Add cluster › Hetzner), or sets up as a
+//! sandbox server (Sandboxes › Add server › Cloud provider).
//!
//! - **The connection** is a team's (Integrations › Hetzner, team admins):
//! clusters belong to a team, and the servers it makes are billed to the
@@ -11,9 +12,10 @@
//! single-use Tailscale key and a single-use code ([`user_data`]). On its
//! first boot it joins the tailnet, trades the code for its join details
//! over the tailnet ([`bootstrap`]), and runs join.sh (or connect.sh for
-//! a new cluster) with them. The server carries this installation's label
-//! (`pyo=`, `pyo-cluster=`) and Pyo's firewall: the
-//! web from anywhere and Tailscale's port, nothing else.
+//! a new cluster) with them, or sandbox.sh for a sandbox server. The
+//! server carries this installation's label (`pyo=`, and
+//! `pyo-cluster=` or `pyo-sandbox-server=`) and Pyo's
+//! firewall: the web from anywhere and Tailscale's port, nothing else.
//! - **Following it** ([`tick`]): Creating → Booting → Joining → Ready,
//! with the Tailnet Lock sign command when its key waits for a signature.
//! After [`TIMEOUT`] it stops waiting and says why.
@@ -50,6 +52,8 @@ use crate::{
pub const LABEL: &str = "pyo";
/// The cluster a server was made for.
pub const CLUSTER_LABEL: &str = "pyo-cluster";
+/// The sandbox server a server was made as.
+pub const SANDBOX_LABEL: &str = "pyo-sandbox-server";
/// How long Pyo waits for a new server to be ready.
pub const TIMEOUT: Duration = Duration::from_secs(15 * 60);
/// The providers there are.
@@ -235,19 +239,44 @@ pub trait Provider {
async fn firewall(&self, install: &str) -> Result;
}
+/// What a server is made for: a cluster, or a sandbox server.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Owner<'a> {
+ Cluster(&'a str),
+ Sandbox(&'a str),
+}
+
+impl<'a> Owner<'a> {
+ /// Its label at the provider.
+ pub fn label(self) -> (&'static str, &'a str) {
+ match self {
+ Owner::Cluster(id) => (CLUSTER_LABEL, id),
+ Owner::Sandbox(id) => (SANDBOX_LABEL, id),
+ }
+ }
+
+ pub fn id(self) -> &'a str {
+ self.label().1
+ }
+}
+
/// The labels a new server carries.
-pub fn labels(install: &str, cluster_id: &str) -> BTreeMap {
+pub fn labels(install: &str, owner: Owner) -> BTreeMap {
+ let (key, value) = owner.label();
BTreeMap::from([
(LABEL.to_owned(), install.to_owned()),
- (CLUSTER_LABEL.to_owned(), cluster_id.to_owned()),
+ (key.to_owned(), value.to_owned()),
])
}
/// Whether Pyo may delete a server: it carries this installation's label,
-/// and the cluster's when one is given.
-pub fn ours(r: &Remote, install: &str, cluster_id: Option<&str>) -> bool {
+/// and its cluster's or sandbox server's when one is given.
+pub fn ours(r: &Remote, install: &str, owner: Option) -> bool {
r.labels.get(LABEL).map(String::as_str) == Some(install)
- && cluster_id.is_none_or(|c| r.labels.get(CLUSTER_LABEL).map(String::as_str) == Some(c))
+ && owner.is_none_or(|o| {
+ let (key, value) = o.label();
+ r.labels.get(key).map(String::as_str) == Some(value)
+ })
}
/// Deletes a server, only when it carries this installation's label
@@ -256,12 +285,12 @@ pub async fn delete(
p: &P,
id: &str,
install: &str,
- cluster_id: Option<&str>,
+ owner: Option>,
) -> Result<(), CloudError> {
let Some(r) = p.server(id).await? else {
return Ok(());
};
- if !ours(&r, install, cluster_id) {
+ if !ours(&r, install, owner) {
return Err(CloudError::NotOurs);
}
p.delete_unchecked(id).await
@@ -306,6 +335,7 @@ pub struct Boot<'a> {
pub pyo: &'a str,
/// Its cloud server ID.
pub server_id: &'a str,
+ /// Its cluster, or the sandbox server it becomes.
pub cluster_id: &'a str,
pub role: bootstrap::Role,
/// Its name in the cluster and on the tailnet.
@@ -346,7 +376,7 @@ pub fn user_data(b: &Boot, ssh_key: Option<&str>) -> ApiResult {
));
}
let mut out = String::from(
- "#cloud-config\n# Made by Pyo: joins this server to a cluster over your tailnet.\nssh_pwauth: false\n",
+ "#cloud-config\n# Made by Pyo: joins this server to Pyo over your tailnet.\nssh_pwauth: false\n",
);
if let Some(k) = ssh_key {
// JSON's quoting is YAML's double-quoted string.
@@ -549,6 +579,7 @@ struct Row {
id: String,
team_id: String,
cluster_id: Option,
+ sandbox_server_id: Option,
name: String,
remote_id: String,
state: String,
@@ -568,6 +599,10 @@ pub struct Seen {
pub in_cluster: Option,
/// Tailnet Lock waits for its key to be signed: the command.
pub sign: Option,
+ /// It becomes a sandbox server, not a cluster's.
+ pub sandbox: bool,
+ /// The sandbox server can't run sandboxes: why.
+ pub refused: Option,
}
/// The next state of a server being made, and why it failed, if it did.
@@ -578,6 +613,9 @@ pub fn next_state(seen: &Seen, age: Duration) -> (&'static str, Option)
let Some(remote) = seen.remote.as_deref() else {
return ("failed", Some("It was deleted at Hetzner.".into()));
};
+ if let Some(why) = &seen.refused {
+ return ("failed", Some(format!("It can't run sandboxes. {why}")));
+ }
let state = if seen.in_cluster.is_some() || seen.on_tailnet {
"joining"
} else if matches!(remote, "running" | "starting") {
@@ -596,6 +634,9 @@ pub fn next_state(seen: &Seen, age: Duration) -> (&'static str, Option)
_ if seen.sign.is_some() => {
"Its key wasn't signed in 15 minutes. Sign it, or delete the server."
}
+ _ if seen.sandbox => {
+ "It joined your tailnet but didn't become a sandbox server in 15 minutes. Read /var/log/pyo-join.log on it."
+ }
_ => {
"It joined your tailnet but not the cluster in 15 minutes. Read /var/log/pyo-join.log on it."
}
@@ -621,7 +662,7 @@ async fn tailnet_devices(s: &AppState) -> Option>
pub async fn tick(s: &AppState) -> ApiResult<()> {
bootstrap::prune(&s.db).await?;
let rows: Vec = sqlx::query_as(
- "SELECT id, team_id, cluster_id, name, remote_id, state,
+ "SELECT id, team_id, cluster_id, sandbox_server_id, name, remote_id, state,
extract(epoch FROM now() - created_at)::float8 AS age_secs, sign_command, first_server
FROM pyo.cloud_servers WHERE state IN ('creating', 'booting', 'joining', 'removing')
ORDER BY created_at",
@@ -691,8 +732,9 @@ async fn follow(
devices: Option<&[crate::tailscale::Device]>,
) -> ApiResult<()> {
let remote = p.server(&r.remote_id).await?;
- let in_cluster: Option = match &r.cluster_id {
- Some(c) => sqlx::query_scalar(
+ let mut refused = None;
+ let in_cluster: Option = match (&r.cluster_id, &r.sandbox_server_id) {
+ (Some(c), _) => sqlx::query_scalar(
"SELECT report->>'status' = 'ready' FROM pyo.servers WHERE cluster_id = $1 AND name = $2",
)
.bind(c)
@@ -700,7 +742,26 @@ async fn follow(
.fetch_optional(&s.db)
.await?
.map(|x: Option| x.unwrap_or(false)),
- None => None,
+ // A sandbox server counts once sandbox.sh redeemed its code, and is
+ // ready once its daemon reports.
+ (None, Some(sbs)) => {
+ let row: Option<(String, bool, Option)> = sqlx::query_as(
+ "SELECT state, joined_at IS NOT NULL, error FROM pyo.sandbox_servers WHERE id = $1",
+ )
+ .bind(sbs)
+ .fetch_optional(&s.db)
+ .await?;
+ match row {
+ Some((state, _, _)) if state == "ready" => Some(true),
+ Some((state, _, error)) if state == "refused" => {
+ refused = Some(error.unwrap_or_default());
+ Some(false)
+ }
+ Some((_, true, _)) => Some(false),
+ _ => None,
+ }
+ }
+ (None, None) => None,
};
let mine: Vec<&crate::tailscale::Device> = devices
.map(|d| device_of(d, &r.name).collect())
@@ -714,6 +775,8 @@ async fn follow(
on_tailnet: !mine.is_empty(),
in_cluster,
sign: sign.clone(),
+ sandbox: r.sandbox_server_id.is_some(),
+ refused,
};
let (mut state, error) = next_state(&seen, Duration::from_secs_f64(r.age_secs.max(0.0)));
// It traded its code over the tailnet: Joining, even when the tailnet's
@@ -836,7 +899,12 @@ async fn finish_removal(
tracing::warn!(server = %r.name, error = %e, "couldn't take the server off the tailnet");
}
}
- match delete(p, &r.remote_id, install, r.cluster_id.as_deref()).await {
+ let owner = match (&r.cluster_id, &r.sandbox_server_id) {
+ (Some(c), _) => Some(Owner::Cluster(c)),
+ (None, Some(sbs)) => Some(Owner::Sandbox(sbs)),
+ (None, None) => None,
+ };
+ match delete(p, &r.remote_id, install, owner).await {
Ok(()) => {
sqlx::query(
"UPDATE pyo.cloud_servers SET state = 'deleted', error = NULL, sign_command = NULL, deleted_at = now(), updated_at = now() WHERE id = $1",
@@ -872,7 +940,8 @@ pub async fn snapshot(db: &PgPool, team_ids: &[String]) -> ApiResult<(Value, Val
.fetch_all(db)
.await?;
let servers: Vec = sqlx::query_scalar(
- "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'provider', provider, 'clusterId', cluster_id, 'name', name,
+ "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'provider', provider, 'clusterId', cluster_id,
+ 'sandboxServerId', sandbox_server_id, 'name', name,
'firstServer', first_server, 'builder', builder, 'location', location, 'serverType', server_type,
'monthly', monthly, 'currency', currency, 'ipv4', ipv4, 'state', state, 'error', error,
'signCommand', sign_command, 'createdBy', created_by, 'createdAt', pyo.ms(created_at),
@@ -898,7 +967,7 @@ mod tests {
location: "fsn1",
server_type: ty,
user_data,
- labels: labels(INSTALL, cluster),
+ labels: labels(INSTALL, Owner::Cluster(cluster)),
firewall: None,
}
}
@@ -968,10 +1037,14 @@ mod tests {
assert_eq!(h.server(&r.id).await.unwrap().unwrap().id, r.id);
// The same name again: Hetzner's names are unique in a project.
assert_eq!(h.create(&c).await.unwrap_err(), CloudError::NameTaken);
- delete(&h, &r.id, INSTALL, Some("cl_one")).await.unwrap();
+ delete(&h, &r.id, INSTALL, Some(Owner::Cluster("cl_one")))
+ .await
+ .unwrap();
assert_eq!(h.server(&r.id).await.unwrap(), None);
// Gone already is fine.
- delete(&h, &r.id, INSTALL, Some("cl_one")).await.unwrap();
+ delete(&h, &r.id, INSTALL, Some(Owner::Cluster("cl_one")))
+ .await
+ .unwrap();
// Ids are numbers: anything else is never sent.
assert_eq!(h.server("1/../../firewalls").await.unwrap(), None);
}
@@ -985,13 +1058,35 @@ mod tests {
json!({ "pyo": "ffffffffffffffffffffffffffffffff", "pyo-cluster": "cl_one" }),
);
let other_cluster = m.add_server(json!({ "pyo": INSTALL, "pyo-cluster": "cl_two" }));
- for id in [unlabelled, other_pyo, other_cluster] {
+ let sandbox = m.add_server(json!({ "pyo": INSTALL, "pyo-sandbox-server": "sbs_one" }));
+ for id in [unlabelled, other_pyo, other_cluster, sandbox] {
assert_eq!(
- delete(&h, &id.to_string(), INSTALL, Some("cl_one")).await,
+ delete(&h, &id.to_string(), INSTALL, Some(Owner::Cluster("cl_one"))).await,
Err(CloudError::NotOurs)
);
}
- assert_eq!(m.0.lock().unwrap().servers.len(), 3);
+ // A sandbox server only by its own label.
+ assert_eq!(
+ delete(
+ &h,
+ &other_cluster.to_string(),
+ INSTALL,
+ Some(Owner::Sandbox("sbs_one"))
+ )
+ .await,
+ Err(CloudError::NotOurs)
+ );
+ assert_eq!(
+ delete(
+ &h,
+ &sandbox.to_string(),
+ INSTALL,
+ Some(Owner::Sandbox("sbs_two"))
+ )
+ .await,
+ Err(CloudError::NotOurs)
+ );
+ assert_eq!(m.0.lock().unwrap().servers.len(), 4);
assert!(
!m.0.lock()
.unwrap()
@@ -1004,6 +1099,14 @@ mod tests {
delete(&h, &other_cluster.to_string(), INSTALL, None)
.await
.unwrap();
+ delete(
+ &h,
+ &sandbox.to_string(),
+ INSTALL,
+ Some(Owner::Sandbox("sbs_one")),
+ )
+ .await
+ .unwrap();
assert_eq!(m.0.lock().unwrap().servers.len(), 2);
}
@@ -1243,6 +1346,8 @@ mod tests {
on_tailnet,
in_cluster,
sign: sign.map(str::to_owned),
+ sandbox: false,
+ refused: None,
};
let soon = Duration::from_secs(60);
let late = TIMEOUT + Duration::from_secs(1);
@@ -1282,5 +1387,22 @@ mod tests {
let (s, why) = next_state(&seen(None, false, None, None), soon);
assert_eq!(s, "failed");
assert!(why.unwrap().contains("deleted at Hetzner"));
+
+ // A sandbox server: refused at once, and its own words when late.
+ let sbx = |in_cluster, refused: Option<&str>| Seen {
+ sandbox: true,
+ refused: refused.map(str::to_owned),
+ ..seen(Some("running"), true, in_cluster, None)
+ };
+ assert_eq!(next_state(&sbx(Some(true), None), soon), ("ready", None));
+ let (s, why) = next_state(&sbx(Some(false), Some("runsc isn't installed.")), soon);
+ assert_eq!(s, "failed");
+ assert_eq!(
+ why.unwrap(),
+ "It can't run sandboxes. runsc isn't installed."
+ );
+ let (s, why) = next_state(&sbx(Some(false), None), late);
+ assert_eq!(s, "failed");
+ assert!(why.unwrap().contains("didn't become a sandbox server"));
}
}
diff --git a/crates/pyo-api/src/github.rs b/crates/pyo-api/src/github.rs
index e5c5d25..ed8ccd7 100644
--- a/crates/pyo-api/src/github.rs
+++ b/crates/pyo-api/src/github.rs
@@ -210,7 +210,7 @@ async fn created(
webhook,
}) = take(&q.state).await
else {
- return Ok(Redirect::to("/settings?github=expired"));
+ return Ok(Redirect::to("/github?github=expired"));
};
if user_id != user.id || !user.is_admin {
return Err(ApiError::forbidden(
@@ -225,7 +225,7 @@ async fn created(
.map_err(anyhow::Error::from)?;
if !res.status().is_success() {
tracing::warn!(status = %res.status(), "GitHub app conversion failed");
- return Ok(Redirect::to("/settings?github=failed"));
+ return Ok(Redirect::to("/github?github=app-failed"));
}
let v: Value = res.json().await.map_err(anyhow::Error::from)?;
let seal = |k: &str| s.sealer.seal(v[k].as_str().unwrap_or(""));
@@ -246,7 +246,8 @@ async fn created(
.bind(serde_json::to_value(&app).map_err(anyhow::Error::from)?)
.execute(&s.db)
.await?;
- Ok(Redirect::to("/settings?github=created"))
+ // Back to GitHub in Settings, which says how it went.
+ Ok(Redirect::to("/github?github=created"))
}
// ── Installations (teams) ────────────────────────────────────────────────────
@@ -304,6 +305,7 @@ async fn installed(
.fetch_optional(&s.db)
.await?;
let Some(t) = linked else {
+ // The dashboard opens the current team's GitHub card (/?github=).
return Ok(Redirect::to("/?github=expired"));
};
let mut tx = s.db.begin().await?;
diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs
index 705c578..15ef499 100644
--- a/crates/pyo-api/src/join.rs
+++ b/crates/pyo-api/src/join.rs
@@ -26,6 +26,9 @@ pub const PUBLISHED_SCRIPTS_URL: &str = "https://pyo.dev";
/// 3. Pyo on the tailnet ([`crate::web::tailnet_api_url`]), then its HTTPS
/// dashboard;
/// 4. pyo.dev, when Pyo is only on its own machine or in its cluster.
+///
+/// Only for machines already on the tailnet, or told Pyo's public URL. A
+/// command with a Tailscale key uses [`scripts_url_before_tailnet`].
pub fn scripts_url(reach: Option<&str>) -> String {
let configured = std::env::var("PYO_SCRIPTS_URL").ok();
let tailnet = crate::web::tailnet_api_url().map(|u| friendly(&u));
@@ -40,8 +43,8 @@ pub fn scripts_url(reach: Option<&str>) -> String {
}
fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String {
- if let Some(u) = configured.map(str::trim).filter(|u| !u.is_empty()) {
- return u.trim_end_matches('/').to_owned();
+ if let Some(u) = setting(configured) {
+ return u;
}
match pyo
.iter()
@@ -54,6 +57,38 @@ fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String {
}
}
+/// Where a command that gives the machine a Tailscale key (`--tailscale-key`)
+/// fetches its script. The machine joins the tailnet only once the script
+/// runs, so the script can't come from Pyo on the tailnet. Nor from Pyo on
+/// its public IP: that address may not be reachable from outside (a server
+/// behind NAT, like Starlink's). So:
+/// 1. `PYO_SCRIPTS_URL`, when set;
+/// 2. else pyo.dev. Its sandbox.sh runs the script of the release
+/// `--version` names. Its join.sh and connect.sh run the latest
+/// release's script, and join.sh's `--k3s` keeps the server on the
+/// cluster's k3s.
+///
+/// A dev build's version may not be a published release, so pyo.dev has no
+/// sandbox.sh for it and its wrapper fails to download one: dev builds set
+/// `PYO_SCRIPTS_URL` (to a mirror, or a Pyo the new machine can reach).
+///
+/// Commands for machines already on the tailnet use [`scripts_url`].
+pub fn scripts_url_before_tailnet() -> String {
+ pick_scripts_url_before_tailnet(std::env::var("PYO_SCRIPTS_URL").ok().as_deref())
+}
+
+pub(crate) fn pick_scripts_url_before_tailnet(configured: Option<&str>) -> String {
+ setting(configured).unwrap_or_else(|| PUBLISHED_SCRIPTS_URL.to_owned())
+}
+
+/// `PYO_SCRIPTS_URL`, when it's set to something.
+fn setting(configured: Option<&str>) -> Option {
+ configured
+ .map(str::trim)
+ .filter(|u| !u.is_empty())
+ .map(|u| u.trim_end_matches('/').to_owned())
+}
+
/// The host of an http(s) URL, lowercase, without brackets or port.
fn url_host(url: &str) -> Option {
let rest = url
@@ -813,6 +848,63 @@ mod tests {
assert_eq!(pick_scripts_url(None, &[None, None]), PUBLISHED_SCRIPTS_URL);
}
+ #[test]
+ fn a_machine_off_the_tailnet_fetches_scripts_from_the_setting_or_pyo_dev() {
+ let pick = pick_scripts_url_before_tailnet;
+ assert_eq!(
+ pick(Some(" https://mirror.example.com/ ")),
+ "https://mirror.example.com"
+ );
+ // Unset or blank: pyo.dev, never Pyo itself (on the tailnet, or on
+ // a public IP that may sit behind NAT).
+ for unset in [None, Some(""), Some(" ")] {
+ assert_eq!(pick(unset), PUBLISHED_SCRIPTS_URL);
+ }
+ }
+
+ #[test]
+ fn commands_with_a_tailscale_key_fetch_their_script_off_the_tailnet() {
+ let base = pick_scripts_url_before_tailnet(None);
+ // join.sh: pyo.dev runs the latest release's join.sh, and --k3s keeps
+ // the server on the cluster's k3s.
+ assert_eq!(
+ command(
+ &base,
+ &Join {
+ server_ip: "server-1.tail1234.ts.net",
+ token: "t",
+ server_name: "server-2",
+ k3s_version: "v1.37.2+k3s1",
+ tailscale_key: Some("tskey-auth-1"),
+ builder: false,
+ registry: None,
+ },
+ ),
+ "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'"
+ );
+ assert_eq!(
+ connect_command(
+ &base,
+ "https://pyo.tail1234.ts.net",
+ "abc",
+ Some("tskey-auth-1"),
+ Some(("pyo-registry.tail1234.ts.net", "def"))
+ ),
+ "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'"
+ );
+ // A mirror the operator set.
+ assert!(
+ connect_command(
+ &pick_scripts_url_before_tailnet(Some("https://mirror.example.com")),
+ "https://pyo.tail1234.ts.net",
+ "abc",
+ Some("k"),
+ None
+ )
+ .starts_with("curl -fsSL https://mirror.example.com/connect.sh | ")
+ );
+ }
+
#[test]
fn plain_http_only_to_private_addresses() {
for ok in [
diff --git a/crates/pyo-api/src/lib.rs b/crates/pyo-api/src/lib.rs
index b422d15..67cdf9e 100644
--- a/crates/pyo-api/src/lib.rs
+++ b/crates/pyo-api/src/lib.rs
@@ -366,6 +366,7 @@ pub async fn run(ext: Extensions) -> anyhow::Result<()> {
tokio::spawn(tailscale::runner(state.clone()));
tokio::spawn(cdn::keeper(state.clone()));
tokio::spawn(sandbox::runner::runner(state.clone()));
+ tokio::spawn(sandbox::base::seed(state.clone()));
for task in tasks {
tokio::spawn(task(state.clone()));
}
diff --git a/crates/pyo-api/src/mcp.rs b/crates/pyo-api/src/mcp.rs
index 03d05f0..dba3850 100644
--- a/crates/pyo-api/src/mcp.rs
+++ b/crates/pyo-api/src/mcp.rs
@@ -688,9 +688,12 @@ async fn run_endpoint(s: &AppState, user: &User, id: &str, args: &Value) -> Resu
"listSandboxServers" => {
json_of(crate::sandbox::api::list_servers(st(), me(), Query(args_as(args)?)).await)
}
- "removeSandboxServer" => {
- json_of(crate::sandbox::api::remove_server(st(), me(), path_id()?).await)
+ "createCloudSandboxServer" => {
+ json_of(crate::sandbox::api::add_cloud_server(st(), me(), Json(args_as(args)?)).await)
}
+ "removeSandboxServer" => json_of(
+ crate::sandbox::api::remove_server(st(), me(), path_id()?, Query(args_as(args)?)).await,
+ ),
"listSandboxTemplates" => {
json_of(crate::sandbox::api::list_templates(st(), me(), Query(args_as(args)?)).await)
}
diff --git a/crates/pyo-api/src/openapi.rs b/crates/pyo-api/src/openapi.rs
index cf9b654..7c700be 100644
--- a/crates/pyo-api/src/openapi.rs
+++ b/crates/pyo-api/src/openapi.rs
@@ -85,6 +85,7 @@ use crate::{auth::KeyAccess, error::ErrorBody, ops::Op};
crate::sandbox::api::close_port,
crate::sandbox::api::list_servers,
crate::sandbox::api::add_server,
+ crate::sandbox::api::add_cloud_server,
crate::sandbox::api::remove_server,
crate::sandbox::api::issue_certificate,
crate::sandbox::api::create_dns_record,
diff --git a/crates/pyo-api/src/ops.rs b/crates/pyo-api/src/ops.rs
index 9536807..1364901 100644
--- a/crates/pyo-api/src/ops.rs
+++ b/crates/pyo-api/src/ops.rs
@@ -330,9 +330,16 @@ pub async fn team_of(
/// Top-level paths the dashboard owns can't be team slugs.
// "pyo": Pyo registry keeps Pyo's own builds under pyo/.
-const RESERVED_SLUGS: [&str; 18] = [
+// "backups" to "updates": the installation's settings pages
+// (SETTINGS_PAGES in dash/src/lib/pyo/paths.ts).
+const RESERVED_SLUGS: [&str; 23] = [
"profile",
"settings",
+ "backups",
+ "email",
+ "github",
+ "observability",
+ "updates",
"oauth",
"api-keys",
"mcp",
@@ -742,7 +749,7 @@ async fn reach(
let api = crate::web::tailnet_api_url().ok_or_else(|| {
ApiError::conflict("Pyo's server isn't on your tailnet yet. Try again in a minute.")
})?;
- let key = crate::tailscale::join_key(s, what).await?;
+ let key = crate::tailscale::join_key(s, what, "this server").await?;
let door = crate::join::friendly(&deploy::registry_push());
let registry = if door.is_empty() {
None
@@ -817,9 +824,11 @@ async fn connect_command(
));
}
let (api, key, registry) = reach(tx, s, cluster_id, "Pyo: connect a cluster").await?;
+ // The machine joins the tailnet in connect.sh (--tailscale-key), so it
+ // fetches the script off the tailnet.
Ok((
crate::join::connect_command(
- &crate::join::scripts_url(Some(&api)),
+ &crate::join::scripts_url_before_tailnet(),
&api,
token,
Some(&key),
@@ -832,7 +841,8 @@ async fn connect_command(
/// What a new server of a cluster joins with (a k3s join token the agent
/// makes, valid an hour): join.sh's details and a fresh Tailscale key.
struct JoinParts {
- /// Where join.sh is.
+ /// Where join.sh is for a server already on the tailnet. A command
+ /// with the Tailscale key uses [`crate::join::scripts_url_before_tailnet`].
base: String,
server_ip: String,
token: String,
@@ -873,9 +883,12 @@ async fn join_command(
wake: &mut Vec,
) -> ApiResult {
let p = join_parts(tx, s, team_id, cluster_id, server_name, wake).await?;
- let command = crate::join::command(&p.base, &p.join(server_name, builder, true));
+ // The server joins the tailnet in join.sh (--tailscale-key), so it
+ // fetches the script off the tailnet.
+ let base = crate::join::scripts_url_before_tailnet();
+ let command = crate::join::command(&base, &p.join(server_name, builder, true));
Ok(
- json!({ "ok": true, "command": command, "scriptUrl": format!("{}/join.sh", p.base), "expiresAt": now_ms() + p.ttl as i64 * 1000, "tailscale": true }),
+ json!({ "ok": true, "command": command, "scriptUrl": format!("{base}/join.sh"), "expiresAt": now_ms() + p.ttl as i64 * 1000, "tailscale": true }),
)
}
@@ -975,7 +988,7 @@ fn cloud_ssh_key(k: Option) -> ApiResult> {
}
/// A price as people read it: "€4.29".
-fn money(currency: &str, amount: f64) -> String {
+pub(crate) fn money(currency: &str, amount: f64) -> String {
match currency {
"EUR" => format!("€{amount:.2}"),
"USD" => format!("${amount:.2}"),
@@ -1006,31 +1019,32 @@ async fn cloud_name_free(
}
/// A server to make at the provider.
-struct CloudNew<'a> {
- id: &'a str,
- team_id: &'a str,
- cluster_id: &'a str,
- name: &'a str,
- location: &'a str,
- server_type: &'a str,
- monthly: f64,
- currency: &'a str,
+pub(crate) struct CloudNew<'a> {
+ pub id: &'a str,
+ pub team_id: &'a str,
+ /// Its cluster, or the sandbox server it becomes.
+ pub owner: crate::cloud::Owner<'a>,
+ pub name: &'a str,
+ pub location: &'a str,
+ pub server_type: &'a str,
+ pub monthly: f64,
+ pub currency: &'a str,
/// The cluster's first server (connect.sh).
- first: bool,
- builder: bool,
+ pub first: bool,
+ pub builder: bool,
/// Single-use, for its user data.
- tailscale_key: &'a str,
- /// What its code opens: join.sh's or connect.sh's details.
- material: crate::cloud::bootstrap::Material,
- ssh_key: Option<&'a str>,
- by: &'a str,
+ pub tailscale_key: &'a str,
+ /// What its code opens: join.sh's, connect.sh's or sandbox.sh's details.
+ pub material: crate::cloud::bootstrap::Material,
+ pub ssh_key: Option<&'a str>,
+ pub by: &'a str,
}
/// Makes the server at the provider, labelled as this installation's and
/// behind its firewall, with user data that holds only its Tailscale key
/// and a single-use code (cloud/bootstrap.rs), and keeps it to follow
/// (cloud/mod.rs `tick`). The last step before the operation commits.
-async fn make_cloud_server(
+pub(crate) async fn make_cloud_server(
s: &AppState,
p: &P,
n: CloudNew<'_>,
@@ -1043,10 +1057,15 @@ async fn make_cloud_server(
"Pyo's server isn't on your tailnet yet. Try again in a minute.",
));
}
- let role = if n.first {
- bootstrap::Role::Connect
- } else {
- bootstrap::Role::Join
+ use crate::cloud::Owner;
+ let role = match n.owner {
+ Owner::Sandbox(_) => bootstrap::Role::Sandbox,
+ Owner::Cluster(_) if n.first => bootstrap::Role::Connect,
+ Owner::Cluster(_) => bootstrap::Role::Join,
+ };
+ let (cluster_id, sandbox_server_id) = match n.owner {
+ Owner::Cluster(c) => (Some(c), None),
+ Owner::Sandbox(x) => (None, Some(x)),
};
// Checked before anything is made: its details are kept sealed.
n.material.render()?;
@@ -1061,7 +1080,7 @@ async fn make_cloud_server(
&crate::cloud::Boot {
pyo: &format!("http://{door}"),
server_id: n.id,
- cluster_id: n.cluster_id,
+ cluster_id: n.owner.id(),
role,
name: n.name,
tailscale_key: n.tailscale_key,
@@ -1077,7 +1096,7 @@ async fn make_cloud_server(
location: n.location,
server_type: n.server_type,
user_data: &user_data,
- labels: crate::cloud::labels(&install, n.cluster_id),
+ labels: crate::cloud::labels(&install, n.owner),
firewall: Some(&firewall),
})
.await?;
@@ -1087,12 +1106,13 @@ async fn make_cloud_server(
"cloud server created"
);
sqlx::query(
- "INSERT INTO pyo.cloud_servers (id, team_id, provider, cluster_id, name, first_server, builder, location, server_type, monthly, currency, remote_id, ipv4, state, created_by)
- VALUES ($1, $2, 'hetzner', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 'creating', $13)",
+ "INSERT INTO pyo.cloud_servers (id, team_id, provider, cluster_id, sandbox_server_id, name, first_server, builder, location, server_type, monthly, currency, remote_id, ipv4, state, created_by)
+ VALUES ($1, $2, 'hetzner', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, 'creating', $14)",
)
.bind(n.id)
.bind(n.team_id)
- .bind(n.cluster_id)
+ .bind(cluster_id)
+ .bind(sandbox_server_id)
.bind(n.name)
.bind(n.first)
.bind(n.builder)
@@ -1105,10 +1125,10 @@ async fn make_cloud_server(
.bind(n.by)
.execute(&mut **tx)
.await?;
- bootstrap::keep(tx, &s.sealer, &code, n.id, n.cluster_id, role, &n.material).await?;
+ bootstrap::keep(tx, &s.sealer, &code, n.id, n.owner.id(), role, &n.material).await?;
Ok(json!({
- "ok": true, "id": n.id, "clusterId": n.cluster_id, "name": n.name, "state": "creating",
- "monthly": n.monthly, "currency": n.currency,
+ "ok": true, "id": n.id, "clusterId": cluster_id, "sandboxServerId": sandbox_server_id,
+ "name": n.name, "state": "creating", "monthly": n.monthly, "currency": n.currency,
}))
}
@@ -1123,16 +1143,17 @@ async fn delete_cloud_server(
confirm: bool,
wake: &mut Vec,
) -> ApiResult {
- let (team_id, cluster_id, name, first, state, monthly, currency): (
+ let (team_id, cluster_id, sandbox_server_id, name, first, state, monthly, currency): (
String,
Option,
+ Option,
String,
bool,
String,
f64,
String,
) = sqlx::query_as(
- "SELECT team_id, cluster_id, name, first_server, state, monthly, currency FROM pyo.cloud_servers WHERE id = $1 AND state <> 'deleted'",
+ "SELECT team_id, cluster_id, sandbox_server_id, name, first_server, state, monthly, currency FROM pyo.cloud_servers WHERE id = $1 AND state <> 'deleted'",
)
.bind(id)
.fetch_optional(&mut **tx)
@@ -1185,6 +1206,11 @@ async fn delete_cloud_server(
remove_node(tx, user, &server_id, !online, wake).await?;
}
}
+ // A sandbox server goes with it: its sandboxes end with the server.
+ if let Some(sbs) = &sandbox_server_id {
+ crate::sandbox::forget_server(tx, sbs, "Its sandbox server was deleted at Hetzner.")
+ .await?;
+ }
sqlx::query("UPDATE pyo.cloud_servers SET state = 'removing', error = NULL, sign_command = NULL, updated_at = now() WHERE id = $1")
.bind(id)
.execute(&mut **tx)
@@ -7032,7 +7058,7 @@ async fn apply(
CloudNew {
id: &id,
team_id: &team_id,
- cluster_id: &cluster_id,
+ owner: crate::cloud::Owner::Cluster(&cluster_id),
name: &name,
location: &location,
server_type: &server_type,
@@ -7125,7 +7151,7 @@ async fn apply(
CloudNew {
id: &server_id,
team_id: &team_id,
- cluster_id: &id,
+ owner: crate::cloud::Owner::Cluster(&id),
name: &server,
location: &location,
server_type: &server_type,
diff --git a/crates/pyo-api/src/sandbox/api.rs b/crates/pyo-api/src/sandbox/api.rs
index 52b8637..c909421 100644
--- a/crates/pyo-api/src/sandbox/api.rs
+++ b/crates/pyo-api/src/sandbox/api.rs
@@ -12,7 +12,7 @@ use axum::{
use serde::Deserialize;
use serde_json::{Value, json};
-use super::{AddServer, Connection, CreateSandbox, CreateTemplate, Need};
+use super::{AddCloudServer, AddServer, Connection, CreateSandbox, CreateTemplate, Need};
use crate::{
AppState,
auth::SessionUser,
@@ -33,6 +33,7 @@ pub fn routes() -> Router {
axum::routing::delete(close_port),
)
.route("/sandbox-servers", get(list_servers).post(add_server))
+ .route("/sandbox-servers/cloud", post(add_cloud_server))
.route(
"/sandbox-servers/{id}",
axum::routing::delete(remove_server),
@@ -328,7 +329,7 @@ pub(crate) async fn token(
#[utoipa::path(get, path = "/sandbox-servers", operation_id = "listSandboxServers", tag = "Sandboxes",
params(TeamQuery),
responses(
- (status = 200, description = "`servers`, each with `id`, `teamId` (null for the installation's), `name`, `address`, `state` (\"joining\", \"ready\", \"unreachable\" or \"refused\"), `enabled`, `capability` (`runtime`, `kvm`, `bootMs`, `arch`, `cpus`, `memory`, `disk` and the checks), `summary` (one line about it), `capacity`, `templates`, `daemonVersion`, `lastReportAt`, `error`, `sandboxes` (how many run there), `createdAt`, `edgeDomain` and, with an edge domain, `edge`: its `domain`, `https` (whether it serves HTTPS), `cert` (`state`: \"none\", \"issuing\", \"ready\" or \"failed\", `notAfter`, `issuedAt`, `error`, `served`), `cloudflare` (the connected zone that covers it: `teamId`, `zone`, `canEdit`, or null) and `dns` (`name`, the `records` to point it at the server, and what Pyo `made`).", body = Object),
+ (status = 200, description = "`servers`, each with `id`, `teamId` (null for the installation's), `name`, `address`, `state` (\"joining\", \"ready\", \"unreachable\" or \"refused\"), `enabled`, `runtime` (\"firecracker\" or \"gvisor\", what runs its sandboxes, null until it reports), `capability` (`runtime`, `kvm`, `bootMs`, `arch`, `cpus`, `memory`, `disk`, `gvisorVersion` and the checks), `summary` (one line about it), `capacity`, `templates`, `daemonVersion`, `lastReportAt`, `error`, `sandboxes` (how many run there), `createdAt`, `edgeDomain` and, with an edge domain, `edge`: its `domain`, `https` (whether it serves HTTPS), `cert` (`state`: \"none\", \"issuing\", \"ready\" or \"failed\", `notAfter`, `issuedAt`, `error`, `served`), `cloudflare` (the connected zone that covers it: `teamId`, `zone`, `canEdit`, or null) and `dns` (`name`, the `records` to point it at the server, and what Pyo `made`).", body = Object),
(status = 404, description = "No such team, or you aren't in it.", body = ErrorBody),
),
extensions(("x-pyo-access" = json!("read"))))]
@@ -367,23 +368,64 @@ pub(crate) async fn add_server(
Ok(Json(super::add_server(&s, &user, req).await?))
}
+/// Buy a sandbox server at a cloud provider
+///
+/// Makes a server in the team's Hetzner project and sets it up as a sandbox
+/// server, the way Pyo makes a cluster's servers there. Read the locations,
+/// server types and prices with getCloudOptions first. On its first boot it
+/// joins your tailnet and runs sandbox.sh with a one-time code. Hetzner
+/// Cloud servers have no KVM, so it runs sandboxes on gVisor. Follow it in
+/// the server's `cloud` (creating, booting, joining, ready) and `state`.
+/// Without `confirm` it answers 409 with the monthly price. Confirm that
+/// with the user, then send it again with `confirm: true`. Team owners and
+/// admins. Needs Tailscale and the team's Hetzner connection.
+#[utoipa::path(post, path = "/sandbox-servers/cloud", operation_id = "createCloudSandboxServer", tag = "Sandboxes",
+ request_body(content = AddCloudServer),
+ responses(
+ (status = 200, description = "`server` (as listSandboxServers shows it) and `cloudServer` (`id`, `name`, `state`, `monthly`, `currency`).", body = Object),
+ (status = 400, description = "A bad name, location or server type, or Tailscale or Hetzner isn't set up.", body = ErrorBody),
+ (status = 409, description = "`needsConfirm` with the monthly price, a name in use, a type Hetzner can't make now, or Pyo isn't on the tailnet yet.", body = ErrorBody),
+ (status = 502, description = "Hetzner failed.", body = ErrorBody),
+ ),
+ extensions(("x-pyo-access" = json!("full"))))]
+pub(crate) async fn add_cloud_server(
+ State(s): State,
+ SessionUser(user): SessionUser,
+ Json(req): Json,
+) -> ApiResult> {
+ Ok(Json(super::add_cloud_server(&s, &user, req).await?))
+}
+
+#[derive(Debug, Default, Deserialize, utoipa::IntoParams)]
+#[into_params(parameter_in = Query)]
+pub struct RemoveQuery {
+ /// Delete a server Pyo made at a cloud provider: without it the answer
+ /// is 409 with the monthly cost that stops.
+ #[serde(default)]
+ confirm: bool,
+}
+
/// Remove a sandbox server
///
/// Deletes its sandboxes where it still answers, and forgets it. To take the
-/// daemon off the server, run `sandbox.sh --uninstall` there.
+/// daemon off the server, run `sandbox.sh --uninstall` there. A server Pyo
+/// made at a cloud provider leaves your tailnet and is deleted there, which
+/// takes `confirm`.
#[utoipa::path(delete, path = "/sandbox-servers/{id}", operation_id = "removeSandboxServer", tag = "Sandboxes",
- params(("id" = String, Path, description = "The sandbox server's id, `sbs_…`.")),
+ params(("id" = String, Path, description = "The sandbox server's id, `sbs_…`."), RemoveQuery),
responses(
- (status = 200, description = "`ok`, and `stopped`: how many sandboxes ended.", body = Object),
+ (status = 200, description = "`ok`, `stopped` (how many sandboxes ended) and `deleting` (whether it's being deleted at the provider).", body = Object),
(status = 404, description = "No such sandbox server.", body = ErrorBody),
+ (status = 409, description = "`needsConfirm`: it's deleted at the provider, with the monthly cost that stops.", body = ErrorBody),
),
extensions(("x-pyo-access" = json!("full"))))]
pub(crate) async fn remove_server(
State(s): State,
SessionUser(user): SessionUser,
Path(id): Path,
+ Query(q): Query,
) -> ApiResult> {
- Ok(Json(super::remove_server(&s, &user, &id).await?))
+ Ok(Json(super::remove_server(&s, &user, &id, q.confirm).await?))
}
/// Get a sandbox server's certificate
@@ -468,8 +510,9 @@ pub(crate) async fn list_templates(
/// Create a sandbox template
///
/// A template made from an OCI image, like "node:22", or an image in Pyo's
-/// registry. Every sandbox server that may use it pulls the image, makes a
-/// root disk from it, boots it once and snapshots it. The template is
+/// registry. Every sandbox server that may use it pulls the image and makes
+/// a root disk from it. A Firecracker server boots it once and snapshots
+/// it, and a gVisor server starts sandboxes from the disk. The template is
/// ready once one server has it.
#[utoipa::path(post, path = "/sandbox-templates", operation_id = "createSandboxTemplate", tag = "Sandboxes",
request_body(content = CreateTemplate),
diff --git a/crates/pyo-api/src/sandbox/base.rs b/crates/pyo-api/src/sandbox/base.rs
new file mode 100644
index 0000000..70c39fc
--- /dev/null
+++ b/crates/pyo-api/src/sandbox/base.rs
@@ -0,0 +1,356 @@
+//! Pyo's built-in sandbox template `base`: Debian 13 with the common tools
+//! (deploy/images/sandbox-base.Dockerfile). It's the installation's, so
+//! every team has it, and a sandbox created without a template starts
+//! from it.
+//!
+//! - Seeding: when a worker starts, and in every sandbox round, Pyo makes
+//! sure `base` exists and points at its release's image
+//! (`PYO_SANDBOX_BASE_IMAGE`, pinned by digest in release.json). It's
+//! idempotent.
+//! - On update: a new release has a new image, so `base` gets a new
+//! revision (the digest's start). Servers make it beside the old one in
+//! the background, and [`super::templates`] moves new sandboxes over
+//! once it's ready and deletes the old one once no sandbox uses it.
+//! An older Pyo still running during the update never moves it back.
+//! - Dev builds have no image of their own: they use the latest release
+//! Pyo found when it checked for updates, or wait for one.
+//! - Nobody deletes it, and no other template may be named `base`.
+
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use pyo_proto::{parse_version, pinned_image};
+use serde_json::{Value, json};
+
+use crate::AppState;
+
+/// Its name, which sandboxes ask for.
+pub const NAME: &str = "base";
+/// Its id, the same on every installation.
+pub const ID: &str = "stp_base";
+/// What it runs, shown beside its image.
+pub const OS: &str = "Debian 13";
+
+/// The image to seed `base` from, and the release it belongs to.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Wanted {
+ pub image: String,
+ pub release: String,
+}
+
+/// This release's image, from `PYO_SANDBOX_BASE_IMAGE`. None for a build
+/// that isn't a release (scripts/pyo-up.sh leaves it empty).
+pub fn from_env() -> Option {
+ let image = std::env::var("PYO_SANDBOX_BASE_IMAGE").ok()?;
+ let image = image.trim();
+ pinned_image(image).then(|| Wanted {
+ image: image.to_owned(),
+ release: crate::updates::current().to_owned(),
+ })
+}
+
+/// The revision servers keep an image under: the first 12 hex digits of
+/// its digest.
+pub fn revision(image: &str) -> Option {
+ let (_, digest) = image.split_once("@sha256:")?;
+ (digest.len() >= 12).then(|| digest[..12].to_owned())
+}
+
+/// The template's source for an image.
+pub fn source(w: &Wanted) -> Value {
+ json!({ "kind": "image", "image": w.image, "os": OS, "release": w.release })
+}
+
+/// What seeding does, given the template's source now (None: there's no
+/// `base` yet).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Seed {
+ Insert,
+ /// A new image: a new revision.
+ Move,
+ Keep,
+}
+
+pub fn decide(have: Option<&Value>, want: &Wanted) -> Seed {
+ let Some(src) = have else {
+ return Seed::Insert;
+ };
+ if src["image"].as_str() == Some(want.image.as_str()) {
+ return Seed::Keep;
+ }
+ // A newer Pyo already moved it: this one is on its way out.
+ let theirs = src["release"].as_str().and_then(parse_version);
+ match (theirs, parse_version(&want.release)) {
+ (Some(a), Some(b)) if a > b => Seed::Keep,
+ _ => Seed::Move,
+ }
+}
+
+/// The image `base` should have now: this release's, else (a dev build)
+/// the latest release found by the update check.
+async fn wanted(s: &AppState) -> Option {
+ if let Some(w) = from_env() {
+ return Some(w);
+ }
+ let found = crate::updates::checked(&s.db).await.ok()?.found?;
+ let image = found.release.images.sandbox_base;
+ let pinned = pinned_image(&image);
+ pinned.then_some(Wanted {
+ image,
+ release: found.release.version,
+ })
+}
+
+/// Said once per process, so a dev build doesn't repeat it every round.
+static SAID_WAITING: AtomicBool = AtomicBool::new(false);
+
+/// Makes sure `base` exists and has the image it should. Tells the servers
+/// at once when that changed.
+pub async fn ensure(s: &AppState) -> anyhow::Result<()> {
+ let Some(want) = wanted(s).await else {
+ if !SAID_WAITING.swap(true, Ordering::Relaxed) {
+ tracing::info!(
+ "sandbox template base: this build has no image for it, and no release was found yet, so it's added later"
+ );
+ }
+ return Ok(());
+ };
+ seed_from(s, &want).await.map(|_| ())
+}
+
+/// Seeds `base` from `want`, and says what it did.
+pub async fn seed_from(s: &AppState, want: &Wanted) -> anyhow::Result {
+ let want = want.clone();
+ let rev = revision(&want.image);
+ let mut tx = s.db.begin().await?;
+ let have: Option =
+ sqlx::query_scalar("SELECT source FROM pyo.sandbox_templates WHERE id = $1 FOR UPDATE")
+ .bind(ID)
+ .fetch_optional(&mut *tx)
+ .await?;
+ let seed = decide(have.as_ref(), &want);
+ match seed {
+ Seed::Keep => return Ok(Seed::Keep),
+ Seed::Insert => {
+ let res = sqlx::query(
+ "INSERT INTO pyo.sandbox_templates (id, team_id, name, source, state, created_by, builtin, revision)
+ VALUES ($1, NULL, $2, $3, 'pending', 'Pyo', true, $4)
+ ON CONFLICT (id) DO NOTHING",
+ )
+ .bind(ID)
+ .bind(NAME)
+ .bind(source(&want))
+ .bind(&rev)
+ .execute(&mut *tx)
+ .await;
+ if let Err(e) = res {
+ if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("23505") {
+ tracing::warn!(
+ "sandbox template base: the installation has another template named base, so Pyo's isn't added"
+ );
+ return Ok(Seed::Keep);
+ }
+ return Err(e.into());
+ }
+ tracing::info!(image = %want.image, "sandbox template base added");
+ }
+ Seed::Move => {
+ sqlx::query(
+ "UPDATE pyo.sandbox_templates
+ SET source = $2, revision = $3, builtin = true, error = NULL, updated_at = now()
+ WHERE id = $1",
+ )
+ .bind(ID)
+ .bind(source(&want))
+ .bind(&rev)
+ .execute(&mut *tx)
+ .await?;
+ tracing::info!(image = %want.image, "sandbox template base moved to a new image");
+ }
+ }
+ tx.commit().await?;
+ tokio::spawn(super::templates::push(s.clone(), ID.to_owned()));
+ Ok(seed)
+}
+
+/// On a worker's start: [`ensure`], retried while the database is busy.
+pub async fn seed(s: AppState) {
+ for attempt in 1..=10u64 {
+ match ensure(&s).await {
+ Ok(()) => return,
+ Err(e) => {
+ tracing::warn!(attempt, error = %format!("{e:#}"), "sandbox template base: couldn't add it, retrying");
+ tokio::time::sleep(std::time::Duration::from_secs(15 * attempt)).await;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const A: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ const B: &str = "ghcr.io/upsetdev/pyo/sandbox-base:0.4.2@sha256:cccccccccccc0000000000000000000000000000000000000000000000000000";
+
+ fn want(image: &str, release: &str) -> Wanted {
+ Wanted {
+ image: image.into(),
+ release: release.into(),
+ }
+ }
+
+ #[test]
+ fn seeding_is_idempotent() {
+ let w = want(A, "0.4.1");
+ assert_eq!(decide(None, &w), Seed::Insert);
+ let src = source(&w);
+ assert_eq!(decide(Some(&src), &w), Seed::Keep, "the same image again");
+ assert_eq!(src["os"], "Debian 13");
+ assert_eq!(src["kind"], "image");
+ }
+
+ #[test]
+ fn a_new_release_moves_it_forward_only() {
+ let old = source(&want(A, "0.4.1"));
+ let new = want(B, "0.4.2");
+ assert_eq!(decide(Some(&old), &new), Seed::Move);
+ // The older Pyo, still running during the update, leaves it.
+ assert_eq!(decide(Some(&source(&new)), &want(A, "0.4.1")), Seed::Keep);
+ // A rebuilt image of the same release moves it too.
+ assert_eq!(decide(Some(&old), &want(B, "0.4.1")), Seed::Move);
+ // One without a release (made by hand) is taken over.
+ assert_eq!(
+ decide(
+ Some(&json!({ "kind": "image", "image": "debian:13" })),
+ &new
+ ),
+ Seed::Move
+ );
+ }
+
+ #[test]
+ fn revisions_come_from_the_digest() {
+ assert_eq!(revision(A).as_deref(), Some("aaaaaaaaaaaa"));
+ assert_eq!(revision(B).as_deref(), Some("cccccccccccc"));
+ assert_eq!(revision("debian:13"), None);
+ let name = super::super::templates::server_name(ID, revision(A).as_deref());
+ assert_eq!(name, "stp_base.aaaaaaaaaaaa");
+ assert!(pyo_proto::sandbox::is_template_name(&name));
+ }
+
+ #[test]
+ fn only_a_pinned_image_from_the_env_counts() {
+ // SAFETY: tests in this module don't read it concurrently.
+ unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", "") };
+ assert_eq!(from_env(), None, "a dev build (pyo-up.sh)");
+ unsafe {
+ std::env::set_var(
+ "PYO_SANDBOX_BASE_IMAGE",
+ "127.0.0.1:5005/pyo/sandbox-base:dev",
+ )
+ };
+ assert_eq!(from_env(), None, "not pinned");
+ unsafe { std::env::set_var("PYO_SANDBOX_BASE_IMAGE", A) };
+ assert_eq!(from_env(), Some(want(A, crate::updates::current())));
+ unsafe { std::env::remove_var("PYO_SANDBOX_BASE_IMAGE") };
+ assert_eq!(from_env(), None);
+ }
+
+ /// Against a real Postgres (DATABASE_URL, a throwaway one, in a database
+ /// of its own): seeding adds `base` once, moves it to a new release's
+ /// image, never back, and nobody deletes it or takes its name.
+ #[tokio::test]
+ #[ignore = "needs DATABASE_URL (a throwaway Postgres)"]
+ async fn seeds_base_in_the_database() {
+ use sqlx::PgPool;
+ use sqlx::postgres::PgConnectOptions;
+ let opts: PgConnectOptions = std::env::var("DATABASE_URL").unwrap().parse().unwrap();
+ let admin = PgPool::connect_with(opts.clone()).await.unwrap();
+ let name = format!("pyo_base_{}", std::process::id());
+ for q in [
+ format!("DROP DATABASE IF EXISTS {name}"),
+ format!("CREATE DATABASE {name}"),
+ ] {
+ sqlx::raw_sql(sqlx::AssertSqlSafe(q))
+ .execute(&admin)
+ .await
+ .unwrap();
+ }
+ let db = PgPool::connect_with(opts.database(&name)).await.unwrap();
+ crate::migrate::run(&db).await.unwrap();
+ let s = super::super::e2e::test_state(db.clone());
+ let row = || async {
+ sqlx::query_as::<_, (String, String, Value, Option, bool, Option)>(
+ "SELECT name, state, source, revision, builtin, team_id FROM pyo.sandbox_templates WHERE id = $1",
+ )
+ .bind(ID)
+ .fetch_one(&db)
+ .await
+ .unwrap()
+ };
+
+ let old = want(A, "0.4.1");
+ assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Insert);
+ assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep, "idempotent");
+ let (n, state, src, rev, builtin, team) = row().await;
+ assert_eq!(
+ (n.as_str(), state.as_str(), rev.as_deref(), builtin, team),
+ ("base", "pending", Some("aaaaaaaaaaaa"), true, None)
+ );
+ assert_eq!(src["image"], A);
+ let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pyo.sandbox_templates")
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert_eq!(count, 1);
+
+ // Every team sees it, as the installation's, at its current revision.
+ let views = super::super::template_views(&db, &["tm_any".into()], None)
+ .await
+ .unwrap();
+ assert_eq!(views[0]["name"], "base");
+ assert_eq!(views[0]["builtin"], true);
+ assert_eq!(views[0]["source"]["os"], "Debian 13");
+
+ // A new release moves it. The old Pyo, still running, doesn't move it back.
+ let new = want(B, "0.4.2");
+ assert_eq!(seed_from(&s, &new).await.unwrap(), Seed::Move);
+ assert_eq!(seed_from(&s, &old).await.unwrap(), Seed::Keep);
+ let (_, _, src, rev, ..) = row().await;
+ assert_eq!(
+ (src["image"].as_str(), rev.as_deref()),
+ (Some(B), Some("cccccccccccc"))
+ );
+
+ // Nobody deletes it, and no team takes its name.
+ let user = crate::auth::User {
+ id: "usr_base".into(),
+ name: "Admin".into(),
+ email: "admin@base.test".into(),
+ timezone: "UTC".into(),
+ is_admin: true,
+ must_set_up: false,
+ key: None,
+ };
+ let e = super::super::delete_template(&s, &user, ID)
+ .await
+ .unwrap_err();
+ assert_eq!(e.status, axum::http::StatusCode::CONFLICT, "{}", e.message);
+ let e = super::super::create_template(
+ &s,
+ &user,
+ serde_json::from_value(json!({ "name": "base", "image": "debian:13" })).unwrap(),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(e.status, axum::http::StatusCode::CONFLICT, "{}", e.message);
+
+ db.close().await;
+ sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
+ "DROP DATABASE {name} WITH (FORCE)"
+ )))
+ .execute(&admin)
+ .await
+ .unwrap();
+ }
+}
diff --git a/crates/pyo-api/src/sandbox/e2e.rs b/crates/pyo-api/src/sandbox/e2e.rs
index 65affe6..893f8fc 100644
--- a/crates/pyo-api/src/sandbox/e2e.rs
+++ b/crates/pyo-api/src/sandbox/e2e.rs
@@ -604,6 +604,101 @@ async fn the_control_plane_drives_a_real_daemon() {
tokio::time::sleep(Duration::from_millis(500)).await;
}
+ // ── The built-in template `base`: the default, and how a release moves it ──
+ // Two small public images stand in for two releases' sandbox-base.
+ use super::base;
+ sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1")
+ .bind(base::ID)
+ .execute(&db)
+ .await
+ .unwrap();
+ let old = base::Wanted {
+ image:
+ "alpine:3.21@sha256:bc21487101ae47172509130ee1092cc273a831968d64b617776d1f69c81e42ab"
+ .into(),
+ release: "0.0.1".into(),
+ };
+ let new = base::Wanted {
+ image:
+ "alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f"
+ .into(),
+ release: "0.0.2".into(),
+ };
+ let rev = |w: &base::Wanted| {
+ super::templates::server_name(base::ID, base::revision(&w.image).as_deref())
+ };
+ let (old_name, new_name) = (rev(&old), rev(&new));
+ assert_eq!(base::seed_from(&s, &old).await.unwrap(), base::Seed::Insert);
+ wait_for(&s, &team, &server, &old_name, false).await;
+ // No template: base.
+ let no_template: super::CreateSandbox =
+ serde_json::from_value(json!({ "teamId": team, "network": "none", "ttl": "5m" })).unwrap();
+ let first = super::create(&s, &user, no_template.clone()).await.unwrap();
+ assert_eq!(first["sandbox"]["template"], "base", "{first}");
+ let first_id = first["sandbox"]["id"].as_str().unwrap().to_owned();
+ let on: String =
+ sqlx::query_scalar("SELECT spec->>'template' FROM pyo.sandboxes WHERE id = $1")
+ .bind(&first_id)
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert_eq!(on, old_name);
+ let r = Daemon::new(&address)
+ .exec(
+ first["connection"]["token"].as_str().unwrap(),
+ &first_id,
+ &exec_req("cat /etc/alpine-release"),
+ )
+ .await
+ .unwrap();
+ assert!(r.stdout.starts_with("3.21"), "{r:?}");
+
+ // A new release: made beside the old one, new sandboxes move over.
+ assert_eq!(base::seed_from(&s, &new).await.unwrap(), base::Seed::Move);
+ assert_eq!(
+ base::seed_from(&s, &old).await.unwrap(),
+ base::Seed::Keep,
+ "the old Pyo doesn't move it back"
+ );
+ wait_for(&s, &team, &server, &new_name, false).await;
+ let second = super::create(&s, &user, no_template).await.unwrap();
+ let second_id = second["sandbox"]["id"].as_str().unwrap().to_owned();
+ let r = Daemon::new(&address)
+ .exec(
+ second["connection"]["token"].as_str().unwrap(),
+ &second_id,
+ &exec_req("cat /etc/alpine-release"),
+ )
+ .await
+ .unwrap();
+ assert!(r.stdout.starts_with("3.22"), "{r:?}");
+ // The running sandbox keeps the old disk, which goes once it ends.
+ super::runner::round(&s).await.unwrap();
+ assert!(ready_on(&db, &team, &server).await.contains(&old_name));
+ let r = Daemon::new(&address)
+ .exec(
+ first["connection"]["token"].as_str().unwrap(),
+ &first_id,
+ &exec_req("cat /etc/alpine-release"),
+ )
+ .await
+ .unwrap();
+ assert!(r.stdout.starts_with("3.21"), "{r:?}");
+ super::delete(&s, &user, &first_id).await.unwrap();
+ wait_for(&s, &team, &server, &old_name, true).await;
+ let views = super::template_views(&db, std::slice::from_ref(&team), Some(base::ID))
+ .await
+ .unwrap();
+ assert_eq!(views[0]["state"], "ready", "{}", views[0]);
+ println!("e2e base: {}", views[0]["servers"]);
+ super::delete(&s, &user, &second_id).await.unwrap();
+ sqlx::query("DELETE FROM pyo.sandbox_templates WHERE id = $1")
+ .bind(base::ID)
+ .execute(&db)
+ .await
+ .unwrap();
+ super::templates::forget(s.clone(), base::ID.into()).await;
+
sqlx::query("DELETE FROM pyo.teams WHERE id = $1")
.bind(&team)
.execute(&db)
@@ -617,7 +712,38 @@ async fn the_control_plane_drives_a_real_daemon() {
keys::reset_cache();
}
-fn test_state(db: PgPool) -> crate::AppState {
+/// The templates a server reports ready, by the names it keeps them under.
+async fn ready_on(db: &PgPool, team: &str, server: &str) -> Vec {
+ let srv = &super::server_views(db, &[team.to_owned()], Some(server))
+ .await
+ .unwrap()[0];
+ srv["templates"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter(|t| t["state"] == "ready" || t["state"].is_null())
+ .filter_map(|t| t["name"].as_str().map(str::to_owned))
+ .collect()
+}
+
+/// Runs the runner until the server has `name` ready, or (`gone`) no longer has it.
+async fn wait_for(s: &crate::AppState, team: &str, server: &str, name: &str, gone: bool) {
+ let t0 = std::time::Instant::now();
+ loop {
+ super::runner::round(s).await.unwrap();
+ let names = ready_on(&s.db, team, server).await;
+ if names.iter().any(|n| n == name) != gone {
+ return;
+ }
+ assert!(
+ t0.elapsed() < Duration::from_secs(300),
+ "{name} gone={gone}: {names:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ }
+}
+
+pub(super) fn test_state(db: PgPool) -> crate::AppState {
let config = crate::Config {
agent_api_url: String::new(),
agent_image: String::new(),
diff --git a/crates/pyo-api/src/sandbox/join.rs b/crates/pyo-api/src/sandbox/join.rs
index ec37b8b..95aeb2e 100644
--- a/crates/pyo-api/src/sandbox/join.rs
+++ b/crates/pyo-api/src/sandbox/join.rs
@@ -221,6 +221,8 @@ pub fn daemon_config(
"jailer": format!("{LIB_DIR}/jailer"),
"kernel": format!("{LIB_DIR}/vmlinux"),
"envd": format!("{LIB_DIR}/pyo-envd"),
+ // sandbox.sh puts gVisor's runsc there on servers without KVM.
+ "runsc": format!("{LIB_DIR}/runsc"),
"keys": keys,
"jailUid": jail_uid,
"jailGid": jail_gid,
@@ -382,7 +384,8 @@ fn q(s: &str) -> String {
/// The command that makes a machine a sandbox server.
pub struct Command<'a> {
- /// Where scripts come from ([`crate::join::scripts_url`]).
+ /// Where scripts come from, off the tailnet the server isn't on yet
+ /// ([`crate::join::scripts_url_before_tailnet`]).
pub base: &'a str,
/// Pyo's tailnet door, "pyo.tail1234.ts.net:5006".
pub door: &'a str,
@@ -468,6 +471,7 @@ mod tests {
assert_eq!(v["listen"], "100.64.0.7:7070");
assert_eq!(v["keys"][0]["publicKey"], "AAAA");
assert_eq!(v["jailUid"], 998);
+ assert_eq!(v["runsc"], "/usr/local/lib/pyo-sandbox/runsc");
assert_eq!(
daemon_config("s", "fd7a:115c:a1e0::9", 1, 1, &keys)["listen"],
"[fd7a:115c:a1e0::9]:7070"
@@ -476,8 +480,10 @@ mod tests {
#[test]
fn the_command_quotes_everything() {
+ // The server isn't on the tailnet yet: pyo.dev.
+ let base = crate::join::pick_scripts_url_before_tailnet(None);
let c = Command {
- base: "https://pyo.tail1234.ts.net/install",
+ base: &base,
door: "pyo.tail1234.ts.net:5006",
server_id: "sbs_one",
name: "it's sbx",
@@ -488,7 +494,7 @@ mod tests {
};
assert_eq!(
c.render(),
- "curl -fsSL https://pyo.tail1234.ts.net/install/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'"
+ "curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'"
);
let c = Command {
edge_domain: Some("sbx.example.com"),
@@ -502,6 +508,38 @@ mod tests {
);
}
+ #[test]
+ fn a_new_server_fetches_sandbox_sh_off_the_tailnet() {
+ let render = |base: &str| {
+ Command {
+ base,
+ door: "pyo.tail1234.ts.net:5006",
+ server_id: "sbs_one",
+ name: "sbx",
+ code: "abc",
+ tailscale_key: "tskey-auth-1",
+ version: "0.4.0",
+ edge_domain: None,
+ }
+ .render()
+ };
+ let pick = crate::join::pick_scripts_url_before_tailnet;
+ // PYO_SCRIPTS_URL wins.
+ assert!(
+ render(&pick(Some("https://mirror.example.com/")))
+ .starts_with("curl -fsSL https://mirror.example.com/sandbox.sh | bash -s -- ")
+ );
+ // Else pyo.dev, which runs the sandbox.sh of the --version release.
+ for unset in [None, Some(" ")] {
+ let c = render(&pick(unset));
+ assert!(
+ c.starts_with("curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- ")
+ && c.contains(" --version '0.4.0'"),
+ "{c}"
+ );
+ }
+ }
+
#[test]
fn refusals_are_plain() {
for r in [
diff --git a/crates/pyo-api/src/sandbox/mcp.rs b/crates/pyo-api/src/sandbox/mcp.rs
index f164d5e..95c7390 100644
--- a/crates/pyo-api/src/sandbox/mcp.rs
+++ b/crates/pyo-api/src/sandbox/mcp.rs
@@ -39,13 +39,13 @@ pub fn defs() -> Vec {
vec![
json!({ "name": "create_sandbox", "title": "Create sandbox", "description": "Create an isolated machine (a sandbox) on the team's sandbox servers, to run code, commands and files in. It ends after its lifetime (5 minutes when omitted). Returns its id.", "inputSchema": schema(json!({
"team": team,
- "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them)" },
+ "template": { "type": "string", "description": "A sandbox template name, like node-22 (list_sandbox_templates shows them). base when omitted: Debian 13 with git, Python, uv, Node.js and pnpm" },
"cpus": { "type": "integer", "minimum": 1, "maximum": 32 },
"memory_mib": { "type": "integer", "minimum": 128 },
"ttl": { "type": "string", "description": "How long it lives, like 30m or 1h (24h at most)" },
"env": { "type": "object", "additionalProperties": { "type": "string" } },
"network": { "type": "string", "enum": ["internet", "none"] },
- }), &["template"]), "annotations": write }),
+ }), &[]), "annotations": write }),
json!({ "name": "run_code", "title": "Run code", "description": "Run a piece of code in a sandbox and wait for it: its output and exit code.", "inputSchema": schema(json!({
"sandbox_id": sandbox,
"code": { "type": "string" },
@@ -149,8 +149,9 @@ pub async fn run(s: &AppState, user: &User, name: &str, args: &Value) -> Result<
match name {
"create_sandbox" => {
let team = team_id(s, user, args).await?;
- let mut req = json!({ "teamId": team, "template": arg(args, "template")? });
+ let mut req = json!({ "teamId": team });
for (from, to) in [
+ ("template", "template"),
("cpus", "cpus"),
("memory_mib", "memoryMib"),
("ttl", "ttl"),
diff --git a/crates/pyo-api/src/sandbox/mod.rs b/crates/pyo-api/src/sandbox/mod.rs
index 1011d8e..49b0f90 100644
--- a/crates/pyo-api/src/sandbox/mod.rs
+++ b/crates/pyo-api/src/sandbox/mod.rs
@@ -5,8 +5,11 @@
//!
//! - [`keys`]: the installation's signing key and the tokens.
//! - [`join`]: a sandbox server's one-time join code and sandbox.sh.
+//! [`add_cloud_server`] makes one at a cloud provider instead, which runs
+//! sandbox.sh from its first boot (cloud/bootstrap.rs).
//! - [`scheduler`]: which server a new sandbox runs on.
//! - [`templates`]: which servers make which templates, and their state.
+//! [`base`]: Pyo's built-in template, kept in step with the release.
//! - [`limits`]: per-team caps, checked on create.
//! - [`daemon`]: Pyo's calls to a daemon over the tailnet.
//! - [`edgecert`]: the edge proxies' wildcard certificates and DNS records.
@@ -20,6 +23,7 @@
//! sandbox server exists ([`available`]).
pub mod api;
+pub mod base;
pub mod daemon;
#[cfg(test)]
mod e2e;
@@ -325,7 +329,13 @@ pub async fn server_views(
'daemonVersion', s.daemon_version, 'lastReportAt', pyo.ms(s.last_report_at),
'error', s.error, 'createdBy', s.created_by, 'createdAt', pyo.ms(s.created_at),
'joinedAt', pyo.ms(s.joined_at), 'edgeDomain', s.edge_domain,
- 'sandboxes', (SELECT count(*) FROM pyo.sandboxes x WHERE x.server_id = s.id AND x.state NOT IN ('stopped', 'failed')))
+ 'sandboxes', (SELECT count(*) FROM pyo.sandboxes x WHERE x.server_id = s.id AND x.state NOT IN ('stopped', 'failed')),
+ 'cloud', (SELECT jsonb_build_object('id', c.id, 'provider', c.provider, 'location', c.location,
+ 'serverType', c.server_type, 'monthly', c.monthly, 'currency', c.currency, 'ipv4', c.ipv4,
+ 'state', c.state, 'error', c.error, 'signCommand', c.sign_command,
+ 'createdAt', pyo.ms(c.created_at), 'deadlineAt', pyo.ms(c.created_at + interval '15 minutes'))
+ FROM pyo.cloud_servers c WHERE c.sandbox_server_id = s.id AND c.state <> 'deleted'
+ ORDER BY c.created_at DESC LIMIT 1))
FROM pyo.sandbox_servers s
WHERE (s.team_id IS NULL OR s.team_id = ANY($1)) AND ($2::text IS NULL OR s.id = $2)
ORDER BY s.team_id NULLS FIRST, s.name",
@@ -335,10 +345,11 @@ pub async fn server_views(
.fetch_all(db)
.await?;
for r in &mut rows {
- let summary = serde_json::from_value::(r["capability"].clone())
- .ok()
- .map(|c| c.summary());
- r["summary"] = json!(summary);
+ let cap = serde_json::from_value::(r["capability"].clone()).ok();
+ r["summary"] = json!(cap.as_ref().map(|c| c.summary()));
+ // What runs its sandboxes: "firecracker" or "gvisor", null until it
+ // reports (or when it can run neither).
+ r["runtime"] = json!(cap.and_then(|c| runtime_name(c.runtime)));
}
edgecert::attach(db, team_ids, &mut rows).await?;
Ok(rows)
@@ -352,6 +363,7 @@ pub async fn template_views(
sqlx::query_scalar(
"SELECT jsonb_build_object(
'id', t.id, 'teamId', t.team_id, 'name', t.name, 'source', t.source, 'state', t.state,
+ 'builtin', t.builtin, 'revision', t.revision,
'snapshots', t.snapshots, 'error', t.error, 'createdBy', t.created_by,
'createdAt', pyo.ms(t.created_at), 'updatedAt', pyo.ms(t.updated_at),
'servers', (SELECT coalesce(jsonb_agg(jsonb_build_object(
@@ -364,7 +376,7 @@ pub async fn template_views(
'bytes', m.e->'bytes', 'snapshots', coalesce(m.e->'snapshots', '[]')) ORDER BY s.name), '[]')
FROM pyo.sandbox_servers s
LEFT JOIN LATERAL (SELECT x AS e FROM jsonb_array_elements(s.templates) x
- WHERE x->>'name' = t.id LIMIT 1) m ON true
+ WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') LIMIT 1) m ON true
WHERE (s.team_id IS NULL OR s.team_id = t.team_id OR t.team_id IS NULL)
AND (s.team_id IS NULL OR s.team_id = ANY($1))
AND (m.e IS NOT NULL OR s.state NOT IN ('joining', 'refused'))))
@@ -390,8 +402,10 @@ pub struct CreateSandbox {
#[serde(default)]
pub team_id: Option,
/// A sandbox template's name (or id): the team's own, else the
- /// installation's.
- pub template: String,
+ /// installation's. "base" (Pyo's built-in Debian 13 template) when
+ /// omitted.
+ #[serde(default)]
+ pub template: Option,
/// 1 when omitted.
#[serde(default)]
pub cpus: Option,
@@ -447,14 +461,30 @@ pub fn parse_ttl_value(v: Option<&Value>) -> ApiResult {
Ok(secs)
}
+/// The template a create asks for: the one named, else [`base::NAME`].
+pub fn template_asked(template: Option<&str>) -> &str {
+ template
+ .map(str::trim)
+ .filter(|t| !t.is_empty())
+ .unwrap_or(base::NAME)
+}
+
+/// A template as a create finds it.
+struct AskedTemplate {
+ id: String,
+ name: String,
+ /// The name its servers keep it under (see [`templates::server_name`]).
+ server_name: String,
+}
+
/// The template a name stands for: the team's, else the installation's.
async fn template_for(
tx: &mut PgConnection,
team_id: &str,
name: &str,
-) -> ApiResult<(String, String, String)> {
- let row: Option<(String, String, String)> = sqlx::query_as(
- "SELECT id, name, state FROM pyo.sandbox_templates
+) -> ApiResult {
+ let row: Option<(String, String, String, Option)> = sqlx::query_as(
+ "SELECT id, name, state, revision FROM pyo.sandbox_templates
WHERE (team_id = $1 OR team_id IS NULL) AND (name = $2 OR id = $2)
ORDER BY team_id NULLS LAST LIMIT 1",
)
@@ -462,13 +492,25 @@ async fn template_for(
.bind(name)
.fetch_optional(tx)
.await?;
- let (id, name, state) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?;
+ let Some((id, name, state, revision)) = row else {
+ return Err(if name == base::NAME {
+ ApiError::conflict(
+ "The base sandbox template isn't ready yet. Try again soon, or pass a template.",
+ )
+ } else {
+ ApiError::not_found("Sandbox template")
+ });
+ };
if state == "failed" {
return Err(ApiError::conflict(format!(
"The sandbox template {name} failed to build. Make it again."
)));
}
- Ok((id, name, state))
+ Ok(AskedTemplate {
+ server_name: templates::server_name(&id, revision.as_deref()),
+ id,
+ name,
+ })
}
/// The servers a team's sandbox can go to, with their addresses.
@@ -595,8 +637,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult<
.bind(&team_id)
.execute(&mut *tx)
.await?;
- let (template_id, template_name, _) =
- template_for(&mut tx, &team_id, req.template.trim()).await?;
+ let template = template_for(&mut tx, &team_id, template_asked(req.template.as_deref())).await?;
if let Some(p) = &req.project_id {
let ok: bool = sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM pyo.projects WHERE id = $1 AND team_id = $2)",
@@ -609,7 +650,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult<
return Err(ApiError::not_found("Project"));
}
}
- let mut spec = SandboxSpec::new(new_sandbox_id(), &team_id, &template_id);
+ let mut spec = SandboxSpec::new(new_sandbox_id(), &team_id, &template.server_name);
spec.cpus = req.cpus.unwrap_or(proto::DEFAULT_CPUS);
spec.memory_mib = req.memory_mib.unwrap_or(proto::DEFAULT_MEMORY_MIB);
spec.disk_mib = req.disk_mib.unwrap_or(proto::DEFAULT_DISK_MIB);
@@ -625,15 +666,18 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult<
limits::check(&lim, &used, &spec).map_err(ApiError::forbidden)?;
let servers = candidates(&mut tx, &team_id).await?;
let list: Vec = servers.iter().map(|(c, _)| c.clone()).collect();
- let chosen = scheduler::pick(&list, &spec).map_err(|n| {
- let n = match n {
- scheduler::NoServer::NoTemplate(_) => {
- scheduler::NoServer::NoTemplate(template_name.clone())
- }
- n => n,
- };
- ApiError::new(n.status(), n.message())
- })?;
+ let (chosen, server_name) =
+ scheduler::pick_revision(&list, &spec, &template.id).map_err(|n| {
+ let n = match n {
+ scheduler::NoServer::NoTemplate(_) => {
+ scheduler::NoServer::NoTemplate(template.name.clone())
+ }
+ n => n,
+ };
+ ApiError::new(n.status(), n.message())
+ })?;
+ // The server may still start it from an older revision of the template.
+ spec.template = server_name;
let address = servers
.iter()
.find(|(c, _)| c.id == chosen.id)
@@ -653,7 +697,7 @@ pub async fn create(s: &AppState, user: &User, req: CreateSandbox) -> ApiResult<
.bind(&team_id)
.bind(&req.project_id)
.bind(&server_id)
- .bind(&template_id)
+ .bind(&template.id)
.bind(json!(kept))
.bind(&user.name)
.bind(ttl_secs as f64)
@@ -998,8 +1042,7 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult<
}
crate::cloud::needs_tailscale(&s.db).await?;
let door = crate::deploy::registry_push();
- let api = crate::web::tailnet_api_url();
- let (false, Some(api)) = (door.is_empty(), api) else {
+ let (false, Some(_)) = (door.is_empty(), crate::web::tailnet_api_url()) else {
return Err(ApiError::conflict(
"Pyo's server isn't on your tailnet yet. Try again in a minute.",
));
@@ -1046,8 +1089,12 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult<
};
let code = join::new_code();
join::keep(&mut tx, &id, &code).await?;
- let ts_key =
- crate::tailscale::join_key(s, &format!("Pyo: sandbox server {}", req.name)).await?;
+ let ts_key = crate::tailscale::join_key(
+ s,
+ &format!("Pyo: sandbox server {}", req.name),
+ "a sandbox server",
+ )
+ .await?;
if let Some(t) = &req.team_id {
crate::ops::log_activity(
&mut tx,
@@ -1062,7 +1109,9 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult<
.await?;
}
tx.commit().await?;
- let base = crate::join::scripts_url(Some(&crate::join::friendly(&api)));
+ // The server joins the tailnet in sandbox.sh, so it fetches the script
+ // off the tailnet.
+ let base = crate::join::scripts_url_before_tailnet();
let door = crate::join::friendly(&door);
let command = join::Command {
base: &base,
@@ -1087,10 +1136,36 @@ pub async fn add_server(s: &AppState, user: &User, req: AddServer) -> ApiResult<
}))
}
+/// Ends a sandbox server's sandboxes in Pyo's records and forgets the
+/// server. Its daemon isn't told: see [`remove_server`]. The number of
+/// sandboxes that ended.
+pub async fn forget_server(
+ tx: &mut Transaction<'_, Postgres>,
+ id: &str,
+ why: &str,
+) -> sqlx::Result {
+ let live: Vec = sqlx::query_scalar(
+ "SELECT id FROM pyo.sandboxes WHERE server_id = $1 AND state NOT IN ('stopped', 'failed')",
+ )
+ .bind(id)
+ .fetch_all(&mut **tx)
+ .await?;
+ for x in &live {
+ end(tx, x, "stopped", Some(why)).await?;
+ }
+ sqlx::query("DELETE FROM pyo.sandbox_servers WHERE id = $1")
+ .bind(id)
+ .execute(&mut **tx)
+ .await?;
+ Ok(live.len())
+}
+
/// Removes a sandbox server: its sandboxes are deleted where it still
/// answers, and Pyo forgets it. The daemon stays installed on the server
-/// until `sandbox.sh --uninstall` runs there.
-pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult {
+/// until `sandbox.sh --uninstall` runs there. One Pyo made at a cloud
+/// provider is deleted there too, which takes `confirm` (409 with the
+/// monthly cost that stops until then).
+pub async fn remove_server(s: &AppState, user: &User, id: &str, confirm: bool) -> ApiResult {
let row: Option<(Option, Option, String)> =
sqlx::query_as("SELECT team_id, address, name FROM pyo.sandbox_servers WHERE id = $1")
.bind(id)
@@ -1102,6 +1177,21 @@ pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult require(&mut tx, user, t, Need::Manage).await?,
None => require_install_admin(user)?,
}
+ let cloud: Option<(String, f64, String)> = sqlx::query_as(
+ "SELECT id, monthly, currency FROM pyo.cloud_servers
+ WHERE sandbox_server_id = $1 AND state NOT IN ('deleted', 'removing') ORDER BY created_at DESC LIMIT 1",
+ )
+ .bind(id)
+ .fetch_optional(&mut *tx)
+ .await?;
+ if let Some((_, monthly, currency)) = &cloud
+ && !confirm
+ {
+ return Err(ApiError::needs_confirm(format!(
+ "Deletes {name} at Hetzner. {} a month stops.",
+ crate::ops::money(currency, *monthly)
+ )));
+ }
let live: Vec = sqlx::query_scalar(
"SELECT id FROM pyo.sandboxes WHERE server_id = $1 AND state NOT IN ('stopped', 'failed')",
)
@@ -1123,34 +1213,197 @@ pub async fn remove_server(s: &AppState, user: &User, id: &str) -> ApiResult,
+ /// A location's `name` from getCloudOptions ("fsn1").
+ pub location: String,
+ /// A server type's `name` from getCloudOptions ("cx22").
+ pub server_type: String,
+ /// A domain whose wildcard points at the server, as for
+ /// `addSandboxServer`.
+ #[serde(default)]
+ pub edge_domain: Option,
+ /// Your OpenSSH public key, to sign in to it over your tailnet.
+ #[serde(default)]
+ pub ssh_key: Option,
+ /// Without it the answer is 409 with the monthly price.
+ #[serde(default)]
+ pub confirm: bool,
+}
+
+/// Makes a sandbox server at a cloud provider: its row (joining), and the
+/// server, which joins the tailnet on its first boot and runs sandbox.sh
+/// with a one-time code. Cloud servers have no KVM, so it runs gVisor.
+pub async fn add_cloud_server(s: &AppState, user: &User, req: AddCloudServer) -> ApiResult {
+ crate::util::require_dns_label(&req.name, "Server name")?;
+ let edge = edge_domain(req.edge_domain.as_deref())?;
+ if !matches!(req.provider.as_deref(), None | Some("hetzner")) {
+ return Err(ApiError::bad_request("The provider is \"hetzner\"."));
+ }
+ let ssh_key = match req
+ .ssh_key
+ .as_deref()
+ .map(str::trim)
+ .filter(|k| !k.is_empty())
+ {
+ Some(k) if crate::cloud::valid_ssh_key(k) => Some(k.to_owned()),
+ Some(_) => {
+ return Err(ApiError::bad_request(
+ "Paste an OpenSSH public key on one line, like the contents of ~/.ssh/id_ed25519.pub.",
+ ));
+ }
+ None => None,
+ };
+ let mut tx = s.db.begin().await?;
+ require(&mut tx, user, &req.team_id, Need::Manage).await?;
+ crate::cloud::needs_tailscale(&s.db).await?;
+ let door = crate::deploy::registry_push();
+ let api = crate::web::tailnet_api_url();
+ let (false, Some(api)) = (door.is_empty(), api) else {
+ return Err(ApiError::conflict(
+ "Pyo's server isn't on your tailnet yet. Try again in a minute.",
+ ));
+ };
+ if !s.sealer.enabled() {
+ return Err(ApiError::new(
+ StatusCode::SERVICE_UNAVAILABLE,
+ "This installation has no PYO_SECRET_KEY, so it can't sign sandbox tokens.",
+ ));
+ }
+ let taken: bool = sqlx::query_scalar(
+ "SELECT EXISTS (SELECT 1 FROM pyo.sandbox_servers WHERE team_id = $1 AND name = $2)",
+ )
+ .bind(&req.team_id)
+ .bind(&req.name)
+ .fetch_one(&mut *tx)
+ .await?;
+ if taken {
+ return Err(ApiError::conflict(
+ "A sandbox server has that name already.",
+ ));
+ }
+ let p = crate::cloud::provider(&s.db, &req.team_id).await?;
+ let catalog = crate::cloud::catalog(&s.db, &req.team_id).await?;
+ let monthly = catalog.pick(&req.location, &req.server_type)?;
+ let price = crate::ops::money(&catalog.currency, monthly);
+ if !req.confirm {
+ return Err(ApiError::needs_confirm(format!(
+ "{} costs {price} a month at Hetzner ({} in {}, before VAT).",
+ req.name, req.server_type, req.location
+ )));
+ }
+ let id = crate::ids::new_id("sbs");
+ sqlx::query("INSERT INTO pyo.sandbox_servers (id, team_id, name, state, created_by, edge_domain) VALUES ($1, $2, $3, 'joining', $4, $5)")
+ .bind(&id)
+ .bind(&req.team_id)
+ .bind(&req.name)
+ .bind(&user.name)
+ .bind(&edge)
+ .execute(&mut *tx)
+ .await?;
+ let ts_key = crate::tailscale::join_key(
+ s,
+ &format!("Pyo: sandbox server {}", req.name),
+ "a sandbox server",
+ )
+ .await?;
+ // It's on the tailnet before it asks. Its join code comes with the
+ // details, made when it asks (cloud/bootstrap.rs).
+ let base = crate::join::scripts_url(Some(&crate::join::friendly(&api)));
+ let door = crate::join::friendly(&door);
+ let mut args: Vec<(&str, Option)> = vec![
+ ("--pyo", Some(door)),
+ ("--server", Some(id.clone())),
+ ("--name", Some(req.name.clone())),
+ ("--version", Some(env!("CARGO_PKG_VERSION").into())),
+ ];
+ if let Some(d) = &edge {
+ args.push(("--edge-domain", Some(d.clone())));
+ }
+ let material = crate::cloud::bootstrap::Material::new(format!("{base}/sandbox.sh"), &args);
+ let cloud_id = crate::ids::new_id("cs");
+ let made = crate::ops::make_cloud_server(
+ s,
+ &p,
+ crate::ops::CloudNew {
+ id: &cloud_id,
+ team_id: &req.team_id,
+ owner: crate::cloud::Owner::Sandbox(&id),
+ name: &req.name,
+ location: &req.location,
+ server_type: &req.server_type,
+ monthly,
+ currency: &catalog.currency,
+ first: false,
+ builder: false,
+ tailscale_key: &ts_key,
+ material,
+ ssh_key: ssh_key.as_deref(),
+ by: &user.name,
+ },
+ &mut tx,
+ )
+ .await?;
+ crate::ops::log_activity(
+ &mut tx,
+ &req.team_id,
+ &user.name,
+ "sandbox.server",
+ "info",
+ &format!(
+ "Creating the sandbox server {} at Hetzner ({}, {price} a month)",
+ req.name, req.server_type
+ ),
+ json!({ "kind": "sandboxServer", "id": id, "label": req.name }),
+ json!({ "cloudServerId": cloud_id }),
+ )
+ .await?;
+ tx.commit().await?;
+ let server = server_views(&s.db, std::slice::from_ref(&req.team_id), Some(&id))
+ .await?
+ .into_iter()
+ .next();
+ Ok(json!({ "server": server, "cloudServer": made }))
}
#[derive(Debug, Clone, Deserialize, utoipa::ToSchema)]
@@ -1176,6 +1429,11 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) ->
"A template name is lowercase letters, digits, dashes, dots and underscores, like node-22.",
));
}
+ if req.name == base::NAME {
+ return Err(ApiError::conflict(
+ "base is Pyo's built-in sandbox template. Pick another name.",
+ ));
+ }
let image = req.image.trim();
if image.is_empty() || image.len() > 512 || image.contains(char::is_whitespace) {
return Err(ApiError::bad_request("Give the image, like node:22."));
@@ -1213,12 +1471,17 @@ pub async fn create_template(s: &AppState, user: &User, req: CreateTemplate) ->
/// Deletes a sandbox template, and has the servers delete their copies.
/// Refused while sandboxes run from it.
pub async fn delete_template(s: &AppState, user: &User, id: &str) -> ApiResult {
- let row: Option<(Option, String)> =
- sqlx::query_as("SELECT team_id, name FROM pyo.sandbox_templates WHERE id = $1")
+ let row: Option<(Option, String, bool)> =
+ sqlx::query_as("SELECT team_id, name, builtin FROM pyo.sandbox_templates WHERE id = $1")
.bind(id)
.fetch_optional(&s.db)
.await?;
- let (team_id, name) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?;
+ let (team_id, name, builtin) = row.ok_or_else(|| ApiError::not_found("Sandbox template"))?;
+ if builtin {
+ return Err(ApiError::conflict(format!(
+ "{name} is Pyo's built-in sandbox template, so it can't be deleted."
+ )));
+ }
let mut tx = s.db.begin().await?;
match &team_id {
Some(t) => require(&mut tx, user, t, Need::Use).await?,
@@ -1352,4 +1615,13 @@ mod tests {
.is_err()
);
}
+
+ #[test]
+ fn create_without_a_template_uses_base() {
+ let r: CreateSandbox = serde_json::from_value(json!({})).unwrap();
+ assert_eq!(r.template, None);
+ assert_eq!(template_asked(r.template.as_deref()), "base");
+ assert_eq!(template_asked(Some(" ")), "base");
+ assert_eq!(template_asked(Some(" node-22 ")), "node-22");
+ }
}
diff --git a/crates/pyo-api/src/sandbox/runner.rs b/crates/pyo-api/src/sandbox/runner.rs
index 66dddb4..fe6538f 100644
--- a/crates/pyo-api/src/sandbox/runner.rs
+++ b/crates/pyo-api/src/sandbox/runner.rs
@@ -67,6 +67,10 @@ struct Server {
/// One round: reports, then lifetimes.
pub async fn round(s: &AppState) -> anyhow::Result<()> {
let _ = super::join::prune(&s.db).await;
+ // A dev build's `base` comes once an update check finds a release.
+ if let Err(e) = super::base::ensure(s).await {
+ tracing::warn!(error = %format!("{e:#}"), "sandboxes: couldn't keep the base template");
+ }
let servers: Vec = sqlx::query_as(
"SELECT id, team_id, address, state FROM pyo.sandbox_servers WHERE address IS NOT NULL AND state <> 'refused'",
)
diff --git a/crates/pyo-api/src/sandbox/scheduler.rs b/crates/pyo-api/src/sandbox/scheduler.rs
index 78417c5..605648b 100644
--- a/crates/pyo-api/src/sandbox/scheduler.rs
+++ b/crates/pyo-api/src/sandbox/scheduler.rs
@@ -155,6 +155,42 @@ pub fn pick<'a>(servers: &'a [Candidate], spec: &SandboxSpec) -> Result<&'a Cand
.ok_or(NoServer::Full)
}
+/// [`pick`] for a template that may have revisions (template `id`, kept
+/// as `spec.template` now): a server with the current revision ready,
+/// else one that still has an older one ready, while the new one is made.
+/// Returns the server and the name of the revision it starts from.
+pub fn pick_revision<'a>(
+ servers: &'a [Candidate],
+ spec: &SandboxSpec,
+ id: &str,
+) -> Result<(&'a Candidate, String), NoServer> {
+ let first = match pick(servers, spec) {
+ Ok(c) => return Ok((c, spec.template.clone())),
+ Err(e) => e,
+ };
+ if !matches!(first, NoServer::NoTemplate(_)) {
+ return Err(first);
+ }
+ let mut older: Vec<&str> = servers
+ .iter()
+ .flat_map(|c| super::templates::older_ready(&c.templates, id, &spec.template))
+ .collect();
+ older.sort_unstable();
+ older.dedup();
+ let mut why = first;
+ for name in older {
+ let mut s = spec.clone();
+ s.template = name.to_owned();
+ match pick(servers, &s) {
+ Ok(c) => return Ok((c, s.template)),
+ // Say why the older one can't run it either (full, say).
+ Err(e) if !matches!(e, NoServer::NoTemplate(_)) => why = e,
+ Err(_) => {}
+ }
+ }
+ Err(why)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -301,6 +337,45 @@ mod tests {
assert_eq!(pick_id(&[building, ready], &spec()).unwrap(), "ready");
}
+ #[test]
+ fn new_sandboxes_move_to_a_new_revision_once_it_is_ready() {
+ let named = |id: &str, name: &str, state: TemplateState| {
+ let mut c = server(id, Runtime::Firecracker, 8);
+ c.templates[0].name = name.into();
+ c.templates[0].state = state;
+ c
+ };
+ let mut s = spec();
+ s.template = "stp_base.new".into();
+ // Only the old one is ready: it's used.
+ let mut a = named("a", "stp_base.old", TemplateState::Ready);
+ a.templates.push(TemplateReport {
+ name: "stp_base.new".into(),
+ state: TemplateState::Building,
+ ..Default::default()
+ });
+ let (c, name) = pick_revision(std::slice::from_ref(&a), &s, "stp_base").unwrap();
+ assert_eq!((c.id.as_str(), name.as_str()), ("a", "stp_base.old"));
+ // Once a server has the new one ready, it's preferred.
+ let b = named("b", "stp_base.new", TemplateState::Ready);
+ let both = [a.clone(), b];
+ let (c, name) = pick_revision(&both, &s, "stp_base").unwrap();
+ assert_eq!((c.id.as_str(), name.as_str()), ("b", "stp_base.new"));
+ // Another template's revisions don't count.
+ let other = named("c", "stp_basement.x", TemplateState::Ready);
+ assert_eq!(
+ pick_revision(&[other], &s, "stp_base").map(|(c, _)| c.id.clone()),
+ Err(NoServer::NoTemplate("stp_base.new".into()))
+ );
+ // The old one's server is full: that's the reason given.
+ let mut full = named("a", "stp_base.old", TemplateState::Ready);
+ full.capacity.memory_available = 0;
+ assert_eq!(
+ pick_revision(&[full], &s, "stp_base").map(|(c, _)| c.id.clone()),
+ Err(NoServer::Full)
+ );
+ }
+
#[test]
fn internet_needs_a_server_with_networking() {
let mut off = server("off", Runtime::Firecracker, 8);
diff --git a/crates/pyo-api/src/sandbox/templates.rs b/crates/pyo-api/src/sandbox/templates.rs
index 20faea8..561f4d1 100644
--- a/crates/pyo-api/src/sandbox/templates.rs
+++ b/crates/pyo-api/src/sandbox/templates.rs
@@ -14,6 +14,11 @@
//! could make it failed.
//! - A template Pyo no longer has is deleted from the servers that made
//! it. One put on a server by hand is never touched.
+//! - A template with a revision (Pyo's built-in `base`, see [`super::base`])
+//! is kept on the servers as `.`. A new image is a new
+//! revision: servers make it beside the old one, new sandboxes move to it
+//! once it's ready there, and the old one is deleted once no sandbox on
+//! that server uses it (running and paused ones keep their disk).
//!
//! TODO(sandbox): templates from a repo or a Dockerfile, through Pyo's
//! builder. The builder runs as a deployment's build job today (deploy.rs
@@ -39,6 +44,46 @@ pub struct Template {
pub id: String,
pub team_id: Option,
pub source: Value,
+ pub revision: Option,
+}
+
+impl Template {
+ /// The name servers keep it under.
+ pub fn server_name(&self) -> String {
+ server_name(&self.id, self.revision.as_deref())
+ }
+}
+
+/// The name servers keep a template under: its id, and its revision when
+/// it has one.
+pub fn server_name(id: &str, revision: Option<&str>) -> String {
+ match revision {
+ Some(r) => format!("{id}.{r}"),
+ None => id.to_owned(),
+ }
+}
+
+/// Whether `name`, as a server keeps it, is a revision of the template `id`
+/// (the current one or an older one).
+pub fn is_revision_of(name: &str, id: &str) -> bool {
+ name == id
+ || name
+ .strip_prefix(id)
+ .is_some_and(|rest| rest.starts_with('.'))
+}
+
+/// Revisions of template `id` a server has ready, other than `current`.
+pub fn older_ready<'a>(
+ reports: &'a [TemplateReport],
+ id: &'a str,
+ current: &'a str,
+) -> impl Iterator- {
+ reports
+ .iter()
+ .filter(move |r| {
+ r.name != current && is_revision_of(&r.name, id) && r.state == TemplateState::Ready
+ })
+ .map(|r| r.name.as_str())
}
/// Whether a server may use a template: an installation's server takes
@@ -69,7 +114,7 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand {
.collect();
match (in_pyos_registry(image, &doors), push.is_empty()) {
(Some(path), false) => DaemonCommand::PrepareTemplate {
- template: t.id.clone(),
+ template: t.server_name(),
// The door on the tailnet, as other machines pull from it.
image: format!("{push}/{path}"),
plain_http: true,
@@ -80,7 +125,7 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand {
max_mib: None,
},
_ => DaemonCommand::PrepareTemplate {
- template: t.id.clone(),
+ template: t.server_name(),
image: image.to_owned(),
plain_http: false,
auth: None,
@@ -90,19 +135,36 @@ pub fn prepare_command(key: &SigningKey, t: &Template) -> DaemonCommand {
}
/// What to tell a server after its report: templates to prepare, and
-/// templates it made that Pyo no longer has (or that aren't for it).
+/// templates it made that Pyo no longer has (or that aren't for it). Both
+/// by the name the server keeps them under. An older revision stays until
+/// the current one is ready there, so new sandboxes have one to start from.
pub fn plan(
server_team: Option<&str>,
report: &ServerReport,
templates: &[Template],
) -> (Vec
, Vec) {
- let wanted: HashSet<&str> = templates
+ let mine: Vec<&Template> = templates
.iter()
.filter(|t| eligible(server_team, t.team_id.as_deref()))
- .map(|t| t.id.as_str())
+ .collect();
+ let names: Vec = mine.iter().map(|t| t.server_name()).collect();
+ let wanted: HashSet<&str> = names.iter().map(String::as_str).collect();
+ // Templates whose current revision isn't ready here yet.
+ let waiting: Vec<&str> = mine
+ .iter()
+ .zip(&names)
+ .filter(|(_, n)| {
+ !report
+ .templates
+ .iter()
+ .any(|r| &r.name == *n && r.state == TemplateState::Ready)
+ })
+ .map(|(t, _)| t.id.as_str())
.collect();
let has: HashSet<&str> = report.templates.iter().map(|t| t.name.as_str()).collect();
- let mut prepare: Vec = if report.capability.runtime == Runtime::Firecracker {
+ // Firecracker servers boot and snapshot a template. gVisor servers make
+ // the same disk and start sandboxes cold from it.
+ let mut prepare: Vec = if report.capability.runtime != Runtime::None {
wanted
.iter()
.filter(|id| !has.contains(*id))
@@ -117,6 +179,7 @@ pub fn plan(
.iter()
// Only what the server made for Pyo: one put there by hand stays.
.filter(|t| t.build.is_some() && !wanted.contains(t.name.as_str()))
+ .filter(|t| !waiting.iter().any(|id| is_revision_of(&t.name, id)))
.map(|t| t.name.clone())
.collect();
stale.sort();
@@ -124,7 +187,7 @@ pub fn plan(
}
pub async fn all(db: &PgPool) -> sqlx::Result> {
- sqlx::query_as("SELECT id, team_id, source FROM pyo.sandbox_templates")
+ sqlx::query_as("SELECT id, team_id, source, revision FROM pyo.sandbox_templates")
.fetch_all(db)
.await
}
@@ -147,7 +210,7 @@ pub async fn sync_server(
let d = Daemon::new(address);
let token = keys::admin_token(key, server_id);
for id in prepare {
- let Some(t) = templates.iter().find(|t| t.id == id) else {
+ let Some(t) = templates.iter().find(|t| t.server_name() == id) else {
continue;
};
match d.command(&token, &prepare_command(key, t)).await {
@@ -187,7 +250,7 @@ pub async fn sync_server(
pub async fn push(s: AppState, template_id: String) {
let res = async {
let Some(t) = sqlx::query_as::<_, Template>(
- "SELECT id, team_id, source FROM pyo.sandbox_templates WHERE id = $1",
+ "SELECT id, team_id, source, revision FROM pyo.sandbox_templates WHERE id = $1",
)
.bind(&template_id)
.fetch_optional(&s.db)
@@ -198,10 +261,14 @@ pub async fn push(s: AppState, template_id: String) {
let servers: Vec<(String, Option, String)> = sqlx::query_as(
"SELECT id, team_id, address FROM pyo.sandbox_servers
WHERE state = 'ready' AND enabled AND address IS NOT NULL
- AND capability->>'runtime' = 'firecracker'",
+ AND capability->>'runtime' IN ('firecracker', 'gvisor')",
)
.fetch_all(&s.db)
.await?;
+ // No server yet: each makes it once it joins.
+ if servers.is_empty() {
+ return anyhow::Ok(());
+ }
let key = keys::signing_key(&s.db, &s.sealer)
.await
.map_err(|e| anyhow::anyhow!(e.message))?;
@@ -225,14 +292,12 @@ pub async fn push(s: AppState, template_id: String) {
}
}
-/// Tells the servers that have a deleted template to delete it. The runner
-/// catches the ones that don't answer now.
+/// Tells the servers that have a deleted template to delete it, every
+/// revision of it. The runner catches the ones that don't answer now.
pub async fn forget(s: AppState, template_id: String) {
- let servers: Vec<(String, String)> = match sqlx::query_as(
- "SELECT id, address FROM pyo.sandbox_servers
- WHERE address IS NOT NULL AND templates @> jsonb_build_array(jsonb_build_object('name', $1::text))",
+ let servers: Vec<(String, String, Value)> = match sqlx::query_as(
+ "SELECT id, address, templates FROM pyo.sandbox_servers WHERE address IS NOT NULL",
)
- .bind(&template_id)
.fetch_all(&s.db)
.await
{
@@ -245,16 +310,22 @@ pub async fn forget(s: AppState, template_id: String) {
let Ok(key) = keys::signing_key(&s.db, &s.sealer).await else {
return;
};
- for (id, address) in servers {
+ for (id, address, templates) in servers {
+ let reports: Vec = serde_json::from_value(templates).unwrap_or_default();
let token = keys::admin_token(&key, &id);
- let _ = Daemon::new(&address)
- .command(
- &token,
- &DaemonCommand::DeleteTemplate {
- template: template_id.clone(),
- },
- )
- .await;
+ for r in reports
+ .iter()
+ .filter(|r| is_revision_of(&r.name, &template_id))
+ {
+ let _ = Daemon::new(&address)
+ .command(
+ &token,
+ &DaemonCommand::DeleteTemplate {
+ template: r.name.clone(),
+ },
+ )
+ .await;
+ }
}
}
@@ -278,25 +349,36 @@ pub fn roll_up(reports: &[TemplateReport], missing: usize) -> (&'static str, Opt
("pending", None)
}
-/// Rolls every template's state up from the servers' latest reports.
+/// Rolls every template's state up from the servers' latest reports, by
+/// its current revision. While a new revision is made, a template whose
+/// older one is ready on a server stays ready: new sandboxes start from it.
pub async fn update_states(db: &PgPool) -> sqlx::Result<()> {
- let rows: Vec<(String, String, Option, Value, i64)> = sqlx::query_as(
+ let rows: Vec<(String, String, Option, Value, i64, bool)> = sqlx::query_as(
"SELECT t.id, t.state, t.error,
coalesce(jsonb_agg(m.e) FILTER (WHERE m.e IS NOT NULL), '[]'),
count(s.id) FILTER (WHERE m.e IS NULL AND s.state = 'ready' AND s.enabled
- AND s.capability->>'runtime' = 'firecracker')
+ AND s.capability->>'runtime' IN ('firecracker', 'gvisor')),
+ coalesce(bool_or(o.ready), false)
FROM pyo.sandbox_templates t
LEFT JOIN pyo.sandbox_servers s
ON s.team_id IS NULL OR t.team_id IS NULL OR s.team_id = t.team_id
LEFT JOIN LATERAL (SELECT x AS e FROM jsonb_array_elements(s.templates) x
- WHERE x->>'name' = t.id LIMIT 1) m ON true
+ WHERE x->>'name' = t.id || coalesce('.' || t.revision, '') LIMIT 1) m ON true
+ LEFT JOIN LATERAL (SELECT true AS ready FROM jsonb_array_elements(s.templates) x
+ WHERE t.revision IS NOT NULL AND starts_with(x->>'name', t.id || '.')
+ AND x->>'name' <> t.id || '.' || t.revision
+ AND coalesce(x->>'state', 'ready') = 'ready' LIMIT 1) o ON true
GROUP BY t.id",
)
.fetch_all(db)
.await?;
- for (id, state, error, reports, missing) in rows {
+ for (id, state, error, reports, missing, older_ready) in rows {
let reports: Vec = serde_json::from_value(reports).unwrap_or_default();
- let (new, why) = roll_up(&reports, missing.max(0) as usize);
+ let (new, why) = match roll_up(&reports, missing.max(0) as usize) {
+ ("ready", why) => ("ready", why),
+ _ if older_ready => ("ready", None),
+ r => r,
+ };
if new != state || why != error {
sqlx::query(
"UPDATE pyo.sandbox_templates SET state = $2, error = $3, updated_at = now() WHERE id = $1",
@@ -323,6 +405,7 @@ mod tests {
id: id.into(),
team_id: team.map(str::to_owned),
source: json!({ "kind": "image", "image": "alpine" }),
+ revision: None,
}
}
@@ -396,11 +479,76 @@ mod tests {
"the installation's server takes all"
);
let (prepare, stale) = plan(Some("tm_a"), &report(Runtime::Gvisor, vec![]), &templates);
+ assert_eq!(
+ prepare,
+ vec!["stp_a", "stp_i"],
+ "gVisor servers make them too"
+ );
+ assert!(stale.is_empty());
+ let (prepare, _) = plan(Some("tm_a"), &report(Runtime::None, vec![]), &templates);
assert!(
prepare.is_empty(),
- "gVisor servers can't make templates yet"
+ "a server that runs nothing makes nothing"
);
- assert!(stale.is_empty());
+ }
+
+ #[test]
+ fn revisions_are_made_beside_the_old_one() {
+ let mut base = t("stp_base", None);
+ base.revision = Some("bbbb".into());
+ assert_eq!(base.server_name(), "stp_base.bbbb");
+ assert!(is_revision_of("stp_base.aaaa", "stp_base"));
+ assert!(is_revision_of("stp_base", "stp_base"));
+ assert!(!is_revision_of("stp_basement", "stp_base"));
+ let templates = [base];
+ // The new revision is made, the old one kept until it's ready.
+ let r = report(
+ Runtime::Firecracker,
+ vec![made("stp_base.aaaa", TemplateState::Ready, None)],
+ );
+ let (prepare, stale) = plan(None, &r, &templates);
+ assert_eq!(prepare, vec!["stp_base.bbbb"]);
+ assert!(stale.is_empty(), "kept while the new one isn't ready");
+ let r = report(
+ Runtime::Firecracker,
+ vec![
+ made("stp_base.aaaa", TemplateState::Ready, None),
+ made("stp_base.bbbb", TemplateState::Building, None),
+ ],
+ );
+ let (prepare, stale) = plan(None, &r, &templates);
+ assert!(prepare.is_empty() && stale.is_empty());
+ assert_eq!(
+ older_ready(&r.templates, "stp_base", "stp_base.bbbb").collect::>(),
+ vec!["stp_base.aaaa"],
+ "new sandboxes still start from the old one"
+ );
+ // Once the new one is ready, the old one goes.
+ let r = report(
+ Runtime::Firecracker,
+ vec![
+ made("stp_base.aaaa", TemplateState::Ready, None),
+ made("stp_base.bbbb", TemplateState::Ready, None),
+ ],
+ );
+ let (prepare, stale) = plan(None, &r, &templates);
+ assert!(prepare.is_empty());
+ assert_eq!(stale, vec!["stp_base.aaaa"]);
+ assert_eq!(
+ older_ready(&r.templates, "stp_base", "stp_base.bbbb").count(),
+ 1
+ );
+ }
+
+ #[test]
+ fn a_revision_is_prepared_under_its_name() {
+ let key = SigningKey::generate("sbk_t");
+ let mut base = t("stp_base", None);
+ base.revision = Some("bbbb".into());
+ let DaemonCommand::PrepareTemplate { template, .. } = prepare_command(&key, &base) else {
+ panic!("not a prepare");
+ };
+ assert_eq!(template, "stp_base.bbbb");
}
#[test]
diff --git a/crates/pyo-api/src/tailscale.rs b/crates/pyo-api/src/tailscale.rs
index 8cc7448..e6a4019 100644
--- a/crates/pyo-api/src/tailscale.rs
+++ b/crates/pyo-api/src/tailscale.rs
@@ -461,7 +461,7 @@ impl Client {
let body = json!({
"capabilities": { "devices": { "create": { "reusable": false, "ephemeral": false, "preauthorized": true, "tags": [TAG] } } },
"expirySeconds": 3600,
- "description": description,
+ "description": key_description(description),
});
let v = self
.call(reqwest::Method::POST, "/tailnet/-/keys", Some(&body))
@@ -483,6 +483,19 @@ impl Client {
}
}
+/// What Tailscale takes as a key's description: letters, digits, spaces,
+/// dashes and underscores, at most 50 characters. "Pyo: join web-1"
+/// becomes "Pyo join web-1".
+fn key_description(s: &str) -> String {
+ let kept: String = s
+ .chars()
+ .filter(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_'))
+ .collect();
+ let mut out = kept.split_whitespace().collect::>().join(" ");
+ out.truncate(50);
+ out.trim_end().to_owned()
+}
+
/// A machine on the tailnet ([`Client::device_details`]).
#[derive(Debug, Clone, PartialEq)]
pub struct Device {
@@ -1008,17 +1021,23 @@ pub async fn wanted_for(
/// Tailscale made no auth key: the details go to the log, the user gets one
/// plain sentence and where to look.
-fn no_auth_key(e: anyhow::Error) -> ApiError {
- tracing::warn!(error = %format!("{e:#}"), "Tailscale made no auth key");
- ApiError::bad_request("Tailscale made no auth key. Check the Tailscale setup in Integrations.")
+/// `server` names what is being added, e.g. "a sandbox server".
+fn no_auth_key(server: &'static str) -> impl FnOnce(anyhow::Error) -> ApiError {
+ move |e| {
+ tracing::warn!(error = %format!("{e:#}"), "Tailscale made no auth key");
+ ApiError::bad_request(format!("Set up Tailscale in Integrations to add {server}."))
+ }
}
/// A join command's auth key: single-use, preauthorized, tag:pyo, an hour.
-pub async fn join_key(s: &AppState, what: &str) -> ApiResult {
+pub async fn join_key(s: &AppState, what: &str, server: &'static str) -> ApiResult {
let Some((_, c)) = saved_client(s).await? else {
return Err(ApiError::bad_request(NEEDED));
};
- c.auth_key(what).await.map(|(_, k)| k).map_err(no_auth_key)
+ c.auth_key(what)
+ .await
+ .map(|(_, k)| k)
+ .map_err(no_auth_key(server))
}
/// Whether a tailnet domain someone typed can be had: Pyo's reserved names
@@ -1335,13 +1354,26 @@ pub async fn host_report(
let (_, key) = c
.auth_key("Pyo: its own server")
.await
- .map_err(no_auth_key)?;
+ .map_err(no_auth_key("this server"))?;
tracing::info!("this machine: joining the tailnet");
Ok(json!({ "state": "join", "authKey": key }))
}
#[cfg(test)]
mod tests {
+ #[test]
+ fn key_descriptions_are_what_tailscale_takes() {
+ use super::key_description;
+ assert_eq!(key_description("Pyo: join web-1"), "Pyo join web-1");
+ assert_eq!(
+ key_description("Pyo: sandbox server sbx_1"),
+ "Pyo sandbox server sbx_1"
+ );
+ assert_eq!(key_description("Pyo: its own server"), "Pyo its own server");
+ let long = key_description(&format!("Pyo: join {}", "a".repeat(80)));
+ assert_eq!(long.len(), 50);
+ }
+
use super::*;
/// A mock Tailscale API: a Service ("docker", someone else's) that the
diff --git a/crates/pyo-builder/Cargo.toml b/crates/pyo-builder/Cargo.toml
index 632e48c..5ea440f 100644
--- a/crates/pyo-builder/Cargo.toml
+++ b/crates/pyo-builder/Cargo.toml
@@ -10,7 +10,7 @@ anyhow = "1.0.104"
base64 = "0.23.1"
futures = "0.3.34"
hex = "0.4.3"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] }
serde = "1.0.229"
serde_json = "1.0.154"
diff --git a/crates/pyo-cli/Cargo.toml b/crates/pyo-cli/Cargo.toml
index dd9e7e2..a7c3dd5 100644
--- a/crates/pyo-cli/Cargo.toml
+++ b/crates/pyo-cli/Cargo.toml
@@ -24,4 +24,4 @@ sha2 = "0.11.1"
tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "net", "io-util", "io-std", "time", "sync", "fs"] }
tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] }
toml = "0.9"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
diff --git a/crates/pyo-cli/src/sandbox.rs b/crates/pyo-cli/src/sandbox.rs
index 62622e5..a3ee79d 100644
--- a/crates/pyo-cli/src/sandbox.rs
+++ b/crates/pyo-cli/src/sandbox.rs
@@ -22,7 +22,8 @@ pub enum Command {
/// Make a sandbox from a sandbox template and print its id.
Create {
/// The sandbox template, like node-22 (see pyo sandbox templates ls).
- template: String,
+ /// Default base: Debian 13 with git, Python, uv, Node.js and pnpm.
+ template: Option,
/// CPUs. Default 1.
#[arg(long)]
cpus: Option,
@@ -186,6 +187,30 @@ pub enum ServersCommand {
#[arg(long, value_name = "DOMAIN")]
edge_domain: Option,
},
+ /// Buy a sandbox server at Hetzner. Pyo sets it up over your tailnet,
+ /// and it runs sandboxes on gVisor. Owners and admins.
+ Create {
+ /// Its name, lowercase letters, digits and dashes.
+ name: String,
+ /// A location, e.g. fsn1 (see pyo server types).
+ #[arg(long)]
+ location: String,
+ /// A server type, e.g. cx22 (see pyo server types).
+ #[arg(long = "type")]
+ server_type: String,
+ /// Serve sandbox ports at -.DOMAIN. Point *.DOMAIN at the server.
+ #[arg(long, value_name = "DOMAIN")]
+ edge_domain: Option,
+ /// Your SSH public key file, to sign in over your tailnet.
+ #[arg(long, value_name = "FILE")]
+ ssh_key: Option,
+ /// Don't ask to confirm the price.
+ #[arg(long, short = 'y')]
+ yes: bool,
+ /// Start it and exit, without following it.
+ #[arg(long)]
+ detach: bool,
+ },
/// The sandbox servers the team can use.
Ls {
/// Print it as JSON.
@@ -238,7 +263,7 @@ pub async fn run(api: &Api, team: Option<&str>, cmd: Command) -> Result<()> {
let team = team_id(api, team).await?;
let body = create_body(CreateArgs {
team: &team,
- template: &template,
+ template: template.as_deref(),
cpus,
memory,
disk,
@@ -385,7 +410,7 @@ async fn lifecycle(api: &Api, id: &str, what: &str, done: &str) -> Result<()> {
pub struct CreateArgs<'a> {
pub team: &'a str,
- pub template: &'a str,
+ pub template: Option<&'a str>,
pub cpus: Option,
pub memory: Option,
pub disk: Option,
@@ -400,12 +425,13 @@ pub struct CreateArgs<'a> {
/// The createSandbox body. Left-out values take the API's defaults.
pub fn create_body(a: CreateArgs) -> Result {
- let mut b = json!({ "teamId": a.team, "template": a.template });
+ let mut b = json!({ "teamId": a.team });
let mut set = |k: &str, v: Value| {
if !v.is_null() {
b[k] = v;
}
};
+ set("template", json!(a.template));
set("cpus", json!(a.cpus));
set("memoryMib", json!(a.memory));
set("diskMib", json!(a.disk));
@@ -460,7 +486,7 @@ fn size(v: &Value) -> String {
fn print_sandboxes(list: &Value) {
let list = list.as_array().map(Vec::as_slice).unwrap_or(&[]);
if list.is_empty() {
- println!("No sandboxes. Make one: pyo sandbox create ");
+ println!("No sandboxes. Make one: pyo sandbox create");
return;
}
let now = crate::api::now_ms();
@@ -619,6 +645,52 @@ async fn servers(api: &Api, team: Option<&str>, cmd: ServersCommand) -> Result<(
println!("{}", r["command"].as_str().unwrap_or(""));
Ok(())
}
+ ServersCommand::Create {
+ name,
+ location,
+ server_type,
+ edge_domain,
+ ssh_key,
+ yes,
+ detach,
+ } => {
+ let ssh_key = match ssh_key {
+ Some(p) => Some(
+ std::fs::read_to_string(&p)
+ .with_context(|| format!("Can't read {}", p.display()))?
+ .trim()
+ .to_owned(),
+ ),
+ None => None,
+ };
+ let mut body = json!({
+ "teamId": team, "name": name, "location": location,
+ "serverType": server_type, "sshKey": ssh_key,
+ });
+ if let Some(d) = edge_domain {
+ body["edgeDomain"] = json!(d);
+ }
+ let path = "/api/v1/sandbox-servers/cloud";
+ let r = match api.post(path, &body).await {
+ Err(e) if e.downcast_ref::().is_some() => {
+ println!("{e}");
+ if !yes && !crate::server::yes_to("Go ahead? [y/N] ")? {
+ bail!("Nothing was changed.");
+ }
+ body["confirm"] = json!(true);
+ api.post(path, &body).await?
+ }
+ r => r?,
+ };
+ ui::ok(&format!(
+ "Creating {name} at Hetzner. It runs sandboxes on gVisor."
+ ));
+ if detach {
+ return Ok(());
+ }
+ let id = r["cloudServer"]["id"].as_str().unwrap_or("").to_owned();
+ crate::server::follow(api, &id).await
+ }
ServersCommand::Ls { json } => {
let r = api
.get(&format!("/api/v1/sandbox-servers?team={team}"))
@@ -636,10 +708,19 @@ async fn servers(api: &Api, team: Option<&str>, cmd: ServersCommand) -> Result<(
.iter()
.map(|x| {
let c = &x["capability"];
+ // A server Pyo is still making at Hetzner says how far it is.
+ let state = match x["cloud"]["state"].as_str() {
+ Some(cs) if cs != "ready" => format!("{cs} (Hetzner)"),
+ _ => x["state"].as_str().unwrap_or("").to_owned(),
+ };
[
x["name"].as_str().unwrap_or("").to_owned(),
- x["state"].as_str().unwrap_or("").to_owned(),
- c["runtime"].as_str().unwrap_or("-").to_owned(),
+ state,
+ x["runtime"]
+ .as_str()
+ .or(c["runtime"].as_str())
+ .unwrap_or("-")
+ .to_owned(),
match (c["cpus"].as_u64(), c["memory"].as_u64()) {
(Some(cpu), Some(m)) => format!("{cpu} CPU, {} GB", m >> 30),
_ => "-".into(),
@@ -670,16 +751,41 @@ async fn servers(api: &Api, team: Option<&str>, cmd: ServersCommand) -> Result<(
}
}
let id = x["id"].as_str().unwrap_or("");
- api.delete(&format!("/api/v1/sandbox-servers/{}", seg(id)))
- .await?;
- ui::ok(&format!(
- "Removed {name}. Run sandbox.sh --uninstall on it to take Pyo off."
- ));
+ let path = format!("/api/v1/sandbox-servers/{}", seg(id));
+ // One Pyo made at Hetzner is deleted there: its price first.
+ match api.delete(&path).await {
+ Err(e) if e.downcast_ref::().is_some() => {
+ println!("{e}");
+ if !yes && !crate::server::yes_to("Go ahead? [y/N] ")? {
+ bail!("Nothing was changed.");
+ }
+ api.delete(&format!("{path}?confirm=true")).await?;
+ ui::ok(&format!("Deleting {name} at Hetzner."));
+ }
+ r => {
+ r?;
+ ui::ok(&format!(
+ "Removed {name}. Run sandbox.sh --uninstall on it to take Pyo off."
+ ));
+ }
+ }
Ok(())
}
}
}
+/// A template's image as the list shows it. Pyo's built-in one says what
+/// it runs and leaves out its version: Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base.
+fn template_image(t: &Value) -> String {
+ let image = t["source"]["image"].as_str().unwrap_or("-");
+ match t["source"]["os"].as_str() {
+ Some(os) if t["builtin"] == true => {
+ format!("{os} · {}", pyo_proto::image_repo(image))
+ }
+ _ => image.to_owned(),
+ }
+}
+
async fn templates(api: &Api, team: Option<&str>, cmd: TemplatesCommand) -> Result<()> {
let team = team_id(api, team).await?;
match cmd {
@@ -703,7 +809,7 @@ async fn templates(api: &Api, team: Option<&str>, cmd: TemplatesCommand) -> Resu
.map(|t| {
[
t["name"].as_str().unwrap_or("").to_owned(),
- t["source"]["image"].as_str().unwrap_or("-").to_owned(),
+ template_image(t),
t["state"].as_str().unwrap_or("").to_owned(),
]
})
@@ -1006,7 +1112,7 @@ mod tests {
let env = vec!["A=1".to_owned(), "B=x=y".to_owned()];
let b = create_body(CreateArgs {
team: "tm_1",
- template: "node-22",
+ template: Some("node-22"),
cpus: Some(2),
memory: None,
disk: None,
@@ -1032,12 +1138,43 @@ mod tests {
);
}
+ #[test]
+ fn no_template_leaves_it_to_pyo() {
+ let b = create_body(CreateArgs {
+ team: "tm_1",
+ template: None,
+ cpus: None,
+ memory: None,
+ disk: None,
+ ttl: None,
+ pause_when_idle: None,
+ env: &[],
+ labels: &[],
+ network: None,
+ isolation: None,
+ project: None,
+ })
+ .unwrap();
+ assert_eq!(b, json!({ "teamId": "tm_1" }), "Pyo uses base");
+ }
+
+ #[test]
+ fn the_builtin_template_shows_what_it_runs() {
+ let base = json!({ "builtin": true, "source": { "image": "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:ab", "os": "Debian 13" } });
+ assert_eq!(
+ template_image(&base),
+ "Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base"
+ );
+ let own = json!({ "builtin": false, "source": { "image": "node:22" } });
+ assert_eq!(template_image(&own), "node:22");
+ }
+
#[test]
fn env_and_labels_need_a_key() {
let bad = vec!["=1".to_owned()];
let e = create_body(CreateArgs {
team: "t",
- template: "x",
+ template: Some("x"),
cpus: None,
memory: None,
disk: None,
diff --git a/crates/pyo-cli/src/server.rs b/crates/pyo-cli/src/server.rs
index 184374a..a8772c5 100644
--- a/crates/pyo-cli/src/server.rs
+++ b/crates/pyo-cli/src/server.rs
@@ -112,7 +112,7 @@ pub async fn disconnect(api: &Api, team_ref: Option<&str>, yes: bool) -> Result<
Ok(())
}
-fn yes_to(q: &str) -> Result {
+pub(crate) fn yes_to(q: &str) -> Result {
Ok(matches!(
ui::ask(q)?.trim().to_lowercase().as_str(),
"y" | "yes"
@@ -281,7 +281,7 @@ pub async fn create(api: &Api, n: New<'_>) -> Result<()> {
}
/// Follows a new server to ready, saying each step.
-async fn follow(api: &Api, id: &str) -> Result<()> {
+pub(crate) async fn follow(api: &Api, id: &str) -> Result<()> {
let mut last = String::new();
let mut sign_shown = false;
loop {
diff --git a/crates/pyo-envd/Cargo.toml b/crates/pyo-envd/Cargo.toml
index 57fad45..b4c8450 100644
--- a/crates/pyo-envd/Cargo.toml
+++ b/crates/pyo-envd/Cargo.toml
@@ -12,5 +12,5 @@ publish.workspace = true
[dependencies]
libc = "0.2.191"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
serde_json.workspace = true
diff --git a/crates/pyo-flows/Cargo.toml b/crates/pyo-flows/Cargo.toml
index 0b987fa..a8964f1 100644
--- a/crates/pyo-flows/Cargo.toml
+++ b/crates/pyo-flows/Cargo.toml
@@ -8,7 +8,7 @@ publish.workspace = true
[dependencies]
anyhow = "1.0.104"
libc = "0.2.191"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls"] }
serde_json = "1.0.154"
tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "time", "sync"] }
diff --git a/crates/pyo-proto/src/lib.rs b/crates/pyo-proto/src/lib.rs
index 959b313..a67436e 100644
--- a/crates/pyo-proto/src/lib.rs
+++ b/crates/pyo-proto/src/lib.rs
@@ -1137,13 +1137,18 @@ pub struct ReleaseImages {
pub postgres17: String,
#[serde(default)]
pub postgres18: String,
+ /// Pyo Sandbox's built-in template `base`, multi-arch
+ /// (deploy/images/sandbox-base.Dockerfile). Pyo seeds the template
+ /// from it; nothing runs it on a cluster.
+ #[serde(default)]
+ pub sandbox_base: String,
}
impl ReleaseImages {
/// Every image, by its key in release.json. A release pins each image
/// scripts/release-manifest.ts writes; adding one there means adding
/// it here (a test compares the two).
- pub fn all(&self) -> [(&'static str, &str); 14] {
+ pub fn all(&self) -> [(&'static str, &str); 15] {
[
("pyo", &self.pyo),
("agent", &self.agent),
@@ -1159,6 +1164,7 @@ impl ReleaseImages {
("postgres16", &self.postgres16),
("postgres17", &self.postgres17),
("postgres18", &self.postgres18),
+ ("sandboxBase", &self.sandbox_base),
]
}
}
@@ -2563,6 +2569,7 @@ mod release_tests {
postgres16: img("postgres-16"),
postgres17: img("postgres-17"),
postgres18: img("postgres-18"),
+ sandbox_base: img("sandbox-base"),
},
manifests: ReleaseManifests {
platform: vec![
@@ -2674,10 +2681,10 @@ mod release_tests {
images
.as_object_mut()
.unwrap()
- .retain(|k, v| !(k == "static" && v == ""));
+ .retain(|k, v| !(matches!(k.as_str(), "static" | "sandboxBase") && v == ""));
assert_eq!(images, raw["images"]);
for (name, image) in r.images.all() {
- if name == "static" && image.is_empty() {
+ if matches!(name, "static" | "sandboxBase") && image.is_empty() {
continue;
}
assert!(pinned_image(image), "{name}: {image}");
diff --git a/crates/pyo-proto/src/sandbox/mod.rs b/crates/pyo-proto/src/sandbox/mod.rs
index 4dbb077..0156128 100644
--- a/crates/pyo-proto/src/sandbox/mod.rs
+++ b/crates/pyo-proto/src/sandbox/mod.rs
@@ -139,6 +139,10 @@ pub struct CapabilityReport {
pub cpu_vendor: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub firecracker_version: String,
+ /// The runsc release on gVisor servers, like `release-20261109.0`.
+ /// Paused sandboxes only resume under the same one.
+ #[serde(default, skip_serializing_if = "String::is_empty")]
+ pub gvisor_version: String,
/// Whether sandboxes on this server can get a network.
#[serde(default)]
pub network: bool,
@@ -766,7 +770,8 @@ pub enum DaemonCommand {
key: SecretPem,
},
/// Makes a template from an OCI image: pulls it, unpacks its layers
- /// into a root disk with `pyo-envd`, boots it once and snapshots it.
+ /// into a root disk with `pyo-envd`, then on Firecracker boots it once
+ /// and snapshots it.
/// Answers at once. The report shows how far it got. A template the
/// server has already is left as it is.
#[serde(rename_all = "camelCase")]
diff --git a/crates/pyo-sandbox/Cargo.toml b/crates/pyo-sandbox/Cargo.toml
index 4584404..65fa1c7 100644
--- a/crates/pyo-sandbox/Cargo.toml
+++ b/crates/pyo-sandbox/Cargo.toml
@@ -16,7 +16,7 @@ http-body-util = "0.1.5"
hyper = { version = "1.12.0", features = ["client", "http1", "server"] }
hyper-util = { version = "0.1.21", features = ["tokio"] }
libc = "0.2.191"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto", features = ["sandbox-token"] }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto", features = ["sandbox-token"] }
serde.workspace = true
serde_json.workspace = true
tokio.workspace = true
diff --git a/crates/pyo-sandbox/scripts/bundle.sh b/crates/pyo-sandbox/scripts/bundle.sh
index 120d119..f31018f 100755
--- a/crates/pyo-sandbox/scripts/bundle.sh
+++ b/crates/pyo-sandbox/scripts/bundle.sh
@@ -13,10 +13,17 @@
# release pinned in firecracker.env, and checked against its pinned sha256
# and the release's own SHA256SUMS. The tarball is flat: pyo-sandbox,
# pyo-envd, firecracker, jailer and vmlinux.
+#
+# Beside it goes pyo-sandbox-gvisor-linux-.tar.gz with its .sha256:
+# runsc and its gvisor-bin/ from the gVisor release pinned in gvisor.env,
+# checked against its pinned sha512. Only servers without KVM fetch it,
+# since it's about 180 MB unpacked.
set -euo pipefail
here=$(cd "$(dirname "$0")" && pwd)
# shellcheck source-path=SCRIPTDIR source=firecracker.env
. "$here/firecracker.env"
+# shellcheck source-path=SCRIPTDIR source=gvisor.env
+. "$here/gvisor.env"
arch="" sandbox="" envd="" kernel="" out=""
while [ $# -gt 0 ]; do
@@ -30,8 +37,8 @@ while [ $# -gt 0 ]; do
esac
done
case "$arch" in
- x86_64) fc_sha=$FC_SHA256_X86_64 ;;
- aarch64) fc_sha=$FC_SHA256_AARCH64 ;;
+ x86_64) fc_sha=$FC_SHA256_X86_64 gv_sha=$GVISOR_SHA512_X86_64 ;;
+ aarch64) fc_sha=$FC_SHA256_AARCH64 gv_sha=$GVISOR_SHA512_AARCH64 ;;
*) echo "--arch is x86_64 or aarch64." >&2; exit 2 ;;
esac
for f in "$sandbox" "$envd" "$kernel"; do
@@ -67,3 +74,19 @@ tar --sort=name --owner=0 --group=0 --numeric-owner --mtime=@0 -C "$b" -cf - \
(cd "$b" && ls -l --block-size=K pyo-sandbox pyo-envd firecracker jailer vmlinux)
echo "$out/$name: $(du -h "$out/$name" | cut -f1)"
cat "$out/$name.sha256"
+
+# gVisor: runsc and gvisor-bin/, as the release has them.
+gv_tar=$work/gvisor.tar.zstd
+curl -fsSL --proto '=https' --retry 3 -o "$gv_tar" \
+ "https://storage.googleapis.com/gvisor/releases/release/$GVISOR_RELEASE/$arch/gvisor.tar.zstd"
+echo "$gv_sha $gv_tar" | sha512sum -c --quiet - || { echo "gVisor $GVISOR_RELEASE doesn't match its pinned sha512." >&2; exit 1; }
+g=$work/gvisor
+mkdir "$g"
+tar --zstd -xf "$gv_tar" -C "$g" runsc gvisor-bin
+[ -x "$g/runsc" ] && [ -d "$g/gvisor-bin" ] || { echo "The gVisor release has no runsc or gvisor-bin." >&2; exit 1; }
+gv_name=pyo-sandbox-gvisor-linux-$arch.tar.gz
+tar --sort=name --owner=0 --group=0 --numeric-owner --mtime=@0 -C "$g" -cf - \
+ runsc gvisor-bin | gzip -n -9 > "$out/$gv_name"
+(cd "$out" && sha256sum "$gv_name" > "$gv_name.sha256")
+echo "$out/$gv_name: $(du -h "$out/$gv_name" | cut -f1)"
+cat "$out/$gv_name.sha256"
diff --git a/crates/pyo-sandbox/scripts/gvisor-assets.sh b/crates/pyo-sandbox/scripts/gvisor-assets.sh
new file mode 100755
index 0000000..42cfc87
--- /dev/null
+++ b/crates/pyo-sandbox/scripts/gvisor-assets.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Fetches runsc for the pyo-sandbox gVisor e2e test: the release pinned in
+# gvisor.env, checked against its pinned sha512. Everything goes into
+# .cache/sandbox-gvisor at the repo root, which git ignores. Prints the env
+# var the test reads.
+set -euo pipefail
+here=$(cd "$(dirname "$0")" && pwd)
+root=$(cd "$here/../../.." && pwd)
+cache=${PYO_SANDBOX_GVISOR_CACHE:-$root/.cache/sandbox-gvisor}
+# shellcheck source-path=SCRIPTDIR source=gvisor.env
+. "$here/gvisor.env"
+case "$(uname -m)" in
+ x86_64 | amd64) arch=x86_64 sum=$GVISOR_SHA512_X86_64 ;;
+ aarch64 | arm64) arch=aarch64 sum=$GVISOR_SHA512_AARCH64 ;;
+ *) echo "gVisor runs on x86-64 or arm64." >&2; exit 1 ;;
+esac
+tgz=$cache/dl/gvisor-$GVISOR_RELEASE-$arch.tar.zstd
+mkdir -p "$cache/dl"
+if [ ! -f "$tgz" ] || ! echo "$sum $tgz" | sha512sum -c --quiet - 2>/dev/null; then
+ curl -fsSL --proto '=https' --retry 3 -o "$tgz.part" \
+ "https://storage.googleapis.com/gvisor/releases/release/$GVISOR_RELEASE/$arch/gvisor.tar.zstd"
+ mv "$tgz.part" "$tgz"
+fi
+echo "$sum $tgz" | sha512sum -c --quiet -
+bin=$cache/$GVISOR_RELEASE-$arch
+if [ ! -x "$bin/runsc" ]; then
+ rm -rf "$bin.tmp"
+ mkdir -p "$bin.tmp"
+ tar --zstd -xf "$tgz" -C "$bin.tmp" runsc gvisor-bin
+ rm -rf "$bin"
+ mv "$bin.tmp" "$bin"
+fi
+echo "PYO_SANDBOX_RUNSC=$bin/runsc"
diff --git a/crates/pyo-sandbox/scripts/gvisor.env b/crates/pyo-sandbox/scripts/gvisor.env
new file mode 100644
index 0000000..b30f1ca
--- /dev/null
+++ b/crates/pyo-sandbox/scripts/gvisor.env
@@ -0,0 +1,8 @@
+# The gVisor release gVisor sandbox servers run, and the sha512 of its
+# gvisor.tar.zstd for each architecture (from the release's .sha512
+# files). Read by gvisor-assets.sh and bundle.sh. A paused gVisor sandbox
+# only resumes under the same release, so changing it ends the pauses of
+# servers that update.
+GVISOR_RELEASE=20261109.0
+GVISOR_SHA512_X86_64=0a0041fd862f4ea3042a9db8d41c742f48a01f909699adc2a0256e46d85e53f97b82b9fd154b3dce636f62b8d557677639f91c77d13f3f875ff9fa535275393c
+GVISOR_SHA512_AARCH64=a0b5d911512ef5fe70e833821e653a90c4d06c7027246ee49423f1d2fe2b2a509389a637797248958b90e49a069bb079ecde860efe1871b1d980ec0fe17e18ac
diff --git a/crates/pyo-sandbox/src/config.rs b/crates/pyo-sandbox/src/config.rs
index c79eec4..86f2e35 100644
--- a/crates/pyo-sandbox/src/config.rs
+++ b/crates/pyo-sandbox/src/config.rs
@@ -28,8 +28,12 @@ pub struct Config {
pub jailer: Option,
/// The guest kernel.
pub kernel: PathBuf,
- /// pyo-envd, for the detection test boot.
+ /// pyo-envd, for the detection test boot and every template.
pub envd: PathBuf,
+ /// runsc, for servers without KVM (gVisor). Its `gvisor-bin` sits
+ /// next to it.
+ #[serde(default = "default_runsc")]
+ pub runsc: PathBuf,
/// Pyo's public keys for checking tokens.
pub keys: Vec,
/// Memory kept back for the host, in MiB. Default: a tenth of the
@@ -70,6 +74,9 @@ pub struct Config {
fn default_root() -> PathBuf {
PathBuf::from("/var/lib/pyo-sandbox")
}
+fn default_runsc() -> PathBuf {
+ PathBuf::from("runsc")
+}
fn default_overcommit() -> f64 {
1.0
}
diff --git a/crates/pyo-sandbox/src/detect.rs b/crates/pyo-sandbox/src/detect.rs
index 4494d9d..2cf32f5 100644
--- a/crates/pyo-sandbox/src/detect.rs
+++ b/crates/pyo-sandbox/src/detect.rs
@@ -7,7 +7,8 @@
//! 2. A test boot: a tiny microVM with a 5 s timeout, booted with the same
//! fast boot args as every sandbox. Under 300 ms is native KVM, slower is
//! nested. The threshold only holds with those args.
-//! 3. Without KVM: `runsc --platform=systrap do true` as root.
+//! 3. Without KVM: `runsc --platform=systrap --network=none do true` as
+//! root, which proves gVisor runs. The report names the runsc release.
//! 4. Neither: the server is refused with a plain reason.
//!
//! `/dev/net/tun`, nftables and another firewall's FORWARD policy only
@@ -34,6 +35,7 @@ pub struct Inputs<'a> {
pub fc: &'a Firecracker,
pub layout: &'a Layout,
pub envd: &'a Path,
+ pub runsc: &'a Path,
}
pub async fn detect(inputs: Inputs<'_>) -> CapabilityReport {
@@ -151,18 +153,16 @@ pub async fn detect(inputs: Inputs<'_>) -> CapabilityReport {
}
}
- // TODO(gvisor): the gVisor driver comes in a later step. Detection
- // already finds it, so the report is right. The manager refuses to
- // create gVisor sandboxes until then.
- match gvisor().await {
- Ok(()) => {
+ match gvisor(inputs.runsc).await {
+ Ok(version) => {
r.runtime = Runtime::Gvisor;
r.kvm = Kvm::None;
r.checks.push(Check {
name: "gvisor".into(),
ok: true,
- detail: "runsc --platform=systrap works".into(),
+ detail: format!("runsc {version} works on systrap"),
});
+ r.gvisor_version = version;
}
Err(e) => {
r.checks.push(Check {
@@ -306,12 +306,13 @@ async fn build_test_image(envd: &Path, out: &Path) -> anyhow::Result<()> {
Ok(())
}
-async fn gvisor() -> Result<(), String> {
+/// Whether runsc runs a sandbox here, and its release.
+async fn gvisor(runsc: &Path) -> Result {
if !is_root() {
return Err("runsc needs root".into());
}
- let run = Command::new("runsc")
- .args(["--platform=systrap", "do", "true"])
+ let run = Command::new(runsc)
+ .args(["--platform=systrap", "--network=none", "do", "true"])
.kill_on_drop(true)
.output();
match tokio::time::timeout(Duration::from_secs(30), run).await {
@@ -320,7 +321,12 @@ async fn gvisor() -> Result<(), String> {
Err("runsc isn't installed".into())
}
Ok(Err(e)) => Err(format!("runsc didn't start: {e}")),
- Ok(Ok(o)) if o.status.success() => Ok(()),
+ Ok(Ok(o)) if o.status.success() => {
+ let v = Command::new(runsc).arg("--version").output().await;
+ Ok(v.ok()
+ .and_then(|o| crate::gvisor::parse_version(&String::from_utf8_lossy(&o.stdout)))
+ .unwrap_or_else(|| "unknown".into()))
+ }
Ok(Ok(o)) => Err(format!(
"runsc failed: {}",
String::from_utf8_lossy(&o.stderr).trim()
diff --git a/crates/pyo-sandbox/src/envd.rs b/crates/pyo-sandbox/src/envd.rs
index c8e5821..02d91c6 100644
--- a/crates/pyo-sandbox/src/envd.rs
+++ b/crates/pyo-sandbox/src/envd.rs
@@ -3,8 +3,13 @@
//! Firecracker exposes the guest's vsock as a Unix socket on the host. The
//! host connects to it, writes `CONNECT \n` and gets `OK \n`. From
//! there on the bytes are the guest's. envd's first frame is `Hello`.
+//!
+//! On gVisor envd dials the daemon instead (see `gvisor`). Those
+//! connections carry a [`Guard`], so the daemon can cut them all before a
+//! checkpoint.
use std::path::Path;
+use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::{Context, anyhow, bail};
@@ -23,6 +28,8 @@ pub const DEFAULT_EXEC_TIMEOUT: Duration = Duration::from_secs(60);
const PORT_LINGER: Duration = Duration::from_secs(30);
pub struct Conn {
+ /// Dropped before the halves, so a guard never outlives its socket.
+ guard: Option>,
r: OwnedReadHalf,
w: OwnedWriteHalf,
dec: Decoder,
@@ -30,6 +37,20 @@ pub struct Conn {
next_stream: u32,
}
+/// A second handle on a connection's socket, to shut it down from outside
+/// while someone else uses it. It goes when the connection does.
+#[derive(Debug)]
+pub struct Guard(std::os::fd::OwnedFd);
+
+impl Guard {
+ /// Ends the connection both ways. Its user reads EOF.
+ pub fn shutdown(&self) {
+ use std::os::fd::AsRawFd;
+ // SAFETY: shutdown on a socket fd we own.
+ unsafe { libc::shutdown(self.0.as_raw_fd(), libc::SHUT_RDWR) };
+ }
+}
+
impl Conn {
/// Connects through Firecracker's vsock socket, retrying until
/// `timeout`. After a restore envd is back within a few ms.
@@ -80,11 +101,24 @@ impl Conn {
Self::start(s, &[]).await
}
+ /// A connection envd dialed, with a guard that can cut it.
+ pub async fn dialed(s: UnixStream) -> anyhow::Result<(Conn, Arc)> {
+ let fd = {
+ use std::os::fd::AsFd;
+ s.as_fd().try_clone_to_owned()?
+ };
+ let guard = Arc::new(Guard(fd));
+ let mut c = Self::start(s, &[]).await?;
+ c.guard = Some(guard.clone());
+ Ok((c, guard))
+ }
+
async fn start(s: UnixStream, already: &[u8]) -> anyhow::Result {
let (r, w) = s.into_split();
let mut dec = Decoder::default();
dec.push(already);
let mut c = Conn {
+ guard: None,
r,
w,
dec,
@@ -103,9 +137,12 @@ impl Conn {
Ok(c)
}
+ /// The guard stays with the reader, which lives as long as the
+ /// connection is used.
pub fn into_split(self) -> (FrameReader, OwnedWriteHalf) {
(
FrameReader {
+ _guard: self.guard,
r: self.r,
dec: self.dec,
},
@@ -415,6 +452,7 @@ impl Conn {
}
pub struct FrameReader {
+ _guard: Option>,
r: OwnedReadHalf,
dec: Decoder,
}
diff --git a/crates/pyo-sandbox/src/gvisor.rs b/crates/pyo-sandbox/src/gvisor.rs
index a803768..e804e41 100644
--- a/crates/pyo-sandbox/src/gvisor.rs
+++ b/crates/pyo-sandbox/src/gvisor.rs
@@ -1,24 +1,851 @@
-//! TODO(gvisor): the gVisor driver, for servers without usable KVM, such
-//! as Hetzner Cloud. Detection already finds runsc and reports
-//! `runtime: gvisor`. The manager refuses gVisor sandboxes until this
-//! lands. It needs root and a box where runsc runs (not this laptop).
+//! The gVisor driver, for servers without usable KVM, such as Hetzner
+//! Cloud. Detection picks it when Firecracker can't run (see `detect`).
//!
-//! The design (PLAN.md "Runtime" and the gVisor spike, items 15 to 22):
-//! - `runsc --platform=systrap`, never the KVM platform. runsc runs as
-//! root. One OCI bundle per sandbox, with the template's root
-//! filesystem read-only and an overlay for writes.
-//! - envd runs in dial mode (`PYO_ENVD_DIAL=unix:/pyo/host.sock`) with
-//! `--host-uds=open`. It never listens, because a bound host socket
-//! blocks `runsc checkpoint`. The daemon listens on a per-sandbox
-//! `host.sock`, and envd's first frame on each connection is the ready
-//! signal. Before a checkpoint the daemon closes the listener and every
-//! connection.
-//! - Templates are prepared root filesystems with cold starts (37 ms), not
-//! memory snapshots. Restores cost more and don't share pages.
-//! - Pause and resume use `runsc checkpoint` and `runsc restore`. Images
-//! are small (15 MiB, 4.8 MiB with flate) and go to Garage as they are.
-//! The cache is keyed by the runsc release.
-//! - Memory is the sandbox cgroup's `memory.current` (`cgroupsPath`), which
-//! matches the host within 1 MiB.
-//! - Starts are CPU bound, about 13 per second per vCPU, so the scheduler
-//! spreads bursts.
+//! - **runsc as root**, `--platform=systrap`, never the KVM platform: on a
+//! cloud VM KVM is nested, and slower at everything (gVisor spike,
+//! item 18). Its state lives under `/runsc`.
+//! - **One OCI bundle per sandbox** in its directory. The root is a host
+//! overlay: the template's disk, mounted read-only once per template
+//! (`templates//root`), under a writable layer on the sandbox's own
+//! ext4 image of `diskMib`, so a sandbox can't fill the server's disk.
+//! runsc's own overlay is off (`--overlay2=none`), so writes go to that
+//! disk, not into memory, and a checkpoint stays small.
+//! - **envd dials out.** It runs as the sandbox's PID 1 with
+//! `PYO_ENVD_DIAL=unix:/pyo/host.sock` and `--host-uds=open`. `/pyo` is
+//! the sandbox directory's `pyo/`, where the daemon listens on
+//! `host.sock` ([`Door`]). envd's first frame on each connection is the
+//! ready signal, and it keeps one idle connection waiting. It never
+//! listens, because a bound host socket blocks `runsc checkpoint`.
+//! - **Network**: a namespace and veth pair per sandbox behind the same
+//! nftables table as Firecracker's taps (`network`), handed to runsc as
+//! the OCI network namespace with `--network=sandbox`. "No network" is a
+//! fresh namespace with `--network=none`.
+//! - **Memory** is the sandbox cgroup's `memory.current`
+//! (`/`), which holds the sentry and the gofer
+//! and matches the host within 1 MiB.
+//! - **Templates** are cold starts (37 ms in the spike), not memory
+//! snapshots: a gVisor restore costs more and shares no pages.
+//! - **Pause** is `runsc checkpoint` into the sandbox directory, after the
+//! daemon closed its listener and every envd connection. **Resume** is
+//! `runsc restore`. Images are small (15 MiB for a warm Python sandbox).
+//! They need the same runsc release, which the capability report names.
+
+use std::net::Ipv4Addr;
+use std::path::{Path, PathBuf};
+use std::process::Stdio;
+use std::sync::{Arc, Mutex as StdMutex, Weak};
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, anyhow, bail};
+use serde_json::{Value, json};
+use tokio::net::UnixListener;
+use tokio::process::Command;
+
+use crate::envd::{self, Guard};
+use crate::firecracker::{DirHandle, process_alive};
+use crate::storage;
+
+pub const PLATFORM: &str = "systrap";
+/// The sandbox directory's part that is `/pyo` inside.
+pub const PYO_DIR: &str = "pyo";
+pub const HOST_SOCK: &str = "host.sock";
+/// Where envd dials, inside the sandbox.
+pub const DIAL: &str = "unix:/pyo/host.sock";
+const ROOTFS_DIR: &str = "rootfs";
+const SCRATCH_IMG: &str = "scratch.ext4";
+const SCRATCH_DIR: &str = "scratch";
+const RESOLV: &str = "resolv.conf";
+/// The checkpoint of a paused sandbox.
+pub const IMAGE_DIR: &str = "pause";
+const LOG: &str = "runsc.log";
+/// Memory the sentry and the gofer take on top of the guest's.
+const CGROUP_OVERHEAD: u64 = 64 << 20;
+const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+
+/// How the daemon runs runsc.
+#[derive(Debug, Clone)]
+pub struct Gvisor {
+ pub runsc: PathBuf,
+ /// runsc's `--root`: its state of every sandbox.
+ pub state: PathBuf,
+ /// The cgroup each sandbox goes under, like the jailer's.
+ pub cgroup_parent: String,
+ /// One template mount at a time.
+ mounting: Arc>,
+}
+
+/// A sandbox to start.
+pub struct Machine<'a> {
+ /// The instance ID: runsc's container ID and the directory's name.
+ pub id: &'a str,
+ pub dir: &'a Path,
+ /// The template's root, mounted ([`Gvisor::template_root`]).
+ pub template_root: &'a Path,
+ /// An empty ext4 image of the sandbox's disk size, copied once.
+ pub scratch: Option<&'a Path>,
+ pub cpus: u32,
+ pub memory_mib: u32,
+ /// The lease's network namespace, or None for no network.
+ pub netns: Option<&'a str>,
+ pub dns: &'a [Ipv4Addr],
+ /// The cgroup it goes under ([`Gvisor::cgroup_parent`]).
+ pub cgroup_parent: &'a str,
+}
+
+/// A running sandbox.
+#[derive(Debug)]
+pub struct Container {
+ pub id: String,
+ pub dir: PathBuf,
+ /// The sandbox process on the host.
+ pub pid: u32,
+ pub network: bool,
+ pub door: Arc,
+ gv: Gvisor,
+}
+
+/// Where envd dials in: the listener on `pyo/host.sock` and the
+/// connections it gave out.
+#[derive(Debug)]
+pub struct Door {
+ handle: DirHandle,
+ listener: tokio::sync::Mutex>,
+ conns: StdMutex>>,
+}
+
+impl Door {
+ /// Listens on the directory's `host.sock`, in place of any old one.
+ pub fn open(pyo_dir: &Path) -> anyhow::Result> {
+ let door = Arc::new(Door {
+ handle: DirHandle::open(pyo_dir)?,
+ listener: tokio::sync::Mutex::new(None),
+ conns: StdMutex::new(Vec::new()),
+ });
+ door.listen_now()?;
+ Ok(door)
+ }
+
+ fn bind(&self) -> anyhow::Result {
+ let path = self.handle.path(HOST_SOCK);
+ let _ = std::fs::remove_file(&path);
+ UnixListener::bind(&path).context("listening for envd")
+ }
+
+ fn listen_now(&self) -> anyhow::Result<()> {
+ let l = self.bind()?;
+ *self
+ .listener
+ .try_lock()
+ .map_err(|_| anyhow!("the door is busy"))? = Some(l);
+ Ok(())
+ }
+
+ /// Listens again after [`Door::close`].
+ pub async fn reopen(&self) -> anyhow::Result<()> {
+ let l = self.bind()?;
+ *self.listener.lock().await = Some(l);
+ Ok(())
+ }
+
+ /// The next connection envd dialed, past its `Hello`.
+ pub async fn accept(&self, timeout: Duration) -> anyhow::Result {
+ let until = Instant::now() + timeout;
+ let stream = {
+ let guard = self.listener.lock().await;
+ let l = guard
+ .as_ref()
+ .ok_or_else(|| anyhow!("the sandbox is being paused"))?;
+ match tokio::time::timeout_at(until.into(), l.accept()).await {
+ Ok(r) => r.context("accepting envd")?.0,
+ Err(_) => bail!("envd didn't answer within {} ms", timeout.as_millis()),
+ }
+ };
+ let left = until.saturating_duration_since(Instant::now());
+ let (conn, guard) =
+ tokio::time::timeout(left.max(Duration::from_secs(1)), envd::Conn::dialed(stream))
+ .await
+ .map_err(|_| anyhow!("envd didn't say hello"))??;
+ let mut conns = self.conns.lock().unwrap_or_else(|p| p.into_inner());
+ conns.retain(|w| w.strong_count() > 0);
+ conns.push(Arc::downgrade(&guard));
+ Ok(conn)
+ }
+
+ /// Stops listening and cuts every connection, which a checkpoint needs.
+ /// envd goes back to dialing.
+ pub async fn close(&self) {
+ *self.listener.lock().await = None;
+ let _ = std::fs::remove_file(self.handle.path(HOST_SOCK));
+ let conns = std::mem::take(&mut *self.conns.lock().unwrap_or_else(|p| p.into_inner()));
+ for g in conns.iter().filter_map(Weak::upgrade) {
+ g.shutdown();
+ }
+ }
+}
+
+impl Gvisor {
+ pub fn new(runsc: PathBuf, state: PathBuf, cgroup_parent: String) -> Gvisor {
+ Gvisor {
+ runsc,
+ state,
+ cgroup_parent,
+ mounting: Arc::new(tokio::sync::Mutex::new(())),
+ }
+ }
+
+ /// `runsc --version`'s release, like `release-20261109.0`.
+ pub async fn version(&self) -> anyhow::Result {
+ let out = Command::new(&self.runsc)
+ .arg("--version")
+ .output()
+ .await
+ .with_context(|| format!("running {}", self.runsc.display()))?;
+ parse_version(&String::from_utf8_lossy(&out.stdout))
+ .ok_or_else(|| anyhow!("runsc --version said nothing useful"))
+ }
+
+ /// runsc's global flags, the same for every command on a sandbox.
+ pub fn flags(&self, network: bool) -> Vec {
+ vec![
+ format!("--root={}", self.state.display()),
+ format!("--platform={PLATFORM}"),
+ format!("--network={}", if network { "sandbox" } else { "none" }),
+ "--host-uds=open".into(),
+ "--overlay2=none".into(),
+ "--cpu-num-from-quota".into(),
+ ]
+ }
+
+ fn cmd(&self, network: bool) -> Command {
+ let mut c = Command::new(&self.runsc);
+ c.args(self.flags(network))
+ .stdin(Stdio::null())
+ .kill_on_drop(true);
+ c
+ }
+
+ /// A runsc command that ends on its own, with its output.
+ async fn run(&self, network: bool, args: &[&str]) -> anyhow::Result {
+ let out = self
+ .cmd(network)
+ .args(args)
+ .output()
+ .await
+ .with_context(|| format!("starting {}", self.runsc.display()))?;
+ if !out.status.success() {
+ bail!(
+ "runsc {}: {}",
+ args.first().copied().unwrap_or_default(),
+ last_line(&String::from_utf8_lossy(&out.stderr))
+ );
+ }
+ Ok(String::from_utf8_lossy(&out.stdout).into_owned())
+ }
+
+ /// `run --detach` and `restore --detach` leave the sandbox holding
+ /// their stdout and stderr, so those go to a log file, never a pipe.
+ async fn run_detached(&self, network: bool, dir: &Path, args: &[&str]) -> anyhow::Result<()> {
+ let log = std::fs::OpenOptions::new()
+ .create(true)
+ .append(true)
+ .open(dir.join(LOG))?;
+ let status = self
+ .cmd(network)
+ .args(args)
+ .stdout(log.try_clone()?)
+ .stderr(log)
+ .status()
+ .await
+ .with_context(|| format!("starting {}", self.runsc.display()))?;
+ if !status.success() {
+ let text = std::fs::read_to_string(dir.join(LOG)).unwrap_or_default();
+ bail!(
+ "runsc {}: {}",
+ args.first().copied().unwrap_or_default(),
+ last_line(&text)
+ );
+ }
+ Ok(())
+ }
+
+ /// The sandbox's state and host pid, from `runsc state`.
+ pub async fn state(&self, id: &str) -> anyhow::Result<(String, u32)> {
+ let out = self.run(false, &["state", id]).await?;
+ let v: Value = serde_json::from_str(&out).context("reading runsc state")?;
+ Ok((
+ v["status"].as_str().unwrap_or_default().to_string(),
+ v["pid"].as_u64().unwrap_or(0) as u32,
+ ))
+ }
+
+ /// The template's disk, mounted read-only at `templates//root`
+ /// (once, for every sandbox of it).
+ pub async fn template_root(&self, rootfs: &Path) -> anyhow::Result {
+ let dir = rootfs
+ .parent()
+ .ok_or_else(|| anyhow!("no template directory"))?
+ .join("root");
+ let _turn = self.mounting.lock().await;
+ if is_mount(&dir) {
+ return Ok(dir);
+ }
+ tokio::fs::create_dir_all(&dir).await?;
+ mount(&["-o", "loop,ro"], rootfs, &dir).await?;
+ Ok(dir)
+ }
+
+ /// The root and the files runsc reads: the writable layer on its own
+ /// disk, the overlay, `resolv.conf` and `config.json`. Done again
+ /// after a reboot, for a paused sandbox.
+ async fn prepare(&self, m: &Machine<'_>) -> anyhow::Result<()> {
+ let dir = m.dir;
+ let pyo = dir.join(PYO_DIR);
+ let rootfs = dir.join(ROOTFS_DIR);
+ let scratch = dir.join(SCRATCH_DIR);
+ for d in [&pyo, &rootfs, &scratch] {
+ tokio::fs::create_dir_all(d).await?;
+ }
+ set_mode(&pyo, 0o700)?;
+ let img = dir.join(SCRATCH_IMG);
+ if !img.exists() {
+ let src = m
+ .scratch
+ .ok_or_else(|| anyhow!("the sandbox's disk is missing"))?;
+ storage::copy_sparse(src, &img).await?;
+ }
+ if !is_mount(&scratch) {
+ mount(&["-o", "loop"], &img, &scratch).await?;
+ }
+ for d in ["upper", "work"] {
+ tokio::fs::create_dir_all(scratch.join(d)).await?;
+ }
+ if !is_mount(&rootfs) {
+ let opts = format!(
+ "lowerdir={},upperdir={},workdir={}",
+ m.template_root.display(),
+ scratch.join("upper").display(),
+ scratch.join("work").display()
+ );
+ mount(
+ &["-t", "overlay", "-o", &opts],
+ Path::new("overlay"),
+ &rootfs,
+ )
+ .await?;
+ }
+ tokio::fs::write(dir.join(RESOLV), resolv_conf(m.dns)).await?;
+ let spec = oci_spec(m, &rootfs, &pyo, &dir.join(RESOLV));
+ tokio::fs::write(dir.join("config.json"), serde_json::to_vec_pretty(&spec)?).await?;
+ Ok(())
+ }
+
+ /// Starts a sandbox and waits for envd's first connection.
+ pub async fn start(
+ &self,
+ m: &Machine<'_>,
+ timeout: Duration,
+ ) -> anyhow::Result<(Container, Duration)> {
+ let t0 = Instant::now();
+ self.prepare(m).await?;
+ let door = Door::open(&m.dir.join(PYO_DIR))?;
+ let network = m.netns.is_some();
+ let bundle = format!("--bundle={}", m.dir.display());
+ self.run_detached(network, m.dir, &["run", "--detach", &bundle, m.id])
+ .await?;
+ self.finish(m.id, m.dir, network, door, t0, timeout).await
+ }
+
+ /// Resumes a checkpoint made by [`Gvisor::checkpoint`].
+ pub async fn restore(
+ &self,
+ m: &Machine<'_>,
+ timeout: Duration,
+ ) -> anyhow::Result<(Container, Duration)> {
+ let t0 = Instant::now();
+ self.prepare(m).await?;
+ let door = Door::open(&m.dir.join(PYO_DIR))?;
+ let network = m.netns.is_some();
+ // A container left from the checkpoint would take the ID.
+ let _ = self.run(network, &["delete", "--force", m.id]).await;
+ let image = format!("--image-path={}", m.dir.join(IMAGE_DIR).display());
+ let bundle = format!("--bundle={}", m.dir.display());
+ self.run_detached(
+ network,
+ m.dir,
+ &["restore", "--detach", &image, &bundle, m.id],
+ )
+ .await?;
+ let started = self.finish(m.id, m.dir, network, door, t0, timeout).await?;
+ let _ = tokio::fs::remove_dir_all(m.dir.join(IMAGE_DIR)).await;
+ Ok(started)
+ }
+
+ async fn finish(
+ &self,
+ id: &str,
+ dir: &Path,
+ network: bool,
+ door: Arc,
+ t0: Instant,
+ timeout: Duration,
+ ) -> anyhow::Result<(Container, Duration)> {
+ let ready = door.accept(timeout).await;
+ let pid = self.state(id).await.map(|(_, pid)| pid).unwrap_or(0);
+ let c = Container {
+ id: id.into(),
+ dir: dir.into(),
+ pid,
+ network,
+ door,
+ gv: self.clone(),
+ };
+ match ready {
+ // Dropped unused: envd dials the next one.
+ Ok(_hello) => Ok((c, t0.elapsed())),
+ Err(e) => {
+ let log = std::fs::read_to_string(dir.join(LOG)).unwrap_or_default();
+ let mut c = c;
+ c.kill().await;
+ Err(e.context(format!("the sandbox didn't start. {}", last_line(&log))))
+ }
+ }
+ }
+
+ /// A sandbox a previous daemon started, when runsc still runs it.
+ pub async fn adopt(&self, id: &str, dir: &Path, network: bool) -> Option {
+ let (status, pid) = self.state(id).await.ok()?;
+ if status != "running" || !process_alive(pid) {
+ return None;
+ }
+ let door = Door::open(&dir.join(PYO_DIR)).ok()?;
+ Some(Container {
+ id: id.into(),
+ dir: dir.into(),
+ pid,
+ network,
+ door,
+ gv: self.clone(),
+ })
+ }
+
+ /// Takes everything of a sandbox off the host but its directory: the
+ /// container, if runsc still has it, and its mounts.
+ pub async fn teardown(&self, id: &str, dir: &Path) {
+ for network in [false, true] {
+ if self.run(network, &["delete", "--force", id]).await.is_ok() {
+ break;
+ }
+ }
+ unmount_below(dir).await;
+ }
+}
+
+impl Container {
+ pub fn exited(&self) -> bool {
+ self.pid == 0 || !process_alive(self.pid)
+ }
+
+ pub async fn kill(&mut self) {
+ self.door.close().await;
+ let _ = self.gv.run(self.network, &["kill", &self.id, "KILL"]).await;
+ let _ = self
+ .gv
+ .run(self.network, &["delete", "--force", &self.id])
+ .await;
+ for _ in 0..200 {
+ if !process_alive(self.pid) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ }
+ }
+
+ /// Writes a checkpoint to the sandbox's `pause/` and stops it. Its
+ /// mounts stay for the restore. On failure it keeps running.
+ pub async fn checkpoint(&mut self) -> anyhow::Result<()> {
+ let image = self.dir.join(IMAGE_DIR);
+ let _ = tokio::fs::remove_dir_all(&image).await;
+ tokio::fs::create_dir_all(&image).await?;
+ self.door.close().await;
+ let arg = format!("--image-path={}", image.display());
+ let mut last = None;
+ // envd may be halfway through a dial when the door closes.
+ for wait in [0, 50, 150, 400] {
+ tokio::time::sleep(Duration::from_millis(wait)).await;
+ match self
+ .gv
+ .run(self.network, &["checkpoint", &arg, &self.id])
+ .await
+ {
+ Ok(_) => {
+ let _ = self
+ .gv
+ .run(self.network, &["delete", "--force", &self.id])
+ .await;
+ return Ok(());
+ }
+ Err(e) => last = Some(e),
+ }
+ }
+ let _ = tokio::fs::remove_dir_all(&image).await;
+ self.door.reopen().await?;
+ Err(last.unwrap_or_else(|| anyhow!("runsc checkpoint failed")))
+ }
+}
+
+/// `runsc --version`'s first line: "runsc version release-20261109.0".
+pub fn parse_version(text: &str) -> Option {
+ let line = text.lines().next()?;
+ let v = line.strip_prefix("runsc version ")?.trim();
+ (!v.is_empty()).then(|| v.to_string())
+}
+
+fn last_line(text: &str) -> String {
+ text.lines()
+ .rev()
+ .map(str::trim)
+ .find(|l| !l.is_empty())
+ .unwrap_or("no output")
+ .chars()
+ .take(400)
+ .collect()
+}
+
+pub fn resolv_conf(dns: &[Ipv4Addr]) -> String {
+ let mut s = String::new();
+ for d in dns {
+ s.push_str(&format!("nameserver {d}\n"));
+ }
+ s
+}
+
+/// The OCI runtime spec runsc starts the sandbox from.
+pub fn oci_spec(m: &Machine<'_>, rootfs: &Path, pyo: &Path, resolv: &Path) -> Value {
+ let caps = json!([
+ "CAP_CHOWN",
+ "CAP_DAC_OVERRIDE",
+ "CAP_FOWNER",
+ "CAP_FSETID",
+ "CAP_KILL",
+ "CAP_SETGID",
+ "CAP_SETUID",
+ "CAP_SETPCAP",
+ "CAP_NET_BIND_SERVICE",
+ "CAP_NET_RAW",
+ "CAP_SYS_CHROOT",
+ "CAP_MKNOD",
+ "CAP_AUDIT_WRITE",
+ "CAP_SETFCAP"
+ ]);
+ let mut mounts = vec![
+ json!({"destination": "/proc", "type": "proc", "source": "proc"}),
+ json!({"destination": "/dev", "type": "tmpfs", "source": "tmpfs",
+ "options": ["nosuid", "strictatime", "mode=755", "size=65536k"]}),
+ json!({"destination": "/dev/pts", "type": "devpts", "source": "devpts",
+ "options": ["nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"]}),
+ json!({"destination": "/dev/shm", "type": "tmpfs", "source": "shm",
+ "options": ["nosuid", "noexec", "nodev", "mode=1777", "size=65536k"]}),
+ json!({"destination": "/sys", "type": "sysfs", "source": "sysfs",
+ "options": ["nosuid", "noexec", "nodev", "ro"]}),
+ json!({"destination": "/pyo", "type": "bind", "source": pyo,
+ "options": ["rbind", "rw"]}),
+ ];
+ if m.netns.is_some() {
+ mounts.push(json!({"destination": "/etc/resolv.conf", "type": "bind",
+ "source": resolv, "options": ["rbind", "ro"]}));
+ }
+ let network = match m.netns {
+ Some(path) => json!({"type": "network", "path": path}),
+ None => json!({"type": "network"}),
+ };
+ let cpus = m.cpus.max(1) as u64;
+ json!({
+ "ociVersion": "1.0.2",
+ "process": {
+ "terminal": false,
+ "user": {"uid": 0, "gid": 0},
+ "args": ["/sbin/pyo-envd"],
+ "env": [
+ format!("PATH={PATH}"),
+ "HOME=/root",
+ format!("PYO_ENVD_DIAL={DIAL}"),
+ ],
+ "cwd": "/",
+ "capabilities": {
+ "bounding": caps, "effective": caps, "permitted": caps
+ },
+ "rlimits": [{"type": "RLIMIT_NOFILE", "hard": 1_048_576u64, "soft": 1_048_576u64}],
+ "noNewPrivileges": false
+ },
+ "root": {"path": rootfs, "readonly": false},
+ "hostname": "sandbox",
+ "mounts": mounts,
+ "linux": {
+ "namespaces": [
+ {"type": "pid"}, {"type": "ipc"}, {"type": "uts"}, {"type": "mount"}, network
+ ],
+ "cgroupsPath": format!("/{}/{}", m.cgroup_parent, m.id),
+ "resources": {
+ "memory": {"limit": ((m.memory_mib as u64) << 20) + CGROUP_OVERHEAD},
+ "cpu": {"quota": cpus * 100_000, "period": 100_000u64}
+ }
+ }
+ })
+}
+
+fn set_mode(p: &Path, mode: u32) -> anyhow::Result<()> {
+ use std::os::unix::fs::PermissionsExt;
+ std::fs::set_permissions(p, std::fs::Permissions::from_mode(mode))
+ .with_context(|| format!("setting the mode of {}", p.display()))
+}
+
+async fn mount(opts: &[&str], src: &Path, dst: &Path) -> anyhow::Result<()> {
+ let out = Command::new("mount")
+ .args(opts)
+ .arg(src)
+ .arg(dst)
+ .stdin(Stdio::null())
+ .output()
+ .await
+ .context("starting mount")?;
+ if !out.status.success() {
+ bail!(
+ "mounting {}: {}",
+ dst.display(),
+ String::from_utf8_lossy(&out.stderr).trim()
+ );
+ }
+ Ok(())
+}
+
+/// Mount points from /proc/self/mountinfo (field 5, with `\040` and the
+/// like decoded).
+pub fn mount_points(mountinfo: &str) -> Vec {
+ mountinfo
+ .lines()
+ .filter_map(|l| l.split(' ').nth(4))
+ .map(|p| PathBuf::from(unescape(p)))
+ .collect()
+}
+
+fn unescape(s: &str) -> String {
+ let b = s.as_bytes();
+ let mut out = Vec::with_capacity(b.len());
+ let mut i = 0;
+ while i < b.len() {
+ if b[i] == b'\\'
+ && i + 3 < b.len()
+ && b[i + 1..i + 4].iter().all(|c| (b'0'..=b'7').contains(c))
+ {
+ let v = (b[i + 1] - b'0') * 64 + (b[i + 2] - b'0') * 8 + (b[i + 3] - b'0');
+ out.push(v);
+ i += 4;
+ } else {
+ out.push(b[i]);
+ i += 1;
+ }
+ }
+ String::from_utf8_lossy(&out).into_owned()
+}
+
+fn mounts_now() -> Vec {
+ mount_points(&std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default())
+}
+
+pub fn is_mount(p: &Path) -> bool {
+ mounts_now().iter().any(|m| m == p)
+}
+
+/// The mounts at or under `dir`, deepest first.
+pub fn mounts_below(points: &[PathBuf], dir: &Path) -> Vec {
+ let mut v: Vec = points
+ .iter()
+ .filter(|m| m.starts_with(dir))
+ .cloned()
+ .collect();
+ v.sort_by_key(|m| std::cmp::Reverse(m.components().count()));
+ v.dedup();
+ v
+}
+
+/// Unmounts everything at or under `dir`, so removing it never reaches
+/// into a template or another disk. A busy mount is detached lazily.
+pub async fn unmount_below(dir: &Path) {
+ for m in mounts_below(&mounts_now(), dir) {
+ let ok = Command::new("umount")
+ .arg(&m)
+ .stdin(Stdio::null())
+ .output()
+ .await
+ .map(|o| o.status.success())
+ .unwrap_or(false);
+ if !ok {
+ let _ = Command::new("umount").arg("-l").arg(&m).output().await;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn versions() {
+ assert_eq!(
+ parse_version("runsc version release-20261109.0\nspec: 1.1.0-rc.1\n").as_deref(),
+ Some("release-20261109.0")
+ );
+ assert_eq!(parse_version(""), None);
+ assert_eq!(parse_version("something else"), None);
+ }
+
+ #[test]
+ fn mountinfo_and_what_goes_first() {
+ let info = "22 1 8:1 / / rw - ext4 /dev/sda1 rw\n\
+ 90 22 7:0 / /var/lib/pyo-sandbox/templates/t/root ro - ext4 /dev/loop0 ro\n\
+ 91 22 7:1 / /var/lib/pyo-sandbox/sandboxes/a/scratch rw - ext4 /dev/loop1 rw\n\
+ 92 22 0:50 / /var/lib/pyo-sandbox/sandboxes/a/rootfs rw - overlay overlay rw\n\
+ 93 92 0:51 / /var/lib/pyo-sandbox/sandboxes/a/rootfs/with\\040space rw - tmpfs t rw\n";
+ let points = mount_points(info);
+ assert_eq!(points.len(), 5);
+ assert!(points.contains(&PathBuf::from(
+ "/var/lib/pyo-sandbox/sandboxes/a/rootfs/with space"
+ )));
+ let below = mounts_below(&points, Path::new("/var/lib/pyo-sandbox/sandboxes/a"));
+ assert_eq!(
+ below,
+ vec![
+ PathBuf::from("/var/lib/pyo-sandbox/sandboxes/a/rootfs/with space"),
+ PathBuf::from("/var/lib/pyo-sandbox/sandboxes/a/scratch"),
+ PathBuf::from("/var/lib/pyo-sandbox/sandboxes/a/rootfs"),
+ ]
+ );
+ // Never the template or a sandbox with a longer name.
+ assert!(
+ mounts_below(&points, Path::new("/var/lib/pyo-sandbox/sandboxes/"))
+ .iter()
+ .all(|m| !m.starts_with("/var/lib/pyo-sandbox/templates"))
+ );
+ assert!(mounts_below(&points, Path::new("/var/lib/pyo-sandbox/sandboxes/ab")).is_empty());
+ }
+
+ #[test]
+ fn the_spec_runs_envd_in_dial_mode_with_limits() {
+ let dns = [Ipv4Addr::new(1, 1, 1, 1)];
+ let m = Machine {
+ id: "sbx-abc",
+ dir: Path::new("/d"),
+ template_root: Path::new("/t"),
+ scratch: None,
+ cpus: 2,
+ memory_mib: 512,
+ netns: Some("/var/run/netns/pyosbx3"),
+ dns: &dns,
+ cgroup_parent: "pyo-sandbox",
+ };
+ let s = oci_spec(
+ &m,
+ Path::new("/d/rootfs"),
+ Path::new("/d/pyo"),
+ Path::new("/d/resolv.conf"),
+ );
+ assert_eq!(s["process"]["args"], json!(["/sbin/pyo-envd"]));
+ assert!(
+ s["process"]["env"]
+ .as_array()
+ .unwrap()
+ .contains(&json!("PYO_ENVD_DIAL=unix:/pyo/host.sock"))
+ );
+ assert_eq!(s["root"]["path"], json!("/d/rootfs"));
+ assert_eq!(
+ s["linux"]["resources"]["memory"]["limit"],
+ json!((512u64 << 20) + CGROUP_OVERHEAD)
+ );
+ assert_eq!(s["linux"]["resources"]["cpu"]["quota"], json!(200_000));
+ assert_eq!(s["linux"]["cgroupsPath"], json!("/pyo-sandbox/sbx-abc"));
+ let ns = s["linux"]["namespaces"].as_array().unwrap();
+ assert!(ns.contains(&json!({"type": "network", "path": "/var/run/netns/pyosbx3"})));
+ let mounts = s["mounts"].as_array().unwrap();
+ assert!(mounts.iter().any(|x| x["destination"] == "/pyo"));
+ assert!(
+ mounts
+ .iter()
+ .any(|x| x["destination"] == "/etc/resolv.conf")
+ );
+
+ let none = Machine { netns: None, ..m };
+ let s = oci_spec(
+ &none,
+ Path::new("/d/rootfs"),
+ Path::new("/d/pyo"),
+ Path::new("/d/resolv.conf"),
+ );
+ let ns = s["linux"]["namespaces"].as_array().unwrap();
+ assert!(ns.contains(&json!({"type": "network"})));
+ assert!(
+ !s["mounts"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .any(|x| x["destination"] == "/etc/resolv.conf")
+ );
+ assert_eq!(resolv_conf(&dns), "nameserver 1.1.1.1\n");
+ }
+
+ /// envd's side of the door, as dial mode does it: connect, then Hello.
+ async fn dial(sock: &Path) -> tokio::net::UnixStream {
+ use pyo_proto::sandbox::wire::{Frame, Hello, Kind};
+ use tokio::io::AsyncWriteExt;
+ let mut s = tokio::net::UnixStream::connect(sock).await.unwrap();
+ s.write_all(&Frame::json(Kind::Hello, 0, &Hello::ours("0.0.0")).encode())
+ .await
+ .unwrap();
+ s
+ }
+
+ /// What a checkpoint needs: the door closes and every connection it
+ /// gave out ends, also one still in use, and it opens again after.
+ #[tokio::test]
+ async fn the_door_cuts_every_connection_on_close() {
+ use tokio::io::AsyncReadExt;
+ let dir = std::env::temp_dir().join(format!("pyo-gv-door-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let sock = dir.join(HOST_SOCK);
+ let door = Door::open(&dir).unwrap();
+
+ let mut envd_side = dial(&sock).await;
+ let conn = door.accept(Duration::from_secs(2)).await.unwrap();
+ let (_reader, _writer) = conn.into_split();
+
+ door.close().await;
+ assert!(!sock.exists(), "no socket while closed");
+ let mut buf = [0u8; 8];
+ let n = tokio::time::timeout(Duration::from_secs(2), envd_side.read(&mut buf))
+ .await
+ .expect("envd's end sees the cut")
+ .unwrap();
+ assert_eq!(n, 0);
+ assert!(door.accept(Duration::from_millis(50)).await.is_err());
+
+ door.reopen().await.unwrap();
+ let _again = dial(&sock).await;
+ door.accept(Duration::from_secs(2)).await.unwrap();
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn every_command_gets_the_same_flags() {
+ let g = Gvisor::new(
+ "runsc".into(),
+ "/var/lib/pyo-sandbox/runsc".into(),
+ "pyo-sandbox".into(),
+ );
+ let f = g.flags(true);
+ assert!(f.contains(&"--platform=systrap".to_string()));
+ assert!(f.contains(&"--network=sandbox".to_string()));
+ assert!(f.contains(&"--host-uds=open".to_string()));
+ assert!(f.contains(&"--overlay2=none".to_string()));
+ assert!(g.flags(false).contains(&"--network=none".to_string()));
+ }
+}
diff --git a/crates/pyo-sandbox/src/lib.rs b/crates/pyo-sandbox/src/lib.rs
index 8105ce5..fb875ad 100644
--- a/crates/pyo-sandbox/src/lib.rs
+++ b/crates/pyo-sandbox/src/lib.rs
@@ -1,5 +1,6 @@
//! The Pyo Sandbox daemon. It runs on dedicated sandbox servers, outside
-//! clusters, and turns Pyo's commands into Firecracker microVMs. Clients
+//! clusters, and turns Pyo's commands into Firecracker microVMs, or gVisor
+//! sandboxes on servers without KVM. Clients
//! talk to it directly with short-lived tokens Pyo signs, so Pyo isn't in
//! the hot path.
diff --git a/crates/pyo-sandbox/src/main.rs b/crates/pyo-sandbox/src/main.rs
index 479c0b5..e33fef1 100644
--- a/crates/pyo-sandbox/src/main.rs
+++ b/crates/pyo-sandbox/src/main.rs
@@ -9,8 +9,9 @@ use pyo_sandbox::config::Config;
use pyo_sandbox::detect::{self, Inputs};
use pyo_sandbox::edge::Edge;
use pyo_sandbox::firecracker::{Firecracker, Jailer};
+use pyo_sandbox::gvisor::Gvisor;
use pyo_sandbox::manager::Manager;
-use pyo_sandbox::network::Network;
+use pyo_sandbox::network::{LinkKind, Network};
use pyo_sandbox::storage::Layout;
#[derive(Parser)]
@@ -85,6 +86,7 @@ async fn main() -> anyhow::Result<()> {
fc: &s.fc,
layout: &s.layout,
envd: &s.cfg.envd,
+ runsc: &s.cfg.runsc,
})
.await;
// Only checks. `serve` puts the table in place, and reports
@@ -101,19 +103,17 @@ async fn main() -> anyhow::Result<()> {
/// Sandbox networking, when this server can have it, and the check for the
/// report. Taps belong to the jail user, so Firecracker can open them after
-/// dropping root.
+/// dropping root. gVisor sandboxes get a veth pair in a namespace instead.
async fn open_network(s: &Setup, runtime: Runtime) -> (Option, Check) {
- let res = if runtime != Runtime::Firecracker {
- Err(anyhow::anyhow!(
- "Sandbox networking needs Firecracker for now."
- ))
- } else {
- let (uid, gid) = match &s.fc.jailer {
- Some(j) => (j.uid, j.gid),
- // SAFETY: plain getters.
- None => unsafe { (libc::geteuid(), libc::getegid()) },
- };
- Network::open(s.cfg.network.clone(), uid, gid).await
+ let (uid, gid) = match &s.fc.jailer {
+ Some(j) => (j.uid, j.gid),
+ // SAFETY: plain getters.
+ None => unsafe { (libc::geteuid(), libc::getegid()) },
+ };
+ let res = match runtime {
+ Runtime::Firecracker => Network::open(s.cfg.network.clone(), LinkKind::Tap, uid, gid).await,
+ Runtime::Gvisor => Network::open(s.cfg.network.clone(), LinkKind::Veth, uid, gid).await,
+ Runtime::None => Err(anyhow::anyhow!("This server can't run sandboxes.")),
};
match res {
Ok(n) => {
@@ -146,6 +146,7 @@ async fn serve(s: Setup) -> anyhow::Result<()> {
fc: &s.fc,
layout: &s.layout,
envd: &s.cfg.envd,
+ runsc: &s.cfg.runsc,
})
.await;
let (net, check) = open_network(&s, capability.runtime).await;
@@ -158,7 +159,24 @@ async fn serve(s: Setup) -> anyhow::Result<()> {
let server_id = s.cfg.server_id.clone();
let listen = s.cfg.listen;
let edge_cfg = s.cfg.edge.clone();
- let mgr = Arc::new(Manager::new(s.cfg, s.layout, s.fc, capability).with_network(net));
+ // gVisor sandboxes live in the plain sandboxes directory: no jail.
+ let gvisor = (capability.runtime == Runtime::Gvisor).then(|| {
+ Gvisor::new(
+ s.cfg.runsc.clone(),
+ s.cfg.root.join("runsc"),
+ s.cfg.cgroup_parent.clone(),
+ )
+ });
+ let mut layout = s.layout;
+ if gvisor.is_some() {
+ layout.jailed = false;
+ layout.create().await?;
+ }
+ let mgr = Arc::new(
+ Manager::new(s.cfg, layout, s.fc, capability)
+ .with_network(net)
+ .with_gvisor(gvisor),
+ );
let adopted = mgr.adopt().await;
if adopted > 0 {
tracing::info!(adopted, "took over sandboxes from before the restart");
diff --git a/crates/pyo-sandbox/src/manager.rs b/crates/pyo-sandbox/src/manager.rs
index c5eb8d2..0b21a51 100644
--- a/crates/pyo-sandbox/src/manager.rs
+++ b/crates/pyo-sandbox/src/manager.rs
@@ -1,5 +1,6 @@
//! Sandboxes on this server: create, delete, pause, resume, lifetimes and
-//! the report Pyo reads.
+//! the report Pyo reads. Each runs as a Firecracker VM or, on servers
+//! without KVM, under gVisor ([`Instance`]).
//!
//! The daemon enforces lifetimes itself, so sandboxes still end on time
//! while Pyo restarts. Each sandbox's record is written to `sandbox.json`
@@ -22,7 +23,8 @@ use tokio::sync::Mutex;
use crate::capacity::{self, MemorySource};
use crate::config::Config;
use crate::envd;
-use crate::firecracker::{Firecracker, Machine, Nic, SCRATCH, Vm};
+use crate::firecracker::{DirHandle, Firecracker, Machine, Nic, SCRATCH, Vm};
+use crate::gvisor::{self, Container, Gvisor};
use crate::network::{Lease, Network};
use crate::storage::{self, Layout, snapshot_key};
@@ -43,10 +45,46 @@ fn runtime(e: anyhow::Error) -> DaemonError {
err(ErrorCode::Runtime, format!("{e:#}"))
}
+/// What runs a sandbox.
+#[derive(Debug)]
+pub enum Instance {
+ Vm(Vm),
+ Gvisor(Container),
+}
+
+impl Instance {
+ pub fn pid(&self) -> u32 {
+ match self {
+ Instance::Vm(v) => v.pid,
+ Instance::Gvisor(c) => c.pid,
+ }
+ }
+
+ pub fn exited(&mut self) -> bool {
+ match self {
+ Instance::Vm(v) => v.exited(),
+ Instance::Gvisor(c) => c.exited(),
+ }
+ }
+
+ pub async fn kill(&mut self) {
+ match self {
+ Instance::Vm(v) => v.kill().await,
+ Instance::Gvisor(c) => c.kill().await,
+ }
+ }
+}
+
+/// How to reach a sandbox's envd, taken while holding its lock.
+enum Dial {
+ Vsock(DirHandle, PathBuf),
+ Door(Arc),
+}
+
pub struct Sandbox {
pub spec: SandboxSpec,
status: StdMutex,
- vm: Mutex>,
+ vm: Mutex >,
/// The tap and addresses of a sandbox with network. Kept while paused.
pub lease: Option,
}
@@ -81,6 +119,8 @@ pub struct Manager {
pub cfg: Config,
pub layout: Layout,
pub fc: Firecracker,
+ /// runsc, on gVisor servers.
+ pub gv: Option,
capability: StdMutex,
sandboxes: StdMutex>>,
net: Option>,
@@ -103,6 +143,7 @@ impl Manager {
cfg,
layout,
fc,
+ gv: None,
capability: StdMutex::new(capability),
sandboxes: StdMutex::new(HashMap::new()),
net: None,
@@ -112,6 +153,12 @@ impl Manager {
}
}
+ /// runsc, which a gVisor server runs its sandboxes with.
+ pub fn with_gvisor(mut self, gv: Option) -> Manager {
+ self.gv = gv;
+ self
+ }
+
/// Sandbox networking. `adopt` puts its table in place, and the
/// capability report says `network: true` once that worked.
pub fn with_network(mut self, net: Option) -> Manager {
@@ -178,8 +225,16 @@ impl Manager {
})
}
- fn memory_source(&self, id: &str, pid: u32) -> MemorySource {
- MemorySource::for_vm(self.layout.jailed, &self.cfg.cgroup_parent, id, pid)
+ fn memory_source(&self, id: &str, inst: &Instance) -> MemorySource {
+ match inst {
+ Instance::Vm(vm) => {
+ MemorySource::for_vm(self.layout.jailed, &self.cfg.cgroup_parent, id, vm.pid)
+ }
+ // runsc puts the sentry and the gofer in the sandbox's cgroup.
+ Instance::Gvisor(c) => {
+ MemorySource::for_vm(true, &self.cfg.cgroup_parent, &c.id, c.pid)
+ }
+ }
}
pub fn status(&self, id: &str) -> Result {
@@ -188,7 +243,7 @@ impl Manager {
if let Ok(vm) = s.vm.try_lock()
&& let Some(vm) = vm.as_ref()
{
- st.memory_used = self.memory_source(id, vm.pid).read();
+ st.memory_used = self.memory_source(id, vm).read();
}
Ok(st)
}
@@ -268,11 +323,11 @@ impl Manager {
};
return Err(err(ErrorCode::Unsupported, msg));
}
- if cap.runtime == Runtime::Gvisor {
- // TODO(gvisor): run the sandbox under runsc. See gvisor.rs.
+ let on_gvisor = cap.runtime == Runtime::Gvisor;
+ if on_gvisor && self.gv.is_none() {
return Err(err(
ErrorCode::Unsupported,
- "gVisor sandboxes aren't available on this server yet.",
+ "This server has no runsc, so it can't run gVisor sandboxes.",
));
}
let net = match spec.network {
@@ -308,7 +363,8 @@ impl Manager {
}
let key = self.snapshot_key(net.is_some(), spec.cpus, spec.memory_mib, spec.disk_mib);
let snap_dir = self.layout.template_snapshot(&spec.template, &key);
- let from_snapshot = snap_dir.join("vm.snap").exists();
+ // gVisor starts cold: its restores cost more and share no pages.
+ let from_snapshot = !on_gvisor && snap_dir.join("vm.snap").exists();
if self
.sandboxes
.lock()
@@ -340,7 +396,11 @@ impl Manager {
team: spec.team.clone(),
template: spec.template.clone(),
state: SandboxState::Creating,
- runtime: Runtime::Firecracker,
+ runtime: if on_gvisor {
+ Runtime::Gvisor
+ } else {
+ Runtime::Firecracker
+ },
cpus: spec.cpus,
memory_mib: spec.memory_mib,
disk_mib: spec.disk_mib,
@@ -407,15 +467,22 @@ impl Manager {
});
let dir = self.layout.vm_dir(&spec.id);
- let started = if from_snapshot {
+ let started = if on_gvisor {
+ self.gvisor_start(&spec, &dir, &rootfs, lease.as_ref(), false)
+ .await
+ .map(|(c, took)| (Instance::Gvisor(c), took))
+ } else if from_snapshot {
self.start_from_snapshot(&spec, &dir, &rootfs, &snap_dir, nic)
.await
+ .map(|(vm, took)| (Instance::Vm(vm), took))
} else {
- self.cold_start(&spec, &dir, &rootfs, nic).await
+ self.cold_start(&spec, &dir, &rootfs, nic)
+ .await
+ .map(|(vm, took)| (Instance::Vm(vm), took))
};
match started {
Ok((vm, took)) => {
- let pid = vm.pid;
+ let pid = vm.pid();
*vm_slot = Some(vm);
let st = sbx.update(|s| {
let now = crate::now_ms();
@@ -433,7 +500,7 @@ impl Manager {
.lock()
.unwrap_or_else(|p| p.into_inner())
.remove(&spec.id);
- let _ = tokio::fs::remove_dir_all(self.layout.sandbox_top(&spec.id)).await;
+ self.remove_files(&spec.id).await;
give_back(lease).await;
tracing::warn!(id = %spec.id, error = %format!("{e:#}"), "sandbox didn't start");
Err(runtime(e))
@@ -441,6 +508,66 @@ impl Manager {
}
}
+ /// Starts, or with `resume` restores, a sandbox under runsc.
+ async fn gvisor_start(
+ &self,
+ spec: &SandboxSpec,
+ dir: &Path,
+ rootfs: &Path,
+ lease: Option<&Lease>,
+ resume: bool,
+ ) -> anyhow::Result<(Container, Duration)> {
+ let gv = self
+ .gv
+ .as_ref()
+ .ok_or_else(|| anyhow::anyhow!("this server has no runsc"))?;
+ tokio::fs::create_dir_all(dir).await?;
+ let template_root = gv.template_root(rootfs).await?;
+ let scratch = if resume {
+ None
+ } else {
+ Some(storage::empty_ext4(&self.layout.cache(), spec.disk_mib).await?)
+ };
+ let netns = lease
+ .zip(self.net.as_ref())
+ .and_then(|(l, n)| n.netns_path(l));
+ let dns = self
+ .net
+ .as_ref()
+ .map(|n| n.cfg.dns.clone())
+ .unwrap_or_default();
+ let id = storage::instance_id(&spec.id);
+ let m = gvisor::Machine {
+ id: &id,
+ dir,
+ template_root: &template_root,
+ scratch: scratch.as_deref(),
+ cpus: spec.cpus,
+ memory_mib: spec.memory_mib,
+ netns: netns.as_deref(),
+ dns: &dns,
+ cgroup_parent: &gv.cgroup_parent,
+ };
+ if resume {
+ if let (Some(l), Some(n)) = (lease, &self.net) {
+ n.readdress(l).await?;
+ }
+ gv.restore(&m, BOOT_TIMEOUT).await
+ } else {
+ gv.start(&m, BOOT_TIMEOUT).await
+ }
+ }
+
+ /// Removes a sandbox's directory. On gVisor its container and mounts go
+ /// first, so nothing reaches into the template's disk.
+ async fn remove_files(&self, id: &str) {
+ let top = self.layout.sandbox_top(id);
+ if let Some(gv) = &self.gv {
+ gv.teardown(&storage::instance_id(id), &top).await;
+ }
+ let _ = tokio::fs::remove_dir_all(top).await;
+ }
+
pub fn snapshot_key(&self, network: bool, cpus: u32, memory_mib: u32, disk_mib: u32) -> String {
snapshot_key(
&self.capability().firecracker_version,
@@ -603,6 +730,8 @@ impl Manager {
if let Some(mut v) = vm.take() {
v.kill().await;
}
+ // Its mounts and runsc state go before its network namespace.
+ self.remove_files(id).await;
if let (Some(l), Some(n)) = (&sbx.lease, &self.net) {
n.release(l).await;
}
@@ -611,7 +740,6 @@ impl Manager {
.lock()
.unwrap_or_else(|p| p.into_inner())
.remove(id);
- let _ = tokio::fs::remove_dir_all(self.layout.sandbox_top(id)).await;
tracing::info!(id, "sandbox deleted");
Ok(())
}
@@ -631,14 +759,31 @@ impl Manager {
"Only a running sandbox can be paused.",
));
}
- let Some(mut vm) = slot.take() else {
+ let Some(inst) = slot.take() else {
return Err(err(ErrorCode::Conflict, "The sandbox has no VM."));
};
sbx.update(|s| s.state = SandboxState::Pausing);
+ let mut vm = match inst {
+ Instance::Vm(vm) => vm,
+ Instance::Gvisor(mut c) => {
+ return match c.checkpoint().await {
+ Ok(()) => {
+ let st = sbx.update(|s| s.state = SandboxState::Paused);
+ self.persist(&sbx, None);
+ Ok(st)
+ }
+ Err(e) => {
+ *slot = Some(Instance::Gvisor(c));
+ sbx.update(|s| s.state = SandboxState::Running);
+ Err(runtime(e))
+ }
+ };
+ }
+ };
let snap = vm.snapshot(PAUSE).await;
if let Err(e) = snap {
let _ = vm.resume().await;
- *slot = Some(vm);
+ *slot = Some(Instance::Vm(vm));
sbx.update(|s| s.state = SandboxState::Running);
return Err(runtime(e));
}
@@ -661,6 +806,27 @@ impl Manager {
sbx.update(|s| s.state = SandboxState::Resuming);
let dir = self.layout.vm_dir(id);
let rootfs = self.layout.template_rootfs(&sbx.spec.template);
+ if sbx.status().runtime == Runtime::Gvisor {
+ return match self
+ .gvisor_start(&sbx.spec, &dir, &rootfs, sbx.lease.as_ref(), true)
+ .await
+ {
+ Ok((c, took)) => {
+ let pid = c.pid;
+ *slot = Some(Instance::Gvisor(c));
+ let st = sbx.update(|s| {
+ s.state = SandboxState::Running;
+ s.start_ms = Some(took.as_millis() as u32);
+ });
+ self.persist(&sbx, Some(pid));
+ Ok(st)
+ }
+ Err(e) => {
+ sbx.update(|s| s.state = SandboxState::Paused);
+ Err(runtime(e))
+ }
+ };
+ }
let guest = match (&sbx.lease, &self.net) {
(Some(l), Some(n)) => Some(n.guest_network(l)),
_ => None,
@@ -681,7 +847,7 @@ impl Manager {
match self.fc.restore(&m, PAUSE, BOOT_TIMEOUT).await {
Ok((vm, took)) => {
let pid = vm.pid;
- *slot = Some(vm);
+ *slot = Some(Instance::Vm(vm));
// The restored VM has the memory file mapped, so it can go.
for ext in ["snap", "mem"] {
let _ = tokio::fs::remove_file(dir.join(format!("{PAUSE}.{ext}"))).await;
@@ -738,7 +904,7 @@ impl Manager {
return Err(err(ErrorCode::Conflict, "The sandbox has ended."));
}
let st = sbx.update(|s| f(&mut s.ports));
- self.persist(&sbx, vm.as_ref().map(|v| v.pid));
+ self.persist(&sbx, vm.as_ref().map(Instance::pid));
Ok(st)
}
@@ -757,10 +923,14 @@ impl Manager {
/// A new connection to the sandbox's envd.
pub async fn envd(&self, id: &str) -> Result<(Arc, envd::Conn)> {
let sbx = self.find(id)?;
- let (_pin, vsock) = {
+ let dial = {
let vm = sbx.vm.lock().await;
match (sbx.status().state, vm.as_ref()) {
- (SandboxState::Running, Some(vm)) => vm.vsock_pinned().map_err(runtime)?,
+ (SandboxState::Running, Some(Instance::Vm(vm))) => {
+ let (pin, path) = vm.vsock_pinned().map_err(runtime)?;
+ Dial::Vsock(pin, path)
+ }
+ (SandboxState::Running, Some(Instance::Gvisor(c))) => Dial::Door(c.door.clone()),
(SandboxState::Paused, _) => {
return Err(err(
ErrorCode::Conflict,
@@ -770,9 +940,11 @@ impl Manager {
_ => return Err(err(ErrorCode::Conflict, "The sandbox isn't running.")),
}
};
- let conn = envd::Conn::connect(&vsock, ENVD_TIMEOUT)
- .await
- .map_err(runtime)?;
+ let conn = match dial {
+ Dial::Vsock(_pin, vsock) => envd::Conn::connect(&vsock, ENVD_TIMEOUT).await,
+ Dial::Door(door) => door.accept(ENVD_TIMEOUT).await,
+ }
+ .map_err(runtime)?;
Ok((sbx, conn))
}
@@ -878,11 +1050,23 @@ impl Manager {
let rec: Option = std::fs::read(dir.join("sandbox.json"))
.ok()
.and_then(|b| serde_json::from_slice(&b).ok());
+ // A sandbox of the other runtime can't be kept: KVM came or went.
+ let runtime_now = self.capability().runtime;
let keep = match &rec {
+ Some(r) if r.status.runtime != runtime_now => None,
Some(r) if r.status.state == SandboxState::Paused => Some(None),
+ Some(r) if runtime_now == Runtime::Gvisor => match &self.gv {
+ Some(gv) => gv
+ .adopt(&name, &dir, r.lease.is_some())
+ .await
+ .map(|c| Some(Instance::Gvisor(c))),
+ None => None,
+ },
Some(r) => match r.pid {
Some(pid) if crate::firecracker::is_firecracker(pid) => {
- Vm::adopt(&r.spec.id, &dir, pid).ok().map(Some)
+ Vm::adopt(&r.spec.id, &dir, pid)
+ .ok()
+ .map(|vm| Some(Instance::Vm(vm)))
}
_ => None,
},
@@ -907,6 +1091,9 @@ impl Manager {
adopted += 1;
}
_ => {
+ if let Some(gv) = &self.gv {
+ gv.teardown(&name, &self.layout.sandbox_top(&name)).await;
+ }
let _ = tokio::fs::remove_dir_all(self.layout.sandbox_top(&name)).await;
}
}
diff --git a/crates/pyo-sandbox/src/network.rs b/crates/pyo-sandbox/src/network.rs
index 84c7db2..8f136aa 100644
--- a/crates/pyo-sandbox/src/network.rs
+++ b/crates/pyo-sandbox/src/network.rs
@@ -1,6 +1,7 @@
-//! Sandbox networking on Firecracker: a tap device per VM, an address per
-//! VM from a private pool, NAT out of the host's default route, and one
-//! nftables table the daemon owns.
+//! Sandbox networking: a tap device per VM on Firecracker, or a network
+//! namespace with a veth pair per sandbox on gVisor. An address per
+//! sandbox from a private pool, NAT out of the host's default route, and
+//! one nftables table the daemon owns.
//!
//! ## Addresses
//!
@@ -57,8 +58,16 @@
//!
//! The daemon also turns on IPv4 forwarding, and turns off IPv6 on taps.
//!
-//! gVisor servers will get a network namespace per sandbox with a veth
-//! pair, behind the same table (gVisor spike, item 22).
+//! ## gVisor
+//!
+//! On gVisor the lease's link is a veth pair instead of a tap. Its host end
+//! has the tap's name (`pyosbx`) and the host address, so every rule
+//! above holds as it is. The other end is `eth0` in the network namespace
+//! `pyosbx`, with the guest address and the default route already set.
+//! runsc's netstack takes them from there (`--network=sandbox`), so nothing
+//! is set inside the sandbox. runsc moves them out of the namespace as it
+//! takes them, so a restore puts them back first (`readdress`). The
+//! namespace goes with the lease.
use std::collections::BTreeSet;
use std::fmt;
@@ -80,6 +89,16 @@ pub const GUEST_IFACE: &str = "eth0";
/// The MAC every guest gets. Each tap is its own link, so they can share.
pub const GUEST_MAC: &str = "06:00:c6:12:00:02";
+/// What a lease's link is.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LinkKind {
+ /// A tap Firecracker opens.
+ Tap,
+ /// A veth pair whose other end is in the lease's network namespace
+ /// (gVisor).
+ Veth,
+}
+
/// Ranges a sandbox never reaches, besides the pool itself.
pub const ALWAYS_BLOCKED: &[&str] = &[
"0.0.0.0/8",
@@ -258,6 +277,7 @@ impl Lease {
pub struct Network {
pub cfg: NetworkConfig,
+ pub link: LinkKind,
/// Who owns the taps: the jail user, so Firecracker can open them after
/// it dropped root.
tap_uid: u32,
@@ -269,7 +289,12 @@ impl Network {
/// Checks the host can do sandbox networking and the pool is free.
/// Changes nothing on the host. The table and IPv4 forwarding come with
/// [`Network::sync`].
- pub async fn open(cfg: NetworkConfig, tap_uid: u32, tap_gid: u32) -> anyhow::Result {
+ pub async fn open(
+ cfg: NetworkConfig,
+ link: LinkKind,
+ tap_uid: u32,
+ tap_gid: u32,
+ ) -> anyhow::Result {
if cfg.disabled {
bail!("Sandbox networking is turned off in the config.");
}
@@ -290,7 +315,7 @@ impl Network {
bail!("Sandbox networking needs `{tool}`, and it isn't installed.");
}
}
- if !std::path::Path::new("/dev/net/tun").exists() {
+ if link == LinkKind::Tap && !std::path::Path::new("/dev/net/tun").exists() {
bail!("Sandbox networking needs /dev/net/tun.");
}
let routes = run_out("ip", &["-j", "-4", "route", "show", "table", "all"]).await?;
@@ -302,6 +327,7 @@ impl Network {
}
Ok(Network {
cfg,
+ link,
tap_uid,
tap_gid,
used: Mutex::new(BTreeSet::new()),
@@ -328,6 +354,11 @@ impl Network {
lease.guest_network(&self.cfg.dns)
}
+ /// The lease's network namespace on gVisor, as runsc takes it.
+ pub fn netns_path(&self, lease: &Lease) -> Option {
+ (self.link == LinkKind::Veth).then(|| format!("{NETNS_DIR}/{}", lease.tap))
+ }
+
fn used(&self) -> std::sync::MutexGuard<'_, BTreeSet> {
self.used.lock().unwrap_or_else(|p| p.into_inner())
}
@@ -372,6 +403,11 @@ impl Network {
{
tracing::warn!(tap = %lease.tap, error = %format!("{e:#}"), "can't remove the tap");
}
+ if netns_exists(&lease.tap)
+ && let Err(e) = run("ip", &["netns", "del", &lease.tap]).await
+ {
+ tracing::warn!(netns = %lease.tap, error = %format!("{e:#}"), "can't remove the network namespace");
+ }
self.used().remove(&lease.index);
}
@@ -379,6 +415,9 @@ impl Network {
if tap_exists(&lease.tap) {
run("ip", &["link", "del", &lease.tap]).await?;
}
+ if self.link == LinkKind::Veth {
+ return self.make_veth(lease).await;
+ }
let (uid, gid) = (self.tap_uid.to_string(), self.tap_gid.to_string());
run(
"ip",
@@ -396,6 +435,92 @@ impl Network {
run("ip", &["link", "set", &lease.tap, "up"]).await
}
+ /// Gives the namespace's `eth0` its address and default route back.
+ /// runsc takes them out of the namespace when it starts a sandbox, and
+ /// `runsc restore` skips an `eth0` without an address, so a resumed
+ /// sandbox would have no network. Run before each restore.
+ pub async fn readdress(&self, lease: &Lease) -> anyhow::Result<()> {
+ if self.link != LinkKind::Veth {
+ return Ok(());
+ }
+ let ns = lease.tap.as_str();
+ let guest = format!("{}/30", lease.guest);
+ let gw = lease.host.to_string();
+ let steps: [&[&str]; 2] = [
+ &["addr", "replace", &guest, "dev", GUEST_IFACE],
+ &["route", "replace", "default", "via", &gw],
+ ];
+ for step in steps {
+ let mut args = vec!["-n", ns];
+ args.extend_from_slice(step);
+ run("ip", &args)
+ .await
+ .context("giving the sandbox its address back")?;
+ }
+ Ok(())
+ }
+
+ /// The lease's namespace with `eth0` in it, and the host end of the
+ /// pair named like a tap. See "gVisor" above.
+ async fn make_veth(&self, lease: &Lease) -> anyhow::Result<()> {
+ let ns = lease.tap.as_str();
+ if netns_exists(ns) {
+ run("ip", &["netns", "del", ns]).await?;
+ }
+ run("ip", &["netns", "add", ns]).await?;
+ let made = async {
+ run(
+ "ip",
+ &[
+ "link",
+ "add",
+ &lease.tap,
+ "type",
+ "veth",
+ "peer",
+ "name",
+ GUEST_IFACE,
+ "netns",
+ ns,
+ ],
+ )
+ .await?;
+ let _ = std::fs::write(
+ format!("/proc/sys/net/ipv6/conf/{}/disable_ipv6", lease.tap),
+ "1",
+ );
+ let host = format!("{}/30", lease.host);
+ let guest = format!("{}/30", lease.guest);
+ let gw = lease.host.to_string();
+ run("ip", &["addr", "add", &host, "dev", &lease.tap]).await?;
+ run("ip", &["link", "set", &lease.tap, "up"]).await?;
+ let steps: [&[&str]; 5] = [
+ &["sysctl", "-qw", "net.ipv6.conf.all.disable_ipv6=1"],
+ &["ip", "addr", "add", &guest, "dev", GUEST_IFACE],
+ &["ip", "link", "set", GUEST_IFACE, "address", GUEST_MAC, "up"],
+ &["ip", "link", "set", "lo", "up"],
+ &["ip", "route", "add", "default", "via", &gw],
+ ];
+ for step in steps {
+ let mut args = vec!["netns", "exec", ns];
+ args.extend_from_slice(step);
+ // sysctl may be missing. The rest may not fail.
+ if let Err(e) = run("ip", &args).await
+ && step[0] != "sysctl"
+ {
+ return Err(e);
+ }
+ }
+ anyhow::Ok(())
+ }
+ .await;
+ if made.is_err() {
+ let _ = run("ip", &["link", "del", &lease.tap]).await;
+ let _ = run("ip", &["netns", "del", ns]).await;
+ }
+ made
+ }
+
/// Puts the table in place for the sandboxes a restarted daemon kept,
/// in one atomic step. Each lease comes with whether its VM runs. Makes
/// taps that went missing for paused sandboxes, and removes taps no
@@ -424,6 +549,10 @@ impl Network {
tracing::info!(%tap, "removing a tap no sandbox owns");
let _ = run("ip", &["link", "del", &tap]).await;
}
+ for ns in stray_netns(&dir_entries(NETNS_DIR), &ls) {
+ tracing::info!(%ns, "removing a network namespace no sandbox owns");
+ let _ = run("ip", &["netns", "del", &ns]).await;
+ }
Ok(())
}
@@ -734,6 +863,35 @@ fn tap_exists(tap: &str) -> bool {
std::path::Path::new("/sys/class/net").join(tap).exists()
}
+/// Where `ip netns` keeps named namespaces.
+pub const NETNS_DIR: &str = "/var/run/netns";
+
+fn netns_exists(name: &str) -> bool {
+ std::path::Path::new(NETNS_DIR).join(name).exists()
+}
+
+fn dir_entries(dir: &str) -> Vec {
+ std::fs::read_dir(dir)
+ .into_iter()
+ .flatten()
+ .flatten()
+ .map(|e| e.file_name().to_string_lossy().into_owned())
+ .collect()
+}
+
+/// Namespaces named like the daemon's that no lease owns.
+pub fn stray_netns(names: &[String], leases: &[Lease]) -> Vec {
+ names
+ .iter()
+ .filter(|n| {
+ n.strip_prefix(TAP_PREFIX)
+ .is_some_and(|i| !i.is_empty() && i.bytes().all(|b| b.is_ascii_digit()))
+ })
+ .filter(|n| !leases.iter().any(|l| &l.tap == *n))
+ .cloned()
+ .collect()
+}
+
async fn nft(script: &str) -> anyhow::Result<()> {
let mut child = Command::new("nft")
.args(["-f", "-"])
@@ -780,6 +938,7 @@ mod tests {
fn net() -> Network {
Network {
cfg: NetworkConfig::default(),
+ link: LinkKind::Tap,
tap_uid: 1,
tap_gid: 1,
used: Mutex::new(BTreeSet::new()),
@@ -850,6 +1009,21 @@ mod tests {
stray_taps(links, &[n.lease_at(0)]).unwrap(),
vec!["pyosbx7".to_string()]
);
+ let names: Vec = ["pyosbx0", "pyosbx3", "pyosbxx", "cni-1"]
+ .map(String::from)
+ .to_vec();
+ assert_eq!(stray_netns(&names, &[n.lease_at(0)]), vec!["pyosbx3"]);
+ }
+
+ #[test]
+ fn gvisor_leases_get_a_namespace() {
+ let mut n = net();
+ assert_eq!(n.netns_path(&n.lease_at(4)), None, "taps have none");
+ n.link = LinkKind::Veth;
+ assert_eq!(
+ n.netns_path(&n.lease_at(4)).as_deref(),
+ Some("/var/run/netns/pyosbx4")
+ );
}
#[test]
diff --git a/crates/pyo-sandbox/src/oci.rs b/crates/pyo-sandbox/src/oci.rs
index be1605b..627da25 100644
--- a/crates/pyo-sandbox/src/oci.rs
+++ b/crates/pyo-sandbox/src/oci.rs
@@ -63,17 +63,23 @@ impl ImageRef {
if image.is_empty() || image.len() > 512 || image.contains(char::is_whitespace) {
bail!("{image:?} isn't an image reference.");
}
+ // A tag's colon comes after the last slash. A colon before it is a
+ // registry port.
+ let untagged = |n: &str| -> (usize, Option) {
+ let slash = n.rfind('/').map(|i| i + 1).unwrap_or(0);
+ (slash, n[slash..].rfind(':'))
+ };
let (name, reference) = match image.split_once('@') {
- Some((n, d)) => (n, d.to_string()),
- None => {
- // A tag's colon comes after the last slash. A colon before
- // it is a registry port.
- let slash = image.rfind('/').map(|i| i + 1).unwrap_or(0);
- match image[slash..].rfind(':') {
- Some(i) => (&image[..slash + i], image[slash + i + 1..].to_string()),
- None => (image, "latest".to_string()),
- }
- }
+ // A digest wins over a tag beside it (`repo:1@sha256:…`), as in
+ // Docker. Pyo's release pins its images that way.
+ Some((n, d)) => match untagged(n) {
+ (slash, Some(i)) => (&n[..slash + i], d.to_string()),
+ _ => (n, d.to_string()),
+ },
+ None => match untagged(image) {
+ (slash, Some(i)) => (&image[..slash + i], image[slash + i + 1..].to_string()),
+ _ => (image, "latest".to_string()),
+ },
};
let (registry, repository) = match name.split_once('/') {
Some((first, rest))
@@ -113,6 +119,8 @@ impl ImageRef {
}
let ref_ok = if let Some(hex) = reference.strip_prefix("sha256:") {
hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())
+ } else if image.contains('@') {
+ false
} else {
!reference.is_empty()
&& reference.len() <= 128
@@ -674,6 +682,19 @@ mod tests {
ImageRef::parse(&format!("100.64.0.1:5006/acme/app/api@{d}")).unwrap(),
r("100.64.0.1:5006", "acme/app/api", &d)
);
+ // As Pyo's release pins its images: the digest counts, not the tag.
+ assert_eq!(
+ ImageRef::parse(&format!("ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@{d}")).unwrap(),
+ r("ghcr.io", "upsetdev/pyo/sandbox-base", &d)
+ );
+ assert_eq!(
+ ImageRef::parse(&format!("127.0.0.1:5000/pyo/x:e2e@{d}")).unwrap(),
+ r("127.0.0.1:5000", "pyo/x", &d)
+ );
+ assert_eq!(
+ ImageRef::parse(&format!("alpine:3.21@{d}")).unwrap(),
+ r(hub, "library/alpine", &d)
+ );
assert_eq!(
ImageRef::parse("localhost:5000/x").unwrap(),
r("localhost:5000", "x", "latest")
diff --git a/crates/pyo-sandbox/src/templates.rs b/crates/pyo-sandbox/src/templates.rs
index ea69a65..7e925be 100644
--- a/crates/pyo-sandbox/src/templates.rs
+++ b/crates/pyo-sandbox/src/templates.rs
@@ -2,7 +2,9 @@
//!
//! A build pulls the image ([`crate::oci`]), unpacks its layers into a root
//! disk with `pyo-envd` at `/sbin/pyo-envd` ([`crate::rootfs`]), then boots
-//! it once and snapshots it per network mode at the default size. Pyo only
+//! it once and snapshots it per network mode at the default size. gVisor
+//! servers make the same disk and stop there: their sandboxes start cold
+//! from it, mounted read-only (see `gvisor`). Pyo only
//! says which image and gives a registry login. It reads the progress from
//! the report:
//!
@@ -208,10 +210,10 @@ impl Manager {
}
let image_ref =
ImageRef::parse(image).map_err(|e| err(ErrorCode::InvalidSpec, format!("{e:#}")))?;
- if self.capability().runtime != Runtime::Firecracker {
+ if self.capability().runtime == Runtime::None {
return Err(err(
ErrorCode::Unsupported,
- "This server can't make templates: it needs Firecracker for now.",
+ "This server can't run sandboxes, so it makes no templates.",
));
}
if let Some(r) = self.template_report(name) {
@@ -407,6 +409,11 @@ impl Manager {
r.rootfs_bytes = rootfs_bytes;
});
+ // gVisor starts every sandbox cold from the disk: nothing to snapshot.
+ if self.capability().runtime == Runtime::Gvisor {
+ return Ok(());
+ }
+
// Boot it once per network mode and snapshot it.
stopped()?;
b.set(|r| r.phase = Some("snapshotting".into()));
@@ -475,6 +482,8 @@ impl Manager {
format!("The sandbox template {name} isn't on this server."),
));
}
+ // gVisor servers have its disk mounted.
+ crate::gvisor::unmount_below(&dir).await;
tokio::fs::remove_dir_all(&dir)
.await
.map_err(|e| err(ErrorCode::Internal, e.to_string()))?;
diff --git a/crates/pyo-sandbox/tests/e2e.rs b/crates/pyo-sandbox/tests/e2e.rs
index 863897d..2d187ee 100644
--- a/crates/pyo-sandbox/tests/e2e.rs
+++ b/crates/pyo-sandbox/tests/e2e.rs
@@ -791,6 +791,7 @@ async fn boots_real_microvms() {
jailer: None,
kernel: kernel.clone(),
envd: envd.clone(),
+ runsc: "runsc".into(),
keys: vec![keys.key.verifying_key()],
host_reserve_mib: Some(1024),
memory_overcommit: 1.0,
@@ -823,6 +824,7 @@ async fn boots_real_microvms() {
fc: &fc,
layout: &layout,
envd: &envd,
+ runsc: std::path::Path::new("runsc"),
})
.await;
note("detect_total_ms", (t.elapsed().as_millis() as u64).into());
diff --git a/crates/pyo-sandbox/tests/gvisor_e2e.rs b/crates/pyo-sandbox/tests/gvisor_e2e.rs
new file mode 100644
index 0000000..0451864
--- /dev/null
+++ b/crates/pyo-sandbox/tests/gvisor_e2e.rs
@@ -0,0 +1,470 @@
+//! Sandboxes under gVisor (runsc), end to end. Runs only with
+//! `PYO_SANDBOX_GVISOR_E2E=1` as root, so plain `cargo test` skips it. It
+//! mounts disks, makes network namespaces and the `pyo_sandbox` nftables
+//! table, so run it in a throwaway VM or a CI runner, never on a laptop or a
+//! real server:
+//!
+//! ```text
+//! crates/pyo-sandbox/scripts/gvisor-assets.sh # runsc, checked, into .cache/
+//! cargo build --release -p pyo-envd --target x86_64-unknown-linux-musl
+//! cargo test -p pyo-sandbox --test gvisor_e2e --no-run
+//! # as root:
+//! PYO_SANDBOX_GVISOR_E2E=1 PYO_SANDBOX_RUNSC=... PYO_SANDBOX_ENVD=... ./gvisor_e2e- --nocapture
+//! ```
+//!
+//! The template is the host's static busybox plus envd. Firecracker is
+//! given a path that doesn't exist, so detection falls back to gVisor even
+//! on a host with KVM, like a cloud server without it would.
+//!
+//! What it checks: detection picks gVisor and names the runsc release.
+//! Exec, env, files, a port through envd, the disk cap and memory from the
+//! sandbox cgroup. Internet with the host and cloud metadata blocked, and
+//! "no network" with no interface. Pause and resume keep files and
+//! processes. A daemon restart adopts running and paused sandboxes. The
+//! lifetime ends a sandbox. Delete leaves no mount, namespace or runsc
+//! state behind.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::Arc;
+use std::time::Duration;
+
+use pyo_proto::sandbox::wire::Cmd;
+use pyo_proto::sandbox::{
+ ExecRequest, ExecResult, NetworkMode, Runtime, SandboxSpec, SandboxState, sandbox_id_from,
+};
+use pyo_sandbox::config::Config;
+use pyo_sandbox::detect::{self, Inputs};
+use pyo_sandbox::firecracker::Firecracker;
+use pyo_sandbox::gvisor::{self, Gvisor};
+use pyo_sandbox::manager::Manager;
+use pyo_sandbox::network::{LinkKind, NETNS_DIR, Network, NetworkConfig, TABLE, TAP_PREFIX};
+use pyo_sandbox::storage::{self, Layout};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+const TEAM: &str = "team-gvisor";
+
+fn enabled() -> bool {
+ if std::env::var("PYO_SANDBOX_GVISOR_E2E").as_deref() != Ok("1") {
+ eprintln!("skipping: set PYO_SANDBOX_GVISOR_E2E=1, as root in a throwaway VM");
+ return false;
+ }
+ if !pyo_sandbox::config::is_root() {
+ eprintln!("skipping: runsc needs root");
+ return false;
+ }
+ true
+}
+
+fn env_path(var: &str) -> PathBuf {
+ let p = PathBuf::from(std::env::var_os(var).unwrap_or_else(|| panic!("{var} isn't set")));
+ assert!(p.exists(), "{var}: {} is missing", p.display());
+ p
+}
+
+fn sh(cmd: &str) -> (bool, String) {
+ let out = Command::new("sh").args(["-c", cmd]).output().unwrap();
+ let text = format!(
+ "{}{}",
+ String::from_utf8_lossy(&out.stdout),
+ String::from_utf8_lossy(&out.stderr)
+ );
+ (out.status.success(), text)
+}
+
+fn must(cmd: &str) -> String {
+ let (ok, out) = sh(cmd);
+ assert!(ok, "{cmd}: {out}");
+ out
+}
+
+fn host_ip() -> String {
+ let out = must("ip -4 -o route get 1.1.1.1");
+ out.split_whitespace()
+ .skip_while(|w| *w != "src")
+ .nth(1)
+ .expect("a src address")
+ .to_string()
+}
+
+fn netns() -> Vec {
+ std::fs::read_dir(NETNS_DIR)
+ .into_iter()
+ .flatten()
+ .flatten()
+ .map(|e| e.file_name().to_string_lossy().into_owned())
+ .filter(|n| n.starts_with(TAP_PREFIX))
+ .collect()
+}
+
+fn mounts_under(dir: &Path) -> Vec {
+ let info = std::fs::read_to_string("/proc/self/mountinfo").unwrap();
+ gvisor::mounts_below(&gvisor::mount_points(&info), dir)
+}
+
+/// A template disk: the host's static busybox and envd.
+async fn busybox_template(layout: &Layout, envd: &Path) {
+ let dir = layout.root.join("template-src");
+ let _ = std::fs::remove_dir_all(&dir);
+ for d in [
+ "bin", "sbin", "proc", "sys", "dev", "tmp", "run", "root", "mnt", "etc", "www",
+ ] {
+ std::fs::create_dir_all(dir.join(d)).unwrap();
+ }
+ std::fs::copy("/usr/bin/busybox", dir.join("bin/busybox")).unwrap();
+ let list = Command::new("/usr/bin/busybox")
+ .arg("--list")
+ .output()
+ .unwrap();
+ for applet in String::from_utf8_lossy(&list.stdout).lines() {
+ let p = dir.join("bin").join(applet);
+ if applet != "busybox" && !p.exists() {
+ std::os::unix::fs::symlink("busybox", p).unwrap();
+ }
+ }
+ std::fs::copy(envd, dir.join("sbin/pyo-envd")).unwrap();
+ std::fs::write(dir.join("etc/passwd"), "root:x:0:0:root:/root:/bin/sh\n").unwrap();
+ std::fs::write(dir.join("www/index.html"), "hello from gvisor\n").unwrap();
+ let out = layout.template_rootfs("busybox");
+ std::fs::create_dir_all(out.parent().unwrap()).unwrap();
+ storage::ext4_from_dir(&dir, &out, 64, "busybox")
+ .await
+ .unwrap();
+ std::fs::remove_dir_all(&dir).unwrap();
+}
+
+/// Ends this run's sandboxes and takes its mounts, namespaces and table
+/// away, also when the test fails.
+struct Cleanup {
+ root: PathBuf,
+ runsc: PathBuf,
+}
+
+impl Drop for Cleanup {
+ fn drop(&mut self) {
+ let state = self.root.join("runsc");
+ let _ = sh(&format!(
+ "for c in $({r} --root={s} list --quiet 2>/dev/null); do {r} --root={s} delete --force $c; done",
+ r = self.runsc.display(),
+ s = state.display()
+ ));
+ // Its own thread: a failing test drops this inside the test's
+ // runtime, where block_on panics.
+ let root = self.root.clone();
+ let _ = std::thread::spawn(move || {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(gvisor::unmount_below(&root));
+ })
+ .join();
+ for ns in netns() {
+ let _ = sh(&format!("ip netns del {ns}"));
+ }
+ let _ = sh(&format!(
+ "for l in $(ls /sys/class/net | grep '^{TAP_PREFIX}'); do ip link del $l; done"
+ ));
+ let _ = sh(&format!("nft delete table inet {TABLE}"));
+ let _ = std::fs::remove_dir_all(&self.root);
+ }
+}
+
+struct Env {
+ cfg: Config,
+ layout: Layout,
+ fc: Firecracker,
+ envd: PathBuf,
+}
+
+impl Env {
+ /// What the daemon does on start: detect, open the network, adopt.
+ async fn daemon(&self) -> Arc {
+ let mut cap = detect::detect(Inputs {
+ fc: &self.fc,
+ layout: &self.layout,
+ envd: &self.envd,
+ runsc: &self.cfg.runsc,
+ })
+ .await;
+ assert_eq!(cap.runtime, Runtime::Gvisor, "{}", cap.summary());
+ assert!(
+ cap.gvisor_version.starts_with("release-"),
+ "{}",
+ cap.gvisor_version
+ );
+ let net = Network::open(self.cfg.network.clone(), LinkKind::Veth, 0, 0)
+ .await
+ .expect("networking opens");
+ cap.network = false;
+ let gv = Gvisor::new(
+ self.cfg.runsc.clone(),
+ self.cfg.root.join("runsc"),
+ self.cfg.cgroup_parent.clone(),
+ );
+ let mgr = Manager::new(self.cfg.clone(), self.layout.clone(), self.fc.clone(), cap)
+ .with_network(Some(net))
+ .with_gvisor(Some(gv));
+ mgr.adopt().await;
+ assert!(mgr.capability().network, "networking is on after adopt");
+ Arc::new(mgr)
+ }
+}
+
+fn spec(n: u8, network: NetworkMode) -> SandboxSpec {
+ let mut s = SandboxSpec::new(sandbox_id_from([n; 16]), TEAM, "busybox");
+ s.memory_mib = 256;
+ s.disk_mib = 128;
+ s.network = network;
+ s.env = BTreeMap::from([("GREETING".into(), "hi".into())]);
+ s
+}
+
+async fn exec(mgr: &Manager, id: &str, cmd: &str) -> ExecResult {
+ mgr.exec(
+ id,
+ &ExecRequest {
+ cmd: Cmd::Shell(cmd.into()),
+ cwd: None,
+ env: BTreeMap::new(),
+ stdin: None,
+ timeout_ms: Some(30_000),
+ },
+ )
+ .await
+ .unwrap_or_else(|e| panic!("{cmd}: {}", e.message))
+}
+
+async fn ok(mgr: &Manager, id: &str, cmd: &str) -> String {
+ let r = exec(mgr, id, cmd).await;
+ assert_eq!(r.exit_code, Some(0), "{cmd}: {} {}", r.stdout, r.stderr);
+ r.stdout
+}
+
+async fn connects(mgr: &Manager, id: &str, addr: &str, port: u16) -> bool {
+ let r = exec(mgr, id, &format!("nc -w 3 {addr} {port} &2; exit 3").await;
+ assert_eq!((r.exit_code, r.stderr.as_str()), (Some(3), "oops\n"));
+ // gVisor's own kernel answers, not the host's.
+ let dmesg = ok(&mgr, &a.id, "dmesg 2>/dev/null | head -3; true").await;
+ println!("gvisor dmesg: {dmesg}");
+ assert_eq!(
+ ok(&mgr, &a.id, "ls /sys/class/net | grep -v '^lo$' | wc -l")
+ .await
+ .trim(),
+ "0",
+ "no network means no interface"
+ );
+ {
+ let (_, mut c) = mgr.envd(&a.id).await.unwrap();
+ c.write_file("/root/note.txt", b"kept", None).await.unwrap();
+ assert_eq!(c.read_file("/root/note.txt", 1024).await.unwrap(), b"kept");
+ }
+ // The template's disk is shared and read-only. Writes land in the
+ // sandbox's own layer.
+ ok(&mgr, &a.id, "echo changed > /www/index.html").await;
+ let tmpl_root = layout.template_dir("busybox").join("root");
+ assert_eq!(
+ std::fs::read_to_string(tmpl_root.join("www/index.html")).unwrap(),
+ "hello from gvisor\n"
+ );
+ // The disk is 128 MiB, so 200 MiB can't fit.
+ let r = exec(
+ &mgr,
+ &a.id,
+ "dd if=/dev/zero of=/root/big bs=1M count=200 2>&1; echo rc=$?",
+ )
+ .await;
+ assert!(
+ !r.stdout.contains("rc=0"),
+ "the disk cap holds: {}",
+ r.stdout
+ );
+ ok(&mgr, &a.id, "rm -f /root/big").await;
+ let used = mgr.status(&a.id).unwrap().memory_used;
+ println!("gvisor memory.current: {used:?}");
+ assert!(used.is_some_and(|m| m > 0), "memory comes from the cgroup");
+
+ // ── A port, through envd like the edge proxy does it ──
+ ok(&mgr, &a.id, "httpd -p 8080 -h /www").await;
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ {
+ let (_, c) = mgr.envd(&a.id).await.unwrap();
+ let mut s = c.port_stream(8080).await.unwrap();
+ s.write_all(b"GET /index.html HTTP/1.0\r\n\r\n")
+ .await
+ .unwrap();
+ let mut body = String::new();
+ tokio::time::timeout(Duration::from_secs(10), s.read_to_string(&mut body))
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(body.contains("changed"), "{body}");
+ }
+
+ // ── Pause and resume: files and processes stay ──
+ // No nohup: Ubuntu's busybox-static doesn't have it.
+ ok(&mgr, &a.id, "sleep 1000 >/dev/null 2>&1 &").await;
+ let st = mgr.pause(&a.id).await.unwrap();
+ assert_eq!(st.state, SandboxState::Paused);
+ assert!(
+ mgr.exec(
+ &a.id,
+ &ExecRequest {
+ cmd: Cmd::Shell("true".into()),
+ cwd: None,
+ env: BTreeMap::new(),
+ stdin: None,
+ timeout_ms: None
+ }
+ )
+ .await
+ .is_err(),
+ "a paused sandbox doesn't run commands"
+ );
+ let st = mgr.resume(&a.id).await.unwrap();
+ assert_eq!(st.state, SandboxState::Running);
+ println!("gvisor resume: {:?} ms", st.start_ms);
+ assert_eq!(ok(&mgr, &a.id, "cat /root/note.txt").await, "kept");
+ assert!(
+ ok(&mgr, &a.id, "ps | grep -c '[s]leep 1000'").await.trim() != "0",
+ "the process survived the pause"
+ );
+
+ // ── Internet, with the host and cloud metadata blocked ──
+ let b = spec(2, NetworkMode::Internet);
+ mgr.create(b.clone()).await.unwrap();
+ assert_eq!(
+ netns().len(),
+ 1,
+ "one namespace for the one networked sandbox"
+ );
+ assert!(
+ connects(&mgr, &b.id, "1.1.1.1", 53).await,
+ "the internet answers"
+ );
+ let resolv = ok(&mgr, &b.id, "cat /etc/resolv.conf").await;
+ assert!(resolv.contains("nameserver"), "{resolv}");
+ assert!(
+ !connects(&mgr, &b.id, &host_ip(), 22).await,
+ "the host is blocked"
+ );
+ assert!(
+ !connects(&mgr, &b.id, "169.254.169.254", 80).await,
+ "cloud metadata is blocked"
+ );
+
+ // ── A daemon restart adopts the running sandboxes ──
+ let paused_c = spec(3, NetworkMode::Internet);
+ mgr.create(paused_c.clone()).await.unwrap();
+ mgr.pause(&paused_c.id).await.unwrap();
+ drop(mgr);
+ let mgr = env.daemon().await;
+ assert_eq!(ok(&mgr, &a.id, "cat /root/note.txt").await, "kept");
+ assert!(connects(&mgr, &b.id, "1.1.1.1", 53).await);
+ assert_eq!(
+ mgr.status(&paused_c.id).unwrap().state,
+ SandboxState::Paused
+ );
+ mgr.resume(&paused_c.id).await.unwrap();
+ assert!(connects(&mgr, &paused_c.id, "1.1.1.1", 53).await);
+
+ // ── The lifetime ends a sandbox ──
+ mgr.set_ttl(&paused_c.id, 1).unwrap();
+ tokio::time::sleep(Duration::from_millis(1500)).await;
+ mgr.sweep().await;
+ assert!(mgr.find(&paused_c.id).is_err(), "its lifetime is over");
+
+ // ── Delete leaves nothing behind ──
+ for x in [&a, &b] {
+ mgr.delete(&x.id).await.unwrap();
+ }
+ let sandboxes = layout.sandboxes();
+ assert!(
+ mounts_under(&sandboxes).is_empty(),
+ "{:?}",
+ mounts_under(&sandboxes)
+ );
+ assert!(netns().is_empty(), "{:?}", netns());
+ assert!(storage::dir_names(&sandboxes).is_empty());
+ mgr.delete_template("busybox").await.unwrap();
+ // runsc keeps a null-netns mount in its own state directory, for
+ // every sandbox to come. It stays.
+ let state = root.join("runsc");
+ let left: Vec<_> = mounts_under(&root)
+ .into_iter()
+ .filter(|m| !m.starts_with(&state))
+ .collect();
+ assert!(left.is_empty(), "{left:?}");
+}
diff --git a/crates/pyo-sandbox/tests/network_e2e.rs b/crates/pyo-sandbox/tests/network_e2e.rs
index d024226..5a2cd59 100644
--- a/crates/pyo-sandbox/tests/network_e2e.rs
+++ b/crates/pyo-sandbox/tests/network_e2e.rs
@@ -38,7 +38,7 @@ use pyo_sandbox::detect::{self, Inputs};
use pyo_sandbox::firecracker::{Firecracker, Jailer};
use pyo_sandbox::manager::Manager;
use pyo_sandbox::network::{
- Network, NetworkConfig, RULE_COMMENT, TABLE, TAP_PREFIX, forward_drops, tagged_rules,
+ LinkKind, Network, NetworkConfig, RULE_COMMENT, TABLE, TAP_PREFIX, forward_drops, tagged_rules,
};
use pyo_sandbox::storage::{self, Layout};
@@ -278,6 +278,7 @@ impl Env {
fc: &self.fc,
layout: &self.layout,
envd: &self.envd,
+ runsc: &self.cfg.runsc,
})
.await;
assert_eq!(cap.runtime, Runtime::Firecracker, "{}", cap.summary());
@@ -285,7 +286,7 @@ impl Env {
Some(j) => (j.uid, j.gid),
None => (0, 0),
};
- let net = Network::open(self.cfg.network.clone(), uid, gid)
+ let net = Network::open(self.cfg.network.clone(), LinkKind::Tap, uid, gid)
.await
.expect("networking opens");
cap.network = false;
@@ -395,6 +396,7 @@ async fn sandbox_networking() {
jailer: jailer.clone(),
kernel: kernel.clone(),
envd: envd.clone(),
+ runsc: "runsc".into(),
keys: vec![],
host_reserve_mib: Some(512),
memory_overcommit: 1.0,
diff --git a/crates/pyo-sandbox/tests/template_e2e.rs b/crates/pyo-sandbox/tests/template_e2e.rs
index ab8ab9c..64e0eac 100644
--- a/crates/pyo-sandbox/tests/template_e2e.rs
+++ b/crates/pyo-sandbox/tests/template_e2e.rs
@@ -112,11 +112,16 @@ fn shell(cmd: &str) -> ExecRequest {
}
}
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn makes_a_template_from_a_public_image() {
- if !enabled() {
- return;
- }
+/// A daemon in dev mode on this machine, its files under `.cache`.
+struct Rig {
+ mgr: Arc,
+ st: AppState,
+ token: String,
+ layout: Layout,
+ _cleanup: Cleanup,
+}
+
+async fn rig(name: &str) -> Rig {
let root = repo_root();
let cache = root.join(".cache/sandbox-e2e");
let target = root.join(
@@ -137,11 +142,9 @@ async fn makes_a_template_from_a_public_image() {
p.display()
);
}
- let image = std::env::var("PYO_SANDBOX_E2E_IMAGE").unwrap_or("python:3.13-alpine".into());
- let python = image.starts_with("python");
- let run = input("PYO_SANDBOX_E2E_RUN", cache.clone())
- .join(format!("tmpl-run-{}", std::process::id()));
+ let run =
+ input("PYO_SANDBOX_E2E_RUN", cache.clone()).join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&run);
let _cleanup = Cleanup(run.clone());
let key = SigningKey::generate("e2e");
@@ -153,6 +156,7 @@ async fn makes_a_template_from_a_public_image() {
jailer: None,
kernel: kernel.clone(),
envd: envd.clone(),
+ runsc: "runsc".into(),
keys: vec![key.verifying_key()],
host_reserve_mib: Some(1024),
memory_overcommit: 1.0,
@@ -177,6 +181,7 @@ async fn makes_a_template_from_a_public_image() {
fc: &fc,
layout: &layout,
envd: &envd,
+ runsc: std::path::Path::new("runsc"),
})
.await;
assert_eq!(cap.runtime, Runtime::Firecracker, "{}", cap.summary());
@@ -188,6 +193,29 @@ async fn makes_a_template_from_a_public_image() {
server_id: SERVER.into(),
};
let token = admin(&key);
+ Rig {
+ mgr,
+ st,
+ token,
+ layout,
+ _cleanup,
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn makes_a_template_from_a_public_image() {
+ if !enabled() {
+ return;
+ }
+ let image = std::env::var("PYO_SANDBOX_E2E_IMAGE").unwrap_or("python:3.13-alpine".into());
+ let python = image.starts_with("python");
+ let Rig {
+ mgr,
+ st,
+ token,
+ layout,
+ _cleanup,
+ } = rig("tmpl-run").await;
let mut summary = serde_json::Map::new();
let mut note = |k: &str, v: serde_json::Value| {
println!("template e2e {k}: {v}");
@@ -393,3 +421,78 @@ fn t_start(t: &TemplateReport) -> Instant {
fn t0(t: &TemplateReport) -> u64 {
pyo_sandbox::now_ms().saturating_sub(t.build.as_ref().map(|b| b.started_at).unwrap_or(0))
}
+
+/// Pyo's built-in template `base` (deploy/images/sandbox-base.Dockerfile),
+/// from the image named by `PYO_SANDBOX_E2E_BASE_IMAGE`: CI builds it and
+/// pushes it to a registry on 127.0.0.1. Skipped without it.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn makes_the_base_template() {
+ if !enabled() {
+ return;
+ }
+ let Ok(image) = std::env::var("PYO_SANDBOX_E2E_BASE_IMAGE") else {
+ eprintln!("skipping: set PYO_SANDBOX_E2E_BASE_IMAGE to the base image to make");
+ return;
+ };
+ let plain_http = image.starts_with("127.0.0.1:") || image.starts_with("localhost:");
+ let Rig {
+ mgr,
+ layout,
+ _cleanup,
+ ..
+ } = rig("base-run").await;
+ let name = "stp_base.e2e";
+ mgr.prepare_template(name, &image, plain_http, None, None)
+ .unwrap();
+ let done = loop {
+ let t = mgr
+ .template_report(name)
+ .expect("the template is in the report");
+ if t.state != TemplateState::Building {
+ break t;
+ }
+ assert!(
+ Instant::now().duration_since(t_start(&t)) < Duration::from_secs(900),
+ "the build took too long"
+ );
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ };
+ let b = done.build.clone().unwrap();
+ assert_eq!(done.state, TemplateState::Ready, "{:?}", b.error);
+ println!(
+ "base e2e: image {} MiB, root disk {} MiB, pull {:?} ms, unpack {:?} ms, snapshot {:?} ms",
+ b.image_bytes >> 20,
+ b.rootfs_bytes >> 20,
+ b.pull_ms,
+ b.unpack_ms,
+ b.snapshot_ms
+ );
+ assert!(
+ done.has_snapshot(NetworkMode::None, 1, 512, 2048),
+ "{done:?}"
+ );
+
+ // A sandbox from it has the tools, as root, with no network.
+ let mut spec = SandboxSpec::new(sandbox_id_from([9; 16]), TEAM, name);
+ spec.network = NetworkMode::None;
+ let s = mgr.create(spec.clone()).await.unwrap();
+ assert!(s.from_snapshot);
+ for (cmd, want) in [
+ ("git --version", "git version 2."),
+ ("python3 --version", "Python 3.13"),
+ ("uv --version", "uv 0."),
+ ("node --version", "v26."),
+ ("pnpm --version", "12."),
+ ("npm --version", ""),
+ ("gcc --version", "gcc"),
+ ("head -1 /etc/os-release", "Debian GNU/Linux 13"),
+ ] {
+ let out = mgr.exec(&spec.id, &shell(cmd)).await.unwrap();
+ println!("base e2e {cmd}: {}", out.stdout.trim());
+ assert_eq!(out.exit_code, Some(0), "{cmd}: {out:?}");
+ assert!(out.stdout.contains(want), "{cmd}: {}", out.stdout);
+ }
+ mgr.delete(&spec.id).await.unwrap();
+ mgr.delete_template(name).await.unwrap();
+ assert!(!layout.template_dir(name).exists());
+}
diff --git a/crates/pyo-static/Cargo.toml b/crates/pyo-static/Cargo.toml
index 0db6897..b68440d 100644
--- a/crates/pyo-static/Cargo.toml
+++ b/crates/pyo-static/Cargo.toml
@@ -11,7 +11,7 @@ anyhow.workspace = true
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json"] }
futures.workspace = true
hex = "0.4.3"
-pyo-proto = { version = "0.4.0", path = "../pyo-proto" }
+pyo-proto = { version = "0.4.1", path = "../pyo-proto" }
reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] }
serde.workspace = true
serde_json.workspace = true
diff --git a/dash/scripts/sandbox-templates.test.ts b/dash/scripts/sandbox-templates.test.ts
new file mode 100644
index 0000000..ca6cb58
--- /dev/null
+++ b/dash/scripts/sandbox-templates.test.ts
@@ -0,0 +1,36 @@
+// Pyo's built-in sandbox template in the dashboard: how its image reads,
+// and that new sandboxes start on it.
+// bun test scripts/sandbox-templates.test.ts
+import { expect, test } from "bun:test";
+import { defaultTemplate, imageRepo, templateImage } from "../src/lib/pyo/sandboxTemplates";
+
+const base = {
+ name: "base",
+ state: "ready",
+ builtin: true,
+ source: {
+ image: "ghcr.io/upsetdev/pyo/sandbox-base:0.4.1@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+ os: "Debian 13",
+ },
+};
+
+test("base reads as what it runs and where it comes from", () => {
+ expect(templateImage(base)).toBe("Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base");
+ expect(templateImage({ source: { image: "node:22" } })).toBe("node:22");
+ expect(templateImage({ source: null })).toBe("");
+});
+
+test("images lose their tag and digest, not a registry's port", () => {
+ expect(imageRepo("ghcr.io/a/b:1@sha256:ab")).toBe("ghcr.io/a/b");
+ expect(imageRepo("127.0.0.1:5005/pyo/x")).toBe("127.0.0.1:5005/pyo/x");
+ expect(imageRepo("127.0.0.1:5005/pyo/x:dev")).toBe("127.0.0.1:5005/pyo/x");
+});
+
+test("new sandboxes start on base once it's ready", () => {
+ const node = { name: "node-22", state: "ready" };
+ expect(defaultTemplate([node, base])).toBe("base");
+ expect(defaultTemplate([node, { ...base, state: "building" }])).toBe("node-22");
+ expect(defaultTemplate([{ ...base, state: "building" }])).toBe("");
+ // A team's own template named base isn't Pyo's.
+ expect(defaultTemplate([node, { name: "base", state: "ready" }])).toBe("node-22");
+});
diff --git a/dash/src/components/pyo/Cloudflare.tsx b/dash/src/components/pyo/Cloudflare.tsx
index 49cbfd5..40580f5 100644
--- a/dash/src/components/pyo/Cloudflare.tsx
+++ b/dash/src/components/pyo/Cloudflare.tsx
@@ -6,7 +6,6 @@ import { AppLink } from "~/components/ui/AppLink";
import { Button } from "~/components/ui/Button";
import { ConfirmDialog } from "~/components/ui/Dialog";
import { SecretInput, Switch } from "~/components/ui/Field";
-import { Panel } from "~/components/ui/Panel";
import { timeAgo } from "~/lib/pyo/format";
import { isFreeDomain } from "~/lib/pyo/freeDomain";
import {
@@ -26,7 +25,7 @@ import type { Application, Domain } from "~/lib/pyo/types";
const isAdmin = (role?: string) => role === "owner" || role === "admin";
/** Integrations › Cloudflare: connect, test, replace and remove the token. */
-export function CloudflarePanel() {
+export function CloudflareSettings() {
const s = usePyo();
const account = cloudflareIn(s);
const admin = isAdmin(roleIn(s));
@@ -100,10 +99,7 @@ export function CloudflarePanel() {
);
return (
-
+ <>
@@ -198,7 +194,7 @@ export function CloudflarePanel() {
toast({ title: "Cloudflare token removed" });
}}
/>
-
+ >
);
}
diff --git a/dash/src/components/pyo/Databases.tsx b/dash/src/components/pyo/Databases.tsx
index fd5ac5f..2a5dbf5 100644
--- a/dash/src/components/pyo/Databases.tsx
+++ b/dash/src/components/pyo/Databases.tsx
@@ -143,6 +143,7 @@ export function Choices({
/** One pick-one card: an optional icon, a name and a line under it. */
export function Choice({
icon: IconCmp,
+ mark,
label,
detail,
selected,
@@ -150,6 +151,8 @@ export function Choice({
disabled = false,
}: {
icon?: Icon;
+ /** A brand logo in place of `icon`, drawn in its own colours. */
+ mark?: ReactNode;
label: string;
detail: ReactNode;
selected: boolean;
@@ -167,7 +170,14 @@ export function Choice({
onClick={onSelect}
className={`${optionClass({ selected })} flex w-full min-w-0 items-center gap-3 px-3 py-2.5 disabled:cursor-not-allowed disabled:opacity-50`}
>
- {IconCmp ? (
+ {mark ? (
+
+ {mark}
+
+ ) : IconCmp ? (
{
};
/** Add a connection, or replace one's token. */
-function TokenDialog({
+export function TokenDialog({
connection,
onClose,
}: {
diff --git a/dash/src/components/pyo/GithubSettings.tsx b/dash/src/components/pyo/GithubSettings.tsx
index 850f872..da56675 100644
--- a/dash/src/components/pyo/GithubSettings.tsx
+++ b/dash/src/components/pyo/GithubSettings.tsx
@@ -1,42 +1,20 @@
-import {
- ArrowSquareOutIcon,
- CheckCircleIcon,
- MinusCircleIcon,
-} from "@phosphor-icons/react";
import { useEffect, useState } from "react";
import { BrandMark, GITHUB_LOGO } from "~/components/pyo/common";
import { Button } from "~/components/ui/Button";
import { ConfirmDialog } from "~/components/ui/Dialog";
import { Field, Input } from "~/components/ui/Field";
import { Panel } from "~/components/ui/Panel";
-import { StatusNote } from "~/components/ui/StatusNote";
import { api, usePyo } from "~/lib/pyo/store";
import { toast } from "~/lib/pyo/toast";
-// This installation's GitHub App. An admin creates it once with GitHub's
-// manifest flow; teams then connect their GitHub accounts in Integrations.
-
-/** GitHub can only deliver webhooks to a public https address. */
-const publicHttps = (url: string) => {
- const m = /^https:\/\/([^/:]+)/.exec(url.trim());
- return (
- !!m &&
- !/^(localhost|127\.|10\.|192\.168\.|100\.)|\.local(host)?$/.test(m[1])
- );
-};
+// This installation's GitHub App, on Settings (/github). The installation
+// admin creates it once with GitHub's manifest flow. GitHub sends them back
+// to /github?github=created. Teams then connect their own accounts in the
+// GitHub card on Integrations.
+/** Create the app, or see and remove it. */
export function GithubSettingsSection() {
const s = usePyo();
- const admin = !!s.user.isAdmin;
- const app = s.github;
- const [publicUrl, setPublicUrl] = useState(
- s.email?.publicUrl ??
- (typeof window === "undefined" ? "" : window.location.origin),
- );
- const [org, setOrg] = useState("");
- const [busy, setBusy] = useState(false);
- const [removing, setRemoving] = useState(false);
-
// GitHub sends the admin back with ?github=created (or an error).
useEffect(() => {
const q = new URLSearchParams(window.location.search).get("github");
@@ -44,8 +22,7 @@ export function GithubSettingsSection() {
if (q === "created")
toast({
title: "GitHub App created",
- description:
- "Teams connect GitHub in Integrations.",
+ description: "Teams connect GitHub in Integrations.",
});
else
toast({
@@ -57,158 +34,162 @@ export function GithubSettingsSection() {
});
window.history.replaceState(null, "", window.location.pathname);
}, []);
-
- if (app)
+ if (!s.user.isAdmin)
return (
-
-
}
- title={`${app.name} is set up`}
- >
- Teams connect GitHub in Integrations.
-
-
-
- -
-
- {app.slug}{" "}
-
-
-
- - {app.owner}
- -
- {app.webhookActive
- ? "Instant, by webhook"
- : "Checked every minute"}
-
- -
-
{app.publicUrl}
-
-
- {admin ? (
-
- setRemoving(true)}
- >
- Remove
-
-
- ) : null}
-
-
setRemoving(false)}
- title="Remove the GitHub App?"
- description="Teams lose private repos and deploy on push."
- confirmLabel="Remove"
- onConfirm={async () => {
- if (await api.removeGithubApp())
- toast({ title: "GitHub App removed" });
- }}
- />
-
+
+
+ Only the installation's admin manages the GitHub App.
+
+
);
+ return (
+
+ {s.github ? (
+
+ ) : (
+
+
+
+ )}
+
+ );
+}
- const webhook = publicHttps(publicUrl);
+/** The installation's GitHub App, for its admin: where it is, and removing it. */
+export function GithubAppRow() {
+ const s = usePyo();
+ const [removing, setRemoving] = useState(false);
+ const app = s.github;
+ if (!app) return null;
return (
-
-
}
- title="GitHub isn't set up"
- >
- Adds private repos, deploy on push and commit statuses.
- {admin ? "" : " The installation's admin can set it up."}
-
- {admin ? (
-
-
-
- {(p) => (
- setPublicUrl(e.target.value)}
- />
- )}
-
-
+
+
+
+
+
-
- {
- setBusy(true);
- const r = await api.githubManifest(publicUrl.trim(), org);
- if (!r) return setBusy(false);
- // GitHub's manifest flow: post the manifest from the browser.
- const form = document.createElement("form");
- form.method = "post";
- form.action = r.action;
- const field = document.createElement("input");
- field.type = "hidden";
- field.name = "manifest";
- field.value = r.manifest;
- form.appendChild(field);
- document.body.appendChild(form);
- form.submit();
- }}
- >
- {" "}
- {busy ? "Opening GitHub…" : "Create GitHub App"}
-
-
-
- ) : null}
-
+ GitHub App · {app.slug}
+
+
+
+ For every team · owned by {app.owner} ·{" "}
+ {app.webhookActive
+ ? "pushes by webhook"
+ : "pushes checked every minute"}
+
+
+ setRemoving(true)}
+ >
+ Remove
+
+
+ setRemoving(false)}
+ title="Remove the GitHub App?"
+ description="Every team loses private repos and deploy on push."
+ confirmLabel="Remove"
+ onConfirm={async () => {
+ if (await api.removeGithubApp()) toast({ title: "GitHub App removed" });
+ }}
+ />
+ >
);
}
-function Item({
- label,
- children,
-}: {
- label: string;
- children: React.ReactNode;
-}) {
+/** GitHub can only deliver webhooks to a public https address. */
+const publicHttps = (url: string) => {
+ const m = /^https:\/\/([^/:]+)/.exec(url.trim());
+ return (
+ !!m &&
+ !/^(localhost|127\.|10\.|192\.168\.|100\.)|\.local(host)?$/.test(m[1])
+ );
+};
+
+/** Create the GitHub App: the dashboard's address, and an organization. */
+export function GithubAppSetup() {
+ const s = usePyo();
+ const [publicUrl, setPublicUrl] = useState(
+ s.email?.publicUrl ??
+ (typeof window === "undefined" ? "" : window.location.origin),
+ );
+ const [org, setOrg] = useState("");
+ const [busy, setBusy] = useState(false);
+ const webhook = publicHttps(publicUrl);
return (
-
-
{label}
- {children}
-
+
);
}
diff --git a/dash/src/components/pyo/Hetzner.tsx b/dash/src/components/pyo/Hetzner.tsx
index 581e350..4884e45 100644
--- a/dash/src/components/pyo/Hetzner.tsx
+++ b/dash/src/components/pyo/Hetzner.tsx
@@ -10,7 +10,7 @@ import { Badge, type Tone } from "~/components/ui/Badge";
import { Button } from "~/components/ui/Button";
import { ConfirmDialog } from "~/components/ui/Dialog";
import { Field, Input, Select, Textarea } from "~/components/ui/Field";
-import { Callout, Panel } from "~/components/ui/Panel";
+import { Callout } from "~/components/ui/Panel";
import { StatusNote } from "~/components/ui/StatusNote";
import { CopyButton } from "~/components/ui/Table";
import { site } from "~/config/site";
@@ -102,7 +102,9 @@ export function DeleteCloudServerDialog({
title={server ? `Delete ${server.name} at Hetzner?` : ""}
description={
server
- ? `Its services move off it first. Then it leaves the cluster and your tailnet, and Hetzner deletes it. ${money(server.currency, server.monthly)} a month stops.`
+ ? server.sandboxServerId
+ ? `Its sandboxes end. Then it leaves your tailnet, and Hetzner deletes it. ${money(server.currency, server.monthly)} a month stops.`
+ : `Its services move off it first. Then it leaves the cluster and your tailnet, and Hetzner deletes it. ${money(server.currency, server.monthly)} a month stops.`
: ""
}
confirmLabel="Delete server"
@@ -110,21 +112,21 @@ export function DeleteCloudServerDialog({
);
}
-/** Integrations › Hetzner: the team's project token, and the servers Pyo made. */
-export function HetznerPanel() {
+/**
+ * The team's Hetzner project token: paste, check and save. In Integrations,
+ * and inline where a server is bought before Hetzner is connected.
+ */
+export function HetznerConnectForm({
+ onSaved,
+ onCancel,
+}: {
+ onSaved?: () => void;
+ onCancel?: () => void;
+}) {
const s = usePyo();
- const admin = useTeamAdmin();
- const conn = hetznerOf(s);
const [token, setToken] = useState("");
- const [replacing, setReplacing] = useState(false);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
- const [seen, setSeen] = useState("");
- const [removing, setRemoving] = useState(false);
- const [deleting, setDeleting] = useState(null);
- const servers = s.cloudServers.filter((c) => c.teamId === s.currentTeamId);
- const title = "Hetzner";
- const description = "For this team. Pyo makes servers in your Hetzner project.";
const save = async () => {
setBusy(true);
setError("");
@@ -132,9 +134,59 @@ export function HetznerPanel() {
setBusy(false);
if ("error" in r) return setError(r.error);
setToken("");
- setReplacing(false);
toast({ title: "Hetzner connected" });
+ onSaved?.();
};
+ return (
+
+ );
+}
+
+/** Integrations › Hetzner: the team's project token, and the servers Pyo made. */
+export function HetznerSettings() {
+ const s = usePyo();
+ const admin = useTeamAdmin();
+ const conn = hetznerOf(s);
+ const [replacing, setReplacing] = useState(false);
+ const [busy, setBusy] = useState(false);
+ const [seen, setSeen] = useState("");
+ const [removing, setRemoving] = useState(false);
+ const [deleting, setDeleting] = useState(null);
+ const servers = s.cloudServers.filter((c) => c.teamId === s.currentTeamId);
const test = async () => {
setBusy(true);
const r = await api.testHetzner();
@@ -143,7 +195,7 @@ export function HetznerPanel() {
else setSeen(r.data.locations.map((l) => l.city).join(", "));
};
return (
-
+ <>
{!s.tailscale ? (
@@ -163,41 +215,10 @@ export function HetznerPanel() {
{conn ? Connected : Not connected }
) : !conn || replacing ? (
-
+ setReplacing(false)}
+ onCancel={conn ? () => setReplacing(false) : undefined}
+ />
) : (
<>
@@ -227,8 +248,8 @@ export function HetznerPanel() {
{c.name}
- {cluster?.name ?? "Cluster removed"} · {c.serverType} · {c.location} ·{" "}
- {money(c.currency, c.monthly)}/mo
+ {c.sandboxServerId ? "Sandbox server" : (cluster?.name ?? "Cluster removed")} ·{" "}
+ {c.serverType} · {c.location} · {money(c.currency, c.monthly)}/mo
{c.error ?
{c.error}
: null}
@@ -259,7 +280,7 @@ export function HetznerPanel() {
confirmLabel="Remove"
/>
setDeleting(null)} />
-
+ >
);
}
@@ -398,7 +419,10 @@ export function CloudProgress({ server }: { server: CloudServer }) {
icon={ }
title={`Deleting ${server.name}`}
>
- {server.error ?? "Its services move off it, then Hetzner deletes it."}
+ {server.error ??
+ (server.sandboxServerId
+ ? "It leaves your tailnet, then Hetzner deletes it."
+ : "Its services move off it, then Hetzner deletes it.")}
);
return (
diff --git a/dash/src/components/pyo/IntegrationCard.tsx b/dash/src/components/pyo/IntegrationCard.tsx
new file mode 100644
index 0000000..96ce7b3
--- /dev/null
+++ b/dash/src/components/pyo/IntegrationCard.tsx
@@ -0,0 +1,141 @@
+import { CaretRightIcon, DotsThreeVerticalIcon } from "@phosphor-icons/react";
+import type { ReactNode } from "react";
+import { Dot, type Tone } from "~/components/ui/Badge";
+import { Button } from "~/components/ui/Button";
+import { Menu, type MenuItem } from "~/components/ui/Menu";
+import { site } from "~/config/site";
+
+// One integration on Integrations: a header with its icon, name and
+// category, a body that says what it does, and a footer with how it stands
+// (connected) or its setup guide (not connected).
+
+export type IntegrationCardProps = {
+ id: string;
+ name: string;
+ category: string;
+ icon: ReactNode;
+ description: string;
+ connected: boolean;
+ /** One real figure, connected only. */
+ stat?: { label: string; value: ReactNode } | null;
+ /** What it enables, not connected only. */
+ /** How it stands, connected only. */
+ status?: { tone: Tone; label: string } | null;
+ /** The docs page (pyo.dev/docs/). */
+ docs?: string;
+ /** Not connected: the Connect button (or why there isn't one). */
+ action?: ReactNode;
+ /** Connected: what can be done with it. */
+ menu?: MenuItem[];
+ /** Just linked to: a ring for a moment. */
+ highlight?: boolean;
+};
+
+/** A docs link as a plain anchor. */
+export const docsUrl = (page: string) => `${site.docsUrl}/${page}`;
+
+export function IntegrationCard({
+ id,
+ name,
+ category,
+ icon,
+ description,
+ connected,
+ stat,
+ status,
+ docs,
+ action,
+ menu,
+ highlight,
+}: IntegrationCardProps) {
+ return (
+
+
+
+ {icon}
+
+
+
+ {name}
+
+
+ {category}
+
+
+ {connected ? (
+
+
+ Connected
+
+ ) : (
+ {action}
+ )}
+ {menu?.length && connected ? (
+ (
+
+
+
+ )}
+ />
+ ) : null}
+
+
+
+ {/* Two lines for the description and one for the stat, always,
+ so every card is the same height. */}
+
+ {description}
+
+
+ {connected && stat ? stat.label : "\u00a0"}
+
+ {connected && stat ? stat.value : null}
+
+
+
+
+ {connected ? (
+ status ? (
+ <>
+ {/* The header's pill already says connected: a dot only when
+ something needs a look or is under way. */}
+ {status.tone !== "success" ? : null}
+ {status.label}
+ >
+ ) : null
+ ) : null}
+ {!connected && docs ? (
+
+ Setup guide
+
+
+ ) : null}
+
+
+
+ );
+}
diff --git a/dash/src/components/pyo/RegistryLogins.tsx b/dash/src/components/pyo/RegistryLogins.tsx
index 4561d36..10f9051 100644
--- a/dash/src/components/pyo/RegistryLogins.tsx
+++ b/dash/src/components/pyo/RegistryLogins.tsx
@@ -7,7 +7,6 @@ import { Badge } from "~/components/ui/Badge";
import { Button } from "~/components/ui/Button";
import { ConfirmDialog, Dialog } from "~/components/ui/Dialog";
import { Field, Input, SecretInput, Select } from "~/components/ui/Field";
-import { Panel } from "~/components/ui/Panel";
import { timeAgo } from "~/lib/pyo/format";
import { api, usePyo } from "~/lib/pyo/store";
import { toast } from "~/lib/pyo/toast";
@@ -45,20 +44,14 @@ const PRESETS = [
const preset = (host: string) => PRESETS.find((p) => p.host === host);
-export function RegistryLoginsPanel() {
+/** Integrations › Registries: the built-in registry and the team's logins. */
+export function RegistryLoginsSettings() {
const s = usePyo();
const logins = s.registryLogins.filter((l) => l.teamId === s.currentTeamId);
const [adding, setAdding] = useState(false);
const [removing, setRemoving] = useState(null);
return (
- setAdding(true)}>
- Add login
-
- }
- >
+ <>
@@ -117,10 +110,15 @@ export function RegistryLoginsPanel() {
})}
) : (
-
+
No logins yet. Public images don't need one.
)}
+
+
setAdding(true)}>
+ Add login
+
+
setAdding(false)} />
-
+ >
);
}
-function AddLoginDialog({
+export function AddLoginDialog({
open,
onClose,
}: {
diff --git a/dash/src/components/pyo/SecretStores.tsx b/dash/src/components/pyo/SecretStores.tsx
index 8e229fa..1c7270b 100644
--- a/dash/src/components/pyo/SecretStores.tsx
+++ b/dash/src/components/pyo/SecretStores.tsx
@@ -1,5 +1,6 @@
import { KeyIcon, PlusIcon, TrashIcon } from "@phosphor-icons/react";
import { useState } from "react";
+import { Badge } from "~/components/ui/Badge";
import { Button } from "~/components/ui/Button";
import { ConfirmDialog, Dialog } from "~/components/ui/Dialog";
import { Field, Input, Select, Textarea } from "~/components/ui/Field";
@@ -114,7 +115,7 @@ function optionsOf(f: Form): Record {
}
/** Connect a store, or change one. Its credential is never shown again. */
-function StoreDialog({
+export function StoreDialog({
store,
onClose,
}: {
@@ -321,8 +322,8 @@ function StoreDialog({
);
}
-/** Integrations: the team's stores. */
-export function SecretStoresPanel() {
+/** Integrations › Secret stores: the default, then the team's stores. */
+export function SecretStoresSettings() {
const s = usePyo();
const stores = s.secretStores.filter((x) => x.teamId === s.currentTeamId);
const [adding, setAdding] = useState(false);
@@ -333,16 +334,23 @@ export function SecretStoresPanel() {
a.secretImports?.some((i) => i.storeId === store.id),
).length;
return (
- setAdding(true)}>
- Connect
-
- }
- >
+ <>
+
+
+
+
+
+
+
Cluster Secrets
+
+ Default. Stored in the target cluster.
+
+
+ Active
+
+
{stores.length ? (
-
+
{stores.map((store) => {
const used = uses(store);
return (
@@ -383,10 +391,15 @@ export function SecretStoresPanel() {
})}
) : (
-
+
Vault, AWS, GCP, Azure, Doppler and more.
)}
+
+
setAdding(true)}>
+ Connect
+
+
{adding ? setAdding(false)} /> : null}
{editing ? (
setEditing(null)} />
@@ -401,7 +414,7 @@ export function SecretStoresPanel() {
if (removing) void api.removeSecretStore(removing.id);
}}
/>
-
+ >
);
}
diff --git a/dash/src/components/pyo/Tailscale.tsx b/dash/src/components/pyo/Tailscale.tsx
index 2bebf24..6816204 100644
--- a/dash/src/components/pyo/Tailscale.tsx
+++ b/dash/src/components/pyo/Tailscale.tsx
@@ -5,7 +5,7 @@ import { Badge } from "~/components/ui/Badge";
import { Button, ButtonLink } from "~/components/ui/Button";
import { ConfirmDialog } from "~/components/ui/Dialog";
import { Input, Switch } from "~/components/ui/Field";
-import { Callout, Panel } from "~/components/ui/Panel";
+import { Callout } from "~/components/ui/Panel";
import { site } from "~/config/site";
import { api, usePyo } from "~/lib/pyo/store";
import { toast } from "~/lib/pyo/toast";
@@ -28,7 +28,7 @@ export function NeedsTailscale() {
Servers join over your tailnet. Set up Tailscale first.{" "}
{s.user.isAdmin ? (
{
@@ -245,31 +245,27 @@ function HostMove() {
}
/** Integrations › Tailscale: the OAuth client for the whole Pyo (admins). */
-export function TailscalePanel() {
+export function TailscaleSettings() {
const s = usePyo();
const ts = s.tailscale;
const [replacing, setReplacing] = useState(false);
const [checks, setChecks] = useState([]);
const [testing, setTesting] = useState(false);
const [removing, setRemoving] = useState(false);
- const title = "Tailscale";
- const description = "For the whole Pyo. Servers join your tailnet with it.";
if (!s.user.isAdmin)
return (
-
-
-
-
- {ts ? `On ${ts.suffix ?? "your tailnet"}` : "An installation admin can set it up."}
-
- {ts ? Set up : Not set up }
-
-
-
-
+
+
+
+ {ts ? `On ${ts.suffix ?? "your tailnet"}` : "An installation admin can set it up."}
+
+ {ts ? Set up : Not set up }
+
+
+
);
return (
-
+ <>
{!ts || replacing ? (
@@ -337,6 +333,6 @@ export function TailscalePanel() {
description="Pyo's names on your tailnet are deleted. Servers stay on it. The public dashboard opens again."
confirmLabel="Remove"
/>
-
+ >
);
}
diff --git a/dash/src/components/pyo/TemplateGallery.tsx b/dash/src/components/pyo/TemplateGallery.tsx
index 6788343..d6fb0ed 100644
--- a/dash/src/components/pyo/TemplateGallery.tsx
+++ b/dash/src/components/pyo/TemplateGallery.tsx
@@ -46,7 +46,7 @@ export function TemplateCard({
action?: string;
}) {
return (
-
+
{/* The whole card is the target; the Details button sits above it. */}
) : null}
-
+
{t.description}
diff --git a/dash/src/components/pyo/TrafficCanvas.tsx b/dash/src/components/pyo/TrafficCanvas.tsx
index 1d45af5..95e2b4a 100644
--- a/dash/src/components/pyo/TrafficCanvas.tsx
+++ b/dash/src/components/pyo/TrafficCanvas.tsx
@@ -470,7 +470,7 @@ function Numbers({ t, r }: { t: ServiceTraffic; r: Region }) {
return (
Request numbers need observability storage.{" "}
-
+
Set it up
diff --git a/dash/src/components/pyo/UpdateSettings.tsx b/dash/src/components/pyo/UpdateSettings.tsx
index fae222e..b15cd9e 100644
--- a/dash/src/components/pyo/UpdateSettings.tsx
+++ b/dash/src/components/pyo/UpdateSettings.tsx
@@ -11,7 +11,8 @@ import {
} from "@phosphor-icons/react";
import { type ReactNode, useState } from "react";
import { Badge, type Tone } from "~/components/ui/Badge";
-import { Button, ButtonLink } from "~/components/ui/Button";
+import { Link } from "@tanstack/react-router";
+import { Button, buttonClass } from "~/components/ui/Button";
import { Dialog } from "~/components/ui/Dialog";
import { Field, Segmented, Select, Switch } from "~/components/ui/Field";
import { Panel } from "~/components/ui/Panel";
@@ -670,9 +671,12 @@ function BackupHint({ u }: { u: Updates }) {
: "Pyo's own data hasn't been backed up yet"
}
action={
-
+
Backups
-
+
}
>
{old
diff --git a/dash/src/components/pyo/common.tsx b/dash/src/components/pyo/common.tsx
index 20fc60f..e68b93e 100644
--- a/dash/src/components/pyo/common.tsx
+++ b/dash/src/components/pyo/common.tsx
@@ -75,10 +75,13 @@ export const CLOUDFLARE_LOGO = siCloudflare;
export function BrandMark({
icon,
size = 16,
+ tone = "auto",
}: {
/** A Simple Icon, or a template's logo: one path and its colour. */
icon: Pick;
size?: number;
+ /** "brand" keeps a dark brand color (Hetzner's red), "mono" takes the text color (Vault's pale yellow). */
+ tone?: "auto" | "brand" | "mono";
}) {
const [r, g, b] = [0, 2, 4].map(
(i) => parseInt(icon.hex.slice(i, i + 2), 16) / 255,
@@ -90,7 +93,11 @@ export function BrandMark({
width={size}
height={size}
aria-hidden
- fill={luminance < 0.35 ? "currentColor" : `#${icon.hex}`}
+ fill={
+ tone === "mono" || (tone === "auto" && luminance < 0.35)
+ ? "currentColor"
+ : `#${icon.hex}`
+ }
className="shrink-0"
>
diff --git a/dash/src/components/pyo/sandbox/AddServerFlow.tsx b/dash/src/components/pyo/sandbox/AddServerFlow.tsx
new file mode 100644
index 0000000..541f98a
--- /dev/null
+++ b/dash/src/components/pyo/sandbox/AddServerFlow.tsx
@@ -0,0 +1,230 @@
+import { CheckCircleIcon, CircleNotchIcon, TerminalWindowIcon } from "@phosphor-icons/react";
+import { useState } from "react";
+import { siHetzner } from "simple-icons";
+import { BrandMark } from "~/components/pyo/common";
+import {
+ CloudProgress,
+ CloudServerFields,
+ HetznerConnectForm,
+ hetznerOf,
+ priceOf,
+ useCloudOptions,
+ type CloudChoice,
+} from "~/components/pyo/Hetzner";
+import { NeedsTailscale } from "~/components/pyo/Tailscale";
+import { Button, ButtonLink } from "~/components/ui/Button";
+import { ConfirmDialog } from "~/components/ui/Dialog";
+import { Field, Input } from "~/components/ui/Field";
+import { OptionTile, OptionTiles } from "~/components/ui/OptionTile";
+import { Callout, Panel } from "~/components/ui/Panel";
+import { DNS_LABEL } from "~/lib/pyo/format";
+import { sandboxApi } from "~/lib/pyo/sandbox";
+import { refresh, roleIn, sandboxServersIn, usePyo } from "~/lib/pyo/store";
+import { AddSandboxServer, RuntimeBadge } from "./common";
+
+// Add a sandbox server: two square tiles first. "Hetzner" has Pyo buy the
+// server and set it up (or connects Hetzner first). "Command line" is the
+// one-line sandbox.sh for a server you have.
+
+type How = "hetzner" | "command";
+
+/** What sandbox.sh takes for --edge-domain. */
+const EDGE_DOMAIN = /^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,62}$/;
+
+export function AddSandboxServerFlow() {
+ const [how, setHow] = useState(null);
+ // Once Pyo is buying one, only its progress shows.
+ const [made, setMade] = useState<{ id: string; name: string } | null>(null);
+ if (made) return ;
+ return (
+
+
+ }
+ label="Hetzner"
+ detail="Pyo buys a server and sets it up."
+ selected={how === "hetzner"}
+ onSelect={() => setHow("hetzner")}
+ />
+ setHow("command")}
+ />
+
+ {how === "hetzner" ? (
+
+ ) : how === "command" ? (
+
+
+
+ ) : null}
+
+ );
+}
+
+/** The first free "sandbox-". */
+function freeName(taken: string[]) {
+ for (let i = taken.length + 1; ; i++) if (!taken.includes(`sandbox-${i}`)) return `sandbox-${i}`;
+}
+
+function CloudServer({ onMade }: { onMade: (m: { id: string; name: string }) => void }) {
+ const s = usePyo();
+ const role = roleIn(s);
+ const canManage = role === "owner" || role === "admin";
+ const servers = sandboxServersIn(s);
+ const [suggested] = useState(() => freeName(servers.map((x) => x.name)));
+ const [name, setName] = useState("");
+ const [edge, setEdge] = useState("");
+ const [choice, setChoice] = useState({ location: "", serverType: "", sshKey: "" });
+ const [confirming, setConfirming] = useState(false);
+ const [creating, setCreating] = useState(false);
+ const [error, setError] = useState("");
+ const connected = !!hetznerOf(s);
+ const { options } = useCloudOptions(connected && !!s.tailscale);
+ const value = name || suggested;
+ const edgeDomain = edge.trim().replace(/\.$/, "");
+ const nameError = !DNS_LABEL.test(value)
+ ? "Lowercase letters, digits and dashes."
+ : servers.some((x) => x.name === value && x.teamId === s.currentTeamId)
+ ? "A sandbox server has this name."
+ : "";
+ const edgeError = !edgeDomain || EDGE_DOMAIN.test(edgeDomain) ? "" : "A domain, like sbx.example.com.";
+ const price = priceOf(options, choice);
+
+ if (!canManage)
+ return (
+
+ Ask a team owner or admin to add a sandbox server.
+
+ );
+
+ const create = async () => {
+ setConfirming(false);
+ setCreating(true);
+ setError("");
+ try {
+ const r = await sandboxApi.addCloudServer({
+ teamId: s.currentTeamId,
+ name: value,
+ provider: "hetzner",
+ location: choice.location,
+ serverType: choice.serverType,
+ ...(edgeDomain ? { edgeDomain } : {}),
+ ...(choice.sshKey.trim() ? { sshKey: choice.sshKey.trim() } : {}),
+ confirm: true,
+ });
+ await refresh().catch(() => undefined);
+ if (r.server) onMade({ id: r.server.id, name: value });
+ } catch (e) {
+ setError((e as Error).message);
+ } finally {
+ setCreating(false);
+ }
+ };
+
+ return (
+
+
+
+ {s.tailscale && !connected ? (
+
+ ) : null}
+ {s.tailscale && connected ? (
+ <>
+
+
+ {(p) => (
+ setName(e.target.value.toLowerCase())} />
+ )}
+
+
+ {(p) => (
+ setEdge(e.target.value.toLowerCase())}
+ />
+ )}
+
+
+
+
+ Hetzner Cloud servers have no KVM, so sandboxes run on gVisor.
+
+ {error ?
{error} : null}
+
setConfirming(true)}
+ >
+ {creating ? "Creating…" : price ? `Create for ${price}/mo` : "Pick a server type"}
+
+
setConfirming(false)}
+ onConfirm={() => void create()}
+ tone="primary"
+ title={`Create ${value} at Hetzner?`}
+ description={`${choice.serverType} in ${choice.location}. ${price ?? ""} a month, before VAT.`}
+ confirmLabel="Create server"
+ />
+ >
+ ) : null}
+
+
+ );
+}
+
+/** A server Pyo is buying, until it runs sandboxes. */
+function CloudSetup({ id, name }: { id: string; name: string }) {
+ const s = usePyo();
+ const server = s.sandboxServers.find((x) => x.id === id);
+ const cloud = s.cloudServers.find((c) => c.sandboxServerId === id);
+ const ready = server?.state === "ready";
+ return (
+
+
+ {cloud ? (
+
+ ) : !server ? (
+
It was removed.
+ ) : (
+
+ Creating it at Hetzner…
+
+ )}
+ {server && ready ? (
+
+
+
+ {server.name} runs sandboxes.
+
+
+
+ New sandbox
+
+
+ ) : cloud && cloud.state !== "failed" && cloud.state !== "removing" ? (
+
+ Takes a few minutes. You can leave this page.
+
+ ) : null}
+
+
+ );
+}
diff --git a/dash/src/components/pyo/sandbox/SandboxServers.tsx b/dash/src/components/pyo/sandbox/SandboxServers.tsx
index 3c3d01f..e800293 100644
--- a/dash/src/components/pyo/sandbox/SandboxServers.tsx
+++ b/dash/src/components/pyo/sandbox/SandboxServers.tsx
@@ -7,20 +7,15 @@ import {
TrashIcon,
} from "@phosphor-icons/react";
import { useEffect, useState } from "react";
-import { Button } from "~/components/ui/Button";
+import { CloudStateBadge, money } from "~/components/pyo/Hetzner";
+import { Button, ButtonLink } from "~/components/ui/Button";
import { ConfirmDialog } from "~/components/ui/Dialog";
import { Menu } from "~/components/ui/Menu";
import { Meter } from "~/components/ui/Panel";
import { cloudflareZoneFor, refresh, roleIn, sandboxServersIn, usePyo } from "~/lib/pyo/store";
import { sandboxApi, type SandboxServer } from "~/lib/pyo/sandbox";
import { toast } from "~/lib/pyo/toast";
-import {
- AddSandboxServerDialog,
- RuntimeBadge,
- ServerState,
- serverCapacity,
- serverUse,
-} from "./common";
+import { RuntimeBadge, ServerState, serverCapacity, serverUse } from "./common";
/**
* Sandbox servers on the servers page, next to the clusters: a third kind,
@@ -31,7 +26,6 @@ export function SandboxServers() {
const servers = sandboxServersIn(s);
const role = roleIn(s);
const [removing, setRemoving] = useState(null);
- const [adding, setAdding] = useState(false);
const issuing = servers.some((x) => x.edge?.cert.state === "issuing");
useEffect(() => {
if (!issuing) return;
@@ -100,27 +94,37 @@ export function SandboxServers() {
{role === "owner" || role === "admin" ? (
- setAdding(true)}>
+
Add server
-
+
) : null}
{servers.map((x) => {
const use = serverUse(x);
return (
-
+
{x.name}
-
+ {x.cloud && x.cloud.state !== "ready" ? (
+
+ ) : (
+
+ )}
{x.teamId ? null : Installation }
{x.address ? {x.address} · : null}
{serverCapacity(x)}
+ {x.cloud ? (
+
+ Hetzner · {x.cloud.serverType} · {x.cloud.location} · {money(x.cloud.currency, x.cloud.monthly)}/mo
+
+ ) : null}
+ {x.cloud?.error ?
{x.cloud.error}
: null}
{x.edge ?
: null}
{x.state !== "ready" && (x.error || x.summary) ? (
@@ -163,23 +167,18 @@ export function SandboxServers() {
);
})}
- setAdding(false)} />
setRemoving(null)}
- title={`Remove ${removing?.name ?? "server"}?`}
- description={
- removing?.sandboxes
- ? `Its ${removing.sandboxes} sandbox${removing.sandboxes === 1 ? "" : "es"} end. Run sandbox.sh --uninstall on the server to take Pyo off it.`
- : "Run sandbox.sh --uninstall on the server to take Pyo off it."
- }
- confirmLabel="Remove server"
+ title={removing?.cloud ? `Delete ${removing.name} at Hetzner?` : `Remove ${removing?.name ?? "server"}?`}
+ description={removeText(removing)}
+ confirmLabel={removing?.cloud ? "Delete server" : "Remove server"}
onConfirm={() => {
const x = removing;
if (!x) return;
void sandboxApi
- .removeServer(x.id)
- .then(() => toast({ title: `Removed ${x.name}` }))
+ .removeServer(x.id, !!x.cloud)
+ .then(() => toast({ title: x.cloud ? `Deleting ${x.name}` : `Removed ${x.name}` }))
.catch((e: Error) => toast({ tone: "error", title: e.message }))
.finally(() => void refresh().catch(() => undefined));
}}
@@ -188,6 +187,16 @@ export function SandboxServers() {
);
}
+/** What removing a server does: its sandboxes end, and one Pyo made goes at Hetzner. */
+function removeText(x: SandboxServer | null) {
+ if (!x) return "";
+ const n = x.sandboxes;
+ const ending = n ? `Its ${n} sandbox${n === 1 ? "" : "es"} end. ` : "";
+ return x.cloud
+ ? `${ending}It leaves your tailnet, and Hetzner deletes it. ${money(x.cloud.currency, x.cloud.monthly)} a month stops.`
+ : `${ending}Run sandbox.sh --uninstall on the server to take Pyo off it.`;
+}
+
const day = (at: number) =>
new Date(at).toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" });
diff --git a/dash/src/components/pyo/sandbox/common.tsx b/dash/src/components/pyo/sandbox/common.tsx
index 83b9b5a..2bb14f4 100644
--- a/dash/src/components/pyo/sandbox/common.tsx
+++ b/dash/src/components/pyo/sandbox/common.tsx
@@ -1,10 +1,10 @@
import { CheckCircleIcon, CircleNotchIcon } from "@phosphor-icons/react";
import { useEffect, useState } from "react";
import { NeedsTailscale } from "~/components/pyo/Tailscale";
+import { AppLink } from "~/components/ui/AppLink";
import { LiveCommand } from "~/components/pyo/LiveCommand";
import { Badge, Dot } from "~/components/ui/Badge";
import { Button } from "~/components/ui/Button";
-import { Dialog } from "~/components/ui/Dialog";
import { Field, Input } from "~/components/ui/Field";
import { Callout } from "~/components/ui/Panel";
import { Tooltip } from "~/components/ui/Tooltip";
@@ -85,6 +85,9 @@ export function serverUse(x: SandboxServer) {
return { used: Math.min(usable, c.memoryCommitted), total: usable };
}
+/** How Pyo's "set up Tailscale" errors start: the word Tailscale links to its card. */
+const TAILSCALE = "Set up Tailscale";
+
/** What sandbox.sh takes for --edge-domain. */
const EDGE_DOMAIN = /^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,62}$/;
@@ -210,7 +213,21 @@ export function AddSandboxServer({ onDone }: { onDone?: () => void }) {
)}
- {error ? {error} : null}
+ {error ? (
+
+ {error.startsWith(TAILSCALE) ? (
+ <>
+ Set up{" "}
+
+ Tailscale
+
+ {error.slice(TAILSCALE.length)}
+ >
+ ) : (
+ error
+ )}
+
+ ) : null}
{busy ? "Preparing…" : "Get the command"}
@@ -220,18 +237,6 @@ export function AddSandboxServer({ onDone }: { onDone?: () => void }) {
);
}
-export function AddSandboxServerDialog({ open, onClose }: { open: boolean; onClose: () => void }) {
- return (
-
- {open ? (
-
- ) : null}
-
- );
-}
-
export function useSandboxes(teamId: string) {
return useQuery(`sandboxes:${teamId}`, () => sandboxApi.list(teamId, "all"), {
refreshMs: 4000,
diff --git a/dash/src/components/pyo/wizard/steps.tsx b/dash/src/components/pyo/wizard/steps.tsx
index 5758e4c..5d547e3 100644
--- a/dash/src/components/pyo/wizard/steps.tsx
+++ b/dash/src/components/pyo/wizard/steps.tsx
@@ -1985,7 +1985,7 @@ function ImageReference({
Public image.{" "}
Private? Add a login
diff --git a/dash/src/components/shell/AppShell.tsx b/dash/src/components/shell/AppShell.tsx
index 444e403..f37d12e 100644
--- a/dash/src/components/shell/AppShell.tsx
+++ b/dash/src/components/shell/AppShell.tsx
@@ -31,7 +31,7 @@ import {
} from "@phosphor-icons/react";
import { ProjectIcon } from "~/components/ui/ProjectIcon";
import { Link, useNavigate, useParams } from "@tanstack/react-router";
-import { currentSlug, useTeamPathname, CLUSTER_TABS } from "~/lib/pyo/paths";
+import { currentSlug, useTeamPathname, CLUSTER_TABS, SETTINGS_PAGES } from "~/lib/pyo/paths";
import { AppLink } from "~/components/ui/AppLink";
import { UserAvatar } from "~/components/ui/UserAvatar";
import { Fragment, useEffect, useState, type ReactNode } from "react";
@@ -195,7 +195,7 @@ export function AppShell({ children }: { children: ReactNode }) {
);
}
-/** On /profile, /settings, /api-keys and /mcp (outside any team): back to the team, then the account's own pages. */
+/** On /profile, the settings pages (/backups, /email…), /api-keys and /mcp (outside any team): back to the team, then the account's own pages. */
/** The account's sidebar: back to the team, then profile, settings and developer pages. */
function AccountNav({
@@ -214,7 +214,12 @@ function AccountNav({
Icon: Icon,
dot = false,
) => {
- const active = pathname === to || pathname.startsWith(`${to}/`);
+ const active =
+ pathname === to ||
+ pathname.startsWith(`${to}/`) ||
+ // The settings pages sit at the root (/backups, /email).
+ (to === "/settings" &&
+ SETTINGS_PAGES.some((p) => pathname === p || pathname.startsWith(`${p}/`)));
return (
void navigate({
- to: updateAvailable ? "/settings/updates" : "/settings",
+ to: updateAvailable ? "/updates" : "/settings",
}),
},
{
diff --git a/dash/src/components/ui/OptionTile.tsx b/dash/src/components/ui/OptionTile.tsx
new file mode 100644
index 0000000..8f29e24
--- /dev/null
+++ b/dash/src/components/ui/OptionTile.tsx
@@ -0,0 +1,55 @@
+import type { Icon } from "@phosphor-icons/react";
+import { useId, type ReactNode } from "react";
+import { optionClass } from "./fieldStyles";
+
+/** Square pick-one tiles in a row: the first choice of a flow. */
+export function OptionTiles({ label, children }: { label: string; children: ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+/** One square tile: an icon (or a brand mark), a name and a short line under it. */
+export function OptionTile({
+ icon: IconCmp,
+ mark,
+ label,
+ detail,
+ selected,
+ onSelect,
+}: {
+ icon?: Icon;
+ /** A logo in place of the icon, in its own colors. */
+ mark?: ReactNode;
+ label: string;
+ detail: ReactNode;
+ selected: boolean;
+ onSelect: () => void;
+}) {
+ const id = useId();
+ return (
+
+
+ {mark ?? (IconCmp ? : null)}
+
+
+ {label}
+
+ {detail}
+
+
+
+ );
+}
diff --git a/dash/src/lib/pyo/paths.ts b/dash/src/lib/pyo/paths.ts
index 3eeefb8..eefdaf4 100644
--- a/dash/src/lib/pyo/paths.ts
+++ b/dash/src/lib/pyo/paths.ts
@@ -17,6 +17,27 @@ export function teamPath(slug: string, path: string) {
: `/${slug}${path}`;
}
+/**
+ * The installation's settings pages, at the root like /profile (/backups,
+ * /email). They show inside the Settings layout, the first is where
+ * /settings opens. Team slugs can't take these (RESERVED_SLUGS in the API).
+ */
+export const SETTINGS_PAGES = [
+ "/backups",
+ "/email",
+ "/github",
+ "/observability",
+ "/updates",
+] as const;
+
+/** For an old /settings/ URL: the same page at the root, query and hash kept. */
+export function movedSettingsHref(
+ page: (typeof SETTINGS_PAGES)[number],
+ location: { searchStr: string; hash: string },
+) {
+ return `${page}${location.searchStr}${location.hash ? `#${location.hash}` : ""}`;
+}
+
/** The team in the URL, or the last-used team on pages outside one. */
export function useTeamSlug(): string {
const params = useParams({ strict: false }) as { team?: string };
diff --git a/dash/src/lib/pyo/sandbox.ts b/dash/src/lib/pyo/sandbox.ts
index 9d63396..83e0ce6 100644
--- a/dash/src/lib/pyo/sandbox.ts
+++ b/dash/src/lib/pyo/sandbox.ts
@@ -5,6 +5,10 @@
// ports are served on the server's edge proxy instead.
import { request } from "./backend";
+import { BASE_TEMPLATE } from "./sandboxTemplates";
+import type { CloudServerState } from "./types";
+
+export { BASE_TEMPLATE, defaultTemplate, imageRepo, templateImage } from "./sandboxTemplates";
// ── Types ──────────────────────────────────────────────────────────────────
@@ -157,13 +161,55 @@ export type SandboxServer = {
createdAt: number;
edgeDomain: string | null;
edge?: SandboxEdge;
+ /** What runs its sandboxes, once it reports. */
+ runtime?: "firecracker" | "gvisor" | null;
+ /** The server Pyo made for it at a cloud provider, until it's deleted there. */
+ cloud?: SandboxServerCloud | null;
+};
+
+export type SandboxServerCloud = {
+ /** The cloud server's id, `cs_…`. */
+ id: string;
+ provider: "hetzner";
+ location: string;
+ serverType: string;
+ /** A month, before VAT. */
+ monthly: number;
+ currency: string;
+ ipv4: string | null;
+ state: CloudServerState;
+ error: string | null;
+ signCommand: string | null;
+ createdAt: number;
+ deadlineAt: number;
+};
+
+/** Where Pyo can buy a sandbox server. More plug in here. */
+export const CLOUD_PROVIDERS = [
+ { value: "hetzner", label: "Hetzner", detail: "Billed to your Hetzner project." },
+] as const;
+
+export type CloudProvider = (typeof CLOUD_PROVIDERS)[number]["value"];
+
+export type AddCloudServer = {
+ teamId: string;
+ name: string;
+ provider: CloudProvider;
+ location: string;
+ serverType: string;
+ edgeDomain?: string;
+ sshKey?: string;
+ confirm: boolean;
};
export type SandboxTemplate = {
id: string;
teamId: string | null;
name: string;
- source: { kind: string; image?: string } | null;
+ /** `os` and `release` on Pyo's built-in template. */
+ source: { kind: string; image?: string; os?: string; release?: string } | null;
+ /** Pyo's own (base): kept in step with Pyo's release, never deleted. */
+ builtin?: boolean;
state: "pending" | "building" | "ready" | "failed";
/** Each sandbox server that may use it, and how far it got there. */
servers: SandboxTemplateServer[];
@@ -245,9 +291,15 @@ export const sandboxApi = {
"/api/v1/sandbox-servers",
{ method: "POST", body },
),
- removeServer: (id: string) =>
- request<{ ok: boolean; stopped: number }>(
- `/api/v1/sandbox-servers/${enc(id)}`,
+ addCloudServer: (body: AddCloudServer) =>
+ request<{ server: SandboxServer | null; cloudServer: { id: string; name: string } }>(
+ "/api/v1/sandbox-servers/cloud",
+ { method: "POST", body },
+ ),
+ /** `confirm`: also delete it at the cloud provider Pyo made it at. */
+ removeServer: (id: string, confirm = false) =>
+ request<{ ok: boolean; stopped: number; deleting?: boolean }>(
+ `/api/v1/sandbox-servers/${enc(id)}${confirm ? "?confirm=true" : ""}`,
{ method: "DELETE" },
),
issueCert: (id: string, teamId?: string) =>
@@ -500,7 +552,9 @@ export const LIFETIMES = [
/** The TypeScript SDK call that makes the same sandbox as the form. */
export function sdkSnippet(b: Omit) {
- const lines: string[] = [` template: ${JSON.stringify(b.template)},`];
+ // base is the default, so it's left out.
+ const lines: string[] =
+ b.template && b.template !== BASE_TEMPLATE ? [` template: ${JSON.stringify(b.template)},`] : [];
if (b.cpus) lines.push(` cpus: ${b.cpus},`);
if (b.memoryMib) lines.push(` memoryMib: ${b.memoryMib},`);
if (b.ttl) lines.push(` ttl: ${JSON.stringify(b.ttl)},`);
diff --git a/dash/src/lib/pyo/sandboxTemplates.ts b/dash/src/lib/pyo/sandboxTemplates.ts
new file mode 100644
index 0000000..ea9e43c
--- /dev/null
+++ b/dash/src/lib/pyo/sandboxTemplates.ts
@@ -0,0 +1,35 @@
+// Sandbox templates as the dashboard shows them: Pyo's built-in `base`
+// (crates/pyo-api/src/sandbox/base.rs) and the rest. No imports, so
+// dash/scripts tests them as they are.
+
+type TemplateLike = {
+ name?: string;
+ state?: string;
+ builtin?: boolean;
+ source?: { image?: string; os?: string } | null;
+};
+
+/** The template a sandbox starts from when none is given. */
+export const BASE_TEMPLATE = "base";
+
+/** An image without its tag and digest: ghcr.io/a/b:1@sha256:… is ghcr.io/a/b. */
+export function imageRepo(image: string) {
+ const name = image.split("@")[0];
+ const i = name.lastIndexOf(":");
+ return i > name.lastIndexOf("/") ? name.slice(0, i) : name;
+}
+
+/** A template's image as lists show it. Pyo's built-in one says what it
+ * runs and leaves out its version: "Debian 13 · ghcr.io/upsetdev/pyo/sandbox-base". */
+export function templateImage(t: TemplateLike) {
+ const image = t.source?.image;
+ if (!image) return "";
+ return t.builtin && t.source?.os ? `${t.source.os} · ${imageRepo(image)}` : image;
+}
+
+/** The template a new sandbox form starts on: base once it's ready, else
+ * the first ready one. */
+export function defaultTemplate(list: TemplateLike[]) {
+ const ready = list.filter((t) => t.state === "ready");
+ return (ready.find((t) => t.builtin && t.name === BASE_TEMPLATE) ?? ready[0])?.name ?? "";
+}
diff --git a/dash/src/lib/pyo/store.ts b/dash/src/lib/pyo/store.ts
index f5a3254..90d819f 100644
--- a/dash/src/lib/pyo/store.ts
+++ b/dash/src/lib/pyo/store.ts
@@ -497,6 +497,13 @@ async function attempt(
/** Slugs a team can't take because a top-level route already owns them. */
const RESERVED_SLUGS = new Set([
"profile",
+ "settings",
+ // The installation's settings pages (SETTINGS_PAGES in paths.ts).
+ "backups",
+ "email",
+ "github",
+ "observability",
+ "updates",
"login",
"signup",
"forgot-password",
diff --git a/dash/src/lib/pyo/types.ts b/dash/src/lib/pyo/types.ts
index 070d09a..0bcf307 100644
--- a/dash/src/lib/pyo/types.ts
+++ b/dash/src/lib/pyo/types.ts
@@ -1809,8 +1809,10 @@ export type CloudServer = {
id: string;
teamId: string;
provider: "hetzner";
- /** Null once its cluster was removed. */
+ /** Null once its cluster was removed, and for a sandbox server. */
clusterId: string | null;
+ /** The sandbox server it becomes, instead of a cluster's. */
+ sandboxServerId?: string | null;
name: string;
/** The cluster's first server (its control plane). */
firstServer: boolean;
diff --git a/dash/src/routeTree.gen.ts b/dash/src/routeTree.gen.ts
index 843ada8..7373adb 100644
--- a/dash/src/routeTree.gen.ts
+++ b/dash/src/routeTree.gen.ts
@@ -13,12 +13,12 @@ import { Route as AppRouteImport } from './routes/_app'
import { Route as AuthRouteImport } from './routes/_auth'
import { Route as AppIndexRouteImport } from './routes/_app/index'
import { Route as AppTeamRouteImport } from './routes/_app/$team'
+import { Route as AppSettingsRouteImport } from './routes/_app/_settings'
import { Route as AppApiKeysRouteImport } from './routes/_app/api-keys'
import { Route as AppCliLoginRouteImport } from './routes/_app/cli-login'
import { Route as AppInviteRouteImport } from './routes/_app/invite'
import { Route as AppMcpRouteImport } from './routes/_app/mcp'
import { Route as AppProfileRouteImport } from './routes/_app/profile'
-import { Route as AppSettingsRouteImport } from './routes/_app/settings'
import { Route as AuthForgotPasswordRouteImport } from './routes/_auth/forgot-password'
import { Route as AuthLoginRouteImport } from './routes/_auth/login'
import { Route as AuthResetPasswordRouteImport } from './routes/_auth/reset-password'
@@ -43,13 +43,14 @@ import { Route as AppTeamRegistryRouteImport } from './routes/_app/$team/registr
import { Route as AppTeamSecurityRouteImport } from './routes/_app/$team/security'
import { Route as AppTeamSettingsRouteImport } from './routes/_app/$team/settings'
import { Route as AppTeamTemplatesRouteImport } from './routes/_app/$team/templates'
+import { Route as AppSettingsBackupsRouteImport } from './routes/_app/_settings/backups'
+import { Route as AppSettingsEmailRouteImport } from './routes/_app/_settings/email'
+import { Route as AppSettingsGithubRouteImport } from './routes/_app/_settings/github'
+import { Route as AppSettingsObservabilityRouteImport } from './routes/_app/_settings/observability'
+import { Route as AppSettingsUpdatesRouteImport } from './routes/_app/_settings/updates'
import { Route as AppOauthAuthorizeRouteImport } from './routes/_app/oauth/authorize'
import { Route as AppSettingsIndexRouteImport } from './routes/_app/settings/index'
-import { Route as AppSettingsBackupsRouteImport } from './routes/_app/settings/backups'
-import { Route as AppSettingsEmailRouteImport } from './routes/_app/settings/email'
-import { Route as AppSettingsGithubRouteImport } from './routes/_app/settings/github'
-import { Route as AppSettingsObservabilityRouteImport } from './routes/_app/settings/observability'
-import { Route as AppSettingsUpdatesRouteImport } from './routes/_app/settings/updates'
+import { Route as AppSettingsPageRouteImport } from './routes/_app/settings/$page'
import { Route as AppTeamProjectIdIndexRouteImport } from './routes/_app/$team/$projectId/index'
import { Route as AppTeamProjectIdDeploymentsRouteImport } from './routes/_app/$team/$projectId/deployments'
import { Route as AppTeamProjectIdNetworkingRouteImport } from './routes/_app/$team/$projectId/networking'
@@ -131,6 +132,7 @@ import { Route as AppTeamPostgresDatabaseIdConnectRouteImport } from './routes/_
import { Route as AppTeamPostgresDatabaseIdExtensionsRouteImport } from './routes/_app/$team/postgres/$databaseId/extensions'
import { Route as AppTeamPostgresDatabaseIdLogsRouteImport } from './routes/_app/$team/postgres/$databaseId/logs'
import { Route as AppTeamPostgresDatabaseIdSettingsRouteImport } from './routes/_app/$team/postgres/$databaseId/settings'
+import { Route as AppTeamSandboxesServersNewRouteImport } from './routes/_app/$team/sandboxes/servers/new'
import { Route as AppTeamClustersClusterIdServersNewRouteImport } from './routes/_app/$team/clusters/$clusterId_/servers/new'
const AppRoute = AppRouteImport.update({
@@ -151,6 +153,10 @@ const AppTeamRoute = AppTeamRouteImport.update({
path: '/$team',
getParentRoute: () => AppRoute,
} as any)
+const AppSettingsRoute = AppSettingsRouteImport.update({
+ id: '/_settings',
+ getParentRoute: () => AppRoute,
+} as any)
const AppApiKeysRoute = AppApiKeysRouteImport.update({
id: '/api-keys',
path: '/api-keys',
@@ -176,11 +182,6 @@ const AppProfileRoute = AppProfileRouteImport.update({
path: '/profile',
getParentRoute: () => AppRoute,
} as any)
-const AppSettingsRoute = AppSettingsRouteImport.update({
- id: '/settings',
- path: '/settings',
- getParentRoute: () => AppRoute,
-} as any)
const AuthForgotPasswordRoute = AuthForgotPasswordRouteImport.update({
id: '/forgot-password',
path: '/forgot-password',
@@ -301,16 +302,6 @@ const AppTeamTemplatesRoute = AppTeamTemplatesRouteImport.update({
path: '/templates',
getParentRoute: () => AppTeamRoute,
} as any)
-const AppOauthAuthorizeRoute = AppOauthAuthorizeRouteImport.update({
- id: '/oauth/authorize',
- path: '/oauth/authorize',
- getParentRoute: () => AppRoute,
-} as any)
-const AppSettingsIndexRoute = AppSettingsIndexRouteImport.update({
- id: '/',
- path: '/',
- getParentRoute: () => AppSettingsRoute,
-} as any)
const AppSettingsBackupsRoute = AppSettingsBackupsRouteImport.update({
id: '/backups',
path: '/backups',
@@ -337,6 +328,21 @@ const AppSettingsUpdatesRoute = AppSettingsUpdatesRouteImport.update({
path: '/updates',
getParentRoute: () => AppSettingsRoute,
} as any)
+const AppOauthAuthorizeRoute = AppOauthAuthorizeRouteImport.update({
+ id: '/oauth/authorize',
+ path: '/oauth/authorize',
+ getParentRoute: () => AppRoute,
+} as any)
+const AppSettingsIndexRoute = AppSettingsIndexRouteImport.update({
+ id: '/settings/',
+ path: '/settings/',
+ getParentRoute: () => AppRoute,
+} as any)
+const AppSettingsPageRoute = AppSettingsPageRouteImport.update({
+ id: '/settings/$page',
+ path: '/settings/$page',
+ getParentRoute: () => AppRoute,
+} as any)
const AppTeamProjectIdIndexRoute = AppTeamProjectIdIndexRouteImport.update({
id: '/',
path: '/',
@@ -798,6 +804,12 @@ const AppTeamPostgresDatabaseIdSettingsRoute =
path: '/settings',
getParentRoute: () => AppTeamPostgresDatabaseIdRoute,
} as any)
+const AppTeamSandboxesServersNewRoute =
+ AppTeamSandboxesServersNewRouteImport.update({
+ id: '/sandboxes/servers/new',
+ path: '/sandboxes/servers/new',
+ getParentRoute: () => AppTeamRoute,
+ } as any)
const AppTeamClustersClusterIdServersNewRoute =
AppTeamClustersClusterIdServersNewRouteImport.update({
id: '/clusters/$clusterId_/servers/new',
@@ -813,7 +825,6 @@ export interface FileRoutesByFullPath {
'/invite': typeof AppInviteRoute
'/mcp': typeof AppMcpRoute
'/profile': typeof AppProfileRoute
- '/settings': typeof AppSettingsRouteWithChildren
'/forgot-password': typeof AuthForgotPasswordRoute
'/login': typeof AuthLoginRoute
'/reset-password': typeof AuthResetPasswordRoute
@@ -837,12 +848,13 @@ export interface FileRoutesByFullPath {
'/$team/security': typeof AppTeamSecurityRoute
'/$team/settings': typeof AppTeamSettingsRouteWithChildren
'/$team/templates': typeof AppTeamTemplatesRoute
+ '/backups': typeof AppSettingsBackupsRoute
+ '/email': typeof AppSettingsEmailRoute
+ '/github': typeof AppSettingsGithubRoute
+ '/observability': typeof AppSettingsObservabilityRoute
+ '/updates': typeof AppSettingsUpdatesRoute
'/oauth/authorize': typeof AppOauthAuthorizeRoute
- '/settings/backups': typeof AppSettingsBackupsRoute
- '/settings/email': typeof AppSettingsEmailRoute
- '/settings/github': typeof AppSettingsGithubRoute
- '/settings/observability': typeof AppSettingsObservabilityRoute
- '/settings/updates': typeof AppSettingsUpdatesRoute
+ '/settings/$page': typeof AppSettingsPageRoute
'/$team/': typeof AppTeamIndexRoute
'/settings/': typeof AppSettingsIndexRoute
'/$team/$projectId/deployments': typeof AppTeamProjectIdDeploymentsRoute
@@ -921,6 +933,7 @@ export interface FileRoutesByFullPath {
'/$team/postgres/$databaseId/extensions': typeof AppTeamPostgresDatabaseIdExtensionsRoute
'/$team/postgres/$databaseId/logs': typeof AppTeamPostgresDatabaseIdLogsRoute
'/$team/postgres/$databaseId/settings': typeof AppTeamPostgresDatabaseIdSettingsRoute
+ '/$team/sandboxes/servers/new': typeof AppTeamSandboxesServersNewRoute
'/$team/$projectId/$serviceId/': typeof AppTeamProjectIdServiceIdIndexRoute
'/$team/clusters/$clusterId/': typeof AppTeamClustersClusterIdIndexRoute
'/$team/kv/$databaseId/': typeof AppTeamKvDatabaseIdIndexRoute
@@ -954,12 +967,13 @@ export interface FileRoutesByTo {
'/$team/registry': typeof AppTeamRegistryRoute
'/$team/security': typeof AppTeamSecurityRoute
'/$team/templates': typeof AppTeamTemplatesRoute
+ '/backups': typeof AppSettingsBackupsRoute
+ '/email': typeof AppSettingsEmailRoute
+ '/github': typeof AppSettingsGithubRoute
+ '/observability': typeof AppSettingsObservabilityRoute
+ '/updates': typeof AppSettingsUpdatesRoute
'/oauth/authorize': typeof AppOauthAuthorizeRoute
- '/settings/backups': typeof AppSettingsBackupsRoute
- '/settings/email': typeof AppSettingsEmailRoute
- '/settings/github': typeof AppSettingsGithubRoute
- '/settings/observability': typeof AppSettingsObservabilityRoute
- '/settings/updates': typeof AppSettingsUpdatesRoute
+ '/settings/$page': typeof AppSettingsPageRoute
'/$team': typeof AppTeamIndexRoute
'/settings': typeof AppSettingsIndexRoute
'/$team/$projectId/deployments': typeof AppTeamProjectIdDeploymentsRoute
@@ -1033,6 +1047,7 @@ export interface FileRoutesByTo {
'/$team/postgres/$databaseId/extensions': typeof AppTeamPostgresDatabaseIdExtensionsRoute
'/$team/postgres/$databaseId/logs': typeof AppTeamPostgresDatabaseIdLogsRoute
'/$team/postgres/$databaseId/settings': typeof AppTeamPostgresDatabaseIdSettingsRoute
+ '/$team/sandboxes/servers/new': typeof AppTeamSandboxesServersNewRoute
'/$team/$projectId/$serviceId': typeof AppTeamProjectIdServiceIdIndexRoute
'/$team/clusters/$clusterId': typeof AppTeamClustersClusterIdIndexRoute
'/$team/kv/$databaseId': typeof AppTeamKvDatabaseIdIndexRoute
@@ -1045,12 +1060,12 @@ export interface FileRoutesById {
'/_app': typeof AppRouteWithChildren
'/_auth': typeof AuthRouteWithChildren
'/_app/$team': typeof AppTeamRouteWithChildren
+ '/_app/_settings': typeof AppSettingsRouteWithChildren
'/_app/api-keys': typeof AppApiKeysRoute
'/_app/cli-login': typeof AppCliLoginRoute
'/_app/invite': typeof AppInviteRoute
'/_app/mcp': typeof AppMcpRoute
'/_app/profile': typeof AppProfileRoute
- '/_app/settings': typeof AppSettingsRouteWithChildren
'/_auth/forgot-password': typeof AuthForgotPasswordRoute
'/_auth/login': typeof AuthLoginRoute
'/_auth/reset-password': typeof AuthResetPasswordRoute
@@ -1075,12 +1090,13 @@ export interface FileRoutesById {
'/_app/$team/security': typeof AppTeamSecurityRoute
'/_app/$team/settings': typeof AppTeamSettingsRouteWithChildren
'/_app/$team/templates': typeof AppTeamTemplatesRoute
+ '/_app/_settings/backups': typeof AppSettingsBackupsRoute
+ '/_app/_settings/email': typeof AppSettingsEmailRoute
+ '/_app/_settings/github': typeof AppSettingsGithubRoute
+ '/_app/_settings/observability': typeof AppSettingsObservabilityRoute
+ '/_app/_settings/updates': typeof AppSettingsUpdatesRoute
'/_app/oauth/authorize': typeof AppOauthAuthorizeRoute
- '/_app/settings/backups': typeof AppSettingsBackupsRoute
- '/_app/settings/email': typeof AppSettingsEmailRoute
- '/_app/settings/github': typeof AppSettingsGithubRoute
- '/_app/settings/observability': typeof AppSettingsObservabilityRoute
- '/_app/settings/updates': typeof AppSettingsUpdatesRoute
+ '/_app/settings/$page': typeof AppSettingsPageRoute
'/_app/$team/': typeof AppTeamIndexRoute
'/_app/settings/': typeof AppSettingsIndexRoute
'/_app/$team/$projectId/deployments': typeof AppTeamProjectIdDeploymentsRoute
@@ -1159,6 +1175,7 @@ export interface FileRoutesById {
'/_app/$team/postgres/$databaseId/extensions': typeof AppTeamPostgresDatabaseIdExtensionsRoute
'/_app/$team/postgres/$databaseId/logs': typeof AppTeamPostgresDatabaseIdLogsRoute
'/_app/$team/postgres/$databaseId/settings': typeof AppTeamPostgresDatabaseIdSettingsRoute
+ '/_app/$team/sandboxes/servers/new': typeof AppTeamSandboxesServersNewRoute
'/_app/$team/$projectId_/$serviceId/': typeof AppTeamProjectIdServiceIdIndexRoute
'/_app/$team/clusters/$clusterId/': typeof AppTeamClustersClusterIdIndexRoute
'/_app/$team/kv/$databaseId/': typeof AppTeamKvDatabaseIdIndexRoute
@@ -1176,7 +1193,6 @@ export interface FileRouteTypes {
| '/invite'
| '/mcp'
| '/profile'
- | '/settings'
| '/forgot-password'
| '/login'
| '/reset-password'
@@ -1200,12 +1216,13 @@ export interface FileRouteTypes {
| '/$team/security'
| '/$team/settings'
| '/$team/templates'
+ | '/backups'
+ | '/email'
+ | '/github'
+ | '/observability'
+ | '/updates'
| '/oauth/authorize'
- | '/settings/backups'
- | '/settings/email'
- | '/settings/github'
- | '/settings/observability'
- | '/settings/updates'
+ | '/settings/$page'
| '/$team/'
| '/settings/'
| '/$team/$projectId/deployments'
@@ -1284,6 +1301,7 @@ export interface FileRouteTypes {
| '/$team/postgres/$databaseId/extensions'
| '/$team/postgres/$databaseId/logs'
| '/$team/postgres/$databaseId/settings'
+ | '/$team/sandboxes/servers/new'
| '/$team/$projectId/$serviceId/'
| '/$team/clusters/$clusterId/'
| '/$team/kv/$databaseId/'
@@ -1317,12 +1335,13 @@ export interface FileRouteTypes {
| '/$team/registry'
| '/$team/security'
| '/$team/templates'
+ | '/backups'
+ | '/email'
+ | '/github'
+ | '/observability'
+ | '/updates'
| '/oauth/authorize'
- | '/settings/backups'
- | '/settings/email'
- | '/settings/github'
- | '/settings/observability'
- | '/settings/updates'
+ | '/settings/$page'
| '/$team'
| '/settings'
| '/$team/$projectId/deployments'
@@ -1396,6 +1415,7 @@ export interface FileRouteTypes {
| '/$team/postgres/$databaseId/extensions'
| '/$team/postgres/$databaseId/logs'
| '/$team/postgres/$databaseId/settings'
+ | '/$team/sandboxes/servers/new'
| '/$team/$projectId/$serviceId'
| '/$team/clusters/$clusterId'
| '/$team/kv/$databaseId'
@@ -1407,12 +1427,12 @@ export interface FileRouteTypes {
| '/_app'
| '/_auth'
| '/_app/$team'
+ | '/_app/_settings'
| '/_app/api-keys'
| '/_app/cli-login'
| '/_app/invite'
| '/_app/mcp'
| '/_app/profile'
- | '/_app/settings'
| '/_auth/forgot-password'
| '/_auth/login'
| '/_auth/reset-password'
@@ -1437,12 +1457,13 @@ export interface FileRouteTypes {
| '/_app/$team/security'
| '/_app/$team/settings'
| '/_app/$team/templates'
+ | '/_app/_settings/backups'
+ | '/_app/_settings/email'
+ | '/_app/_settings/github'
+ | '/_app/_settings/observability'
+ | '/_app/_settings/updates'
| '/_app/oauth/authorize'
- | '/_app/settings/backups'
- | '/_app/settings/email'
- | '/_app/settings/github'
- | '/_app/settings/observability'
- | '/_app/settings/updates'
+ | '/_app/settings/$page'
| '/_app/$team/'
| '/_app/settings/'
| '/_app/$team/$projectId/deployments'
@@ -1521,6 +1542,7 @@ export interface FileRouteTypes {
| '/_app/$team/postgres/$databaseId/extensions'
| '/_app/$team/postgres/$databaseId/logs'
| '/_app/$team/postgres/$databaseId/settings'
+ | '/_app/$team/sandboxes/servers/new'
| '/_app/$team/$projectId_/$serviceId/'
| '/_app/$team/clusters/$clusterId/'
| '/_app/$team/kv/$databaseId/'
@@ -1564,6 +1586,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppTeamRouteImport
parentRoute: typeof AppRoute
}
+ '/_app/_settings': {
+ id: '/_app/_settings'
+ path: ''
+ fullPath: '/'
+ preLoaderRoute: typeof AppSettingsRouteImport
+ parentRoute: typeof AppRoute
+ }
'/_app/api-keys': {
id: '/_app/api-keys'
path: '/api-keys'
@@ -1599,13 +1628,6 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppProfileRouteImport
parentRoute: typeof AppRoute
}
- '/_app/settings': {
- id: '/_app/settings'
- path: '/settings'
- fullPath: '/settings'
- preLoaderRoute: typeof AppSettingsRouteImport
- parentRoute: typeof AppRoute
- }
'/_auth/forgot-password': {
id: '/_auth/forgot-password'
path: '/forgot-password'
@@ -1774,55 +1796,62 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppTeamTemplatesRouteImport
parentRoute: typeof AppTeamRoute
}
- '/_app/oauth/authorize': {
- id: '/_app/oauth/authorize'
- path: '/oauth/authorize'
- fullPath: '/oauth/authorize'
- preLoaderRoute: typeof AppOauthAuthorizeRouteImport
- parentRoute: typeof AppRoute
- }
- '/_app/settings/': {
- id: '/_app/settings/'
- path: '/'
- fullPath: '/settings/'
- preLoaderRoute: typeof AppSettingsIndexRouteImport
- parentRoute: typeof AppSettingsRoute
- }
- '/_app/settings/backups': {
- id: '/_app/settings/backups'
+ '/_app/_settings/backups': {
+ id: '/_app/_settings/backups'
path: '/backups'
- fullPath: '/settings/backups'
+ fullPath: '/backups'
preLoaderRoute: typeof AppSettingsBackupsRouteImport
parentRoute: typeof AppSettingsRoute
}
- '/_app/settings/email': {
- id: '/_app/settings/email'
+ '/_app/_settings/email': {
+ id: '/_app/_settings/email'
path: '/email'
- fullPath: '/settings/email'
+ fullPath: '/email'
preLoaderRoute: typeof AppSettingsEmailRouteImport
parentRoute: typeof AppSettingsRoute
}
- '/_app/settings/github': {
- id: '/_app/settings/github'
+ '/_app/_settings/github': {
+ id: '/_app/_settings/github'
path: '/github'
- fullPath: '/settings/github'
+ fullPath: '/github'
preLoaderRoute: typeof AppSettingsGithubRouteImport
parentRoute: typeof AppSettingsRoute
}
- '/_app/settings/observability': {
- id: '/_app/settings/observability'
+ '/_app/_settings/observability': {
+ id: '/_app/_settings/observability'
path: '/observability'
- fullPath: '/settings/observability'
+ fullPath: '/observability'
preLoaderRoute: typeof AppSettingsObservabilityRouteImport
parentRoute: typeof AppSettingsRoute
}
- '/_app/settings/updates': {
- id: '/_app/settings/updates'
+ '/_app/_settings/updates': {
+ id: '/_app/_settings/updates'
path: '/updates'
- fullPath: '/settings/updates'
+ fullPath: '/updates'
preLoaderRoute: typeof AppSettingsUpdatesRouteImport
parentRoute: typeof AppSettingsRoute
}
+ '/_app/oauth/authorize': {
+ id: '/_app/oauth/authorize'
+ path: '/oauth/authorize'
+ fullPath: '/oauth/authorize'
+ preLoaderRoute: typeof AppOauthAuthorizeRouteImport
+ parentRoute: typeof AppRoute
+ }
+ '/_app/settings/': {
+ id: '/_app/settings/'
+ path: '/settings'
+ fullPath: '/settings/'
+ preLoaderRoute: typeof AppSettingsIndexRouteImport
+ parentRoute: typeof AppRoute
+ }
+ '/_app/settings/$page': {
+ id: '/_app/settings/$page'
+ path: '/settings/$page'
+ fullPath: '/settings/$page'
+ preLoaderRoute: typeof AppSettingsPageRouteImport
+ parentRoute: typeof AppRoute
+ }
'/_app/$team/$projectId/': {
id: '/_app/$team/$projectId/'
path: '/'
@@ -2390,6 +2419,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppTeamPostgresDatabaseIdSettingsRouteImport
parentRoute: typeof AppTeamPostgresDatabaseIdRoute
}
+ '/_app/$team/sandboxes/servers/new': {
+ id: '/_app/$team/sandboxes/servers/new'
+ path: '/sandboxes/servers/new'
+ fullPath: '/$team/sandboxes/servers/new'
+ preLoaderRoute: typeof AppTeamSandboxesServersNewRouteImport
+ parentRoute: typeof AppTeamRoute
+ }
'/_app/$team/clusters/$clusterId_/servers/new': {
id: '/_app/$team/clusters/$clusterId_/servers/new'
path: '/clusters/$clusterId/servers/new'
@@ -2664,6 +2700,7 @@ interface AppTeamRouteChildren {
AppTeamProjectsIndexRoute: typeof AppTeamProjectsIndexRoute
AppTeamSandboxesIndexRoute: typeof AppTeamSandboxesIndexRoute
AppTeamClustersClusterIdServerIdRoute: typeof AppTeamClustersClusterIdServerIdRoute
+ AppTeamSandboxesServersNewRoute: typeof AppTeamSandboxesServersNewRoute
AppTeamClustersClusterIdServersNewRoute: typeof AppTeamClustersClusterIdServersNewRoute
}
@@ -2717,6 +2754,7 @@ const AppTeamRouteChildren: AppTeamRouteChildren = {
AppTeamProjectsIndexRoute: AppTeamProjectsIndexRoute,
AppTeamSandboxesIndexRoute: AppTeamSandboxesIndexRoute,
AppTeamClustersClusterIdServerIdRoute: AppTeamClustersClusterIdServerIdRoute,
+ AppTeamSandboxesServersNewRoute: AppTeamSandboxesServersNewRoute,
AppTeamClustersClusterIdServersNewRoute:
AppTeamClustersClusterIdServersNewRoute,
}
@@ -2730,7 +2768,6 @@ interface AppSettingsRouteChildren {
AppSettingsGithubRoute: typeof AppSettingsGithubRoute
AppSettingsObservabilityRoute: typeof AppSettingsObservabilityRoute
AppSettingsUpdatesRoute: typeof AppSettingsUpdatesRoute
- AppSettingsIndexRoute: typeof AppSettingsIndexRoute
}
const AppSettingsRouteChildren: AppSettingsRouteChildren = {
@@ -2739,7 +2776,6 @@ const AppSettingsRouteChildren: AppSettingsRouteChildren = {
AppSettingsGithubRoute: AppSettingsGithubRoute,
AppSettingsObservabilityRoute: AppSettingsObservabilityRoute,
AppSettingsUpdatesRoute: AppSettingsUpdatesRoute,
- AppSettingsIndexRoute: AppSettingsIndexRoute,
}
const AppSettingsRouteWithChildren = AppSettingsRoute._addFileChildren(
@@ -2748,26 +2784,30 @@ const AppSettingsRouteWithChildren = AppSettingsRoute._addFileChildren(
interface AppRouteChildren {
AppTeamRoute: typeof AppTeamRouteWithChildren
+ AppSettingsRoute: typeof AppSettingsRouteWithChildren
AppApiKeysRoute: typeof AppApiKeysRoute
AppCliLoginRoute: typeof AppCliLoginRoute
AppInviteRoute: typeof AppInviteRoute
AppMcpRoute: typeof AppMcpRoute
AppProfileRoute: typeof AppProfileRoute
- AppSettingsRoute: typeof AppSettingsRouteWithChildren
AppIndexRoute: typeof AppIndexRoute
AppOauthAuthorizeRoute: typeof AppOauthAuthorizeRoute
+ AppSettingsPageRoute: typeof AppSettingsPageRoute
+ AppSettingsIndexRoute: typeof AppSettingsIndexRoute
}
const AppRouteChildren: AppRouteChildren = {
AppTeamRoute: AppTeamRouteWithChildren,
+ AppSettingsRoute: AppSettingsRouteWithChildren,
AppApiKeysRoute: AppApiKeysRoute,
AppCliLoginRoute: AppCliLoginRoute,
AppInviteRoute: AppInviteRoute,
AppMcpRoute: AppMcpRoute,
AppProfileRoute: AppProfileRoute,
- AppSettingsRoute: AppSettingsRouteWithChildren,
AppIndexRoute: AppIndexRoute,
AppOauthAuthorizeRoute: AppOauthAuthorizeRoute,
+ AppSettingsPageRoute: AppSettingsPageRoute,
+ AppSettingsIndexRoute: AppSettingsIndexRoute,
}
const AppRouteWithChildren = AppRoute._addFileChildren(AppRouteChildren)
diff --git a/dash/src/routes/_app/$team/backups.tsx b/dash/src/routes/_app/$team/backups.tsx
index 6e95d56..99ab9c7 100644
--- a/dash/src/routes/_app/$team/backups.tsx
+++ b/dash/src/routes/_app/$team/backups.tsx
@@ -11,7 +11,7 @@ import {
ArrowCounterClockwiseIcon,
GearSixIcon,
} from "@phosphor-icons/react";
-import { createFileRoute } from "@tanstack/react-router";
+import { createFileRoute, redirect } from "@tanstack/react-router";
import { useState, type ReactNode } from "react";
import { AppLink } from "~/components/ui/AppLink";
import { Badge, Dot, type Tone } from "~/components/ui/Badge";
@@ -37,8 +37,13 @@ import type { Database, TeamBackupRun, TeamBackups } from "~/lib/pyo/types";
// The team's backups on one screen: how things stand (with every database's
// backups in one graph), the databases, the volumes, and what ran lately.
// Adding goes through "Add backup"; restoring and where backups go have
-// their own buttons beside it.
+// their own buttons beside it. Backups of Pyo's own data are on /backups,
+// and the old ?tab=pyo link goes there.
export const Route = createFileRoute("/_app/$team/backups")({
+ beforeLoad: ({ search }) => {
+ if ((search as { tab?: unknown }).tab === "pyo")
+ throw redirect({ to: "/backups", replace: true });
+ },
head: () => ({ meta: seo({ title: "Backups" }) }),
component: () => (
diff --git a/dash/src/routes/_app/$team/clusters/$clusterId_/servers/new.tsx b/dash/src/routes/_app/$team/clusters/$clusterId_/servers/new.tsx
index 093ea5f..e4c6431 100644
--- a/dash/src/routes/_app/$team/clusters/$clusterId_/servers/new.tsx
+++ b/dash/src/routes/_app/$team/clusters/$clusterId_/servers/new.tsx
@@ -226,7 +226,7 @@ function JoinForm({
{!hetzner ? (
Connect Hetzner first.{" "}
-
+
Integrations
diff --git a/dash/src/routes/_app/$team/clusters/new.tsx b/dash/src/routes/_app/$team/clusters/new.tsx
index ca1abc0..cc65a88 100644
--- a/dash/src/routes/_app/$team/clusters/new.tsx
+++ b/dash/src/routes/_app/$team/clusters/new.tsx
@@ -310,7 +310,7 @@ function HetznerCluster({
{!conn ? (
Connect Hetzner first.{" "}
-
+
Integrations
diff --git a/dash/src/routes/_app/$team/integrations.tsx b/dash/src/routes/_app/$team/integrations.tsx
index 57687a4..a442913 100644
--- a/dash/src/routes/_app/$team/integrations.tsx
+++ b/dash/src/routes/_app/$team/integrations.tsx
@@ -1,29 +1,504 @@
-import { KeyIcon, TreeStructureIcon } from "@phosphor-icons/react";
import { createFileRoute, Link } from "@tanstack/react-router";
-import { RegistryLoginsPanel } from "~/components/pyo/RegistryLogins";
-import { SecretStoresPanel } from "~/components/pyo/SecretStores";
-import { CloudflarePanel } from "~/components/pyo/Cloudflare";
-import { TailscalePanel } from "~/components/pyo/Tailscale";
-import { GitlabRows } from "~/components/pyo/GitConnections";
-import { HetznerPanel } from "~/components/pyo/Hetzner";
-import { useEffect, useState, type ReactNode } from "react";
-import { Badge, ClusterStatusBadge } from "~/components/ui/Badge";
-import { Button } from "~/components/ui/Button";
-import { Page, PageHeader, Panel } from "~/components/ui/Panel";
+import { siGitlab, siHetzner, siTailscale, siVault } from "simple-icons";
+import { useCallback, useEffect, useState, type ReactNode } from "react";
+import { z } from "zod";
+import {
+ IntegrationCard,
+ docsUrl,
+ type IntegrationCardProps,
+} from "~/components/pyo/IntegrationCard";
+import {
+ AddLoginDialog,
+ RegistryLoginsSettings,
+} from "~/components/pyo/RegistryLogins";
+import { SecretStoresSettings, StoreDialog } from "~/components/pyo/SecretStores";
+import { CloudflareSettings } from "~/components/pyo/Cloudflare";
+import { TailscaleSettings } from "~/components/pyo/Tailscale";
+import { GitlabRows, TokenDialog } from "~/components/pyo/GitConnections";
+import { HetznerSettings, hetznerOf } from "~/components/pyo/Hetzner";
+import { useTeamAdmin } from "~/components/pyo/database/NewDatabase";
+import { Badge, type Tone } from "~/components/ui/Badge";
+import { Button, buttonClass } from "~/components/ui/Button";
+import { ConfirmDialog, Dialog } from "~/components/ui/Dialog";
+import type { MenuItem } from "~/components/ui/Menu";
+import { Page, PageHeader, NoMatches } from "~/components/ui/Panel";
+import { SearchInput } from "~/components/ui/Table";
+import { Tabs } from "~/components/ui/Tabs";
+import { timeAgo } from "~/lib/pyo/format";
import { seo } from "~/lib/seo";
-import { api, clustersIn, usePyo } from "~/lib/pyo/store";
+import {
+ api,
+ cloudflareIn,
+ roleIn,
+ usePyo,
+ type PyoState,
+} from "~/lib/pyo/store";
import { toast } from "~/lib/pyo/toast";
-import { BrandMark, GITHUB_LOGO } from "~/components/pyo/common";
-import { currentSlug } from "~/lib/pyo/paths";
+import { BrandMark, CLOUDFLARE_LOGO, DOCKER_LOGO, GITHUB_LOGO } from "~/components/pyo/common";
-// Everything Pyo connects to: third-party services (each a panel, so more
-// can be added), then Git, registries, clusters and secret stores.
-// Notification channels live in Settings.
+// Everything Pyo connects to, as cards: a filter by category, a search, and
+// each card's settings in a dialog. "/integrations#" scrolls to a card
+// and opens its settings. Notification channels live in Settings.
export const Route = createFileRoute("/_app/$team/integrations")({
+ // ?github=: back from GitHub (see useGithubReturn).
+ validateSearch: z.object({ github: z.string().optional().catch(undefined) }),
head: () => ({ meta: seo({ title: "Integrations" }) }),
component: Integrations,
});
+const CATEGORIES = [
+ "Git",
+ "DNS",
+ "Networking",
+ "Servers",
+ "Registries",
+ "Secrets",
+] as const;
+type Category = (typeof CATEGORIES)[number];
+
+type Integration = Omit & {
+ category: Category;
+ /** Its settings, in a dialog from the menu or a link to the card. */
+ settings?: { title: string; description?: string; body: ReactNode };
+};
+
+type Removing = "tailscale" | "hetzner" | "cloudflare" | null;
+
+const plural = (n: number, one: string, many = `${one}s`) =>
+ `${n} ${n === 1 ? one : many}`;
+
+const oldest = (xs: { createdAt: number }[]) =>
+ Math.min(...xs.map((x) => x.createdAt));
+
+/** Repositories the team's services deploy from, through a connection. */
+function reposLinked(s: PyoState, linked: (src: { installationId?: number; gitConnectionId?: string }) => boolean) {
+ const repos = new Set();
+ for (const a of s.services) {
+ if (a.teamId !== s.currentTeamId || a.source.type !== "git") continue;
+ if (linked(a.source)) repos.add(a.source.repoUrl);
+ }
+ return repos.size;
+}
+
+function Integrations() {
+ const s = usePyo();
+ const teamAdmin = useTeamAdmin();
+ const cfAdmin = ["owner", "admin"].includes(roleIn(s) ?? "");
+ const [filter, setFilter] = useState("all");
+ const [q, setQ] = useState("");
+ const [open, setOpen] = useState(null);
+ const [adding, setAdding] = useState<"gitlab" | "registries" | "secrets" | null>(null);
+ const [removing, setRemoving] = useState(null);
+ const [highlight, setHighlight] = useState(null);
+
+ useGithubReturn(() => setOpen("github"));
+
+ const team = s.currentTeamId;
+ const ts = s.tailscale;
+ const hetzner = hetznerOf(s);
+ const cloudServers = s.cloudServers.filter((c) => c.teamId === team);
+ const installs = s.githubInstallations.filter((g) => g.teamId === team);
+ const gitlab = s.gitConnections.filter((c) => c.teamId === team);
+ const cf = cloudflareIn(s);
+ const logins = s.registryLogins.filter((l) => l.teamId === team);
+ const stores = s.secretStores.filter((x) => x.teamId === team);
+
+ const connect = (label = "Connect", onClick?: () => void, disabled = false) => (
+
+ {label}
+
+ );
+ const askAdmin = Ask an admin ;
+ const settingsItem = (id: string, label = "Settings"): MenuItem => ({
+ label,
+ onSelect: () => setOpen(id),
+ });
+ const guideItem = (page: string): MenuItem => ({
+ label: "Setup guide",
+ href: docsUrl(page),
+ });
+
+ const test = {
+ tailscale: async () => {
+ const r = await api.testTailscale();
+ if ("error" in r) return toast({ tone: "error", title: r.error });
+ const bad = r.data.checks.filter((c) => !c.ok);
+ toast(
+ bad.length
+ ? { tone: "error", title: "Tailscale has problems", description: bad.map((c) => c.message).join(" ") }
+ : { title: "Tailscale works" },
+ );
+ },
+ hetzner: async () => {
+ const r = await api.testHetzner();
+ if ("error" in r) return toast({ tone: "error", title: r.error });
+ toast({ title: "Hetzner works", description: r.data.locations.map((l) => l.city).join(", ") });
+ },
+ cloudflare: async () => {
+ const r = await api.testCloudflare();
+ if (!r) return;
+ toast(
+ r.problems?.length
+ ? { tone: "error", title: "Cloudflare has problems", description: r.problems.join(" ") }
+ : { title: "Cloudflare token works" },
+ );
+ },
+ };
+
+ const tsStatus = (): { tone: Tone; label: string } | null => {
+ if (!ts) return null;
+ if (ts.hostMove?.state === "failed") return { tone: "danger", label: "This server couldn't move" };
+ if (ts.hostMove?.state === "moving") return { tone: "progress", label: "Moving this server" };
+ if (ts.dashboardError || ts.registryError) return { tone: "warning", label: "Needs attention" };
+ if (!ts.onTailnet) return { tone: "progress", label: "Joining tailnet" };
+ if (ts.dashboardName && !ts.dashboard) return { tone: "progress", label: "Getting certificate" };
+ return null;
+ };
+
+ const failedServers = cloudServers.filter((c) => c.state === "failed").length;
+ const gitlabProjects = gitlab.reduce((n, c) => n + (c.info.projects ?? 0), 0);
+ const gitlabProblems = gitlab.some((c) => c.info.problems?.length);
+ const cfLacking = cf?.zones.filter((z) => !z.canEdit).length ?? 0;
+ const lastUsed = Math.max(0, ...logins.map((l) => l.lastUsedAt ?? 0));
+ const storeUses = s.services.filter(
+ (a) => a.teamId === team && a.secretImports?.some((i) => stores.some((x) => x.id === i.storeId)),
+ ).length;
+
+ const all: Integration[] = [
+ {
+ id: "tailscale",
+ name: "Tailscale",
+ category: "Networking",
+ icon: ,
+ description: "Servers join your tailnet with it.",
+ connected: !!ts,
+ stat: ts?.suffix ? { label: "Tailnet", value: {ts.suffix} } : null,
+ status: tsStatus(),
+ docs: "tailscale",
+ action: s.user.isAdmin ? connect("Connect", () => setOpen("tailscale")) : askAdmin,
+ menu: [
+ settingsItem("tailscale"),
+ ...(s.user.isAdmin
+ ? [
+ { label: "Test", onSelect: () => void test.tailscale() },
+ guideItem("tailscale"),
+ { type: "separator" as const },
+ { label: "Remove", danger: true, onSelect: () => setRemoving("tailscale") },
+ ]
+ : [guideItem("tailscale")]),
+ ],
+ settings: { title: "Tailscale", description: "For the whole Pyo. Servers join your tailnet with it.", body: },
+ },
+ {
+ id: "hetzner",
+ name: "Hetzner",
+ category: "Servers",
+ icon: ,
+ description: "Pyo makes servers in your Hetzner project.",
+ connected: !!hetzner,
+ stat: hetzner ? { label: "Servers made", value: cloudServers.length } : null,
+ status: hetzner
+ ? failedServers
+ ? { tone: "danger", label: `${plural(failedServers, "server")} failed` }
+ : { tone: "success", label: `Added ${timeAgo(hetzner.createdAt)}` }
+ : null,
+ docs: "hetzner",
+ action: teamAdmin ? connect("Connect", () => setOpen("hetzner")) : askAdmin,
+ menu: [
+ settingsItem("hetzner"),
+ ...(teamAdmin
+ ? [
+ { label: "Test", onSelect: () => void test.hetzner() },
+ guideItem("hetzner"),
+ { type: "separator" as const },
+ { label: "Remove", danger: true, onSelect: () => setRemoving("hetzner") },
+ ]
+ : [guideItem("hetzner")]),
+ ],
+ settings: { title: "Hetzner", description: "For this team. Pyo makes servers in your Hetzner project.", body: },
+ },
+ {
+ id: "github",
+ name: "GitHub",
+ category: "Git",
+ icon: ,
+ description: "Private repos, and a deploy on every push.",
+ connected: !!s.github && installs.length > 0,
+ stat: { label: "Repos linked", value: reposLinked(s, (src) => installs.some((g) => g.id === src.installationId)) },
+ status: installs.length
+ ? { tone: "success", label: `Added ${timeAgo(oldest(installs))}` }
+ : null,
+ docs: "github",
+ action: s.github
+ ? connect("Connect", () => setOpen("github"))
+ : s.user.isAdmin
+ ? connect("Set up", () => setOpen("github"))
+ : askAdmin,
+ menu: [settingsItem("github", "Accounts"), guideItem("github")],
+ settings: { title: "GitHub", body: },
+ },
+ {
+ id: "gitlab",
+ name: "GitLab",
+ category: "Git",
+ icon: ,
+ description: "gitlab.com or your own GitLab, with an access token.",
+ connected: gitlab.length > 0,
+ stat: gitlabProjects
+ ? { label: "Projects", value: gitlabProjects }
+ : { label: "Repos linked", value: reposLinked(s, (src) => gitlab.some((c) => c.id === src.gitConnectionId)) },
+ status: gitlab.length
+ ? gitlabProblems
+ ? { tone: "warning", label: "Needs attention" }
+ : { tone: "success", label: `Added ${timeAgo(oldest(gitlab))}` }
+ : null,
+ docs: "gitlab",
+ action: connect("Connect", () => setAdding("gitlab")),
+ menu: [
+ settingsItem("gitlab", "Connections"),
+ { label: "Connect another GitLab", onSelect: () => setAdding("gitlab") },
+ guideItem("gitlab"),
+ ],
+ settings: { title: "GitLab", body: },
+ },
+ {
+ id: "cloudflare",
+ name: "Cloudflare",
+ category: "DNS",
+ icon: ,
+ description: "Creates DNS records for domains in your zones.",
+ connected: !!cf,
+ stat: cf ? { label: "Zones", value: cf.zones.length - cfLacking } : null,
+ status: cf
+ ? cfLacking
+ ? { tone: "warning", label: `${plural(cfLacking, "zone")} lack access` }
+ : { tone: "success", label: `Tested ${timeAgo(cf.testedAt)}` }
+ : null,
+ docs: "cloudflare",
+ action: cfAdmin ? connect("Connect", () => setOpen("cloudflare")) : askAdmin,
+ menu: [
+ settingsItem("cloudflare"),
+ ...(cfAdmin
+ ? [
+ { label: "Test", onSelect: () => void test.cloudflare() },
+ guideItem("cloudflare"),
+ { type: "separator" as const },
+ { label: "Remove", danger: true, onSelect: () => setRemoving("cloudflare") },
+ ]
+ : [guideItem("cloudflare")]),
+ ],
+ settings: { title: "Cloudflare", description: "Creates DNS records for domains in your zones.", body: },
+ },
+ {
+ id: "registries",
+ name: "Container registries",
+ category: "Registries",
+ icon: ,
+ description: "Logins for private images. Services pull with them.",
+ connected: logins.length > 0,
+ stat: { label: "Logins", value: logins.length },
+ status: logins.length
+ ? lastUsed
+ ? { tone: "success", label: `Used ${timeAgo(lastUsed)}` }
+ : { tone: "neutral", label: "Not used yet" }
+ : null,
+ docs: "registries",
+ action: connect("Add login", () => setAdding("registries")),
+ menu: [
+ settingsItem("registries", "Logins"),
+ { label: "Add login", onSelect: () => setAdding("registries") },
+ guideItem("registries"),
+ ],
+ settings: { title: "Container registries", body: },
+ },
+ {
+ id: "secrets",
+ name: "Secret stores",
+ category: "Secrets",
+ icon: ,
+ description: "Services import secrets from your store on each deploy.",
+ connected: stores.length > 0,
+ stat: { label: "Services importing", value: storeUses },
+ status: stores.length
+ ? { tone: "success", label: `Added ${timeAgo(oldest(stores))}` }
+ : null,
+ docs: "external-secrets",
+ action: connect("Connect", () => setAdding("secrets")),
+ menu: [
+ settingsItem("secrets", "Stores"),
+ { label: "Connect another store", onSelect: () => setAdding("secrets") },
+ guideItem("external-secrets"),
+ ],
+ settings: { title: "Secret stores", body: },
+ },
+ ];
+
+ // Connected first, each group in the order above.
+ const cards = [...all.filter((x) => x.connected), ...all.filter((x) => !x.connected)];
+ const connectedN = all.filter((x) => x.connected).length;
+ const used = CATEGORIES.filter((c) => all.some((x) => x.category === c));
+ const needle = q.trim().toLowerCase();
+ const shown = cards.filter(
+ (x) =>
+ (filter === "all" || (filter === "connected" ? x.connected : x.category === filter)) &&
+ (!needle ||
+ [x.name, x.category, x.description].some((t) => t.toLowerCase().includes(needle))),
+ );
+
+ // "/integrations#tailscale": show the card, ring it, open its settings.
+ const ids = all.map((x) => x.id).join(",");
+ const reveal = useCallback(() => {
+ const id = window.location.hash.slice(1);
+ if (!id || !ids.split(",").includes(id)) return;
+ setFilter("all");
+ setQ("");
+ setOpen(id);
+ setHighlight(id);
+ requestAnimationFrame(() => document.getElementById(id)?.scrollIntoView({ block: "center" }));
+ setTimeout(() => setHighlight((h) => (h === id ? null : h)), 2000);
+ }, [ids]);
+ useEffect(() => {
+ reveal();
+ window.addEventListener("hashchange", reveal);
+ return () => window.removeEventListener("hashchange", reveal);
+ }, [reveal]);
+ const close = () => {
+ setOpen(null);
+ if (window.location.hash)
+ window.history.replaceState(window.history.state, "", window.location.pathname + window.location.search);
+ };
+
+ const current = all.find((x) => x.id === open);
+
+ return (
+
+
+
+
+
+ ({ value: c, label: c })),
+ ]}
+ />
+
+
+
+ {connectedN} connected · {all.length - connectedN} available
+
+
+
+
+ {shown.length ? (
+
+ {shown.map(({ settings: _settings, ...x }) => (
+
+ ))}
+
+ ) : (
+
{
+ setQ("");
+ setFilter("all");
+ }}
+ />
+ )}
+
+
+ {current?.settings?.body}
+
+ {adding === "gitlab" ? setAdding(null)} /> : null}
+ setAdding(null)} />
+ {adding === "secrets" ? setAdding(null)} /> : null}
+ setRemoving(null)}
+ onConfirm={() => {
+ setRemoving(null);
+ void api.removeTailscale();
+ }}
+ title="Remove Tailscale?"
+ description="Pyo's names on your tailnet are deleted. Servers stay on it. The public dashboard opens again."
+ confirmLabel="Remove"
+ />
+ setRemoving(null)}
+ onConfirm={() => {
+ setRemoving(null);
+ void api.removeHetzner();
+ }}
+ title="Remove Hetzner?"
+ description="Servers Pyo made there keep running."
+ confirmLabel="Remove"
+ />
+ setRemoving(null)}
+ onConfirm={() => {
+ setRemoving(null);
+ void api.removeCloudflare();
+ toast({ title: "Cloudflare token removed" });
+ }}
+ title="Remove the Cloudflare token?"
+ description="Records Pyo made stay in Cloudflare."
+ confirmLabel="Remove"
+ />
+
+ );
+}
+
+const GITHUB_ERRORS: Record = {
+ taken: "Another team has connected this GitHub account.",
+ denied: "Your GitHub account can't access that installation.",
+ expired: "That took too long. Try again.",
+};
+
+/**
+ * Back from GitHub after installing the app (?github=connected, or an
+ * error): say how it went and open the GitHub card.
+ */
+function useGithubReturn(open: () => void) {
+ useEffect(() => {
+ const q = new URLSearchParams(window.location.search).get("github");
+ if (!q) return;
+ if (q === "connected")
+ toast({
+ title: "GitHub connected",
+ description:
+ "Pick a repository in New deploy. Pushes then deploy on their own.",
+ });
+ else
+ toast({
+ tone: "error",
+ title: GITHUB_ERRORS[q] ?? "GitHub didn't connect. Try again.",
+ });
+ window.history.replaceState(null, "", window.location.pathname);
+ open();
+ // Once, on arrival.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+}
+
function Row({
icon,
title,
@@ -49,67 +524,6 @@ function Row({
);
}
-function Integrations() {
- const s = usePyo();
- return (
-
-
-
-
-
-
-
-
-
-
-
-
-
- {clustersIn(s).map((c) => (
-
}
- title={
-
- {c.name}
-
- }
- body={`${c.connection.method === "agent" ? `Agent ${c.connection.agentVersion}` : "Kubeconfig"}`}
- right={ }
- />
- ))}
- {!clustersIn(s).length ? (
-
}
- title="No clusters"
- body="Add a cluster under Clusters."
- right={null}
- />
- ) : null}
-
-
-
-
-
}
- title="Cluster Secrets"
- body="Default. Stored in the target cluster."
- right={Active }
- />
-
-
-
-
- );
-}
-
/** The team's GitHub accounts, through this installation's GitHub App. */
function GithubRows() {
const s = usePyo();
@@ -117,33 +531,13 @@ function GithubRows() {
const installs = s.githubInstallations.filter(
(g) => g.teamId === s.currentTeamId,
);
- useEffect(() => {
- const q = new URLSearchParams(window.location.search).get("github");
- if (!q) return;
- if (q === "connected")
- toast({
- title: "GitHub connected",
- description:
- "Pick a repository in New deploy. Pushes then deploy on their own.",
- });
- else
- toast({
- tone: "error",
- title:
- q === "taken"
- ? "Another team has connected this GitHub account."
- : q === "denied"
- ? "Your GitHub account can't access that installation."
- : "GitHub didn't connect. Try again.",
- });
- window.history.replaceState(null, "", window.location.pathname);
- }, []);
const connect = async () => {
setBusy(true);
const r = await api.githubInstallUrl();
if (r) window.location.assign(r.url);
else setBusy(false);
};
+ // No GitHub App yet: the installation admin creates it in Settings.
if (!s.github)
return (
- Set up in Settings
+
+ Set up
) : (
Not set up
diff --git a/dash/src/routes/_app/$team/sandboxes/index.tsx b/dash/src/routes/_app/$team/sandboxes/index.tsx
index 32cec82..28b9483 100644
--- a/dash/src/routes/_app/$team/sandboxes/index.tsx
+++ b/dash/src/routes/_app/$team/sandboxes/index.tsx
@@ -9,11 +9,11 @@ import {
} from "@phosphor-icons/react";
import { createFileRoute, useNavigate } from "@tanstack/react-router";
import { useState } from "react";
+import { CloudProgress } from "~/components/pyo/Hetzner";
import {
act,
- AddSandboxServer,
- AddSandboxServerDialog,
SandboxStatus,
+ ServerState,
useNow,
useSandboxes,
} from "~/components/pyo/sandbox/common";
@@ -44,6 +44,7 @@ import {
type SandboxLimits,
} from "~/lib/pyo/sandbox";
import { roleIn, sandboxServersIn, usePyo } from "~/lib/pyo/store";
+import type { SandboxServer } from "~/lib/pyo/sandbox";
import { seo } from "~/lib/seo";
export const Route = createFileRoute("/_app/$team/sandboxes/")({
@@ -51,8 +52,10 @@ export const Route = createFileRoute("/_app/$team/sandboxes/")({
component: SandboxesPage,
});
-// The team's sandboxes. Until a sandbox server joins, the page is where you
-// add one. Sandboxes aren't in the snapshot, so the list polls on its own.
+// The team's sandboxes. With no sandbox server and no sandboxes it's an
+// empty state like the databases', whose button opens Add sandbox server
+// (sandboxes/servers/new). Servers still being set up show above the list.
+// Sandboxes aren't in the snapshot, so the list polls on its own.
type Filter = "live" | "running" | "paused" | "ended";
@@ -60,27 +63,16 @@ function SandboxesPage() {
const s = usePyo();
const servers = sandboxServersIn(s);
const q = useSandboxes(s.currentTeamId);
- const [adding, setAdding] = useState(false);
const [filter, setFilter] = useState("live");
const [template, setTemplate] = useState("");
const [deleting, setDeleting] = useState(null);
const role = roleIn(s);
const canUse = role !== "viewer";
+ const canManage = role === "owner" || role === "admin";
const now = useNow();
-
- // Until a server has joined, the page is where you add one (a joining
- // server stays here, so its command stays on screen).
- if (!servers.some((x) => x.state === "ready" || x.state === "unreachable"))
- return (
-
-
-
-
-
-
- );
+ const pending = servers.filter(
+ (x) => x.state === "joining" || x.state === "refused" || (x.cloud && x.cloud.state !== "ready"),
+ );
const all = q.status === "success" ? q.data.sandboxes : [];
const live = all.filter((x) => !ended(x));
@@ -101,11 +93,35 @@ function SandboxesPage() {
...(canUse
? [{ label: "New sandbox", icon: , href: "/sandboxes/new" }]
: []),
- { label: "Add server", icon: , onSelect: () => setAdding(true) },
+ ...(canManage
+ ? [{ label: "Add server", icon: , href: "/sandboxes/servers/new" }]
+ : []),
]}
/>
);
+ // Nothing yet: the same empty state as the databases'.
+ if (!servers.length && (q.status !== "success" || !all.length))
+ return (
+
+
+
+
+ Add a sandbox server
+
+ ) : undefined
+ }
+ />
+
+
+ );
+
return (
}
/>
+ {pending.length ? : null}
{q.status === "success" ? (
)}
- setAdding(false)} />
setDeleting(null)}
@@ -191,6 +207,35 @@ function SandboxesPage() {
);
}
+/** Sandbox servers not running sandboxes yet: Pyo's progress at Hetzner, or the join. */
+function PendingServers({ servers }: { servers: SandboxServer[] }) {
+ const s = usePyo();
+ return (
+
+
+
+ );
+}
+
function Usage({
limits,
inUse,
diff --git a/dash/src/routes/_app/$team/sandboxes/new.tsx b/dash/src/routes/_app/$team/sandboxes/new.tsx
index 5314d11..1052a94 100644
--- a/dash/src/routes/_app/$team/sandboxes/new.tsx
+++ b/dash/src/routes/_app/$team/sandboxes/new.tsx
@@ -1,4 +1,4 @@
-import { CaretDownIcon, CodeIcon } from "@phosphor-icons/react";
+import { CaretDownIcon, CircleNotchIcon, CodeIcon, HardDrivesIcon } from "@phosphor-icons/react";
import { createFileRoute, useNavigate } from "@tanstack/react-router";
import { useState } from "react";
import { useSandboxes } from "~/components/pyo/sandbox/common";
@@ -6,22 +6,24 @@ import { AppLink } from "~/components/ui/AppLink";
import { Button, ButtonLink } from "~/components/ui/Button";
import { Field, Segmented, Select, Switch, Textarea } from "~/components/ui/Field";
import { Highlighted } from "~/components/ui/Highlighted";
-import { Callout, Page, PageHeader, Panel } from "~/components/ui/Panel";
+import { Callout, EmptyState, Page, PageHeader, Panel } from "~/components/ui/Panel";
import { Reveal } from "~/components/ui/Reveal";
import { CopyButton } from "~/components/ui/Table";
import { parseEnv } from "~/lib/pyo/dotenv";
import { currentSlug } from "~/lib/pyo/paths";
import { useQuery } from "~/lib/pyo/query";
import {
+ defaultTemplate,
LIFETIMES,
SIZES,
rememberConnection,
sandboxApi,
sdkSnippet,
sizeLabel,
+ templateImage,
type CreateSandbox,
} from "~/lib/pyo/sandbox";
-import { projectsIn, sandboxServersIn, usePyo } from "~/lib/pyo/store";
+import { projectsIn, roleIn, sandboxServersIn, usePyo } from "~/lib/pyo/store";
import { seo } from "~/lib/seo";
export const Route = createFileRoute("/_app/$team/sandboxes/new")({
@@ -41,6 +43,51 @@ function pairs(text: string) {
}
function NewSandbox() {
+ const s = usePyo();
+ const servers = sandboxServersIn(s);
+ if (servers.some((x) => x.state === "ready")) return ;
+ // No server that runs sandboxes yet: the form waits for one.
+ const joining = servers.find((x) => x.state === "joining");
+ const canManage = ["owner", "admin"].includes(roleIn(s) ?? "");
+ return (
+
+
+
+ {joining ? (
+
+ See {joining.name}
+
+ }
+ />
+ ) : (
+
+ Add a sandbox server
+
+ ) : undefined
+ }
+ />
+ )}
+
+
+ );
+}
+
+function NewSandboxForm() {
const s = usePyo();
const navigate = useNavigate();
const team = s.currentTeamId;
@@ -65,7 +112,7 @@ function NewSandbox() {
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
- const picked = template || ready.find((t) => t.state === "ready")?.name || "";
+ const picked = template || defaultTemplate(ready);
const sz = SIZES.find((x) => x.value === size)!;
const limits = list.status === "success" ? list.data.limits : null;
const inUse = list.status === "success" ? list.data.inUse : null;
@@ -137,7 +184,7 @@ function NewSandbox() {
{ready.map((t) => (
{t.name}
- {t.source?.image ? ` · ${t.source.image}` : ""}
+ {templateImage(t) ? ` · ${templateImage(t)}` : ""}
{t.state !== "ready" ? ` (${t.state})` : ""}
))}
diff --git a/dash/src/routes/_app/$team/sandboxes/servers/new.tsx b/dash/src/routes/_app/$team/sandboxes/servers/new.tsx
new file mode 100644
index 0000000..8baccf6
--- /dev/null
+++ b/dash/src/routes/_app/$team/sandboxes/servers/new.tsx
@@ -0,0 +1,14 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { AddSandboxServerFlow } from "~/components/pyo/sandbox/AddServerFlow";
+import { Page, PageHeader } from "~/components/ui/Panel";
+import { seo } from "~/lib/seo";
+
+export const Route = createFileRoute("/_app/$team/sandboxes/servers/new")({
+ head: () => ({ meta: seo({ title: "Add sandbox server" }) }),
+ component: () => (
+
+
+
+
+ ),
+});
diff --git a/dash/src/routes/_app/$team/sandboxes/templates.tsx b/dash/src/routes/_app/$team/sandboxes/templates.tsx
index e7593d2..fe53da4 100644
--- a/dash/src/routes/_app/$team/sandboxes/templates.tsx
+++ b/dash/src/routes/_app/$team/sandboxes/templates.tsx
@@ -10,7 +10,7 @@ import { EmptyState, ErrorState, Page, PageHeader, Panel } from "~/components/ui
import { Table, Td, Th, Tr } from "~/components/ui/Table";
import { megabytes, timeAgo } from "~/lib/pyo/format";
import { useQuery } from "~/lib/pyo/query";
-import { sandboxApi, type SandboxTemplate, type SandboxTemplateServer } from "~/lib/pyo/sandbox";
+import { sandboxApi, templateImage, type SandboxTemplate, type SandboxTemplateServer } from "~/lib/pyo/sandbox";
import { roleIn, usePyo } from "~/lib/pyo/store";
import { toast } from "~/lib/pyo/toast";
import { seo } from "~/lib/seo";
@@ -60,7 +60,7 @@ function TemplatesPage() {
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [removing, setRemoving] = useState(null);
- const canRemove = (t: SandboxTemplate) => (t.teamId ? canCreate : !!s.user.isAdmin);
+ const canRemove = (t: SandboxTemplate) => !t.builtin && (t.teamId ? canCreate : !!s.user.isAdmin);
const nameError = name && !NAME.test(name) ? "Lowercase letters, digits, dashes, dots and underscores." : "";
const create = async () => {
@@ -108,7 +108,7 @@ function TemplatesPage() {
{t.name}
{t.teamId ? null : Installation }
- {t.source?.image ?? "—"}
+ {templateImage(t) || "—"}
diff --git a/dash/src/routes/_app/$team/templates.tsx b/dash/src/routes/_app/$team/templates.tsx
index 5dac1cc..1c410bf 100644
--- a/dash/src/routes/_app/$team/templates.tsx
+++ b/dash/src/routes/_app/$team/templates.tsx
@@ -14,7 +14,7 @@ function TemplatesPage() {
const s = usePyo();
const navigate = useNavigate();
return (
-
+
URLs redirect here.
// Team settings stay under each team (/:team/settings).
-export const Route = createFileRoute("/_app/settings")({
+export const Route = createFileRoute("/_app/_settings")({
head: () => ({ meta: seo({ title: "Settings" }) }),
component: InstallationSettings,
});
@@ -43,19 +46,15 @@ function GithubIcon({ size = 14 }: { size?: number }) {
}
const TABS: {
- to: string;
+ to: (typeof SETTINGS_PAGES)[number];
label: string;
icon: Icon | ComponentType<{ size?: number }>;
}[] = [
- { to: "/settings/backups", label: "Backups", icon: CloudArrowUpIcon },
- { to: "/settings/email", label: "Email", icon: EnvelopeSimpleIcon },
- {
- to: "/settings/observability",
- label: "Observability",
- icon: ChartLineUpIcon,
- },
- { to: "/settings/github", label: "GitHub", icon: GithubIcon },
- { to: "/settings/updates", label: "Updates", icon: ArrowCircleUpIcon },
+ { to: "/backups", label: "Backups", icon: CloudArrowUpIcon },
+ { to: "/email", label: "Email", icon: EnvelopeSimpleIcon },
+ { to: "/github", label: "GitHub", icon: GithubIcon },
+ { to: "/observability", label: "Observability", icon: ChartLineUpIcon },
+ { to: "/updates", label: "Updates", icon: ArrowCircleUpIcon },
];
function InstallationSettings() {
@@ -89,7 +88,7 @@ function InstallationSettings() {
>
{t.label}
- {t.to === "/settings/updates" && updateAvailable ? (
+ {t.to === "/updates" && updateAvailable ? (
(
(
(
-
+
(
diff --git a/dash/src/routes/_app/settings/updates.tsx b/dash/src/routes/_app/_settings/updates.tsx
similarity index 79%
rename from dash/src/routes/_app/settings/updates.tsx
rename to dash/src/routes/_app/_settings/updates.tsx
index 0a69da9..937a217 100644
--- a/dash/src/routes/_app/settings/updates.tsx
+++ b/dash/src/routes/_app/_settings/updates.tsx
@@ -1,9 +1,9 @@
import { createFileRoute } from "@tanstack/react-router";
import { UpdatesSection } from "~/components/pyo/UpdateSettings";
import { ManagedOr } from "~/components/pyo/Managed";
-import { SettingsSection } from "../settings";
+import { SettingsSection } from "../_settings";
-export const Route = createFileRoute("/_app/settings/updates")({
+export const Route = createFileRoute("/_app/_settings/updates")({
component: () => (
{
const team = currentTeam(getState());
+ if (team && search.github)
+ throw redirect({
+ to: "/$team/integrations",
+ params: { team: team.slug },
+ search: { github: search.github },
+ hash: "github",
+ replace: true,
+ });
if (team)
throw redirect({
to: search.onboarding ? "/$team/onboarding" : "/$team",
diff --git a/dash/src/routes/_app/settings/$page.tsx b/dash/src/routes/_app/settings/$page.tsx
new file mode 100644
index 0000000..d1fd202
--- /dev/null
+++ b/dash/src/routes/_app/settings/$page.tsx
@@ -0,0 +1,15 @@
+import { createFileRoute, redirect } from "@tanstack/react-router";
+import { movedSettingsHref, SETTINGS_PAGES } from "~/lib/pyo/paths";
+
+// The settings pages moved to the root (/settings/email is now /email).
+// Old links land on the same page with their query kept, so GitHub's
+// /settings/github?github= still works. Unknown pages open Settings.
+export const Route = createFileRoute("/_app/settings/$page")({
+ beforeLoad: ({ params, location }) => {
+ const page = SETTINGS_PAGES.find((p) => p === `/${params.page}`);
+ throw redirect({
+ href: movedSettingsHref(page ?? SETTINGS_PAGES[0], location),
+ replace: true,
+ });
+ },
+});
diff --git a/dash/src/routes/_app/settings/index.tsx b/dash/src/routes/_app/settings/index.tsx
index 0a72293..7f02912 100644
--- a/dash/src/routes/_app/settings/index.tsx
+++ b/dash/src/routes/_app/settings/index.tsx
@@ -1,8 +1,9 @@
import { createFileRoute, redirect } from "@tanstack/react-router";
+import { SETTINGS_PAGES } from "~/lib/pyo/paths";
-// /settings opens on Backups.
+// /settings opens on the first settings page.
export const Route = createFileRoute("/_app/settings/")({
beforeLoad: () => {
- throw redirect({ to: "/settings/backups", replace: true });
+ throw redirect({ to: SETTINGS_PAGES[0], replace: true });
},
});
diff --git a/deploy/images/sandbox-base.Dockerfile b/deploy/images/sandbox-base.Dockerfile
new file mode 100644
index 0000000..19aaadc
--- /dev/null
+++ b/deploy/images/sandbox-base.Dockerfile
@@ -0,0 +1,75 @@
+# Pyo Sandbox's built-in template `base`: Debian 13 with the tools agents
+# expect. The release publishes it multi-arch as
+# ghcr.io/upsetdev/pyo/sandbox-base:, and release.json pins its
+# digest (images.sandboxBase), which Pyo seeds the `base` template from
+# (crates/pyo-api/src/sandbox/base.rs).
+#
+# Templates don't use an image's User, Env or WorkingDir yet: pyo-envd
+# runs every command as root in /root, with HOME=/root, LANG=C.UTF-8 and
+# PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin. So
+# everything here works as root with that PATH, and nothing relies on ENV.
+#
+# Versions are pinned. Bump them together, and Debian's point release
+# with them. About 790 MB unpacked on amd64 (about 275 MB to pull), most of
+# it the compilers, Node.js and pnpm.
+
+ARG NODE_VERSION=26.13.0
+ARG NODE_SHA256_AMD64=d28a96af6c38596f6d9d97c0ee531c13d238d757742387d1928de822d959e997
+ARG NODE_SHA256_ARM64=406fe3fbde1c054adff8b297c16fdb5af6659db327c1ac8acd46b56ef1af9a92
+ARG COREPACK_VERSION=0.36.0
+ARG PNPM_VERSION=12.16.1
+
+# uv and uvx, Astral's static binaries for each architecture.
+FROM ghcr.io/astral-sh/uv:0.13.6@sha256:af22a52f5eafef3d78e551599d405c02df61919d18028f0cea8eda93bc9986d0 AS uv
+
+FROM debian:13.7-slim@sha256:a72b4433de1cf7d6bae0c4cee74d0547f58601a643b3375f070c51600d63b6b2
+ARG NODE_VERSION NODE_SHA256_AMD64 NODE_SHA256_ARM64 COREPACK_VERSION PNPM_VERSION
+ARG TARGETARCH
+LABEL org.opencontainers.image.source=https://github.com/upsetdev/pyo \
+ org.opencontainers.image.description="Pyo Sandbox's base template: Debian 13 with git, Python, uv, Node.js and pnpm"
+
+# C.UTF-8 is built into Debian's libc, so it needs no locales package.
+ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8
+
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ bash-completion build-essential ca-certificates curl git iproute2 \
+ iputils-ping jq less lsof netcat-openbsd openssh-client procps python3 \
+ python3-pip python3-venv sudo tzdata unzip vim-tiny wget xz-utils zip; \
+ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb; \
+ # A sandbox is thrown away: let pip install for the system Python,
+ # as agents expect.
+ rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED; \
+ ln -s /usr/bin/python3 /usr/local/bin/python; \
+ ln -s /usr/bin/vim.tiny /usr/local/bin/vim
+
+# Node.js LTS, the official build, checked against its sha256.
+RUN set -eux; \
+ case "$TARGETARCH" in \
+ amd64) arch=x64; sum=$NODE_SHA256_AMD64 ;; \
+ arm64) arch=arm64; sum=$NODE_SHA256_ARM64 ;; \
+ *) echo "No Node.js for $TARGETARCH" >&2; exit 1 ;; \
+ esac; \
+ f="node-v$NODE_VERSION-linux-$arch.tar.xz"; \
+ curl -fsSLo "/tmp/$f" "https://nodejs.org/dist/v$NODE_VERSION/$f"; \
+ echo "$sum /tmp/$f" | sha256sum -c -; \
+ tar -xJf "/tmp/$f" -C /usr/local --strip-components=1 --no-same-owner \
+ --exclude='*/CHANGELOG.md' --exclude='*/README.md' --exclude='*/LICENSE' --exclude='*/doc' \
+ --exclude='*/include/node/openssl/archs'; \
+ rm "/tmp/$f"; \
+ # pnpm through corepack, which Node no longer ships. pnpm is fetched
+ # now, so it works in a sandbox without network too.
+ npm install -g "corepack@$COREPACK_VERSION"; \
+ corepack enable pnpm; \
+ COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack install -g "pnpm@$PNPM_VERSION"; \
+ npm cache clean --force; \
+ rm -rf /tmp/* /root/.npm; \
+ node --version; npm --version; pnpm --version
+
+COPY --from=uv /uv /uvx /usr/local/bin/
+
+RUN uv --version && python3 --version && git --version
+
+WORKDIR /root
+CMD ["bash"]
diff --git a/deploy/k8s/pyo.yaml b/deploy/k8s/pyo.yaml
index 46d92e8..dc0b426 100644
--- a/deploy/k8s/pyo.yaml
+++ b/deploy/k8s/pyo.yaml
@@ -209,6 +209,8 @@ spec:
- { name: PYO_POSTGRES_IMAGE_18, value: "__IMAGES__/postgres:18-__TAG__" }
- { name: PYO_MYSQL_OPERATOR_IMAGE, value: "__IMAGES__/mysql-operator:__TAG__" }
- { name: PYO_MYSQL_BACKUP_IMAGE, value: "__IMAGES__/mysql-backup:__TAG__" }
+ # Pyo Sandbox's built-in template `base` (sandbox/base.rs).
+ - { name: PYO_SANDBOX_BASE_IMAGE, value: "__IMAGES__/sandbox-base:__TAG__" }
- { name: RUST_LOG, value: "pyo_api=info" }
- { name: CLICKHOUSE_URL, value: "http://pyo-clickhouse.pyo-system.svc:8123" }
- { name: CLICKHOUSE_USER, value: pyo }
diff --git a/deploy/release-notices.json b/deploy/release-notices.json
index 33cb360..1db5c96 100644
--- a/deploy/release-notices.json
+++ b/deploy/release-notices.json
@@ -1,5 +1,5 @@
{
- "version": "0.4.0",
+ "version": "0.4.1",
"minFrom": null,
"manualOnly": false,
"beforeYouUpdate": []
diff --git a/deploy/sandbox.sh b/deploy/sandbox.sh
index 19f586c..dfb9a8f 100644
--- a/deploy/sandbox.sh
+++ b/deploy/sandbox.sh
@@ -23,6 +23,10 @@
# --edge-cert F the certificate for *.D (PEM, full chain). Default:
# /etc/pyo-sandbox/edge/cert.pem
# --edge-key F its private key. Default: /etc/pyo-sandbox/edge/key.pem
+# --gvisor-bundle X gVisor's files from a file or URL instead of the
+# release (PYO_SANDBOX_GVISOR_BUNDLE), for servers without
+# KVM. Checked against the signed release.json of
+# --version, or not at all with --dev.
#
# Run it again with --version to update the daemon. A server that has joined
# needs no code then. Running sandboxes keep running.
@@ -33,7 +37,9 @@
#
# It joins your tailnet, installs the daemon (pyo-sandbox), pyo-envd,
# Firecracker, its jailer and the guest kernel, and checks what this server
-# can run: Firecracker where it has KVM, gVisor elsewhere. Then it trades
+# can run: Firecracker where it has KVM, gVisor elsewhere. A server without
+# KVM gets gVisor's runsc too, from pyo-sandbox-gvisor-linux-.tar.gz
+# beside the sandbox files, and is checked again. Then it trades
# the code for the daemon's config at Pyo over the tailnet, and starts the
# daemon. The daemon answers only on the tailnet.
#
@@ -266,6 +272,7 @@ RELEASE_ISSUER=https://token.actions.githubusercontent.com
defaults() {
PYO="" SERVER="" NAME="" CODE="" VERSION=${PYO_VERSION:-} BUNDLE=${PYO_SANDBOX_BUNDLE:-}
BUNDLE_SHA256=${PYO_SANDBOX_BUNDLE_SHA256:-} DEV=${PYO_SANDBOX_DEV:-}
+ GVISOR_BUNDLE=${PYO_SANDBOX_GVISOR_BUNDLE:-} RUNTIME="" KVM="" BOOT="" REASON=""
TS_AUTHKEY=${TS_AUTHKEY:-} UNINSTALL=""
RELEASES=${PYO_RELEASES:-https://github.com/upsetdev/pyo/releases/download}
LIB=/usr/local/lib/pyo-sandbox BIN=/usr/local/bin/pyo-sandbox
@@ -294,6 +301,7 @@ parse_args() {
--version) VERSION="${2:-}"; shift 2 ;;
--bundle) BUNDLE="${2:-}"; shift 2 ;;
--bundle-sha256) BUNDLE_SHA256="${2:-}"; shift 2 ;;
+ --gvisor-bundle) GVISOR_BUNDLE="${2:-}"; shift 2 ;;
--dev) DEV=1; shift ;;
--edge-domain) EDGE_DOMAIN="${2:-}"; shift 2 ;;
--edge-cert) EDGE_CERT="${2:-}"; shift 2 ;;
@@ -301,7 +309,7 @@ parse_args() {
--uninstall) UNINSTALL=1; shift ;;
-h | --help)
echo "Usage: sandbox.sh --pyo HOST:PORT --server ID --name NAME --code CODE --version X [--tailscale-key KEY]"
- echo " [--bundle FILE|URL [--bundle-sha256 SUM | --dev]]"
+ echo " [--bundle FILE|URL [--bundle-sha256 SUM | --dev]] [--gvisor-bundle FILE|URL]"
echo " [--edge-domain DOMAIN [--edge-cert FILE] [--edge-key FILE]]"
echo " sandbox.sh --uninstall"
exit 0
@@ -406,10 +414,12 @@ release_json() {
fail "release.json's signature doesn't check out: it isn't the one Pyo's release workflow signed. Nothing was changed."
}
-# The sandbox files' SHA-256 for this architecture, from release.json's
-# "sandbox" (scripts/release-manifest.ts writes it pretty-printed).
+# The SHA-256 of this architecture's files under a key of release.json:
+# "sandbox" (the default) or "sandboxGvisor". scripts/release-manifest.ts
+# writes it pretty-printed.
release_sum() {
- sed -n '/^ "sandbox": {/,/^ }/s/^ "'"$ARCH"'": "\([0-9a-f]\{64\}\)",*$/\1/p' "$WORK/release.json" | head -n1
+ local key=${1:-sandbox}
+ sed -n '/^ "'"$key"'": {/,/^ }/s/^ "'"$ARCH"'": "\([0-9a-f]\{64\}\)",*$/\1/p' "$WORK/release.json" | head -n1
}
# The sandbox server's files, checked, before anything changes:
@@ -447,6 +457,50 @@ fetch_bundle() {
done
}
+# gVisor's runsc and its gvisor-bin/, checked, for a server without KVM:
+# the release's against its signed release.json, files from
+# --gvisor-bundle the same way, or not at all with --dev.
+fetch_gvisor() {
+ local tgz=$WORK/gvisor.tar.gz src=$GVISOR_BUNDLE sum=""
+ if [ -z "$DEV" ]; then
+ [ -n "$VERSION" ] || fail "gVisor's files are checked against a release: give --version X, or --dev for files you built yourself."
+ [ -s "$WORK/release.json" ] || release_json
+ sum=$(release_sum sandboxGvisor)
+ [ -n "$sum" ] || fail "Pyo $VERSION doesn't list gVisor's checksum, so it can't be checked. Use a newer release."
+ fi
+ if [ -n "$src" ] && [ -f "$src" ]; then
+ cp "$src" "$tgz"
+ else
+ [ -n "$src" ] || src=$(release_url "pyo-sandbox-gvisor-linux-$ARCH.tar.gz")
+ quietly "Downloading gVisor…" curl -fsSL --proto '=https,http' --retry 3 -o "$tgz" "$src" ||
+ fail "Couldn't download gVisor from $src. Check this machine's internet connection and run the same command again."
+ fi
+ if [ -n "$sum" ]; then
+ echo "$sum $tgz" | sha256sum -c --quiet - >/dev/null 2>&1 ||
+ fail "gVisor's files don't match their checksum. Run the same command again."
+ else
+ echo "gVisor's files aren't checked (--dev)."
+ fi
+ rm -rf "$WORK/gvisor" && mkdir -p "$WORK/gvisor"
+ tar -xzf "$tgz" -C "$WORK/gvisor" 2>/dev/null || fail "gVisor's files aren't a .tar.gz. Run the same command again."
+ [ -s "$WORK/gvisor/runsc" ] && [ -d "$WORK/gvisor/gvisor-bin" ] || fail "gVisor's files have no runsc."
+}
+
+# runsc finds gvisor-bin/ next to itself. Into place by rename, so running
+# sandboxes keep the binaries they started with.
+install_gvisor() {
+ install -d -m 755 "$LIB"
+ rm -rf "$LIB/.gvisor-bin.new"
+ cp -R "$WORK/gvisor/gvisor-bin" "$LIB/.gvisor-bin.new"
+ rm -rf "$LIB/gvisor-bin" && mv "$LIB/.gvisor-bin.new" "$LIB/gvisor-bin"
+ install -m 755 "$WORK/gvisor/runsc" "$LIB/.runsc.new" && mv -f "$LIB/.runsc.new" "$LIB/runsc"
+}
+
+add_gvisor() {
+ fetch_gvisor
+ install_gvisor
+}
+
join_tailnet() {
command -v tailscale >/dev/null || quietly "Installing Tailscale…" tailscale_install ||
fail "Tailscale didn't install. Run the same command again."
@@ -486,7 +540,7 @@ detect_config() {
cat < "$WORK/capability.json"; }
-# What this server can run. Stops here, before the code is used, when it
-# can run neither Firecracker nor gVisor. The detection's own output shows
-# only when it fails.
-detect() {
+# The detection once, its fields into RUNTIME, KVM, BOOT and REASON.
+detect_once() {
detect_config > "$WORK/detect.json"
quietly "Checking what this server can run…" detect_run ||
fail "Couldn't check what this server can run. Run the same command again."
- local runtime kvm boot reason
- runtime=$(report_field runtime "$WORK/capability.json")
- kvm=$(report_field kvm "$WORK/capability.json")
- boot=$(report_field bootMs "$WORK/capability.json")
- case "$runtime:$kvm" in
- firecracker:native) echo "Firecracker on KVM. A test VM booted in ${boot:-?} ms." ;;
- firecracker:*) echo "Firecracker on nested KVM. A test VM booted in ${boot:-?} ms, so sandboxes start slower." ;;
+ RUNTIME=$(report_field runtime "$WORK/capability.json")
+ KVM=$(report_field kvm "$WORK/capability.json")
+ BOOT=$(report_field bootMs "$WORK/capability.json")
+ REASON=$(report_field reason "$WORK/capability.json")
+}
+
+# What this server can run. Without Firecracker it gets gVisor and is
+# checked again. Stops here, before the code is used, when it can run
+# neither. The detection's own output shows only when it fails.
+detect() {
+ detect_once
+ if [ "$RUNTIME" != firecracker ] && [ ! -x "$LIB/runsc" ]; then
+ add_gvisor
+ detect_once
+ fi
+ case "$RUNTIME:$KVM" in
+ firecracker:native) echo "Firecracker on KVM. A test VM booted in ${BOOT:-?} ms." ;;
+ firecracker:*) echo "Firecracker on nested KVM. A test VM booted in ${BOOT:-?} ms, so sandboxes start slower." ;;
gvisor:*) echo "gVisor, because this server has no usable KVM." ;;
- *)
- reason=$(report_field reason "$WORK/capability.json")
- fail "This server can't run sandboxes. ${reason:-It has neither KVM nor gVisor.}"
- ;;
+ *) fail "This server can't run sandboxes. ${REASON:-It has neither KVM nor gVisor.}" ;;
esac
}
@@ -654,10 +714,22 @@ uninstall() {
say "Removing the Pyo sandbox daemon"
systemctl disable --now pyo-sandbox >/dev/null 2>&1 || true
# Its sandboxes: the VMs it started outlive it by design.
- local p
+ local p c m
for p in $(pgrep -x firecracker || true); do
case "$(readlink "/proc/$p/cwd" 2>/dev/null)" in "$DATA"/*) kill -9 "$p" 2>/dev/null || true ;; esac
done
+ # gVisor's sandboxes, their disks and network namespaces.
+ if [ -x "$LIB/runsc" ]; then
+ for c in $("$LIB/runsc" --root="$DATA/runsc" list --quiet 2>/dev/null || true); do
+ "$LIB/runsc" --root="$DATA/runsc" delete --force "$c" >/dev/null 2>&1 || true
+ done
+ fi
+ for m in $(awk -v d="$DATA/" 'index($5, d) == 1 { print $5 }' /proc/self/mountinfo | sort -r); do
+ umount "$m" 2>/dev/null || umount -l "$m" 2>/dev/null || true
+ done
+ for c in $(ip netns list 2>/dev/null | awk '/^pyosbx[0-9]+/ { print $1 }'); do
+ ip netns del "$c" 2>/dev/null || true
+ done
# Its firewall rules: the nftables table, and the iptables rules it adds
# when another firewall drops forwarded traffic (they carry its comment).
nft delete table inet pyo_sandbox >/dev/null 2>&1 || true
@@ -704,6 +776,8 @@ main() {
join_tailnet
make_user
install_files
+ # A gVisor server's runsc follows the release too.
+ [ ! -x "$LIB/runsc" ] || add_gvisor
detect
if [ -n "$CODE" ]; then
redeem
diff --git a/scripts/pyo-up.sh b/scripts/pyo-up.sh
index b78a9b8..21cd360 100755
--- a/scripts/pyo-up.sh
+++ b/scripts/pyo-up.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Build Pyo's images with the in-cluster BuildKit and (re)deploy Pyo into
-# k3s. Run as your normal user after scripts/setup-host.sh:
+# k3s. Run as your normal user. On a machine without k3s it offers to run
+# scripts/setup-host.sh first:
#
# scripts/pyo-up.sh # build what changed and deploy
# scripts/pyo-up.sh --only pyo,agent # build just these images
@@ -274,9 +275,21 @@ as_root() {
# k3s (and its kubectl) come from the one-time root setup.
if ! command -v kubectl >/dev/null || [ ! -r "$KUBECONFIG" ]; then
- echo "k3s isn't installed yet. Run the one-time host setup first:" >&2
- echo " sudo bash scripts/setup-host.sh" >&2
- exit 1
+ answer=""
+ if [ -t 0 ]; then
+ read -r -p "This machine isn't set up for Pyo yet. Set it up now? [y/N] " answer || answer=""
+ fi
+ case "$answer" in
+ [yY] | [yY][eE][sS]) sudo bash scripts/setup-host.sh ;;
+ *)
+ echo "Set up this machine first: sudo bash scripts/setup-host.sh" >&2
+ exit 1
+ ;;
+ esac
+ if ! command -v kubectl >/dev/null || [ ! -r "$KUBECONFIG" ]; then
+ echo "The setup didn't finish. Run it again: sudo bash scripts/setup-host.sh" >&2
+ exit 1
+ fi
fi
if ! kubectl get nodes >/dev/null 2>&1; then
echo "k3s is installed but its API isn't answering. Check: sudo systemctl status k3s" >&2
@@ -762,6 +775,9 @@ for image in "${!TAG_OF[@]}"; do
*) IMAGE_SED+=(-e "s|__IMAGES__/$image:__TAG__|$IMAGES/$image:${TAG_OF[$image]}|g") ;;
esac
done
+# The sandbox template image isn't built here: Pyo then seeds `base` from
+# the latest release it found (sandbox/base.rs), or waits for one.
+IMAGE_SED+=(-e "s|__IMAGES__/sandbox-base:__TAG__||g")
PYO_YAML=$(sed "${IMAGE_SED[@]}" -e "s|__IMAGES__|$IMAGES|g" deploy/k8s/pyo.yaml)
if grep -q "__TAG__" <<<"$PYO_YAML"; then
echo "pyo-up: deploy/k8s/pyo.yaml has an image scripts/build-images.sh doesn't build:" >&2
diff --git a/scripts/release-manifest.ts b/scripts/release-manifest.ts
index 08cce3a..8e12b81 100644
--- a/scripts/release-manifest.ts
+++ b/scripts/release-manifest.ts
@@ -7,9 +7,11 @@
// --images ghcr.io/upsetdev/pyo \
// --pyo sha256:… --agent sha256:… --builder sha256:… --flows sha256:… --waf sha256:… --geoip sha256:… \
// --db-controller sha256:… --kv sha256:… --static sha256:… --mysql-operator sha256:… --mysql-backup sha256:… \
+// --sandbox-base sha256:… \
// --postgres-16 sha256:… --postgres-17 sha256:… --postgres-18 sha256:… \
// --agent-access "$(pyo-agent access-hash)" \
-// --sandbox-x86_64 --sandbox-aarch64 > release.json
+// --sandbox-x86_64 --sandbox-aarch64 \
+// --sandbox-gvisor-x86_64 --sandbox-gvisor-aarch64 > release.json
//
// Images are pinned by digest. deploy/k8s/{platform,pyo}.yaml become JSON
// objects. Notices for this release come from
@@ -18,9 +20,10 @@
// roles the release's agent is installed with (`pyo-agent access-hash`):
// installations on other roles tell their admin, before updating, to run
// each cluster's connect command afterwards. `sandbox` lists the SHA-256 of
-// each architecture's sandbox server files (pyo-sandbox-linux-.tar.gz):
-// deploy/sandbox.sh checks them against it once release.json's signature
-// checks out.
+// each architecture's sandbox server files (pyo-sandbox-linux-.tar.gz),
+// and `sandboxGvisor` those of runsc (pyo-sandbox-gvisor-linux-.tar.gz)
+// for servers without KVM: deploy/sandbox.sh checks them against it once
+// release.json's signature checks out.
import { readdirSync, readFileSync } from "node:fs";
@@ -63,6 +66,9 @@ const names = {
// (deploy/images/moco).
"mysql-operator": "mysqlOperator",
"mysql-backup": "mysqlBackup",
+ // Pyo Sandbox's built-in template `base` (multi-arch). Pyo reads it
+ // from PYO_SANDBOX_BASE_IMAGE and seeds the template from it.
+ "sandbox-base": "sandboxBase",
} as const;
const byName: Record = Object.fromEntries(Object.keys(names).map((n) => [n, ref(n)]));
const pinned: Record = Object.fromEntries(Object.entries(names).map(([n, key]) => [key, byName[n]]));
@@ -128,13 +134,16 @@ const k3s = {
};
// The sandbox server files' SHA-256, by architecture (sandbox.sh's ARCH).
-const sandbox = Object.fromEntries(
- ["x86_64", "aarch64"].map((arch) => {
- const sum = need(`sandbox-${arch}`).toLowerCase();
- if (!sha256.test(sum)) fail(`--sandbox-${arch} isn't a sha256: ${sum}`);
- return [arch, sum];
- }),
-);
+const sums = (flag: string) =>
+ Object.fromEntries(
+ ["x86_64", "aarch64"].map((arch) => {
+ const sum = need(`${flag}-${arch}`).toLowerCase();
+ if (!sha256.test(sum)) fail(`--${flag}-${arch} isn't a sha256: ${sum}`);
+ return [arch, sum];
+ }),
+ );
+const sandbox = sums("sandbox");
+const sandboxGvisor = sums("sandbox-gvisor");
const release = {
version,
@@ -152,6 +161,7 @@ const release = {
beforeYouUpdate: forThis ? (notices.beforeYouUpdate ?? []) : [],
agentAccess,
sandbox,
+ sandboxGvisor,
};
// Pretty-printed, two spaces: deploy/install.sh reads it with sed.
console.log(JSON.stringify(release, null, 2));
diff --git a/scripts/sandbox.test.sh b/scripts/sandbox.test.sh
index 9d768d2..43f8de6 100644
--- a/scripts/sandbox.test.sh
+++ b/scripts/sandbox.test.sh
@@ -148,10 +148,56 @@ out=$( (fetch_bundle) 2>&1)
check "a release without the checksums is refused" "$out" "doesn't list the sandbox files' checksum" has
unset -f curl cosign
+# gVisor's files, for servers without KVM: checked against release.json's
+# "sandboxGvisor" like the sandbox files against "sandbox".
+mkdir -p "$tmp/g/gvisor-bin"
+echo runsc >"$tmp/g/runsc" && echo sentry >"$tmp/g/gvisor-bin/gvisor_sentry"
+tar -czf "$tmp/gvisor.tar.gz" -C "$tmp/g" runsc gvisor-bin
+gv_good=$(sha256sum "$tmp/gvisor.tar.gz" | cut -d' ' -f1)
+curl() {
+ local a o="" url=""
+ for a in "$@"; do case "$a" in http*) url=$a ;; esac; done
+ echo "$url" >>"$tmp/urls"
+ for a in "$@"; do
+ if [ "$o" = 1 ]; then
+ case "$url" in
+ */release.json) printf '{\n "version": "0.4.0",\n "sandbox": {\n "x86_64": "%s"\n },\n "sandboxGvisor": {\n "x86_64": "%s",\n "aarch64": "%s"\n }\n}\n' "$(printf '%064d' 3)" "$gv_sum" "$(printf '%064d' 4)" >"$a" ;;
+ *.sigstore.json) echo '{}' >"$a" ;;
+ *) cp "$tmp/gvisor.tar.gz" "$a" ;;
+ esac
+ o=""
+ fi
+ [ "$a" = -o ] && o=1
+ done
+ return 0
+}
+cosign() { :; }
+defaults
+ARCH=x86_64 VERSION=0.4.0 LIB=$tmp/lib WORK=$tmp/work-gv1 gv_sum=$gv_good
+mkdir -p "$WORK"
+out=$( (add_gvisor && ls "$LIB" "$LIB/gvisor-bin") 2>&1)
+check "gVisor comes from the release" "$(cat "$tmp/urls")" "/v0.4.0/pyo-sandbox-gvisor-linux-x86_64.tar.gz$" has
+check "and is installed next to its gvisor-bin" "$out" "gvisor_sentry" has
+check "with runsc" "$(cat "$LIB/runsc")" "^runsc$" has
+check "release_sum reads sandboxGvisor" "$(release_sum sandboxGvisor)" "^$gv_good$" has
+check "and still sandbox by default" "$(release_sum)" "^0{63}3$" has
+WORK=$tmp/work-gv2 gv_sum=$(printf '%064d' 5)
+mkdir -p "$WORK"
+out=$( (add_gvisor && echo "went on") 2>&1)
+check "gVisor that doesn't match is refused" "$out" "gVisor's files don't match their checksum" has
+check "and goes no further" "$out" "went on" lacks
+WORK=$tmp/work-gv3 VERSION="" DEV=""
+mkdir -p "$WORK"
+out=$( (fetch_gvisor) 2>&1)
+check "unchecked gVisor needs --dev" "$out" "give --version X, or --dev" has
+unset -f curl cosign
+rm -f "$tmp/urls"
+
# ── Detection ──
defaults
-WORK=$tmp/work3 JAIL_UID=998 JAIL_GID=997 BIN=$tmp/pyo-sandbox
+WORK=$tmp/work3 JAIL_UID=998 JAIL_GID=997 BIN=$tmp/pyo-sandbox LIB=$tmp/lib-detect
mkdir -p "$WORK"
+add_gvisor() { echo "Adding gVisor"; }
fake_detect() { # the daemon's `detect`, printing $1
printf '#!/bin/sh\ncat <"$BIN"
chmod +x "$BIN"
@@ -189,7 +235,13 @@ fake_detect '{
"kvm": "none",
"arch": "x86_64"
}'
-check "gVisor is reported" "$( (detect) 2>&1)" "gVisor, because this server has no usable KVM" has
+out=$( (detect) 2>&1)
+check "gVisor is reported" "$out" "gVisor, because this server has no usable KVM" has
+check "a server without KVM gets gVisor first" "$out" "^Adding gVisor$" has
+check "the detection gets runsc's path" "$(detect_config)" '"runsc": "[^"]*/runsc"' has
+mkdir -p "$LIB" && touch "$LIB/runsc" && chmod +x "$LIB/runsc"
+check "gVisor once is enough" "$( (detect) 2>&1)" "Adding gVisor" lacks
+rm -rf "$LIB"
printf '#!/bin/sh\necho "kvm: permission denied" >&2\nexit 1\n' >"$BIN"
out=$( (detect && echo "went on") 2>&1)
check "a failed detection shows why" "$out" "kvm: permission denied" has