diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93d595b..5848868 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,32 @@ jobs: - run: bun test ./dash/scripts - run: bun run build + # The dashboard bundles pyo-templates' catalog as catalog.json. It has to + # match pyo-templates' main, or Pyo ships old templates. + templates: + name: Template catalog + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: upsetdev/pyo-templates + ref: main + path: .pyo-templates + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.2 + - name: Build pyo-templates + working-directory: .pyo-templates + run: bun install --frozen-lockfile && bun run build + - name: catalog.json matches pyo-templates + run: | + bun dash/scripts/sync-templates.ts .pyo-templates/dist/v1 --check || { + echo "::error::dash/src/lib/pyo/catalog.json doesn't match pyo-templates' main. Run \`bun run build\` in pyo-templates, then \`bun scripts/sync-templates.ts\` in dash/, and commit catalog.json." + exit 1 + } + rust: name: Rust runs-on: ubuntu-24.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef86eef..2315dd2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,9 @@ name: Release # Pushing a tag like v0.1.0 publishes that version: signed images at # ghcr.io/upsetdev/pyo, and a GitHub release with the CLI for each -# platform, the sandbox server files and release.json (what the -# dashboard's updates apply). See RELEASING.md. +# platform, the sandbox server files, the install scripts and release.json +# (what the dashboard's updates apply). Then pyo.dev is rebuilt, to serve +# the release's install scripts. See RELEASING.md. on: push: @@ -42,6 +43,23 @@ jobs: bun-version: 1.4.2 - run: bun install --frozen-lockfile - run: bun run typecheck + # The templates this release bundles are pyo-templates' main, as CI + # checks. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: upsetdev/pyo-templates + ref: main + path: .pyo-templates + persist-credentials: false + - name: Build pyo-templates + working-directory: .pyo-templates + run: bun install --frozen-lockfile && bun run build + - name: catalog.json matches pyo-templates + run: | + bun dash/scripts/sync-templates.ts .pyo-templates/dist/v1 --check || { + echo "::error::dash/src/lib/pyo/catalog.json doesn't match pyo-templates' main. Run \`bun run build\` in pyo-templates, then \`bun scripts/sync-templates.ts\` in dash/, commit catalog.json on dev, and release again." + exit 1 + } # The WAF's processor (waf/). - name: WAF tests run: docker run --rm -v "$PWD/waf:/src" -w /src -e CGO_ENABLED=0 golang:1.26 sh -c 'go vet ./... && go test ./...' @@ -198,6 +216,10 @@ jobs: done # What Pyo needs to update itself from the dashboard: the images # pinned by the digests that were just signed, and the manifests. + # Also the install scripts as the release publishes them, written to + # release-scripts/ (connect.sh, join.sh and sandbox.sh with this + # version on their pyo:release-version line), and their SHA-256: + # pyo.dev serves only scripts that match these signed sums. - name: Write release.json env: VERSION: ${{ needs.verify.outputs.version }} @@ -213,8 +235,9 @@ jobs: --sandbox-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256)" \ --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" \ --sandbox-gvisor-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256)" \ - --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" > release.json - jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, migrations: (.migrations | length)}' release.json + --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" \ + --scripts release-scripts > release.json + jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, scripts, migrations: (.migrations | length)}' release.json # The release's own upgrader reads it before it's signed: an image # it doesn't accept, or an agentAccess that isn't its agent's roles, # fails the release here, not on installations. @@ -263,19 +286,23 @@ jobs: - name: Write the release notes env: GH_TOKEN: ${{ github.token }} - run: bun run scripts/release-notes.ts --to "$GITHUB_REF_NAME" > release-notes.md && cat release-notes.md - # install.sh goes out as an asset named exactly that: pyo.dev/install.sh - # fetches releases/latest/download/install.sh (stable releases only). - # connect.sh beside it: install.sh --uninstall runs its own release's. - # pyo.dev/connect.sh and pyo.dev/join.sh fetch connect.sh and join.sh - # the same way (Pyo's commands use them when it has no address other - # machines can reach). pyo.dev/cli.sh fetches cli.sh, the CLI's - # installer. pyo.dev/sandbox.sh fetches sandbox.sh, which sets up - # sandbox servers with the pyo-sandbox-linux-.tar.gz beside it. - - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 - with: - body_path: release-notes.md - files: | + run: bun run scripts/release-notes.ts --to "$GITHUB_REF_NAME" --release release.json > release-notes.md && cat release-notes.md + # The release goes up as a draft with every asset, and is published + # once they're all there: with immutable releases (on for this repo), + # a published release's assets and tag can't change. Run again after + # a failure, this job replaces the draft's assets. A release that's + # published already is left as it is. + # + # The install scripts are release-scripts/'s: install.sh (pyo.dev/install.sh + # is the latest release's), connect.sh (also for install.sh --uninstall), + # join.sh, sandbox.sh (beside the sandbox files it installs) and cli.sh, + # the CLI's installer. + - name: Upload the release as a draft + id: draft + env: + GH_TOKEN: ${{ github.token }} + run: | + files=( pyo-linux-x86_64 pyo-linux-x86_64.sha256 cli-platforms/pyo-linux-aarch64 @@ -288,11 +315,11 @@ jobs: cli-platforms/pyo-windows-x86_64.exe.sha256 release.json release.json.sigstore.json - deploy/install.sh - deploy/cli.sh - crates/pyo-api/assets/connect.sh - crates/pyo-api/assets/join.sh - deploy/sandbox.sh + release-scripts/install.sh + release-scripts/cli.sh + release-scripts/connect.sh + release-scripts/join.sh + release-scripts/sandbox.sh sandbox/pyo-sandbox-linux-x86_64.tar.gz sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256 sandbox/pyo-sandbox-linux-aarch64.tar.gz @@ -301,3 +328,68 @@ jobs: sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256 + ) + if ! draft=$(gh release view "$GITHUB_REF_NAME" --json isDraft --jq .isDraft 2>&1); then + grep -q "release not found" <<<"$draft" || { echo "$draft"; exit 1; } + draft=none + fi + case "$draft" in + false) + echo "::notice::$GITHUB_REF_NAME is published already, so its assets stay as they are." + echo "published=true" >> "$GITHUB_OUTPUT" + ;; + true) + gh release edit "$GITHUB_REF_NAME" --notes-file release-notes.md + gh release upload "$GITHUB_REF_NAME" --clobber "${files[@]}" + ;; + none) + gh release create "$GITHUB_REF_NAME" --draft --verify-tag --title "$GITHUB_REF_NAME" \ + --notes-file release-notes.md "${files[@]}" + ;; + esac + # What the draft holds now: each install script as release.json lists it. + - name: Check the draft's install scripts + if: steps.draft.outputs.published != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "$GITHUB_REF_NAME" --pattern '*.sh' --dir uploaded + jq -r '.scripts | to_entries[] | "\(.value) uploaded/\(.key)"' release.json | sha256sum -c + [ "$(ls uploaded | wc -l)" = "$(jq '.scripts | length' release.json)" ] + - name: Publish the release + if: steps.draft.outputs.published != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: gh release edit "$GITHUB_REF_NAME" --draft=false --latest + + # pyo.dev serves the latest release's install scripts, and each release's + # at pyo.dev/v//. Its build takes them from the published + # release only when they match the sums in its signed release.json. This + # asks Cloudflare's Workers Builds for that build, through a deploy hook + # whose URL is the secret PYO_WEB_DEPLOY_HOOK (Cloudflare dashboard > + # Workers & Pages > pyo-web > Settings > Builds > Deploy Hooks). When it + # fails, run this job again: the release stays as it is. + site: + name: Rebuild pyo.dev + needs: release + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: Ask Workers Builds to build pyo-web + env: + HOOK: ${{ secrets.PYO_WEB_DEPLOY_HOOK }} + run: | + if [ -z "$HOOK" ]; then + echo "::error::The secret PYO_WEB_DEPLOY_HOOK isn't set, so pyo.dev wasn't rebuilt and still serves the previous release's scripts. Make a deploy hook for pyo-web in Cloudflare (Workers & Pages > pyo-web > Settings > Builds > Deploy Hooks), save its URL as this repo's PYO_WEB_DEPLOY_HOOK secret, and run this job again." + exit 1 + fi + case "$HOOK" in + https://api.cloudflare.com/client/v4/workers/builds/deploy_hooks/*) ;; + *) echo "::error::PYO_WEB_DEPLOY_HOOK isn't a Workers Builds deploy hook URL."; exit 1 ;; + esac + out=$(curl -fsS -X POST --retry 3 --max-time 30 "$HOOK") || { echo "::error::The deploy hook didn't answer. Check that it still exists in Cloudflare."; exit 1; } + if [ "$(echo "$out" | jq -r '.success // empty' 2>/dev/null)" = false ]; then + echo "::error::Cloudflare refused the build: $(echo "$out" | jq -c '.errors')" + exit 1 + fi + echo "Build started: $(echo "$out" | jq -r '.result.build_uuid // .build_uuid // "see the Cloudflare dashboard"' 2>/dev/null)" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63abb2c..5901443 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,7 +149,8 @@ Then run `bun run dev`. The template catalog in `dash/src/lib/pyo/catalog.json` comes from [pyo-templates](https://github.com/upsetdev/pyo-templates). Don't edit it by hand. Run `bun scripts/sync-templates.ts` in `dash/` with pyo-templates built -next to this repository. +next to this repository. CI and the release check that it matches +pyo-templates' `main`. ## Run the checks diff --git a/Cargo.lock b/Cargo.lock index 5d770c1..478a1c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pyo-agent" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base64 0.23.1", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "pyo-api" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "argon2 0.6.0", @@ -3380,6 +3380,8 @@ dependencies = [ "futures", "hex", "hmac", + "hyper", + "hyper-util", "instant-acme", "jiff", "jsonwebtoken", @@ -3413,7 +3415,7 @@ dependencies = [ [[package]] name = "pyo-builder" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base64 0.23.1", @@ -3432,7 +3434,7 @@ dependencies = [ [[package]] name = "pyo-cli" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "clap", @@ -3452,7 +3454,7 @@ dependencies = [ [[package]] name = "pyo-db-controller" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base64 0.23.1", @@ -3474,7 +3476,7 @@ dependencies = [ [[package]] name = "pyo-envd" -version = "0.4.2" +version = "0.4.3" dependencies = [ "libc", "pyo-proto", @@ -3483,7 +3485,7 @@ dependencies = [ [[package]] name = "pyo-flows" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "libc", @@ -3497,7 +3499,7 @@ dependencies = [ [[package]] name = "pyo-kv" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "axum", @@ -3514,7 +3516,7 @@ dependencies = [ [[package]] name = "pyo-proto" -version = "0.4.2" +version = "0.4.3" dependencies = [ "base64 0.23.1", "regex", @@ -3528,7 +3530,7 @@ dependencies = [ [[package]] name = "pyo-resp" -version = "0.4.2" +version = "0.4.3" dependencies = [ "futures", "socket2", @@ -3537,7 +3539,7 @@ dependencies = [ [[package]] name = "pyo-sandbox" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "axum", @@ -3567,7 +3569,7 @@ dependencies = [ [[package]] name = "pyo-static" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index cd5ef6d..12bb862 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/pyo-proto", "crates/pyo-api", "crates/pyo-agent", "crates/pyo-builder", "crates/pyo-cli", "crates/pyo-flows", "crates/pyo-resp", "crates/pyo-kv", "crates/pyo-db-controller", "crates/pyo-static", "crates/pyo-envd", "crates/pyo-sandbox"] [workspace.package] -version = "0.4.2" +version = "0.4.3" edition = "2024" license = "Apache-2.0" publish = false diff --git a/README.md b/README.md index 47bd280..7999972 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ Building from source or contributing? See [CONTRIBUTING.md](CONTRIBUTING.md). - [pyo-docs](https://github.com/upsetdev/pyo-docs): the documentation, published at [pyo.dev/docs](https://pyo.dev/docs). - [pyo-web](https://github.com/upsetdev/pyo-web): the website, [pyo.dev](https://pyo.dev). - [pyo-templates](https://github.com/upsetdev/pyo-templates): the one-click templates. +- [pyo-sandbox-images](https://github.com/upsetdev/pyo-sandbox-images): the images of the built-in sandbox templates, base and desktop. ## License diff --git a/RELEASING.md b/RELEASING.md index 07a7510..7c3dbcb 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -23,7 +23,8 @@ to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,pos (all from `deploy/images/`, at the same version. The Rust binaries are compiled once, by the `binaries` job, and the Rust images' jobs take them from that run's build cache), -and creates a GitHub release with the CLI and `release.json`. The CLI is +and creates a GitHub release with the CLI, the install scripts and +`release.json`. The CLI is `pyo-linux-x86_64`, `pyo-linux-aarch64`, `pyo-macos-aarch64`, `pyo-macos-x86_64` and `pyo-windows-x86_64.exe`, each with its `.sha256`, plus its installer `deploy/cli.sh`. Linux x86-64's comes from the Rust @@ -84,8 +85,17 @@ The release's text lists the changes since the previous release (see Don't move or reuse a tag. If a release is bad, publish the next version. -Publish releases as stable releases (not drafts or pre-releases): installations -only see GitHub's "latest" release. +The repo has GitHub's immutable releases on: once a release is published, +its assets and its tag can't change. So the workflow uploads the release +as a draft with every asset, checks the install scripts in it, and only +then publishes it, as a stable release marked latest (installations only +see GitHub's "latest" release). If the job fails before that, run it again: +it replaces the draft's assets. A release that's published already is left +as it is. The release's text can still be edited. + +Then the `site` job rebuilds pyo.dev (see +[Install scripts on pyo.dev](#install-scripts-on-pyodev)). If it fails, run +that job again. ## Release notes @@ -168,6 +178,12 @@ release carries it as an asset. Installations find it in **Settings › Updates* - `sandbox`: the SHA-256 of each architecture's sandbox server files. `deploy/sandbox.sh` checks release.json's signature with cosign, as `install.sh` does, then the files against these. +- `scripts`: the SHA-256 of each install script the release publishes + (`install.sh`, `cli.sh`, `connect.sh`, `join.sh`, `sandbox.sh`). The + script writes them to `--scripts `, with this version on the + `pyo:release-version` line of `connect.sh`, `join.sh` and `sandbox.sh`, + and the release uploads them from there. pyo.dev serves only scripts + that match these sums. The release notes list them too. To try it before tagging: @@ -178,7 +194,9 @@ bun run scripts/release-manifest.ts --version 0.1.0 --images ghcr.io/upsetdev/py --mysql-operator sha256:<64 hex> --mysql-backup sha256:<64 hex> \ --postgres-16 sha256:<64 hex> --postgres-17 sha256:<64 hex> --postgres-18 sha256:<64 hex> \ --agent-access "$(cargo run -q -p pyo-agent -- access-hash)" \ - --sandbox-x86_64 <64 hex> --sandbox-aarch64 <64 hex> + --sandbox-x86_64 <64 hex> --sandbox-aarch64 <64 hex> \ + --sandbox-gvisor-x86_64 <64 hex> --sandbox-gvisor-aarch64 <64 hex> \ + --scripts /tmp/release-scripts ``` ### Notices @@ -234,10 +252,35 @@ certificates are valid to 2031), `install.sh` with cosign (the machine's, or a pinned download checked by SHA-256). The images' own signatures aren't verified in the cluster. `pyo-up.sh --release` checks them. +## Install scripts on pyo.dev + +pyo.dev serves the scripts each release publishes, as they are: + +- `https://pyo.dev/