From f2746316718d752a297e8b3e5d5ba7f70f25e466 Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Tue, 17 Nov 2026 22:40:39 +0700 Subject: [PATCH 01/22] ci: check the template catalog against pyo-templates The dashboard bundles pyo-templates' catalog as catalog.json, and nothing noticed when it fell behind. CI and the release now build pyo-templates' main and fail when catalog.json doesn't match it, saying how to refresh it. --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ .github/workflows/release.yml | 17 +++++++++++++++++ CONTRIBUTING.md | 3 ++- dash/scripts/sync-templates.ts | 4 +++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93d595b..5848868 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,32 @@ jobs: - run: bun test ./dash/scripts - run: bun run build + # The dashboard bundles pyo-templates' catalog as catalog.json. It has to + # match pyo-templates' main, or Pyo ships old templates. + templates: + name: Template catalog + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: upsetdev/pyo-templates + ref: main + path: .pyo-templates + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.2 + - name: Build pyo-templates + working-directory: .pyo-templates + run: bun install --frozen-lockfile && bun run build + - name: catalog.json matches pyo-templates + run: | + bun dash/scripts/sync-templates.ts .pyo-templates/dist/v1 --check || { + echo "::error::dash/src/lib/pyo/catalog.json doesn't match pyo-templates' main. Run \`bun run build\` in pyo-templates, then \`bun scripts/sync-templates.ts\` in dash/, and commit catalog.json." + exit 1 + } + rust: name: Rust runs-on: ubuntu-24.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef86eef..59ac339 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,6 +42,23 @@ jobs: bun-version: 1.4.2 - run: bun install --frozen-lockfile - run: bun run typecheck + # The templates this release bundles are pyo-templates' main, as CI + # checks. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: upsetdev/pyo-templates + ref: main + path: .pyo-templates + persist-credentials: false + - name: Build pyo-templates + working-directory: .pyo-templates + run: bun install --frozen-lockfile && bun run build + - name: catalog.json matches pyo-templates + run: | + bun dash/scripts/sync-templates.ts .pyo-templates/dist/v1 --check || { + echo "::error::dash/src/lib/pyo/catalog.json doesn't match pyo-templates' main. Run \`bun run build\` in pyo-templates, then \`bun scripts/sync-templates.ts\` in dash/, commit catalog.json on dev, and release again." + exit 1 + } # The WAF's processor (waf/). - name: WAF tests run: docker run --rm -v "$PWD/waf:/src" -w /src -e CGO_ENABLED=0 golang:1.26 sh -c 'go vet ./... && go test ./...' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63abb2c..5901443 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,7 +149,8 @@ Then run `bun run dev`. The template catalog in `dash/src/lib/pyo/catalog.json` comes from [pyo-templates](https://github.com/upsetdev/pyo-templates). Don't edit it by hand. Run `bun scripts/sync-templates.ts` in `dash/` with pyo-templates built -next to this repository. +next to this repository. CI and the release check that it matches +pyo-templates' `main`. ## Run the checks diff --git a/dash/scripts/sync-templates.ts b/dash/scripts/sync-templates.ts index b212f3f..b334386 100644 --- a/dash/scripts/sync-templates.ts +++ b/dash/scripts/sync-templates.ts @@ -58,7 +58,9 @@ const text = `${JSON.stringify({ ...index, logos }, null, 2)}\n`; if (check) { const current = existsSync(out) ? readFileSync(out, "utf8") : ""; if (current !== text) { - console.error("src/lib/pyo/catalog.json is out of date: run bun scripts/sync-templates.ts"); + console.error( + "src/lib/pyo/catalog.json is out of date. Run `bun run build` in pyo-templates, then `bun scripts/sync-templates.ts` in dash/, and commit catalog.json.", + ); process.exit(1); } console.log(`catalog.json is up to date (${index.templates.length} templates)`); From 313bc3506a5ff0c499b35bf0fe0946d027addf2a Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Tue, 17 Nov 2026 23:22:32 +0700 Subject: [PATCH 02/22] docs(sandbox): say the desktop's screen is voxl, not noVNC The desktop image serves the screen with voxl on 6080. vnc.html, where noVNC's page was, goes on to voxl's page, so desktopUrl stays as it is. The comments and the template's API description say voxl now, and that voxl's page needs HTTPS or localhost to decode video. --- crates/pyo-api/src/sandbox/base.rs | 2 +- crates/pyo-api/src/sandbox/mod.rs | 2 +- crates/pyo-sandbox/src/api.rs | 2 +- crates/pyo-sandbox/tests/e2e.rs | 2 +- dash/scripts/sandbox-templates.test.ts | 2 +- dash/src/lib/pyo/sandboxTemplates.ts | 10 ++++++---- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/pyo-api/src/sandbox/base.rs b/crates/pyo-api/src/sandbox/base.rs index a1e0637..a20149e 100644 --- a/crates/pyo-api/src/sandbox/base.rs +++ b/crates/pyo-api/src/sandbox/base.rs @@ -2,7 +2,7 @@ //! ([`super::images`]): //! - `base`: Ubuntu with the tools coding agents expect. A sandbox created //! without a template starts from it. -//! - `desktop`: `base` with a desktop, noVNC on port 6080 and a browser with +//! - `desktop`: `base` with a desktop, voxl on port 6080 and a browser with //! CDP on port 9222, started on first use. Its sandboxes get 2 CPUs and //! 2 GiB unless the create says otherwise. //! diff --git a/crates/pyo-api/src/sandbox/mod.rs b/crates/pyo-api/src/sandbox/mod.rs index 4321872..c1dd99d 100644 --- a/crates/pyo-api/src/sandbox/mod.rs +++ b/crates/pyo-api/src/sandbox/mod.rs @@ -526,7 +526,7 @@ pub struct CreateSandbox { /// A sandbox template's name (or id): the team's own, else the /// installation's. "base" (Pyo's built-in Ubuntu 24.04 template) when /// omitted. "desktop" is Pyo's built-in desktop with a browser: CDP on - /// port 9222 and the screen (noVNC) on port 6080. A server that doesn't + /// port 9222 and the screen (voxl) on port 6080. A server that doesn't /// have Pyo's template yet makes it first, so the first sandbox from it /// there takes longer. #[serde(default)] diff --git a/crates/pyo-sandbox/src/api.rs b/crates/pyo-sandbox/src/api.rs index 51ea506..17d5aa6 100644 --- a/crates/pyo-sandbox/src/api.rs +++ b/crates/pyo-sandbox/src/api.rs @@ -21,7 +21,7 @@ //! //! `.../web/{port}/` serves a web page on a port of the sandbox to the //! browser that opens it, over the tailnet, like the dashboard's Open -//! desktop (noVNC on 6080). The port needn't be open on the edge. The +//! desktop (voxl on 6080). The port needn't be open on the edge. The //! token comes once as `?pyo_token=`, which sets an HttpOnly cookie for //! that sandbox's pages (SameSite=Lax, so other sites can't use it) and //! redirects, then in the cookie, or in `Authorization`. The rest of the diff --git a/crates/pyo-sandbox/tests/e2e.rs b/crates/pyo-sandbox/tests/e2e.rs index 9223d88..1f86fa5 100644 --- a/crates/pyo-sandbox/tests/e2e.rs +++ b/crates/pyo-sandbox/tests/e2e.rs @@ -474,7 +474,7 @@ async fn edge_proxy(state: &Arc, keys: &Keys, run: &Path, id: &str) { } assert_eq!(echo, b"ping through the edge\n"); - // And through the daemon's API (noVNC's WebSocket on Open desktop). + // And through the daemon's API (the desktop's WebSocket on Open desktop). let api = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let api_port = api.local_addr().unwrap().port(); tokio::spawn(async move { diff --git a/dash/scripts/sandbox-templates.test.ts b/dash/scripts/sandbox-templates.test.ts index f62c6cf..2a8edd9 100644 --- a/dash/scripts/sandbox-templates.test.ts +++ b/dash/scripts/sandbox-templates.test.ts @@ -144,7 +144,7 @@ test("the desktop needs room, and opens through the daemon", () => { expect(url.searchParams.has("app")).toBe(false); const chromium = new URL(desktopUrl("http://100.64.0.2:7070", "sbx_abc", "t", "chromium")); expect(chromium.searchParams.get("app")).toBe("chromium"); - // noVNC finds its WebSocket beside the page. + // The page finds its WebSocket beside it. expect(url.searchParams.has("path")).toBe(false); }); diff --git a/dash/src/lib/pyo/sandboxTemplates.ts b/dash/src/lib/pyo/sandboxTemplates.ts index a0e0cf8..4c0fd57 100644 --- a/dash/src/lib/pyo/sandboxTemplates.ts +++ b/dash/src/lib/pyo/sandboxTemplates.ts @@ -12,7 +12,7 @@ type TemplateLike = { /** The template a sandbox starts from when none is given. */ export const BASE_TEMPLATE = "base"; -/** Pyo's desktop: the screen in noVNC on DESKTOP_PORT, the browser's CDP on 9222. */ +/** Pyo's desktop: the screen in voxl on DESKTOP_PORT, the browser's CDP on 9222. */ export const DESKTOP_TEMPLATE = "desktop"; export const DESKTOP_PORT = 6080; @@ -89,10 +89,12 @@ export const minSize = (t: TemplateLike | undefined) => (t && isDesktop(t) ? DES /** An app the desktop can bring to the front when its screen opens. */ export type DesktopApp = "chromium"; -/** The desktop's screen in a browser: noVNC through the daemon's +/** The desktop's screen in a browser: voxl through the daemon's * `/web/{port}/` (crates/pyo-sandbox api.rs), which takes the token once - * and keeps it in a cookie for that sandbox's pages. noVNC finds its - * WebSocket beside the page (websockify). */ + * and keeps it in a cookie for that sandbox's pages. vnc.html, where + * noVNC's page was, goes on to voxl's page beside it, which finds its + * WebSocket there too. voxl decodes video with WebCodecs, which browsers + * offer only on HTTPS pages and localhost. */ export function desktopUrl(daemon: string, sandboxId: string, token: string, app?: DesktopApp) { const prefix = `v1/sandboxes/${encodeURIComponent(sandboxId)}/web/${DESKTOP_PORT}/`; const q = new URLSearchParams({ From fea601ca1efecd02436ac222ca5dfe39d7b34f17 Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Tue, 17 Nov 2026 23:57:46 +0700 Subject: [PATCH 03/22] feat(gateway): serve https on a cloud server's nat'd public ip AWS, Google Cloud, Azure, Alibaba Cloud and Tencent Cloud map a VM's public IP to it with 1:1 NAT, so the IP isn't on its network card and Pyo never set up https://pyo..nip.io there. - locate.rs asks the cloud's metadata service for that IP (each cloud at once, a second at most; Oracle's trace IP, as its metadata has none) and looks again after a reboot, when a cloud may change it - Pyo tries its public dashboard on that IP too: Let's Encrypt issuing the certificate is the reachability check, never a request from the server to itself. Issued, the IP counts as the server's own and free domains go there. Refused, cert-manager retries hourly and up, and the agent makes the certificate again after 6 hours at most - a pyo-public-nat Service puts those IPs' ports 80 and 443 on the proxies inside the cluster (externalIPs), so cert-manager's self check and in-cluster requests don't depend on the cloud's hairpin - the dashboard says which ports to allow while the IP is blocked --- crates/pyo-agent/src/gateway.rs | 157 ++++++++++++++ crates/pyo-agent/src/locate.rs | 294 ++++++++++++++++++++++---- crates/pyo-agent/src/main.rs | 1 + crates/pyo-agent/src/observe.rs | 6 + crates/pyo-agent/src/publicdash.rs | 132 +++++++++++- crates/pyo-agent/src/tls.rs | 2 +- crates/pyo-api/src/agent.rs | 6 + crates/pyo-api/src/freedomain.rs | 113 +++++++++- crates/pyo-api/src/ops.rs | 2 +- crates/pyo-api/src/publicdash.rs | 154 ++++++++++++-- crates/pyo-api/src/snapshot.rs | 6 + crates/pyo-api/src/traffic.rs | 6 +- crates/pyo-proto/src/lib.rs | 20 ++ crates/pyo-proto/src/rbac.rs | 80 ++++++- crates/pyo-proto/src/system.rs | 38 ++++ dash/src/components/pyo/Tailscale.tsx | 12 +- dash/src/lib/pyo/types.ts | 4 +- 17 files changed, 949 insertions(+), 84 deletions(-) diff --git a/crates/pyo-agent/src/gateway.rs b/crates/pyo-agent/src/gateway.rs index 9e291e4..e2e2108 100644 --- a/crates/pyo-agent/src/gateway.rs +++ b/crates/pyo-agent/src/gateway.rs @@ -1621,6 +1621,105 @@ pub async fn set_tailnet_web(client: &Client, on: bool) { } } +/// The public IPs behind the cloud's NAT: every server's that runs a proxy +/// (all but build-only ones) and has one its cloud maps to it (locate.rs), +/// unless it's on its own network card too. Sorted, once each. +pub fn public_nat_ips(nodes: &[k8s_openapi::api::core::v1::Node]) -> Vec { + let mut ips: Vec = nodes + .iter() + .filter(|n| { + n.labels() + .get(pyo_proto::BUILDER_NODE_LABEL) + .map(String::as_str) + != Some("true") + }) + .filter_map(|n| { + let a = n.annotations(); + let ip = a.get(pyo_proto::CLOUD_PUBLIC_IPV4_ANNOTATION)?; + let on_card = a + .get(pyo_proto::NIC_PUBLIC_IPV4_ANNOTATION) + .is_some_and(|v| v.split(',').any(|c| c.trim() == ip)); + (crate::locate::public_v4(ip) && !on_card).then(|| ip.clone()) + }) + .collect(); + ips.sort(); + ips.dedup(); + ips +} + +/// The NAT Service was refused (said once). +static PUBLIC_NAT_REFUSED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// The cluster's own requests to the public IPs behind the cloud's NAT go +/// to the proxies (system.rs `PUBLIC_NAT_SERVICE`), while it takes public +/// traffic (from each heartbeat's reply): the Service written or removed +/// when that differs. +pub async fn set_public_nat(client: &Client, on: bool) { + let services: Api = + Api::namespaced(client.clone(), GATEWAY_NS); + let wanted = if on { + match Api::::all(client.clone()) + .list(&ListParams::default()) + .await + { + Ok(l) => public_nat_ips(&l.items), + Err(e) => { + tracing::warn!(error = %e, "couldn't list servers for the public IPs behind NAT"); + return; + } + } + } else { + vec![] + }; + let current = match services.get_opt(system::PUBLIC_NAT_SERVICE).await { + Ok(s) => s.map(|s| { + let spec = s.spec.unwrap_or_default(); + let mut ips = spec.external_ips.unwrap_or_default(); + ips.sort(); + let ports = serde_json::to_value(spec.ports.unwrap_or_default()).unwrap_or_default(); + (ips, ports) + }), + Err(e) => { + tracing::warn!(error = %e, "couldn't read the public NAT Service"); + return; + } + }; + // The same rule as the tailnet websites' Service: on, with addresses. + match tailnet_web_change(on, wanted, current) { + None => {} + Some(None) => match services + .delete(system::PUBLIC_NAT_SERVICE, &DeleteParams::default()) + .await + { + Ok(_) => tracing::info!("no public IPs behind NAT"), + Err(e) => tracing::warn!(error = %e, "couldn't remove the public NAT Service"), + }, + Some(Some(ips)) => match services + .patch( + system::PUBLIC_NAT_SERVICE, + &pp(), + &Patch::Apply(system::public_nat_service(&ips)), + ) + .await + { + Ok(_) => { + PUBLIC_NAT_REFUSED.store(false, Ordering::Relaxed); + tracing::info!( + ?ips, + "public IPs behind NAT reach the proxies from within the cluster" + ) + } + Err(kube::Error::Api(e)) if e.code == 403 => { + if !PUBLIC_NAT_REFUSED.swap(true, Ordering::Relaxed) { + tracing::warn!(error = %e, "couldn't write the public NAT Service (run the cluster's connect command)"); + } + } + Err(e) => tracing::warn!(error = %e, "couldn't write the public NAT Service"), + }, + } +} + async fn ensure_once(client: &Client) -> anyhow::Result<()> { let proxies = envoy_proxies(client); let current = proxies @@ -1922,6 +2021,64 @@ mod tests { assert_eq!(tailnet_web_change(true, vec![], None), None); } + /// The public IPs behind the cloud's NAT: those the cloud maps to the + /// servers that run a proxy, not ones on their own cards, nor anything + /// that isn't a public IPv4 address. + #[test] + fn public_ips_behind_nat() { + let node = |name: &str, labels: Value, annotations: Value| { + serde_json::from_value::(json!({ + "metadata": { "name": name, "labels": labels, "annotations": annotations }, + })) + .unwrap() + }; + let nodes = [ + node( + "a", + json!({}), + json!({ "pyo/cloud-public-ipv4": "43.156.1.2", "pyo/nic-public-ipv4": "" }), + ), + node( + "b", + json!({}), + json!({ "pyo/cloud-public-ipv4": "3.120.1.2" }), + ), + // On its card: the internet and the cluster reach it directly. + node( + "c", + json!({}), + json!({ "pyo/cloud-public-ipv4": "203.0.113.7", "pyo/nic-public-ipv4": "203.0.113.7" }), + ), + // Build-only: no proxy there. + node( + "d", + json!({ "pyo/builder": "true" }), + json!({ "pyo/cloud-public-ipv4": "34.72.1.2" }), + ), + // None, or not a public address. + node("e", json!({}), json!({ "pyo/cloud-public-ipv4": "" })), + node( + "f", + json!({}), + json!({ "pyo/cloud-public-ipv4": "10.3.8.13" }), + ), + node("g", json!({}), json!({})), + node( + "h", + json!({}), + json!({ "pyo/cloud-public-ipv4": "43.156.1.2" }), + ), + ]; + assert_eq!(public_nat_ips(&nodes), ["3.120.1.2", "43.156.1.2"]); + let svc = system::public_nat_service(&public_nat_ips(&nodes)); + assert_eq!(svc["spec"]["ports"], system::tailnet_web_ports()); + assert_eq!(svc["spec"]["externalTrafficPolicy"], "Local"); + assert_eq!( + svc["spec"]["externalIPs"], + json!(["3.120.1.2", "43.156.1.2"]) + ); + } + #[test] fn a_failed_chart_points_at_its_log() { assert_eq!(chart_failed(false, 2), None); diff --git a/crates/pyo-agent/src/locate.rs b/crates/pyo-agent/src/locate.rs index a2fa24b..e22395e 100644 --- a/crates/pyo-agent/src/locate.rs +++ b/crates/pyo-agent/src/locate.rs @@ -1,16 +1,25 @@ -//! Where each server is: its public address as the internet sees it, and -//! its Tailscale address. +//! Where each server is: its public address as the internet sees it, the +//! one its cloud maps to it, and its Tailscale address. //! //! Node addresses are often private or tailnet IPs, which say nothing about -//! location. So, for each ready server not checked lately, the agent runs a -//! one-off pod on it (host network) that asks Cloudflare's trace endpoint -//! for the server's public IP and country, once over IPv4 and once over -//! IPv6 (a server can have one of each), and lists the server's own IPv4 -//! addresses (its Tailscale one, and any public one on a network card, -//! among them), and keeps the answers as node annotations. The heartbeat -//! reports them; the API looks the IP up (GeoIP) for the city, and points -//! the cluster's free domains at the public or the Tailscale address (a -//! server behind NAT has only a private address on its network card). +//! location. So, for each ready server not checked lately (or started +//! again since), the agent runs a one-off pod on it (host network) that +//! asks Cloudflare's trace endpoint for the server's public IP and country, +//! once over IPv4 and once over IPv6 (a server can have one of each), asks +//! the cloud's metadata service for the public IP the cloud maps to it, +//! and lists the server's own IPv4 addresses (its Tailscale one, and any +//! public one on a network card, among them), and keeps the answers as +//! node annotations. The heartbeat reports them; the API looks the IP up +//! (GeoIP) for the city, and points the cluster's free domains at the +//! public or the Tailscale address. +//! +//! A cloud server often has only a private address on its network card: +//! AWS, Google Cloud, Azure, Alibaba Cloud and Tencent Cloud map its public +//! IP to it (1:1 NAT). Their metadata says which IP that is, so Pyo tries +//! it (pyo-api's publicdash.rs). What the trace sees isn't proof of that: +//! it can be a home router's or a shared NAT gateway's, which don't lead +//! back to the server. Oracle Cloud's metadata has no public IP: there the +//! trace's counts, as nothing else leads out of the server. //! //! Cloudflare's trace answers over HTTPS from a nearby anycast address, on //! both families (cloudflare.com has A and AAAA records), with the country @@ -27,9 +36,9 @@ use kube::{ api::{DeleteParams, ListParams, LogParams, Patch, PatchParams, PostParams}, }; use pyo_proto::{ - LOCATED_AT_ANNOTATION, NIC_PUBLIC_IPV4_ANNOTATION, PUBLIC_COUNTRY_ANNOTATION, - PUBLIC_IP_ANNOTATION, PUBLIC_IPV4_ANNOTATION, PUBLIC_IPV6_ANNOTATION, TAILNET_IPV4_ANNOTATION, - now_ms, + CLOUD_PUBLIC_IPV4_ANNOTATION, LOCATED_AT_ANNOTATION, LOCATED_BOOT_ANNOTATION, + NIC_PUBLIC_IPV4_ANNOTATION, PUBLIC_COUNTRY_ANNOTATION, PUBLIC_IP_ANNOTATION, + PUBLIC_IPV4_ANNOTATION, PUBLIC_IPV6_ANNOTATION, TAILNET_IPV4_ANNOTATION, now_ms, }; use serde_json::json; @@ -43,6 +52,31 @@ const LOOKUP_ENV: &str = "PYO_PUBLIC_IP_LOOKUP"; /// How often a server is checked again (its address can change). const EVERY: Duration = Duration::from_secs(24 * 3600); +/// What the pod runs (`$1`: the lookup URL, empty for none): the server's +/// own IPv4 addresses ("a " lines); the public IP its cloud's metadata +/// service names ("c " lines, each service asked at once, a second at +/// most: AWS (IMDSv2, else v1, and clouds that answer like it), Google +/// Cloud, Azure, Alibaba Cloud, Tencent Cloud, Scaleway; "c oracle" on +/// Oracle Cloud, whose metadata has none); then the lookup once per +/// family, each answer's lines prefixed with it (a family the server lacks +/// just fails). The URL is an argument, never part of the script. +const SCRIPT: &str = r#"ip -4 -o addr show 2>/dev/null | sed "s/^/a /" +m=169.254.169.254 +c() { x=$(curl -fsS -m 1 "$@" 2>/dev/null) && [ -n "$x" ] && echo "c $x"; } +(t=$(curl -fsS -m 1 -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" "http://$m/latest/api/token" 2>/dev/null) + if [ -n "$t" ]; then c -H "X-aws-ec2-metadata-token: $t" "http://$m/latest/meta-data/public-ipv4"; else c "http://$m/latest/meta-data/public-ipv4"; fi) & +c -H "Metadata-Flavor: Google" "http://$m/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" & +c -H "Metadata: true" "http://$m/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2021-02-01&format=text" & +(t=$(curl -fsS -m 1 -X PUT -H "X-aliyun-ecs-metadata-token-ttl-seconds: 60" http://100.100.100.200/latest/api/token 2>/dev/null) + h="X-aliyun-ecs-metadata-token: $t" + c -H "$h" http://100.100.100.200/latest/meta-data/eipv4 || c -H "$h" http://100.100.100.200/latest/meta-data/public-ipv4) & +c http://169.254.0.23/latest/meta-data/public-ipv4 & +(x=$(curl -fsS -m 1 http://169.254.42.42/conf 2>/dev/null | sed -n "s/^PUBLIC_IP_ADDRESS=//p") && [ -n "$x" ] && echo "c $x") & +(curl -fsS -m 1 -H "Authorization: Bearer Oracle" "http://$m/opc/v2/instance/id" >/dev/null 2>&1 && echo "c oracle") & +wait +[ -z "$1" ] || for f in 4 6; do curl -$f -s -m 8 "$1" | sed "s/^/v$f /"; done +exit 0"#; + /// The lookup URL, or None when it's off. fn lookup_url(env: Option<&str>) -> Option { match env.map(str::trim) { @@ -89,11 +123,24 @@ async fn locate_all(client: &Client, url: Option<&str>) -> anyhow::Result<()> { .get(LOCATED_AT_ANNOTATION) .and_then(|v| v.parse::().ok()) .unwrap_or(0); - // Never looked for its Tailscale address or its network cards' public - // ones: now (an agent that didn't yet). - let tailnet_known = n.annotations().contains_key(TAILNET_IPV4_ANNOTATION) - && n.annotations().contains_key(NIC_PUBLIC_IPV4_ANNOTATION); - if !ready || (tailnet_known && now_ms() - checked < EVERY.as_millis() as i64) { + // Never looked for its Tailscale address, its network cards' public + // ones or its cloud's: now (an agent that didn't yet). + let all_known = [ + TAILNET_IPV4_ANNOTATION, + NIC_PUBLIC_IPV4_ANNOTATION, + CLOUD_PUBLIC_IPV4_ANNOTATION, + ] + .iter() + .all(|a| n.annotations().contains_key(*a)); + // Started again since: a cloud may have given it another public IP. + let boot = n + .status + .as_ref() + .and_then(|s| s.node_info.as_ref()) + .map(|i| i.boot_id.clone()) + .unwrap_or_default(); + let same_boot = n.annotations().get(LOCATED_BOOT_ANNOTATION) == Some(&boot); + if !ready || (all_known && same_boot && now_ms() - checked < EVERY.as_millis() as i64) { continue; } let name = n.name_any(); @@ -112,6 +159,8 @@ async fn locate_all(client: &Client, url: Option<&str>) -> anyhow::Result<()> { let mut annotations = json!({ TAILNET_IPV4_ANNOTATION: tailnet, LOCATED_AT_ANNOTATION: now_ms().to_string(), NIC_PUBLIC_IPV4_ANNOTATION: found.nic_public.join(","), + CLOUD_PUBLIC_IPV4_ANNOTATION: found.cloud.clone().unwrap_or_default(), + LOCATED_BOOT_ANNOTATION: boot, }); if url.is_some() { annotations[PUBLIC_IP_ANNOTATION] = json!(ip); @@ -123,7 +172,7 @@ async fn locate_all(client: &Client, url: Option<&str>) -> anyhow::Result<()> { nodes .patch(&name, &PatchParams::default(), &Patch::Merge(&patch)) .await?; - tracing::info!(node = %name, %ip, %country, %tailnet, "located server"); + tracing::info!(node = %name, %ip, %country, %tailnet, cloud = ?found.cloud, "located server"); } Err(e) => tracing::warn!(node = %name, error = %e, "couldn't locate server"), } @@ -141,6 +190,8 @@ struct Found { tailnet: Option, /// Public IPv4 addresses on its own network cards. nic_public: Vec, + /// The public IPv4 address its cloud maps to it. + cloud: Option, } /// Public IPs, country code and Tailscale address of one server. @@ -164,11 +215,7 @@ async fn locate(client: &Client, node: &str, url: Option<&str>) -> anyhow::Resul "tolerations": [{ "operator": "Exists" }], "containers": [{ "name": "locate", "image": IMAGE, - // The server's own IPv4 addresses ("a " lines); then the - // lookup once per family, each answer's lines prefixed with - // it (a family the server lacks just fails). The URL is an - // argument, never part of the script; empty: no lookup. - "command": ["sh", "-c", "ip -4 -o addr show 2>/dev/null | sed \"s/^/a /\"; [ -z \"$1\" ] || for f in 4 6; do curl -$f -s -m 8 \"$1\" | sed \"s/^/v$f /\"; done; exit 0", "locate", url.unwrap_or("")], + "command": ["sh", "-c", SCRIPT, "locate", url.unwrap_or("")], "resources": { "limits": { "cpu": "100m", "memory": "32Mi" } }, "securityContext": { "runAsNonRoot": true, "runAsUser": 65534, "runAsGroup": 65534, "allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": { "drop": ["ALL"] } }, }], @@ -247,7 +294,9 @@ fn parse_trace(out: &str) -> Option { let (v4, v6) = (ip("v4", false), ip("v6", true)); let tailnet = tailnet_of(out); let nic_public = nic_public_of(out); - if v4.is_none() && v6.is_none() && tailnet.is_none() && nic_public.is_empty() { + let cloud = cloud_of(out, v4.as_deref()); + if v4.is_none() && v6.is_none() && tailnet.is_none() && nic_public.is_empty() && cloud.is_none() + { return None; } let country = field("v4", "loc") @@ -260,9 +309,48 @@ fn parse_trace(out: &str) -> Option { country, tailnet, nic_public, + cloud, }) } +/// A public IPv4 address: not private, tailnet/CGNAT, loopback, +/// link-local, unspecified, broadcast or multicast. +pub fn public_v4(ip: &str) -> bool { + ip.parse::().is_ok_and(|a| { + let [x, y, ..] = a.octets(); + !(a.is_private() + || a.is_loopback() + || a.is_link_local() + || a.is_unspecified() + || a.is_broadcast() + || a.is_multicast() + || x >= 240 + || x == 0 + || (x == 100 && (64..=127).contains(&y))) + }) +} + +/// The public IPv4 address the cloud's metadata named ("c " lines), or on +/// Oracle Cloud ("c oracle", its metadata has none) the one the lookup +/// found over IPv4 (`v4`). +fn cloud_of(out: &str, v4: Option<&str>) -> Option { + let said: Vec<&str> = out + .lines() + .filter_map(|l| l.strip_prefix("c ")) + .map(str::trim) + .collect(); + said.iter() + .find(|ip| public_v4(ip)) + .map(|ip| (*ip).to_owned()) + .or_else(|| { + said.contains(&"oracle") + .then_some(v4) + .flatten() + .filter(|ip| public_v4(ip)) + .map(str::to_owned) + }) +} + /// The server's own IPv4 addresses ("a " lines of `ip -4 -o addr`), each /// with its line. fn own_addresses(out: &str) -> impl Iterator { @@ -281,18 +369,7 @@ fn own_addresses(out: &str) -> impl Iterator { fn nic_public_of(out: &str) -> Vec { let mut ips: Vec = own_addresses(out) .map(|(_, ip)| ip) - .filter(|ip| { - ip.parse::().is_ok_and(|a| { - let [x, y, ..] = a.octets(); - !(a.is_private() - || a.is_loopback() - || a.is_link_local() - || a.is_unspecified() - || a.is_broadcast() - || a.is_multicast() - || (x == 100 && (64..=127).contains(&y))) - }) - }) + .filter(|ip| public_v4(ip)) .collect(); ips.dedup(); ips @@ -325,7 +402,8 @@ mod tests { v6: Some("2001:db8::7".into()), country: "DE".into(), tailnet: None, - nic_public: vec![] + nic_public: vec![], + cloud: None }) ); let v6_only = "v6 ip=2001:db8::7\nv6 loc=SG\n"; @@ -336,7 +414,8 @@ mod tests { v6: Some("2001:db8::7".into()), country: "SG".into(), tailnet: None, - nic_public: vec![] + nic_public: vec![], + cloud: None }) ); assert_eq!(parse_trace("nope"), None); @@ -348,7 +427,8 @@ mod tests { v6: None, country: "SG".into(), tailnet: None, - nic_public: vec![] + nic_public: vec![], + cloud: None }) ); // A lookup answering with just the address. @@ -359,7 +439,8 @@ mod tests { v6: None, country: String::new(), tailnet: None, - nic_public: vec![] + nic_public: vec![], + cloud: None }) ); // An IPv4 answer over IPv6 (no IPv6 route, a proxy) isn't an IPv6 address. @@ -381,7 +462,8 @@ mod tests { v6: None, country: "SG".into(), tailnet: Some("100.101.67.2".into()), - nic_public: vec![] + nic_public: vec![], + cloud: None }) ); // No internet (or the lookup off): the Tailscale address alone. @@ -417,6 +499,132 @@ mod tests { assert!(tailnet_v4("100.127.0.1") && !tailnet_v4("100.128.0.1") && !tailnet_v4("10.0.0.1")); } + /// What the pod printed on cloud servers behind 1:1 NAT. + #[test] + fn the_clouds_public_ip() { + // Tencent Cloud: a private card, the metadata names the public IP. + let tencent = "a 2: eth0 inet 10.3.8.13/22 brd 10.3.11.255 scope global eth0\n\ + c 43.156.1.2\nv4 ip=43.156.1.2\nv4 loc=SG\n"; + let f = parse_trace(tencent).unwrap(); + assert_eq!(f.cloud.as_deref(), Some("43.156.1.2")); + assert!(f.nic_public.is_empty()); + // Even with the lookup off: the metadata is the cloud's own. + assert_eq!( + parse_trace("a 2: ens5 inet 172.31.5.10/20 scope global ens5\nc 3.120.1.2\n") + .unwrap() + .cloud + .as_deref(), + Some("3.120.1.2") + ); + // A home server: only the trace, no cloud IP (that's the router's). + assert_eq!( + parse_trace("a 2: eth0 inet 192.168.1.5/24 scope global eth0\nv4 ip=81.2.69.160\n") + .unwrap() + .cloud, + None + ); + // Oracle Cloud: its metadata has no public IP, the trace's counts. + let oracle = + "a 2: enp0s6 inet 10.0.0.37/24 scope global enp0s6\nc oracle\nv4 ip=129.146.1.2\n"; + assert_eq!( + parse_trace(oracle).unwrap().cloud.as_deref(), + Some("129.146.1.2") + ); + assert_eq!(parse_trace("c oracle\n"), None); + // An error page, a private or a reserved address isn't one. + for junk in [ + "c ", + "c 10.3.8.13", + "c 100.101.67.2", + "c 999.1.2.3", + "c 250.1.2.3", + ] { + assert_eq!(cloud_of(junk, None), None, "{junk}"); + } + } + + /// The pod's script asks every cloud for a second at most, and never + /// has the lookup URL in it. + #[test] + fn the_script_asks_the_clouds_briefly() { + for host in [ + "m=169.254.169.254", + "/latest/meta-data/public-ipv4", + "computeMetadata", + "Metadata: true", + "100.100.100.200", + "169.254.0.23", + "169.254.42.42", + "Bearer Oracle", + ] { + assert!(SCRIPT.contains(host), "{host}"); + } + let asks = SCRIPT.matches("curl -fsS").count(); + assert_eq!(asks, SCRIPT.matches("curl -fsS -m 1 ").count()); + assert!(!SCRIPT.contains("cloudflare")); + } + + /// The script itself (sh), with the server's commands stubbed: `ip`, + /// and `curl` answering as Tencent Cloud's metadata does (no other + /// cloud's), then as Oracle Cloud's. + #[test] + fn the_script_finds_the_clouds_public_ip() { + let dir = std::env::temp_dir().join(format!("pyo-locate-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let stub = |name: &str, body: &str| { + let p = dir.join(name); + std::fs::write(&p, format!("#!/bin/sh\n{body}\n")).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + }; + stub( + "ip", + "echo '2: eth0 inet 10.3.8.13/22 brd 10.3.11.255 scope global eth0'", + ); + let run = |cloud: &str| { + stub( + "curl", + &format!( + r#"for a; do url=$a; done +case "{cloud}:$url" in + tencent:http://169.254.0.23/latest/meta-data/public-ipv4) printf 43.156.1.2 ;; + oracle:http://169.254.169.254/opc/v2/instance/id) printf ocid1.instance ;; + *:https://trace.example/) printf 'ip=129.146.1.2\nloc=US\n' ;; + *) exit 7 ;; +esac"# + ), + ); + let out = std::process::Command::new("sh") + .arg("-c") + .arg(SCRIPT) + .arg("locate") + .arg(if cloud == "oracle" { + "https://trace.example/" + } else { + "" + }) + .env("PATH", format!("{}:/usr/bin:/bin", dir.display())) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).into_owned() + }; + let out = run("tencent"); + assert!(out.contains("\nc 43.156.1.2\n"), "{out}"); + let f = parse_trace(&out).unwrap(); + assert_eq!(f.cloud.as_deref(), Some("43.156.1.2")); + assert_eq!(f.v4, None); + let out = run("oracle"); + assert_eq!( + parse_trace(&out).unwrap().cloud.as_deref(), + Some("129.146.1.2"), + "{out}" + ); + // Nowhere: no cloud IP. + let out = run("none"); + assert_eq!(parse_trace(&out), None, "{out}"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn lookup_can_change_or_be_off() { assert_eq!(lookup_url(None).as_deref(), Some(LOOKUP)); diff --git a/crates/pyo-agent/src/main.rs b/crates/pyo-agent/src/main.rs index 73b4afa..d83d247 100644 --- a/crates/pyo-agent/src/main.rs +++ b/crates/pyo-agent/src/main.rs @@ -216,6 +216,7 @@ async fn heartbeat_loop(agent: Agent) { gpu::apply_policies(&agent.kube, &reply.gpu_policies).await; if let Some(on) = reply.public_traffic { gateway::set_public_traffic(&agent.kube, on).await; + gateway::set_public_nat(&agent.kube, on).await; if let Err(e) = rollout::set_public_ports(&agent.kube, on).await { tracing::warn!(error = %format!("{e:#}"), "couldn't change public ports"); } diff --git a/crates/pyo-agent/src/observe.rs b/crates/pyo-agent/src/observe.rs index 01fcff8..eee01d9 100644 --- a/crates/pyo-agent/src/observe.rs +++ b/crates/pyo-agent/src/observe.rs @@ -466,6 +466,11 @@ pub async fn heartbeat(client: &Client) -> anyhow::Result { .annotations() .get(pyo_proto::PUBLIC_COUNTRY_ANNOTATION) .cloned(), + cloud_public_ip: n + .annotations() + .get(pyo_proto::CLOUD_PUBLIC_IPV4_ANNOTATION) + .filter(|ip| !ip.is_empty()) + .cloned(), nic_public_ips: n .annotations() .get(pyo_proto::NIC_PUBLIC_IPV4_ANNOTATION) @@ -787,6 +792,7 @@ pub async fn heartbeat(client: &Client) -> anyhow::Result { ) .await, certificates: crate::tls::report(client).await, + public_dashboard: crate::publicdash::report(client).await, host_proof: None, updates: pyo_proto::UPDATES_VERSION, scope: None, diff --git a/crates/pyo-agent/src/publicdash.rs b/crates/pyo-agent/src/publicdash.rs index 72cf555..586dcf8 100644 --- a/crates/pyo-agent/src/publicdash.rs +++ b/crates/pyo-agent/src/publicdash.rs @@ -1,15 +1,110 @@ //! Pyo's dashboard on the public IP (pyo-proto system.rs //! `public_dashboard_objects`), on the cluster Pyo runs on: applied while -//! the API asks for it, removed once it's closed. +//! the API asks for it, removed once it's closed. Its certificate is how +//! Pyo learns the internet reaches the public IP: Let's Encrypt issues it +//! only when it reached the server on port 80 (the heartbeat reports it). +//! +//! A try that failed (ports 80 and 443 closed in a cloud firewall) is +//! tried again by cert-manager, an hour later, then two, four and so on. +//! Past [`RETRY_CAP`], the certificate is made again, which starts that +//! over: ports opened later are picked up within hours, and Let's +//! Encrypt's limit of 5 failed checks per name an hour is never near. The +//! install script, run again, removes a refused one (15 minutes after its +//! try at the soonest) to have it tried now. -use kube::Client; -use kube::api::{DeleteParams, Patch, PatchParams}; -use pyo_proto::{PublicDashboard, system}; +use std::time::Duration; + +use kube::api::{DeleteParams, DynamicObject, Patch, PatchParams}; +use kube::{Api, Client}; +use pyo_proto::{CertificateReport, PublicDashboard, system}; + +/// The longest wait between two tries at the certificate. +const RETRY_CAP: Duration = Duration::from_secs(6 * 3600); + +fn certificates(kube: &Client) -> Api { + Api::namespaced_with( + kube.clone(), + system::GATEWAY_NS, + &crate::tls::certificate_resource(), + ) +} + +/// The certificate as cert-manager has it, for the heartbeat (none while +/// there's no public dashboard). +pub async fn report(kube: &Client) -> Option { + let c = certificates(kube) + .get_opt(system::PUBLIC_DASHBOARD_TLS) + .await + .ok()??; + let hostname = c.data["spec"]["dnsNames"][0].as_str()?.to_owned(); + let created = c + .metadata + .creation_timestamp + .as_ref() + .map(|t| t.0.as_second()); + Some(CertificateReport { + hostname, + ..crate::tls::state(&c.data["status"], created) + }) +} + +/// Its last try failed longer than [`RETRY_CAP`] ago (unix seconds `now`): +/// time to make it again rather than wait for cert-manager's next try. +fn retry_now(status: &serde_json::Value, now: i64) -> bool { + let ready = status["conditions"] + .as_array() + .into_iter() + .flatten() + .any(|c| c["type"] == "Ready" && c["status"] == "True"); + let failed_at = status["lastFailureTime"] + .as_str() + .and_then(|t| { + serde_json::from_value::(t.into()) + .ok() + }) + .map(|t| t.0.as_second()); + !ready && failed_at.is_some_and(|t| now - t >= RETRY_CAP.as_secs() as i64) +} + +/// A certificate whose try failed long ago is removed, so it's made again +/// at once (with a fresh try) by [`apply`]; so is one that's gone (the +/// install script removes a refused one to try again now). +async fn retry_failed(kube: &Client) { + let api = certificates(kube); + let c = match api.get_opt(system::PUBLIC_DASHBOARD_TLS).await { + Ok(Some(c)) => c, + Ok(None) => { + if let Ok(mut a) = APPLIED.lock() { + *a = None; + } + return; + } + Err(_) => return, + }; + if !retry_now(&c.data["status"], pyo_proto::now_ms() / 1000) { + return; + } + match api + .delete(system::PUBLIC_DASHBOARD_TLS, &DeleteParams::default()) + .await + { + Ok(_) => { + tracing::info!("public dashboard: trying its certificate again"); + if let Ok(mut a) = APPLIED.lock() { + *a = None; + } + } + Err(e) => tracing::warn!(error = %e, "couldn't renew the public dashboard's certificate"), + } +} /// What was last applied (a heartbeat that changes nothing writes nothing). static APPLIED: std::sync::Mutex>> = std::sync::Mutex::new(None); pub async fn apply(kube: &Client, wanted: &PublicDashboard) { + if wanted.host.is_some() { + retry_failed(kube).await; + } if APPLIED .lock() .map(|a| a.as_ref() == Some(&wanted.host)) @@ -68,3 +163,32 @@ pub async fn apply(kube: &Client, wanted: &PublicDashboard) { } } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn a_failed_try_is_made_again_after_the_cap() { + let failed = |at: &str| json!({ "lastFailureTime": at, "conditions": [{ "type": "Ready", "status": "False" }] }); + let t = |s: &str| { + serde_json::from_value::(json!(s)) + .unwrap() + .0 + .as_second() + }; + let now = t("2026-11-17T18:00:00Z"); + // cert-manager tries again within the cap itself. + assert!(!retry_now(&failed("2026-11-17T17:00:00Z"), now)); + assert!(!retry_now(&failed("2026-11-17T12:00:01Z"), now)); + assert!(retry_now(&failed("2026-11-17T12:00:00Z"), now)); + // Issued (an older failure kept), on its way, or never failed: no. + assert!(!retry_now( + &json!({ "lastFailureTime": "2026-11-16T00:00:00Z", "conditions": [{ "type": "Ready", "status": "True" }] }), + now + )); + assert!(!retry_now(&json!({ "conditions": [] }), now)); + assert!(!retry_now(&json!({}), now)); + } +} diff --git a/crates/pyo-agent/src/tls.rs b/crates/pyo-agent/src/tls.rs index fac0646..a3d6237 100644 --- a/crates/pyo-agent/src/tls.rs +++ b/crates/pyo-agent/src/tls.rs @@ -102,7 +102,7 @@ pub async fn report(client: &Client) -> Vec { /// A Certificate's status as the heartbeat reports it (no service or /// hostname yet). -fn state(status: &serde_json::Value, created: Option) -> CertificateReport { +pub(crate) fn state(status: &serde_json::Value, created: Option) -> CertificateReport { let cond = |t: &str| { status["conditions"] .as_array() diff --git a/crates/pyo-api/src/agent.rs b/crates/pyo-api/src/agent.rs index 05bfe85..823f948 100644 --- a/crates/pyo-api/src/agent.rs +++ b/crates/pyo-api/src/agent.rs @@ -436,6 +436,12 @@ async fn heartbeat( } // Volumes: their state, deletions confirmed, claims Pyo has no volume for. crate::volumes::reported(&mut tx, &cluster.id, &hb.volumes).await?; + // Let's Encrypt's word on Pyo's dashboard on the public IP: issued + // there, the IP behind the cloud's NAT is reached from outside, and its + // free domains may go there (freedomain.rs). + if same_cluster { + crate::publicdash::reported(&mut tx, hb.public_dashboard.as_ref()).await?; + } // Free domains on another of its addresses (a private one, one it no // longer has, the other mode's) follow its free-domain address, live. let rehomed = crate::freedomain::rehome(&mut tx, &cluster.id).await?; diff --git a/crates/pyo-api/src/freedomain.rs b/crates/pyo-api/src/freedomain.rs index 49e42e2..1c4e13b 100644 --- a/crates/pyo-api/src/freedomain.rs +++ b/crates/pyo-api/src/freedomain.rs @@ -17,10 +17,16 @@ //! //! Unset, a cluster uses its public IP when it takes public traffic and //! the server has that address itself (on a network card, or as its -//! ExternalIP) or a load balancer gives it one. When the public IP is only -//! what the internet sees (behind NAT: a home router, some providers), -//! ports 80 and 443 may not reach the server, so Tailscale. Without an -//! address there's no free domain. +//! ExternalIP) or a load balancer gives it one. A cloud server often has +//! only a private address on its card, its public IP mapped to it by the +//! cloud (1:1 NAT, named by the cloud's metadata: locate.rs): Pyo tries +//! its dashboard's certificate there (publicdash.rs), and once Let's +//! Encrypt issued it, the internet reaches that IP, so it counts as the +//! server's own ([`Cluster::reached`]). When the public IP is only what +//! the internet sees (behind NAT: a home router, a shared gateway), or +//! Let's Encrypt can't reach it (a cloud firewall), ports 80 and 443 may +//! not reach the server, so Tailscale. Without an address there's no free +//! domain. //! //! Free domains Pyo made on another of the cluster's addresses (a private //! one, the other mode's, one a server no longer has) or in the older @@ -92,6 +98,10 @@ pub struct Cluster { pub tailnet_web: Vec, /// Its servers' reports, in a stable order. pub servers: Vec, + /// The public IP behind the cloud's NAT that Let's Encrypt reached (it + /// issued the dashboard's certificate there): on the cluster Pyo runs + /// on (publicdash.rs `reached`). + pub reached: Option, } /// Servers that take public traffic, ready ones first. @@ -106,6 +116,12 @@ fn entry(servers: &[Value]) -> impl Iterator { .chain(entry().filter(|r| r["status"] != json!("ready"))) } +/// The public IPv4 address the server's cloud maps to it (1:1 NAT), from +/// the cloud's metadata (locate.rs). +fn cloud_v4(r: &Value) -> Option { + r["cloudPublicIp"].as_str().and_then(public_v4) +} + /// A public IPv4 address the server has itself: its ExternalIP, a node /// address, or one on a network card (what the agent found, locate.rs). fn direct_v4(r: &Value) -> Option { @@ -147,8 +163,9 @@ impl Cluster { /// Its public IPv4 address when the internet reaches it directly: a /// public load balancer address, or one a server that takes public - /// traffic has itself (on a network card, or as its ExternalIP). Not - /// one only the outbound lookup found (behind NAT). + /// traffic has itself (on a network card, or as its ExternalIP), or + /// the one its cloud maps to it once Let's Encrypt reached it there. + /// Not one only the outbound lookup found (behind NAT). pub fn direct_public_ip(&self) -> Option { if !self.public_traffic { return None; @@ -157,6 +174,32 @@ impl Cluster { .iter() .find_map(|i| public_v4(i)) .or_else(|| entry(&self.servers).find_map(direct_v4)) + .or_else(|| self.reached.filter(|ip| self.cloud_ip() == Some(*ip))) + } + + /// The public IPv4 address the cloud maps to a server that takes + /// public traffic (1:1 NAT), from the cloud's metadata. + pub fn cloud_ip(&self) -> Option { + if !self.public_traffic { + return None; + } + entry(&self.servers).find_map(cloud_v4) + } + + /// Where Pyo's dashboard goes on the public IP (publicdash.rs), while + /// it takes public traffic: where its free domains are public, and + /// behind the cloud's NAT (the dashboard's certificate is how Pyo + /// learns Let's Encrypt reaches it there), unless they were set to + /// Tailscale. + pub fn dashboard_ip(&self) -> Option { + if !self.public_traffic { + return None; + } + match (self.mode(), self.set.as_deref()) { + (Mode::Public, _) => self.public_ip(), + (Mode::Tailscale, None) => self.cloud_ip(), + (Mode::Tailscale, Some(_)) => None, + } } /// Its public IP is known only from what the internet sees: the server @@ -272,6 +315,7 @@ fn addresses(r: &Value) -> impl Iterator + '_ { [ "internalIP", "externalIP", + "cloudPublicIp", "publicIp", "publicIpv4", "publicIpv6", @@ -305,13 +349,13 @@ pub async fn load( tx: &mut Transaction<'_, Postgres>, cluster_id: &str, ) -> ApiResult> { - let row: Option<(String, bool, Option, Value)> = sqlx::query_as( - "SELECT team_id, public_traffic, free_domain_address, capabilities FROM pyo.clusters WHERE id = $1", + let row: Option<(String, bool, Option, Value, bool)> = sqlx::query_as( + "SELECT team_id, public_traffic, free_domain_address, capabilities, same_cluster FROM pyo.clusters WHERE id = $1", ) .bind(cluster_id) .fetch_optional(&mut **tx) .await?; - let Some((team_id, public_traffic, set, caps)) = row else { + let Some((team_id, public_traffic, set, caps, same_cluster)) = row else { return Ok(None); }; let servers: Vec = @@ -327,6 +371,11 @@ pub async fn load( lb_ips: ips(&caps, "loadBalancerIps"), tailnet_web: ips(&caps, "tailnetWebIps"), servers, + reached: if same_cluster { + crate::publicdash::reached(&mut **tx).await? + } else { + None + }, }, ))) } @@ -507,6 +556,7 @@ mod tests { }, tailnet_web: vec![], servers, + reached: None, } } @@ -630,6 +680,51 @@ mod tests { ); } + /// A cloud server behind 1:1 NAT: its cloud's metadata names the public + /// IP. Pyo tries its dashboard there, and once Let's Encrypt reached it, + /// the IP is the server's own: free domains go there too. + #[test] + fn the_clouds_public_ip_counts_once_reached() { + let cloud = json!({ "name": "lighthouse", "status": "ready", "internalIP": "10.3.8.13", "addresses": ["10.3.8.13"], + "cloudPublicIp": "43.156.1.2", "publicIpv4": "43.156.1.2", "tailnetIp": "100.101.67.2" }); + let mut c = cluster(true, None, vec![cloud.clone()]); + assert_eq!((c.mode(), c.behind_nat()), (Mode::Tailscale, true)); + assert_eq!(c.dashboard_ip(), Some(v4("43.156.1.2"))); + c.reached = Some(v4("43.156.1.2")); + assert_eq!( + (c.mode(), c.address(), c.behind_nat()), + (Mode::Public, Some(v4("43.156.1.2")), false) + ); + assert_eq!(c.dashboard_ip(), Some(v4("43.156.1.2"))); + // The cloud gave it another IP since (a restart): tried there again. + c.reached = Some(v4("43.156.9.9")); + assert_eq!(c.mode(), Mode::Tailscale); + assert_eq!(c.dashboard_ip(), Some(v4("43.156.1.2"))); + // Behind a home router (only what the lookup saw): not tried. + assert_eq!(cluster(true, None, vec![nat_server()]).dashboard_ip(), None); + // Free domains set to Tailscale, or no public traffic: none. + assert_eq!( + cluster(true, Some("tailscale"), vec![cloud.clone()]).dashboard_ip(), + None + ); + assert_eq!( + cluster(false, None, vec![cloud.clone()]).dashboard_ip(), + None + ); + // A public card: as ever. + assert_eq!( + cluster(true, None, vec![direct_server()]).dashboard_ip(), + Some(v4("203.0.113.7")) + ); + // The cloud's word comes before the lookup's. + let both = json!({ "name": "a", "status": "ready", "internalIP": "10.0.0.5", + "cloudPublicIp": "43.156.1.2", "publicIpv4": "198.51.100.1" }); + assert_eq!( + cluster(true, Some("public"), vec![both]).public_ip(), + Some(v4("43.156.1.2")) + ); + } + #[test] fn only_servers_taking_public_traffic_count() { let builder = json!({ "name": "a-builder", "status": "ready", "builder": true, "publicIpv4": "198.51.100.1" }); diff --git a/crates/pyo-api/src/ops.rs b/crates/pyo-api/src/ops.rs index 584141e..9ee7f15 100644 --- a/crates/pyo-api/src/ops.rs +++ b/crates/pyo-api/src/ops.rs @@ -2131,7 +2131,7 @@ pub(crate) enum Op { /// server but build-only ones (ports 80 and 443, and public TCP and UDP /// ports through it). Off: services are only reachable inside the cluster. SetClusterPublicTraffic { cluster_id: String, on: bool }, - /// Where the cluster's free domains (`..nip.io`) point: "public" (its public IP, for everyone) or "tailscale" (its Tailscale IP, only from the tailnet). With "tailscale", services are reachable from your tailnet on ports 80 and 443 whether or not the cluster takes public traffic. Free domains on it move to the new address in place. By default it's public when the cluster takes public traffic and a server has its public IP itself (on a network card or as its ExternalIP) or a load balancer gives it one. Behind NAT, it's Tailscale. + /// Where the cluster's free domains (`..nip.io`) point: "public" (its public IP, for everyone) or "tailscale" (its Tailscale IP, only from the tailnet). With "tailscale", services are reachable from your tailnet on ports 80 and 443 whether or not the cluster takes public traffic. Free domains on it move to the new address in place. By default it's public when the cluster takes public traffic and a server has its public IP itself (on a network card or as its ExternalIP), a load balancer gives it one, or its cloud maps one to it and Let's Encrypt reached it there. Behind NAT, it's Tailscale. SetClusterFreeDomainAddress { cluster_id: String, address: String }, /// Add an access rule to the end of a service's firewall. Allow and block rules are checked in order, and the first match decides. With no match, the client gets in. Rate limit rules then apply to websites, and each matching one counts the request, per client address and per server. AddFirewallRule { diff --git a/crates/pyo-api/src/publicdash.rs b/crates/pyo-api/src/publicdash.rs index cd01415..a2f83c7 100644 --- a/crates/pyo-api/src/publicdash.rs +++ b/crates/pyo-api/src/publicdash.rs @@ -2,8 +2,19 @@ //! the cluster Pyo runs on, with Let's Encrypt's certificate (the agent's //! publicdash.rs), so one server needs no Tailscale. An admin can close it //! once Pyo is on the tailnet; removing Tailscale opens it again. +//! +//! Behind a cloud's NAT (the public IP mapped to the server, from the +//! cloud's metadata), Let's Encrypt is the check that the internet reaches +//! that IP: Pyo asks for the certificate there as well, and Let's Encrypt +//! issues it only once it reached the server on port 80 from outside. The +//! server never checks its own public IP (a cloud may not send its own +//! requests back in). Issued, the IP counts as the server's own: free +//! domains go there too ([`reached`], freedomain.rs). Refused (ports 80 +//! and 443 closed in a cloud firewall), cert-manager tries again an hour +//! later, then less often, at most every few hours (the agent's +//! publicdash.rs): ports opened later are picked up by themselves. -use std::time::Duration; +use std::{net::Ipv4Addr, time::Duration}; use serde_json::{Value, json}; use sqlx::PgPool; @@ -45,7 +56,8 @@ pub async fn set_open(db: &PgPool, open: bool) -> ApiResult<()> { /// Its name while it's open: "pyo..nip.io" of the cluster Pyo /// runs on, when that takes public traffic and its free domains are on -/// its public IP. +/// its public IP, or it's behind the cloud's NAT (freedomain.rs +/// `dashboard_ip`). pub async fn host(db: &PgPool) -> ApiResult> { if closed(db).await? { return Ok(None); @@ -58,14 +70,73 @@ pub async fn host(db: &PgPool) -> ApiResult> { let mut tx = db.begin().await?; let c = crate::freedomain::load(&mut tx, &id).await?; tx.commit().await?; - // Only where its free domains are public too: behind NAT (free domains - // on Tailscale) Let's Encrypt can't reach it, and every try would count - // against its limits. - Ok( - c.filter(|(_, c)| c.public_traffic && c.mode() == crate::freedomain::Mode::Public) - .and_then(|(_, c)| c.address()) - .and_then(|ip| pyo_proto::system::public_dashboard_host(&ip.to_string())), + // Not behind a home router or a shared gateway (only what the internet + // sees of the server's requests): Let's Encrypt can't reach it there. + Ok(c.and_then(|(_, c)| c.dashboard_ip()) + .and_then(|ip| pyo_proto::system::public_dashboard_host(&ip.to_string()))) +} + +/// Let's Encrypt's last word on the dashboard's certificate, as kept: +/// the public IP and whether it was issued ("ready") or refused ("failed"). +async fn reach<'e, E: sqlx::PgExecutor<'e>>(e: E) -> ApiResult> { + let v: Option = + sqlx::query_scalar("SELECT value FROM pyo.settings WHERE key = 'public_reach'") + .fetch_optional(e) + .await?; + Ok(v.and_then(|v| { + let ip = v["ip"].as_str()?.parse().ok()?; + Some((ip, v["state"].as_str()?.to_owned())) + })) +} + +/// The public IP Let's Encrypt reached: it issued the dashboard's +/// certificate there. +pub async fn reached<'e, E: sqlx::PgExecutor<'e>>(e: E) -> ApiResult> { + Ok(reach(e) + .await? + .filter(|(_, state)| state == "ready") + .map(|(ip, _)| ip)) +} + +/// What a heartbeat of the cluster Pyo runs on says of the dashboard's +/// certificate: issued or refused is kept (a try on its way changes +/// nothing). +pub fn outcome(report: &pyo_proto::CertificateReport) -> Option<(Ipv4Addr, &'static str)> { + let ip = crate::freedomain::ip_in(&report.hostname)?; + if report.ready { + Some((ip, "ready")) + } else if report.failed { + Some((ip, "failed")) + } else { + None + } +} + +/// Keeps what a heartbeat of the cluster Pyo runs on says of the +/// dashboard's certificate (see [`outcome`]). +pub async fn reported( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + report: Option<&pyo_proto::CertificateReport>, +) -> ApiResult<()> { + let Some((ip, state)) = report.and_then(outcome) else { + return Ok(()); + }; + if reach(&mut **tx) + .await? + .is_some_and(|(i, s)| i == ip && s == state) + { + return Ok(()); + } + tracing::info!(%ip, state, "public dashboard's certificate"); + sqlx::query( + "INSERT INTO pyo.settings (key, value) VALUES ('public_reach', jsonb_build_object('ip', $1::text, 'state', $2::text)) + ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = now()", ) + .bind(ip.to_string()) + .bind(state) + .execute(&mut **tx) + .await?; + Ok(()) } /// The heartbeat reply's `publicDashboard`, for the cluster Pyo runs on. @@ -76,14 +147,18 @@ pub async fn wanted_for(db: &PgPool, same_cluster: bool) -> ApiResult ApiResult<()> { let want = host(&s.db).await?; let have = crate::web::addresses().public; match want { None if have.is_some() => crate::tailscale::save_address(&s.db, "public", None).await?, - Some(h) if have.as_deref() != Some(h.as_str()) && answers(&h).await => { + Some(h) + if have.as_deref() != Some(h.as_str()) + && (issued(&s.db, &h).await? || answers(&h).await) => + { tracing::info!(host = %h, "public dashboard ready"); crate::tailscale::save_address(&s.db, "public", Some(&h)).await?; } @@ -92,6 +167,12 @@ pub async fn tick(s: &AppState) -> ApiResult<()> { Ok(()) } +/// Let's Encrypt issued the certificate of `host` ("pyo..nip.io"). +async fn issued(db: &PgPool, host: &str) -> ApiResult { + let ip = crate::freedomain::ip_in(host); + Ok(ip.is_some() && reached(db).await? == ip) +} + async fn answers(host: &str) -> bool { let Ok(c) = reqwest::Client::builder() .timeout(Duration::from_secs(10)) @@ -105,14 +186,57 @@ async fn answers(host: &str) -> bool { .is_ok_and(|r| r.status().is_success()) } -/// For the dashboard (admins): its name, and whether it's open and ready. +/// For the dashboard (admins): its name, whether it's open and ready, and +/// whether Let's Encrypt couldn't reach the public IP (`blocked`: ports 80 +/// and 443 closed in a cloud firewall). pub async fn snapshot(db: &PgPool) -> ApiResult { let closed = closed(db).await?; - // Behind NAT (or without public traffic) there's none to open or close. - let possible = closed || host(db).await?.is_some(); + let wanted = host(db).await?; + // Behind a router's NAT (or without public traffic) there's none to + // open or close. + let possible = closed || wanted.is_some(); + let ip = wanted.as_deref().and_then(crate::freedomain::ip_in); + let blocked = ip.is_some() + && reach(db) + .await? + .is_some_and(|(i, s)| Some(i) == ip && s == "failed"); Ok(json!({ "open": !closed, "host": crate::web::addresses().public, "possible": possible, + "blocked": blocked, })) } + +#[cfg(test)] +mod tests { + use super::*; + + fn cert(host: &str, ready: bool, failed: bool) -> pyo_proto::CertificateReport { + pyo_proto::CertificateReport { + service_id: String::new(), + hostname: host.into(), + ready, + message: String::new(), + failed, + failure: String::new(), + issuing_since: None, + } + } + + /// Issued or refused is what counts; a try on its way says nothing. + #[test] + fn lets_encrypt_has_the_last_word() { + let ip: Ipv4Addr = "43.156.1.2".parse().unwrap(); + assert_eq!( + outcome(&cert("pyo.43.156.1.2.nip.io", true, false)), + Some((ip, "ready")) + ); + assert_eq!( + outcome(&cert("pyo.43.156.1.2.nip.io", false, true)), + Some((ip, "failed")) + ); + assert_eq!(outcome(&cert("pyo.43.156.1.2.nip.io", false, false)), None); + assert_eq!(outcome(&cert("pyo.example.com", true, false)), None); + } +} diff --git a/crates/pyo-api/src/snapshot.rs b/crates/pyo-api/src/snapshot.rs index 5ed8d27..b05a083 100644 --- a/crates/pyo-api/src/snapshot.rs +++ b/crates/pyo-api/src/snapshot.rs @@ -143,8 +143,14 @@ pub async fn build(db: &PgPool, user: &User, agent_api_url: &str) -> ApiResult bool { } /// A server's public address of each family, from what its node reports: -/// its ExternalIP (set on purpose), then the ones the agent found, then the -/// cluster's own addresses. +/// its ExternalIP (set on purpose), then the ones the agent found (the one +/// its cloud maps to it first), then the cluster's own addresses. pub fn public_families(r: &Value) -> (Option, Option) { - let mut all: Vec<&str> = ["externalIP", "publicIpv4", "publicIpv6"] + let mut all: Vec<&str> = ["externalIP", "cloudPublicIp", "publicIpv4", "publicIpv6"] .iter() .filter_map(|k| r[*k].as_str()) .collect(); diff --git a/crates/pyo-proto/src/lib.rs b/crates/pyo-proto/src/lib.rs index e3aa8d9..3eb3c97 100644 --- a/crates/pyo-proto/src/lib.rs +++ b/crates/pyo-proto/src/lib.rs @@ -45,6 +45,13 @@ pub struct Heartbeat { /// TLS certificates of service domains (cert-manager). #[serde(default)] pub certificates: Vec, + /// The certificate of Pyo's dashboard on the public IP + /// ([`system::public_dashboard_objects`]), while there is one: on the + /// cluster Pyo runs on. Its `hostname` is "pyo..nip.io"; + /// Let's Encrypt issuing it shows the internet reaches that IP on + /// port 80. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub public_dashboard: Option, /// Proof that this is the cluster Pyo runs on ("This machine"): what /// [`host_proof`] makes of the host's bootstrap key (its /// pyo-system/pyo-bootstrap Secret, only there) and this agent's token. @@ -549,6 +556,12 @@ pub struct ServerReport { /// it there directly (not behind NAT). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub nic_public_ips: Vec, + /// The public IPv4 address its cloud maps to it (1:1 NAT, as on AWS, + /// Google Cloud, Azure, Alibaba Cloud, Tencent Cloud), from the cloud's + /// metadata: its own, unlike what the internet sees of its requests + /// (that can be a router's or a shared gateway's). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cloud_public_ip: Option, /// NVIDIA GPUs Kubernetes can hand out here (allocatable nvidia.com/gpu). #[serde(default)] pub gpus: u32, @@ -644,6 +657,13 @@ pub const LOCATED_AT_ANNOTATION: &str = "pyo/located-at"; pub const TAILNET_IPV4_ANNOTATION: &str = "pyo/tailnet-ipv4"; /// Public IPv4 addresses on its own network cards, comma-separated ("" for none). pub const NIC_PUBLIC_IPV4_ANNOTATION: &str = "pyo/nic-public-ipv4"; +/// The public IPv4 address its cloud maps to it (1:1 NAT), from the cloud's +/// metadata ("" for none: not on a cloud, or no public IP there). +pub const CLOUD_PUBLIC_IPV4_ANNOTATION: &str = "pyo/cloud-public-ipv4"; +/// The boot it was last looked at in (its node's boot ID): a server that +/// started again is looked at again, as a cloud may give it another public +/// IP then. +pub const LOCATED_BOOT_ANNOTATION: &str = "pyo/located-boot"; #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] diff --git a/crates/pyo-proto/src/rbac.rs b/crates/pyo-proto/src/rbac.rs index 002b980..87c53e0 100644 --- a/crates/pyo-proto/src/rbac.rs +++ b/crates/pyo-proto/src/rbac.rs @@ -265,7 +265,7 @@ pub fn objects(upgrades: bool) -> Vec { { "apiGroups": ["networking.k8s.io"], "resources": ["networkpolicies"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::FIREWALL_POLICY] }, // Websites on the servers' Tailscale addresses: its own Service. { "apiGroups": [""], "resources": ["services"], "verbs": ["create"] }, - { "apiGroups": [""], "resources": ["services"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WEB_SERVICE, system::TAILNET_VIP_SERVICE] }, + { "apiGroups": [""], "resources": ["services"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WEB_SERVICE, system::TAILNET_VIP_SERVICE, system::PUBLIC_NAT_SERVICE] }, // The Tailscale Services' wanted list, for the servers' tailnet pods. { "apiGroups": [""], "resources": ["configmaps"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WANTED] }, ]), @@ -583,6 +583,9 @@ pub fn admission(waf_image: &str, geoip_image: &str) -> Vec { (request.resource.resource == 'services' && variables.objName == '{vip_svc}' && variables.sub == '' && (request.operation == 'DELETE' || (object.spec.all(k, k in {tw_keys}) && \ object.spec.type == 'ClusterIP' && object.spec.externalTrafficPolicy == 'Local' && object.spec.selector == {tw_selector} && object.spec.ports == {vip_ports} && \ size(object.spec.externalIPs) >= 1 && size(object.spec.externalIPs) <= 1024 && object.spec.externalIPs.all(ip, ip.matches({tw_v4}) || ip.matches({tw_v6}))))) || \ + (request.resource.resource == 'services' && variables.objName == '{nat_svc}' && variables.sub == '' && (request.operation == 'DELETE' || (object.spec.all(k, k in {tw_keys}) && \ + object.spec.type == 'ClusterIP' && object.spec.externalTrafficPolicy == 'Local' && object.spec.selector == {tw_selector} && object.spec.ports == {tw_ports} && \ + size(object.spec.externalIPs) >= 1 && size(object.spec.externalIPs) <= 64 && object.spec.externalIPs.all(ip, ip.matches({nat_v4}) && !ip.matches({nat_private}))))) || \ (request.resource.resource == 'configmaps' && variables.objName == '{tn_wanted}' && (request.operation == 'DELETE' || (has(object.data) && object.data.all(k, k == '{tn_wanted_key}')))) || \ (request.resource.resource == 'gateways' && variables.objName != '{tailnet_gw}' && (request.operation == 'DELETE' ? variables.objName != '{http_gw}' : (variables.objName == '{http_gw}' ? object.spec == {http_spec} : \ (object.metadata.labels[?'{label}'].orValue('') == 'https' && object.spec.gatewayClassName == 'pyo' && object.spec.listeners.all(l, l.port == 443 && l.protocol == 'HTTPS' && has(l.hostname) && !l.hostname.lowerAscii().endsWith('{tn_suffix}') && l.tls.mode == 'Terminate' && l.tls.certificateRefs.all(c, c.name.startsWith('{tls}') && (!has(c.namespace) || c.namespace == '{gw}') && c.kind == 'Secret') && has(l.allowedRoutes) && l.allowedRoutes.namespaces.from == 'Same')) || \ @@ -646,6 +649,12 @@ pub fn admission(waf_image: &str, geoip_image: &str) -> Vec { tw_ports = cel_literal(&system::tailnet_web_ports()), tw_v4 = cel_literal(&json!(system::TAILNET_IPV4_RE)), tw_v6 = cel_literal(&json!(system::TAILNET_IPV6_RE)), + // The cluster's own requests to the public IPs behind the cloud's + // NAT (system.rs): this Service only, onto the proxies, the + // websites' ports, and public IPv4 addresses alone. + nat_svc = system::PUBLIC_NAT_SERVICE, + nat_v4 = cel_literal(&json!(system::IPV4_RE)), + nat_private = cel_literal(&json!(system::NOT_PUBLIC_IPV4_RE)), label = system::GATEWAY_LABEL, tls = system::TLS_SECRET_PREFIX, pull = system::ORIGIN_PULL_SECRET, @@ -1317,7 +1326,7 @@ mod tests { svc, [ &json!({ "apiGroups": [""], "resources": ["services"], "verbs": ["create"] }), - &json!({ "apiGroups": [""], "resources": ["services"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WEB_SERVICE, system::TAILNET_VIP_SERVICE] }), + &json!({ "apiGroups": [""], "resources": ["services"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WEB_SERVICE, system::TAILNET_VIP_SERVICE, system::PUBLIC_NAT_SERVICE] }), ] ); let policy = admission("w", "g"); @@ -1375,6 +1384,73 @@ mod tests { } } + /// The Service for the public IPs behind the cloud's NAT: pinned like + /// the tailnet websites' one, on public IPv4 addresses only. + #[test] + fn the_public_nat_service_is_pinned() { + let policy = admission("w", "g"); + let all = policy[0]["spec"]["variables"] + .as_array() + .unwrap() + .iter() + .find(|v| v["name"] == "gatewayAllowed") + .unwrap()["expression"] + .as_str() + .unwrap(); + let start = all + .find("variables.objName == 'pyo-public-nat'") + .expect("the NAT Service's clause"); + let clause = &all[start..start + all[start..].find(")))) || ").unwrap()]; + for part in [ + "object.spec.type == 'ClusterIP' && object.spec.externalTrafficPolicy == 'Local'" + .to_owned(), + format!( + "object.spec.selector == {}", + cel_literal(&system::proxy_pods()["matchLabels"]) + ), + format!( + "object.spec.ports == {}", + cel_literal(&system::tailnet_web_ports()) + ), + "size(object.spec.externalIPs) <= 64".to_owned(), + format!( + "ip.matches({}) && !ip.matches({})", + cel_literal(&json!(system::IPV4_RE)), + cel_literal(&json!(system::NOT_PUBLIC_IPV4_RE)) + ), + ] { + assert!(clause.contains(&part), "{part}\n{clause}"); + } + // The regexes as the policy's RE2 reads them. + let v4 = regex::Regex::new(system::IPV4_RE).unwrap(); + let private = regex::Regex::new(system::NOT_PUBLIC_IPV4_RE).unwrap(); + for ip in [ + "43.156.1.2", + "3.120.1.2", + "203.0.113.7", + "100.128.0.1", + "172.32.0.1", + ] { + assert!(v4.is_match(ip) && !private.is_match(ip), "{ip}"); + } + for ip in [ + "10.3.8.13", + "172.17.0.1", + "192.168.1.5", + "100.101.67.2", + "127.0.0.1", + "169.254.169.254", + "0.1.2.3", + "224.0.0.1", + "255.255.255.255", + ] { + assert!(!v4.is_match(ip) || private.is_match(ip), "{ip}"); + } + for ip in ["999.1.2.3", "1.2.3", "1.2.3.4.5", "fd7a:115c:a1e0::1"] { + assert!(!v4.is_match(ip), "{ip}"); + } + } + #[test] fn the_geoip_image_is_the_installations() { let img = "registry.example.com/pyo/geoip:7@sha256:bb"; diff --git a/crates/pyo-proto/src/system.rs b/crates/pyo-proto/src/system.rs index 496e16f..f34f42a 100644 --- a/crates/pyo-proto/src/system.rs +++ b/crates/pyo-proto/src/system.rs @@ -1158,6 +1158,44 @@ pub fn tailnet_web_service(ips: &[String]) -> Value { }) } +// ── The public IP behind the cloud's NAT ─────────────────────────────────── +// +// A cloud server whose public IP the cloud maps to it (1:1 NAT, locate.rs) +// doesn't have that IP itself: a request from the server to it (cert- +// manager's check of its own HTTP-01 answer before Let's Encrypt is asked, +// a service calling another's address) would leave for the cloud's NAT and +// may never come back (clouds don't all send it back in). The agent's +// `PUBLIC_NAT_SERVICE` puts those IPs' ports 80 and 443 on the proxies +// within the cluster, as the internet reaches them from outside: kube-proxy +// takes such a request to a proxy (externalIPs). Nothing from outside +// carries these addresses (the NAT made them the private ones), so this +// changes nothing for visitors. The admission policy pins it: this name, +// the websites' ports, the proxies' pods, and public IPv4 addresses only. + +/// The agent's Service for the cluster's own requests to the public IPs +/// behind the cloud's NAT. +pub const PUBLIC_NAT_SERVICE: &str = "pyo-public-nat"; +/// An IPv4 address, as the admission policy checks it. +pub const IPV4_RE: &str = r"^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$"; +/// An IPv4 address that isn't public: this network, private, tailnet +/// (CGNAT), loopback, link-local, multicast or reserved. +pub const NOT_PUBLIC_IPV4_RE: &str = r"^(0|10|127|22[4-9]|2[3-5][0-9])\.|^169\.254\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\.|^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\."; + +/// The Service, on these public IPs (`ips`, at least one). +pub fn public_nat_service(ips: &[String]) -> Value { + json!({ + "apiVersion": "v1", "kind": "Service", + "metadata": { "name": PUBLIC_NAT_SERVICE, "namespace": GATEWAY_NS, "labels": { "app.kubernetes.io/managed-by": "pyo" } }, + "spec": { + "type": "ClusterIP", + "externalIPs": ips, + "externalTrafficPolicy": "Local", + "selector": proxy_pods()["matchLabels"], + "ports": tailnet_web_ports(), + }, + }) +} + /// Spec fields the API server may fill in on it, besides those it sets. pub const TAILNET_WEB_DEFAULTED: [&str; 6] = [ "clusterIP", diff --git a/dash/src/components/pyo/Tailscale.tsx b/dash/src/components/pyo/Tailscale.tsx index 6816204..1bad45e 100644 --- a/dash/src/components/pyo/Tailscale.tsx +++ b/dash/src/components/pyo/Tailscale.tsx @@ -216,11 +216,13 @@ function PublicDashboardRow() { description={ !pd.possible ? "Off. Ports 80 and 443 don't reach this server." - : locked - ? "Stays on until Pyo is on your tailnet." - : pd.open - ? (pd.host ?? "Starting") - : undefined + : pd.open && !pd.host && pd.blocked + ? "Allow ports 80 and 443 in your cloud firewall." + : locked + ? "Stays on until Pyo is on your tailnet." + : pd.open + ? (pd.host ?? "Starting") + : undefined } /> diff --git a/dash/src/lib/pyo/types.ts b/dash/src/lib/pyo/types.ts index 0bcf307..9add092 100644 --- a/dash/src/lib/pyo/types.ts +++ b/dash/src/lib/pyo/types.ts @@ -1873,8 +1873,10 @@ export type PublicDashboard = { open: boolean; /** "pyo.203.0.113.7.nip.io", once it answers over HTTPS. */ host: string | null; - /** It can be open: ports 80 and 443 reach Pyo's server (not behind NAT). */ + /** It can be open: Pyo's server has a public IP of its own (not behind a router's NAT). */ possible: boolean; + /** Let's Encrypt couldn't reach that IP: ports 80 and 443 are closed in a cloud firewall. */ + blocked?: boolean; }; // ── Integrations ─────────────────────────────────────────────────────────── From d47fcd9e619cb33ab35471f3f3978d809c72e32e Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Wed, 18 Nov 2026 00:00:06 +0700 Subject: [PATCH 04/22] style(scripts): show steps as a checklist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install, connect, join, sandbox and pyo-up scripts drew each step over the last one and printed "==>" headings in bold even into logs. The shared pyo:progress block now keeps finished steps on screen: a ticking "→ Downloading k3s… 0:12" becomes "āœ“ Downloaded k3s 0:12", a failure "āœ— Downloading k3s" before the error, and a wait that goes on by itself "ā— Starting the dashboard". say, ok_line and dot_line come with it, so headings, last lines and status dots look the same in every script. Colors only on a terminal without NO_COLOR. In a log, nothing is drawn over and each line comes once. --- crates/pyo-api/assets/connect.sh | 104 +++++++++++++++++++++---- crates/pyo-api/assets/join.sh | 99 ++++++++++++++++++++---- crates/pyo-api/src/join.rs | 14 ++-- deploy/install.sh | 114 ++++++++++++++++++++++----- deploy/sandbox.sh | 113 ++++++++++++++++++++++----- scripts/build-images.sh | 2 +- scripts/install.test.sh | 47 +++++++++-- scripts/pyo-up.sh | 129 ++++++++++++++++++++++++------- scripts/sandbox.test.sh | 21 ++--- 9 files changed, 520 insertions(+), 123 deletions(-) diff --git a/crates/pyo-api/assets/connect.sh b/crates/pyo-api/assets/connect.sh index 8f693fb..110b330 100644 --- a/crates/pyo-api/assets/connect.sh +++ b/crates/pyo-api/assets/connect.sh @@ -55,11 +55,33 @@ become_root() { # --uninstall uses kubectl as you with KUBECONFIG set, or without k3s here. case " $* " in *" --uninstall "*) [ -n "${KUBECONFIG:-}" ] || ! command -v k3s >/dev/null || become_root "$@" ;; *) become_root "$@" ;; esac # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) -# A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line -# comes once (not again for a step with the same line), then with its time -# every 20 seconds while it runs. progress_start MESSAGE, then -# progress_tick as often as it likes, then progress_end. +# Steps, as a checklist. On a terminal, the step that runs ticks on one +# line ("→ Downloading k3s… 0:12"), then stays: done ("āœ“ Downloaded k3s +# 0:12", the time once it took a while), failed ("āœ— Downloading k3s") or +# still going on its own ("ā— Starting the dashboard"), in color unless +# NO_COLOR is set. Elsewhere (piped, a CI log), no colors and no line +# drawn over: each step's line comes once (not again for a step with the +# same line), then with its time every 20 seconds while it runs, then how +# it ended (once for steps alike). progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end [ok|fail|wait] +# [WHAT IT DID]. And: say TEXT, a heading. ok_line TEXT, the last line +# once all went well. dot_line green|yellow|red|dim TEXT, how something +# is (running, starting, failed, stopped). +progress_color() { [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; } +# progress_paint SGR TEXT: TEXT in that color, when there are colors (it +# prints itself: in $(...) there's never a terminal). +progress_paint() { + if progress_color; then printf '\033[%sm%s\033[0m' "$1" "$2"; else printf '%s' "$2"; fi +} +progress_time() { printf '%d:%02d' $(($1 / 60)) $(($1 % 60)); } +say() { progress_paint 1 "→ $*"; echo; } +ok_line() { progress_paint '1;32' "āœ“ $*"; echo; } +dot_line() { + local c=2 + case $1 in green) c=32 ;; yellow) c=33 ;; red) c=31 ;; esac + progress_paint "$c" ā— + printf ' %s\n' "$2" +} progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -68,33 +90,82 @@ progress_tick() { [ -n "${PROGRESS_MSG:-}" ] || return 0 local t=$((SECONDS - PROGRESS_T0)) if [ -t 1 ]; then - printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '\r\033[K' + progress_paint 36 → + printf ' %s ' "$PROGRESS_MSG" + progress_paint 2 "$(progress_time "$t")" elif [ "$t" -ge "$PROGRESS_NEXT" ]; then if [ "$PROGRESS_NEXT" -gt 0 ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '→ %s %s\n' "$PROGRESS_MSG" "$(progress_time "$t")" elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then - printf '%s\n' "$PROGRESS_MSG" + printf '→ %s\n' "$PROGRESS_MSG" fi PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } +# What a step did, from what it was doing: "Downloading k3s…" is +# "Downloaded k3s". +progress_done() { + local m=${1%…} w + w=${m%% *} + case $w in + Applying) w=Applied ;; Building) w=Built ;; Checking) w=Checked ;; + Connecting) w=Connected ;; Copying) w=Copied ;; Creating) w=Created ;; + Deleting) w=Deleted ;; Downloading) w=Downloaded ;; Getting) w=Got ;; + Installing) w=Installed ;; Joining) w=Joined ;; Moving) w=Moved ;; + Removing) w=Removed ;; Restarting) w=Restarted ;; Setting) w=Set ;; + Starting) w=Started ;; Stopping) w=Stopped ;; Updating) w=Updated ;; + Waiting) w=Waited ;; + esac + printf '%s%s' "$w" "${m#"${m%% *}"}" +} progress_end() { [ -n "${PROGRESS_MSG:-}" ] || return 0 + local t=$((SECONDS - PROGRESS_T0)) done="" [ ! -t 1 ] || printf '\r\033[K' + [ "${1:-ok}" != ok ] || done=${2:-$(progress_done "$PROGRESS_MSG")} + # The same step again (a file each): said once. + if [ -n "$done" ] && [ "$done" = "${PROGRESS_DONE:-}" ]; then + PROGRESS_MSG="" + return 0 + fi + PROGRESS_DONE=$done + case ${1:-ok} in + ok) + progress_paint 32 āœ“ + printf ' %s' "$done" + ;; + fail) + progress_paint 31 "āœ— ${PROGRESS_MSG%…}" + t=0 + ;; + *) + progress_paint 33 ā— + printf ' %s' "${2:-${PROGRESS_MSG%…}}" + ;; + esac + if [ "$t" -ge 5 ]; then + printf ' ' + progress_paint 2 "$(progress_time "$t")" + fi + echo PROGRESS_MSG="" } -# wait_until MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds until it -# succeeds, or a failure once SECONDS have passed. Nothing to show when it +# wait_until [-f] MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds +# until it succeeds, or a failure once SECONDS have passed: the step goes +# on by itself ("ā—"), or with -f it failed ("āœ—"). Nothing to show when it # succeeds at once. wait_until() { + local end=wait + [ "$1" != -f ] || { end=fail; shift; } local msg=$1 limit=$2 t0=$SECONDS shift 2 "$@" && return 0 progress_start "$msg" until "$@"; do if [ $((SECONDS - t0)) -ge "$limit" ]; then - progress_end + progress_end "$end" return 1 fi sleep 1 @@ -118,7 +189,7 @@ quietly() { progress_tick done wait "$pid" || rc=$? - progress_end + if [ "$rc" = 0 ]; then progress_end; else progress_end fail; fi [ "$rc" = 0 ] || cat "$log" >&2 rm -f "$log" return "$rc" @@ -1114,7 +1185,7 @@ else fi k3s_up() { k3s kubectl get nodes >/dev/null 2>&1; } -wait_until "Starting k3s…" 180 k3s_up || { echo "k3s didn't start. Restart this machine, then run the same command again." >&2; exit 1; } +wait_until -f "Starting k3s…" 180 k3s_up || { echo "k3s didn't start. Restart this machine, then run the same command again." >&2; exit 1; } # An existing k3s didn't get the labels at registration: add them now. if [ -n "$GPU_LABELS" ] && k3s kubectl get node "$(hostname)" >/dev/null 2>&1; then # shellcheck disable=SC2086 # one label per word @@ -1173,7 +1244,7 @@ server_minor() { # the policy, quietly. apply_agent() { local minor i delay=5 until=$((SECONDS + 300)) - echo "Installing the Pyo agent…" + progress_start "Installing the Pyo agent…" while :; do if k3s kubectl get --raw=/readyz --request-timeout=20s >/dev/null 2>&1 && k3s kubectl apply --server-side --force-conflicts --field-manager=pyo --request-timeout=120s -f "$MANIFEST" >/dev/null; then @@ -1186,10 +1257,9 @@ apply_agent() { return 0 fi if [ "$SECONDS" -ge "$until" ]; then - progress_end + progress_end fail return 1 fi - [ -n "${PROGRESS_MSG:-}" ] || progress_start "Still installing the Pyo agent…" for ((i = 0; i < delay; i++)); do sleep 1 progress_tick @@ -1207,4 +1277,4 @@ if ! quietly "Starting the Pyo agent…" k3s kubectl -n pyo-system rollout statu echo "The Pyo agent didn't start. Run the same command again in a minute." >&2 exit 1 fi -echo "Connected $(hostname). It shows up in Pyo in a few seconds." +ok_line "Connected $(hostname). It shows up in Pyo in a few seconds." diff --git a/crates/pyo-api/assets/join.sh b/crates/pyo-api/assets/join.sh index 1bc0229..61f21df 100644 --- a/crates/pyo-api/assets/join.sh +++ b/crates/pyo-api/assets/join.sh @@ -44,11 +44,33 @@ become_root() { # from here, without what came before. become_root "$@" # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) -# A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line -# comes once (not again for a step with the same line), then with its time -# every 20 seconds while it runs. progress_start MESSAGE, then -# progress_tick as often as it likes, then progress_end. +# Steps, as a checklist. On a terminal, the step that runs ticks on one +# line ("→ Downloading k3s… 0:12"), then stays: done ("āœ“ Downloaded k3s +# 0:12", the time once it took a while), failed ("āœ— Downloading k3s") or +# still going on its own ("ā— Starting the dashboard"), in color unless +# NO_COLOR is set. Elsewhere (piped, a CI log), no colors and no line +# drawn over: each step's line comes once (not again for a step with the +# same line), then with its time every 20 seconds while it runs, then how +# it ended (once for steps alike). progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end [ok|fail|wait] +# [WHAT IT DID]. And: say TEXT, a heading. ok_line TEXT, the last line +# once all went well. dot_line green|yellow|red|dim TEXT, how something +# is (running, starting, failed, stopped). +progress_color() { [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; } +# progress_paint SGR TEXT: TEXT in that color, when there are colors (it +# prints itself: in $(...) there's never a terminal). +progress_paint() { + if progress_color; then printf '\033[%sm%s\033[0m' "$1" "$2"; else printf '%s' "$2"; fi +} +progress_time() { printf '%d:%02d' $(($1 / 60)) $(($1 % 60)); } +say() { progress_paint 1 "→ $*"; echo; } +ok_line() { progress_paint '1;32' "āœ“ $*"; echo; } +dot_line() { + local c=2 + case $1 in green) c=32 ;; yellow) c=33 ;; red) c=31 ;; esac + progress_paint "$c" ā— + printf ' %s\n' "$2" +} progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -57,33 +79,82 @@ progress_tick() { [ -n "${PROGRESS_MSG:-}" ] || return 0 local t=$((SECONDS - PROGRESS_T0)) if [ -t 1 ]; then - printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '\r\033[K' + progress_paint 36 → + printf ' %s ' "$PROGRESS_MSG" + progress_paint 2 "$(progress_time "$t")" elif [ "$t" -ge "$PROGRESS_NEXT" ]; then if [ "$PROGRESS_NEXT" -gt 0 ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '→ %s %s\n' "$PROGRESS_MSG" "$(progress_time "$t")" elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then - printf '%s\n' "$PROGRESS_MSG" + printf '→ %s\n' "$PROGRESS_MSG" fi PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } +# What a step did, from what it was doing: "Downloading k3s…" is +# "Downloaded k3s". +progress_done() { + local m=${1%…} w + w=${m%% *} + case $w in + Applying) w=Applied ;; Building) w=Built ;; Checking) w=Checked ;; + Connecting) w=Connected ;; Copying) w=Copied ;; Creating) w=Created ;; + Deleting) w=Deleted ;; Downloading) w=Downloaded ;; Getting) w=Got ;; + Installing) w=Installed ;; Joining) w=Joined ;; Moving) w=Moved ;; + Removing) w=Removed ;; Restarting) w=Restarted ;; Setting) w=Set ;; + Starting) w=Started ;; Stopping) w=Stopped ;; Updating) w=Updated ;; + Waiting) w=Waited ;; + esac + printf '%s%s' "$w" "${m#"${m%% *}"}" +} progress_end() { [ -n "${PROGRESS_MSG:-}" ] || return 0 + local t=$((SECONDS - PROGRESS_T0)) done="" [ ! -t 1 ] || printf '\r\033[K' + [ "${1:-ok}" != ok ] || done=${2:-$(progress_done "$PROGRESS_MSG")} + # The same step again (a file each): said once. + if [ -n "$done" ] && [ "$done" = "${PROGRESS_DONE:-}" ]; then + PROGRESS_MSG="" + return 0 + fi + PROGRESS_DONE=$done + case ${1:-ok} in + ok) + progress_paint 32 āœ“ + printf ' %s' "$done" + ;; + fail) + progress_paint 31 "āœ— ${PROGRESS_MSG%…}" + t=0 + ;; + *) + progress_paint 33 ā— + printf ' %s' "${2:-${PROGRESS_MSG%…}}" + ;; + esac + if [ "$t" -ge 5 ]; then + printf ' ' + progress_paint 2 "$(progress_time "$t")" + fi + echo PROGRESS_MSG="" } -# wait_until MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds until it -# succeeds, or a failure once SECONDS have passed. Nothing to show when it +# wait_until [-f] MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds +# until it succeeds, or a failure once SECONDS have passed: the step goes +# on by itself ("ā—"), or with -f it failed ("āœ—"). Nothing to show when it # succeeds at once. wait_until() { + local end=wait + [ "$1" != -f ] || { end=fail; shift; } local msg=$1 limit=$2 t0=$SECONDS shift 2 "$@" && return 0 progress_start "$msg" until "$@"; do if [ $((SECONDS - t0)) -ge "$limit" ]; then - progress_end + progress_end "$end" return 1 fi sleep 1 @@ -107,7 +178,7 @@ quietly() { progress_tick done wait "$pid" || rc=$? - progress_end + if [ "$rc" = 0 ]; then progress_end; else progress_end fail; fi [ "$rc" = 0 ] || cat "$log" >&2 rm -f "$log" return "$rc" @@ -704,7 +775,7 @@ export K3S_URL="https://$SERVER_IP:6443" K3S_TOKEN quietly "Installing k3s and joining the cluster…" k3s_install "$K3S_DIR" agent || { echo "k3s didn't install. Run the same command again." >&2; exit 1; } if [ -n "$BUILDER" ]; then - echo "Joined as $NODE_NAME, for builds only. The builder starts on it in a minute." + ok_line "Joined as $NODE_NAME, for builds only. The builder starts on it in a minute." else - echo "Joined as $NODE_NAME. It shows up in Pyo in a few seconds." + ok_line "Joined as $NODE_NAME. It shows up in Pyo in a few seconds." fi diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index e057afb..7d16617 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -1127,15 +1127,12 @@ if apply_agent; then echo "ok after $applies"; else echo "failed after $applies" let (code, out, err) = apply_agent(&version_json("37", "37"), 3); assert_eq!(code, 0, "{err}"); assert!(out.trim().ends_with("ok after 4"), "{out}"); - // What it's doing, then its progress line once and with the time - // every 20 seconds off a terminal, and no Kubernetes errors or tries. - assert!(out.starts_with("Installing the Pyo agent…")); - assert!(out.contains("\nStill installing the Pyo agent…\n"), "{out}"); + // The step once, then with the time every 20 seconds off a + // terminal, then done, and no Kubernetes errors or tries. + assert!(out.starts_with("→ Installing the Pyo agent…\n"), "{out}"); assert!(!out.contains("agent… 0:00"), "{out}"); - assert!( - out.contains("Still installing the Pyo agent… 0:20"), - "{out}" - ); + assert!(out.contains("→ Installing the Pyo agent… 0:20"), "{out}"); + assert!(out.contains("\nāœ“ Installed the Pyo agent "), "{out}"); assert!(!out.contains("try ") && !err.contains("try "), "{out}{err}"); } @@ -1166,6 +1163,7 @@ if apply_agent; then echo "ok after $applies"; else echo "failed after $applies" assert_eq!(code, 0, "{err}"); let last = out.trim().lines().last().unwrap(); assert!(last.starts_with("failed after "), "{out}"); + assert!(out.contains("\nāœ— Installing the Pyo agent\n"), "{out}"); let tries: u32 = last["failed after ".len()..].parse().unwrap(); assert!((10..=14).contains(&tries), "{tries} tries"); assert!(!out.contains("older than") && !err.contains("older than")); diff --git a/deploy/install.sh b/deploy/install.sh index a1856fc..a72d816 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -40,9 +40,8 @@ defaults() { UNINSTALL="" PURGE="" KEEP_K3S="" REMOVE_K3S="" DRY_RUN="" } -say() { printf '\033[1m==> %s\033[0m\n' "$*"; } fail() { - progress_end + progress_end fail printf 'Error: %s\n' "$*" >&2 exit 1 } @@ -50,11 +49,33 @@ fail() { kc() { k3s kubectl "$@"; } # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) -# A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line -# comes once (not again for a step with the same line), then with its time -# every 20 seconds while it runs. progress_start MESSAGE, then -# progress_tick as often as it likes, then progress_end. +# Steps, as a checklist. On a terminal, the step that runs ticks on one +# line ("→ Downloading k3s… 0:12"), then stays: done ("āœ“ Downloaded k3s +# 0:12", the time once it took a while), failed ("āœ— Downloading k3s") or +# still going on its own ("ā— Starting the dashboard"), in color unless +# NO_COLOR is set. Elsewhere (piped, a CI log), no colors and no line +# drawn over: each step's line comes once (not again for a step with the +# same line), then with its time every 20 seconds while it runs, then how +# it ended (once for steps alike). progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end [ok|fail|wait] +# [WHAT IT DID]. And: say TEXT, a heading. ok_line TEXT, the last line +# once all went well. dot_line green|yellow|red|dim TEXT, how something +# is (running, starting, failed, stopped). +progress_color() { [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; } +# progress_paint SGR TEXT: TEXT in that color, when there are colors (it +# prints itself: in $(...) there's never a terminal). +progress_paint() { + if progress_color; then printf '\033[%sm%s\033[0m' "$1" "$2"; else printf '%s' "$2"; fi +} +progress_time() { printf '%d:%02d' $(($1 / 60)) $(($1 % 60)); } +say() { progress_paint 1 "→ $*"; echo; } +ok_line() { progress_paint '1;32' "āœ“ $*"; echo; } +dot_line() { + local c=2 + case $1 in green) c=32 ;; yellow) c=33 ;; red) c=31 ;; esac + progress_paint "$c" ā— + printf ' %s\n' "$2" +} progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -63,33 +84,82 @@ progress_tick() { [ -n "${PROGRESS_MSG:-}" ] || return 0 local t=$((SECONDS - PROGRESS_T0)) if [ -t 1 ]; then - printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '\r\033[K' + progress_paint 36 → + printf ' %s ' "$PROGRESS_MSG" + progress_paint 2 "$(progress_time "$t")" elif [ "$t" -ge "$PROGRESS_NEXT" ]; then if [ "$PROGRESS_NEXT" -gt 0 ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '→ %s %s\n' "$PROGRESS_MSG" "$(progress_time "$t")" elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then - printf '%s\n' "$PROGRESS_MSG" + printf '→ %s\n' "$PROGRESS_MSG" fi PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } +# What a step did, from what it was doing: "Downloading k3s…" is +# "Downloaded k3s". +progress_done() { + local m=${1%…} w + w=${m%% *} + case $w in + Applying) w=Applied ;; Building) w=Built ;; Checking) w=Checked ;; + Connecting) w=Connected ;; Copying) w=Copied ;; Creating) w=Created ;; + Deleting) w=Deleted ;; Downloading) w=Downloaded ;; Getting) w=Got ;; + Installing) w=Installed ;; Joining) w=Joined ;; Moving) w=Moved ;; + Removing) w=Removed ;; Restarting) w=Restarted ;; Setting) w=Set ;; + Starting) w=Started ;; Stopping) w=Stopped ;; Updating) w=Updated ;; + Waiting) w=Waited ;; + esac + printf '%s%s' "$w" "${m#"${m%% *}"}" +} progress_end() { [ -n "${PROGRESS_MSG:-}" ] || return 0 + local t=$((SECONDS - PROGRESS_T0)) done="" [ ! -t 1 ] || printf '\r\033[K' + [ "${1:-ok}" != ok ] || done=${2:-$(progress_done "$PROGRESS_MSG")} + # The same step again (a file each): said once. + if [ -n "$done" ] && [ "$done" = "${PROGRESS_DONE:-}" ]; then + PROGRESS_MSG="" + return 0 + fi + PROGRESS_DONE=$done + case ${1:-ok} in + ok) + progress_paint 32 āœ“ + printf ' %s' "$done" + ;; + fail) + progress_paint 31 "āœ— ${PROGRESS_MSG%…}" + t=0 + ;; + *) + progress_paint 33 ā— + printf ' %s' "${2:-${PROGRESS_MSG%…}}" + ;; + esac + if [ "$t" -ge 5 ]; then + printf ' ' + progress_paint 2 "$(progress_time "$t")" + fi + echo PROGRESS_MSG="" } -# wait_until MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds until it -# succeeds, or a failure once SECONDS have passed. Nothing to show when it +# wait_until [-f] MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds +# until it succeeds, or a failure once SECONDS have passed: the step goes +# on by itself ("ā—"), or with -f it failed ("āœ—"). Nothing to show when it # succeeds at once. wait_until() { + local end=wait + [ "$1" != -f ] || { end=fail; shift; } local msg=$1 limit=$2 t0=$SECONDS shift 2 "$@" && return 0 progress_start "$msg" until "$@"; do if [ $((SECONDS - t0)) -ge "$limit" ]; then - progress_end + progress_end "$end" return 1 fi sleep 1 @@ -113,7 +183,7 @@ quietly() { progress_tick done wait "$pid" || rc=$? - progress_end + if [ "$rc" = 0 ]; then progress_end; else progress_end fail; fi [ "$rc" = 0 ] || cat "$log" >&2 rm -f "$log" return "$rc" @@ -535,7 +605,7 @@ REGISTRIES install -d -m 700 /etc/pyo echo "k3s was installed by Pyo's install.sh on $(date -u +%F)." > /etc/pyo/k3s-installed fi - wait_until "Starting k3s…" 180 k3s_up || fail "k3s didn't start. Restart this machine, then run the same command again." + wait_until -f "Starting k3s…" 180 k3s_up || fail "k3s didn't start. Restart this machine, then run the same command again." } k3s_up() { kc get nodes >/dev/null 2>&1; } @@ -1109,7 +1179,6 @@ YAML progress_tick done done - progress_end local logs outcome logs=$(kc -n "$NS" logs "job/$name" 2>/dev/null || true) outcome=$(printf '%s\n' "$logs" | grep pyoUpgradeOutcome | tail -n1 || true) @@ -1117,8 +1186,9 @@ YAML # Its last line says how it went; without it (logs gone), the Job's state. # When it failed, what it did goes to stderr first. case "$outcome" in - *'"ok":true'*) say "Pyo $R_VERSION is ready" ;; + *'"ok":true'*) progress_end ok "Started Pyo $R_VERSION" ;; *'"ok":false'*) + progress_end fail upgrader_log "$logs" local why why=$(printf '%s' "$outcome" | sed -n 's/.*"message":"\([^"\\]*\)".*/\1/p' | head -n1) @@ -1127,12 +1197,14 @@ YAML ;; *) case "$st" in - 1/*) say "Pyo $R_VERSION is ready" ;; + 1/*) progress_end ok "Started Pyo $R_VERSION" ;; */1) + progress_end fail upgrader_log "$logs" fail "Pyo $R_VERSION didn't start. The version that was running keeps running. Run the same command again to try again." ;; *) + progress_end fail upgrader_log "$logs" fail "Pyo $R_VERSION didn't start in time. Run the same command again to try again." ;; @@ -1541,7 +1613,7 @@ uninstall_pyo() { echo "Dry run done: nothing was changed." return 0 fi - say "Pyo is uninstalled" + ok_line "Pyo is uninstalled" if [ -n "$keep_storage" ]; then echo "Kept the volumes' files (Pyo's own database, registry and your volumes and databases) in:" echo " $KEPT_DIR/storage" @@ -1580,7 +1652,7 @@ main() { # Pyo's scripts on this machine, like pyo-reset-login, as this # release has them. host_units - say "Pyo $R_VERSION is already installed. Nothing to update" + ok_line "Pyo $R_VERSION is already installed. Nothing to update" pyo_addresses print_sign_in return 0 @@ -1595,7 +1667,7 @@ main() { ADMIN_PASSWORD="" run_upgrader "$fresh" STARTING="" - wait_until "Waiting for the dashboard…" 300 dashboard_up || STARTING=1 + wait_until "Starting the dashboard…" 300 dashboard_up || STARTING=1 if [ -n "$fresh" ]; then # The first sign-in is printed below. count_install diff --git a/deploy/sandbox.sh b/deploy/sandbox.sh index 86bc20a..31b95c7 100644 --- a/deploy/sandbox.sh +++ b/deploy/sandbox.sh @@ -63,11 +63,33 @@ set -euo pipefail # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) -# A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line -# comes once (not again for a step with the same line), then with its time -# every 20 seconds while it runs. progress_start MESSAGE, then -# progress_tick as often as it likes, then progress_end. +# Steps, as a checklist. On a terminal, the step that runs ticks on one +# line ("→ Downloading k3s… 0:12"), then stays: done ("āœ“ Downloaded k3s +# 0:12", the time once it took a while), failed ("āœ— Downloading k3s") or +# still going on its own ("ā— Starting the dashboard"), in color unless +# NO_COLOR is set. Elsewhere (piped, a CI log), no colors and no line +# drawn over: each step's line comes once (not again for a step with the +# same line), then with its time every 20 seconds while it runs, then how +# it ended (once for steps alike). progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end [ok|fail|wait] +# [WHAT IT DID]. And: say TEXT, a heading. ok_line TEXT, the last line +# once all went well. dot_line green|yellow|red|dim TEXT, how something +# is (running, starting, failed, stopped). +progress_color() { [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; } +# progress_paint SGR TEXT: TEXT in that color, when there are colors (it +# prints itself: in $(...) there's never a terminal). +progress_paint() { + if progress_color; then printf '\033[%sm%s\033[0m' "$1" "$2"; else printf '%s' "$2"; fi +} +progress_time() { printf '%d:%02d' $(($1 / 60)) $(($1 % 60)); } +say() { progress_paint 1 "→ $*"; echo; } +ok_line() { progress_paint '1;32' "āœ“ $*"; echo; } +dot_line() { + local c=2 + case $1 in green) c=32 ;; yellow) c=33 ;; red) c=31 ;; esac + progress_paint "$c" ā— + printf ' %s\n' "$2" +} progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -76,33 +98,82 @@ progress_tick() { [ -n "${PROGRESS_MSG:-}" ] || return 0 local t=$((SECONDS - PROGRESS_T0)) if [ -t 1 ]; then - printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '\r\033[K' + progress_paint 36 → + printf ' %s ' "$PROGRESS_MSG" + progress_paint 2 "$(progress_time "$t")" elif [ "$t" -ge "$PROGRESS_NEXT" ]; then if [ "$PROGRESS_NEXT" -gt 0 ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '→ %s %s\n' "$PROGRESS_MSG" "$(progress_time "$t")" elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then - printf '%s\n' "$PROGRESS_MSG" + printf '→ %s\n' "$PROGRESS_MSG" fi PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } +# What a step did, from what it was doing: "Downloading k3s…" is +# "Downloaded k3s". +progress_done() { + local m=${1%…} w + w=${m%% *} + case $w in + Applying) w=Applied ;; Building) w=Built ;; Checking) w=Checked ;; + Connecting) w=Connected ;; Copying) w=Copied ;; Creating) w=Created ;; + Deleting) w=Deleted ;; Downloading) w=Downloaded ;; Getting) w=Got ;; + Installing) w=Installed ;; Joining) w=Joined ;; Moving) w=Moved ;; + Removing) w=Removed ;; Restarting) w=Restarted ;; Setting) w=Set ;; + Starting) w=Started ;; Stopping) w=Stopped ;; Updating) w=Updated ;; + Waiting) w=Waited ;; + esac + printf '%s%s' "$w" "${m#"${m%% *}"}" +} progress_end() { [ -n "${PROGRESS_MSG:-}" ] || return 0 + local t=$((SECONDS - PROGRESS_T0)) done="" [ ! -t 1 ] || printf '\r\033[K' + [ "${1:-ok}" != ok ] || done=${2:-$(progress_done "$PROGRESS_MSG")} + # The same step again (a file each): said once. + if [ -n "$done" ] && [ "$done" = "${PROGRESS_DONE:-}" ]; then + PROGRESS_MSG="" + return 0 + fi + PROGRESS_DONE=$done + case ${1:-ok} in + ok) + progress_paint 32 āœ“ + printf ' %s' "$done" + ;; + fail) + progress_paint 31 "āœ— ${PROGRESS_MSG%…}" + t=0 + ;; + *) + progress_paint 33 ā— + printf ' %s' "${2:-${PROGRESS_MSG%…}}" + ;; + esac + if [ "$t" -ge 5 ]; then + printf ' ' + progress_paint 2 "$(progress_time "$t")" + fi + echo PROGRESS_MSG="" } -# wait_until MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds until it -# succeeds, or a failure once SECONDS have passed. Nothing to show when it +# wait_until [-f] MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds +# until it succeeds, or a failure once SECONDS have passed: the step goes +# on by itself ("ā—"), or with -f it failed ("āœ—"). Nothing to show when it # succeeds at once. wait_until() { + local end=wait + [ "$1" != -f ] || { end=fail; shift; } local msg=$1 limit=$2 t0=$SECONDS shift 2 "$@" && return 0 progress_start "$msg" until "$@"; do if [ $((SECONDS - t0)) -ge "$limit" ]; then - progress_end + progress_end "$end" return 1 fi sleep 1 @@ -126,7 +197,7 @@ quietly() { progress_tick done wait "$pid" || rc=$? - progress_end + if [ "$rc" = 0 ]; then progress_end; else progress_end fail; fi [ "$rc" = 0 ] || cat "$log" >&2 rm -f "$log" return "$rc" @@ -288,9 +359,8 @@ defaults() { EDGE_HTTP=80 EDGE_HTTPS=443 } -say() { printf '\033[1m==> %s\033[0m\n' "$*"; } fail() { - progress_end + progress_end fail printf 'Error: %s\n' "$*" >&2 exit 1 } @@ -576,9 +646,9 @@ detect() { detect_once fi case "$RUNTIME:$KVM" in - firecracker:native) echo "Firecracker on KVM. A test VM booted in ${BOOT:-?} ms." ;; - firecracker:*) echo "Firecracker on nested KVM. A test VM booted in ${BOOT:-?} ms, so sandboxes start slower." ;; - gvisor:*) echo "gVisor, because this server has no usable KVM." ;; + firecracker:native) dot_line green "Firecracker on KVM. A test VM booted in ${BOOT:-?} ms." ;; + firecracker:*) dot_line yellow "Firecracker on nested KVM. A test VM booted in ${BOOT:-?} ms, so sandboxes start slower." ;; + gvisor:*) dot_line yellow "gVisor, because this server has no usable KVM." ;; *) fail "This server can't run sandboxes. ${REASON:-It has neither KVM nor gVisor.}" ;; esac } @@ -710,7 +780,7 @@ start_daemon() { systemctl daemon-reload systemctl enable pyo-sandbox >/dev/null 2>&1 systemctl restart pyo-sandbox - wait_until "Starting the sandbox daemon…" 60 daemon_up && return 0 + wait_until -f "Starting the sandbox daemon…" 60 daemon_up && return 0 journalctl -u pyo-sandbox -n 20 --no-pager >&2 || true fail "The sandbox daemon didn't start. Its last log lines are above. Run the same command again." } @@ -759,7 +829,8 @@ uninstall() { ufw delete allow "$EDGE_HTTPS/tcp" >/dev/null 2>&1 || true fi fi - say "Done. This machine is still on your tailnet. You can remove it in the Tailscale admin console." + ok_line "Removed the Pyo sandbox daemon" + echo "This machine is still on your tailnet. You can remove it in the Tailscale admin console." } cleanup() { [ -z "${WORK:-}" ] || rm -rf "$WORK"; } @@ -794,9 +865,9 @@ main() { firewall start_daemon if [ -n "$CODE" ]; then - say "$NAME is a sandbox server now. It shows up in Pyo in a few seconds." + ok_line "$NAME is a sandbox server now. It shows up in Pyo in a few seconds." else - say "The sandbox daemon is updated. Its sandboxes kept running." + ok_line "The sandbox daemon is updated. Its sandboxes kept running." fi } diff --git a/scripts/build-images.sh b/scripts/build-images.sh index a004c05..28de206 100755 --- a/scripts/build-images.sh +++ b/scripts/build-images.sh @@ -92,7 +92,7 @@ if [ -t 1 ] && [ "${TERM:-dumb}" != dumb ]; then YELLOW=$'\033[33m' CYAN=$'\033[36m' RESET=$'\033[0m' fi fi -log() { printf '%s==> %s%s\n' "$BOLD" "$*" "$RESET"; } +log() { printf '%s→ %s%s\n' "$BOLD" "$*" "$RESET"; } WORK=$(mktemp -d) DRAWN=0 INTERRUPTED="" diff --git a/scripts/install.test.sh b/scripts/install.test.sh index 9ec09d9..f93d02d 100755 --- a/scripts/install.test.sh +++ b/scripts/install.test.sh @@ -191,8 +191,8 @@ ips="10.0.2.15 203.0.113.7" : >"$asked" out=$(pyo_addresses 300) check "a public IP: waits five minutes" "$(wc -l <"$asked")" "^1[45][0-9]$" has -check "a public IP: says it waits, as it goes" "$out" "^Waiting for the HTTPS address \(up to 5 minutes\)…$" has -check "a public IP: every 20 seconds in a log" "$out" "^Waiting for the HTTPS address \(up to 5 minutes\)… 4:40$" has +check "a public IP: says it waits, as it goes" "$out" "^→ Waiting for the HTTPS address \(up to 5 minutes\)…$" has +check "a public IP: every 20 seconds in a log" "$out" "^→ Waiting for the HTTPS address \(up to 5 minutes\)… 4:40$" has check "a public IP: not more often" "$out" "… 0:1[0-9]$" lacks meta='{"publicUrl":"https://pyo.203.0.113.7.nip.io"}' : >"$asked" @@ -200,22 +200,57 @@ out=$(pyo_addresses 300) check "an HTTPS address: no wait" "$out" "Waiting" lacks check "an HTTPS address: shown" "$out" "https://pyo\.203\.0\.113\.7\.nip\.io" has check "an HTTPS address: no Tailscale hint" "$out" "Tailscale" lacks -unset -f hostname curl +unset -f hostname curl sleep # quietly: the output of what it runs only when that fails. -out=$(quietly "Doing it…" bash -c 'echo noisy; exit 0' 2>&1) -check "quietly: progress" "$out" "^Doing it…$" has +out=$(quietly "Downloading it…" bash -c 'echo noisy; exit 0' 2>&1) +check "quietly: progress" "$out" "^→ Downloading it…$" has +check "quietly: done, as done" "$out" "^āœ“ Downloaded it$" has check "quietly: no output when it works" "$out" "noisy" lacks # Off a terminal, two steps with the same line show it once. out=$( (quietly "Fetching…" true; quietly "Fetching…" true) 2>&1) -check "quietly: a repeated line once" "$(grep -c '^Fetching…$' <<<"$out")" "^1$" has +check "quietly: a repeated line once" "$(grep -c '^→ Fetching…$' <<<"$out")" "^1$" has out=$(quietly "Doing it…" bash -c 'echo what went wrong; exit 3' 2>&1) && rc=0 || rc=$? check "quietly: the output when it fails" "$out" "what went wrong" has +check "quietly: the step failed" "$out" "^āœ— Doing it$" has check "quietly: and its exit code" "$rc" "^3$" has # A failure ends the progress line first. out=$( (progress_start "Working…"; fail "Broke.") 2>&1 || true) check "fail after progress" "$out" "Error: Broke\." has +check "fail: the step failed first" "$out" $'^āœ— Working\nError: Broke\.$' has + +# Off a terminal: no colors, nothing drawn over. +out=$( (say "Installing Pyo 9.9.9"; ok_line "Pyo 9.9.9 is installed"; dot_line green "Running"; quietly "Starting k3s…" true) 2>&1) +check "no terminal: no escapes" "$out" $'\033|\r' lacks +check "no terminal: a heading" "$out" "^→ Installing Pyo 9\.9\.9$" has +check "no terminal: the last line" "$out" "^āœ“ Pyo 9\.9\.9 is installed$" has +check "no terminal: a status" "$out" "^ā— Running$" has +check "no terminal: a step done" "$out" "^āœ“ Started k3s$" has +# A step's time, once it took a while. +sleep() { SECONDS=$((SECONDS + ${1%.*})); } +out=$( (progress_start "Downloading k3s…"; sleep 12; progress_end) 2>&1) +check "a long step: its time" "$out" "^āœ“ Downloaded k3s 0:12$" has +out=$( (progress_start "Starting Pyo 9.9.9 (a few minutes the first time)…"; progress_end ok "Started Pyo 9.9.9") 2>&1) +check "a step's own words when done" "$out" "^āœ“ Started Pyo 9\.9\.9$" has unset -f sleep +# On a terminal (script(1) gives one): the step ticks on its line, then +# stays as a checklist, in color. NO_COLOR: the same, without colors. +if command -v script >/dev/null; then + blk=$(mktemp) + sed -n '/^# >>> pyo:progress/,/^# <<< pyo:progress/p' deploy/install.sh >"$blk" + tty_run() { script -qec "bash -c '. $blk; $1'" /dev/null 2>/dev/null | tr -d '\r' || true; } + out=$(tty_run 'say Installing; quietly "Downloading k3s…" sleep 1; quietly "Checking it…" false; ok_line Done; dot_line yellow Starting') + check "terminal: a bold heading" "$out" $'\033\\[1m→ Installing\033\\[0m' has + check "terminal: a ticking step in cyan" "$out" $'\033\\[36m→\033\\[0m Downloading k3s…' has + check "terminal: a green check" "$out" $'\033\\[32māœ“\033\\[0m Downloaded k3s' has + check "terminal: a red cross" "$out" $'\033\\[31māœ— Checking it\033\\[0m' has + check "terminal: the last line in green" "$out" $'\033\\[1;32māœ“ Done\033\\[0m' has + check "terminal: a yellow dot" "$out" $'\033\\[33mā—\033\\[0m Starting' has + out=$(NO_COLOR=1 tty_run 'say Installing; quietly "Downloading k3s…" true') + check "NO_COLOR: no colors on a terminal" "$out" $'\033\\[[0-9;]*m' lacks + check "NO_COLOR: the checklist still" "$out" "āœ“ Downloaded k3s" has + rm -f "$blk" +fi [ "$failed" = 0 ] && echo "install.sh: all checks passed" exit "$failed" diff --git a/scripts/pyo-up.sh b/scripts/pyo-up.sh index f85cdb3..9229d6a 100755 --- a/scripts/pyo-up.sh +++ b/scripts/pyo-up.sh @@ -38,12 +38,12 @@ IMAGES=127.0.0.1:5005/pyo RELEASE_IMAGES=ghcr.io/upsetdev/pyo RELEASE_SIGNER='^https://github\.com/upsetdev/pyo/\.github/workflows/release\.yml@refs/tags/v' -TTY="" BOLD="" RED="" RESET="" +TTY="" BOLD="" RED="" GREEN="" CYAN="" DIM="" RESET="" if [ -t 1 ] && [ "${TERM:-dumb}" != dumb ]; then TTY=1 - [ -n "${NO_COLOR:-}" ] || BOLD=$'\033[1m' RED=$'\033[31m' RESET=$'\033[0m' + [ -n "${NO_COLOR:-}" ] || BOLD=$'\033[1m' RED=$'\033[31m' GREEN=$'\033[32m' CYAN=$'\033[36m' DIM=$'\033[2m' RESET=$'\033[0m' fi -log() { printf '%s==> %s%s\n' "$BOLD" "$*" "$RESET"; } +log() { printf '%s→ %s%s\n' "$BOLD" "$*" "$RESET"; } # The sandbox templates' images: published (the default) or built here. SANDBOX_SRC="" @@ -159,31 +159,37 @@ trap 'exit 130' INT TERM fmt_time() { printf '%d:%02d' "$(($1 / 60))" "$(($1 % 60))"; } +# A step as the install script shows one (pyo:progress): "→ label 0:12" +# while it runs, then "āœ“ label 0:12" or "āœ— label". SPIN_LABEL="" SPIN_T0=0 SPIN_N=0 SPIN_NEXT=0 spin_start() { SPIN_LABEL=$1 SPIN_T0=$SECONDS SPIN_N=0 SPIN_NEXT=$((SECONDS + 15)) SPINNING=1 - if [ -n "$TTY" ]; then printf '\033[?25l'; else printf ' %s ...\n' "$SPIN_LABEL"; fi + if [ -n "$TTY" ]; then printf '\033[?25l'; else printf '→ %s\n' "$SPIN_LABEL"; fi } spin_tick() { - local el=$((SECONDS - SPIN_T0)) line cols chars='|/-'"\\" + local el=$((SECONDS - SPIN_T0)) line cols if [ -n "$TTY" ]; then cols=$(tput cols 2>/dev/null) || cols=80 [[ "$cols" =~ ^[0-9]+$ ]] && [ "$cols" -ge 20 ] || cols=80 - line="${chars:$((SPIN_N % 4)):1} $SPIN_LABEL $(fmt_time "$el")" - printf '\r\033[2K%s' "${line:0:$((cols - 1))}" + line="$SPIN_LABEL $(fmt_time "$el")" + printf '\r\033[2K%s→%s %s' "$CYAN" "$RESET" "${line:0:$((cols - 3))}" elif [ "$SECONDS" -ge "$SPIN_NEXT" ]; then - printf ' %s %s\n' "$SPIN_LABEL" "$(fmt_time "$el")" + printf '→ %s %s\n' "$SPIN_LABEL" "$(fmt_time "$el")" SPIN_NEXT=$((SECONDS + 15)) fi SPIN_N=$((SPIN_N + 1)) } # Clears the spinner's line, to print something else. spin_clear() { if [ -n "$TTY" ]; then printf '\r\033[2K'; fi; } -spin_stop() { # what to say, e.g. "pyo ready" +spin_stop() { # ok|fail, what to say, e.g. "pyo ready" spin_clear if [ -n "$TTY" ]; then printf '\033[?25h'; fi SPINNING="" - printf ' %s (%s)\n' "$1" "$(fmt_time "$((SECONDS - SPIN_T0))")" + if [ "$1" = ok ]; then + printf '%sāœ“%s %s %s%s%s\n' "$GREEN" "$RESET" "$2" "$DIM" "$(fmt_time "$((SECONDS - SPIN_T0))")" "$RESET" + else + printf '%sāœ— %s%s\n' "$RED" "$2" "$RESET" + fi } # Ticks until process $1 ends, calling function $2 (if any) every 3 s; # returns the process's exit status. @@ -212,7 +218,7 @@ run_step() { # label command... spin_start "$label" "$@" >"$STEP_OUT" 2>&1 & spin_while "$!" || rc=$? - if [ "$rc" -eq 0 ]; then spin_stop "$label: done"; else spin_stop "$label: failed"; fi + if [ "$rc" -eq 0 ]; then spin_stop ok "$label"; else spin_stop fail "$label"; fi return "$rc" } @@ -283,10 +289,10 @@ wait_rollout() { # kind/name timeout kubectl -n "$NS" rollout status "$1" --timeout="$2" >"$out" 2>&1 & spin_while "$!" rollout_hook || rc=$? if [ "$rc" -eq 0 ]; then - spin_stop "${1#*/} ready" + spin_stop ok "${1#*/} ready" return 0 fi - spin_stop "${RED}${1#*/} isn't ready${RESET} after $2" + spin_stop fail "${1#*/} isn't ready after $2" sed 's/^/ | /' "$out" >&2 [ -z "$WAIT_SELECTOR" ] || kubectl -n "$NS" get pods -l "$WAIT_SELECTOR" -o wide 2>&1 | sed 's/^/ | /' >&2 || true return "$rc" @@ -360,11 +366,33 @@ fi # joins the tailnet by itself and Pyo is also https://pyo..ts.net. # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) -# A long wait shows it's still going. On a terminal, one line ticks -# ("Starting Pyo… 0:42"). Elsewhere (piped, a CI log), each step's line -# comes once (not again for a step with the same line), then with its time -# every 20 seconds while it runs. progress_start MESSAGE, then -# progress_tick as often as it likes, then progress_end. +# Steps, as a checklist. On a terminal, the step that runs ticks on one +# line ("→ Downloading k3s… 0:12"), then stays: done ("āœ“ Downloaded k3s +# 0:12", the time once it took a while), failed ("āœ— Downloading k3s") or +# still going on its own ("ā— Starting the dashboard"), in color unless +# NO_COLOR is set. Elsewhere (piped, a CI log), no colors and no line +# drawn over: each step's line comes once (not again for a step with the +# same line), then with its time every 20 seconds while it runs, then how +# it ended (once for steps alike). progress_start MESSAGE, then +# progress_tick as often as it likes, then progress_end [ok|fail|wait] +# [WHAT IT DID]. And: say TEXT, a heading. ok_line TEXT, the last line +# once all went well. dot_line green|yellow|red|dim TEXT, how something +# is (running, starting, failed, stopped). +progress_color() { [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; } +# progress_paint SGR TEXT: TEXT in that color, when there are colors (it +# prints itself: in $(...) there's never a terminal). +progress_paint() { + if progress_color; then printf '\033[%sm%s\033[0m' "$1" "$2"; else printf '%s' "$2"; fi +} +progress_time() { printf '%d:%02d' $(($1 / 60)) $(($1 % 60)); } +say() { progress_paint 1 "→ $*"; echo; } +ok_line() { progress_paint '1;32' "āœ“ $*"; echo; } +dot_line() { + local c=2 + case $1 in green) c=32 ;; yellow) c=33 ;; red) c=31 ;; esac + progress_paint "$c" ā— + printf ' %s\n' "$2" +} progress_start() { PROGRESS_MSG=$1 PROGRESS_T0=$SECONDS PROGRESS_NEXT=0 progress_tick @@ -373,33 +401,82 @@ progress_tick() { [ -n "${PROGRESS_MSG:-}" ] || return 0 local t=$((SECONDS - PROGRESS_T0)) if [ -t 1 ]; then - printf '\r\033[K%s %d:%02d' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '\r\033[K' + progress_paint 36 → + printf ' %s ' "$PROGRESS_MSG" + progress_paint 2 "$(progress_time "$t")" elif [ "$t" -ge "$PROGRESS_NEXT" ]; then if [ "$PROGRESS_NEXT" -gt 0 ]; then - printf '%s %d:%02d\n' "$PROGRESS_MSG" $((t / 60)) $((t % 60)) + printf '→ %s %s\n' "$PROGRESS_MSG" "$(progress_time "$t")" elif [ "$PROGRESS_MSG" != "${PROGRESS_LAST:-}" ]; then - printf '%s\n' "$PROGRESS_MSG" + printf '→ %s\n' "$PROGRESS_MSG" fi PROGRESS_LAST=$PROGRESS_MSG PROGRESS_NEXT=$((t + 20)) fi return 0 } +# What a step did, from what it was doing: "Downloading k3s…" is +# "Downloaded k3s". +progress_done() { + local m=${1%…} w + w=${m%% *} + case $w in + Applying) w=Applied ;; Building) w=Built ;; Checking) w=Checked ;; + Connecting) w=Connected ;; Copying) w=Copied ;; Creating) w=Created ;; + Deleting) w=Deleted ;; Downloading) w=Downloaded ;; Getting) w=Got ;; + Installing) w=Installed ;; Joining) w=Joined ;; Moving) w=Moved ;; + Removing) w=Removed ;; Restarting) w=Restarted ;; Setting) w=Set ;; + Starting) w=Started ;; Stopping) w=Stopped ;; Updating) w=Updated ;; + Waiting) w=Waited ;; + esac + printf '%s%s' "$w" "${m#"${m%% *}"}" +} progress_end() { [ -n "${PROGRESS_MSG:-}" ] || return 0 + local t=$((SECONDS - PROGRESS_T0)) done="" [ ! -t 1 ] || printf '\r\033[K' + [ "${1:-ok}" != ok ] || done=${2:-$(progress_done "$PROGRESS_MSG")} + # The same step again (a file each): said once. + if [ -n "$done" ] && [ "$done" = "${PROGRESS_DONE:-}" ]; then + PROGRESS_MSG="" + return 0 + fi + PROGRESS_DONE=$done + case ${1:-ok} in + ok) + progress_paint 32 āœ“ + printf ' %s' "$done" + ;; + fail) + progress_paint 31 "āœ— ${PROGRESS_MSG%…}" + t=0 + ;; + *) + progress_paint 33 ā— + printf ' %s' "${2:-${PROGRESS_MSG%…}}" + ;; + esac + if [ "$t" -ge 5 ]; then + printf ' ' + progress_paint 2 "$(progress_time "$t")" + fi + echo PROGRESS_MSG="" } -# wait_until MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds until it -# succeeds, or a failure once SECONDS have passed. Nothing to show when it +# wait_until [-f] MESSAGE SECONDS COMMAND...: COMMAND every 2 seconds +# until it succeeds, or a failure once SECONDS have passed: the step goes +# on by itself ("ā—"), or with -f it failed ("āœ—"). Nothing to show when it # succeeds at once. wait_until() { + local end=wait + [ "$1" != -f ] || { end=fail; shift; } local msg=$1 limit=$2 t0=$SECONDS shift 2 "$@" && return 0 progress_start "$msg" until "$@"; do if [ $((SECONDS - t0)) -ge "$limit" ]; then - progress_end + progress_end "$end" return 1 fi sleep 1 @@ -423,7 +500,7 @@ quietly() { progress_tick done wait "$pid" || rc=$? - progress_end + if [ "$rc" = 0 ]; then progress_end; else progress_end fail; fi [ "$rc" = 0 ] || cat "$log" >&2 rm -f "$log" return "$rc" @@ -913,7 +990,7 @@ run_step "Connecting this machine to Pyo" "${SUDO[@]}" /usr/local/lib/pyo/connec sed 's/^/ | /' "$STEP_OUT" echo -echo "Pyo $TAG is up." +ok_line "Pyo $TAG is up" pyo_addresses 120 if [ -n "$ADMIN_PASSWORD" ]; then echo diff --git a/scripts/sandbox.test.sh b/scripts/sandbox.test.sh index feffeb5..3d5f9dc 100644 --- a/scripts/sandbox.test.sh +++ b/scripts/sandbox.test.sh @@ -98,12 +98,12 @@ curl() { echo "curl: (6) Could not resolve host" >&2; return 6; } WORK=$tmp/work2b BUNDLE="" DEV="" BUNDLE_SHA256="" mkdir -p "$WORK" out=$( (fetch_bundle) 2>&1) -check "the release's checksums come first" "$out" "^Downloading the release's checksums…$" has +check "the release's checksums come first" "$out" "^→ Downloading the release's checksums…$" has check "a failed download says to run it again" "$out" "Couldn't download release.json of Pyo 0.4.0.*run the same command again\.$" has WORK=$tmp/work2b2 BUNDLE_SHA256=$(printf '%064d' 0) mkdir -p "$WORK" out=$( (fetch_bundle) 2>&1) -check "the download shows progress" "$out" "^Downloading the sandbox files…$" has +check "the download shows progress" "$out" "^→ Downloading the sandbox files…$" has check "and its failure says to run it again" "$out" "Couldn't download the sandbox files from https://github.com/.*run the same command again\.$" has # The release: release.json lists the files' checksums, and its signature # is checked with cosign first. @@ -129,7 +129,7 @@ cosign() { echo "$*" >"$tmp/cosign-args"; [ -z "${cosign_fails:-}" ] || { echo " WORK=$tmp/work2c BUNDLE_SHA256="" fake_sum=$good mkdir -p "$WORK" out=$( (fetch_bundle && ls "$WORK/bundle") 2>&1) -check "the release's signature is checked" "$out" "^Checking the release's signature…$" has +check "the release's signature is checked" "$out" "^→ Checking the release's signature…$" has check "with cosign, as install.sh does" "$(cat "$tmp/cosign-args")" "verify-blob --bundle .*release.json.sigstore.json --certificate-identity-regexp \^https://github\\\\.com/upsetdev/pyo/.*--certificate-oidc-issuer https://token.actions.githubusercontent.com .*release.json$" has check "a release's files that match are used" "$out" "vmlinux" has WORK=$tmp/work2d fake_sum=$(printf '%064d' 2) @@ -217,8 +217,8 @@ fake_detect '{ ] }' out=$( (detect) 2>&1) -check "Firecracker on KVM is reported" "$out" "Firecracker on KVM. A test VM booted in 73 ms." has -check "the detection shows progress" "$out" "^Checking what this server can run…$" has +check "Firecracker on KVM is reported" "$out" "^ā— Firecracker on KVM. A test VM booted in 73 ms." has +check "the detection shows progress" "$out" "^→ Checking what this server can run…$" has check "and no raw JSON" "$out" '"runtime"|[{}]' lacks check "the detection gets this server's jail user" "$(detect_config)" '"jailUid": 998, "jailGid": 997' has fake_detect '{ @@ -262,7 +262,7 @@ tailscale() { } rm -f "$tmp/ts-up" out=$( (join_tailnet && echo "ip=$TS_IP pyo=$PYO") 2>&1) -check "joining the tailnet shows progress" "$out" "^Joining your tailnet…$" has +check "joining the tailnet shows progress" "$out" "^→ Joining your tailnet…$" has check "and hides Tailscale's own output" "$out" "advertising" lacks check "and gets this server's address" "$out" "^ip=100.64.0.9 pyo=100.64.0.1:5006$" has rm -f "$tmp/ts-up" @@ -303,14 +303,16 @@ out=$( (redeem) 2>&1) check "another server's config is refused" "$out" "isn't this server's config" has fake_body='{"serverId":"sbs_abc123"}' fake_code=200 out=$( (redeem) 2>&1) -check "joining Pyo shows progress" "$out" "^Joining Pyo as sbx-1…$" has +check "joining Pyo shows progress" "$out" "^→ Joining Pyo as sbx-1…$" has +check "then that it joined" "$out" "^āœ“ Joined Pyo as sbx-1$" has # Pyo out of reach: the wait ends after 10 minutes, without curl's words. sleep() { SECONDS=$((SECONDS + ${1%.*})); } fake_body='curl: (7) Failed to connect' fake_code=000 out=$( (redeem) 2>&1) check "an unreachable Pyo says what to do" "$out" "didn't answer over your tailnet in 10 minutes. Check that Pyo is running, then run the same command again\.$" has check "and not curl's error" "$out" "curl:" lacks -check "the wait shows every 20 seconds in a log" "$out" "^Joining Pyo as sbx-1… 0:20$" has +check "the wait shows every 20 seconds in a log" "$out" "^→ Joining Pyo as sbx-1… 0:20$" has +check "then that it failed" "$out" $'^āœ— Joining Pyo as sbx-1\nError: Pyo didn\'t answer' has unset -f curl sleep # ── Starting the daemon ── @@ -326,7 +328,8 @@ check "a daemon that's up at once: nothing but done" "$out" "^started$" has check "and no log" "$out" "bad config" lacks rm -f "$tmp/up" out=$( (start_daemon && echo started) 2>&1) -check "a daemon that won't start shows progress" "$out" "^Starting the sandbox daemon…$" has +check "a daemon that won't start shows progress" "$out" "^→ Starting the sandbox daemon…$" has +check "and that it failed" "$out" "^āœ— Starting the sandbox daemon$" has check "and its log" "$out" "pyo-sandbox: bad config" has check "and says to run it again" "$out" "didn't start. Its last log lines are above. Run the same command again\.$" has check "and doesn't go on" "$out" "started$" lacks From 6be414809c87daee6a3660aae3eade28a9361bc6 Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Wed, 18 Nov 2026 00:00:30 +0700 Subject: [PATCH 05/22] feat(install): list pyo's addresses by kind and wait less MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installer only knew a public IP on the network card, so a cloud VM behind 1:1 NAT showed no HTTPS address, and it picked the first private address it saw, docker's or k3s's included. host_units now writes addresses.sh, which the installer, pyo-up and pyo-reset-login share. It prints one aligned line each, in color on a terminal: āžœ Public: https://pyo..nip.io āžœ Tailnet: http://:8000 āžœ Network: http://:8000 āžœ Local: http://localhost:8000 - Public: the IP on the card, else the one the cloud's metadata names. The installer reads cert-manager's certificate and challenges, stops waiting as soon as Let's Encrypt issued or refused, and waits 90 seconds at most. Refused, the line says which ports to allow. Not there yet, that it comes by itself. Never plain HTTP on that IP - Tailnet: Pyo's tailnet name, else this machine's Tailscale IP when Pyo answers on it - Network: the default route's card, never a docker, k3s, libvirt, multipass or LXD bridge - run again, the installer tries a refused certificate again (15 minutes after the last try at the soonest) --- crates/pyo-api/src/join.rs | 12 +- deploy/install.sh | 323 ++++++++++++++++++++++++++++++++----- scripts/install.test.sh | 323 ++++++++++++++++++++++++++++++++++--- scripts/pyo-up.sh | 316 +++++++++++++++++++++++++++++++----- 4 files changed, 860 insertions(+), 114 deletions(-) diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index 7d16617..bd5b08d 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -428,6 +428,7 @@ mod tests { assert!(s.starts_with("#!/usr/bin/env bash\nset -euo pipefail\nhost_units() {\n")); assert!(s.ends_with("\n}\nhost_units\n")); assert!(s.contains("/usr/local/bin/pyo-reset-login")); + assert!(s.contains("tee /usr/local/lib/pyo/addresses.sh")); assert!(s.contains("http://127.0.0.1:8000/install/host-units.sh")); assert!(s.contains("systemctl enable --now pyo-connect.timer pyo-tailnet.timer")); } @@ -706,7 +707,8 @@ mod tests { } /// install.sh and scripts/pyo-up.sh set up the machine Pyo runs on the - /// same way: its timers, bootstrap key and where Pyo is. + /// same way: its timers, bootstrap key and where Pyo is (host_units + /// writes addresses.sh). #[test] fn install_and_pyo_up_share_the_host_setup() { let function = |file: &str, script: &str, name: &str| -> String { @@ -720,13 +722,7 @@ mod tests { }; let install = include_str!("../../../deploy/install.sh"); let pyo_up = include_str!("../../../scripts/pyo-up.sh"); - for name in [ - "host_units", - "bootstrap_key", - "nic_public_ip", - "pyo_https_url", - "pyo_addresses", - ] { + for name in ["host_units", "bootstrap_key", "pyo_addresses"] { assert_eq!( function("deploy/install.sh", install, name), function("scripts/pyo-up.sh", pyo_up, name), diff --git a/deploy/install.sh b/deploy/install.sh index a72d816..5c9baed 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -806,45 +806,41 @@ check_update() { # ── Where Pyo is, and this machine's helpers ─────────────────────────────── # One server needs no Tailscale: Pyo is on this machine's public IP over # HTTPS (https://pyo..nip.io, Let's Encrypt) until an admin -# closes it, and over plain HTTP on :8000 from this machine and its private -# network. Once Tailscale is set up in Pyo (Integrations), this machine -# joins the tailnet by itself and Pyo is also https://pyo..ts.net. - -# A public IPv4 address on this machine's network cards: not private, -# tailnet (CGNAT), loopback, link-local or multicast. Pyo puts its HTTPS -# address on the public IP only when the machine has one itself (the -# agent's locate.rs, freedomain.rs `direct_public_ip`). -nic_public_ip() { - hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | - grep -vE '^(0|10|127|2(2[4-9]|[3-5][0-9]))\.|^169\.254\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\.|^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.' | - head -n1 || true -} - -# Pyo's HTTPS address, into PYO_HTTPS_URL, once it has one. -pyo_https_url() { - PYO_HTTPS_URL=$(curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | sed -n 's/.*"publicUrl":"\([^"]*\)".*/\1/p' || true) - [ -n "$PYO_HTTPS_URL" ] -} - -# Pyo's addresses. Waits up to $1 seconds for its HTTPS one, when this -# machine has a public IP for it (else it would never come). +# closes it, and over plain HTTP on :8000 from this machine, its private +# network and its tailnet. Once Tailscale is set up in Pyo (Integrations), +# this machine joins the tailnet by itself and Pyo is also +# https://pyo..ts.net. Which addresses there are, and how they're +# shown, is in addresses.sh (host_units writes it). + +# Pyo's addresses (addresses.sh). Where this machine has a public IP of its +# own (on a network card, or the one its cloud maps to it), waits up to $1 +# seconds (90 at most) for the HTTPS address there: until Let's Encrypt +# issued its certificate, or couldn't reach this machine. $2: the last +# line of the steps before them ("Pyo 0.4.3 is installed"). pyo_addresses() { - local lan="" wait=${1:-0} - [ -n "$(nic_public_ip)" ] || wait=0 - PYO_HTTPS_URL="" - pyo_https_url || [ "$wait" = 0 ] || - wait_until "Waiting for the HTTPS address (up to $((wait / 60)) minutes)…" "$wait" pyo_https_url || true - lan=$(hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' | head -n1 || true) + local wait=${1:-0} + # shellcheck source=/dev/null + if ! declare -F addr_print >/dev/null && ! . "${PYO_ADDRESSES_SH:-/usr/local/lib/pyo/addresses.sh}" 2>/dev/null; then + [ -z "${2:-}" ] || ok_line "$2" + echo + echo "Open Pyo at http://localhost:8000" + return 0 + fi + [ "$wait" -le 90 ] || wait=90 + [ "$wait" = 0 ] || addr_wait_wanted || wait=0 + if [ "$wait" != 0 ]; then + ADDR_WAITED=1 + wait_until "Checking the HTTPS address…" "$wait" addr_public_settled || true + fi + [ -z "${2:-}" ] || ok_line "$2" echo # The dashboard didn't answer in time: it still comes up by itself. if [ -n "${STARTING:-}" ] && ! dashboard_up; then - echo "Pyo is still starting. It opens at these addresses in a minute or two." + dot_line yellow "Pyo is still starting. It opens at these addresses in a minute or two." fi echo "Open Pyo:" - [ -z "$PYO_HTTPS_URL" ] || echo " $PYO_HTTPS_URL" - [ -z "$lan" ] || echo " http://$lan:8000 (your network)" - echo " http://localhost:8000 (this machine)" - [ -n "$PYO_HTTPS_URL" ] || echo "For an HTTPS address, set up Tailscale in Pyo." + addr_print open + [ -n "$ADDR_HTTPS" ] || echo "For an HTTPS address, set up Tailscale in Pyo." } # The host's own helpers, as systemd timers (root), every minute: connecting # this machine as a cluster until it's done, and putting it on the tailnet @@ -971,6 +967,249 @@ sum=$(sha256sum <<<"$units" | cut -d' ' -f1) [ "$sum" != "$(cat /etc/pyo/host-units 2>/dev/null)" ] || exit 0 exec bash -c "$units"$'\n'"echo $sum > /etc/pyo/host-units" SH + "${sudo[@]}" tee /usr/local/lib/pyo/addresses.sh >/dev/null <<'ADDR' +# Where Pyo is, as the install script, scripts/pyo-up.sh and pyo-reset-login +# show it. Sourced: it only defines functions. +# +# Public https://pyo..nip.io, with Let's Encrypt's +# certificate. The public IP is this machine's own: on a network +# card, or the one its cloud maps to it (1:1 NAT), from the +# cloud's metadata. Never plain HTTP there: passwords would cross +# the internet in the clear. +# Tailnet Pyo's tailnet name, else this machine's Tailscale IP on :8000 +# when Pyo answers there. +# Network this machine's private address on :8000: its main network +# card's. Never a container, VM or k3s bridge's. +# Local http://localhost:8000 + +# Public IPv4 addresses among the lines on stdin: not private, tailnet +# (CGNAT), loopback, link-local, multicast or reserved. +addr_public_v4s() { + grep -E '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$' | + grep -vE '^(0|10|127|2(2[4-9]|[3-5][0-9]))\.|^169\.254\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\.|^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.' || true +} + +# This machine's IPv4 addresses on its own network cards, "card address" a +# line. Not loopback, tunnels (Tailscale's comes on its own), or the +# bridges of containers, VMs and k3s (docker, br-*, cni0, flannel, libvirt, +# multipass, LXD), nor k3s's pod and service ranges or docker's own. +addr_own_ipv4s() { + local out + if out=$(ip -o -4 addr show 2>/dev/null) && [ -n "$out" ]; then + awk '{ split($4, a, "/"); sub(/@.*/, "", $2); print $2, a[1] }' <<<"$out" + else + hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+(\.[0-9]+){3}$' | sed 's/^/- /' + fi | grep -vE '^(lo|docker[0-9]*|br-[^ ]*|veth[^ ]*|cni[0-9]*|flannel[^ ]*|cali[^ ]*|vxlan[^ ]*|kube-[^ ]*|cilium[^ ]*|virbr[^ ]*|mpqemubr[^ ]*|lxdbr[^ ]*|lxcbr[^ ]*|podman[^ ]*|tailscale[^ ]*|tun[^ ]*|wg[^ ]*|zt[^ ]*) ' | + grep -vE ' (127\.|169\.254\.|10\.4[23]\.|172\.17\.)' || true +} + +# A public IPv4 address on this machine's network cards (as Pyo's agent +# finds them: locate.rs). +addr_nic_public_ip() { + addr_own_ipv4s | cut -d' ' -f2 | addr_public_v4s | head -n1 +} + +# This machine on its private network: an address of the card the default +# route goes through, else the first private one. One address, or none. +addr_network_ip() { + local all dev + all=$(addr_own_ipv4s | grep -E ' (10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' || true) + dev=$(ip -4 route show default 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -n1 || true) + { [ -z "$dev" ] || grep "^$dev " <<<"$all" || true; echo "$all"; } | sed -n '1s/^[^ ]* //p' +} + +# One cloud's metadata service: the public IPv4 address it maps to this +# machine. Each request waits a second at most. +addr_cloud_ask() { + local m=169.254.169.254 t="" h=() + case $1 in + aws) # AWS (IMDSv2, else v1), and clouds that answer like it + t=$(curl -fsS -m 1 -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' "http://$m/latest/api/token" 2>/dev/null) || t="" + [ -z "$t" ] || h=(-H "X-aws-ec2-metadata-token: $t") + curl -fsS -m 1 "${h[@]}" "http://$m/latest/meta-data/public-ipv4" ;; + gcp) curl -fsS -m 1 -H 'Metadata-Flavor: Google' "http://$m/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" ;; + azure) curl -fsS -m 1 -H 'Metadata: true' "http://$m/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2021-02-01&format=text" ;; + alibaba) + t=$(curl -fsS -m 1 -X PUT -H 'X-aliyun-ecs-metadata-token-ttl-seconds: 60' http://100.100.100.200/latest/api/token 2>/dev/null) || t="" + [ -z "$t" ] || h=(-H "X-aliyun-ecs-metadata-token: $t") + curl -fsS -m 1 "${h[@]}" http://100.100.100.200/latest/meta-data/eipv4 || + curl -fsS -m 1 "${h[@]}" http://100.100.100.200/latest/meta-data/public-ipv4 ;; + tencent) curl -fsS -m 1 http://169.254.0.23/latest/meta-data/public-ipv4 ;; + scaleway) curl -fsS -m 1 http://169.254.42.42/conf | sed -n 's/^PUBLIC_IP_ADDRESS=//p' ;; + esac 2>/dev/null + echo +} + +# The public IPv4 address this machine's cloud maps to it, from the cloud's +# metadata: every cloud asked at once, so it takes a second or two. Empty +# elsewhere (a home network, a laptop, a VM). Never what the internet sees +# of this machine's own requests: that can be a router's or a shared +# gateway's address, which doesn't lead back here. +addr_cloud_ip() { + local d c pids=() + d=$(mktemp -d) + for c in aws gcp azure alibaba tencent scaleway; do + addr_cloud_ask "$c" >"$d/$c" & + pids+=($!) + done + wait "${pids[@]}" 2>/dev/null || true + cat "$d"/* 2>/dev/null | tr -d '\r ' | addr_public_v4s | head -n1 + rm -rf "$d" +} + +# Pyo's agent's word for it: the public IP this machine's cloud maps to it, +# as its locate.rs found it (a cloud whose metadata doesn't say, like +# Oracle's). +addr_agent_cloud_ip() { + addr_kc get node "$(hostname | tr 'A-Z' 'a-z')" -o jsonpath='{.metadata.annotations.pyo/cloud-public-ipv4}' 2>/dev/null | + addr_public_v4s | head -n1 +} + +# This machine's own public IPv4 address (looked up once): on a network +# card, else the one its cloud maps to it. Empty without one. +addr_public_ip() { + if [ -z "${ADDR_IP_KNOWN:-}" ]; then + ADDR_IP=$(addr_nic_public_ip) + [ -n "$ADDR_IP" ] || ADDR_IP=$(addr_cloud_ip) + [ -n "$ADDR_IP" ] || ADDR_IP=$(addr_agent_cloud_ip) + ADDR_IP_KNOWN=1 + fi + echo "$ADDR_IP" +} + +addr_kc() { + if [ "$(id -u)" = 0 ]; then k3s kubectl "$@"; else kubectl "$@"; fi +} + +# The certificate of Pyo's HTTPS address on the public IP, as cert-manager +# has it: ADDR_CERT_HOST ("pyo..nip.io", empty when Pyo asked +# for none) and ADDR_CERT_STATE: ready, failed (Let's Encrypt couldn't +# reach this machine on port 80), or pending. A challenge or order Let's +# Encrypt turned down counts as failed at once, before the certificate +# says so. +addr_public_cert() { + local host ready reason made ns=pyo-gateway + ADDR_CERT_HOST="" ADDR_CERT_STATE="" ADDR_CERT_FAILED_AT="" + read -r host ready reason made ADDR_CERT_FAILED_AT < <(addr_kc -n "$ns" get certificate pyo-tls---dashboard \ + -o jsonpath='{.spec.dnsNames[0]} {.status.conditions[?(@.type=="Ready")].status}- {.status.conditions[?(@.type=="Issuing")].reason}- {.metadata.creationTimestamp} {.status.lastFailureTime}' 2>/dev/null || true) || true + [ -n "$host" ] || return 0 + ADDR_CERT_HOST=$host ADDR_CERT_STATE=pending + if [ "$ready" = True- ]; then + ADDR_CERT_STATE=ready + elif [ "$reason" = Failed- ] || [ -n "$ADDR_CERT_FAILED_AT" ] || + addr_kc -n "$ns" get challenges.acme.cert-manager.io,orders.acme.cert-manager.io \ + -o jsonpath='{range .items[*]}{.metadata.creationTimestamp} {.spec.dnsName}{.spec.dnsNames[0]} {.status.state}{"\n"}{end}' 2>/dev/null | + awk -v host="$host" -v made="$made" '$1 >= made && $2 == host && ($3 == "invalid" || $3 == "errored") { found = 1 } END { exit !found }'; then + ADDR_CERT_STATE=failed + fi +} + +# A certificate Let's Encrypt refused 15 minutes ago or more is asked for +# again now (Pyo's agent makes it again at once): running the installer +# again after opening the ports checks them. Not sooner: Let's Encrypt +# allows 5 failed checks for a name an hour. +addr_retry_failed() { + local at + [ "${ADDR_CERT_STATE:-}" = failed ] && [ -n "${ADDR_CERT_FAILED_AT:-}" ] || return 1 + at=$(date -d "$ADDR_CERT_FAILED_AT" +%s 2>/dev/null) || return 1 + [ $(($(date +%s) - at)) -ge 900 ] || return 1 + addr_kc -n pyo-gateway delete certificate pyo-tls---dashboard --wait=false >/dev/null 2>&1 || return 1 + ADDR_CERT_STATE=pending ADDR_CERT_FAILED_AT="" +} + +# Done waiting for the HTTPS address: its certificate is issued, or Let's +# Encrypt couldn't reach this machine. +addr_public_settled() { + addr_public_cert + [ "$ADDR_CERT_STATE" = ready ] || [ "$ADDR_CERT_STATE" = failed ] +} + +# Worth waiting for the HTTPS address: this machine has a public IP of its +# own, and the certificate there isn't settled (or was refused a while ago: +# tried again). Without one asked for yet, not when Pyo is on the tailnet +# (an admin may have closed it). +addr_wait_wanted() { + ! addr_public_settled || addr_retry_failed || return 1 + [ -n "$(addr_public_ip)" ] || return 1 + [ -n "$ADDR_CERT_HOST" ] || [ -z "$(addr_pyo_tailnet)" ] +} + +# Pyo's HTTPS name on the tailnet ("https://pyo..ts.net"), once it +# has one. +addr_pyo_tailnet() { + curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | + sed -n 's/.*"publicUrl":"\(https:\/\/[^"]*\.ts\.net\)\/*".*/\1/p' | head -n1 || true +} + +# Pyo on the tailnet: its HTTPS name there, else this machine's Tailscale +# IP on :8000, when Tailscale is up here and Pyo answers on it. +addr_tailnet() { + local url ip + url=$(addr_pyo_tailnet) + if [ -n "$url" ]; then + echo "$url" + return 0 + fi + command -v tailscale >/dev/null 2>&1 || return 0 + ip=$(tailscale ip -4 2>/dev/null | head -n1 || true) + [ -n "$ip" ] || return 0 + curl -fsS -m 3 -o /dev/null "http://$ip:8000/api/v1/meta" 2>/dev/null && echo "http://$ip:8000" + return 0 +} + +# Colors on a terminal, unless NO_COLOR says not to. +addr_color() { [ -z "${NO_COLOR:-}" ] && [ -t 1 ]; } + +# addr_line LABEL URL [NOTE]: " āžœ Public: https://… (note)", the +# labels lined up. On a terminal: a green arrow, the label in bold, the +# address in cyan. +addr_line() { + local pad note="" + pad=$(printf '%*s' $((9 - ${#1})) '') + [ -z "${3:-}" ] || note=" ($3)" + if addr_color; then + printf ' \033[32māžœ\033[0m \033[1m%s:\033[0m%s\033[36m%s\033[0m%s\n' "$1" "$pad" "$2" "$note" + else + printf ' āžœ %s:%s%s%s\n' "$1" "$pad" "$2" "$note" + fi +} + +# addr_print open|PATH: Pyo's addresses, one line each. "open": for opening +# Pyo, with the HTTPS address on the public IP also while it's on its way +# or blocked, and what to do then. A PATH: links to it that work now. +# ADDR_HTTPS says whether there's a Public or Tailnet line. +addr_print() { + local path="" host="" ip url net note="" + [ "${1:-open}" = open ] || path=$1 + ADDR_HTTPS="" + [ -n "${ADDR_CERT_STATE:-}" ] || addr_public_cert + host=$ADDR_CERT_HOST + # None asked for yet, after a wait for it: it's on its way. + if [ -z "$host" ] && [ -n "${ADDR_WAITED:-}" ]; then + ip=$(addr_public_ip) + [ -z "$ip" ] || host="pyo.$ip.nip.io" + fi + if [ -n "$host" ]; then + case "${ADDR_CERT_STATE:-}" in + ready) ;; + failed) note="allow ports 80 and 443 in your cloud firewall" ;; + *) note="ready in a few minutes" ;; + esac + if [ -z "$note" ] || [ -z "$path" ]; then + addr_line Public "https://$host$path" "$note" + ADDR_HTTPS=1 + fi + fi + url=$(addr_tailnet) + if [ -n "$url" ]; then + addr_line Tailnet "$url$path" + ADDR_HTTPS=1 + fi + net=$(addr_network_ip) + [ -z "$net" ] || addr_line Network "http://$net:8000$path" + addr_line Local "http://localhost:8000$path" +} +ADDR "${sudo[@]}" tee /usr/local/lib/pyo/pyo-reset-login >/dev/null <<'SH' #!/usr/bin/env bash # Prints a one-time link to choose a new password for a Pyo account, for a @@ -1005,15 +1244,14 @@ if [ "$status" != 200 ]; then fi path=$(field path) email=$(field email) minutes=$(sed -n 's/.*"minutes":\([0-9]*\).*/\1/p' <<<"$body") -https=$(curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | sed -n 's/.*"publicUrl":"\([^"]*\)".*/\1/p' || true) -lan=$(hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' | head -n1 || true) +# shellcheck source=/dev/null +. /usr/local/lib/pyo/addresses.sh echo "Open one of these links to choose a new password for $email:" -[ -z "$https" ] || echo " ${https%/}$path" -[ -z "$lan" ] || echo " http://$lan:8000$path (your network)" -echo " http://localhost:8000$path (this machine)" +addr_print "$path" echo "It works once, for ${minutes:-30} minutes." SH "${sudo[@]}" chmod 755 /usr/local/lib/pyo/connect-this-machine.sh /usr/local/lib/pyo/tailnet-this-machine.sh /usr/local/lib/pyo/pyo-reset-login + "${sudo[@]}" chmod 644 /usr/local/lib/pyo/addresses.sh "${sudo[@]}" ln -sfn /usr/local/lib/pyo/pyo-reset-login /usr/local/bin/pyo-reset-login "${sudo[@]}" tee /etc/systemd/system/pyo-connect.service >/dev/null <<'UNIT' [Unit] @@ -1652,8 +1890,7 @@ main() { # Pyo's scripts on this machine, like pyo-reset-login, as this # release has them. host_units - ok_line "Pyo $R_VERSION is already installed. Nothing to update" - pyo_addresses + pyo_addresses 0 "Pyo $R_VERSION is already installed. Nothing to update" print_sign_in return 0 fi @@ -1679,7 +1916,11 @@ main() { # public IP, which the addresses wait for. quietly "Connecting this machine to Pyo…" /usr/local/lib/pyo/connect-this-machine.sh || echo "Couldn't connect this machine to Pyo yet. Pyo tries again every minute." - pyo_addresses 300 + if [ -n "$fresh" ]; then + pyo_addresses 300 "Pyo $R_VERSION is installed" + else + pyo_addresses 300 "Pyo is updated to $R_VERSION" + fi print_sign_in } diff --git a/scripts/install.test.sh b/scripts/install.test.sh index f93d02d..4b0ef5c 100755 --- a/scripts/install.test.sh +++ b/scripts/install.test.sh @@ -172,35 +172,308 @@ rm -rf "$WORK" check "--version asks GitHub for that release" "$(cat "$asked")" "^https://api.github.com/repos/upsetdev/pyo/releases/tags/v1.2.3" has check "once, when it isn't there" "$(wc -l <"$asked")" "^1$" has check "and says so" "$out" "There's no release 1.2.3" has -# The addresses at the end: no wait for an HTTPS address where the machine -# has no public IP (behind NAT, it never comes), and progress while waiting. -# Sleeping only moves the clock. +# ── The addresses at the end (addresses.sh, as host_units writes it) ── +# The machine is stubbed: its cards (ip), its cloud's metadata and Pyo +# (curl), Tailscale, and cert-manager's view of the HTTPS address's +# certificate (k3s kubectl). Sleeping only moves the clock. +lib=$(mktemp) +sed -n "/tee \/usr\/local\/lib\/pyo\/addresses.sh >\/dev\/null <<'ADDR'$/,/^ADDR$/p" deploy/install.sh | sed '1d;$d' >"$lib" +bash -n "$lib" +check "addresses.sh is in host_units" "$(wc -l <"$lib")" "^[0-9]{3}$" has +# shellcheck source=/dev/null +. "$lib" +PYO_ADDRESSES_SH=$lib sleep() { SECONDS=$((SECONDS + ${1%.*})); } -hostname() { echo "$ips"; } -curl() { echo x >>"$asked"; [ -z "$meta" ] || echo "$meta"; } -ips="10.0.2.15 10.42.0.0 172.17.0.1 100.101.67.2 fd7a:115c::1" -meta='{"version":"9.9.9"}' -: >"$asked" +id() { echo 0; } +hostname() { if [ "${1:-}" = -I ]; then echo "$ips"; else echo VM-1; fi; } +ip() { + case "$*" in + "-o -4 addr show") [ -n "$cards" ] && echo "$cards" ;; + "-4 route show default") [ -z "$route" ] || echo "default via 10.0.0.1 dev $route proto dhcp src 10.0.0.9 metric 100" ;; + esac +} +tailscale() { [ "$*" = "ip -4" ] && [ -n "$ts_ip" ] && echo "$ts_ip"; } +# cloud: which metadata service answers (with cloud_ip); meta: Pyo's +# /api/v1/meta; ts_answers: Pyo answers on the Tailscale IP. +curl() { + local url=${*: -1} + echo "$*" >>"$asked" + case "$url" in + http://127.0.0.1:8000/api/v1/meta) [ -n "$meta" ] && echo "$meta" ;; + http://100.*:8000/api/v1/meta) [ -n "$ts_answers" ] ;; + http://169.254.169.254/latest/api/token) [ "$cloud" = aws ] && printf tok ;; + http://169.254.169.254/latest/meta-data/public-ipv4) [ "$cloud" = aws ] && [[ $* == *"X-aws-ec2-metadata-token: tok"* ]] && printf '%s' "$cloud_ip" ;; + *computeMetadata*) [ "$cloud" = gcp ] && [[ $* == *"Metadata-Flavor: Google"* ]] && printf '%s' "$cloud_ip" ;; + *169.254.169.254/metadata/instance*) [ "$cloud" = azure ] && [[ $* == *"Metadata: true"* ]] && printf '%s' "$cloud_ip" ;; + http://100.100.100.200/latest/api/token) return 22 ;; + http://100.100.100.200/latest/meta-data/eipv4) [ "$cloud" = alibaba ] && printf '%s' "$cloud_ip" ;; + http://169.254.0.23/latest/meta-data/public-ipv4) [ "$cloud" = tencent ] && printf '%s' "$cloud_ip" ;; + http://169.254.42.42/conf) [ "$cloud" = scaleway ] && printf 'PRIVATE_IP=10.1.2.3\nPUBLIC_IP_ADDRESS=%s\n' "$cloud_ip" ;; + *) return 7 ;; + esac || return 22 +} +# cert: the certificate as the jsonpath reads it ("" none); the challenge +# is turned down from look number $fail_after on (counted in $polls, a +# file: the reads run in subshells). +k3s() { + shift + case "$*" in + *"get certificate pyo-tls---dashboard"*) + echo x >>"$polls" + [ -n "$cert" ] && echo "$cert" ;; + *challenges.acme*) + # An older try's, turned down: not this certificate's. + [ -z "$cert" ] || echo "2026-11-17T08:00:00Z ${cert%% *} invalid" + [ -n "$cert" ] && [ "$(wc -l <"$polls")" -ge "${fail_after:-999}" ] && echo "2026-11-17T09:00:01Z ${cert%% *} invalid" + return 0 ;; + *"delete certificate pyo-tls---dashboard"*) + echo deleted >>"$asked" + cert="${cert%% *} False- DoesNotExist- 2026-11-17T09:30:00Z " ;; + *"get node vm-1"*) echo "$agent_ip" ;; + esac +} +asked=$(mktemp) polls=$(mktemp) +reset() { + unset ADDR_IP_KNOWN ADDR_IP ADDR_CERT_STATE ADDR_CERT_HOST ADDR_WAITED STARTING + cards="" route="" ips="" ts_ip="" ts_answers="" cloud="" cloud_ip="" meta='{"version":"9.9.9"}' cert="" agent_ip="" fail_after="" failed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + : >"$asked" + : >"$polls" +} +lo='1: lo inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever' +k3s_cards='5: cni0 inet 10.42.0.1/24 brd 10.42.0.255 scope global cni0\ valid_lft forever preferred_lft forever +6: flannel.1 inet 10.42.0.0/32 scope global flannel.1\ valid_lft forever preferred_lft forever' +ts_card='3: tailscale0 inet 100.101.67.2/32 scope global tailscale0\ valid_lft forever preferred_lft forever' + +# A Tencent Cloud server: a private address on its card, the public IP in +# front of it (1:1 NAT) from the metadata, on a tailnet already, the HTTPS +# address issued. The block as the user sees it, lined up. +reset +cards="$lo +2: eth0 inet 10.3.8.13/22 brd 10.3.11.255 scope global eth0\ valid_lft forever preferred_lft forever +$ts_card +$k3s_cards" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 ts_ip=100.101.67.2 ts_answers=1 +cert="pyo.43.156.1.2.nip.io True- Issued- 2026-11-17T09:00:00Z " out=$(pyo_addresses 300) -check "no public IP: one look" "$(wc -l <"$asked")" "^1$" has -check "no public IP: no wait" "$out" "Waiting" lacks -check "no public IP: the network address" "$out" "http://10\.0\.2\.15:8000 \(your network\)" has -check "no public IP: one line on HTTPS" "$out" "^For an HTTPS address, set up Tailscale in Pyo\.$" has -check "no jargon" "$out" "NAT|curl|api/v1|ports? [0-9]" lacks -ips="10.0.2.15 203.0.113.7" -: >"$asked" +want=$'\nOpen Pyo:\n āžœ Public: https://pyo.43.156.1.2.nip.io\n āžœ Tailnet: http://100.101.67.2:8000\n āžœ Network: http://10.3.8.13:8000\n āžœ Local: http://localhost:8000' +same() { if [ "$1" = "$2" ]; then echo same; else printf '%s\n--- wanted:\n%s\n' "$1" "$2"; fi; } +check "a cloud server behind NAT: the whole block" "$(same "$out" "$want")" "^same$" has +check "issued: no wait" "$out" "Waiting" lacks +check "never plain HTTP on the public IP" "$out" "http://43\.156" lacks +check "no k3s address" "$out" "10\.42\." lacks +check "no Tailscale hint with HTTPS" "$out" "set up Tailscale" lacks +check "no colors off a terminal" "$out" $'\033' lacks + +# Let's Encrypt couldn't reach it: the address still, with what to do. +reset +cards="$lo +2: eth0 inet 10.3.8.13/22 brd 10.3.11.255 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +cert="pyo.43.156.1.2.nip.io False- Failed- 2026-11-17T09:00:00Z $failed_at" out=$(pyo_addresses 300) -check "a public IP: waits five minutes" "$(wc -l <"$asked")" "^1[45][0-9]$" has -check "a public IP: says it waits, as it goes" "$out" "^→ Waiting for the HTTPS address \(up to 5 minutes\)…$" has -check "a public IP: every 20 seconds in a log" "$out" "^→ Waiting for the HTTPS address \(up to 5 minutes\)… 4:40$" has -check "a public IP: not more often" "$out" "… 0:1[0-9]$" lacks -meta='{"publicUrl":"https://pyo.203.0.113.7.nip.io"}' -: >"$asked" +check "blocked: the public address and the ports to open" "$out" "^ āžœ Public: https://pyo\.43\.156\.1\.2\.nip\.io \(allow ports 80 and 443 in your cloud firewall\)$" has +check "blocked: failed already, no wait" "$out" "Waiting" lacks +check "blocked: no Tailscale hint" "$out" "set up Tailscale" lacks +check "blocked: not on a tailnet, no Tailnet line" "$out" "Tailnet:" lacks + +# Run again after opening the ports: a try refused 15 minutes ago or more +# is made again now, and waited for. Within 15 minutes, not. +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +failed_at=$(date -u -d '-20 minutes' +%Y-%m-%dT%H:%M:%SZ) +cert="pyo.43.156.1.2.nip.io False- Failed- 2026-11-17T09:00:00Z $failed_at" +out=$(pyo_addresses 300) +check "refused a while ago: tried again" "$(cat "$asked")" "^deleted$" has +check "and waited for" "$out" "^→ Checking the HTTPS address…$" has +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +failed_at=$(date -u -d '-5 minutes' +%Y-%m-%dT%H:%M:%SZ) +cert="pyo.43.156.1.2.nip.io False- Failed- 2026-11-17T09:00:00Z $failed_at" +out=$(pyo_addresses 300) +check "refused just now: not again" "$(cat "$asked")" "^deleted$" lacks +check "and not waited for" "$out" "Checking" lacks +out=$(pyo_addresses) +check "no wait asked: never tried again" "$(cat "$asked")" "^deleted$" lacks + +# A fresh install: the certificate on its way, then Let's Encrypt turns its +# challenge down. The wait stops right then, well before 90 seconds. +reset +cards="$lo +2: ens5 inet 172.31.5.10/20 brd 172.31.15.255 scope global dynamic ens5 +4: docker0 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0" +route=ens5 cloud=aws cloud_ip=3.120.1.2 +cert="pyo.3.120.1.2.nip.io False- DoesNotExist- 2026-11-17T09:00:00Z " fail_after=8 +t0=$SECONDS +out=$(pyo_addresses 300; echo "took $((SECONDS - t0))") +check "AWS: the public IP from IMDSv2" "$out" "https://pyo\.3\.120\.1\.2\.nip\.io \(allow ports 80 and 443" has +check "turned down: the wait stops at once" "$(sed -n 's/^took //p' <<<"$out")" "^1[0-9]$" has +check "says it checks" "$out" "^→ Checking the HTTPS address…$" has +check "and that it did" "$out" "^āœ“ Checked the HTTPS address 0:1[0-9]$" has +check "AWS: the private address, not docker's" "$out" "^ āžœ Network: http://172\.31\.5\.10:8000$" has +check "no docker bridge" "$out" "172\.17\." lacks + +# The certificate isn't there yet after 90 seconds: it comes by itself. +reset +cards="$lo +2: ens4 inet 10.128.0.2/32 scope global dynamic ens4" +route=ens4 cloud=gcp cloud_ip=34.72.1.2 +t0=$SECONDS +out=$(pyo_addresses 300; echo "took $((SECONDS - t0))") +check "Google Cloud: the public IP from the metadata" "$out" "^ āžœ Public: https://pyo\.34\.72\.1\.2\.nip\.io \(ready in a few minutes\)$" has +check "90 seconds at most" "$(sed -n 's/^took //p' <<<"$out")" "^9[0-3]$" has +check "every 20 seconds in a log" "$out" "^→ Checking the HTTPS address… 1:20$" has +check "then it goes on by itself" "$out" "^ā— Checking the HTTPS address 1:3[0-3]$" has + +# Issued while it waits. +reset +cards="$lo +2: eth0 inet 10.0.0.4/24 scope global eth0" +route=eth0 cloud=azure cloud_ip=20.1.2.3 +cert="pyo.20.1.2.3.nip.io False- DoesNotExist- 2026-11-17T09:00:00Z " +k3s_was=$(declare -f k3s) +k3s() { shift; case "$*" in *"get certificate"*) echo x >>"$polls"; if [ "$(wc -l <"$polls")" -ge 4 ]; then echo "pyo.20.1.2.3.nip.io True- Issued- 2026-11-17T09:00:00Z "; else echo "$cert"; fi ;; esac; } out=$(pyo_addresses 300) -check "an HTTPS address: no wait" "$out" "Waiting" lacks -check "an HTTPS address: shown" "$out" "https://pyo\.203\.0\.113\.7\.nip\.io" has -check "an HTTPS address: no Tailscale hint" "$out" "Tailscale" lacks -unset -f hostname curl sleep +eval "$k3s_was" +check "Azure: issued while it waits" "$out" "^ āžœ Public: https://pyo\.20\.1\.2\.3\.nip\.io$" has + +# Alibaba Cloud and Scaleway answer too. Garbage or private answers don't count. +for c in alibaba:47.88.1.2 scaleway:51.15.1.2; do + reset + cloud=${c%%:*} cloud_ip=${c#*:} + check "$cloud: the public IP from the metadata" "$(addr_cloud_ip)" "^${cloud_ip//./\\.}$" has +done +for junk in "404" 999.1.2.3 10.3.8.13 100.101.67.2 ""; do + reset + cloud=tencent cloud_ip=$junk + check "metadata answering \"$junk\" is no public IP" "$(addr_cloud_ip)" "." lacks +done +reset +t0=$SECONDS +addr_cloud_ip >/dev/null +check "every cloud asked at once" "$(grep -c 169.254 "$asked")" "^[4-9]$" has +check "each for a second at most" "$(grep 'curl\|-m' "$asked" | grep -vc -- '-m 1 ')" "^0$" has + +# A Hetzner server: the public IP on its own card, and a private network. +reset +cards="$lo +2: eth0 inet 203.0.113.7/32 scope global dynamic eth0 +3: ens10 inet 10.0.0.2/32 brd 10.0.0.2 scope global dynamic ens10 +$k3s_cards" +route=eth0 +cert="pyo.203.0.113.7.nip.io True- Issued- 2026-11-17T09:00:00Z " +out=$(pyo_addresses 300) +check "a public card: its address, no metadata asked" "$(cat "$asked")" "169\.254|100\.100" lacks +check "a public card: HTTPS on it" "$out" "^ āžœ Public: https://pyo\.203\.0\.113\.7\.nip\.io$" has +check "a public card: the private network's address" "$out" "^ āžœ Network: http://10\.0\.0\.2:8000$" has +check "a public card: never plain HTTP on it" "$out" "http://203\." lacks + +# A laptop with docker, k3s, multipass, libvirt and LXD bridges, on a +# tailnet where Pyo doesn't answer on :8000: just its Wi-Fi address. +reset +cards="$lo +2: wlp0s20f3 inet 192.168.1.23/24 brd 192.168.1.255 scope global dynamic wlp0s20f3 +4: docker0 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0 +7: br-3f2a9c1e inet 172.18.0.1/16 brd 172.18.255.255 scope global br-3f2a9c1e +$k3s_cards +8: mpqemubr0 inet 10.97.0.1/24 brd 10.97.0.255 scope global mpqemubr0 +9: virbr0 inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 +10: lxdbr0 inet 10.10.10.1/24 scope global lxdbr0 +11: veth12ab@if10 inet 10.42.0.9/32 scope global veth12ab +$ts_card" +route=wlp0s20f3 ts_ip=100.101.67.2 +out=$(pyo_addresses 300) +want=$'\nOpen Pyo:\n āžœ Network: http://192.168.1.23:8000\n āžœ Local: http://localhost:8000\nFor an HTTPS address, set up Tailscale in Pyo.' +check "a laptop: one Network line, the hint" "$(same "$out" "$want")" "^same$" has +check "a laptop: no wait" "$out" "Waiting" lacks +check "a laptop: no public line" "$out" "Public" lacks +# Pyo answers on its Tailscale address. +ts_answers=1 +out=$(pyo_addresses 300) +check "a laptop on a tailnet: the Tailnet line" "$out" "^ āžœ Tailnet: http://100\.101\.67\.2:8000$" has +check "a laptop on a tailnet: no Tailscale hint" "$out" "set up Tailscale" lacks +# Pyo's own name on the tailnet comes first. +meta='{"version":"9.9.9","publicUrl":"https://pyo.tail1234.ts.net"}' +out=$(pyo_addresses 300) +check "Pyo's tailnet name" "$out" "^ āžœ Tailnet: https://pyo\.tail1234\.ts\.net$" has +check "only one Tailnet line" "$(grep -c Tailnet <<<"$out")" "^1$" has + +# No ip command: hostname -I, without k3s's or docker's addresses. +reset +ips="10.0.2.15 10.42.0.0 10.42.0.1 172.17.0.1 100.101.67.2 fd7a:115c::1" +out=$(pyo_addresses 300) +check "hostname -I: the network address" "$out" "^ āžœ Network: http://10\.0\.2\.15:8000$" has +check "hostname -I: one Network line" "$(grep -c Network <<<"$out")" "^1$" has + +# Oracle Cloud's metadata doesn't say: what Pyo's agent found does. +reset +cards="2: enp0s6 inet 10.0.0.37/24 scope global enp0s6" +route=enp0s6 agent_ip=129.146.1.2 +cert="pyo.129.146.1.2.nip.io True- Issued- 2026-11-17T09:00:00Z " +check "the agent's cloud IP" "$(addr_public_ip)" "^129\.146\.1\.2$" has + +# Pyo is on the tailnet and an admin closed the public address: no wait, +# no Public line. +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +meta='{"version":"9.9.9","publicUrl":"https://pyo.tail1234.ts.net"}' +out=$(pyo_addresses 300) +check "closed: no wait" "$out" "Waiting" lacks +check "closed: no Public line" "$out" "Public" lacks + +# Run again with nothing to update: no wait, the addresses as they are. +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +cert="pyo.43.156.1.2.nip.io False- DoesNotExist- 2026-11-17T09:00:00Z " +out=$(pyo_addresses) +check "no wait asked: none" "$out" "Waiting" lacks +check "on its way" "$out" "\(ready in a few minutes\)$" has + +# Still starting. +reset +STARTING=1 +dashboard_up() { return 1; } +out=$(pyo_addresses) +check "still starting: said first" "$(sed -n 2p <<<"$out")" "^ā— Pyo is still starting\. It opens at these addresses in a minute or two\.$" has +unset -f dashboard_up + +# Colors on a terminal: a green arrow, the label in bold, the address in +# cyan. NO_COLOR or no terminal: plain. +addr_color_was=$(declare -f addr_color) +addr_color() { [ -z "${NO_COLOR:-}" ]; } +out=$(addr_line Public https://pyo.43.156.1.2.nip.io "allow ports 80 and 443 in your cloud firewall") +check "colors" "$out" $'^ \033\\[32māžœ\033\\[0m \033\\[1mPublic:\033\\[0m \033\\[36mhttps://pyo\\.43\\.156\\.1\\.2\\.nip\\.io\033\\[0m \\(allow ports' has +out=$(NO_COLOR=1 addr_line Local http://localhost:8000) +check "NO_COLOR" "$out" "^ āžœ Local: http://localhost:8000$" has +eval "$addr_color_was" +out=$(addr_line Tailnet http://100.101.67.2:8000) +check "no terminal: plain" "$out" $'\033' lacks + +# pyo-reset-login's links: only addresses that work now. +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 cloud=tencent cloud_ip=43.156.1.2 +cert="pyo.43.156.1.2.nip.io False- Failed- 2026-11-17T09:00:00Z $failed_at" +out=$(addr_print /reset/abc) +check "links: not the blocked address" "$out" "Public" lacks +check "links: the network one" "$out" "^ āžœ Network: http://10\.3\.8\.13:8000/reset/abc$" has +cert="pyo.43.156.1.2.nip.io True- Issued- 2026-11-17T09:00:00Z " +unset ADDR_CERT_STATE +out=$(addr_print /reset/abc) +check "links: the issued one" "$out" "^ āžœ Public: https://pyo\.43\.156\.1\.2\.nip\.io/reset/abc$" has + +# The steps' last line, before the addresses. +reset +cards="2: eth0 inet 10.3.8.13/22 scope global eth0" +route=eth0 +out=$(pyo_addresses 0 "Pyo 9.9.9 is installed") +check "the last step line" "$(sed -n 1p <<<"$out")" "^āœ“ Pyo 9\.9\.9 is installed$" has +check "then the addresses" "$(sed -n 3p <<<"$out")" "^Open Pyo:$" has +unset -f curl hostname ip tailscale k3s id sleep +rm -f "$lib" "$asked" "$polls" # quietly: the output of what it runs only when that fails. out=$(quietly "Downloading it…" bash -c 'echo noisy; exit 0' 2>&1) diff --git a/scripts/pyo-up.sh b/scripts/pyo-up.sh index 9229d6a..2072f07 100755 --- a/scripts/pyo-up.sh +++ b/scripts/pyo-up.sh @@ -361,9 +361,11 @@ fi # ── Where Pyo is, and this machine's helpers ─────────────────────────────── # One server needs no Tailscale: Pyo is on this machine's public IP over # HTTPS (https://pyo..nip.io, Let's Encrypt) until an admin -# closes it, and over plain HTTP on :8000 from this machine and its private -# network. Once Tailscale is set up in Pyo (Integrations), this machine -# joins the tailnet by itself and Pyo is also https://pyo..ts.net. +# closes it, and over plain HTTP on :8000 from this machine, its private +# network and its tailnet. Once Tailscale is set up in Pyo (Integrations), +# this machine joins the tailnet by itself and Pyo is also +# https://pyo..ts.net. Which addresses there are, and how they're +# shown, is in addresses.sh (host_units writes it). # >>> pyo:progress (the same in deploy/install.sh, connect.sh, join.sh, scripts/pyo-up.sh and deploy/sandbox.sh) # Steps, as a checklist. On a terminal, the step that runs ticks on one @@ -506,41 +508,35 @@ quietly() { return "$rc" } # <<< pyo:progress -# A public IPv4 address on this machine's network cards: not private, -# tailnet (CGNAT), loopback, link-local or multicast. Pyo puts its HTTPS -# address on the public IP only when the machine has one itself (the -# agent's locate.rs, freedomain.rs `direct_public_ip`). -nic_public_ip() { - hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | - grep -vE '^(0|10|127|2(2[4-9]|[3-5][0-9]))\.|^169\.254\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\.|^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.' | - head -n1 || true -} - -# Pyo's HTTPS address, into PYO_HTTPS_URL, once it has one. -pyo_https_url() { - PYO_HTTPS_URL=$(curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | sed -n 's/.*"publicUrl":"\([^"]*\)".*/\1/p' || true) - [ -n "$PYO_HTTPS_URL" ] -} - -# Pyo's addresses. Waits up to $1 seconds for its HTTPS one, when this -# machine has a public IP for it (else it would never come). +# Pyo's addresses (addresses.sh). Where this machine has a public IP of its +# own (on a network card, or the one its cloud maps to it), waits up to $1 +# seconds (90 at most) for the HTTPS address there: until Let's Encrypt +# issued its certificate, or couldn't reach this machine. $2: the last +# line of the steps before them ("Pyo 0.4.3 is installed"). pyo_addresses() { - local lan="" wait=${1:-0} - [ -n "$(nic_public_ip)" ] || wait=0 - PYO_HTTPS_URL="" - pyo_https_url || [ "$wait" = 0 ] || - wait_until "Waiting for the HTTPS address (up to $((wait / 60)) minutes)…" "$wait" pyo_https_url || true - lan=$(hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' | head -n1 || true) + local wait=${1:-0} + # shellcheck source=/dev/null + if ! declare -F addr_print >/dev/null && ! . "${PYO_ADDRESSES_SH:-/usr/local/lib/pyo/addresses.sh}" 2>/dev/null; then + [ -z "${2:-}" ] || ok_line "$2" + echo + echo "Open Pyo at http://localhost:8000" + return 0 + fi + [ "$wait" -le 90 ] || wait=90 + [ "$wait" = 0 ] || addr_wait_wanted || wait=0 + if [ "$wait" != 0 ]; then + ADDR_WAITED=1 + wait_until "Checking the HTTPS address…" "$wait" addr_public_settled || true + fi + [ -z "${2:-}" ] || ok_line "$2" echo # The dashboard didn't answer in time: it still comes up by itself. if [ -n "${STARTING:-}" ] && ! dashboard_up; then - echo "Pyo is still starting. It opens at these addresses in a minute or two." + dot_line yellow "Pyo is still starting. It opens at these addresses in a minute or two." fi echo "Open Pyo:" - [ -z "$PYO_HTTPS_URL" ] || echo " $PYO_HTTPS_URL" - [ -z "$lan" ] || echo " http://$lan:8000 (your network)" - echo " http://localhost:8000 (this machine)" - [ -n "$PYO_HTTPS_URL" ] || echo "For an HTTPS address, set up Tailscale in Pyo." + addr_print open + [ -n "$ADDR_HTTPS" ] || echo "For an HTTPS address, set up Tailscale in Pyo." } # The host's own helpers, as systemd timers (root), every minute: connecting # this machine as a cluster until it's done, and putting it on the tailnet @@ -667,6 +663,249 @@ sum=$(sha256sum <<<"$units" | cut -d' ' -f1) [ "$sum" != "$(cat /etc/pyo/host-units 2>/dev/null)" ] || exit 0 exec bash -c "$units"$'\n'"echo $sum > /etc/pyo/host-units" SH + "${sudo[@]}" tee /usr/local/lib/pyo/addresses.sh >/dev/null <<'ADDR' +# Where Pyo is, as the install script, scripts/pyo-up.sh and pyo-reset-login +# show it. Sourced: it only defines functions. +# +# Public https://pyo..nip.io, with Let's Encrypt's +# certificate. The public IP is this machine's own: on a network +# card, or the one its cloud maps to it (1:1 NAT), from the +# cloud's metadata. Never plain HTTP there: passwords would cross +# the internet in the clear. +# Tailnet Pyo's tailnet name, else this machine's Tailscale IP on :8000 +# when Pyo answers there. +# Network this machine's private address on :8000: its main network +# card's. Never a container, VM or k3s bridge's. +# Local http://localhost:8000 + +# Public IPv4 addresses among the lines on stdin: not private, tailnet +# (CGNAT), loopback, link-local, multicast or reserved. +addr_public_v4s() { + grep -E '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$' | + grep -vE '^(0|10|127|2(2[4-9]|[3-5][0-9]))\.|^169\.254\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\.|^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.' || true +} + +# This machine's IPv4 addresses on its own network cards, "card address" a +# line. Not loopback, tunnels (Tailscale's comes on its own), or the +# bridges of containers, VMs and k3s (docker, br-*, cni0, flannel, libvirt, +# multipass, LXD), nor k3s's pod and service ranges or docker's own. +addr_own_ipv4s() { + local out + if out=$(ip -o -4 addr show 2>/dev/null) && [ -n "$out" ]; then + awk '{ split($4, a, "/"); sub(/@.*/, "", $2); print $2, a[1] }' <<<"$out" + else + hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+(\.[0-9]+){3}$' | sed 's/^/- /' + fi | grep -vE '^(lo|docker[0-9]*|br-[^ ]*|veth[^ ]*|cni[0-9]*|flannel[^ ]*|cali[^ ]*|vxlan[^ ]*|kube-[^ ]*|cilium[^ ]*|virbr[^ ]*|mpqemubr[^ ]*|lxdbr[^ ]*|lxcbr[^ ]*|podman[^ ]*|tailscale[^ ]*|tun[^ ]*|wg[^ ]*|zt[^ ]*) ' | + grep -vE ' (127\.|169\.254\.|10\.4[23]\.|172\.17\.)' || true +} + +# A public IPv4 address on this machine's network cards (as Pyo's agent +# finds them: locate.rs). +addr_nic_public_ip() { + addr_own_ipv4s | cut -d' ' -f2 | addr_public_v4s | head -n1 +} + +# This machine on its private network: an address of the card the default +# route goes through, else the first private one. One address, or none. +addr_network_ip() { + local all dev + all=$(addr_own_ipv4s | grep -E ' (10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' || true) + dev=$(ip -4 route show default 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -n1 || true) + { [ -z "$dev" ] || grep "^$dev " <<<"$all" || true; echo "$all"; } | sed -n '1s/^[^ ]* //p' +} + +# One cloud's metadata service: the public IPv4 address it maps to this +# machine. Each request waits a second at most. +addr_cloud_ask() { + local m=169.254.169.254 t="" h=() + case $1 in + aws) # AWS (IMDSv2, else v1), and clouds that answer like it + t=$(curl -fsS -m 1 -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' "http://$m/latest/api/token" 2>/dev/null) || t="" + [ -z "$t" ] || h=(-H "X-aws-ec2-metadata-token: $t") + curl -fsS -m 1 "${h[@]}" "http://$m/latest/meta-data/public-ipv4" ;; + gcp) curl -fsS -m 1 -H 'Metadata-Flavor: Google' "http://$m/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" ;; + azure) curl -fsS -m 1 -H 'Metadata: true' "http://$m/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2021-02-01&format=text" ;; + alibaba) + t=$(curl -fsS -m 1 -X PUT -H 'X-aliyun-ecs-metadata-token-ttl-seconds: 60' http://100.100.100.200/latest/api/token 2>/dev/null) || t="" + [ -z "$t" ] || h=(-H "X-aliyun-ecs-metadata-token: $t") + curl -fsS -m 1 "${h[@]}" http://100.100.100.200/latest/meta-data/eipv4 || + curl -fsS -m 1 "${h[@]}" http://100.100.100.200/latest/meta-data/public-ipv4 ;; + tencent) curl -fsS -m 1 http://169.254.0.23/latest/meta-data/public-ipv4 ;; + scaleway) curl -fsS -m 1 http://169.254.42.42/conf | sed -n 's/^PUBLIC_IP_ADDRESS=//p' ;; + esac 2>/dev/null + echo +} + +# The public IPv4 address this machine's cloud maps to it, from the cloud's +# metadata: every cloud asked at once, so it takes a second or two. Empty +# elsewhere (a home network, a laptop, a VM). Never what the internet sees +# of this machine's own requests: that can be a router's or a shared +# gateway's address, which doesn't lead back here. +addr_cloud_ip() { + local d c pids=() + d=$(mktemp -d) + for c in aws gcp azure alibaba tencent scaleway; do + addr_cloud_ask "$c" >"$d/$c" & + pids+=($!) + done + wait "${pids[@]}" 2>/dev/null || true + cat "$d"/* 2>/dev/null | tr -d '\r ' | addr_public_v4s | head -n1 + rm -rf "$d" +} + +# Pyo's agent's word for it: the public IP this machine's cloud maps to it, +# as its locate.rs found it (a cloud whose metadata doesn't say, like +# Oracle's). +addr_agent_cloud_ip() { + addr_kc get node "$(hostname | tr 'A-Z' 'a-z')" -o jsonpath='{.metadata.annotations.pyo/cloud-public-ipv4}' 2>/dev/null | + addr_public_v4s | head -n1 +} + +# This machine's own public IPv4 address (looked up once): on a network +# card, else the one its cloud maps to it. Empty without one. +addr_public_ip() { + if [ -z "${ADDR_IP_KNOWN:-}" ]; then + ADDR_IP=$(addr_nic_public_ip) + [ -n "$ADDR_IP" ] || ADDR_IP=$(addr_cloud_ip) + [ -n "$ADDR_IP" ] || ADDR_IP=$(addr_agent_cloud_ip) + ADDR_IP_KNOWN=1 + fi + echo "$ADDR_IP" +} + +addr_kc() { + if [ "$(id -u)" = 0 ]; then k3s kubectl "$@"; else kubectl "$@"; fi +} + +# The certificate of Pyo's HTTPS address on the public IP, as cert-manager +# has it: ADDR_CERT_HOST ("pyo..nip.io", empty when Pyo asked +# for none) and ADDR_CERT_STATE: ready, failed (Let's Encrypt couldn't +# reach this machine on port 80), or pending. A challenge or order Let's +# Encrypt turned down counts as failed at once, before the certificate +# says so. +addr_public_cert() { + local host ready reason made ns=pyo-gateway + ADDR_CERT_HOST="" ADDR_CERT_STATE="" ADDR_CERT_FAILED_AT="" + read -r host ready reason made ADDR_CERT_FAILED_AT < <(addr_kc -n "$ns" get certificate pyo-tls---dashboard \ + -o jsonpath='{.spec.dnsNames[0]} {.status.conditions[?(@.type=="Ready")].status}- {.status.conditions[?(@.type=="Issuing")].reason}- {.metadata.creationTimestamp} {.status.lastFailureTime}' 2>/dev/null || true) || true + [ -n "$host" ] || return 0 + ADDR_CERT_HOST=$host ADDR_CERT_STATE=pending + if [ "$ready" = True- ]; then + ADDR_CERT_STATE=ready + elif [ "$reason" = Failed- ] || [ -n "$ADDR_CERT_FAILED_AT" ] || + addr_kc -n "$ns" get challenges.acme.cert-manager.io,orders.acme.cert-manager.io \ + -o jsonpath='{range .items[*]}{.metadata.creationTimestamp} {.spec.dnsName}{.spec.dnsNames[0]} {.status.state}{"\n"}{end}' 2>/dev/null | + awk -v host="$host" -v made="$made" '$1 >= made && $2 == host && ($3 == "invalid" || $3 == "errored") { found = 1 } END { exit !found }'; then + ADDR_CERT_STATE=failed + fi +} + +# A certificate Let's Encrypt refused 15 minutes ago or more is asked for +# again now (Pyo's agent makes it again at once): running the installer +# again after opening the ports checks them. Not sooner: Let's Encrypt +# allows 5 failed checks for a name an hour. +addr_retry_failed() { + local at + [ "${ADDR_CERT_STATE:-}" = failed ] && [ -n "${ADDR_CERT_FAILED_AT:-}" ] || return 1 + at=$(date -d "$ADDR_CERT_FAILED_AT" +%s 2>/dev/null) || return 1 + [ $(($(date +%s) - at)) -ge 900 ] || return 1 + addr_kc -n pyo-gateway delete certificate pyo-tls---dashboard --wait=false >/dev/null 2>&1 || return 1 + ADDR_CERT_STATE=pending ADDR_CERT_FAILED_AT="" +} + +# Done waiting for the HTTPS address: its certificate is issued, or Let's +# Encrypt couldn't reach this machine. +addr_public_settled() { + addr_public_cert + [ "$ADDR_CERT_STATE" = ready ] || [ "$ADDR_CERT_STATE" = failed ] +} + +# Worth waiting for the HTTPS address: this machine has a public IP of its +# own, and the certificate there isn't settled (or was refused a while ago: +# tried again). Without one asked for yet, not when Pyo is on the tailnet +# (an admin may have closed it). +addr_wait_wanted() { + ! addr_public_settled || addr_retry_failed || return 1 + [ -n "$(addr_public_ip)" ] || return 1 + [ -n "$ADDR_CERT_HOST" ] || [ -z "$(addr_pyo_tailnet)" ] +} + +# Pyo's HTTPS name on the tailnet ("https://pyo..ts.net"), once it +# has one. +addr_pyo_tailnet() { + curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | + sed -n 's/.*"publicUrl":"\(https:\/\/[^"]*\.ts\.net\)\/*".*/\1/p' | head -n1 || true +} + +# Pyo on the tailnet: its HTTPS name there, else this machine's Tailscale +# IP on :8000, when Tailscale is up here and Pyo answers on it. +addr_tailnet() { + local url ip + url=$(addr_pyo_tailnet) + if [ -n "$url" ]; then + echo "$url" + return 0 + fi + command -v tailscale >/dev/null 2>&1 || return 0 + ip=$(tailscale ip -4 2>/dev/null | head -n1 || true) + [ -n "$ip" ] || return 0 + curl -fsS -m 3 -o /dev/null "http://$ip:8000/api/v1/meta" 2>/dev/null && echo "http://$ip:8000" + return 0 +} + +# Colors on a terminal, unless NO_COLOR says not to. +addr_color() { [ -z "${NO_COLOR:-}" ] && [ -t 1 ]; } + +# addr_line LABEL URL [NOTE]: " āžœ Public: https://… (note)", the +# labels lined up. On a terminal: a green arrow, the label in bold, the +# address in cyan. +addr_line() { + local pad note="" + pad=$(printf '%*s' $((9 - ${#1})) '') + [ -z "${3:-}" ] || note=" ($3)" + if addr_color; then + printf ' \033[32māžœ\033[0m \033[1m%s:\033[0m%s\033[36m%s\033[0m%s\n' "$1" "$pad" "$2" "$note" + else + printf ' āžœ %s:%s%s%s\n' "$1" "$pad" "$2" "$note" + fi +} + +# addr_print open|PATH: Pyo's addresses, one line each. "open": for opening +# Pyo, with the HTTPS address on the public IP also while it's on its way +# or blocked, and what to do then. A PATH: links to it that work now. +# ADDR_HTTPS says whether there's a Public or Tailnet line. +addr_print() { + local path="" host="" ip url net note="" + [ "${1:-open}" = open ] || path=$1 + ADDR_HTTPS="" + [ -n "${ADDR_CERT_STATE:-}" ] || addr_public_cert + host=$ADDR_CERT_HOST + # None asked for yet, after a wait for it: it's on its way. + if [ -z "$host" ] && [ -n "${ADDR_WAITED:-}" ]; then + ip=$(addr_public_ip) + [ -z "$ip" ] || host="pyo.$ip.nip.io" + fi + if [ -n "$host" ]; then + case "${ADDR_CERT_STATE:-}" in + ready) ;; + failed) note="allow ports 80 and 443 in your cloud firewall" ;; + *) note="ready in a few minutes" ;; + esac + if [ -z "$note" ] || [ -z "$path" ]; then + addr_line Public "https://$host$path" "$note" + ADDR_HTTPS=1 + fi + fi + url=$(addr_tailnet) + if [ -n "$url" ]; then + addr_line Tailnet "$url$path" + ADDR_HTTPS=1 + fi + net=$(addr_network_ip) + [ -z "$net" ] || addr_line Network "http://$net:8000$path" + addr_line Local "http://localhost:8000$path" +} +ADDR "${sudo[@]}" tee /usr/local/lib/pyo/pyo-reset-login >/dev/null <<'SH' #!/usr/bin/env bash # Prints a one-time link to choose a new password for a Pyo account, for a @@ -701,15 +940,14 @@ if [ "$status" != 200 ]; then fi path=$(field path) email=$(field email) minutes=$(sed -n 's/.*"minutes":\([0-9]*\).*/\1/p' <<<"$body") -https=$(curl -fsS -m 5 http://127.0.0.1:8000/api/v1/meta 2>/dev/null | sed -n 's/.*"publicUrl":"\([^"]*\)".*/\1/p' || true) -lan=$(hostname -I 2>/dev/null | tr ' ' '\n' | grep -E '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' | head -n1 || true) +# shellcheck source=/dev/null +. /usr/local/lib/pyo/addresses.sh echo "Open one of these links to choose a new password for $email:" -[ -z "$https" ] || echo " ${https%/}$path" -[ -z "$lan" ] || echo " http://$lan:8000$path (your network)" -echo " http://localhost:8000$path (this machine)" +addr_print "$path" echo "It works once, for ${minutes:-30} minutes." SH "${sudo[@]}" chmod 755 /usr/local/lib/pyo/connect-this-machine.sh /usr/local/lib/pyo/tailnet-this-machine.sh /usr/local/lib/pyo/pyo-reset-login + "${sudo[@]}" chmod 644 /usr/local/lib/pyo/addresses.sh "${sudo[@]}" ln -sfn /usr/local/lib/pyo/pyo-reset-login /usr/local/bin/pyo-reset-login "${sudo[@]}" tee /etc/systemd/system/pyo-connect.service >/dev/null <<'UNIT' [Unit] @@ -989,9 +1227,7 @@ as_root run_step "Connecting this machine to Pyo" "${SUDO[@]}" /usr/local/lib/pyo/connect-this-machine.sh || true sed 's/^/ | /' "$STEP_OUT" -echo -ok_line "Pyo $TAG is up" -pyo_addresses 120 +pyo_addresses 120 "Pyo $TAG is up" if [ -n "$ADMIN_PASSWORD" ]; then echo echo "Sign in with this login, then choose your own email and password:" From 53a02903b6d9da0b69a7fc9fca39f1739f8cb3ed Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Wed, 18 Nov 2026 00:03:56 +0700 Subject: [PATCH 06/22] feat(install): pin each release's install scripts on pyo.dev pyo.dev now serves each release's scripts at pyo.dev/v//, checked against its signed release.json. Commands that fetch scripts from pyo.dev use this release's there, instead of the latest one with --version. connect.sh, join.sh and sandbox.sh get run_release: with --version naming another release, they run that release's own script from pyo.dev/v//. Pyo 0.4.2 and earlier give the latest script their --version, so their commands keep running their own release's scripts. The release workflow writes each script's release on its pyo:release-version line. Pyo's own copy leaves it empty and never runs another. --- crates/pyo-api/assets/connect.sh | 40 ++++++- crates/pyo-api/assets/join.sh | 40 ++++++- crates/pyo-api/src/join.rs | 180 ++++++++++++++++++++++------- crates/pyo-api/src/sandbox/join.rs | 16 ++- deploy/sandbox.sh | 36 ++++++ 5 files changed, 264 insertions(+), 48 deletions(-) diff --git a/crates/pyo-api/assets/connect.sh b/crates/pyo-api/assets/connect.sh index 8f693fb..12cc92a 100644 --- a/crates/pyo-api/assets/connect.sh +++ b/crates/pyo-api/assets/connect.sh @@ -11,7 +11,7 @@ # --tailscale-key joins your tailnet first (a single-use key from Pyo) # --registry Pyo's registry door, for images Pyo builds # --registry-token this cluster's login there (pull only) -# --version the Pyo release, for pyo.dev to run its script (ignored here) +# --version runs this Pyo release's connect.sh instead, from pyo.dev # # Or only joins this machine to the tailnet (Pyo's own machine does this # once Tailscale is set up in Pyo): --tailnet-only, with TS_AUTHKEY. @@ -26,6 +26,42 @@ # --yes don't ask first # --dry-run only say what it would delete # With KUBECONFIG set, it uses kubectl and that cluster instead of k3s's. +# >>> pyo:release (the same in connect.sh, join.sh and deploy/sandbox.sh) +# With --version naming another Pyo release, runs that release's own +# script from pyo.dev, which keeps each release's scripts at +# https://pyo.dev/v//. Pyo 0.4.2 and earlier give pyo.dev's latest +# script their --version. The release workflow writes the release a script +# belongs to on the line marked pyo:release-version, in the copy it +# publishes. Pyo's own copy (/install/) leaves it empty and never runs +# another. +run_release() { + local name=$1 want="" prev="" arg dir rc=0 + local release="" # pyo:release-version + shift + for arg in "$@"; do + [ "$prev" != --version ] || want=${arg#v} + prev=$arg + done + [ -n "$release" ] && [ -n "$want" ] && [ "$want" != "$release" ] || return 0 + if [[ ! $want =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "--version takes a Pyo release, like $release. \"$want\" isn't one." >&2 + exit 2 + fi + dir=$(mktemp -d) + if ! curl -fsSL --proto '=https' --retry 3 -o "$dir/$name" "https://pyo.dev/v/$want/$name"; then + rm -rf "$dir" + echo "pyo.dev has no $name for Pyo $want. Copy the command from your Pyo again." >&2 + exit 1 + fi + # Piped into bash, the rest of this script is still in the pipe. Read it + # out, so curl can finish and the other script doesn't take it as input. + [ -f "${BASH_SOURCE[1]:-}" ] || [ -t 0 ] || cat >/dev/null + bash "$dir/$name" "$@" || rc=$? + rm -rf "$dir" + exit "$rc" +} +# <<< pyo:release +run_release connect.sh "$@" # >>> pyo:become-root (the same in connect.sh and join.sh) # Root, through sudo when it isn't: the script runs again as root, with # the same arguments and Tailscale key. Run from a file, sudo runs that @@ -143,7 +179,7 @@ while [ $# -gt 0 ]; do --dry-run) DRY_RUN=1; shift ;; # install.sh --uninstall on a k3s Pyo didn't install: Pyo itself goes too. --with-pyo) WITH_PYO=1; shift ;; - # The release pyo.dev's wrapper ran this script from. Nothing to do here. + # run_release (above) handled it. --version) shift 2 || shift ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac diff --git a/crates/pyo-api/assets/join.sh b/crates/pyo-api/assets/join.sh index 1bc0229..69420ed 100644 --- a/crates/pyo-api/assets/join.sh +++ b/crates/pyo-api/assets/join.sh @@ -15,7 +15,43 @@ # --builder a build-only server: only Pyo's builder runs on it # --registry Pyo's registry door, for images Pyo builds # --registry-token the cluster's login there (pull only) -# --version the Pyo release, for pyo.dev to run its script (ignored here) +# --version runs this Pyo release's join.sh instead, from pyo.dev +# >>> pyo:release (the same in connect.sh, join.sh and deploy/sandbox.sh) +# With --version naming another Pyo release, runs that release's own +# script from pyo.dev, which keeps each release's scripts at +# https://pyo.dev/v//. Pyo 0.4.2 and earlier give pyo.dev's latest +# script their --version. The release workflow writes the release a script +# belongs to on the line marked pyo:release-version, in the copy it +# publishes. Pyo's own copy (/install/) leaves it empty and never runs +# another. +run_release() { + local name=$1 want="" prev="" arg dir rc=0 + local release="" # pyo:release-version + shift + for arg in "$@"; do + [ "$prev" != --version ] || want=${arg#v} + prev=$arg + done + [ -n "$release" ] && [ -n "$want" ] && [ "$want" != "$release" ] || return 0 + if [[ ! $want =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "--version takes a Pyo release, like $release. \"$want\" isn't one." >&2 + exit 2 + fi + dir=$(mktemp -d) + if ! curl -fsSL --proto '=https' --retry 3 -o "$dir/$name" "https://pyo.dev/v/$want/$name"; then + rm -rf "$dir" + echo "pyo.dev has no $name for Pyo $want. Copy the command from your Pyo again." >&2 + exit 1 + fi + # Piped into bash, the rest of this script is still in the pipe. Read it + # out, so curl can finish and the other script doesn't take it as input. + [ -f "${BASH_SOURCE[1]:-}" ] || [ -t 0 ] || cat >/dev/null + bash "$dir/$name" "$@" || rc=$? + rm -rf "$dir" + exit "$rc" +} +# <<< pyo:release +run_release join.sh "$@" # >>> pyo:become-root (the same in connect.sh and join.sh) # Root, through sudo when it isn't: the script runs again as root, with # the same arguments and Tailscale key. Run from a file, sudo runs that @@ -127,7 +163,7 @@ while [ $# -gt 0 ]; do --builder) BUILDER=1; shift ;; --registry) REGISTRY="$2"; shift 2 ;; --registry-token) REGISTRY_TOKEN="$2"; shift 2 ;; - # The release pyo.dev's wrapper ran this script from. Nothing to do here. + # run_release (above) handled it. --version) shift 2 || shift ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index e057afb..24cf83b 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -14,9 +14,15 @@ pub const CONNECT_SCRIPT: &str = include_str!("../assets/connect.sh"); pub const SANDBOX_SCRIPT: &str = include_str!("../../../deploy/sandbox.sh"); /// Where the scripts are published, for when this Pyo has no address -/// another machine can reach. +/// another machine can reach. pyo.dev serves each release's scripts at +/// `/v//`, only as its signed release.json lists them. pub const PUBLISHED_SCRIPTS_URL: &str = "https://pyo.dev"; +/// This release's scripts on pyo.dev: `https://pyo.dev/v/`. +pub fn published_scripts_url() -> String { + format!("{PUBLISHED_SCRIPTS_URL}/v/{}", env!("CARGO_PKG_VERSION")) +} + /// Where the commands for other machines fetch their scripts from, first /// of: /// 1. `PYO_SCRIPTS_URL`, when set (a mirror, or pyo.dev); @@ -25,7 +31,8 @@ pub const PUBLISHED_SCRIPTS_URL: &str = "https://pyo.dev"; /// which it has to reach anyway; /// 3. Pyo on the tailnet ([`crate::web::tailnet_api_url`]), then its HTTPS /// dashboard; -/// 4. pyo.dev, when Pyo is only on its own machine or in its cluster. +/// 4. pyo.dev ([`published_scripts_url`]), when Pyo is only on its own +/// machine or in its cluster. /// /// Only for machines already on the tailnet, or told Pyo's public URL. A /// command with a Tailscale key uses [`scripts_url_before_tailnet`]. @@ -53,7 +60,7 @@ fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String { .find(|u| outside(u) && secure_transport(u)) { Some(u) => format!("{u}/install"), - None => PUBLISHED_SCRIPTS_URL.to_owned(), + None => published_scripts_url(), } } @@ -63,13 +70,12 @@ fn pick_scripts_url(configured: Option<&str>, pyo: &[Option<&str>]) -> String { /// its public IP: that address may not be reachable from outside (a server /// behind NAT, like Starlink's). So: /// 1. `PYO_SCRIPTS_URL`, when set; -/// 2. else pyo.dev. Its sandbox.sh, join.sh and connect.sh run the script -/// of the release `--version` names ([`pinned`]), and join.sh's `--k3s` -/// keeps the server on the cluster's k3s. +/// 2. else this release's scripts on pyo.dev ([`published_scripts_url`]), +/// and join.sh's `--k3s` keeps the server on the cluster's k3s. /// /// A dev build's version may not be a published release, so pyo.dev has no -/// sandbox.sh for it and its wrapper fails to download one: dev builds set -/// `PYO_SCRIPTS_URL` (to a mirror, or a Pyo the new machine can reach). +/// scripts for it: dev builds set `PYO_SCRIPTS_URL` (to a mirror, or a Pyo +/// the new machine can reach). /// /// Commands for machines already on the tailnet use [`scripts_url`]. pub fn scripts_url_before_tailnet() -> String { @@ -77,15 +83,23 @@ pub fn scripts_url_before_tailnet() -> String { } pub(crate) fn pick_scripts_url_before_tailnet(configured: Option<&str>) -> String { - setting(configured).unwrap_or_else(|| PUBLISHED_SCRIPTS_URL.to_owned()) + setting(configured).unwrap_or_else(published_scripts_url) } -/// `PYO_SCRIPTS_URL`, when it's set to something. +/// `PYO_SCRIPTS_URL`, when it's set to something. pyo.dev itself means +/// this release's scripts there. fn setting(configured: Option<&str>) -> Option { configured .map(str::trim) .filter(|u| !u.is_empty()) - .map(|u| u.trim_end_matches('/').to_owned()) + .map(|u| u.trim_end_matches('/')) + .map(|u| { + if u == PUBLISHED_SCRIPTS_URL { + published_scripts_url() + } else { + u.to_owned() + } + }) } /// The host of an http(s) URL, lowercase, without brackets or port. @@ -269,15 +283,16 @@ pub fn command(base: &str, j: &Join) -> String { ) } -/// A script's arguments for where it's fetched from (`base`). pyo.dev's -/// join.sh and connect.sh run the script of the release `--version` names, -/// else the latest one's: fetched from anywhere but a Pyo (whose own are -/// its release's), they get this Pyo's release. The scripts ignore it. +/// A script's arguments for where it's fetched from (`base`). A Pyo's own +/// scripts and pyo.dev's `/v//` are this release's. From anywhere +/// else (a mirror) they get `--version`: a release's join.sh and connect.sh +/// then run that release's script from pyo.dev when they're another's. pub fn pinned( base: &str, mut args: Vec<(&'static str, Option)>, ) -> Vec<(&'static str, Option)> { - if !base.trim_end_matches('/').ends_with("/install") { + let base = base.trim_end_matches('/'); + if !base.ends_with("/install") && !base.starts_with(&format!("{PUBLISHED_SCRIPTS_URL}/v/")) { args.push(("--version", Some(env!("CARGO_PKG_VERSION").to_owned()))); } args @@ -419,7 +434,9 @@ pub async fn serve_host_units() -> impl axum::response::IntoResponse { mod tests { use super::*; - /// pyo.dev's wrappers run this release's scripts. + /// This release's scripts on pyo.dev. + const V: &str = concat!("https://pyo.dev/v/", env!("CARGO_PKG_VERSION")); + /// From a mirror, a release's scripts run this release's. const PIN: &str = concat!(" --version '", env!("CARGO_PKG_VERSION"), "'"); #[test] @@ -678,6 +695,27 @@ mod tests { block("become-root", "connect.sh", CONNECT_SCRIPT), "join.sh's become_root differs from connect.sh's" ); + // How they run the release --version names (release.yml writes + // each one's release on its pyo:release-version line). + let release = block("release", "connect.sh", CONNECT_SCRIPT); + for (file, script) in [("join.sh", SCRIPT), ("deploy/sandbox.sh", SANDBOX_SCRIPT)] { + assert_eq!( + block("release", file, script), + release, + "{file}'s run_release differs from connect.sh's" + ); + } + for (file, script) in [ + ("connect.sh", CONNECT_SCRIPT), + ("join.sh", SCRIPT), + ("deploy/sandbox.sh", SANDBOX_SCRIPT), + ] { + assert_eq!( + script.matches(RELEASE_LINE).count(), + 1, + "{file} needs exactly one {RELEASE_LINE:?} line for release.yml" + ); + } // How both find Pyo's (or the control plane's) tailnet name. assert_eq!( block("tailnet-host", "join.sh", SCRIPT), @@ -753,8 +791,9 @@ mod tests { #[test] fn builds_command() { + assert_eq!(published_scripts_url(), V); let c = command( - "https://pyo.dev", + V, &Join { server_ip: "100.64.0.1", token: "abc123.0123456789abcdef", @@ -768,7 +807,7 @@ mod tests { assert_eq!( c, format!( - "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server '100.64.0.1' --token 'abc123.0123456789abcdef' --name 'sg-1' --k3s 'v1.36.5+k3s1'{PIN}" + "curl -fsSL {V}/join.sh | bash -s -- --server '100.64.0.1' --token 'abc123.0123456789abcdef' --name 'sg-1' --k3s 'v1.36.5+k3s1'" ) ); let c = command( @@ -791,19 +830,24 @@ mod tests { ); assert_eq!( connect_command( - "https://pyo.dev", + V, "http://100.64.0.1:8000", "abc", None, Some(("100.64.0.1:5005", "def")) ), format!( - "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'http://100.64.0.1:8000' --token 'abc' --registry '100.64.0.1:5005' --registry-token 'def'{PIN}" + "curl -fsSL {V}/connect.sh | bash -s -- --api 'http://100.64.0.1:8000' --token 'abc' --registry '100.64.0.1:5005' --registry-token 'def'" ) ); assert_eq!( - refresh_command("https://pyo.dev"), - format!("curl -fsSL https://pyo.dev/connect.sh | bash -s --{PIN}") + refresh_command(V), + format!("curl -fsSL {V}/connect.sh | bash") + ); + // A mirror's: --version runs this release's, when they're another's. + assert_eq!( + refresh_command("https://mirror.example.com"), + format!("curl -fsSL https://mirror.example.com/connect.sh | bash -s --{PIN}") ); // Pyo's own scripts are its release's. assert_eq!( @@ -881,11 +925,12 @@ mod tests { #[test] fn scripts_come_from_the_setting_then_pyo_then_pyo_dev() { let tailnet = Some("http://100.64.0.1:8000/"); - // The setting wins. + // The setting wins. pyo.dev there is this release's scripts on it. assert_eq!( pick_scripts_url(Some(" https://mirror.example.com/ "), &[tailnet]), "https://mirror.example.com" ); + assert_eq!(pick_scripts_url(Some("https://pyo.dev/"), &[tailnet]), V); // Unset or blank: Pyo, at the first address another machine reaches. for unset in [None, Some(""), Some(" ")] { assert_eq!( @@ -919,13 +964,9 @@ mod tests { "", ]; for u in local { - assert_eq!( - pick_scripts_url(None, &[Some(u)]), - PUBLISHED_SCRIPTS_URL, - "{u}" - ); + assert_eq!(pick_scripts_url(None, &[Some(u)]), V, "{u}"); } - assert_eq!(pick_scripts_url(None, &[None, None]), PUBLISHED_SCRIPTS_URL); + assert_eq!(pick_scripts_url(None, &[None, None]), V); } #[test] @@ -938,15 +979,15 @@ mod tests { // Unset or blank: pyo.dev, never Pyo itself (on the tailnet, or on // a public IP that may sit behind NAT). for unset in [None, Some(""), Some(" ")] { - assert_eq!(pick(unset), PUBLISHED_SCRIPTS_URL); + assert_eq!(pick(unset), V); } } #[test] fn commands_with_a_tailscale_key_fetch_their_script_off_the_tailnet() { let base = pick_scripts_url_before_tailnet(None); - // join.sh: pyo.dev runs the latest release's join.sh, and --k3s keeps - // the server on the cluster's k3s. + // join.sh: this release's on pyo.dev, and --k3s keeps the server on + // the cluster's k3s. assert_eq!( command( &base, @@ -961,7 +1002,7 @@ mod tests { }, ), format!( - "curl -fsSL https://pyo.dev/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'{PIN}" + "curl -fsSL {V}/join.sh | bash -s -- --server 'server-1.tail1234.ts.net' --token 't' --name 'server-2' --k3s 'v1.37.2+k3s1' --tailscale-key 'tskey-auth-1'" ) ); assert_eq!( @@ -973,7 +1014,7 @@ mod tests { Some(("pyo-registry.tail1234.ts.net", "def")) ), format!( - "curl -fsSL https://pyo.dev/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'{PIN}" + "curl -fsSL {V}/connect.sh | bash -s -- --api 'https://pyo.tail1234.ts.net' --token 'abc' --tailscale-key 'tskey-auth-1' --registry 'pyo-registry.tail1234.ts.net' --registry-token 'def'" ) ); // A mirror the operator set. @@ -1026,10 +1067,7 @@ mod tests { assert!(!secure_transport(bad), "{bad}"); } // Never a plain-http public address for the scripts either. - assert_eq!( - pick_scripts_url(None, &[Some("http://pyo.example.com")]), - PUBLISHED_SCRIPTS_URL - ); + assert_eq!(pick_scripts_url(None, &[Some("http://pyo.example.com")]), V); } #[test] @@ -1055,6 +1093,70 @@ mod tests { script[start..start + end].to_owned() } + /// The line release.yml writes a script's release on. + const RELEASE_LINE: &str = "\n local release=\"\" # pyo:release-version\n"; + + /// join.sh's run_release, as release.yml publishes it for `release` + /// (empty: as Pyo serves it), with `args`. pyo.dev has only 0.4.2's + /// join.sh, which prints its arguments and exits with 3. + fn run_release(release: &str, args: &str) -> (i32, String, String) { + let block = script_block("release", SCRIPT).replace( + RELEASE_LINE, + &format!("\n local release=\"{release}\" # pyo:release-version\n"), + ); + bash(&format!( + r#"curl() {{ + local out="" url="" a + while [ $# -gt 0 ]; do + case "$1" in -o) out=$2; shift 2 ;; --proto | --retry) shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac + done + [ "$url" = https://pyo.dev/v/0.4.2/join.sh ] || return 22 + printf '%s\n' 'echo "0.4.2 join.sh: $*"; exit 3' >"$out" +}} +{block} +run_release join.sh {args} +echo "this one runs" +"# + )) + } + + #[test] + fn scripts_run_the_release_their_version_names() { + // Pyo 0.4.2's commands give pyo.dev's latest script --version 0.4.2. + let (code, out, _) = run_release("0.4.3", "--server s --version 0.4.2"); + assert_eq!( + (code, out.as_str()), + (3, "0.4.2 join.sh: --server s --version 0.4.2\n") + ); + let (code, out, _) = run_release("0.4.3", "--version v0.4.2"); + assert_eq!( + (code, out.as_str()), + (3, "0.4.2 join.sh: --version v0.4.2\n") + ); + // Its own release, no --version, or Pyo's own copy: this script. + for (release, args) in [ + ("0.4.3", "--server s --version 0.4.3"), + ("0.4.3", "--server s"), + ("", "--version 0.4.2"), + ] { + assert_eq!( + run_release(release, args), + (0, "this one runs\n".into(), String::new()), + "{release} {args}" + ); + } + // A release pyo.dev doesn't have, or not a release at all. + let (code, out, err) = run_release("0.4.3", "--version 0.3.0"); + assert_eq!((code, out.as_str()), (1, "")); + assert!( + err.contains("pyo.dev has no join.sh for Pyo 0.3.0."), + "{err}" + ); + let (code, _, err) = run_release("0.4.3", "--version '0.4.2;x'"); + assert_eq!(code, 2); + assert!(err.contains("isn't one"), "{err}"); + } + /// Runs `script` with bash: (exit code, stdout, stderr). fn bash(script: &str) -> (i32, String, String) { let out = std::process::Command::new("bash") diff --git a/crates/pyo-api/src/sandbox/join.rs b/crates/pyo-api/src/sandbox/join.rs index e1ae203..b8f2e20 100644 --- a/crates/pyo-api/src/sandbox/join.rs +++ b/crates/pyo-api/src/sandbox/join.rs @@ -481,8 +481,10 @@ mod tests { #[test] fn the_command_quotes_everything() { - // The server isn't on the tailnet yet: pyo.dev. + // The server isn't on the tailnet yet: this release's scripts on + // pyo.dev. let base = crate::join::pick_scripts_url_before_tailnet(None); + assert_eq!(base, crate::join::published_scripts_url()); let c = Command { base: &base, door: "pyo.tail1234.ts.net:5006", @@ -495,7 +497,9 @@ mod tests { }; assert_eq!( c.render(), - "curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'" + format!( + "curl -fsSL {base}/sandbox.sh | bash -s -- --pyo 'pyo.tail1234.ts.net:5006' --server 'sbs_one' --name 'it'\\''s sbx' --code 'abc' --tailscale-key 'tskey-auth-1' --version '0.4.0'" + ) ); let c = Command { edge_domain: Some("sbx.example.com"), @@ -530,12 +534,14 @@ mod tests { render(&pick(Some("https://mirror.example.com/"))) .starts_with("curl -fsSL https://mirror.example.com/sandbox.sh | bash -s -- ") ); - // Else pyo.dev, which runs the sandbox.sh of the --version release. + // Else this release's sandbox.sh on pyo.dev. for unset in [None, Some(" ")] { let c = render(&pick(unset)); assert!( - c.starts_with("curl -fsSL https://pyo.dev/sandbox.sh | bash -s -- ") - && c.contains(" --version '0.4.0'"), + c.starts_with(&format!( + "curl -fsSL https://pyo.dev/v/{}/sandbox.sh | bash -s -- ", + env!("CARGO_PKG_VERSION") + )) && c.contains(" --version '0.4.0'"), "{c}" ); } diff --git a/deploy/sandbox.sh b/deploy/sandbox.sh index 86bc20a..e14676e 100644 --- a/deploy/sandbox.sh +++ b/deploy/sandbox.sh @@ -800,4 +800,40 @@ main() { fi } +# >>> pyo:release (the same in connect.sh, join.sh and deploy/sandbox.sh) +# With --version naming another Pyo release, runs that release's own +# script from pyo.dev, which keeps each release's scripts at +# https://pyo.dev/v//. Pyo 0.4.2 and earlier give pyo.dev's latest +# script their --version. The release workflow writes the release a script +# belongs to on the line marked pyo:release-version, in the copy it +# publishes. Pyo's own copy (/install/) leaves it empty and never runs +# another. +run_release() { + local name=$1 want="" prev="" arg dir rc=0 + local release="" # pyo:release-version + shift + for arg in "$@"; do + [ "$prev" != --version ] || want=${arg#v} + prev=$arg + done + [ -n "$release" ] && [ -n "$want" ] && [ "$want" != "$release" ] || return 0 + if [[ ! $want =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "--version takes a Pyo release, like $release. \"$want\" isn't one." >&2 + exit 2 + fi + dir=$(mktemp -d) + if ! curl -fsSL --proto '=https' --retry 3 -o "$dir/$name" "https://pyo.dev/v/$want/$name"; then + rm -rf "$dir" + echo "pyo.dev has no $name for Pyo $want. Copy the command from your Pyo again." >&2 + exit 1 + fi + # Piped into bash, the rest of this script is still in the pipe. Read it + # out, so curl can finish and the other script doesn't take it as input. + [ -f "${BASH_SOURCE[1]:-}" ] || [ -t 0 ] || cat >/dev/null + bash "$dir/$name" "$@" || rc=$? + rm -rf "$dir" + exit "$rc" +} +# <<< pyo:release +run_release sandbox.sh "$@" main "$@" From e548d6c61da0a88981054b51c346264b5bce0353 Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Wed, 18 Nov 2026 00:03:56 +0700 Subject: [PATCH 07/22] ci(release): list the install scripts' sums in the release notes --release release.json adds an "Install scripts" section with each script's SHA-256, as the signed release.json lists them, and a link to how to check one. --- scripts/release-notes.test.ts | 8 ++++++++ scripts/release-notes.ts | 26 +++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/release-notes.test.ts b/scripts/release-notes.test.ts index b5c6bc9..2191670 100644 --- a/scripts/release-notes.test.ts +++ b/scripts/release-notes.test.ts @@ -78,6 +78,14 @@ describe("notes", () => { expect(notes.startsWith("### āš ļø Before you update\n\n- Postgres restarts once.\n- Update to 0.2.0 first")).toBe(true); expect(notes).toContain("- Automatic updates skip this release."); }); + test("the install scripts' sums, after the changes", () => { + const sum = "a".repeat(64); + const notes = render({ from: "v0.4.2", to: "v0.4.3", commits: [c("fix: f")], scripts: { "install.sh": sum } }); + expect(notes.indexOf("### šŸ” Install scripts")).toBeGreaterThan(notes.indexOf("### šŸž Bug Fixes")); + expect(notes).toContain(`| \`install.sh\` | \`${sum}\` |`); + expect(notes).toEndWith("**Full Changelog**: https://github.com/upsetdev/pyo/compare/v0.4.2...v0.4.3\n"); + expect(render({ from: "v0.4.2", to: "v0.4.3", commits: [] })).not.toContain("Install scripts"); + }); test("the first release", () => { const notes = render({ from: null, to: "v0.1.0", commits: [] }); expect(notes).toBe("Initial release.\n\n**Full Changelog**: https://github.com/upsetdev/pyo/commits/v0.1.0\n"); diff --git a/scripts/release-notes.ts b/scripts/release-notes.ts index bdc9497..4ddb73a 100644 --- a/scripts/release-notes.ts +++ b/scripts/release-notes.ts @@ -27,8 +27,12 @@ // // The notices in deploy/release-notices.json at --to (when its "version" // is this release's) go first, under "Before you update". +// +// --release release.json adds the SHA-256 of each install script it lists +// (`scripts`), under "Install scripts", after the changes. import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; export const REPO = "upsetdev/pyo"; export const MAINTAINERS = ["fransallen"]; @@ -103,6 +107,7 @@ export function render(opts: { logins?: Map; maintainers?: string[]; notices?: Notices | null; + scripts?: Record | null; }): string { const out: string[] = []; const n = opts.notices; @@ -123,6 +128,19 @@ export function render(opts: { const lines = groups.get(s); if (lines?.length) out.push(`### ${title}`, "", ...lines, ""); } + const scripts = Object.entries(opts.scripts ?? {}); + if (scripts.length) { + out.push( + "### šŸ” Install scripts", + "", + "Each script's SHA-256, as the signed release.json lists it. pyo.dev serves these same files. [How to check one](https://pyo.dev/docs/install#verify-a-script)", + "", + "| Script | SHA-256 |", + "| --- | --- |", + ...scripts.map(([name, sum]) => `| \`${name}\` | \`${sum}\` |`), + "", + ); + } if (opts.from) { out.push(`**Full Changelog**: https://github.com/${REPO}/compare/${opts.from}...${opts.to}`); } else { @@ -203,7 +221,13 @@ function main() { if (!args.has("offline")) { for (const c of listed) if (classify(c)) logins.set(c.sha, login(c.sha)); } - process.stdout.write(render({ from, to, commits: listed, logins, maintainers, notices: notices(to) })); + const release = args.get("release"); + const scripts = release ? JSON.parse(readFileSync(release, "utf8")).scripts : null; + if (release && (!scripts || !Object.keys(scripts).length)) { + console.error(`release-notes: ${release} lists no scripts`); + process.exit(1); + } + process.stdout.write(render({ from, to, commits: listed, logins, maintainers, notices: notices(to), scripts })); } if (import.meta.main) main(); From 38da9287a4950eaf50941b615dea740e43bee940 Mon Sep 17 00:00:00 2001 From: Frans Allen Date: Wed, 18 Nov 2026 00:04:13 +0700 Subject: [PATCH 08/22] ci(release): sign the install scripts' sums and publish through a draft release.json lists the SHA-256 of each install script the release publishes (`scripts`), so they're signed with it. release-manifest.ts --scripts writes the scripts as the release publishes them, with the version on the pyo:release-version line of connect.sh, join.sh and sandbox.sh, and the release uploads those. Pyo reads release.json as before: it ignores the new field. For immutable releases, the release goes up as a draft with every asset, the install scripts in it are checked against release.json, and only then is it published. Run again before that, the job replaces the draft's assets. A release that's published already is left as it is. The notes list each script's sum. --- .github/workflows/release.yml | 87 ++++++++++++++++++++++++++--------- RELEASING.md | 45 ++++++++++++++++-- scripts/release-manifest.ts | 38 ++++++++++++++- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59ac339..4786c1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release # Pushing a tag like v0.1.0 publishes that version: signed images at # ghcr.io/upsetdev/pyo, and a GitHub release with the CLI for each -# platform, the sandbox server files and release.json (what the -# dashboard's updates apply). See RELEASING.md. +# platform, the sandbox server files, the install scripts and release.json +# (what the dashboard's updates apply). See RELEASING.md. on: push: @@ -215,6 +215,10 @@ jobs: done # What Pyo needs to update itself from the dashboard: the images # pinned by the digests that were just signed, and the manifests. + # Also the install scripts as the release publishes them, written to + # release-scripts/ (connect.sh, join.sh and sandbox.sh with this + # version on their pyo:release-version line), and their SHA-256: + # pyo.dev serves only scripts that match these signed sums. - name: Write release.json env: VERSION: ${{ needs.verify.outputs.version }} @@ -230,8 +234,9 @@ jobs: --sandbox-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256)" \ --sandbox-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-linux-aarch64.tar.gz.sha256)" \ --sandbox-gvisor-x86_64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256)" \ - --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" > release.json - jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, migrations: (.migrations | length)}' release.json + --sandbox-gvisor-aarch64 "$(cut -d' ' -f1 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256)" \ + --scripts release-scripts > release.json + jq '{version, images, k3s, minFrom, manualOnly, beforeYouUpdate, agentAccess, sandbox, sandboxGvisor, scripts, migrations: (.migrations | length)}' release.json # The release's own upgrader reads it before it's signed: an image # it doesn't accept, or an agentAccess that isn't its agent's roles, # fails the release here, not on installations. @@ -280,19 +285,23 @@ jobs: - name: Write the release notes env: GH_TOKEN: ${{ github.token }} - run: bun run scripts/release-notes.ts --to "$GITHUB_REF_NAME" > release-notes.md && cat release-notes.md - # install.sh goes out as an asset named exactly that: pyo.dev/install.sh - # fetches releases/latest/download/install.sh (stable releases only). - # connect.sh beside it: install.sh --uninstall runs its own release's. - # pyo.dev/connect.sh and pyo.dev/join.sh fetch connect.sh and join.sh - # the same way (Pyo's commands use them when it has no address other - # machines can reach). pyo.dev/cli.sh fetches cli.sh, the CLI's - # installer. pyo.dev/sandbox.sh fetches sandbox.sh, which sets up - # sandbox servers with the pyo-sandbox-linux-.tar.gz beside it. - - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 - with: - body_path: release-notes.md - files: | + run: bun run scripts/release-notes.ts --to "$GITHUB_REF_NAME" --release release.json > release-notes.md && cat release-notes.md + # The release goes up as a draft with every asset, and is published + # once they're all there: with immutable releases (on for this repo), + # a published release's assets and tag can't change. Run again after + # a failure, this job replaces the draft's assets. A release that's + # published already is left as it is. + # + # The install scripts are release-scripts/'s: install.sh (pyo.dev/install.sh + # is the latest release's), connect.sh (also for install.sh --uninstall), + # join.sh, sandbox.sh (beside the sandbox files it installs) and cli.sh, + # the CLI's installer. + - name: Upload the release as a draft + id: draft + env: + GH_TOKEN: ${{ github.token }} + run: | + files=( pyo-linux-x86_64 pyo-linux-x86_64.sha256 cli-platforms/pyo-linux-aarch64 @@ -305,11 +314,11 @@ jobs: cli-platforms/pyo-windows-x86_64.exe.sha256 release.json release.json.sigstore.json - deploy/install.sh - deploy/cli.sh - crates/pyo-api/assets/connect.sh - crates/pyo-api/assets/join.sh - deploy/sandbox.sh + release-scripts/install.sh + release-scripts/cli.sh + release-scripts/connect.sh + release-scripts/join.sh + release-scripts/sandbox.sh sandbox/pyo-sandbox-linux-x86_64.tar.gz sandbox/pyo-sandbox-linux-x86_64.tar.gz.sha256 sandbox/pyo-sandbox-linux-aarch64.tar.gz @@ -318,3 +327,37 @@ jobs: sandbox/pyo-sandbox-gvisor-linux-x86_64.tar.gz.sha256 sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz sandbox/pyo-sandbox-gvisor-linux-aarch64.tar.gz.sha256 + ) + if ! draft=$(gh release view "$GITHUB_REF_NAME" --json isDraft --jq .isDraft 2>&1); then + grep -q "release not found" <<<"$draft" || { echo "$draft"; exit 1; } + draft=none + fi + case "$draft" in + false) + echo "::notice::$GITHUB_REF_NAME is published already, so its assets stay as they are." + echo "published=true" >> "$GITHUB_OUTPUT" + ;; + true) + gh release edit "$GITHUB_REF_NAME" --notes-file release-notes.md + gh release upload "$GITHUB_REF_NAME" --clobber "${files[@]}" + ;; + none) + gh release create "$GITHUB_REF_NAME" --draft --verify-tag --title "$GITHUB_REF_NAME" \ + --notes-file release-notes.md "${files[@]}" + ;; + esac + # What the draft holds now: each install script as release.json lists it. + - name: Check the draft's install scripts + if: steps.draft.outputs.published != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "$GITHUB_REF_NAME" --pattern '*.sh' --dir uploaded + jq -r '.scripts | to_entries[] | "\(.value) uploaded/\(.key)"' release.json | sha256sum -c + [ "$(ls uploaded | wc -l)" = "$(jq '.scripts | length' release.json)" ] + - name: Publish the release + if: steps.draft.outputs.published != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: gh release edit "$GITHUB_REF_NAME" --draft=false --latest + diff --git a/RELEASING.md b/RELEASING.md index 07a7510..0a78ff1 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -23,7 +23,8 @@ to `ghcr.io/upsetdev/pyo/{pyo,agent,builder,flows,waf,geoip,db-controller,kv,pos (all from `deploy/images/`, at the same version. The Rust binaries are compiled once, by the `binaries` job, and the Rust images' jobs take them from that run's build cache), -and creates a GitHub release with the CLI and `release.json`. The CLI is +and creates a GitHub release with the CLI, the install scripts and +`release.json`. The CLI is `pyo-linux-x86_64`, `pyo-linux-aarch64`, `pyo-macos-aarch64`, `pyo-macos-x86_64` and `pyo-windows-x86_64.exe`, each with its `.sha256`, plus its installer `deploy/cli.sh`. Linux x86-64's comes from the Rust @@ -84,8 +85,13 @@ The release's text lists the changes since the previous release (see Don't move or reuse a tag. If a release is bad, publish the next version. -Publish releases as stable releases (not drafts or pre-releases): installations -only see GitHub's "latest" release. +The repo has GitHub's immutable releases on: once a release is published, +its assets and its tag can't change. So the workflow uploads the release +as a draft with every asset, checks the install scripts in it, and only +then publishes it, as a stable release marked latest (installations only +see GitHub's "latest" release). If the job fails before that, run it again: +it replaces the draft's assets. A release that's published already is left +as it is. The release's text can still be edited. ## Release notes @@ -168,6 +174,12 @@ release carries it as an asset. Installations find it in **Settings › Updates* - `sandbox`: the SHA-256 of each architecture's sandbox server files. `deploy/sandbox.sh` checks release.json's signature with cosign, as `install.sh` does, then the files against these. +- `scripts`: the SHA-256 of each install script the release publishes + (`install.sh`, `cli.sh`, `connect.sh`, `join.sh`, `sandbox.sh`). The + script writes them to `--scripts `, with this version on the + `pyo:release-version` line of `connect.sh`, `join.sh` and `sandbox.sh`, + and the release uploads them from there. pyo.dev serves only scripts + that match these sums. The release notes list them too. To try it before tagging: @@ -178,7 +190,9 @@ bun run scripts/release-manifest.ts --version 0.1.0 --images ghcr.io/upsetdev/py --mysql-operator sha256:<64 hex> --mysql-backup sha256:<64 hex> \ --postgres-16 sha256:<64 hex> --postgres-17 sha256:<64 hex> --postgres-18 sha256:<64 hex> \ --agent-access "$(cargo run -q -p pyo-agent -- access-hash)" \ - --sandbox-x86_64 <64 hex> --sandbox-aarch64 <64 hex> + --sandbox-x86_64 <64 hex> --sandbox-aarch64 <64 hex> \ + --sandbox-gvisor-x86_64 <64 hex> --sandbox-gvisor-aarch64 <64 hex> \ + --scripts /tmp/release-scripts ``` ### Notices @@ -234,10 +248,31 @@ certificates are valid to 2031), `install.sh` with cosign (the machine's, or a pinned download checked by SHA-256). The images' own signatures aren't verified in the cluster. `pyo-up.sh --release` checks them. +## Install scripts on pyo.dev + +pyo.dev serves the scripts each release publishes, as they are: + +- `https://pyo.dev/