fix(previews): reap unlabelled stages and stop skipped runs cancelling teardown

.github/workflows/ci.yml

@@ -138,7 +138,7 @@ jobs:
               env:
                   CHANGED_FILES: ${{ steps.filter.outputs.base != 'true' && fromJSON(steps.filter.outputs.changed_count) <= 1000 && steps.filter.outputs.changed_files || '' }}
               run: |
-                  bun test scripts/ci/plan-tests.test.ts
+                  bun test scripts/ci/plan-tests.test.ts scripts/lib/pr-state.test.ts
                   matrix="$(bun scripts/ci/plan-tests.ts)"
                   echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
                   echo "has-tests=$(jq '.include | length > 0' <<< "$matrix")" >> "$GITHUB_OUTPUT"

.github/workflows/cleanup-preview-orphans.yml

@@ -13,6 +13,7 @@ on:
 permissions:
     contents: read
     pull-requests: read # sweep checks each pr-<n> branch's PR state
+    actions: read # stage sweep skips a PR whose preview deploy is queued or running
     id-token: write # Infisical OIDC machine-identity auth
 
 concurrency:

.github/workflows/deploy-pr-preview.yml

@@ -15,8 +15,16 @@ on:
             - unlabeled
             - closed
 
+# Only runs that deploy or tear down share the PR's group (same test as deploy-pr-preview's
+# `if`). A run that skips both gets its own group, so a push to an unlabelled PR can't
+# cancel its teardown mid-destroy and leak the stack (#1564).
 concurrency:
-    group: pr-preview-${{ github.event.pull_request.number }}
+    group: >-
+        ${{ ((github.event.action == 'unlabeled' && github.event.label.name == 'preview')
+            || (github.event.action != 'unlabeled'
+                && contains(github.event.pull_request.labels.*.name, 'preview')))
+        && format('pr-preview-{0}', github.event.pull_request.number)
+        || format('pr-preview-noop-{0}', github.run_id) }}
     cancel-in-progress: true
 
 permissions:
@@ -57,8 +65,9 @@ jobs:
         env:
             # Set `PREVIEW_INFISICAL_ENV_SLUG` once a dedicated preview environment exists.
             INFISICAL_ENV_SLUG: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }}
-            # Each live preview holds one of the account's 25 Hyperdrive configs.
-            MAX_LIVE_PREVIEWS: 8
+            # Each live preview holds a VPC in the prd AWS account, whose us-east-1 quota of 5
+            # leaves 3 beside the default and prd VPCs (also 1 of 25 Hyperdrive configs).
+            MAX_LIVE_PREVIEWS: 3
             # `unlabeled` only reaches here for the `preview` label, so both mean teardown.
             TEARDOWN: ${{ (github.event.action == 'closed' || github.event.action == 'unlabeled') && 'true' || 'false' }}
             # Opt-in collector (prd-only otherwise, see `profile.deploys.collector`).

docs/pr-previews.md

@@ -9,7 +9,7 @@ when the label comes off or the PR closes (`.github/workflows/deploy-pr-preview.
 | :------------------------------------------------------ | :----------------------------------------------------- | :------------------------------------------------------------------------------------- |
 | web, api, ai, alerting, chat-bot, electric-sync Workers | `app-pr-<n>`, `api-pr-<n>`, `sync-pr-<n>` `.maple.dev` | Clerk development instance                                                             |
 | Postgres                                                | Neon branch `pr-<n>` of `NEON_PROJECT_ID`              | Migrated by the deploy (`declarePreviewDb` in `alchemy.run.ts`), deleted on destroy    |
-| Hyperdrive                                              | `maple-db-pr-<n>`, shared by every Worker              | One of the account's 25 configs, hence the live-preview cap                            |
+| Hyperdrive                                              | `maple-db-pr-<n>`, shared by every Worker              | One of the account's 25 configs                                                        |
 | Warehouse                                               | Tinybird branch `pr_<n>`                               | Empty; `scripts/tinybird-pr-branch.ts` swaps host, token, signing key and workspace id |
 | Ingest gateway                                          | `ingest-pr-<n>.maple.dev` (ECS on EC2)                 | Reads ingest keys from the preview's own branch                                        |
 | Electric                                                | `electric-pr-<n>.maple.dev` (ECS Fargate)              | Replicates from the branch; electric-sync points at it                                 |
@@ -48,11 +48,15 @@ and fails the job if a Postgres- or warehouse-backed route returns a 5xx. Withou
 
 ## Limits and cost
 
-- At most `MAX_LIVE_PREVIEWS` (8) open PRs may carry `preview`; the deploy refuses the next one.
+- At most `MAX_LIVE_PREVIEWS` (3) open PRs may carry `preview`; the deploy refuses the next one.
+  Each preview's ingest needs a VPC in the prd AWS account, and us-east-1 allows 5: the default
+  VPC and prd hold 2.
 - A preview runs an EC2 host, two ALBs and a Fargate task while it is live. Neon compute stays
   awake while Electric holds its replication slot.
-- `cleanup-preview-orphans.yml` destroys the stage of any closed PR whose resources are still
-  around (`scripts/preview-stage-orphan-sweep.ts`), then sweeps Tinybird, Workers and Hyperdrive.
+- `cleanup-preview-orphans.yml` sweeps Tinybird, destroys the stage of any PR that is closed or
+  no longer carries `preview` (`scripts/preview-stage-orphan-sweep.ts`), then sweeps Queues,
+  Workers and Hyperdrive. That also reaps a stage whose teardown was cancelled. A stage with a
+  queued or running preview deploy is skipped until the next sweep.
 
 ## One-time setup
 

scripts/cloudflare-worker-orphan-sweep.ts

@@ -26,7 +26,7 @@
  * both binds and consumes its stage's `maple-vcs-sync-pr-<n>` /
  * `maple-planetscale-webhooks-pr-<n>` queues (apps/api/src/resources/queues.ts), so
  * the sweep breaks the cycle in three passes:
- *   1. detach the consumer registrations of closed-PR queues,
+ *   1. detach the consumer registrations of retired-PR queues,
  *   2. delete the workers (their producer bindings die with them),
  *   3. delete the now-unreferenced queues.
  *
@@ -50,7 +50,7 @@
  * GITHUB_TOKEN/GH_TOKEN for the PR-state gate.
  */
 
-import { fetchPrState, hasPrStateCredentials, type PrState } from "./lib/pr-state.ts"
+import { fetchPreviewState, hasPrStateCredentials, type PreviewState } from "./lib/pr-state.ts"
 
 const FAILURE = 1
 
@@ -122,18 +122,18 @@ const main = async (): Promise<void> => {
 
 	// One PR-state lookup per PR number, shared across both passes — a stage
 	// leaks ~8 workers + 2 queues and they all share the same verdict.
-	const stateByPr = new Map<string, PrState>()
-	const prState = async (prNumber: string): Promise<PrState> => {
+	const stateByPr = new Map<string, PreviewState>()
+	const prState = async (prNumber: string): Promise<PreviewState> => {
 		let state = stateByPr.get(prNumber)
 		if (!state) {
-			state = await fetchPrState(prNumber)
+			state = await fetchPreviewState(prNumber)
 			stateByPr.set(prNumber, state)
 		}
 		return state
 	}
 	const failures: string[] = []
 
-	// Pass 1: detach consumer registrations from closed-PR queues, so the
+	// Pass 1: detach consumer registrations from retired-PR queues, so the
 	// worker deletions in pass 2 stop failing with 10064. The queues
 	// themselves are deleted in pass 3, after the workers' producer bindings
 	// are gone (deleting them here fails with 11005).
@@ -164,8 +164,8 @@ const main = async (): Promise<void> => {
 	for (const { id, name, match } of queueCandidates) {
 		const prNumber = match[2] as string
 		const state = await prState(prNumber)
-		if (state === "open") {
-			console.log(`… keeping ${name} (PR #${prNumber} is open)`)
+		if (state === "live") {
+			console.log(`… keeping ${name} (PR #${prNumber} has a live preview)`)
 			continue
 		}
 		if (state === "unknown") {
@@ -190,7 +190,9 @@ const main = async (): Promise<void> => {
 				failures.push(name)
 				continue
 			}
-			console.log(`… detaching consumer ${consumerRef} from ${name} (PR #${prNumber} is closed)`)
+			console.log(
+				`… detaching consumer ${consumerRef} from ${name} (PR #${prNumber} is closed or unlabelled)`,
+			)
 			const detached = await cfRequest(
 				token,
 				`/accounts/${accountId}/queues/${id}/consumers/${consumerRef}`,
@@ -227,15 +229,15 @@ const main = async (): Promise<void> => {
 	for (const { name, match } of candidates) {
 		const prNumber = match[2] as string
 		const state = await prState(prNumber)
-		if (state === "open") {
-			console.log(`… keeping ${name} (PR #${prNumber} is open)`)
+		if (state === "live") {
+			console.log(`… keeping ${name} (PR #${prNumber} has a live preview)`)
 			continue
 		}
 		if (state === "unknown") {
 			console.log(`⚠ skipping ${name} (PR #${prNumber} state unknown)`)
 			continue
 		}
-		console.log(`… deleting orphan ${name} (PR #${prNumber} is closed)`)
+		console.log(`… deleting orphan ${name} (PR #${prNumber} is closed or unlabelled)`)
 		const removed = await cfRequest(token, `/accounts/${accountId}/workers/scripts/${name}?force=true`, {
 			method: "DELETE",
 		})

scripts/hyperdrive-orphan-sweep.ts

@@ -31,7 +31,7 @@
  * GITHUB_TOKEN/GH_TOKEN for the PR-state gate.
  */
 
-import { fetchPrState, hasPrStateCredentials } from "./lib/pr-state.ts"
+import { fetchPreviewState, hasPrStateCredentials } from "./lib/pr-state.ts"
 
 const FAILURE = 1
 
@@ -114,16 +114,16 @@ const main = async (): Promise<void> => {
 
 	const failures: string[] = []
 	for (const { id, name, match } of candidates) {
-		const state = await fetchPrState(match[1] as string)
-		if (state === "open") {
-			console.log(`… keeping ${name} (PR #${match[1]} is open)`)
+		const state = await fetchPreviewState(match[1] as string)
+		if (state === "live") {
+			console.log(`… keeping ${name} (PR #${match[1]} has a live preview)`)
 			continue
 		}
 		if (state === "unknown") {
 			console.log(`⚠ skipping ${name} (PR #${match[1]} state unknown)`)
 			continue
 		}
-		console.log(`… deleting orphan ${name} (PR #${match[1]} is closed)`)
+		console.log(`… deleting orphan ${name} (PR #${match[1]} is closed or unlabelled)`)
 		const removed = await cfRequest(token, `/accounts/${accountId}/hyperdrive/configs/${id}`, {
 			method: "DELETE",
 		})

scripts/lib/pr-state.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, test } from "bun:test"
+import { previewStateOf } from "./pr-state.ts"
+
+describe("previewStateOf", () => {
+	test("an open PR keeps its preview only while it carries the label", () => {
+		expect(previewStateOf({ state: "open", labels: [{ name: "bug" }, { name: "preview" }] })).toBe("live")
+		expect(previewStateOf({ state: "open", labels: [{ name: "preview:collector" }] })).toBe("retired")
+		expect(previewStateOf({ state: "open", labels: [] })).toBe("retired")
+	})
+
+	test("a closed PR is retired even if the label is still on it", () => {
+		expect(previewStateOf({ state: "closed", labels: [{ name: "preview" }] })).toBe("retired")
+	})
+
+	test("an unrecognised payload is unknown, so nothing is deleted", () => {
+		expect(previewStateOf({})).toBe("unknown")
+		expect(previewStateOf({ state: "draft", labels: [{ name: "preview" }] })).toBe("unknown")
+	})
+})

scripts/lib/pr-state.ts

@@ -1,21 +1,74 @@
 /**
- * The PR-state gate shared by the preview orphan sweeps (cleanup-preview-orphans.yml).
+ * The preview-state gate shared by the preview orphan sweeps (cleanup-preview-orphans.yml).
  *
  * Auth: GITHUB_REPOSITORY + GITHUB_TOKEN/GH_TOKEN.
  */
 
-export type PrState = "open" | "closed" | "unknown"
+/** `live`: open and labelled `preview`. `retired`: closed, or open without the label. */
+export type PreviewState = "live" | "retired" | "unknown"
+
+export const PREVIEW_LABEL = "preview"
 
 export const hasPrStateCredentials = (): boolean =>
 	Boolean(
 		process.env.GITHUB_REPOSITORY?.trim() && (process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN)?.trim(),
 	)
 
+/** An open PR that lost its label is retired too: a cancelled teardown leaves its stage up. */
+export const previewStateOf = (pull: {
+	state?: string
+	labels?: ReadonlyArray<{ name?: string }>
+}): PreviewState => {
+	if (pull.state === "closed") return "retired"
+	if (pull.state !== "open") return "unknown"
+	return pull.labels?.some((label) => label.name === PREVIEW_LABEL) ? "live" : "retired"
+}
+
+/**
+ * Whether a preview deploy run for this PR is queued or running, so a sweep never destroys a
+ * stage that a restored label is redeploying. "unknown" when the API call fails.
+ */
+export const hasActivePreviewRun = async (prNumber: string): Promise<boolean | "unknown"> => {
+	const repo = process.env.GITHUB_REPOSITORY?.trim()
+	const token = (process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN)?.trim()
+	if (!repo || !token) return "unknown"
+	try {
+		for (const status of ["queued", "in_progress"]) {
+			const response = await fetch(
+				`https://api.github.com/repos/${repo}/actions/workflows/deploy-pr-preview.yml/runs?status=${status}&per_page=100`,
+				{
+					headers: {
+						Authorization: `Bearer ${token}`,
+						Accept: "application/vnd.github+json",
+						"X-GitHub-Api-Version": "2022-11-28",
+					},
+				},
+			)
+			if (!response.ok) {
+				console.log(`⚠ Could not list ${status} preview runs (HTTP ${response.status})`)
+				return "unknown"
+			}
+			const { workflow_runs: runs = [] } = (await response.json()) as {
+				workflow_runs?: ReadonlyArray<{ pull_requests?: ReadonlyArray<{ number?: number }> }>
+			}
+			if (runs.some((run) => run.pull_requests?.some((pull) => String(pull.number) === prNumber))) {
+				return true
+			}
+		}
+		return false
+	} catch (error) {
+		console.log(
+			`⚠ Could not list preview runs (${error instanceof Error ? error.message : String(error)})`,
+		)
+		return "unknown"
+	}
+}
+
 /**
- * PR state via the GitHub REST API. Returns "unknown" when the API call
- * fails; callers must treat "unknown" as "don't delete", never as "closed".
+ * Preview state via the GitHub REST API. Returns "unknown" when the API call
+ * fails; callers must treat "unknown" as "don't delete", never as "retired".
  */
-export const fetchPrState = async (prNumber: string): Promise<PrState> => {
+export const fetchPreviewState = async (prNumber: string): Promise<PreviewState> => {
 	const repo = process.env.GITHUB_REPOSITORY?.trim()
 	const token = (process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN)?.trim()
 	if (!repo || !token) return "unknown"
@@ -31,8 +84,9 @@ export const fetchPrState = async (prNumber: string): Promise<PrState> => {
 			console.log(`⚠ Could not look up PR #${prNumber} state (HTTP ${response.status})`)
 			return "unknown"
 		}
-		const parsed = (await response.json()) as { state?: string }
-		return parsed.state === "open" ? "open" : parsed.state === "closed" ? "closed" : "unknown"
+		return previewStateOf(
+			(await response.json()) as { state?: string; labels?: ReadonlyArray<{ name?: string }> },
+		)
 	} catch (error) {
 		console.log(
 			`⚠ Could not look up PR #${prNumber} state (${error instanceof Error ? error.message : String(error)})`,

scripts/preview-stage-orphan-sweep.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 /**
- * Tear down whole PR-preview stages whose PR is closed.
+ * Tear down whole PR-preview stages whose PR is closed or lost its `preview` label.
  *
  *   bun scripts/preview-stage-orphan-sweep.ts [--dry-run]
  *
@@ -14,7 +14,7 @@
  *   - Neon branches named `pr-<n>` in NEON_PROJECT_ID (alchemy.run.ts, declarePreviewDb),
  *   - R2 buckets named `maple-replay-blobs-pr-<n>`, which alchemy RETAINS on destroy
  *     (apps/api/src/resources/replay-blobs.ts), so every deployed stage keeps one.
- * For each PR the GitHub API affirmatively reports closed (unknown/open → keep), it runs
+ * For each PR the GitHub API affirmatively reports retired (unknown/live → keep), it runs
  * `alchemy destroy --yes --stage pr-<n>`, then deletes the Neon branch and empties and
  * deletes the R2 bucket. If the destroy fails, the branch and bucket stay so the next run
  * rediscovers the stage and retries. Stages run one at a time; a failure does not stop
@@ -31,7 +31,7 @@
 
 import { spawnSync } from "node:child_process"
 import { join } from "node:path"
-import { fetchPrState, hasPrStateCredentials } from "./lib/pr-state.ts"
+import { fetchPreviewState, hasActivePreviewRun, hasPrStateCredentials } from "./lib/pr-state.ts"
 
 const FAILURE = 1
 const DRY_RUN = process.argv.includes("--dry-run")
@@ -292,18 +292,27 @@ const main = async (): Promise<void> => {
 	for (const prNumber of prNumbers) {
 		const stage = `pr-${prNumber}`
 		const markers = candidates.get(prNumber) ?? []
-		const state = await fetchPrState(prNumber)
-		if (state === "open") {
-			console.log(`… keeping ${stage} (PR #${prNumber} is open)`)
+		const state = await fetchPreviewState(prNumber)
+		if (state === "live") {
+			console.log(`… keeping ${stage} (PR #${prNumber} has a live preview)`)
 			continue
 		}
 		if (state === "unknown") {
 			console.log(`⚠ skipping ${stage} (PR #${prNumber} state unknown)`)
 			continue
 		}
+		const activeRun = await hasActivePreviewRun(prNumber)
+		if (activeRun !== false) {
+			console.log(
+				`⚠ skipping ${stage} (${activeRun === true ? `a preview run for PR #${prNumber} is active` : "preview runs unknown"})`,
+			)
+			continue
+		}
 		const bucket = markers.find((marker) => REPLAY_BUCKET_PATTERN.test(marker))
 		const branchId = neonBranchIdByPr.get(prNumber)
-		console.log(`… tearing down ${stage} (PR #${prNumber} is closed; found ${markers.join(", ")})`)
+		console.log(
+			`… tearing down ${stage} (PR #${prNumber} is closed or unlabelled; found ${markers.join(", ")})`,
+		)
 		if (DRY_RUN) {
 			console.log(
 				`  would destroy ${stage}${branchId ? `, delete Neon branch ${branchId}` : ""}${bucket ? `, empty and delete ${bucket}` : ""}`,