Close ten engineering-standard conformance gaps

.github/pull_request_template.md

@@ -0,0 +1,48 @@
+## What and why
+
+<!-- What changes, and what problem it solves. Link the issue or ADR. -->
+
+## Contract classification (VER-3)
+
+<!--
+Required on any PR that touches a contract. flowctl's contracts are listed in
+docs/engineering-profile.md §2: CLI flags and arguments, exit codes, the
+--output json/ndjson shapes, the pipeline config format, the gRPC API, the
+event envelope, and the on-disk run store.
+
+Delete this section only if the PR touches none of them.
+-->
+
+- [ ] **None** — this PR touches no contract.
+- [ ] **Additive** — new optional field, new flag, new subcommand. Consumers that
+      ignore unknown fields are unaffected.
+- [ ] **Deprecating** — something is scheduled for removal. Notice, migration note
+      and removal date below.
+- [ ] **Breaking** — removed or renamed field, changed type, changed exit code.
+      Needs an ADR and a migration window.
+- [ ] **Semantic** — same wire format, different meaning. Treated as breaking:
+      version bump, parity test, replay decision.
+
+ADR: <!-- docs/adr/NNNN-....md, or "n/a" -->
+
+Migration notes: <!-- what a consumer must do, or "n/a" -->
+
+## Testing
+
+<!--
+- A bug fix needs a test that fails without the fix.
+- Concurrent code needs `go test -race`.
+- Parsers of external input need fuzz tests.
+- Prefer real captured data; invented data is for injecting failures only.
+-->
+
+- [ ] `make lint` passes
+- [ ] `make test` passes
+- [ ] `make test-race` passes
+
+## Checklist
+
+- [ ] Commit messages follow Conventional Commits
+- [ ] `CHANGELOG.md` updated if this is user-visible
+- [ ] Docs updated (README, `docs/`, `--help` text)
+- [ ] No secrets, credentials or customer data in the diff

.github/workflows/pr-tests.yml

@@ -9,73 +9,201 @@ on:
 permissions:
   contents: read
 
+# The merge gate from §9 of the OBSRVR Engineering Standard. Every job here
+# blocks the merge:
+#
+#   lint        gofmt, go vet, staticcheck
+#   test        build and the full suite
+#   race        go test -race                        (TEST-4)
+#   vulncheck   govulncheck                          (SEC-9)
+#   secrets     gitleaks                             (SEC-6)
+#   proto       buf lint and buf breaking            (VER-5)
+#   smoke       onboarding smoke tests
 jobs:
-  vet:
-    name: Vet
+  lint:
+    name: Lint
     runs-on: ubuntu-latest
-    timeout-minutes: 20
-
+    timeout-minutes: 15
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
+      - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           cache: true
 
-      - name: Download dependencies
-        run: go mod download
-
-      - name: Run go vet
+      - name: Check formatting
         run: |
-          PKGS=$(find . \
-            \( -path './.git' -o -path './data' -o -path './bin' -o -path './internal/testfixtures/source' -o -path './internal/testfixtures/processor' -o -path './internal/testfixtures/sink' \) -prune \
-            -o -name '*.go' -print | xargs -r -n1 dirname | sort -u | sed 's#^#./#' | sed 's#\.//#./#')
-          go vet $PKGS
+          unformatted=$(gofmt -l . | grep -v '^data/' || true)
+          if [ -n "$unformatted" ]; then
+            echo "These files need gofmt:"
+            echo "$unformatted"
+            exit 1
+          fi
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: staticcheck
+        uses: dominikh/staticcheck-action@v1
+        with:
+          version: latest
+          install-go: false
 
   test:
     name: Build and test
     runs-on: ubuntu-latest
     timeout-minutes: 20
-
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
+      - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           cache: true
 
       - name: Download dependencies
         run: go mod download
 
+      - name: Verify go.mod is tidy
+        run: |
+          go mod tidy
+          git diff --exit-code go.mod go.sum || {
+            echo "go.mod/go.sum are not tidy; run 'go mod tidy' and commit the result"
+            exit 1
+          }
+
       - name: Verify build
         run: make build
 
       - name: Run test suite
         run: make test
 
+  race:
+    name: Race detector
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      # TEST-4. The pipeline runner and control plane are heavily concurrent;
+      # this job found a real escaped-pointer race in GetComponentStatus the
+      # first time it was switched on.
+      - name: Run tests with the race detector
+        run: make test-race
+
+  vulncheck:
+    name: Vulnerability scan
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      # SEC-9: fail on reachable known vulnerabilities. Findings that genuinely
+      # do not affect flowctl are exempted one at a time, with a written
+      # justification, in .vex/flowctl.openvex.json (SEC-12). The script fails
+      # on anything not covered there, so the gate cannot rot into a no-op.
+      - name: govulncheck
+        run: ./scripts/govulncheck.sh
+
+  secrets:
+    name: Secret scan
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # gitleaks needs history to scan the commits a PR adds.
+          fetch-depth: 0
+
+      # SEC-6: secrets must never reach git.
+      #
+      # The gitleaks *binary* is open source; only the GitHub Action wrapper
+      # requires a paid licence for organization repos. Running the binary
+      # directly keeps the gate without the licence.
+      - name: Install gitleaks
+        env:
+          GITLEAKS_VERSION: 8.21.2
+        run: |
+          curl -sSfL -o /tmp/gitleaks.tar.gz \
+            "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
+          tar -xzf /tmp/gitleaks.tar.gz -C /tmp gitleaks
+          sudo install -m 0755 /tmp/gitleaks /usr/local/bin/gitleaks
+          gitleaks version
+
+      - name: Scan the working tree
+        run: gitleaks dir . --redact --verbose
+
+      - name: Scan the commits this PR adds
+        if: github.event_name == 'pull_request'
+        run: |
+          gitleaks git . --redact --verbose \
+            --log-opts="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
+
+  proto:
+    name: Protobuf contracts
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: buf lint
+        run: buf lint
+
+      # VER-5: field numbers and wire types must not change, and removed
+      # numbers must stay reserved. Compared against the last release rather
+      # than against main, because the released tag is what deployed
+      # components are actually speaking.
+      - name: buf breaking
+        run: |
+          latest=$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || true)
+          if [ -z "$latest" ]; then
+            echo "No release tag found; skipping the breaking-change check."
+            exit 0
+          fi
+          echo "Comparing protobuf against $latest"
+          buf breaking --against ".git#tag=$latest"
+
+      # DATA-5: generated code is regenerated and checked, never hand-edited.
+      - name: Verify generated code is current
+        run: |
+          buf generate
+          git diff --exit-code -- proto/ || {
+            echo "Generated protobuf code is stale; run 'make generate' and commit the result."
+            echo "Plugin versions are pinned in buf.gen.yaml, so this diff means"
+            echo "the .proto files changed without the generated code being updated."
+            exit 1
+          }
+
   smoke:
     name: Onboarding smoke tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
+      - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           cache: true
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 

.github/workflows/release.yml

@@ -7,6 +7,9 @@ on:
 
 permissions:
   contents: write
+  # SEC-10: lets cosign exchange a GitHub OIDC token for a Sigstore
+  # certificate, so releases are signed without a long-lived key.
+  id-token: write
 
 jobs:
   release:
@@ -28,9 +31,34 @@ jobs:
       - name: Download dependencies
         run: go mod download
 
+      # Release gate (§9): the merge gate plus the extras below. These repeat
+      # the PR jobs because a tag can be pushed to a commit that never went
+      # through a PR.
+      - name: Lint
+        run: make lint
+
       - name: Run tests
         run: make test
 
+      - name: Run tests with the race detector
+        run: make test-race
+
+      - name: Vulnerability scan
+        run: ./scripts/govulncheck.sh
+
+      # BUILD-3: a release that cannot be rebuilt byte-for-byte from its tag
+      # is not a release.
+      - name: Verify the build is reproducible
+        run: make verify-repro
+
+      # SEC-10: SBOM generation and keyless signing, both driven by
+      # .goreleaser.yaml.
+      - name: Install syft
+        uses: anchore/sbom-action/download-syft@v0
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v6
         with: