diff --git a/.agentic-loop.yml b/.agentic-loop.yml index cdf875d0..255a1f12 100644 --- a/.agentic-loop.yml +++ b/.agentic-loop.yml @@ -52,9 +52,8 @@ verification: commands: - "npx tsc --noEmit" - "npm run lint" - - "npm run test:run" + - 'npm run quality:verify -- --base "${FALLOW_AUDIT_BASE:?Set the exact approved integration base SHA}"' - "npm run build" - - 'FALLOW_AUDIT_BASE="${FALLOW_AUDIT_BASE:?Set the exact approved integration base SHA}" npm run fallow:audit' - "git diff --check" human: diff --git a/.codex/agents/delivery-worker.toml b/.codex/agents/delivery-worker.toml index 2b66ce06..c0dc7ccf 100644 --- a/.codex/agents/delivery-worker.toml +++ b/.codex/agents/delivery-worker.toml @@ -48,11 +48,12 @@ revision, source/test scope, exclusions, result, evidence path and provenance; and the separate Standards and Spec review inputs. A body digest is computed from the parsed GitHub body bytes without trimming or adding a CLI newline. Use the exact starting origin/develop SHA in -FALLOW_AUDIT_BASE= npm run fallow:audit and verify Fallow's reported range. +npm run quality:verify -- --base and verify the retained bundle with +quality:check as described in docs/agents/quality-verification.md. When every linked issue's acceptance criteria are satisfied, run relevant -focused checks plus npx tsc --noEmit, npm run lint, npm run test:run, npm run -build, FALLOW_AUDIT_BASE= npm run fallow:audit, +focused checks plus npx tsc --noEmit, npm run lint, +npm run quality:verify -- --base , npm run build, and git diff --check. Keep the PR in draft, do not merge, and hand the complete PR diff and exact head to pr-finalizer. Your task ends at that handoff unless the workflow director explicitly returns a bounded diff --git a/.codex/agents/pr-finalizer.toml b/.codex/agents/pr-finalizer.toml index 33fa7bf8..9fe93607 100644 --- a/.codex/agents/pr-finalizer.toml +++ b/.codex/agents/pr-finalizer.toml @@ -30,18 +30,17 @@ docs/agents/handoffs.md. Report Standards and Spec conclusions separately. Finalizer findings name severity, evidence, affected requirement or standard, and status; the Director verifies each finding and adds the disposition and rationale. Use -FALLOW_AUDIT_BASE= npm run fallow:audit and -verify the report range; a bare, branch-upstream, or self-comparison audit is -not delivery evidence. +npm run quality:check -- --base --head +and docs/agents/quality-verification.md to accept the retained bundle. You may edit the PR branch only for a correction or simplification that is directly required by the approved tickets, stays inside their write boundaries and non-goals, preserves public behavior unless repairing a specified defect, and has a convincing verification path. Prefer direct, boring code. Commit all such finalizer changes in one focused commit, push the same branch, and rerun -the affected checks plus npx tsc --noEmit, npm run lint, npm run test:run, npm -run build, FALLOW_AUDIT_BASE= npm run -fallow:audit, and git diff --check. +the affected checks plus npx tsc --noEmit, npm run lint, +npm run quality:verify -- --base , +npm run build, and git diff --check. Return exactly one verdict: diff --git a/.codex/agents/workflow-director.toml b/.codex/agents/workflow-director.toml index 7b807f31..342b634c 100644 --- a/.codex/agents/workflow-director.toml +++ b/.codex/agents/workflow-director.toml @@ -40,9 +40,9 @@ configuration, revision, measured scopes, exclusions, result, evidence location, and provenance; and separate Standards and Spec findings with an explicit disposition and rationale. Compare requirement content when its body or governing decision changes. A changed head, wrong scope, absent evidence, -or wrong checkout invalidates affected approval and evidence. For local Fallow, -pin FALLOW_AUDIT_BASE to the exact starting origin/develop SHA and verify the -reported comparison range. +or wrong checkout invalidates affected approval and evidence. Use +docs/agents/quality-verification.md and quality:check with the exact starting +origin/develop base and candidate head to accept retained quality evidence. Every delivery brief names the outcome, acceptance criteria, non-goals, write boundaries, contract and runtime surfaces, decision and safety gates, diff --git a/.fallowrc.json b/.fallowrc.json index 91c11277..95c9b534 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -25,5 +25,75 @@ "createRenderRoot", "shouldUpdate", "update" - ] + ], + "boundaries": { + "zones": [ + { + "name": "browser", + "patterns": ["src/**"] + }, + { + "name": "shared", + "patterns": ["shared/src/**"] + }, + { + "name": "server", + "patterns": ["server/src/**"] + }, + { + "name": "worker", + "patterns": ["workers/telemetry/src/**"] + }, + { + "name": "tooling", + "patterns": ["scripts/**"] + }, + { + "name": "root-tests", + "patterns": ["tests/**"] + }, + { + "name": "server-tests", + "patterns": ["server/tests/**"] + }, + { + "name": "shared-tests", + "patterns": ["shared/tests/**"] + } + ], + "rules": [ + { + "from": "browser", + "allow": ["browser", "shared"] + }, + { + "from": "shared", + "allow": ["shared"] + }, + { + "from": "server", + "allow": ["server", "shared"] + }, + { + "from": "worker", + "allow": ["worker", "shared"] + }, + { + "from": "tooling", + "allow": ["tooling", "shared"] + }, + { + "from": "root-tests", + "allow": ["browser", "shared", "worker", "tooling", "root-tests"] + }, + { + "from": "server-tests", + "allow": ["server", "shared", "server-tests"] + }, + { + "from": "shared-tests", + "allow": ["shared", "shared-tests"] + } + ] + } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c0adc8a..5fa8e781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,17 @@ permissions: contents: read pull-requests: read +env: + QUALITY_BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + QUALITY_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + jobs: checks: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false @@ -28,14 +33,14 @@ jobs: - name: Install Chromium for Browser Mode run: npx playwright install --with-deps --only-shell chromium - - name: Shared workspace checks - run: npm run shared:check + - name: Shared typecheck + run: npm run shared:typecheck - - name: Server workspace checks - run: npm run server:check + - name: Server typecheck and lint + run: npm run server:typecheck && npm run server:lint - - name: Telemetry worker checks - run: npm run telemetry:typecheck && npm run telemetry:test && npm run telemetry:dry-run + - name: Telemetry worker typecheck and dry run + run: npm run telemetry:typecheck && npm run telemetry:dry-run - name: Typecheck run: npx tsc --noEmit @@ -43,32 +48,34 @@ jobs: - name: Lint run: npm run lint - - name: Tests - run: npm run test:run + - name: Quality evidence + run: | + npm run quality:verify -- --base "$QUALITY_BASE" + npm run quality:check -- --base "$QUALITY_BASE" --head "$QUALITY_HEAD" - - name: Browser tests - run: npm run test:browser + - name: Retain quality evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: quality-${{ env.QUALITY_HEAD }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .fallow/quality + include-hidden-files: true + if-no-files-found: error + retention-days: 14 - name: Build run: npm run build - - name: Fallow audit - uses: fallow-rs/fallow@v2 - with: - command: audit - version: 3.2.0 - artifacts-dir: .fallow/ci - no-cache: true - server-container: runs-on: ubuntu-latest timeout-minutes: 15 env: - PIXEL_FORGE_IMAGE_REVISION: ${{ github.sha }} + PIXEL_FORGE_IMAGE_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} PIXEL_FORGE_SMOKE_PROJECT_NAME: pixel-forge-container-smoke-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Build and smoke the linux/amd64 server image diff --git a/AGENTS.md b/AGENTS.md index 586e8fbc..c4094039 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,14 +124,15 @@ failures, and a regression test for each confirmed bug. or externally visible identifiers that lack a compatibility path. Preserve the existing contract or add an explicit versioned migration. +Before collecting or accepting quality evidence, read +[`docs/agents/quality-verification.md`](docs/agents/quality-verification.md). Before handoff, run the relevant focused checks plus: ```sh npx tsc --noEmit npm run lint -npm run test:run +npm run quality:verify -- --base npm run build -npm run fallow:audit git diff --check ``` diff --git a/docs/agents/change-map.md b/docs/agents/change-map.md index 1ef0cb10..06384a38 100644 --- a/docs/agents/change-map.md +++ b/docs/agents/change-map.md @@ -65,7 +65,7 @@ the listed tests and runner still cover the intended behavior. - **Interface:** [`docs/operations/codex-agent-workflow.md`](../../docs/operations/codex-agent-workflow.md) is the canonical policy; [`AGENTS.md`](../../AGENTS.md), [`.agentic-loop.yml`](../../.agentic-loop.yml), and [`.codex/agents/`](../../.codex/agents/) provide entry, command, and role configuration. - **Callers:** Workflow Director/Sol shapes and reconciles GitHub state; delivery-worker/Luna implements one approved delivery slice; pr-finalizer/Terra reviews the complete draft PR. GitHub issues, PRs, checks, and retained artifacts are the durable readers and writers. -- **Tests:** `.agentic-loop.yml` configures repository gates; focused Markdown/configuration checks validate links and profile parsing, while the application and workspace runners above prove product behavior. `npm run fallow:audit` must use the exact handed-off `FALLOW_AUDIT_BASE` and record its report scope. +- **Tests:** `.agentic-loop.yml` configures repository gates; focused Markdown/configuration checks validate links and profile parsing, while the application and workspace runners above prove product behavior. `npm run quality:verify -- --base ` collects the five profiles and constraints through [`scripts/quality-evidence.mjs`](../../scripts/quality-evidence.mjs). Public refusal exercises run with `npm run quality:test`; see [quality verification](quality-verification.md) before collection or downloaded-bundle acceptance. - **Governing:** Root [`AGENTS.md`](../../AGENTS.md), [`docs/agents/issue-tracker.md`](../../docs/agents/issue-tracker.md), [`docs/agents/triage-labels.md`](../../docs/agents/triage-labels.md), and the live delivery issue govern readiness, task order, evidence, review, and the explicit human merge gate. Browser, shared workspace, server integration, and telemetry Worker commands diff --git a/docs/agents/handoffs.md b/docs/agents/handoffs.md index 146aca35..d1a17855 100644 --- a/docs/agents/handoffs.md +++ b/docs/agents/handoffs.md @@ -52,6 +52,11 @@ can review the new identity. ## Verification record +For collection, downloaded-bundle acceptance, and finalizer rechecks, follow +[quality verification](quality-verification.md). Record the bundle path, manifest +and certification digests, five separate producer scopes, and CI artifact ID, +digest and expiry alongside the command records below. + Each command is an attributable record, not just a pass label: | Field | Required value | diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md new file mode 100644 index 00000000..44748f9c --- /dev/null +++ b/docs/agents/quality-verification.md @@ -0,0 +1,159 @@ +# Quality evidence + +Before collecting delivery evidence, accepting a downloaded bundle, or finalizing +its PR, use these commands from the exact clean candidate checkout: + +```sh +npm run quality:verify -- --base +npm run quality:check -- --base --head +npm run quality:check -- --base --head --bundle +``` + +`quality:verify` invalidates the previous certification before starting a +producer. It builds the shared and server packages, runs five coverage profiles, +executes the shared Node import smoke, checks runtime imports, and reconciles +Fallow reports. It certifies only after all required evidence is valid and the +checkout still has the same HEAD and input hashes. Failure leaves logs and no +certification. `quality:check` reads existing evidence without modifying it or +rerunning producers. + +Run `npm run quality:test` for the focused public-command exercises. These use +small isolated Git repositories and installed Vitest, Chromium, Wrangler, +TypeScript, ESLint and Fallow commands. The root test selection also includes +these tests; their child repositories contain small fixture tests, so they do +not recursively execute the product suite. + +## Producers and measured scope + +The executable inventory is [config.mjs](../../scripts/quality/config.mjs). +The manifest records its resolved source and test paths, configuration hashes, +commands, environment, producer root, tools and results. + +| Profile | Test selection | Production source | Runtime | +| --------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------ | +| `root` | All `tests/**/*.test.ts` except `tests/browser/**` | `src/**/*.ts`, excluding declarations, source test files and `src/main.ts` | Node/happy-dom | +| `browser` | All `tests/browser/**/*.test.ts` | `src/**/*.ts`, excluding declarations; includes `main.ts` | Chromium through Playwright | +| `shared` | `tests/shared/project-contract.test.ts` and `product-events.test.ts` | `shared/src/**/*.ts`, excluding declarations | Node/happy-dom | +| `server` | All `server/tests/**/*.test.ts` except database and storage suites | `server/src/**/*.ts`, excluding declarations | Node | +| `worker` | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding declarations | Node/happy-dom with mocked Worker bindings | + +The root coverage command uses one Vitest worker so its product files and CLI +fixtures do not overlap. This controls this runner’s concurrency; other local +processes can still compete for CPU. Test selection and product timeouts stay +unchanged. The limit is part of the recorded and accepted command. + +Each profile uses the installed Vitest V8 provider and emits its own +Istanbul-shaped JSON map. Every declared source file must appear, including +zero-hit files and empty type-only or barrel maps. The five maps are independent; +there is no combined percentage or coverage threshold. + +`scripts/**` is outside all five coverage maps. The CLI exercises prove tooling +behavior, but they do not produce measured coverage for these modules. Fallow +function scores for tooling therefore retain estimated provenance. + +Absolute includes prevent Vitest 4's substring glob matching from collecting +`shared/src` and Worker files in the root profile. The public fixture preserves +this regression: its root tests import all three areas, but each profile keeps +its declared scope. `excludeAfterRemap` also applies exclusions after source-map +translation. V8 can serialize an unbounded end column as `null`; this remains a +valid location, while hit counts must be nonnegative integers. The remapper also +emits `{start: {}, end: {}}` for the second location of a two-way `if` branch +with no explicit `else`; only that precise implicit-branch sentinel is accepted. + +The Worker command runs from the root npm workspace caller, generates its types, +and records root `INIT_CWD`. It does not measure workerd or deployed bindings. +The plain Node shared import smoke is executed but not covered. Server process +and compatibility subprocesses are likewise outside their parent's V8 map. +Database and object-storage integration suites remain separate provisioned +commands; unit coverage does not imply those integrations ran. Provider and +matching limits are detailed in the +[verified research note](../../specs/research/clean-code-quality-evidence.md). + +## Bundle and acceptance + +Generated evidence stays under ignored `.fallow/quality/`: + +- `identity.json` identifies base/head, clean state, producer root and OS, + Node/npm/tool versions, and sorted hashes of tracked and nonignored untracked + inputs. Configuration and lock files are part of that input inventory. +- `profiles.json` records the five resolved scopes and configuration identities. +- `commands/` retains each command's explicit environment, result, duration, + working directory and raw log. It never serializes the ambient environment. +- Each profile directory keeps `tests.json`, `coverage-final.json`, + `coverage-summary.json`, and `fallow.json`. +- `audit.json`, `audit.sarif.json`, `boundaries.json`, and + `boundary-inventory.json` retain the independent Fallow outputs. +- `review.json` separates blockers from the advisory catalogue. +- `manifest.json` binds records and artifact hashes; `certification.json` binds + its digest to the exact base/head. `failure.log` explains a failed collection. + +Acceptance derives required commands, profile files and configuration from the +expected checkout. It checks artifact hashes, separate and embedded records, +actual executed test files, coverage schema and complete source inventories, +installed tool and registry identity, and the Fallow policy result. Missing, +failed, stale, wrong-profile, empty or foreign evidence fails closed. + +For downloaded evidence, retain the original `producerRoot`. It translates +absolute coverage paths; it never permits reading artifacts outside the selected +bundle. Evidence can move between the verified macOS developer host and Linux CI. +Windows producer and consumer paths are unsupported and unverified in this slice. +Replacing the original root with the download directory invalidates the evidence. + +An input change invalidates acceptance even if HEAD did not change. Restoring +the exact revision and inputs can make the same intact historical bundle valid +again. A failed collection must be rerun from the intended clean candidate; +there is no certification-history ledger. + +## Constraints and advisory review + +Fallow zones enforce local source direction for browser, shared, server, Worker, +tooling and the three test areas. The full-tree boundary scan blocks violations +in unchanged and newly discovered files. ESLint applies explicit package +allowlists to static and literal dynamic imports and rejects computed dynamic +imports in production roots. A new package needs a deliberate allowlist update. +TypeScript remains required for types and globals; it accepts some wrong-runtime +packages, including `node:fs` from shared code, and cannot replace these rules. + +The Fallow audit compares the exact base with `new-only`. Its JSON introduced +identity is joined to native SARIF severity through the installed +`fallow/issue-registry.json`, whose digest is retained. Introduced non-heuristic +errors and every boundary violation block; unknown or ambiguous joins fail. +Inherited errors cannot turn a new warning into a blocker. + +Complexity and duplication remain advisory regardless of native SARIF level or +exit 1. `review.json` lists introduced advisories and inherited configured +hotspots in the changed scope. The Director supplies explicit dispositions, +possibly grouping exact listed IDs with one rationale. The existing inherited +clone `dup:af384e5f` keeps its prior grouped disposition unless its code or +evidence changes. Raw native `npm run fallow:audit` remains a diagnostic command; +the quality command applies the accepted policy. + +Each profile's health report uses `--max-crap 1` only to enumerate function +provenance. Its raw rows are not thousands of separate review requests. Retain +`coverage_source`, `coverage_source_consistency`, `istanbul_matched` and +`istanbul_total`: Fallow can estimate unmatched functions even with a valid V8 +map. Its `coverage_model: istanbul` is an input-format label, not a producer or +proof of measured attribution. + +## CI and remaining checks + +Both CI jobs check out the actual source head. The checks job passes the exact +event base/head to the same public command. Quality collection replaces the five +unit/browser test invocations and native Fallow action. It includes shared Node +smoke; CI separately runs shared typecheck, server typecheck/lint, Worker +typecheck/dry-run, root typecheck/lint, and the browser build. The second job +retains the server container build and smoke. Database/storage integration +commands remain guarded by their separate provisioned targets. + +Before handoff, run focused checks plus root typecheck, lint, exact-base +`quality:verify`, build, and `git diff --check`. Record the applicable workspace +and container checks separately. Evidence acceptance does not replace the +Standards/Spec review, Director dispositions, fresh finalizer after `FIXED`, or +human merge approval. + +CI uploads only `.fallow/quality`, including hidden files, under `if: always()`; +missing artifacts are an error. Artifacts are named with the source head, run ID and attempt, and +retained for 14 days. Record the Actions artifact ID and digest in the handoff. +After expiration, check out the same source revision, install its lockfile and +Chromium, then regenerate with the same exact base. That creates new evidence +and a new artifact identity; an expired artifact is not a passing record. diff --git a/docs/operations/codex-agent-workflow.md b/docs/operations/codex-agent-workflow.md index c87c85bc..d917a328 100644 --- a/docs/operations/codex-agent-workflow.md +++ b/docs/operations/codex-agent-workflow.md @@ -165,20 +165,21 @@ Use the smallest convincing behavior-first test set: Do not generate exhaustive equivalent-value matrices, test private helpers, or add a second harness. +Before collecting, accepting downloaded evidence, or finalizing its PR, read +[quality verification](../agents/quality-verification.md). Before handoff, run relevant focused checks plus: ```sh npx tsc --noEmit npm run lint -npm run test:run +npm run quality:verify -- --base npm run build -FALLOW_AUDIT_BASE= npm run fallow:audit git diff --check ``` The worker records the producer, configuration, exact revision, source/test scope, exclusions, result, and evidence location for every command. It verifies -the Fallow report's comparison range matches the handed-off base; a bare audit +the quality bundle and Fallow comparison range match the handed-off base; a bare audit may select `main` or a branch upstream, and a self-comparison is not delivery evidence. The worker keeps the draft pull request updated against `develop`. After every linked issue's acceptance criteria are satisfied and the diff --git a/eslint.config.js b/eslint.config.js index e37876e2..1344ed25 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,7 +1,59 @@ import js from '@eslint/js'; import tseslint from 'typescript-eslint'; +// Local source edges belong to Fallow. These rules cover package specifiers +// and reject destinations that cannot be checked statically. +function runtimeImports(files, name, packages, node = false) { + const escaped = packages.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + const allowed = `^(?:\\.\\.?/|${node ? 'node:|' : ''}${escaped.length ? `(?:${escaped.join('|')})(?:/|$)` : '(?!)'})`; + const forbidden = `^(?!${allowed.slice(1)})`; + const message = `${name} imports must follow its runtime package contract.`; + return { + files, + rules: { + 'no-restricted-imports': ['error', { patterns: [{ regex: forbidden, message }] }], + 'no-restricted-syntax': [ + 'error', + { + selector: `ImportExpression[source.type='Literal'][source.value=/${forbidden.replaceAll('/', '\\/')}/]`, + message, + }, + { + selector: "ImportExpression[source.type!='Literal']", + message: `${name} computed imports cannot establish a runtime destination.`, + }, + ], + }, + }; +} + export default tseslint.config( + runtimeImports(['src/**/*.ts'], 'Browser', [ + '@pixel-forge/shared', + '@lit-labs/signals', + '@sentry/browser', + 'idb', + 'lit', + 'pako', + 'uuid', + 'virtual:pwa-register', + ]), + runtimeImports(['shared/src/**/*.ts'], 'Shared', []), + runtimeImports( + ['server/src/**/*.ts'], + 'Server', + [ + '@pixel-forge/shared', + '@aws-sdk/client-s3', + '@hono/node-server', + 'drizzle-orm', + 'hono', + 'postgres', + ], + true + ), + runtimeImports(['workers/telemetry/src/**/*.ts'], 'Worker', ['@pixel-forge/shared']), + runtimeImports(['scripts/**/*.{mjs,js,ts}'], 'Tooling', ['@pixel-forge/shared'], true), { ignores: ['dist/', 'node_modules/', 'coverage/'] }, js.configs.recommended, ...tseslint.configs.recommended, diff --git a/package.json b/package.json index 6f72ad10..457f37cd 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,10 @@ "telemetry:dry-run": "wrangler deploy --dry-run --strict --config workers/telemetry/wrangler.jsonc", "telemetry:report": "npm run shared:build && node scripts/telemetry-report.mjs", "telemetry:test": "npm run test --workspace @pixel-forge/telemetry-worker", - "telemetry:typecheck": "npm run typecheck --workspace @pixel-forge/telemetry-worker" + "telemetry:typecheck": "npm run typecheck --workspace @pixel-forge/telemetry-worker", + "quality:verify": "node scripts/quality-evidence.mjs verify", + "quality:check": "node scripts/quality-evidence.mjs check", + "quality:test": "node scripts/quality-evidence.mjs test" }, "dependencies": { "@pixel-forge/shared": "0.1.0", diff --git a/scripts/quality-evidence.mjs b/scripts/quality-evidence.mjs new file mode 100644 index 00000000..b0fced58 --- /dev/null +++ b/scripts/quality-evidence.mjs @@ -0,0 +1,96 @@ +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { coverageCommand, digest, git, identity, profiles } from './quality/config.mjs'; +import { + producerIdentity, + runCommand, + same, + saveManifest, + validateBundle, + writeJson, +} from './quality/bundle.mjs'; +import { collectFallow, validateFallow } from './quality/fallow.mjs'; +import { validateCoverage } from './quality/coverage.mjs'; + +const root = process.cwd(); +const [action, ...args] = process.argv.slice(2); +let bundle = resolve(root, '.fallow/quality'); +try { + const options = {}; + for (let index = 0; index < args.length; index += 2) { + if ( + !['--base', '--head', '--bundle'].includes(args[index]) || + !args[index + 1] || + options[args[index]] + ) + throw new Error('invalid quality command arguments'); + options[args[index]] = args[index + 1]; + } + if (action === 'test') { + if (args.length) throw new Error('quality:test takes no arguments'); + runCommand(root, bundle, 'quality-test', [ + 'node_modules/.bin/vitest', + 'run', + 'tests/scripts/quality-evidence.test.ts', + 'tests/scripts/quality-boundaries.test.ts', + ]); + } else if (action === 'check') { + bundle = resolve(root, options['--bundle'] ?? '.fallow/quality'); + const manifest = validateBundle(root, bundle, options['--base'], options['--head']); + validateFallow(root, bundle, manifest); + console.log(`Accepted quality evidence for ${options['--head']}`); + } else if (action === 'verify') { + if (options['--bundle'] || options['--head']) + throw new Error('quality:verify takes only --base'); + mkdirSync(bundle, { recursive: true }); + rmSync(resolve(bundle, 'certification.json'), { force: true }); + const producer = producerIdentity(root, options['--base']); + const profileList = profiles(producer.inputs); + writeJson(resolve(bundle, 'identity.json'), producer); + writeJson(resolve(bundle, 'profiles.json'), profileList); + const commands = []; + commands.push(runCommand(root, bundle, 'shared-build', ['npm', 'run', 'shared:build'])); + commands.push(runCommand(root, bundle, 'server-build', ['npm', 'run', 'server:build'])); + for (const profile of profileList) { + console.log(`Collecting ${profile.id} (${profile.runtime})`); + commands.push( + runCommand(root, bundle, profile.id, coverageCommand(profile, bundle), profile.cwd) + ); + validateCoverage( + JSON.parse(readFileSync(resolve(bundle, profile.id, 'coverage-final.json'), 'utf8')), + profile, + root + ); + } + commands.push(runCommand(root, bundle, 'shared-node', ['npm', 'run', 'shared:node-test'])); + same( + identity(root), + { head: producer.head, dirty: producer.dirty, inputs: producer.inputs }, + 'post-run checkout' + ); + const manifest = saveManifest(bundle, producer, profileList, commands); + collectFallow(root, bundle, manifest); + validateFallow(root, bundle, manifest); + same( + identity(root), + { head: producer.head, dirty: producer.dirty, inputs: producer.inputs }, + 'post-constraint checkout' + ); + validateBundle(root, bundle, producer.base, producer.head, false); + writeJson(resolve(bundle, 'certification.json'), { + base: producer.base, + head: producer.head, + manifest: digest(readFileSync(resolve(bundle, 'manifest.json'))), + }); + console.log(`Certified quality evidence for ${git(root, 'rev-parse', 'HEAD')}`); + } else { + throw new Error('expected verify, check, or test'); + } +} catch (error) { + if (action === 'verify') { + mkdirSync(bundle, { recursive: true }); + writeFileSync(resolve(bundle, 'failure.log'), `${error.stack}\n`); + } + console.error(error.message); + process.exitCode = 1; +} diff --git a/scripts/quality/bundle.mjs b/scripts/quality/bundle.mjs new file mode 100644 index 00000000..f65ec9cc --- /dev/null +++ b/scripts/quality/bundle.mjs @@ -0,0 +1,214 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { closeSync, mkdirSync, openSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { platform, arch } from 'node:os'; +import { + coverageCommand, + digest, + environment, + git, + identity, + profiles, + readJson, + toolsIdentity, +} from './config.mjs'; +import { validateCoverage } from './coverage.mjs'; + +export const writeJson = (path, value) => + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +export function same(actual, expected, label) { + if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`${label} mismatch`); +} + +function exactRevision(root, base, head) { + if (!/^[a-f0-9]{40}$/.test(base ?? '') || !/^[a-f0-9]{40}$/.test(head ?? '')) + throw new Error('base and head must be exact commit SHAs'); + same(git(root, 'rev-parse', `${base}^{commit}`), base, 'base'); + same(git(root, 'rev-parse', 'HEAD'), head, 'head'); + git(root, 'merge-base', '--is-ancestor', base, head); +} + +export function runCommand(root, bundle, id, command, cwd = '.', allowed = [0], output = null) { + mkdirSync(resolve(bundle, 'commands'), { recursive: true }); + const log = `commands/${id}.log`; + const descriptor = openSync(resolve(bundle, log), 'w'); + const outputDescriptor = output ? openSync(resolve(bundle, output), 'w') : descriptor; + const commandEnvironment = id === 'worker' ? { ...environment, INIT_CWD: root } : environment; + const startedAt = new Date().toISOString(); + const start = Date.now(); + let result; + try { + result = spawnSync(command[0], command.slice(1), { + cwd: resolve(root, cwd), + env: { ...process.env, ...commandEnvironment }, + stdio: ['ignore', outputDescriptor, descriptor], + }); + } finally { + closeSync(descriptor); + if (output) closeSync(outputDescriptor); + } + const record = { + id, + command, + cwd, + environment: commandEnvironment, + startedAt, + durationMs: Date.now() - start, + status: result.status, + error: result.error?.message ?? null, + log, + output, + }; + writeJson(resolve(bundle, `commands/${id}.json`), record); + if (result.error || !allowed.includes(result.status)) + throw new Error(`${id} failed (${result.status}); see ${log}`); + return record; +} + +export function producerIdentity(root, base) { + const inputs = identity(root); + if (inputs.dirty) throw new Error('quality evidence requires a clean committed checkout'); + exactRevision(root, base, inputs.head); + return { + ...inputs, + base, + producerRoot: root, + producerOS: `${platform()}/${arch()}`, + node: process.version, + npm: execFileSync('npm', ['--version'], { encoding: 'utf8' }).trim(), + tools: toolsIdentity(root), + }; +} + +export function artifact(bundle, path) { + if (typeof path !== 'string' || isAbsolute(path) || path.split(/[\\/]/).includes('..')) + throw new Error('artifact path escapes bundle'); + const absolute = resolve(bundle, path); + const located = relative(realpathSync(bundle), realpathSync(absolute)); + if (located.startsWith(`..${sep}`) || isAbsolute(located)) + throw new Error('artifact path escapes bundle'); + return absolute; +} + +export function artifactDigests(bundle, paths) { + return Object.fromEntries( + [...new Set(paths)].sort().map((path) => [path, digest(readFileSync(artifact(bundle, path)))]) + ); +} + +export function validateBundle(root, bundle, base, head, certification = true) { + exactRevision(root, base, head); + const manifest = readJson(artifact(bundle, 'manifest.json')); + const current = identity(root); + if (current.dirty) throw new Error('checkout inputs changed: dirty tree'); + const expectedProfiles = profiles(current.inputs); + const producer = manifest.producer; + same(producer.base, base, 'base'); + same(producer.head, head, 'head'); + same(producer.inputs, current.inputs, 'input inventory'); + same(producer.dirty, '', 'producer dirty state'); + same(producer.tools, toolsIdentity(root), 'installed tools'); + if ( + typeof producer.producerRoot !== 'string' || + !isAbsolute(producer.producerRoot) || + !producer.producerOS || + !/^v\d/.test(producer.node) || + !/^\d/.test(producer.npm) + ) + throw new Error('incomplete producer identity'); + same(manifest.schema, 1, 'bundle schema'); + same(readJson(artifact(bundle, 'identity.json')), producer, 'separate identity'); + same(manifest.profiles, expectedProfiles, 'profile configuration/source/test scope'); + same(readJson(artifact(bundle, 'profiles.json')), manifest.profiles, 'separate profiles'); + same( + manifest.artifacts, + artifactDigests(bundle, Object.keys(manifest.artifacts)), + 'artifact digests' + ); + const requiredCommands = [ + { id: 'shared-build', command: ['npm', 'run', 'shared:build'], cwd: '.' }, + { id: 'server-build', command: ['npm', 'run', 'server:build'], cwd: '.' }, + ...expectedProfiles.map((profile) => ({ + id: profile.id, + command: coverageCommand(profile, resolve(producer.producerRoot, '.fallow/quality')), + cwd: profile.cwd, + })), + { id: 'shared-node', command: ['npm', 'run', 'shared:node-test'], cwd: '.' }, + ]; + same( + manifest.commands.map(({ id, command, cwd }) => ({ id, command, cwd })), + requiredCommands, + 'producer commands' + ); + const requiredArtifacts = ['identity.json', 'profiles.json']; + for (const record of manifest.commands) { + same(record.status, 0, `${record.id} result`); + same(record.error, null, `${record.id} error`); + same( + record.environment, + record.id === 'worker' ? { ...environment, INIT_CWD: producer.producerRoot } : environment, + `${record.id} environment` + ); + same(record.log, `commands/${record.id}.log`, 'command log'); + if ( + !Number.isFinite(record.durationMs) || + record.durationMs < 0 || + !Number.isFinite(Date.parse(record.startedAt)) + ) + throw new Error('invalid command timing'); + same(readJson(artifact(bundle, `commands/${record.id}.json`)), record, 'separate command'); + requiredArtifacts.push(record.log, `commands/${record.id}.json`); + } + for (const profile of expectedProfiles) { + const path = `${profile.id}/coverage-final.json`; + requiredArtifacts.push(path, `${profile.id}/coverage-summary.json`, `${profile.id}/tests.json`); + validateCoverage(readJson(artifact(bundle, path)), profile, producer.producerRoot); + const tests = readJson(artifact(bundle, `${profile.id}/tests.json`)); + if ( + !tests.success || + tests.numFailedTests || + !tests.numTotalTests || + !tests.numPassedTests || + !Array.isArray(tests.testResults) + ) + throw new Error(`${profile.id}: failed or absent test results`); + const executed = tests.testResults + .map((test) => { + const path = test.name.replaceAll('\\', '/'); + if (!path.startsWith(`${producer.producerRoot}/`)) + throw new Error(`${profile.id}: foreign test result`); + return path.slice(producer.producerRoot.length + 1); + }) + .sort(); + same(executed, profile.tests, `${profile.id} executed tests`); + } + for (const path of requiredArtifacts) + if (!manifest.artifacts[path]) throw new Error(`missing artifact digest: ${path}`); + if (certification) + same( + readJson(artifact(bundle, 'certification.json')), + { base, head, manifest: digest(readFileSync(artifact(bundle, 'manifest.json'))) }, + 'certification' + ); + return manifest; +} + +export function saveManifest(bundle, producer, profileList, commands) { + const paths = ['identity.json', 'profiles.json']; + for (const command of commands) paths.push(command.log, `commands/${command.id}.json`); + for (const profile of profileList) + paths.push( + `${profile.id}/coverage-final.json`, + `${profile.id}/coverage-summary.json`, + `${profile.id}/tests.json` + ); + const manifest = { + schema: 1, + producer, + profiles: profileList, + commands, + artifacts: artifactDigests(bundle, paths), + }; + writeJson(resolve(bundle, 'manifest.json'), manifest); + return manifest; +} diff --git a/scripts/quality/config.mjs b/scripts/quality/config.mjs new file mode 100644 index 00000000..705ef3ce --- /dev/null +++ b/scripts/quality/config.mjs @@ -0,0 +1,162 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +export const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +export const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')); +export const git = (root, ...args) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); +export const environment = { CI: 'true', FORCE_COLOR: '0' }; + +export function identity(root) { + const paths = [ + ...new Set( + git(root, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + .split('\0') + .filter(Boolean) + ), + ].sort(); + return { + head: git(root, 'rev-parse', 'HEAD'), + dirty: git(root, 'status', '--porcelain', '--untracked-files=all'), + inputs: paths.map((path) => ({ path, sha256: digest(readFileSync(resolve(root, path))) })), + }; +} + +export function toolsIdentity(root) { + const packages = [ + 'vitest', + '@vitest/coverage-v8', + '@vitest/browser-playwright', + 'playwright', + 'fallow', + 'eslint', + 'typescript', + 'wrangler', + ]; + const locked = readJson(resolve(root, 'package-lock.json')).packages; + return Object.fromEntries( + packages.map((name) => { + const installed = readJson(resolve(root, 'node_modules', name, 'package.json')).version; + if (installed !== locked?.[`node_modules/${name}`]?.version) + throw new Error(`${name}: installed tool does not match package-lock.json`); + return [name, installed]; + }) + ); +} + +export function profiles(inputs) { + const paths = inputs.map((input) => input.path); + const source = (prefix) => + paths.filter( + (path) => path.startsWith(prefix) && path.endsWith('.ts') && !path.endsWith('.d.ts') + ); + const tests = paths.filter((path) => path.endsWith('.test.ts')); + const definitions = [ + { + id: 'root', + runtime: 'Node/happy-dom', + cwd: '.', + command: ['node_modules/.bin/vitest', 'run', '--maxWorkers=1'], + config: ['vitest.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts', 'src/**/*.test.ts', 'src/main.ts'], + sources: source('src/').filter((p) => p !== 'src/main.ts' && !p.endsWith('.test.ts')), + tests: tests.filter((p) => p.startsWith('tests/') && !p.startsWith('tests/browser/')), + testSelection: ['tests/**/*.test.ts'], + testExclusions: ['tests/browser/**/*.test.ts'], + }, + { + id: 'browser', + runtime: 'Chromium/Playwright', + cwd: '.', + command: ['node_modules/.bin/vitest', 'run', '--config', 'vitest.browser.config.ts'], + config: ['vitest.browser.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('src/'), + tests: tests.filter((p) => p.startsWith('tests/browser/')), + testSelection: ['tests/browser/**/*.test.ts'], + testExclusions: [], + }, + { + id: 'shared', + runtime: 'Node/happy-dom', + cwd: '.', + command: [ + 'node_modules/.bin/vitest', + 'run', + 'tests/shared/project-contract.test.ts', + 'tests/shared/product-events.test.ts', + ], + config: ['vitest.config.ts', 'shared/package.json'], + include: 'shared/src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('shared/src/'), + tests: ['tests/shared/product-events.test.ts', 'tests/shared/project-contract.test.ts'], + testSelection: [ + 'tests/shared/project-contract.test.ts', + 'tests/shared/product-events.test.ts', + ], + testExclusions: [], + }, + { + id: 'server', + runtime: 'Node', + cwd: 'server', + command: ['../node_modules/.bin/vitest', 'run', '--config', 'vitest.config.ts'], + config: ['server/vitest.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('server/src/'), + tests: tests.filter( + (p) => p.startsWith('server/tests/') && !/\.(database|storage)\.test\.ts$/.test(p) + ), + testSelection: ['server/tests/**/*.test.ts'], + testExclusions: ['server/tests/**/*.database.test.ts', 'server/tests/**/*.storage.test.ts'], + }, + { + id: 'worker', + runtime: 'Node/happy-dom; mocked Worker bindings, not workerd', + cwd: '.', + command: ['npm', 'run', 'test', '--workspace', '@pixel-forge/telemetry-worker', '--'], + config: [ + 'vitest.config.ts', + 'workers/telemetry/package.json', + 'workers/telemetry/wrangler.jsonc', + ], + include: 'workers/telemetry/src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('workers/telemetry/src/'), + tests: ['tests/workers/telemetry-worker.test.ts'], + testSelection: ['tests/workers/telemetry-worker.test.ts'], + testExclusions: [], + }, + ]; + return definitions.map((profile) => ({ + ...profile, + sources: profile.sources.sort(), + tests: profile.tests.sort(), + environment, + provider: 'v8', + format: 'istanbul', + configs: profile.config.map((path) => inputs.find((input) => input.path === path)), + })); +} + +export function coverageCommand(profile, bundle) { + return [ + ...profile.command, + '--reporter=json', + `--outputFile=${resolve(bundle, profile.id, 'tests.json')}`, + '--coverage.enabled', + '--coverage.excludeAfterRemap', + '--coverage.provider=v8', + '--coverage.reporter=json', + '--coverage.reporter=json-summary', + `--coverage.include=${resolve(bundle, '../..', profile.cwd, profile.include)}`, + ...profile.exclude.map((path) => `--coverage.exclude=${path}`), + `--coverage.reportsDirectory=${resolve(bundle, profile.id)}`, + ]; +} diff --git a/scripts/quality/coverage.mjs b/scripts/quality/coverage.mjs new file mode 100644 index 00000000..a6a8a1ff --- /dev/null +++ b/scripts/quality/coverage.mjs @@ -0,0 +1,103 @@ +import { posix } from 'node:path'; + +function object(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function count(value) { + return Number.isSafeInteger(value) && value >= 0; +} + +// V8 serializes its unbounded end columns (Infinity) as null in JSON. +function location(value) { + return ( + object(value) && + ['start', 'end'].every( + (edge) => + object(value[edge]) && + count(value[edge].line) && + value[edge].line > 0 && + (count(value[edge].column) || (edge === 'end' && value[edge].column === null)) + ) + ); +} + +export function validateCoverage(map, profile, producerRoot) { + if (!object(map) || !Object.keys(map).length) + throw new Error(`${profile.id}: empty or malformed coverage map`); + const root = producerRoot.replaceAll('\\', '/').replace(/\/$/, ''); + const found = []; + for (const [key, file] of Object.entries(map)) { + const path = key.replaceAll('\\', '/'); + if ( + !path.startsWith(`${root}/`) || + posix.normalize(path) !== path || + !object(file) || + file.path !== key + ) + throw new Error(`${profile.id}: foreign or inconsistent coverage path`); + found.push(path.slice(root.length + 1)); + validateFileCounters(file, profile.id); + } + if (JSON.stringify(found.sort()) !== JSON.stringify(profile.sources)) + throw new Error( + `${profile.id}: incomplete or wrong-profile source coverage; missing ${profile.sources.filter((path) => !found.includes(path)).join(', ')}; unexpected ${found.filter((path) => !profile.sources.includes(path)).join(', ')}` + ); +} + +// The V8 remapper emits this sentinel for the implicit else of an if statement. +function implicitElse(value) { + return ( + object(value) && + Object.keys(value).length === 2 && + object(value.start) && + object(value.end) && + Object.keys(value.start).length === 0 && + Object.keys(value.end).length === 0 + ); +} + +function validateFileCounters(file, profileId) { + for (const [counters, mappings] of [ + ['s', 'statementMap'], + ['f', 'fnMap'], + ['b', 'branchMap'], + ]) { + if ( + !object(file[counters]) || + !object(file[mappings]) || + JSON.stringify(Object.keys(file[counters]).sort()) !== + JSON.stringify(Object.keys(file[mappings]).sort()) + ) + throw new Error(`${profileId}: malformed coverage counters`); + for (const [id, value] of Object.entries(file[counters])) { + const mapping = file[mappings][id]; + if (counters === 'b') { + if ( + !Array.isArray(value) || + !value.every(count) || + !object(mapping) || + typeof mapping.type !== 'string' || + !location(mapping.loc) || + !Array.isArray(mapping.locations) || + value.length !== mapping.locations.length || + !mapping.locations.every( + (value, index) => + location(value) || + (mapping.type === 'if' && + mapping.locations.length === 2 && + index === 1 && + implicitElse(value)) + ) + ) + throw new Error(`${profileId}: malformed branch coverage`); + } else if ( + !count(value) || + !location(counters === 's' ? mapping : mapping?.loc) || + (counters === 'f' && (typeof mapping.name !== 'string' || !location(mapping.decl))) + ) { + throw new Error(`${profileId}: malformed statement/function coverage`); + } + } + } +} diff --git a/scripts/quality/fallow.mjs b/scripts/quality/fallow.mjs new file mode 100644 index 00000000..7b76ce6a --- /dev/null +++ b/scripts/quality/fallow.mjs @@ -0,0 +1,265 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { digest, environment, readJson } from './config.mjs'; +import { artifact, artifactDigests, runCommand, same, writeJson } from './bundle.mjs'; + +function findingLocation(finding) { + if (typeof finding.path !== 'string') throw new Error('unsupported Fallow finding location'); + return { path: finding.path, line: finding.line ?? 1, column: (finding.col ?? 0) + 1 }; +} + +function sarifLocation(result) { + const location = result.locations?.[0]?.physicalLocation; + if (!location?.artifactLocation?.uri) throw new Error('missing SARIF finding location'); + return { + path: decodeURI(location.artifactLocation.uri), + line: location.region?.startLine ?? 1, + column: location.region?.startColumn ?? 1, + }; +} + +function reconcile(audit, sarif, registry, base, head) { + if ( + audit.kind !== 'audit' || + audit.base_ref !== base || + !head.startsWith(audit.head_sha) || + audit.head_sha.length < 7 || + audit.attribution?.gate !== 'new-only' + ) + throw new Error('Fallow comparison identity mismatch'); + const results = sarif.runs?.flatMap((run) => run.results ?? []); + if (!Array.isArray(results) || !audit.dead_code) throw new Error('malformed Fallow reports'); + const advisories = []; + const blockers = []; + const matched = new Set(); + for (const [category, findings] of Object.entries(audit.dead_code)) { + if (!Array.isArray(findings) || !findings.length) continue; + const rule = registry.issue_types.find((entry) => entry.result_key === category); + if (!rule?.sarif_rule_ids?.length) throw new Error(`unknown Fallow category: ${category}`); + for (const finding of findings) { + if (typeof finding.introduced !== 'boolean') + throw new Error('missing Fallow introduced identity'); + const location = findingLocation(finding); + const candidates = results.filter( + (result) => + rule.sarif_rule_ids.includes(result.ruleId) && + JSON.stringify(sarifLocation(result)) === JSON.stringify(location) + ); + if (candidates.length !== 1 || matched.has(candidates[0])) + throw new Error(`ambiguous Fallow severity join: ${category} ${JSON.stringify(location)}`); + const result = candidates[0]; + matched.add(result); + if (!['error', 'warning', 'note', 'none'].includes(result.level)) + throw new Error('unknown Fallow severity'); + const entry = { + id: `${result.ruleId}:${location.path}:${location.line}:${location.column}`, + category, + level: result.level, + introduced: finding.introduced, + location, + }; + if (category.startsWith('boundary_') || (finding.introduced && result.level === 'error')) + blockers.push(entry); + else if (finding.introduced && result.level !== 'none') advisories.push(entry); + } + } + const heuristicRules = new Set( + registry.issue_types + .filter((entry) => + [ + 'code-duplication', + 'high-complexity', + 'high-cyclomatic-complexity', + 'high-cognitive-complexity', + 'high-crap-score', + ].includes(entry.id) + ) + .flatMap((entry) => [entry.rule_id, ...(entry.sarif_rule_ids ?? [])]) + ); + for (const result of results) + if (!matched.has(result) && !heuristicRules.has(result.ruleId)) + throw new Error(`unmatched Fallow result: ${result.ruleId}`); + for (const finding of audit.complexity?.findings ?? []) + advisories.push({ + id: `complexity:${finding.path}:${finding.line}:${finding.col}`, + category: 'complexity', + introduced: finding.introduced, + finding, + }); + for (const group of audit.duplication?.clone_groups ?? []) + advisories.push({ + id: group.id ?? group.fingerprint, + category: 'duplication', + introduced: group.introduced, + finding: group, + }); + return { + policy: + 'introduced non-heuristic errors and all boundaries block; complexity and duplication advisory', + blockers, + advisories, + }; +} + +function commands(producer, profiles) { + const fallow = 'node_modules/.bin/fallow'; + const audit = ['audit', '--base', producer.base, '--gate', 'new-only', '--no-cache']; + return [ + { + id: 'boundaries', + command: [fallow, 'dead-code', '--boundary-violations', '--format', 'json', '--no-cache'], + output: 'boundaries.json', + allowed: [0, 1], + }, + { + id: 'boundary-inventory', + command: [fallow, 'list', '--boundaries', '--format', 'json'], + output: 'boundary-inventory.json', + allowed: [0], + }, + { + id: 'runtime-imports', + command: [ + 'node_modules/.bin/eslint', + 'src', + 'shared/src', + 'server/src', + 'workers/telemetry/src', + 'scripts', + ], + output: null, + allowed: [0], + }, + { + id: 'audit-json', + command: [fallow, ...audit, '--format', 'json'], + output: 'audit.json', + allowed: [0, 1], + }, + { + id: 'audit-sarif', + command: [fallow, ...audit, '--format', 'sarif'], + output: 'audit.sarif.json', + allowed: [0, 1], + }, + ...profiles.map((profile) => ({ + id: `health-${profile.id}`, + command: [ + fallow, + 'health', + '--report-only', + '--file-scores', + '--complexity', + '--max-crap', + '1', + '--coverage', + resolve(producer.producerRoot, '.fallow/quality', profile.id, 'coverage-final.json'), + '--coverage-root', + producer.producerRoot, + '--format', + 'json', + '--no-cache', + ], + output: `${profile.id}/fallow.json`, + allowed: [0], + })), + ]; +} + +export function collectFallow(root, bundle, manifest) { + const records = commands(manifest.producer, manifest.profiles).map((item) => + runCommand(root, bundle, item.id, item.command, '.', item.allowed, item.output) + ); + const registry = readJson(resolve(root, 'node_modules/fallow/issue-registry.json')); + const review = reconcile( + readJson(resolve(bundle, 'audit.json')), + readJson(resolve(bundle, 'audit.sarif.json')), + registry, + manifest.producer.base, + manifest.producer.head + ); + writeJson(resolve(bundle, 'review.json'), review); + const paths = ['review.json']; + for (const record of records) + paths.push(record.log, `commands/${record.id}.json`, ...(record.output ? [record.output] : [])); + manifest.fallow = { + registry: digest(readFileSync(resolve(root, 'node_modules/fallow/issue-registry.json'))), + records, + review, + }; + manifest.artifacts = artifactDigests(bundle, [...Object.keys(manifest.artifacts), ...paths]); + writeJson(resolve(bundle, 'manifest.json'), manifest); +} + +export function validateFallow(root, bundle, manifest) { + if (!manifest.fallow) throw new Error('missing Fallow evidence'); + const { records, review } = manifest.fallow; + same( + manifest.fallow.registry, + digest(readFileSync(resolve(root, 'node_modules/fallow/issue-registry.json'))), + 'Fallow registry' + ); + const expected = commands(manifest.producer, manifest.profiles); + same( + records.map(({ id, command, output }) => ({ id, command, output })), + expected.map(({ id, command, output }) => ({ id, command, output })), + 'Fallow command inventory' + ); + for (let index = 0; index < records.length; index++) { + const record = records[index]; + same(record.cwd, '.', 'constraint working directory'); + same(record.environment, environment, 'constraint environment'); + same(record.log, `commands/${record.id}.log`, 'constraint log'); + if ( + !Number.isFinite(record.durationMs) || + record.durationMs < 0 || + !Number.isFinite(Date.parse(record.startedAt)) + ) + throw new Error('invalid constraint timing'); + if (!expected[index].allowed.includes(record.status) || record.error) + throw new Error(`${record.id}: failed constraint producer`); + same(readJson(artifact(bundle, `commands/${record.id}.json`)), record, 'Fallow command record'); + for (const path of [ + record.log, + `commands/${record.id}.json`, + ...(record.output ? [record.output] : []), + ]) + if (!manifest.artifacts[path]) throw new Error(`missing constraint artifact: ${path}`); + } + const boundaries = readJson(artifact(bundle, 'boundaries.json')); + if (boundaries.kind !== 'dead-code' || boundaries.total_issues !== 0) + throw new Error('runtime boundary violation'); + const inventory = readJson(artifact(bundle, 'boundary-inventory.json')); + const configured = readJson(resolve(root, '.fallowrc.json')).boundaries; + same( + inventory.boundaries.zones.map(({ name, patterns }) => ({ name, patterns })), + configured.zones, + 'Fallow zone inventory' + ); + for (const profile of manifest.profiles) { + const health = readJson(artifact(bundle, `${profile.id}/fallow.json`)); + if ( + health.kind !== 'health' || + health.error || + health.version !== manifest.producer.tools.fallow || + health.summary?.coverage_model !== 'istanbul' || + typeof health.summary?.coverage_source_consistency !== 'string' || + !Number.isInteger(health.summary?.istanbul_matched) || + !Number.isInteger(health.summary?.istanbul_total) + ) + throw new Error(`${profile.id}: malformed Fallow provenance`); + } + const reconciled = reconcile( + readJson(artifact(bundle, 'audit.json')), + readJson(artifact(bundle, 'audit.sarif.json')), + readJson(resolve(root, 'node_modules/fallow/issue-registry.json')), + manifest.producer.base, + manifest.producer.head + ); + same(review, reconciled, 'Fallow acceptance policy'); + same(readJson(artifact(bundle, 'review.json')), review, 'separate Fallow review'); + if (review.blockers.length) + throw new Error( + `required Fallow findings: ${review.blockers.map((finding) => finding.id).join(', ')}` + ); +} diff --git a/specs/research/clean-code-quality-evidence.md b/specs/research/clean-code-quality-evidence.md new file mode 100644 index 00000000..7d925456 --- /dev/null +++ b/specs/research/clean-code-quality-evidence.md @@ -0,0 +1,291 @@ +# Pixel Forge quality evidence: verified producers + +Research for [slice #604](https://github.com/Flow-Fly/pixel-forge/issues/604), +under [capability #602](https://github.com/Flow-Fly/pixel-forge/issues/602). +The inspected repository head was +`e94f52e4f8c22f78db93688f5dab3ad95430a652`, in a clean detached research +checkout. This establishes tool mechanics; it does **not** verify a delivery +head or approve a quality gate. + +The existing dependencies can produce five separate measured coverage maps. +Keep Vitest's V8 provider. No dependency change, Istanbul instrumentation, +coverage merge, score threshold, debt cleanup, or performance-test split is +justified by these probes. Fallow consumes the maps but still produces mixed +measured and estimated function attribution. + +## Evidence identity + +The isolated checkout used `npm ci`: Node `22.22.3`, npm `10.9.8`, +Vitest and `@vitest/coverage-v8` `4.1.10`, Playwright and its Vitest provider +`1.62.1` / `4.1.10`, Fallow `3.2.0`, macOS arm64. Chromium headless shell +`151.0.7922.34`, Playwright revision `1234`, was already cached. + +Raw evidence is under +[/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/). +Each `*.command.json` records the actual argument array, working directory, +explicit environment, head, UTC start/end, and exit status. Matching logs, +coverage maps, summaries, and artifact SHA-256 hashes are retained. +[environment.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/environment.json), +[source-input-hashes.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/source-input-hashes.json), +and [artifact-hashes.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/artifact-hashes.json) +identify the inputs and outputs. These are local research artifacts, without a +CI retention guarantee. The retained probe scripts and command records allow +regeneration at the named commit with the locked dependencies. + +## Coverage producers and scopes + +All five probes exited zero and emitted Istanbul-format `coverage-final.json` +through the **V8** provider. JSON format does not imply Istanbul +instrumentation. Each output path below is relative to the evidence directory. + +| Producer and owning configuration | Focused tests actually executed | Verified production scope | Result and map | +| ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------------------------------------------------------ | +| Root Vitest, [vitest.config.ts](../../vitest.config.ts), Node/happy-dom with `fake-indexeddb/auto` | `tests/utils/mask-utils.test.ts` | `src/**/*.ts`, excluding `**/*.d.ts`, `src/main.ts`, `src/**/*.test.ts` | 18 tests; 300 files; `root-scoped/coverage-final.json` | +| Chromium Browser Mode, [vitest.browser.config.ts](../../vitest.browser.config.ts) | `tests/browser/canonical-composition.test.ts` | `src/**/*.ts`, excluding `**/*.d.ts` | 3 tests; 301 files; `browser/coverage-final.json` | +| Shared workspace's root/happy-dom configuration, [shared/package.json](../../shared/package.json) | Both `tests/shared/project-contract.test.ts` and `tests/shared/product-events.test.ts` | `shared/src/**/*.ts`, excluding `**/*.d.ts` | 35 tests; 6 files; `shared/coverage-final.json` | +| Server Node unit runner, [server/vitest.config.ts](../../server/vitest.config.ts) | `server/tests/config.test.ts` | `src/**/*.ts` relative to `server/`, excluding `**/*.d.ts` | 25 tests; 18 files; `server/coverage-final.json` | +| Telemetry workspace's root/happy-dom unit runner, [workers/telemetry/package.json](../../workers/telemetry/package.json) | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding `**/*.d.ts` | 13 tests; 1 file; `worker-npm/coverage-final.json` | + +These source inventories include unexecuted files, not just imported modules. +The broad root probe recorded 283 nonempty files with only zero statement +counts; server recorded 16. Empty maps also occur legitimately for type-only +source or barrels; requiring executable statements in every source file would +reject valid output. Keep the existing root `src/main.ts` exclusion. Its +current configuration also includes two empty `.d.ts` entries; explicitly +excluding declarations makes the runtime source inventory unambiguous. + +The root test selection for a complete evidence run remains +`tests/**/*.test.ts` minus `tests/browser/**/*.test.ts`. Shared and Worker +tests overlap that selection but need their own source scopes and invocation +records. The server unit runner excludes `*.database.test.ts` and +`*.storage.test.ts`. The research selected fewer tests to verify mechanics; +delivery collection must run the complete declared selection. + +Vitest 4 removed `coverage.all`: set `coverage.include` explicitly to add +unexecuted source. Use separate `coverage.reportsDirectory` values because +Vitest cleans that directory. Keep explicit exclusions and apply them after +source remapping where necessary. [Vitest 4 migration](https://v4.vitest.dev/guide/migration), +[coverage configuration](https://v4.vitest.dev/config/coverage), and +[installed-version provider source](https://github.com/vitest-dev/vitest/blob/v4.1.10/packages/coverage-v8/src/provider.ts) +own these behaviors. + +The Chromium probe emitted a Vite resolution warning for the unexecuted +`src/services/pwa-registration.ts` import of `virtual:pwa-register`. +No source file disappeared: all 301 expected non-declaration files remained; +that file had seven zero-hit statements and four functions. Vitest's provider +catches a failed transform and can fall back to original source. Preserve the +warning and inventory check; this is not grounds for excluding PWA files. +Root and browser maps can have different transformed function inventories, +so their counts must not be added or implicitly merged. + +## Reproducible commands + +Use the installed Vitest binary or package scripts, with `CI=true`, and record +the working directory. The probes used these common coverage arguments: + +```sh +--coverage.enabled --coverage.provider=v8 \ +--coverage.reporter=json --coverage.reporter=json-summary \ +--coverage.include='' --coverage.exclude='**/*.d.ts' \ +--coverage.reportsDirectory='' +``` + +Append those arguments to the following invocations. Root additionally repeats +`--coverage.exclude=src/main.ts` and `--coverage.exclude=src/**/*.test.ts` to +preserve its existing exclusions. The optional `lcovonly` reporter was also +probed; it is unnecessary for the proposed Fallow input. + +```sh +# Repository root; shared build is required by the normal package flow. +npm run shared:build +./node_modules/.bin/vitest run tests/utils/mask-utils.test.ts +./node_modules/.bin/vitest run --config vitest.browser.config.ts tests/browser/canonical-composition.test.ts + +# shared/ working directory, matching its Vitest selection. +../node_modules/.bin/vitest run --root .. tests/shared/project-contract.test.ts tests/shared/product-events.test.ts + +# Build from root, then run from server/. +npm run server:build +../node_modules/.bin/vitest run --config vitest.config.ts tests/config.test.ts + +# Repository root. Verified: flags reach Vitest after Wrangler type generation. +npm run test --workspace @pixel-forge/telemetry-worker -- +``` + +The real Worker npm invocation passed 13/13 tests. Calling its inner Vitest +command directly from `workers/telemetry/` without npm's root `INIT_CWD` +failed two configuration tests by resolving the Wrangler path twice. Preserve +the root npm caller and forwarding contract. The shared script ends with a +plain Node command, so appending coverage flags to `npm run shared:test` does +not configure its earlier Vitest command; collect at the actual Vitest call. + +## What these maps do not measure + +- The telemetry runner uses happy-dom and mocked Analytics Engine/rate-limiter + bindings. It measures Worker source executed in those tests, not workerd, + Cloudflare deployment, or real bindings. Vitest V8 requires an exposed V8 + profiler; actual Cloudflare Workers do not provide that interface. + [Vitest coverage providers](https://v4.vitest.dev/guide/coverage). +- `shared/tests/node-import.mjs` passed separately after build. This plain Node + ESM/package-export smoke test is outside Vitest coverage. Browser tests and + server tests can import the compiled shared package; that does not substitute + for the shared source-map evidence. +- Server database/storage integration runners need their guarded external + targets; neither was executed here. Server process and compatibility tests + spawn built Node subprocesses, whose execution is not recorded by the parent + Vitest inspector. Retain these checks with `not-collected` coverage rather + than attributing their execution to the unit map. +- V8 has known limitations around default arguments and statements following + an unexpectedly throwing call. Coverage remains measured execution evidence + with provider limits, not proof that assertions exercise every behavior. + [Remapper limitations](https://github.com/AriPerkkio/ast-v8-to-istanbul#limitations). + +No wall-clock performance budget assertion was found by inspection of +`performance.now`, `Date.now`, timing assertions, and test timeouts. Creative +Run tests use controlled application time; server timeouts bound integration +completion. No local evidence supports importing a separate uninstrumented +performance runner. + +## Fallow 3.2.0 consumption and refusal behavior + +Source was checked at upstream tag `v3.2.0`, commit +`0d568277b6cf91e95c98cb92f8228154e58d6984`, alongside installed CLI help. +`health --coverage` accepts a single Istanbul coverage map JSON, or a directory +containing `coverage-final.json`. It does not accept LCOV or native V8 JSON for +this purpose. `--runtime-coverage` is a different runtime-analysis feature and +is unnecessary here. [Pinned CLI contract](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/main.rs#L1030). + +For diagnostic enumeration only, the probes ran: + +```sh +./node_modules/.bin/fallow health --report-only --file-scores --complexity \ + --max-crap 1 --coverage '' --format json --no-cache +``` + +`--max-crap 1` exposed attribution fields; it is **not a proposed threshold**. +With the broad probe maps, findings inside each corresponding production +scope had the following `coverage_source` values: + +| Input map | `istanbul` findings | `estimated` findings | +| ------------------------------------------- | ------------------: | -------------------: | +| Root, broad diagnostic scope including main | 5,216 | 102 | +| Chromium | 5,215 | 103 | +| Shared | 116 | 1 | +| Server unit | 84 | 0 | +| Worker unit | 10 | 0 | + +These are surfaced findings, not a complete function inventory or a coverage +percentage. No duplicate `(path, name, line, col)` finding identities occurred +in these reports. The whole-repository reports additionally estimate paths +outside the supplied map. `coverage_source: istanbul` names the consumed JSON +format; the producer is still Vitest V8. + +Fallow matches canonical file paths, then function names and source positions, +with limited nearby-name/anonymous fallbacks. It calculates the fraction of +covered statements contained inside each matched function body, falling back +to function hits when no statements exist. Unmatched functions receive an +estimate. For example, Vitest names the second `execute` method `execute_2` +while Fallow names it `execute`, leaving a real method estimated. The shared +unmatched `atob` is an ambient declaration without runtime implementation. +[Pinned matching/scoring implementation](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/engine/src/health/scoring.rs#L730). + +| Explicit coverage input exercise | Observed result | +| -------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| Valid Vitest JSON | Exit 0 under `--report-only`; `coverage_model: istanbul`, mixed function provenance | +| Missing file | Exit 2, JSON `error: true` | +| LCOV text | Exit 2, coverage parse error | +| Empty map `{}` | Exit 0; `coverage_model: istanbul`, zero matches, uniform estimated findings | +| Valid map with foreign absolute file paths | Exit 0; zero matches | +| Same foreign map plus correct absolute `--coverage-root` | Exit 0; all 84 server matches restored | + +Consequently, neither exit 0 nor `coverage_model: istanbul` proves valid +measured evidence. Preserve each finding's `coverage_source`, summary +`coverage_source_consistency`, and `istanbul_matched` / `istanbul_total`. +Use source-root rebasing only after validating the recorded producer root. +Fallow's input loader does not establish base/head or source/test/config +freshness. Its nearby-position matching especially cannot certify freshness. + +Audit classification needs a separate implementation exercise. Source inspection +confirms `kind: audit`, `verdict`, `base_ref`, `head_sha`, and introduced/inherited +category counts. However, `summary.dead_code_has_errors` considers all +changed-scope issues, whereas the default `new-only` verdict checks introduced +issues. That summary cannot replace the existing gate. Native SARIF generation +receives configured rule severities; the audit JSON dead-code serializer does +not. Do not downgrade every exit 1 or infer per-finding severity from category +counts. Preserve the existing new-only contract until a tested classifier can +separate advisory findings from required errors. The fixture appendix below +verifies these distinctions; no global Pixel Forge dead-code baseline was +established here. +[Pinned audit gate](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/audit.rs#L358), +[output contract](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/api/src/audit_output.rs#L25), +and [JSON/SARIF serializers](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/audit_output.rs#L737). + +## Recommended first delivery contract + +Keep five independently identified JSON maps and an executable inventory of +their exact test selection, source files, exclusions, runner/config/tool +versions, environment, commands, base/head, and artifact hashes. A source file +with zero hits must remain visible; type-only files may have empty counters. +Reject missing reports, error payloads, malformed maps, unexpected or absent +source paths, inconsistent internal file paths, stale inputs, and incorrect +base/head before consuming coverage. Check map structure and counters, not +just the file name or existence. + +Retain raw maps, command logs, tool reports, inventory, and input/output hashes +as one identifiable CI artifact. State retention and regenerate from the exact +commit/lockfile/configuration when artifacts expire. Recollection is new +evidence and cannot reuse the expired artifact's identity. +If the output lives under `.fallow/quality`, explicitly enable hidden-file +upload, set missing files to an error, and upload only that generated evidence +directory. Record the artifact ID/digest and configured retention. +[Official upload-artifact options](https://github.com/actions/upload-artifact). + +Keep architecture/runtime checks blocking in their own evidence records. +Keep configured complexity and duplication advisory, with explicit Director +dispositions. Do not apply `--report-only` to a combined command that also owns +required architecture failures. Separate raw measured coverage from Fallow's +mixed advisory interpretation; no merged score is needed. Exercises must +demonstrate acceptance of a correctly identified complete bundle and rejection +of absent, stale, wrong-revision, and wrong-scope reports. + +The research checkout produced only this note for later import into the delivery branch. No configuration, product +code, GitHub state, or dependency lockfile was changed. Focused probes and +`git diff --check` ran; full repository gates remain the delivery worker's +responsibility after implementation. + +## Appendix: native audit gate probes + +Four tiny Git repositories under the evidence directory's `audit-fixtures/` +used real base/head commits, an explicit entry point, and fixture-only rules +`unused-exports: warn` and `unresolved-imports: error`. Installed Fallow ran +`audit --base --gate new-only --format json --no-cache`, then the +same command with `--format sarif`. The script is +[probe-audit.py](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/probe-audit.py); +`audit--.command.json` records each identity and invocation. + +| Candidate change | Audit JSON verdict / exit | Native SARIF | +| ---------------------------------------------------------------- | ------------------------- | ----------------------------------------------------- | +| New unused export configured as warning | `warn` / 0 | `fallow/unused-export`, `warning` | +| New unresolved import configured as error | `fail` / 1 | `fallow/unresolved-import`, `error` | +| Complexity alone, no dead-code finding | `fail` / 1 | `fallow/high-crap-score`, `error` | +| Inherited unresolved-import error plus new unused-export warning | `warn` / 0 | Contains both the inherited `error` and new `warning` | + +The last case proves two unsafe shortcuts: `summary.dead_code_has_errors` +was true despite the correct warning verdict, and SARIF retained the inherited +error without `introduced` or baseline metadata. Audit JSON correctly marked +the unresolved import `introduced: false` and unused export `introduced: true`. +Replacing audit with `dead-code --changed-since --format sarif` +exited 1 on the inherited error, changing the existing new-only contract. + +Installed CLI help limits `--only`/`--skip` to invocations without a subcommand; +there is no verified native audit option to omit just health and duplication. +Do not change thresholds to simulate that omission. A safe acceptance adapter +needs a tested join between JSON introduced findings and native SARIF levels, +with errors for missing, ambiguous, or unsupported identities. The installed +`fallow/issue-registry.json` provides `result_key` and `sarif_rule_ids` mappings +for dead-code categories, avoiding invented pluralization rules. In these +fixtures, file path, line, and column aligned after converting JSON's zero-based +column to SARIF's one-based column. Other finding shapes still need explicit +fixtures before acceptance support is claimed. Complexity/duplication can then +receive advisory dispositions while configured introduced errors remain +blocking; whole-repository boundary checks remain a separately verified scope. diff --git a/tests/scripts/quality-boundaries.test.ts b/tests/scripts/quality-boundaries.test.ts new file mode 100644 index 00000000..b4f98e2f --- /dev/null +++ b/tests/scripts/quality-boundaries.test.ts @@ -0,0 +1,98 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { afterAll, beforeAll, expect, it } from 'vitest'; + +const repository = process.cwd(); +const fixture = mkdtempSync(resolve(tmpdir(), 'pixel-forge-boundaries-')); + +beforeAll(() => { + symlinkSync(resolve(repository, 'node_modules'), resolve(fixture, 'node_modules'), 'dir'); + for (const file of ['eslint.config.js', '.fallowrc.json']) + cpSync(resolve(repository, file), resolve(fixture, file)); + for (const directory of ['src', 'server/src', 'shared/src']) + mkdirSync(resolve(fixture, directory), { recursive: true }); + writeFileSync(resolve(fixture, 'package.json'), '{"type":"module"}'); + writeFileSync(resolve(fixture, '.gitignore'), 'node_modules\n.fallow\n'); + execFileSync('git', ['init', '-q'], { cwd: fixture }); +}); +afterAll(() => rmSync(fixture, { recursive: true, force: true })); + +it('blocks local source crossings even outside a changed-files audit', () => { + writeFileSync(resolve(fixture, 'server/src/index.ts'), 'export const server = 1;'); + writeFileSync( + resolve(fixture, 'src/main.ts'), + "import { server } from '../server/src/index.ts'; console.log(server);" + ); + const result = spawnSync( + resolve(repository, 'node_modules/.bin/fallow'), + ['dead-code', '--boundary-violations', '--format', 'json', '--no-cache'], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status).toBe(1); + expect(JSON.parse(result.stdout).boundary_violations).toHaveLength(1); +}); + +it('rejects static, literal dynamic and computed imports in each production runtime', () => { + const cases = []; + for (const [directory, extension, specifier] of [ + ['src', 'ts', 'hono'], + ['shared/src', 'ts', 'node:fs'], + ['server/src', 'ts', 'lit'], + ['workers/telemetry/src', 'ts', 'lit'], + ['scripts', 'mjs', 'lit'], + ]) { + mkdirSync(resolve(fixture, directory), { recursive: true }); + const sources = [ + `import * as value from '${specifier}'; export { value };`, + `export const value = import('${specifier}');`, + "const destination = './local.js'; export const value = import(destination);", + ]; + for (const [index, source] of sources.entries()) { + const filename = `${directory}/case-${index}.${extension}`; + writeFileSync(resolve(fixture, filename), source); + cases.push(filename); + } + } + const result = spawnSync( + resolve(repository, 'node_modules/.bin/eslint'), + [...cases, '--format', 'json'], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status, result.stderr).toBe(1); + const reports = JSON.parse(result.stdout); + expect(reports).toHaveLength(cases.length); + for (const report of reports) + expect( + report.messages.some((message: { ruleId: string }) => + ['no-restricted-imports', 'no-restricted-syntax'].includes(message.ruleId) + ), + report.filePath + ).toBe(true); +}, 60_000); + +it('keeps a type-accepted wrong-runtime package blocked by the import policy', () => { + writeFileSync( + resolve(fixture, 'shared/src/fixture.ts'), + "import { readFileSync } from 'node:fs'; export { readFileSync };" + ); + const result = spawnSync( + resolve(repository, 'node_modules/.bin/tsc'), + [ + '--noEmit', + '--skipLibCheck', + '--target', + 'es2022', + '--lib', + 'es2022', + '--module', + 'nodenext', + '--types', + 'node', + 'shared/src/fixture.ts', + ], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status, result.stdout + result.stderr).toBe(0); +}); diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts new file mode 100644 index 00000000..ce3fd334 --- /dev/null +++ b/tests/scripts/quality-evidence.test.ts @@ -0,0 +1,393 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const repository = process.cwd(); +const fixture = realpathSync(mkdtempSync(resolve(tmpdir(), 'pixel-forge-quality-'))); +const bundle = resolve(fixture, '.fallow/quality'); +let base: string; +let head: string; +function write(path: string, text: string) { + mkdirSync(dirname(resolve(fixture, path)), { recursive: true }); + writeFileSync(resolve(fixture, path), text); +} +function git(...args: string[]) { + return execFileSync('git', args, { cwd: fixture, encoding: 'utf8' }).trim(); +} +function cli(...args: string[]) { + const result = spawnSync(process.execPath, ['scripts/quality-evidence.mjs', ...args], { + cwd: fixture, + encoding: 'utf8', + env: { ...process.env, CI: 'true' }, + }); + return { status: result.status, text: result.stdout + result.stderr }; +} +function check(path = bundle, candidateBase = base, candidateHead = head) { + return cli('check', '--base', candidateBase, '--head', candidateHead, '--bundle', path); +} + +beforeAll(() => { + for (const path of [ + 'scripts/quality-evidence.mjs', + 'scripts/quality', + 'vitest.config.ts', + 'vitest.browser.config.ts', + 'vite.config.ts', + 'server/vitest.config.ts', + 'eslint.config.js', + '.fallowrc.json', + ]) { + mkdirSync(dirname(resolve(fixture, path)), { recursive: true }); + cpSync(resolve(repository, path), resolve(fixture, path), { recursive: true }); + } + write( + 'vitest.config.ts', + readFileSync(resolve(repository, 'vitest.config.ts'), 'utf8').replace( + "setupFiles: ['fake-indexeddb/auto'],", + '' + ) + ); + write( + 'vitest.browser.config.ts', + readFileSync(resolve(repository, 'vitest.browser.config.ts'), 'utf8').replace( + 'defineConfig({', + `defineConfig({ server: { fs: { allow: [${JSON.stringify(repository)}, ${JSON.stringify(fixture)}] } },` + ) + ); + symlinkSync(resolve(repository, 'node_modules'), resolve(fixture, 'node_modules'), 'dir'); + write( + '.gitignore', + 'node_modules\n.fallow/\n**/dist/\nworkers/telemetry/worker-configuration.d.ts\n' + ); + write( + 'package.json', + JSON.stringify({ + type: 'module', + workspaces: ['workers/telemetry'], + scripts: { + 'shared:build': 'node --eval "0"', + 'server:build': 'node --eval "0"', + 'shared:node-test': 'node shared/tests/node-import.mjs', + }, + }) + ); + write('package-lock.json', readFileSync(resolve(repository, 'package-lock.json'), 'utf8')); + write('shared/package.json', readFileSync(resolve(repository, 'shared/package.json'), 'utf8')); + write( + 'workers/telemetry/package.json', + readFileSync(resolve(repository, 'workers/telemetry/package.json'), 'utf8') + ); + write( + 'workers/telemetry/wrangler.jsonc', + '{"name":"quality-fixture","main":"src/index.ts","compatibility_date":"2026-01-01"}' + ); + write('src/main.ts', 'export const main = 1;\n'); + write('src/value.ts', 'export const value = 2;\n'); + write( + 'src/zero.ts', + 'export function uncalled(flag: boolean) { if (flag) return 1; return 3; }\n' + ); + write('src/types.ts', 'export interface Value { value: number }\n'); + write('shared/src/index.ts', 'export const shared = 3;\n'); + write('server/src/index.ts', 'export const server = 4;\n'); + write('workers/telemetry/src/index.ts', 'export const worker = 5;\n'); + write( + 'shared/tests/node-import.mjs', + 'import assert from "node:assert/strict"; assert.equal(1 + 1, 2);\n' + ); + for (const [path, source, name, value] of [ + ['tests/value.test.ts', '../src/value.ts', 'value', 2], + ['tests/browser/value.test.ts', '../../src/main.ts', 'main', 1], + ['tests/shared/project-contract.test.ts', '../../shared/src/index.ts', 'shared', 3], + ['tests/shared/product-events.test.ts', '../../shared/src/index.ts', 'shared', 3], + ['server/tests/value.test.ts', '../src/index.ts', 'server', 4], + ['tests/workers/telemetry-worker.test.ts', '../../workers/telemetry/src/index.ts', 'worker', 5], + ] as const) + write( + path, + `import { expect, it } from 'vitest'; import { ${name} } from '${source}'; it('executes ${name}', () => expect(${name}).toBe(${value}));\n` + ); + const fallow = JSON.parse(readFileSync(resolve(fixture, '.fallowrc.json'), 'utf8')); + fallow.entry.push('scripts/fixture-entry.mjs'); + fallow.rules = { 'unused-exports': 'warn', 'unresolved-imports': 'error' }; + write('.fallowrc.json', JSON.stringify(fallow)); + write( + 'scripts/fixture-entry.mjs', + "import { used } from './fixture-lib.mjs'; console.log(used);\n" + ); + write('scripts/fixture-lib.mjs', 'export function used() { return 1; }\n'); + git('init', '-q'); + git('config', 'user.email', 'quality@example.invalid'); + git('config', 'user.name', 'Quality fixture'); + git('add', '.'); + git('commit', '-qm', 'fixture base'); + base = git('rev-parse', 'HEAD'); + write('README.md', 'Quality fixture candidate.\n'); + git('add', '.'); + git('commit', '-qm', 'fixture head'); + head = git('rev-parse', 'HEAD'); + const result = cli('verify', '--base', base); + const logs = readdirSync(resolve(bundle, 'commands')) + .filter((name) => name.endsWith('.log')) + .map((name) => readFileSync(resolve(bundle, 'commands', name), 'utf8')) + .join('\n'); + expect(result, result.text + logs).toMatchObject({ status: 0 }); +}, 300_000); + +afterAll(() => rmSync(fixture, { recursive: true, force: true })); + +describe('public quality evidence CLI', () => { + it('collects all five real producers and accepts a relocated intact bundle', () => { + expect(check().status).toBe(0); + const manifest = JSON.parse(readFileSync(resolve(bundle, 'manifest.json'), 'utf8')); + expect(manifest.profiles.map((profile: { id: string }) => profile.id)).toEqual([ + 'root', + 'browser', + 'shared', + 'server', + 'worker', + ]); + const rootMap = JSON.parse(readFileSync(resolve(bundle, 'root/coverage-final.json'), 'utf8')); + expect(rootMap[resolve(fixture, 'src/main.ts')]).toBeUndefined(); + expect( + Object.values(rootMap[resolve(fixture, 'src/zero.ts')].s).every((hits) => hits === 0) + ).toBe(true); + expect(rootMap[resolve(fixture, 'src/types.ts')].s).toEqual({}); + const branches = Object.values(rootMap[resolve(fixture, 'src/zero.ts')].branchMap) as any[]; + expect( + branches.some( + (branch) => JSON.stringify(branch.locations[1]) === JSON.stringify({ start: {}, end: {} }) + ) + ).toBe(true); + const relocated = resolve(fixture, '.fallow/downloaded'); + cpSync(bundle, relocated, { recursive: true }); + expect(check(relocated).status).toBe(0); + expect(readFileSync(resolve(relocated, 'manifest.json'), 'utf8')).toBe( + readFileSync(resolve(bundle, 'manifest.json'), 'utf8') + ); + }); + + it('rejects a manifest symlink outside the selected bundle', () => { + const relocated = resolve(fixture, '.fallow/external-manifest'); + cpSync(bundle, relocated, { recursive: true }); + const manifestPath = resolve(relocated, 'manifest.json'); + rmSync(manifestPath); + symlinkSync(resolve(bundle, 'manifest.json'), manifestPath); + const result = check(relocated); + expect(result.status).toBe(1); + expect(result.text).toContain('artifact path escapes bundle'); + }); + + it('rejects absent, malformed, incomplete, foreign, or mixed artifacts', () => { + expect(check(resolve(fixture, '.fallow/absent')).status).toBe(1); + for (const [path, replacement] of [ + ['manifest.json', '{'], + ['root/coverage-final.json', '{}'], + [ + 'root/coverage-final.json', + readFileSync(resolve(bundle, 'server/coverage-final.json'), 'utf8'), + ], + [ + 'root/coverage-final.json', + readFileSync(resolve(bundle, 'root/coverage-final.json'), 'utf8').replaceAll( + fixture, + '/foreign/root' + ), + ], + ['commands/root.json', '{}'], + ['certification.json', '{}'], + ]) { + const original = readFileSync(resolve(bundle, path)); + writeFileSync(resolve(bundle, path), replacement); + expect(check().status, path).toBe(1); + writeFileSync(resolve(bundle, path), original); + } + }); + + it('validates coverage structure even when a changed artifact has a matching digest', () => { + const path = 'root/coverage-final.json'; + const original = readFileSync(resolve(bundle, path), 'utf8'); + const savedManifest = readFileSync(resolve(bundle, 'manifest.json'), 'utf8'); + const map = JSON.parse(original); + const incomplete = { ...map }; + delete incomplete[resolve(fixture, 'src/zero.ts')]; + const malformedBranch = JSON.parse(original); + const firstBranch = Object.values( + malformedBranch[resolve(fixture, 'src/zero.ts')].branchMap + )[0] as any; + firstBranch.locations[0] = { start: {}, end: {} }; + const malformed = JSON.parse(original); + malformed[resolve(fixture, 'src/value.ts')].s = { 0: -1 }; + for (const [replacement, message] of [ + ['{}', 'empty or malformed'], + [JSON.stringify(incomplete), 'incomplete or wrong-profile'], + [JSON.stringify(malformed), 'malformed'], + [JSON.stringify(malformedBranch), 'malformed branch'], + [original.replaceAll(fixture, '/foreign/root'), 'foreign'], + [readFileSync(resolve(bundle, 'server/coverage-final.json'), 'utf8'), 'wrong-profile'], + ]) { + writeFileSync(resolve(bundle, path), replacement); + const manifest = JSON.parse(savedManifest); + manifest.artifacts[path] = createHash('sha256').update(replacement).digest('hex'); + writeFileSync(resolve(bundle, 'manifest.json'), JSON.stringify(manifest)); + const result = check(); + expect(result.status).toBe(1); + expect(result.text).toContain(message); + } + writeFileSync(resolve(bundle, path), original); + writeFileSync(resolve(bundle, 'manifest.json'), savedManifest); + }); + + it('rejects wrong revisions, tool/config/scope identities and changed inputs', () => { + expect(check(bundle, head).status).toBe(1); + expect(check(bundle, base, base).status).toBe(1); + const path = resolve(bundle, 'manifest.json'); + const original = readFileSync(path, 'utf8'); + for (const mutate of [ + (m: any) => { + m.producer.tools.vitest = '0.0.0'; + }, + (m: any) => { + m.profiles[0].configs = []; + }, + (m: any) => { + m.profiles[0].sources = []; + }, + (m: any) => { + m.profiles[0].tests = []; + }, + (m: any) => { + m.producer.producerRoot = '/foreign/root'; + }, + (m: any) => { + m.artifacts['../outside'] = 'invalid'; + }, + ]) { + const manifest = JSON.parse(original); + mutate(manifest); + writeFileSync(path, JSON.stringify(manifest)); + expect(check().status).toBe(1); + writeFileSync(path, original); + } + write('README.md', 'Changed tracked input.'); + expect(check().status).toBe(1); + git('restore', 'README.md'); + write('new-input.txt', 'Nonignored input.'); + expect(check().status).toBe(1); + rmSync(resolve(fixture, 'new-input.txt')); + expect(check().status).toBe(0); + }, 30_000); + + it('refuses a committed lockfile that disagrees with the installed tool', () => { + const original = readFileSync(resolve(fixture, 'package-lock.json'), 'utf8'); + const lock = JSON.parse(original); + lock.packages['node_modules/vitest'].version = '0.0.0'; + write('package-lock.json', JSON.stringify(lock)); + git('add', '.'); + git('commit', '-qm', 'contradictory tool lock'); + const result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(result.text).toContain('installed tool does not match package-lock.json'); + expect(existsSync(resolve(bundle, 'certification.json'))).toBe(false); + write('package-lock.json', original); + git('add', '.'); + git('commit', '-qm', 'restore tool lock'); + head = git('rev-parse', 'HEAD'); + }); + + it('preserves new-only required errors while native complexity errors stay advisory', () => { + const commit = (source: string) => { + write('scripts/fixture-lib.mjs', source); + git('add', '.'); + git('commit', '-qm', 'audit fixture'); + head = git('rev-parse', 'HEAD'); + }; + commit('export function used() { return 1; }\nexport const unused = 2;\n'); + let result = cli('verify', '--base', base); + expect(result, result.text).toMatchObject({ status: 0 }); + expect( + JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')).advisories.some( + (finding: any) => finding.category === 'unused_exports' + ) + ).toBe(true); + commit("import { missing } from './missing.js';\nexport function used() { return missing; }\n"); + result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(result.text).toContain('required Fallow findings'); + const inheritedBase = head; + commit( + "import { missing } from './missing.js';\nexport function used() { return missing; }\nexport const unused = 2;\n" + ); + result = cli('verify', '--base', inheritedBase); + expect(result, result.text).toMatchObject({ status: 0 }); + const review = JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')); + expect(review.blockers).toEqual([]); + expect(review.advisories.some((finding: any) => finding.category === 'unused_exports')).toBe( + true + ); + commit( + `export function used(value) {\n${Array.from({ length: 25 }, (_, index) => `if (value === ${index}) return ${index};`).join('\n')}\nreturn -1;\n}\n` + ); + result = cli('verify', '--base', base); + expect(result, result.text).toMatchObject({ status: 0 }); + expect(JSON.parse(readFileSync(resolve(bundle, 'audit.json'), 'utf8')).verdict).toBe('fail'); + expect( + JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')).advisories.some( + (finding: any) => finding.category === 'complexity' + ) + ).toBe(true); + const sarifPath = resolve(bundle, 'audit.sarif.json'); + const sarif = JSON.parse(readFileSync(sarifPath, 'utf8')); + const nativeResult = sarif.runs + .flatMap((run: any) => run.results ?? []) + .find((result: any) => result.ruleId === 'fallow/high-crap-score'); + expect(nativeResult.level).toBe('error'); + nativeResult.ruleId = 'fallow/css-broken-reference'; + const changedSarif = JSON.stringify(sarif); + writeFileSync(sarifPath, changedSarif); + const manifest = JSON.parse(readFileSync(resolve(bundle, 'manifest.json'), 'utf8')); + manifest.artifacts['audit.sarif.json'] = createHash('sha256') + .update(changedSarif) + .digest('hex'); + const changedManifest = JSON.stringify(manifest); + writeFileSync(resolve(bundle, 'manifest.json'), changedManifest); + writeFileSync( + resolve(bundle, 'certification.json'), + JSON.stringify({ + base, + head, + manifest: createHash('sha256').update(changedManifest).digest('hex'), + }) + ); + result = check(); + expect(result.status).toBe(1); + expect(result.text).toContain('unmatched Fallow result: fallow/css-broken-reference'); + }, 600_000); + + it('invalidates certification before a failed producer and retains its diagnostics', () => { + write( + 'tests/value.test.ts', + "import { it } from 'vitest'; it('fails', () => { throw new Error('producer failure'); });\n" + ); + git('add', '.'); + git('commit', '-qm', 'failed test producer'); + const result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(existsSync(resolve(bundle, 'certification.json'))).toBe(false); + expect(readFileSync(resolve(bundle, 'root/tests.json'), 'utf8')).toContain('producer failure'); + }, 60_000); +});