From 7a96f372acf6c1ed9c09daa4323c6213e63d7cd1 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 00:48:28 +0200 Subject: [PATCH 01/11] feat(quality): collect and validate revision-bound V8 evidence (#613) --- package.json | 5 +- scripts/quality-evidence.mjs | 87 +++++++++ scripts/quality/bundle.mjs | 208 +++++++++++++++++++++ scripts/quality/config.mjs | 159 ++++++++++++++++ scripts/quality/coverage.mjs | 73 ++++++++ tests/scripts/quality-evidence.test.ts | 242 +++++++++++++++++++++++++ 6 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 scripts/quality-evidence.mjs create mode 100644 scripts/quality/bundle.mjs create mode 100644 scripts/quality/config.mjs create mode 100644 scripts/quality/coverage.mjs create mode 100644 tests/scripts/quality-evidence.test.ts diff --git a/package.json b/package.json index 6f72ad10..457f37cd 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,10 @@ "telemetry:dry-run": "wrangler deploy --dry-run --strict --config workers/telemetry/wrangler.jsonc", "telemetry:report": "npm run shared:build && node scripts/telemetry-report.mjs", "telemetry:test": "npm run test --workspace @pixel-forge/telemetry-worker", - "telemetry:typecheck": "npm run typecheck --workspace @pixel-forge/telemetry-worker" + "telemetry:typecheck": "npm run typecheck --workspace @pixel-forge/telemetry-worker", + "quality:verify": "node scripts/quality-evidence.mjs verify", + "quality:check": "node scripts/quality-evidence.mjs check", + "quality:test": "node scripts/quality-evidence.mjs test" }, "dependencies": { "@pixel-forge/shared": "0.1.0", diff --git a/scripts/quality-evidence.mjs b/scripts/quality-evidence.mjs new file mode 100644 index 00000000..2dc742db --- /dev/null +++ b/scripts/quality-evidence.mjs @@ -0,0 +1,87 @@ +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { coverageCommand, digest, git, identity, profiles } from './quality/config.mjs'; +import { + producerIdentity, + runCommand, + same, + saveManifest, + validateBundle, + writeJson, +} from './quality/bundle.mjs'; +import { validateCoverage } from './quality/coverage.mjs'; + +const root = process.cwd(); +const [action, ...args] = process.argv.slice(2); +let bundle = resolve(root, '.fallow/quality'); +try { + const options = {}; + for (let index = 0; index < args.length; index += 2) { + if ( + !['--base', '--head', '--bundle'].includes(args[index]) || + !args[index + 1] || + options[args[index]] + ) + throw new Error('invalid quality command arguments'); + options[args[index]] = args[index + 1]; + } + if (action === 'test') { + if (args.length) throw new Error('quality:test takes no arguments'); + runCommand(root, bundle, 'quality-test', [ + 'node_modules/.bin/vitest', + 'run', + 'tests/scripts/quality-evidence.test.ts', + 'tests/scripts/quality-boundaries.test.ts', + ]); + } else if (action === 'check') { + bundle = resolve(root, options['--bundle'] ?? '.fallow/quality'); + validateBundle(root, bundle, options['--base'], options['--head']); + console.log(`Accepted quality evidence for ${options['--head']}`); + } else if (action === 'verify') { + if (options['--bundle'] || options['--head']) + throw new Error('quality:verify takes only --base'); + mkdirSync(bundle, { recursive: true }); + rmSync(resolve(bundle, 'certification.json'), { force: true }); + const producer = producerIdentity(root, options['--base']); + const profileList = profiles(producer.inputs); + writeJson(resolve(bundle, 'identity.json'), producer); + writeJson(resolve(bundle, 'profiles.json'), profileList); + const commands = []; + commands.push(runCommand(root, bundle, 'shared-build', ['npm', 'run', 'shared:build'])); + commands.push(runCommand(root, bundle, 'server-build', ['npm', 'run', 'server:build'])); + for (const profile of profileList) { + console.log(`Collecting ${profile.id} (${profile.runtime})`); + commands.push( + runCommand(root, bundle, profile.id, coverageCommand(profile, bundle), profile.cwd) + ); + validateCoverage( + JSON.parse(readFileSync(resolve(bundle, profile.id, 'coverage-final.json'), 'utf8')), + profile, + root + ); + } + commands.push(runCommand(root, bundle, 'shared-node', ['npm', 'run', 'shared:node-test'])); + same( + identity(root), + { head: producer.head, dirty: producer.dirty, inputs: producer.inputs }, + 'post-run checkout' + ); + saveManifest(bundle, producer, profileList, commands); + validateBundle(root, bundle, producer.base, producer.head, false); + writeJson(resolve(bundle, 'certification.json'), { + base: producer.base, + head: producer.head, + manifest: digest(readFileSync(resolve(bundle, 'manifest.json'))), + }); + console.log(`Certified quality evidence for ${git(root, 'rev-parse', 'HEAD')}`); + } else { + throw new Error('expected verify, check, or test'); + } +} catch (error) { + if (action === 'verify') { + mkdirSync(bundle, { recursive: true }); + writeFileSync(resolve(bundle, 'failure.log'), `${error.stack}\n`); + } + console.error(error.message); + process.exitCode = 1; +} diff --git a/scripts/quality/bundle.mjs b/scripts/quality/bundle.mjs new file mode 100644 index 00000000..00bbbf4f --- /dev/null +++ b/scripts/quality/bundle.mjs @@ -0,0 +1,208 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { closeSync, mkdirSync, openSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { platform, arch } from 'node:os'; +import { + coverageCommand, + digest, + environment, + git, + identity, + profiles, + readJson, + toolsIdentity, +} from './config.mjs'; +import { validateCoverage } from './coverage.mjs'; + +export const writeJson = (path, value) => + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +export function same(actual, expected, label) { + if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`${label} mismatch`); +} + +export function exactRevision(root, base, head) { + if (!/^[a-f0-9]{40}$/.test(base ?? '') || !/^[a-f0-9]{40}$/.test(head ?? '')) + throw new Error('base and head must be exact commit SHAs'); + same(git(root, 'rev-parse', `${base}^{commit}`), base, 'base'); + same(git(root, 'rev-parse', 'HEAD'), head, 'head'); + git(root, 'merge-base', '--is-ancestor', base, head); +} + +export function runCommand(root, bundle, id, command, cwd = '.', allowed = [0], output = null) { + mkdirSync(resolve(bundle, 'commands'), { recursive: true }); + const log = `commands/${id}.log`; + const descriptor = openSync(resolve(bundle, log), 'w'); + const outputDescriptor = output ? openSync(resolve(bundle, output), 'w') : descriptor; + const commandEnvironment = id === 'worker' ? { ...environment, INIT_CWD: root } : environment; + const startedAt = new Date().toISOString(); + const start = Date.now(); + let result; + try { + result = spawnSync(command[0], command.slice(1), { + cwd: resolve(root, cwd), + env: { ...process.env, ...commandEnvironment }, + stdio: ['ignore', outputDescriptor, descriptor], + }); + } finally { + closeSync(descriptor); + if (output) closeSync(outputDescriptor); + } + const record = { + id, + command, + cwd, + environment: commandEnvironment, + startedAt, + durationMs: Date.now() - start, + status: result.status, + error: result.error?.message ?? null, + log, + output, + }; + writeJson(resolve(bundle, `commands/${id}.json`), record); + if (result.error || !allowed.includes(result.status)) + throw new Error(`${id} failed (${result.status}); see ${log}`); + return record; +} + +export function producerIdentity(root, base) { + const inputs = identity(root); + if (inputs.dirty) throw new Error('quality evidence requires a clean committed checkout'); + exactRevision(root, base, inputs.head); + return { + ...inputs, + base, + producerRoot: root, + producerOS: `${platform()}/${arch()}`, + node: process.version, + npm: execFileSync('npm', ['--version'], { encoding: 'utf8' }).trim(), + tools: toolsIdentity(root), + }; +} + +export function artifact(bundle, path) { + if (typeof path !== 'string' || isAbsolute(path) || path.split(/[\\/]/).includes('..')) + throw new Error('artifact path escapes bundle'); + const absolute = resolve(bundle, path); + const located = relative(realpathSync(bundle), realpathSync(absolute)); + if (located.startsWith(`..${sep}`) || isAbsolute(located)) + throw new Error('artifact path escapes bundle'); + return absolute; +} + +export function artifactDigests(bundle, paths) { + return Object.fromEntries( + [...new Set(paths)].sort().map((path) => [path, digest(readFileSync(artifact(bundle, path)))]) + ); +} + +export function validateBundle(root, bundle, base, head, certification = true) { + exactRevision(root, base, head); + const manifest = readJson(resolve(bundle, 'manifest.json')); + const current = identity(root); + if (current.dirty) throw new Error('checkout inputs changed: dirty tree'); + const expectedProfiles = profiles(current.inputs); + const producer = manifest.producer; + same(producer.base, base, 'base'); + same(producer.head, head, 'head'); + same(producer.inputs, current.inputs, 'input inventory'); + same(producer.dirty, '', 'producer dirty state'); + same(producer.tools, toolsIdentity(root), 'installed tools'); + if ( + typeof producer.producerRoot !== 'string' || + !isAbsolute(producer.producerRoot) || + !producer.producerOS || + !/^v\d/.test(producer.node) || + !/^\d/.test(producer.npm) + ) + throw new Error('incomplete producer identity'); + same(manifest.schema, 1, 'bundle schema'); + same(readJson(artifact(bundle, 'identity.json')), producer, 'separate identity'); + same(manifest.profiles, expectedProfiles, 'profile configuration/source/test scope'); + same(readJson(artifact(bundle, 'profiles.json')), manifest.profiles, 'separate profiles'); + same( + manifest.artifacts, + artifactDigests(bundle, Object.keys(manifest.artifacts)), + 'artifact digests' + ); + const requiredCommands = [ + { id: 'shared-build', command: ['npm', 'run', 'shared:build'], cwd: '.' }, + { id: 'server-build', command: ['npm', 'run', 'server:build'], cwd: '.' }, + ...expectedProfiles.map((profile) => ({ + id: profile.id, + command: coverageCommand(profile, resolve(producer.producerRoot, '.fallow/quality')), + cwd: profile.cwd, + })), + { id: 'shared-node', command: ['npm', 'run', 'shared:node-test'], cwd: '.' }, + ]; + same( + manifest.commands.map(({ id, command, cwd }) => ({ id, command, cwd })), + requiredCommands, + 'producer commands' + ); + const requiredArtifacts = ['identity.json', 'profiles.json']; + for (const record of manifest.commands) { + same(record.status, 0, `${record.id} result`); + same(record.error, null, `${record.id} error`); + same( + record.environment, + record.id === 'worker' ? { ...environment, INIT_CWD: producer.producerRoot } : environment, + `${record.id} environment` + ); + same(record.log, `commands/${record.id}.log`, 'command log'); + if ( + !Number.isFinite(record.durationMs) || + record.durationMs < 0 || + !Number.isFinite(Date.parse(record.startedAt)) + ) + throw new Error('invalid command timing'); + same(readJson(artifact(bundle, `commands/${record.id}.json`)), record, 'separate command'); + requiredArtifacts.push(record.log, `commands/${record.id}.json`); + } + for (const profile of expectedProfiles) { + const path = `${profile.id}/coverage-final.json`; + requiredArtifacts.push(path, `${profile.id}/coverage-summary.json`, `${profile.id}/tests.json`); + validateCoverage(readJson(artifact(bundle, path)), profile, producer.producerRoot); + const tests = readJson(artifact(bundle, `${profile.id}/tests.json`)); + if ( + !tests.success || + tests.numFailedTests || + !tests.numTotalTests || + !Array.isArray(tests.testResults) + ) + throw new Error(`${profile.id}: failed or absent test results`); + const executed = tests.testResults + .map((test) => test.name.replaceAll('\\', '/').slice(producer.producerRoot.length + 1)) + .sort(); + same(executed, profile.tests, `${profile.id} executed tests`); + } + for (const path of requiredArtifacts) + if (!manifest.artifacts[path]) throw new Error(`missing artifact digest: ${path}`); + if (certification) + same( + readJson(artifact(bundle, 'certification.json')), + { base, head, manifest: digest(readFileSync(resolve(bundle, 'manifest.json'))) }, + 'certification' + ); + return manifest; +} + +export function saveManifest(bundle, producer, profileList, commands) { + const paths = ['identity.json', 'profiles.json']; + for (const command of commands) paths.push(command.log, `commands/${command.id}.json`); + for (const profile of profileList) + paths.push( + `${profile.id}/coverage-final.json`, + `${profile.id}/coverage-summary.json`, + `${profile.id}/tests.json` + ); + const manifest = { + schema: 1, + producer, + profiles: profileList, + commands, + artifacts: artifactDigests(bundle, paths), + }; + writeJson(resolve(bundle, 'manifest.json'), manifest); + return manifest; +} diff --git a/scripts/quality/config.mjs b/scripts/quality/config.mjs new file mode 100644 index 00000000..eea6050a --- /dev/null +++ b/scripts/quality/config.mjs @@ -0,0 +1,159 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +export const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +export const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')); +export const git = (root, ...args) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); +export const environment = { CI: 'true', FORCE_COLOR: '0' }; + +export function identity(root) { + const paths = [ + ...new Set( + git(root, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + .split('\0') + .filter(Boolean) + ), + ].sort(); + return { + head: git(root, 'rev-parse', 'HEAD'), + dirty: git(root, 'status', '--porcelain', '--untracked-files=all'), + inputs: paths.map((path) => ({ path, sha256: digest(readFileSync(resolve(root, path))) })), + }; +} + +export function toolsIdentity(root) { + const packages = [ + 'vitest', + '@vitest/coverage-v8', + '@vitest/browser-playwright', + 'playwright', + 'fallow', + 'eslint', + 'typescript', + 'wrangler', + ]; + return Object.fromEntries( + packages.map((name) => [ + name, + readJson(resolve(root, 'node_modules', name, 'package.json')).version, + ]) + ); +} + +export function profiles(inputs) { + const paths = inputs.map((input) => input.path); + const source = (prefix) => + paths.filter( + (path) => path.startsWith(prefix) && path.endsWith('.ts') && !path.endsWith('.d.ts') + ); + const tests = paths.filter((path) => path.endsWith('.test.ts')); + const definitions = [ + { + id: 'root', + runtime: 'Node/happy-dom', + cwd: '.', + command: ['node_modules/.bin/vitest', 'run'], + config: ['vitest.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts', 'src/**/*.test.ts', 'src/main.ts'], + sources: source('src/').filter((p) => p !== 'src/main.ts' && !p.endsWith('.test.ts')), + tests: tests.filter((p) => p.startsWith('tests/') && !p.startsWith('tests/browser/')), + testSelection: ['tests/**/*.test.ts'], + testExclusions: ['tests/browser/**/*.test.ts'], + }, + { + id: 'browser', + runtime: 'Chromium/Playwright', + cwd: '.', + command: ['node_modules/.bin/vitest', 'run', '--config', 'vitest.browser.config.ts'], + config: ['vitest.browser.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('src/'), + tests: tests.filter((p) => p.startsWith('tests/browser/')), + testSelection: ['tests/browser/**/*.test.ts'], + testExclusions: [], + }, + { + id: 'shared', + runtime: 'Node/happy-dom', + cwd: '.', + command: [ + 'node_modules/.bin/vitest', + 'run', + 'tests/shared/project-contract.test.ts', + 'tests/shared/product-events.test.ts', + ], + config: ['vitest.config.ts', 'shared/package.json'], + include: 'shared/src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('shared/src/'), + tests: ['tests/shared/product-events.test.ts', 'tests/shared/project-contract.test.ts'], + testSelection: [ + 'tests/shared/project-contract.test.ts', + 'tests/shared/product-events.test.ts', + ], + testExclusions: [], + }, + { + id: 'server', + runtime: 'Node', + cwd: 'server', + command: ['../node_modules/.bin/vitest', 'run', '--config', 'vitest.config.ts'], + config: ['server/vitest.config.ts'], + include: 'src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('server/src/'), + tests: tests.filter( + (p) => p.startsWith('server/tests/') && !/\.(database|storage)\.test\.ts$/.test(p) + ), + testSelection: ['server/tests/**/*.test.ts'], + testExclusions: ['server/tests/**/*.database.test.ts', 'server/tests/**/*.storage.test.ts'], + }, + { + id: 'worker', + runtime: 'Node/happy-dom; mocked Worker bindings, not workerd', + cwd: '.', + command: ['npm', 'run', 'test', '--workspace', '@pixel-forge/telemetry-worker', '--'], + config: [ + 'vitest.config.ts', + 'workers/telemetry/package.json', + 'workers/telemetry/wrangler.jsonc', + ], + include: 'workers/telemetry/src/**/*.ts', + exclude: ['**/*.d.ts'], + sources: source('workers/telemetry/src/'), + tests: ['tests/workers/telemetry-worker.test.ts'], + testSelection: ['tests/workers/telemetry-worker.test.ts'], + testExclusions: [], + }, + ]; + return definitions.map((profile) => ({ + ...profile, + sources: profile.sources.sort(), + tests: profile.tests.sort(), + environment, + provider: 'v8', + format: 'istanbul', + configs: profile.config.map((path) => inputs.find((input) => input.path === path)), + })); +} + +export function coverageCommand(profile, bundle) { + return [ + ...profile.command, + '--reporter=json', + `--outputFile=${resolve(bundle, profile.id, 'tests.json')}`, + '--coverage.enabled', + '--coverage.excludeAfterRemap', + '--coverage.provider=v8', + '--coverage.reporter=json', + '--coverage.reporter=json-summary', + `--coverage.include=${resolve(bundle, '../..', profile.cwd, profile.include)}`, + ...profile.exclude.map((path) => `--coverage.exclude=${path}`), + `--coverage.reportsDirectory=${resolve(bundle, profile.id)}`, + ]; +} diff --git a/scripts/quality/coverage.mjs b/scripts/quality/coverage.mjs new file mode 100644 index 00000000..fbd5d067 --- /dev/null +++ b/scripts/quality/coverage.mjs @@ -0,0 +1,73 @@ +import { posix } from 'node:path'; + +function object(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function count(value) { + return Number.isSafeInteger(value) && value >= 0; +} + +function location(value) { + return ( + object(value) && + ['start', 'end'].every( + (edge) => + object(value[edge]) && + count(value[edge].line) && + value[edge].line > 0 && + (count(value[edge].column) || (edge === 'end' && value[edge].column === null)) + ) + ); +} + +export function validateCoverage(map, profile, producerRoot) { + if (!object(map) || !Object.keys(map).length) + throw new Error(`${profile.id}: empty or malformed coverage map`); + const root = producerRoot.replaceAll('\\', '/').replace(/\/$/, ''); + const found = []; + for (const [key, file] of Object.entries(map)) { + const path = key.replaceAll('\\', '/'); + if ( + !path.startsWith(`${root}/`) || + posix.normalize(path) !== path || + !object(file) || + file.path !== key + ) + throw new Error(`${profile.id}: foreign or inconsistent coverage path`); + found.push(path.slice(root.length + 1)); + for (const [counters, mappings] of [ + ['s', 'statementMap'], + ['f', 'fnMap'], + ['b', 'branchMap'], + ]) { + if ( + !object(file[counters]) || + !object(file[mappings]) || + JSON.stringify(Object.keys(file[counters]).sort()) !== + JSON.stringify(Object.keys(file[mappings]).sort()) + ) + throw new Error(`${profile.id}: malformed coverage counters`); + for (const [id, value] of Object.entries(file[counters])) { + const mapping = file[mappings][id]; + if (counters === 'b') { + if ( + !Array.isArray(value) || + !value.every(count) || + !object(mapping) || + !Array.isArray(mapping.locations) || + value.length !== mapping.locations.length || + !mapping.locations.every(location) + ) + throw new Error(`${profile.id}: malformed branch coverage`); + } else if (!count(value) || !location(counters === 's' ? mapping : mapping?.loc)) { + throw new Error(`${profile.id}: malformed statement/function coverage`); + } + } + } + } + if (JSON.stringify(found.sort()) !== JSON.stringify(profile.sources)) + throw new Error( + `${profile.id}: incomplete or wrong-profile source coverage; missing ${profile.sources.filter((path) => !found.includes(path)).join(', ')}; unexpected ${found.filter((path) => !profile.sources.includes(path)).join(', ')}` + ); +} diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts new file mode 100644 index 00000000..9ffb8cbe --- /dev/null +++ b/tests/scripts/quality-evidence.test.ts @@ -0,0 +1,242 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const repository = process.cwd(); +const fixture = realpathSync(mkdtempSync(resolve(tmpdir(), 'pixel-forge-quality-'))); +const bundle = resolve(fixture, '.fallow/quality'); +let base: string; +let head: string; +function write(path: string, text: string) { + mkdirSync(dirname(resolve(fixture, path)), { recursive: true }); + writeFileSync(resolve(fixture, path), text); +} +function git(...args: string[]) { + return execFileSync('git', args, { cwd: fixture, encoding: 'utf8' }).trim(); +} +function cli(...args: string[]) { + const result = spawnSync(process.execPath, ['scripts/quality-evidence.mjs', ...args], { + cwd: fixture, + encoding: 'utf8', + env: { ...process.env, CI: 'true' }, + }); + return { status: result.status, text: result.stdout + result.stderr }; +} +function check(path = bundle, candidateBase = base, candidateHead = head) { + return cli('check', '--base', candidateBase, '--head', candidateHead, '--bundle', path); +} + +beforeAll(() => { + for (const path of [ + 'scripts/quality-evidence.mjs', + 'scripts/quality', + 'vitest.config.ts', + 'vitest.browser.config.ts', + 'server/vitest.config.ts', + 'eslint.config.js', + '.fallowrc.json', + ]) { + mkdirSync(dirname(resolve(fixture, path)), { recursive: true }); + cpSync(resolve(repository, path), resolve(fixture, path), { recursive: true }); + } + write( + 'vitest.config.ts', + readFileSync(resolve(repository, 'vitest.config.ts'), 'utf8').replace( + "setupFiles: ['fake-indexeddb/auto'],", + '' + ) + ); + write( + 'vitest.browser.config.ts', + readFileSync(resolve(repository, 'vitest.browser.config.ts'), 'utf8').replace( + 'defineConfig({', + `defineConfig({ server: { fs: { allow: [${JSON.stringify(repository)}, ${JSON.stringify(fixture)}] } },` + ) + ); + symlinkSync(resolve(repository, 'node_modules'), resolve(fixture, 'node_modules'), 'dir'); + write( + '.gitignore', + 'node_modules\n.fallow/\n**/dist/\nworkers/telemetry/worker-configuration.d.ts\n' + ); + write( + 'package.json', + JSON.stringify({ + type: 'module', + workspaces: ['workers/telemetry'], + scripts: { + 'shared:build': 'node --eval "0"', + 'server:build': 'node --eval "0"', + 'shared:node-test': 'node shared/tests/node-import.mjs', + }, + }) + ); + write('package-lock.json', readFileSync(resolve(repository, 'package-lock.json'), 'utf8')); + write('shared/package.json', readFileSync(resolve(repository, 'shared/package.json'), 'utf8')); + write( + 'workers/telemetry/package.json', + readFileSync(resolve(repository, 'workers/telemetry/package.json'), 'utf8') + ); + write( + 'workers/telemetry/wrangler.jsonc', + '{"name":"quality-fixture","main":"src/index.ts","compatibility_date":"2026-01-01"}' + ); + write('src/main.ts', 'export const main = 1;\n'); + write('src/value.ts', 'export const value = 2;\n'); + write('src/zero.ts', 'export function uncalled() { return 3; }\n'); + write('src/types.ts', 'export interface Value { value: number }\n'); + write('shared/src/index.ts', 'export const shared = 3;\n'); + write('server/src/index.ts', 'export const server = 4;\n'); + write('workers/telemetry/src/index.ts', 'export const worker = 5;\n'); + write( + 'shared/tests/node-import.mjs', + 'import assert from "node:assert/strict"; assert.equal(1 + 1, 2);\n' + ); + for (const [path, source, name, value] of [ + ['tests/value.test.ts', '../src/value.ts', 'value', 2], + ['tests/browser/value.test.ts', '../../src/main.ts', 'main', 1], + ['tests/shared/project-contract.test.ts', '../../shared/src/index.ts', 'shared', 3], + ['tests/shared/product-events.test.ts', '../../shared/src/index.ts', 'shared', 3], + ['server/tests/value.test.ts', '../src/index.ts', 'server', 4], + ['tests/workers/telemetry-worker.test.ts', '../../workers/telemetry/src/index.ts', 'worker', 5], + ] as const) + write( + path, + `import { expect, it } from 'vitest'; import { ${name} } from '${source}'; it('executes ${name}', () => expect(${name}).toBe(${value}));\n` + ); + git('init', '-q'); + git('config', 'user.email', 'quality@example.invalid'); + git('config', 'user.name', 'Quality fixture'); + git('add', '.'); + git('commit', '-qm', 'fixture base'); + base = git('rev-parse', 'HEAD'); + write('README.md', 'Quality fixture candidate.\n'); + git('add', '.'); + git('commit', '-qm', 'fixture head'); + head = git('rev-parse', 'HEAD'); + const result = cli('verify', '--base', base); + const logs = readdirSync(resolve(bundle, 'commands')) + .filter((name) => name.endsWith('.log')) + .map((name) => readFileSync(resolve(bundle, 'commands', name), 'utf8')) + .join('\n'); + expect(result, result.text + logs).toMatchObject({ status: 0 }); +}, 120_000); + +afterAll(() => rmSync(fixture, { recursive: true, force: true })); + +describe('public quality evidence CLI', () => { + it('collects all five real producers and accepts a relocated intact bundle', () => { + expect(check().status).toBe(0); + const manifest = JSON.parse(readFileSync(resolve(bundle, 'manifest.json'), 'utf8')); + expect(manifest.profiles.map((profile: { id: string }) => profile.id)).toEqual([ + 'root', + 'browser', + 'shared', + 'server', + 'worker', + ]); + const rootMap = JSON.parse(readFileSync(resolve(bundle, 'root/coverage-final.json'), 'utf8')); + expect(rootMap[resolve(fixture, 'src/main.ts')]).toBeUndefined(); + expect( + Object.values(rootMap[resolve(fixture, 'src/zero.ts')].s).every((hits) => hits === 0) + ).toBe(true); + expect(rootMap[resolve(fixture, 'src/types.ts')].s).toEqual({}); + const relocated = resolve(fixture, '.fallow/downloaded'); + cpSync(bundle, relocated, { recursive: true }); + expect(check(relocated).status).toBe(0); + expect(readFileSync(resolve(relocated, 'manifest.json'), 'utf8')).toBe( + readFileSync(resolve(bundle, 'manifest.json'), 'utf8') + ); + }); + + it('rejects absent, malformed, incomplete, foreign, or mixed artifacts', () => { + expect(check(resolve(fixture, '.fallow/absent')).status).toBe(1); + for (const [path, replacement] of [ + ['manifest.json', '{'], + ['root/coverage-final.json', '{}'], + [ + 'root/coverage-final.json', + readFileSync(resolve(bundle, 'server/coverage-final.json'), 'utf8'), + ], + [ + 'root/coverage-final.json', + readFileSync(resolve(bundle, 'root/coverage-final.json'), 'utf8').replaceAll( + fixture, + '/foreign/root' + ), + ], + ['commands/root.json', '{}'], + ['certification.json', '{}'], + ]) { + const original = readFileSync(resolve(bundle, path)); + writeFileSync(resolve(bundle, path), replacement); + expect(check().status, path).toBe(1); + writeFileSync(resolve(bundle, path), original); + } + }); + + it('rejects wrong revisions, tool/config/scope identities and changed inputs', () => { + expect(check(bundle, head).status).toBe(1); + expect(check(bundle, base, base).status).toBe(1); + const path = resolve(bundle, 'manifest.json'); + const original = readFileSync(path, 'utf8'); + for (const mutate of [ + (m: any) => { + m.producer.tools.vitest = '0.0.0'; + }, + (m: any) => { + m.profiles[0].configs = []; + }, + (m: any) => { + m.profiles[0].sources = []; + }, + (m: any) => { + m.profiles[0].tests = []; + }, + (m: any) => { + m.producer.producerRoot = '/foreign/root'; + }, + (m: any) => { + m.artifacts['../outside'] = 'invalid'; + }, + ]) { + const manifest = JSON.parse(original); + mutate(manifest); + writeFileSync(path, JSON.stringify(manifest)); + expect(check().status).toBe(1); + writeFileSync(path, original); + } + write('README.md', 'Changed tracked input.'); + expect(check().status).toBe(1); + git('restore', 'README.md'); + write('new-input.txt', 'Nonignored input.'); + expect(check().status).toBe(1); + rmSync(resolve(fixture, 'new-input.txt')); + expect(check().status).toBe(0); + }); + + it('invalidates certification before a failed producer and retains its diagnostics', () => { + write( + 'tests/value.test.ts', + "import { it } from 'vitest'; it('fails', () => { throw new Error('producer failure'); });\n" + ); + git('add', '.'); + git('commit', '-qm', 'failed test producer'); + const result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(existsSync(resolve(bundle, 'certification.json'))).toBe(false); + expect(readFileSync(resolve(bundle, 'root/tests.json'), 'utf8')).toContain('producer failure'); + }, 60_000); +}); From 0959137b3139fcc8c3c0e751fb1f4a6762b1e0a4 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 00:53:41 +0200 Subject: [PATCH 02/11] feat(quality): enforce runtime boundaries and reconcile Fallow evidence (#614) --- .fallowrc.json | 72 ++++++- eslint.config.js | 52 +++++ scripts/quality-evidence.mjs | 13 +- scripts/quality/fallow.mjs | 257 +++++++++++++++++++++++ tests/scripts/quality-boundaries.test.ts | 88 ++++++++ tests/scripts/quality-evidence.test.ts | 81 +++++++ 6 files changed, 560 insertions(+), 3 deletions(-) create mode 100644 scripts/quality/fallow.mjs create mode 100644 tests/scripts/quality-boundaries.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 91c11277..95c9b534 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -25,5 +25,75 @@ "createRenderRoot", "shouldUpdate", "update" - ] + ], + "boundaries": { + "zones": [ + { + "name": "browser", + "patterns": ["src/**"] + }, + { + "name": "shared", + "patterns": ["shared/src/**"] + }, + { + "name": "server", + "patterns": ["server/src/**"] + }, + { + "name": "worker", + "patterns": ["workers/telemetry/src/**"] + }, + { + "name": "tooling", + "patterns": ["scripts/**"] + }, + { + "name": "root-tests", + "patterns": ["tests/**"] + }, + { + "name": "server-tests", + "patterns": ["server/tests/**"] + }, + { + "name": "shared-tests", + "patterns": ["shared/tests/**"] + } + ], + "rules": [ + { + "from": "browser", + "allow": ["browser", "shared"] + }, + { + "from": "shared", + "allow": ["shared"] + }, + { + "from": "server", + "allow": ["server", "shared"] + }, + { + "from": "worker", + "allow": ["worker", "shared"] + }, + { + "from": "tooling", + "allow": ["tooling", "shared"] + }, + { + "from": "root-tests", + "allow": ["browser", "shared", "worker", "tooling", "root-tests"] + }, + { + "from": "server-tests", + "allow": ["server", "shared", "server-tests"] + }, + { + "from": "shared-tests", + "allow": ["shared", "shared-tests"] + } + ] + } } diff --git a/eslint.config.js b/eslint.config.js index e37876e2..1344ed25 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,7 +1,59 @@ import js from '@eslint/js'; import tseslint from 'typescript-eslint'; +// Local source edges belong to Fallow. These rules cover package specifiers +// and reject destinations that cannot be checked statically. +function runtimeImports(files, name, packages, node = false) { + const escaped = packages.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + const allowed = `^(?:\\.\\.?/|${node ? 'node:|' : ''}${escaped.length ? `(?:${escaped.join('|')})(?:/|$)` : '(?!)'})`; + const forbidden = `^(?!${allowed.slice(1)})`; + const message = `${name} imports must follow its runtime package contract.`; + return { + files, + rules: { + 'no-restricted-imports': ['error', { patterns: [{ regex: forbidden, message }] }], + 'no-restricted-syntax': [ + 'error', + { + selector: `ImportExpression[source.type='Literal'][source.value=/${forbidden.replaceAll('/', '\\/')}/]`, + message, + }, + { + selector: "ImportExpression[source.type!='Literal']", + message: `${name} computed imports cannot establish a runtime destination.`, + }, + ], + }, + }; +} + export default tseslint.config( + runtimeImports(['src/**/*.ts'], 'Browser', [ + '@pixel-forge/shared', + '@lit-labs/signals', + '@sentry/browser', + 'idb', + 'lit', + 'pako', + 'uuid', + 'virtual:pwa-register', + ]), + runtimeImports(['shared/src/**/*.ts'], 'Shared', []), + runtimeImports( + ['server/src/**/*.ts'], + 'Server', + [ + '@pixel-forge/shared', + '@aws-sdk/client-s3', + '@hono/node-server', + 'drizzle-orm', + 'hono', + 'postgres', + ], + true + ), + runtimeImports(['workers/telemetry/src/**/*.ts'], 'Worker', ['@pixel-forge/shared']), + runtimeImports(['scripts/**/*.{mjs,js,ts}'], 'Tooling', ['@pixel-forge/shared'], true), { ignores: ['dist/', 'node_modules/', 'coverage/'] }, js.configs.recommended, ...tseslint.configs.recommended, diff --git a/scripts/quality-evidence.mjs b/scripts/quality-evidence.mjs index 2dc742db..b0fced58 100644 --- a/scripts/quality-evidence.mjs +++ b/scripts/quality-evidence.mjs @@ -9,6 +9,7 @@ import { validateBundle, writeJson, } from './quality/bundle.mjs'; +import { collectFallow, validateFallow } from './quality/fallow.mjs'; import { validateCoverage } from './quality/coverage.mjs'; const root = process.cwd(); @@ -35,7 +36,8 @@ try { ]); } else if (action === 'check') { bundle = resolve(root, options['--bundle'] ?? '.fallow/quality'); - validateBundle(root, bundle, options['--base'], options['--head']); + const manifest = validateBundle(root, bundle, options['--base'], options['--head']); + validateFallow(root, bundle, manifest); console.log(`Accepted quality evidence for ${options['--head']}`); } else if (action === 'verify') { if (options['--bundle'] || options['--head']) @@ -66,7 +68,14 @@ try { { head: producer.head, dirty: producer.dirty, inputs: producer.inputs }, 'post-run checkout' ); - saveManifest(bundle, producer, profileList, commands); + const manifest = saveManifest(bundle, producer, profileList, commands); + collectFallow(root, bundle, manifest); + validateFallow(root, bundle, manifest); + same( + identity(root), + { head: producer.head, dirty: producer.dirty, inputs: producer.inputs }, + 'post-constraint checkout' + ); validateBundle(root, bundle, producer.base, producer.head, false); writeJson(resolve(bundle, 'certification.json'), { base: producer.base, diff --git a/scripts/quality/fallow.mjs b/scripts/quality/fallow.mjs new file mode 100644 index 00000000..e0381aba --- /dev/null +++ b/scripts/quality/fallow.mjs @@ -0,0 +1,257 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { digest, environment, readJson } from './config.mjs'; +import { artifact, artifactDigests, runCommand, same, writeJson } from './bundle.mjs'; + +function findingLocation(finding) { + if (typeof finding.path !== 'string') throw new Error('unsupported Fallow finding location'); + return { path: finding.path, line: finding.line ?? 1, column: (finding.col ?? 0) + 1 }; +} + +function sarifLocation(result) { + const location = result.locations?.[0]?.physicalLocation; + if (!location?.artifactLocation?.uri) throw new Error('missing SARIF finding location'); + return { + path: decodeURI(location.artifactLocation.uri), + line: location.region?.startLine ?? 1, + column: location.region?.startColumn ?? 1, + }; +} + +function reconcile(audit, sarif, registry, base, head) { + if ( + audit.kind !== 'audit' || + audit.base_ref !== base || + !head.startsWith(audit.head_sha) || + audit.head_sha.length < 7 || + audit.attribution?.gate !== 'new-only' + ) + throw new Error('Fallow comparison identity mismatch'); + const results = sarif.runs?.flatMap((run) => run.results ?? []); + if (!Array.isArray(results) || !audit.dead_code) throw new Error('malformed Fallow reports'); + const advisories = []; + const blockers = []; + const matched = new Set(); + for (const [category, findings] of Object.entries(audit.dead_code)) { + if (!Array.isArray(findings) || !findings.length) continue; + const rule = registry.issue_types.find((entry) => entry.result_key === category); + if (!rule?.sarif_rule_ids?.length) throw new Error(`unknown Fallow category: ${category}`); + for (const finding of findings) { + if (typeof finding.introduced !== 'boolean') + throw new Error('missing Fallow introduced identity'); + const location = findingLocation(finding); + const candidates = results.filter( + (result) => + rule.sarif_rule_ids.includes(result.ruleId) && + JSON.stringify(sarifLocation(result)) === JSON.stringify(location) + ); + if (candidates.length !== 1 || matched.has(candidates[0])) + throw new Error(`ambiguous Fallow severity join: ${category} ${JSON.stringify(location)}`); + const result = candidates[0]; + matched.add(result); + if (!['error', 'warning', 'note', 'none'].includes(result.level)) + throw new Error('unknown Fallow severity'); + const entry = { + id: `${result.ruleId}:${location.path}:${location.line}:${location.column}`, + category, + level: result.level, + introduced: finding.introduced, + location, + }; + if (category.startsWith('boundary_') || (finding.introduced && result.level === 'error')) + blockers.push(entry); + else if (finding.introduced && result.level !== 'none') advisories.push(entry); + } + } + const heuristicRules = new Set( + registry.issue_types + .filter((entry) => ['health', 'dupes'].includes(entry.command)) + .flatMap((entry) => [entry.rule_id, ...(entry.sarif_rule_ids ?? [])]) + ); + for (const result of results) + if (!matched.has(result) && !heuristicRules.has(result.ruleId)) + throw new Error(`unmatched Fallow result: ${result.ruleId}`); + for (const finding of audit.complexity?.findings ?? []) + advisories.push({ + id: `complexity:${finding.path}:${finding.line}:${finding.col}`, + category: 'complexity', + introduced: finding.introduced, + finding, + }); + for (const group of audit.duplication?.clone_groups ?? []) + advisories.push({ + id: group.id ?? group.fingerprint, + category: 'duplication', + introduced: group.introduced, + finding: group, + }); + return { + policy: + 'introduced non-heuristic errors and all boundaries block; complexity and duplication advisory', + blockers, + advisories, + }; +} + +function commands(producer, profiles) { + const fallow = 'node_modules/.bin/fallow'; + const audit = ['audit', '--base', producer.base, '--gate', 'new-only', '--no-cache']; + return [ + { + id: 'boundaries', + command: [fallow, 'dead-code', '--boundary-violations', '--format', 'json', '--no-cache'], + output: 'boundaries.json', + allowed: [0, 1], + }, + { + id: 'boundary-inventory', + command: [fallow, 'list', '--boundaries', '--format', 'json'], + output: 'boundary-inventory.json', + allowed: [0], + }, + { + id: 'runtime-imports', + command: [ + 'node_modules/.bin/eslint', + 'src', + 'shared/src', + 'server/src', + 'workers/telemetry/src', + 'scripts', + ], + output: null, + allowed: [0], + }, + { + id: 'audit-json', + command: [fallow, ...audit, '--format', 'json'], + output: 'audit.json', + allowed: [0, 1], + }, + { + id: 'audit-sarif', + command: [fallow, ...audit, '--format', 'sarif'], + output: 'audit.sarif.json', + allowed: [0, 1], + }, + ...profiles.map((profile) => ({ + id: `health-${profile.id}`, + command: [ + fallow, + 'health', + '--report-only', + '--file-scores', + '--complexity', + '--max-crap', + '1', + '--coverage', + resolve(producer.producerRoot, '.fallow/quality', profile.id, 'coverage-final.json'), + '--coverage-root', + producer.producerRoot, + '--format', + 'json', + '--no-cache', + ], + output: `${profile.id}/fallow.json`, + allowed: [0], + })), + ]; +} + +export function collectFallow(root, bundle, manifest) { + const records = commands(manifest.producer, manifest.profiles).map((item) => + runCommand(root, bundle, item.id, item.command, '.', item.allowed, item.output) + ); + const registry = readJson(resolve(root, 'node_modules/fallow/issue-registry.json')); + const review = reconcile( + readJson(resolve(bundle, 'audit.json')), + readJson(resolve(bundle, 'audit.sarif.json')), + registry, + manifest.producer.base, + manifest.producer.head + ); + writeJson(resolve(bundle, 'review.json'), review); + const paths = ['review.json']; + for (const record of records) + paths.push(record.log, `commands/${record.id}.json`, ...(record.output ? [record.output] : [])); + manifest.fallow = { + registry: digest(readFileSync(resolve(root, 'node_modules/fallow/issue-registry.json'))), + records, + review, + }; + manifest.artifacts = artifactDigests(bundle, [...Object.keys(manifest.artifacts), ...paths]); + writeJson(resolve(bundle, 'manifest.json'), manifest); +} + +export function validateFallow(root, bundle, manifest) { + if (!manifest.fallow) throw new Error('missing Fallow evidence'); + const { records, review } = manifest.fallow; + same( + manifest.fallow.registry, + digest(readFileSync(resolve(root, 'node_modules/fallow/issue-registry.json'))), + 'Fallow registry' + ); + const expected = commands(manifest.producer, manifest.profiles); + same( + records.map(({ id, command, output }) => ({ id, command, output })), + expected.map(({ id, command, output }) => ({ id, command, output })), + 'Fallow command inventory' + ); + for (let index = 0; index < records.length; index++) { + const record = records[index]; + same(record.cwd, '.', 'constraint working directory'); + same(record.environment, environment, 'constraint environment'); + same(record.log, `commands/${record.id}.log`, 'constraint log'); + if ( + !Number.isFinite(record.durationMs) || + record.durationMs < 0 || + !Number.isFinite(Date.parse(record.startedAt)) + ) + throw new Error('invalid constraint timing'); + if (!expected[index].allowed.includes(record.status) || record.error) + throw new Error(`${record.id}: failed constraint producer`); + same(readJson(artifact(bundle, `commands/${record.id}.json`)), record, 'Fallow command record'); + for (const path of [ + record.log, + `commands/${record.id}.json`, + ...(record.output ? [record.output] : []), + ]) + if (!manifest.artifacts[path]) throw new Error(`missing constraint artifact: ${path}`); + } + const boundaries = readJson(artifact(bundle, 'boundaries.json')); + if (boundaries.kind !== 'dead-code' || boundaries.total_issues !== 0) + throw new Error('runtime boundary violation'); + const inventory = readJson(artifact(bundle, 'boundary-inventory.json')); + const configured = readJson(resolve(root, '.fallowrc.json')).boundaries; + same( + inventory.boundaries.zones.map(({ name, patterns }) => ({ name, patterns })), + configured.zones, + 'Fallow zone inventory' + ); + for (const profile of manifest.profiles) { + const health = readJson(artifact(bundle, `${profile.id}/fallow.json`)); + if ( + health.kind !== 'health' || + health.error || + health.version !== manifest.producer.tools.fallow || + health.summary?.coverage_model !== 'istanbul' || + typeof health.summary?.coverage_source_consistency !== 'string' || + !Number.isInteger(health.summary?.istanbul_matched) || + !Number.isInteger(health.summary?.istanbul_total) + ) + throw new Error(`${profile.id}: malformed Fallow provenance`); + } + const reconciled = reconcile( + readJson(artifact(bundle, 'audit.json')), + readJson(artifact(bundle, 'audit.sarif.json')), + readJson(resolve(root, 'node_modules/fallow/issue-registry.json')), + manifest.producer.base, + manifest.producer.head + ); + same(review, reconciled, 'Fallow acceptance policy'); + same(readJson(artifact(bundle, 'review.json')), review, 'separate Fallow review'); + if (review.blockers.length) + throw new Error( + `required Fallow findings: ${review.blockers.map((finding) => finding.id).join(', ')}` + ); +} diff --git a/tests/scripts/quality-boundaries.test.ts b/tests/scripts/quality-boundaries.test.ts new file mode 100644 index 00000000..fe3b70a1 --- /dev/null +++ b/tests/scripts/quality-boundaries.test.ts @@ -0,0 +1,88 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { afterAll, beforeAll, expect, it } from 'vitest'; + +const repository = process.cwd(); +const fixture = mkdtempSync(resolve(tmpdir(), 'pixel-forge-boundaries-')); + +beforeAll(() => { + symlinkSync(resolve(repository, 'node_modules'), resolve(fixture, 'node_modules'), 'dir'); + for (const file of ['eslint.config.js', '.fallowrc.json']) + cpSync(resolve(repository, file), resolve(fixture, file)); + for (const directory of ['src', 'server/src', 'shared/src']) + mkdirSync(resolve(fixture, directory), { recursive: true }); + writeFileSync(resolve(fixture, 'package.json'), '{"type":"module"}'); + writeFileSync(resolve(fixture, '.gitignore'), 'node_modules\n.fallow\n'); + execFileSync('git', ['init', '-q'], { cwd: fixture }); +}); +afterAll(() => rmSync(fixture, { recursive: true, force: true })); + +it('blocks local source crossings even outside a changed-files audit', () => { + writeFileSync(resolve(fixture, 'server/src/index.ts'), 'export const server = 1;'); + writeFileSync( + resolve(fixture, 'src/main.ts'), + "import { server } from '../server/src/index.ts'; console.log(server);" + ); + const result = spawnSync( + resolve(repository, 'node_modules/.bin/fallow'), + ['dead-code', '--boundary-violations', '--format', 'json', '--no-cache'], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status).toBe(1); + expect(JSON.parse(result.stdout).boundary_violations).toHaveLength(1); +}); + +it('rejects static, literal dynamic and computed imports in each production runtime', () => { + for (const [filename, specifier] of [ + ['src/fixture.ts', 'hono'], + ['shared/src/fixture.ts', 'node:fs'], + ['server/src/fixture.ts', 'lit'], + ['workers/telemetry/src/fixture.ts', 'lit'], + ['scripts/fixture.mjs', 'lit'], + ]) { + for (const source of [ + `import * as value from '${specifier}'; export { value };`, + `export const value = import('${specifier}');`, + "const destination = './local.js'; export const value = import(destination);", + ]) { + const result = spawnSync( + resolve(repository, 'node_modules/.bin/eslint'), + ['--stdin', '--stdin-filename', filename, '--format', 'json'], + { cwd: fixture, encoding: 'utf8', input: source } + ); + expect(result.status, result.stderr).toBe(1); + expect( + JSON.parse(result.stdout)[0].messages.some((message: { ruleId: string }) => + ['no-restricted-imports', 'no-restricted-syntax'].includes(message.ruleId) + ) + ).toBe(true); + } + } +}, 30_000); + +it('keeps a type-accepted wrong-runtime package blocked by the import policy', () => { + writeFileSync( + resolve(fixture, 'shared/src/fixture.ts'), + "import { readFileSync } from 'node:fs'; export { readFileSync };" + ); + const result = spawnSync( + resolve(repository, 'node_modules/.bin/tsc'), + [ + '--noEmit', + '--skipLibCheck', + '--target', + 'es2022', + '--lib', + 'es2022', + '--module', + 'nodenext', + '--types', + 'node', + 'shared/src/fixture.ts', + ], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status, result.stdout + result.stderr).toBe(0); +}); diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 9ffb8cbe..13a30a5b 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; import { cpSync, @@ -116,6 +117,15 @@ beforeAll(() => { path, `import { expect, it } from 'vitest'; import { ${name} } from '${source}'; it('executes ${name}', () => expect(${name}).toBe(${value}));\n` ); + const fallow = JSON.parse(readFileSync(resolve(fixture, '.fallowrc.json'), 'utf8')); + fallow.entry.push('scripts/fixture-entry.mjs'); + fallow.rules = { 'unused-exports': 'warn', 'unresolved-imports': 'error' }; + write('.fallowrc.json', JSON.stringify(fallow)); + write( + 'scripts/fixture-entry.mjs', + "import { used } from './fixture-lib.mjs'; console.log(used);\n" + ); + write('scripts/fixture-lib.mjs', 'export function used() { return 1; }\n'); git('init', '-q'); git('config', 'user.email', 'quality@example.invalid'); git('config', 'user.name', 'Quality fixture'); @@ -187,6 +197,34 @@ describe('public quality evidence CLI', () => { } }); + it('validates coverage structure even when a changed artifact has a matching digest', () => { + const path = 'root/coverage-final.json'; + const original = readFileSync(resolve(bundle, path), 'utf8'); + const savedManifest = readFileSync(resolve(bundle, 'manifest.json'), 'utf8'); + const map = JSON.parse(original); + const incomplete = { ...map }; + delete incomplete[resolve(fixture, 'src/zero.ts')]; + const malformed = JSON.parse(original); + malformed[resolve(fixture, 'src/value.ts')].s = { 0: -1 }; + for (const [replacement, message] of [ + ['{}', 'empty or malformed'], + [JSON.stringify(incomplete), 'incomplete or wrong-profile'], + [JSON.stringify(malformed), 'malformed'], + [original.replaceAll(fixture, '/foreign/root'), 'foreign'], + [readFileSync(resolve(bundle, 'server/coverage-final.json'), 'utf8'), 'wrong-profile'], + ]) { + writeFileSync(resolve(bundle, path), replacement); + const manifest = JSON.parse(savedManifest); + manifest.artifacts[path] = createHash('sha256').update(replacement).digest('hex'); + writeFileSync(resolve(bundle, 'manifest.json'), JSON.stringify(manifest)); + const result = check(); + expect(result.status).toBe(1); + expect(result.text).toContain(message); + } + writeFileSync(resolve(bundle, path), original); + writeFileSync(resolve(bundle, 'manifest.json'), savedManifest); + }); + it('rejects wrong revisions, tool/config/scope identities and changed inputs', () => { expect(check(bundle, head).status).toBe(1); expect(check(bundle, base, base).status).toBe(1); @@ -227,6 +265,49 @@ describe('public quality evidence CLI', () => { expect(check().status).toBe(0); }); + it('preserves new-only required errors while native complexity errors stay advisory', () => { + const commit = (source: string) => { + write('scripts/fixture-lib.mjs', source); + git('add', '.'); + git('commit', '-qm', 'audit fixture'); + head = git('rev-parse', 'HEAD'); + }; + commit('export function used() { return 1; }\nexport const unused = 2;\n'); + let result = cli('verify', '--base', base); + expect(result, result.text).toMatchObject({ status: 0 }); + expect( + JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')).advisories.some( + (finding: any) => finding.category === 'unused_exports' + ) + ).toBe(true); + commit("import { missing } from './missing.js';\nexport function used() { return missing; }\n"); + result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(result.text).toContain('required Fallow findings'); + const inheritedBase = head; + commit( + "import { missing } from './missing.js';\nexport function used() { return missing; }\nexport const unused = 2;\n" + ); + result = cli('verify', '--base', inheritedBase); + expect(result, result.text).toMatchObject({ status: 0 }); + const review = JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')); + expect(review.blockers).toEqual([]); + expect(review.advisories.some((finding: any) => finding.category === 'unused_exports')).toBe( + true + ); + commit( + `export function used(value) {\n${Array.from({ length: 25 }, (_, index) => `if (value === ${index}) return ${index};`).join('\n')}\nreturn -1;\n}\n` + ); + result = cli('verify', '--base', base); + expect(result, result.text).toMatchObject({ status: 0 }); + expect(JSON.parse(readFileSync(resolve(bundle, 'audit.json'), 'utf8')).verdict).toBe('fail'); + expect( + JSON.parse(readFileSync(resolve(bundle, 'review.json'), 'utf8')).advisories.some( + (finding: any) => finding.category === 'complexity' + ) + ).toBe(true); + }, 180_000); + it('invalidates certification before a failed producer and retains its diagnostics', () => { write( 'tests/value.test.ts', From 7fc291c74775458ca15108f1dd9e83d32f26378b Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 00:58:37 +0200 Subject: [PATCH 03/11] ci(quality): retain exact-head evidence and document acceptance (#615) --- .agentic-loop.yml | 3 +- .codex/agents/delivery-worker.toml | 7 +- .codex/agents/pr-finalizer.toml | 11 +- .codex/agents/workflow-director.toml | 6 +- .github/workflows/ci.yml | 45 +-- AGENTS.md | 5 +- docs/agents/change-map.md | 2 +- docs/agents/handoffs.md | 5 + docs/agents/quality-verification.md | 147 +++++++++ docs/operations/codex-agent-workflow.md | 7 +- scripts/quality/bundle.mjs | 8 +- scripts/quality/config.mjs | 11 +- scripts/quality/coverage.mjs | 69 +++-- specs/research/clean-code-quality-evidence.md | 291 ++++++++++++++++++ tests/scripts/quality-evidence.test.ts | 17 + 15 files changed, 561 insertions(+), 73 deletions(-) create mode 100644 docs/agents/quality-verification.md create mode 100644 specs/research/clean-code-quality-evidence.md diff --git a/.agentic-loop.yml b/.agentic-loop.yml index cdf875d0..255a1f12 100644 --- a/.agentic-loop.yml +++ b/.agentic-loop.yml @@ -52,9 +52,8 @@ verification: commands: - "npx tsc --noEmit" - "npm run lint" - - "npm run test:run" + - 'npm run quality:verify -- --base "${FALLOW_AUDIT_BASE:?Set the exact approved integration base SHA}"' - "npm run build" - - 'FALLOW_AUDIT_BASE="${FALLOW_AUDIT_BASE:?Set the exact approved integration base SHA}" npm run fallow:audit' - "git diff --check" human: diff --git a/.codex/agents/delivery-worker.toml b/.codex/agents/delivery-worker.toml index 2b66ce06..c0dc7ccf 100644 --- a/.codex/agents/delivery-worker.toml +++ b/.codex/agents/delivery-worker.toml @@ -48,11 +48,12 @@ revision, source/test scope, exclusions, result, evidence path and provenance; and the separate Standards and Spec review inputs. A body digest is computed from the parsed GitHub body bytes without trimming or adding a CLI newline. Use the exact starting origin/develop SHA in -FALLOW_AUDIT_BASE= npm run fallow:audit and verify Fallow's reported range. +npm run quality:verify -- --base and verify the retained bundle with +quality:check as described in docs/agents/quality-verification.md. When every linked issue's acceptance criteria are satisfied, run relevant -focused checks plus npx tsc --noEmit, npm run lint, npm run test:run, npm run -build, FALLOW_AUDIT_BASE= npm run fallow:audit, +focused checks plus npx tsc --noEmit, npm run lint, +npm run quality:verify -- --base , npm run build, and git diff --check. Keep the PR in draft, do not merge, and hand the complete PR diff and exact head to pr-finalizer. Your task ends at that handoff unless the workflow director explicitly returns a bounded diff --git a/.codex/agents/pr-finalizer.toml b/.codex/agents/pr-finalizer.toml index 33fa7bf8..9fe93607 100644 --- a/.codex/agents/pr-finalizer.toml +++ b/.codex/agents/pr-finalizer.toml @@ -30,18 +30,17 @@ docs/agents/handoffs.md. Report Standards and Spec conclusions separately. Finalizer findings name severity, evidence, affected requirement or standard, and status; the Director verifies each finding and adds the disposition and rationale. Use -FALLOW_AUDIT_BASE= npm run fallow:audit and -verify the report range; a bare, branch-upstream, or self-comparison audit is -not delivery evidence. +npm run quality:check -- --base --head +and docs/agents/quality-verification.md to accept the retained bundle. You may edit the PR branch only for a correction or simplification that is directly required by the approved tickets, stays inside their write boundaries and non-goals, preserves public behavior unless repairing a specified defect, and has a convincing verification path. Prefer direct, boring code. Commit all such finalizer changes in one focused commit, push the same branch, and rerun -the affected checks plus npx tsc --noEmit, npm run lint, npm run test:run, npm -run build, FALLOW_AUDIT_BASE= npm run -fallow:audit, and git diff --check. +the affected checks plus npx tsc --noEmit, npm run lint, +npm run quality:verify -- --base , +npm run build, and git diff --check. Return exactly one verdict: diff --git a/.codex/agents/workflow-director.toml b/.codex/agents/workflow-director.toml index 7b807f31..342b634c 100644 --- a/.codex/agents/workflow-director.toml +++ b/.codex/agents/workflow-director.toml @@ -40,9 +40,9 @@ configuration, revision, measured scopes, exclusions, result, evidence location, and provenance; and separate Standards and Spec findings with an explicit disposition and rationale. Compare requirement content when its body or governing decision changes. A changed head, wrong scope, absent evidence, -or wrong checkout invalidates affected approval and evidence. For local Fallow, -pin FALLOW_AUDIT_BASE to the exact starting origin/develop SHA and verify the -reported comparison range. +or wrong checkout invalidates affected approval and evidence. Use +docs/agents/quality-verification.md and quality:check with the exact starting +origin/develop base and candidate head to accept retained quality evidence. Every delivery brief names the outcome, acceptance criteria, non-goals, write boundaries, contract and runtime surfaces, decision and safety gates, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c0adc8a..731eb448 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,17 @@ permissions: contents: read pull-requests: read +env: + QUALITY_BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + QUALITY_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + jobs: checks: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false @@ -28,14 +33,14 @@ jobs: - name: Install Chromium for Browser Mode run: npx playwright install --with-deps --only-shell chromium - - name: Shared workspace checks - run: npm run shared:check + - name: Shared typecheck + run: npm run shared:typecheck - - name: Server workspace checks - run: npm run server:check + - name: Server typecheck and lint + run: npm run server:typecheck && npm run server:lint - - name: Telemetry worker checks - run: npm run telemetry:typecheck && npm run telemetry:test && npm run telemetry:dry-run + - name: Telemetry worker typecheck and dry run + run: npm run telemetry:typecheck && npm run telemetry:dry-run - name: Typecheck run: npx tsc --noEmit @@ -43,32 +48,34 @@ jobs: - name: Lint run: npm run lint - - name: Tests - run: npm run test:run + - name: Quality evidence + run: | + npm run quality:verify -- --base "$QUALITY_BASE" + npm run quality:check -- --base "$QUALITY_BASE" --head "$QUALITY_HEAD" - - name: Browser tests - run: npm run test:browser + - name: Retain quality evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: quality-${{ env.QUALITY_HEAD }} + path: .fallow/quality + include-hidden-files: true + if-no-files-found: error + retention-days: 14 - name: Build run: npm run build - - name: Fallow audit - uses: fallow-rs/fallow@v2 - with: - command: audit - version: 3.2.0 - artifacts-dir: .fallow/ci - no-cache: true - server-container: runs-on: ubuntu-latest timeout-minutes: 15 env: - PIXEL_FORGE_IMAGE_REVISION: ${{ github.sha }} + PIXEL_FORGE_IMAGE_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} PIXEL_FORGE_SMOKE_PROJECT_NAME: pixel-forge-container-smoke-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Build and smoke the linux/amd64 server image diff --git a/AGENTS.md b/AGENTS.md index 586e8fbc..c4094039 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,14 +124,15 @@ failures, and a regression test for each confirmed bug. or externally visible identifiers that lack a compatibility path. Preserve the existing contract or add an explicit versioned migration. +Before collecting or accepting quality evidence, read +[`docs/agents/quality-verification.md`](docs/agents/quality-verification.md). Before handoff, run the relevant focused checks plus: ```sh npx tsc --noEmit npm run lint -npm run test:run +npm run quality:verify -- --base npm run build -npm run fallow:audit git diff --check ``` diff --git a/docs/agents/change-map.md b/docs/agents/change-map.md index 1ef0cb10..06384a38 100644 --- a/docs/agents/change-map.md +++ b/docs/agents/change-map.md @@ -65,7 +65,7 @@ the listed tests and runner still cover the intended behavior. - **Interface:** [`docs/operations/codex-agent-workflow.md`](../../docs/operations/codex-agent-workflow.md) is the canonical policy; [`AGENTS.md`](../../AGENTS.md), [`.agentic-loop.yml`](../../.agentic-loop.yml), and [`.codex/agents/`](../../.codex/agents/) provide entry, command, and role configuration. - **Callers:** Workflow Director/Sol shapes and reconciles GitHub state; delivery-worker/Luna implements one approved delivery slice; pr-finalizer/Terra reviews the complete draft PR. GitHub issues, PRs, checks, and retained artifacts are the durable readers and writers. -- **Tests:** `.agentic-loop.yml` configures repository gates; focused Markdown/configuration checks validate links and profile parsing, while the application and workspace runners above prove product behavior. `npm run fallow:audit` must use the exact handed-off `FALLOW_AUDIT_BASE` and record its report scope. +- **Tests:** `.agentic-loop.yml` configures repository gates; focused Markdown/configuration checks validate links and profile parsing, while the application and workspace runners above prove product behavior. `npm run quality:verify -- --base ` collects the five profiles and constraints through [`scripts/quality-evidence.mjs`](../../scripts/quality-evidence.mjs). Public refusal exercises run with `npm run quality:test`; see [quality verification](quality-verification.md) before collection or downloaded-bundle acceptance. - **Governing:** Root [`AGENTS.md`](../../AGENTS.md), [`docs/agents/issue-tracker.md`](../../docs/agents/issue-tracker.md), [`docs/agents/triage-labels.md`](../../docs/agents/triage-labels.md), and the live delivery issue govern readiness, task order, evidence, review, and the explicit human merge gate. Browser, shared workspace, server integration, and telemetry Worker commands diff --git a/docs/agents/handoffs.md b/docs/agents/handoffs.md index 146aca35..d1a17855 100644 --- a/docs/agents/handoffs.md +++ b/docs/agents/handoffs.md @@ -52,6 +52,11 @@ can review the new identity. ## Verification record +For collection, downloaded-bundle acceptance, and finalizer rechecks, follow +[quality verification](quality-verification.md). Record the bundle path, manifest +and certification digests, five separate producer scopes, and CI artifact ID, +digest and expiry alongside the command records below. + Each command is an attributable record, not just a pass label: | Field | Required value | diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md new file mode 100644 index 00000000..55376f81 --- /dev/null +++ b/docs/agents/quality-verification.md @@ -0,0 +1,147 @@ +# Quality evidence + +Before collecting delivery evidence, accepting a downloaded bundle, or finalizing +its PR, use these commands from the exact clean candidate checkout: + +```sh +npm run quality:verify -- --base +npm run quality:check -- --base --head +npm run quality:check -- --base --head --bundle +``` + +`quality:verify` invalidates the previous certification before starting a +producer. It builds the shared and server packages, runs five coverage profiles, +executes the shared Node import smoke, checks runtime imports, and reconciles +Fallow reports. It certifies only after all required evidence is valid and the +checkout still has the same HEAD and input hashes. Failure leaves logs and no +certification. `quality:check` reads existing evidence without modifying it or +rerunning producers. + +Run `npm run quality:test` for the focused public-command exercises. These use +small isolated Git repositories and installed Vitest, Chromium, Wrangler, +TypeScript, ESLint and Fallow commands. The root test selection also includes +these tests; their child repositories contain small fixture tests, so they do +not recursively execute the product suite. + +## Producers and measured scope + +The executable inventory is [config.mjs](../../scripts/quality/config.mjs). +The manifest records its resolved source and test paths, configuration hashes, +commands, environment, producer root, tools and results. + +| Profile | Test selection | Production source | Runtime | +| --------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------ | +| `root` | All `tests/**/*.test.ts` except `tests/browser/**` | `src/**/*.ts`, excluding declarations, source test files and `src/main.ts` | Node/happy-dom | +| `browser` | All `tests/browser/**/*.test.ts` | `src/**/*.ts`, excluding declarations; includes `main.ts` | Chromium through Playwright | +| `shared` | `tests/shared/project-contract.test.ts` and `product-events.test.ts` | `shared/src/**/*.ts`, excluding declarations | Node/happy-dom | +| `server` | All `server/tests/**/*.test.ts` except database and storage suites | `server/src/**/*.ts`, excluding declarations | Node | +| `worker` | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding declarations | Node/happy-dom with mocked Worker bindings | + +Each profile uses the installed Vitest V8 provider and emits its own +Istanbul-shaped JSON map. Every declared source file must appear, including +zero-hit files and empty type-only or barrel maps. The five maps are independent; +there is no combined percentage or coverage threshold. + +Absolute includes prevent Vitest 4's substring glob matching from collecting +`shared/src` and Worker files in the root profile. The public fixture preserves +this regression: its root tests import all three areas, but each profile keeps +its declared scope. `excludeAfterRemap` also applies exclusions after source-map +translation. V8 can serialize an unbounded end column as `null`; this remains a +valid location, while hit counts must be nonnegative integers. + +The Worker command runs from the root npm workspace caller, generates its types, +and records root `INIT_CWD`. It does not measure workerd or deployed bindings. +The plain Node shared import smoke is executed but not covered. Server process +and compatibility subprocesses are likewise outside their parent's V8 map. +Database and object-storage integration suites remain separate provisioned +commands; unit coverage does not imply those integrations ran. Provider and +matching limits are detailed in the +[verified research note](../../specs/research/clean-code-quality-evidence.md). + +## Bundle and acceptance + +Generated evidence stays under ignored `.fallow/quality/`: + +- `identity.json` identifies base/head, clean state, producer root and OS, + Node/npm/tool versions, and sorted hashes of tracked and nonignored untracked + inputs. Configuration and lock files are part of that input inventory. +- `profiles.json` records the five resolved scopes and configuration identities. +- `commands/` retains each command's explicit environment, result, duration, + working directory and raw log. It never serializes the ambient environment. +- Each profile directory keeps `tests.json`, `coverage-final.json`, + `coverage-summary.json`, and `fallow.json`. +- `audit.json`, `audit.sarif.json`, `boundaries.json`, and + `boundary-inventory.json` retain the independent Fallow outputs. +- `review.json` separates blockers from the advisory catalogue. +- `manifest.json` binds records and artifact hashes; `certification.json` binds + its digest to the exact base/head. `failure.log` explains a failed collection. + +Acceptance derives required commands, profile files and configuration from the +expected checkout. It checks artifact hashes, separate and embedded records, +actual executed test files, coverage schema and complete source inventories, +installed tool and registry identity, and the Fallow policy result. Missing, +failed, stale, wrong-profile, empty or foreign evidence fails closed. + +For downloaded evidence, retain the original `producerRoot`. It translates +absolute coverage paths; it never permits reading artifacts outside the selected +bundle. The consumer OS need not match the producer OS. Replacing the original +root with the download directory invalidates the evidence. + +An input change invalidates acceptance even if HEAD did not change. Restoring +the exact revision and inputs can make the same intact historical bundle valid +again. A failed collection must be rerun from the intended clean candidate; +there is no certification-history ledger. + +## Constraints and advisory review + +Fallow zones enforce local source direction for browser, shared, server, Worker, +tooling and the three test areas. The full-tree boundary scan blocks violations +in unchanged and newly discovered files. ESLint applies explicit package +allowlists to static and literal dynamic imports and rejects computed dynamic +imports in production roots. A new package needs a deliberate allowlist update. +TypeScript remains required for types and globals; it accepts some wrong-runtime +packages, including `node:fs` from shared code, and cannot replace these rules. + +The Fallow audit compares the exact base with `new-only`. Its JSON introduced +identity is joined to native SARIF severity through the installed +`fallow/issue-registry.json`, whose digest is retained. Introduced non-heuristic +errors and every boundary violation block; unknown or ambiguous joins fail. +Inherited errors cannot turn a new warning into a blocker. + +Complexity and duplication remain advisory regardless of native SARIF level or +exit 1. `review.json` lists introduced advisories and inherited configured +hotspots in the changed scope. The Director supplies explicit dispositions, +possibly grouping exact listed IDs with one rationale. The existing inherited +clone `dup:af384e5f` keeps its prior grouped disposition unless its code or +evidence changes. Raw native `npm run fallow:audit` remains a diagnostic command; +the quality command applies the accepted policy. + +Each profile's health report uses `--max-crap 1` only to enumerate function +provenance. Its raw rows are not thousands of separate review requests. Retain +`coverage_source`, `coverage_source_consistency`, `istanbul_matched` and +`istanbul_total`: Fallow can estimate unmatched functions even with a valid V8 +map. Its `coverage_model: istanbul` is an input-format label, not a producer or +proof of measured attribution. + +## CI and remaining checks + +Both CI jobs check out the actual source head. The checks job passes the exact +event base/head to the same public command. Quality collection replaces the five +unit/browser test invocations and native Fallow action. It includes shared Node +smoke; CI separately runs shared typecheck, server typecheck/lint, Worker +typecheck/dry-run, root typecheck/lint, and the browser build. The second job +retains the server container build and smoke. Database/storage integration +commands remain guarded by their separate provisioned targets. + +Before handoff, run focused checks plus root typecheck, lint, exact-base +`quality:verify`, build, and `git diff --check`. Record the applicable workspace +and container checks separately. Evidence acceptance does not replace the +Standards/Spec review, Director dispositions, fresh finalizer after `FIXED`, or +human merge approval. + +CI uploads only `.fallow/quality`, including hidden files, under `if: always()`; +missing artifacts are an error. Artifacts are named with the source head and +retained for 14 days. Record the Actions artifact ID and digest in the handoff. +After expiration, check out the same source revision, install its lockfile and +Chromium, then regenerate with the same exact base. That creates new evidence +and a new artifact identity; an expired artifact is not a passing record. diff --git a/docs/operations/codex-agent-workflow.md b/docs/operations/codex-agent-workflow.md index c87c85bc..d917a328 100644 --- a/docs/operations/codex-agent-workflow.md +++ b/docs/operations/codex-agent-workflow.md @@ -165,20 +165,21 @@ Use the smallest convincing behavior-first test set: Do not generate exhaustive equivalent-value matrices, test private helpers, or add a second harness. +Before collecting, accepting downloaded evidence, or finalizing its PR, read +[quality verification](../agents/quality-verification.md). Before handoff, run relevant focused checks plus: ```sh npx tsc --noEmit npm run lint -npm run test:run +npm run quality:verify -- --base npm run build -FALLOW_AUDIT_BASE= npm run fallow:audit git diff --check ``` The worker records the producer, configuration, exact revision, source/test scope, exclusions, result, and evidence location for every command. It verifies -the Fallow report's comparison range matches the handed-off base; a bare audit +the quality bundle and Fallow comparison range match the handed-off base; a bare audit may select `main` or a branch upstream, and a self-comparison is not delivery evidence. The worker keeps the draft pull request updated against `develop`. After every linked issue's acceptance criteria are satisfied and the diff --git a/scripts/quality/bundle.mjs b/scripts/quality/bundle.mjs index 00bbbf4f..c8ab32ee 100644 --- a/scripts/quality/bundle.mjs +++ b/scripts/quality/bundle.mjs @@ -168,11 +168,17 @@ export function validateBundle(root, bundle, base, head, certification = true) { !tests.success || tests.numFailedTests || !tests.numTotalTests || + !tests.numPassedTests || !Array.isArray(tests.testResults) ) throw new Error(`${profile.id}: failed or absent test results`); const executed = tests.testResults - .map((test) => test.name.replaceAll('\\', '/').slice(producer.producerRoot.length + 1)) + .map((test) => { + const path = test.name.replaceAll('\\', '/'); + if (!path.startsWith(`${producer.producerRoot}/`)) + throw new Error(`${profile.id}: foreign test result`); + return path.slice(producer.producerRoot.length + 1); + }) .sort(); same(executed, profile.tests, `${profile.id} executed tests`); } diff --git a/scripts/quality/config.mjs b/scripts/quality/config.mjs index eea6050a..e878c8c3 100644 --- a/scripts/quality/config.mjs +++ b/scripts/quality/config.mjs @@ -35,11 +35,14 @@ export function toolsIdentity(root) { 'typescript', 'wrangler', ]; + const locked = readJson(resolve(root, 'package-lock.json')).packages; return Object.fromEntries( - packages.map((name) => [ - name, - readJson(resolve(root, 'node_modules', name, 'package.json')).version, - ]) + packages.map((name) => { + const installed = readJson(resolve(root, 'node_modules', name, 'package.json')).version; + if (installed !== locked?.[`node_modules/${name}`]?.version) + throw new Error(`${name}: installed tool does not match package-lock.json`); + return [name, installed]; + }) ); } diff --git a/scripts/quality/coverage.mjs b/scripts/quality/coverage.mjs index fbd5d067..8ee341c3 100644 --- a/scripts/quality/coverage.mjs +++ b/scripts/quality/coverage.mjs @@ -8,6 +8,7 @@ function count(value) { return Number.isSafeInteger(value) && value >= 0; } +// V8 serializes its unbounded end columns (Infinity) as null in JSON. function location(value) { return ( object(value) && @@ -36,38 +37,48 @@ export function validateCoverage(map, profile, producerRoot) { ) throw new Error(`${profile.id}: foreign or inconsistent coverage path`); found.push(path.slice(root.length + 1)); - for (const [counters, mappings] of [ - ['s', 'statementMap'], - ['f', 'fnMap'], - ['b', 'branchMap'], - ]) { - if ( - !object(file[counters]) || - !object(file[mappings]) || - JSON.stringify(Object.keys(file[counters]).sort()) !== - JSON.stringify(Object.keys(file[mappings]).sort()) - ) - throw new Error(`${profile.id}: malformed coverage counters`); - for (const [id, value] of Object.entries(file[counters])) { - const mapping = file[mappings][id]; - if (counters === 'b') { - if ( - !Array.isArray(value) || - !value.every(count) || - !object(mapping) || - !Array.isArray(mapping.locations) || - value.length !== mapping.locations.length || - !mapping.locations.every(location) - ) - throw new Error(`${profile.id}: malformed branch coverage`); - } else if (!count(value) || !location(counters === 's' ? mapping : mapping?.loc)) { - throw new Error(`${profile.id}: malformed statement/function coverage`); - } - } - } + validateFileCounters(file, profile.id); } if (JSON.stringify(found.sort()) !== JSON.stringify(profile.sources)) throw new Error( `${profile.id}: incomplete or wrong-profile source coverage; missing ${profile.sources.filter((path) => !found.includes(path)).join(', ')}; unexpected ${found.filter((path) => !profile.sources.includes(path)).join(', ')}` ); } + +function validateFileCounters(file, profileId) { + for (const [counters, mappings] of [ + ['s', 'statementMap'], + ['f', 'fnMap'], + ['b', 'branchMap'], + ]) { + if ( + !object(file[counters]) || + !object(file[mappings]) || + JSON.stringify(Object.keys(file[counters]).sort()) !== + JSON.stringify(Object.keys(file[mappings]).sort()) + ) + throw new Error(`${profileId}: malformed coverage counters`); + for (const [id, value] of Object.entries(file[counters])) { + const mapping = file[mappings][id]; + if (counters === 'b') { + if ( + !Array.isArray(value) || + !value.every(count) || + !object(mapping) || + typeof mapping.type !== 'string' || + !location(mapping.loc) || + !Array.isArray(mapping.locations) || + value.length !== mapping.locations.length || + !mapping.locations.every(location) + ) + throw new Error(`${profileId}: malformed branch coverage`); + } else if ( + !count(value) || + !location(counters === 's' ? mapping : mapping?.loc) || + (counters === 'f' && (typeof mapping.name !== 'string' || !location(mapping.decl))) + ) { + throw new Error(`${profileId}: malformed statement/function coverage`); + } + } + } +} diff --git a/specs/research/clean-code-quality-evidence.md b/specs/research/clean-code-quality-evidence.md new file mode 100644 index 00000000..7d925456 --- /dev/null +++ b/specs/research/clean-code-quality-evidence.md @@ -0,0 +1,291 @@ +# Pixel Forge quality evidence: verified producers + +Research for [slice #604](https://github.com/Flow-Fly/pixel-forge/issues/604), +under [capability #602](https://github.com/Flow-Fly/pixel-forge/issues/602). +The inspected repository head was +`e94f52e4f8c22f78db93688f5dab3ad95430a652`, in a clean detached research +checkout. This establishes tool mechanics; it does **not** verify a delivery +head or approve a quality gate. + +The existing dependencies can produce five separate measured coverage maps. +Keep Vitest's V8 provider. No dependency change, Istanbul instrumentation, +coverage merge, score threshold, debt cleanup, or performance-test split is +justified by these probes. Fallow consumes the maps but still produces mixed +measured and estimated function attribution. + +## Evidence identity + +The isolated checkout used `npm ci`: Node `22.22.3`, npm `10.9.8`, +Vitest and `@vitest/coverage-v8` `4.1.10`, Playwright and its Vitest provider +`1.62.1` / `4.1.10`, Fallow `3.2.0`, macOS arm64. Chromium headless shell +`151.0.7922.34`, Playwright revision `1234`, was already cached. + +Raw evidence is under +[/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/). +Each `*.command.json` records the actual argument array, working directory, +explicit environment, head, UTC start/end, and exit status. Matching logs, +coverage maps, summaries, and artifact SHA-256 hashes are retained. +[environment.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/environment.json), +[source-input-hashes.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/source-input-hashes.json), +and [artifact-hashes.json](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/artifact-hashes.json) +identify the inputs and outputs. These are local research artifacts, without a +CI retention guarantee. The retained probe scripts and command records allow +regeneration at the named commit with the locked dependencies. + +## Coverage producers and scopes + +All five probes exited zero and emitted Istanbul-format `coverage-final.json` +through the **V8** provider. JSON format does not imply Istanbul +instrumentation. Each output path below is relative to the evidence directory. + +| Producer and owning configuration | Focused tests actually executed | Verified production scope | Result and map | +| ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------------------------------------------------------ | +| Root Vitest, [vitest.config.ts](../../vitest.config.ts), Node/happy-dom with `fake-indexeddb/auto` | `tests/utils/mask-utils.test.ts` | `src/**/*.ts`, excluding `**/*.d.ts`, `src/main.ts`, `src/**/*.test.ts` | 18 tests; 300 files; `root-scoped/coverage-final.json` | +| Chromium Browser Mode, [vitest.browser.config.ts](../../vitest.browser.config.ts) | `tests/browser/canonical-composition.test.ts` | `src/**/*.ts`, excluding `**/*.d.ts` | 3 tests; 301 files; `browser/coverage-final.json` | +| Shared workspace's root/happy-dom configuration, [shared/package.json](../../shared/package.json) | Both `tests/shared/project-contract.test.ts` and `tests/shared/product-events.test.ts` | `shared/src/**/*.ts`, excluding `**/*.d.ts` | 35 tests; 6 files; `shared/coverage-final.json` | +| Server Node unit runner, [server/vitest.config.ts](../../server/vitest.config.ts) | `server/tests/config.test.ts` | `src/**/*.ts` relative to `server/`, excluding `**/*.d.ts` | 25 tests; 18 files; `server/coverage-final.json` | +| Telemetry workspace's root/happy-dom unit runner, [workers/telemetry/package.json](../../workers/telemetry/package.json) | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding `**/*.d.ts` | 13 tests; 1 file; `worker-npm/coverage-final.json` | + +These source inventories include unexecuted files, not just imported modules. +The broad root probe recorded 283 nonempty files with only zero statement +counts; server recorded 16. Empty maps also occur legitimately for type-only +source or barrels; requiring executable statements in every source file would +reject valid output. Keep the existing root `src/main.ts` exclusion. Its +current configuration also includes two empty `.d.ts` entries; explicitly +excluding declarations makes the runtime source inventory unambiguous. + +The root test selection for a complete evidence run remains +`tests/**/*.test.ts` minus `tests/browser/**/*.test.ts`. Shared and Worker +tests overlap that selection but need their own source scopes and invocation +records. The server unit runner excludes `*.database.test.ts` and +`*.storage.test.ts`. The research selected fewer tests to verify mechanics; +delivery collection must run the complete declared selection. + +Vitest 4 removed `coverage.all`: set `coverage.include` explicitly to add +unexecuted source. Use separate `coverage.reportsDirectory` values because +Vitest cleans that directory. Keep explicit exclusions and apply them after +source remapping where necessary. [Vitest 4 migration](https://v4.vitest.dev/guide/migration), +[coverage configuration](https://v4.vitest.dev/config/coverage), and +[installed-version provider source](https://github.com/vitest-dev/vitest/blob/v4.1.10/packages/coverage-v8/src/provider.ts) +own these behaviors. + +The Chromium probe emitted a Vite resolution warning for the unexecuted +`src/services/pwa-registration.ts` import of `virtual:pwa-register`. +No source file disappeared: all 301 expected non-declaration files remained; +that file had seven zero-hit statements and four functions. Vitest's provider +catches a failed transform and can fall back to original source. Preserve the +warning and inventory check; this is not grounds for excluding PWA files. +Root and browser maps can have different transformed function inventories, +so their counts must not be added or implicitly merged. + +## Reproducible commands + +Use the installed Vitest binary or package scripts, with `CI=true`, and record +the working directory. The probes used these common coverage arguments: + +```sh +--coverage.enabled --coverage.provider=v8 \ +--coverage.reporter=json --coverage.reporter=json-summary \ +--coverage.include='' --coverage.exclude='**/*.d.ts' \ +--coverage.reportsDirectory='' +``` + +Append those arguments to the following invocations. Root additionally repeats +`--coverage.exclude=src/main.ts` and `--coverage.exclude=src/**/*.test.ts` to +preserve its existing exclusions. The optional `lcovonly` reporter was also +probed; it is unnecessary for the proposed Fallow input. + +```sh +# Repository root; shared build is required by the normal package flow. +npm run shared:build +./node_modules/.bin/vitest run tests/utils/mask-utils.test.ts +./node_modules/.bin/vitest run --config vitest.browser.config.ts tests/browser/canonical-composition.test.ts + +# shared/ working directory, matching its Vitest selection. +../node_modules/.bin/vitest run --root .. tests/shared/project-contract.test.ts tests/shared/product-events.test.ts + +# Build from root, then run from server/. +npm run server:build +../node_modules/.bin/vitest run --config vitest.config.ts tests/config.test.ts + +# Repository root. Verified: flags reach Vitest after Wrangler type generation. +npm run test --workspace @pixel-forge/telemetry-worker -- +``` + +The real Worker npm invocation passed 13/13 tests. Calling its inner Vitest +command directly from `workers/telemetry/` without npm's root `INIT_CWD` +failed two configuration tests by resolving the Wrangler path twice. Preserve +the root npm caller and forwarding contract. The shared script ends with a +plain Node command, so appending coverage flags to `npm run shared:test` does +not configure its earlier Vitest command; collect at the actual Vitest call. + +## What these maps do not measure + +- The telemetry runner uses happy-dom and mocked Analytics Engine/rate-limiter + bindings. It measures Worker source executed in those tests, not workerd, + Cloudflare deployment, or real bindings. Vitest V8 requires an exposed V8 + profiler; actual Cloudflare Workers do not provide that interface. + [Vitest coverage providers](https://v4.vitest.dev/guide/coverage). +- `shared/tests/node-import.mjs` passed separately after build. This plain Node + ESM/package-export smoke test is outside Vitest coverage. Browser tests and + server tests can import the compiled shared package; that does not substitute + for the shared source-map evidence. +- Server database/storage integration runners need their guarded external + targets; neither was executed here. Server process and compatibility tests + spawn built Node subprocesses, whose execution is not recorded by the parent + Vitest inspector. Retain these checks with `not-collected` coverage rather + than attributing their execution to the unit map. +- V8 has known limitations around default arguments and statements following + an unexpectedly throwing call. Coverage remains measured execution evidence + with provider limits, not proof that assertions exercise every behavior. + [Remapper limitations](https://github.com/AriPerkkio/ast-v8-to-istanbul#limitations). + +No wall-clock performance budget assertion was found by inspection of +`performance.now`, `Date.now`, timing assertions, and test timeouts. Creative +Run tests use controlled application time; server timeouts bound integration +completion. No local evidence supports importing a separate uninstrumented +performance runner. + +## Fallow 3.2.0 consumption and refusal behavior + +Source was checked at upstream tag `v3.2.0`, commit +`0d568277b6cf91e95c98cb92f8228154e58d6984`, alongside installed CLI help. +`health --coverage` accepts a single Istanbul coverage map JSON, or a directory +containing `coverage-final.json`. It does not accept LCOV or native V8 JSON for +this purpose. `--runtime-coverage` is a different runtime-analysis feature and +is unnecessary here. [Pinned CLI contract](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/main.rs#L1030). + +For diagnostic enumeration only, the probes ran: + +```sh +./node_modules/.bin/fallow health --report-only --file-scores --complexity \ + --max-crap 1 --coverage '' --format json --no-cache +``` + +`--max-crap 1` exposed attribution fields; it is **not a proposed threshold**. +With the broad probe maps, findings inside each corresponding production +scope had the following `coverage_source` values: + +| Input map | `istanbul` findings | `estimated` findings | +| ------------------------------------------- | ------------------: | -------------------: | +| Root, broad diagnostic scope including main | 5,216 | 102 | +| Chromium | 5,215 | 103 | +| Shared | 116 | 1 | +| Server unit | 84 | 0 | +| Worker unit | 10 | 0 | + +These are surfaced findings, not a complete function inventory or a coverage +percentage. No duplicate `(path, name, line, col)` finding identities occurred +in these reports. The whole-repository reports additionally estimate paths +outside the supplied map. `coverage_source: istanbul` names the consumed JSON +format; the producer is still Vitest V8. + +Fallow matches canonical file paths, then function names and source positions, +with limited nearby-name/anonymous fallbacks. It calculates the fraction of +covered statements contained inside each matched function body, falling back +to function hits when no statements exist. Unmatched functions receive an +estimate. For example, Vitest names the second `execute` method `execute_2` +while Fallow names it `execute`, leaving a real method estimated. The shared +unmatched `atob` is an ambient declaration without runtime implementation. +[Pinned matching/scoring implementation](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/engine/src/health/scoring.rs#L730). + +| Explicit coverage input exercise | Observed result | +| -------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| Valid Vitest JSON | Exit 0 under `--report-only`; `coverage_model: istanbul`, mixed function provenance | +| Missing file | Exit 2, JSON `error: true` | +| LCOV text | Exit 2, coverage parse error | +| Empty map `{}` | Exit 0; `coverage_model: istanbul`, zero matches, uniform estimated findings | +| Valid map with foreign absolute file paths | Exit 0; zero matches | +| Same foreign map plus correct absolute `--coverage-root` | Exit 0; all 84 server matches restored | + +Consequently, neither exit 0 nor `coverage_model: istanbul` proves valid +measured evidence. Preserve each finding's `coverage_source`, summary +`coverage_source_consistency`, and `istanbul_matched` / `istanbul_total`. +Use source-root rebasing only after validating the recorded producer root. +Fallow's input loader does not establish base/head or source/test/config +freshness. Its nearby-position matching especially cannot certify freshness. + +Audit classification needs a separate implementation exercise. Source inspection +confirms `kind: audit`, `verdict`, `base_ref`, `head_sha`, and introduced/inherited +category counts. However, `summary.dead_code_has_errors` considers all +changed-scope issues, whereas the default `new-only` verdict checks introduced +issues. That summary cannot replace the existing gate. Native SARIF generation +receives configured rule severities; the audit JSON dead-code serializer does +not. Do not downgrade every exit 1 or infer per-finding severity from category +counts. Preserve the existing new-only contract until a tested classifier can +separate advisory findings from required errors. The fixture appendix below +verifies these distinctions; no global Pixel Forge dead-code baseline was +established here. +[Pinned audit gate](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/audit.rs#L358), +[output contract](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/api/src/audit_output.rs#L25), +and [JSON/SARIF serializers](https://github.com/fallow-rs/fallow/blob/0d568277b6cf91e95c98cb92f8228154e58d6984/crates/cli/src/audit_output.rs#L737). + +## Recommended first delivery contract + +Keep five independently identified JSON maps and an executable inventory of +their exact test selection, source files, exclusions, runner/config/tool +versions, environment, commands, base/head, and artifact hashes. A source file +with zero hits must remain visible; type-only files may have empty counters. +Reject missing reports, error payloads, malformed maps, unexpected or absent +source paths, inconsistent internal file paths, stale inputs, and incorrect +base/head before consuming coverage. Check map structure and counters, not +just the file name or existence. + +Retain raw maps, command logs, tool reports, inventory, and input/output hashes +as one identifiable CI artifact. State retention and regenerate from the exact +commit/lockfile/configuration when artifacts expire. Recollection is new +evidence and cannot reuse the expired artifact's identity. +If the output lives under `.fallow/quality`, explicitly enable hidden-file +upload, set missing files to an error, and upload only that generated evidence +directory. Record the artifact ID/digest and configured retention. +[Official upload-artifact options](https://github.com/actions/upload-artifact). + +Keep architecture/runtime checks blocking in their own evidence records. +Keep configured complexity and duplication advisory, with explicit Director +dispositions. Do not apply `--report-only` to a combined command that also owns +required architecture failures. Separate raw measured coverage from Fallow's +mixed advisory interpretation; no merged score is needed. Exercises must +demonstrate acceptance of a correctly identified complete bundle and rejection +of absent, stale, wrong-revision, and wrong-scope reports. + +The research checkout produced only this note for later import into the delivery branch. No configuration, product +code, GitHub state, or dependency lockfile was changed. Focused probes and +`git diff --check` ran; full repository gates remain the delivery worker's +responsibility after implementation. + +## Appendix: native audit gate probes + +Four tiny Git repositories under the evidence directory's `audit-fixtures/` +used real base/head commits, an explicit entry point, and fixture-only rules +`unused-exports: warn` and `unresolved-imports: error`. Installed Fallow ran +`audit --base --gate new-only --format json --no-cache`, then the +same command with `--format sarif`. The script is +[probe-audit.py](/Users/flow/.codex/artifacts/pixel-forge-quality-adoption/research/probe-audit.py); +`audit--.command.json` records each identity and invocation. + +| Candidate change | Audit JSON verdict / exit | Native SARIF | +| ---------------------------------------------------------------- | ------------------------- | ----------------------------------------------------- | +| New unused export configured as warning | `warn` / 0 | `fallow/unused-export`, `warning` | +| New unresolved import configured as error | `fail` / 1 | `fallow/unresolved-import`, `error` | +| Complexity alone, no dead-code finding | `fail` / 1 | `fallow/high-crap-score`, `error` | +| Inherited unresolved-import error plus new unused-export warning | `warn` / 0 | Contains both the inherited `error` and new `warning` | + +The last case proves two unsafe shortcuts: `summary.dead_code_has_errors` +was true despite the correct warning verdict, and SARIF retained the inherited +error without `introduced` or baseline metadata. Audit JSON correctly marked +the unresolved import `introduced: false` and unused export `introduced: true`. +Replacing audit with `dead-code --changed-since --format sarif` +exited 1 on the inherited error, changing the existing new-only contract. + +Installed CLI help limits `--only`/`--skip` to invocations without a subcommand; +there is no verified native audit option to omit just health and duplication. +Do not change thresholds to simulate that omission. A safe acceptance adapter +needs a tested join between JSON introduced findings and native SARIF levels, +with errors for missing, ambiguous, or unsupported identities. The installed +`fallow/issue-registry.json` provides `result_key` and `sarif_rule_ids` mappings +for dead-code categories, avoiding invented pluralization rules. In these +fixtures, file path, line, and column aligned after converting JSON's zero-based +column to SARIF's one-based column. Other finding shapes still need explicit +fixtures before acceptance support is claimed. Complexity/duplication can then +receive advisory dispositions while configured introduced errors remain +blocking; whole-repository boundary checks remain a separately verified scope. diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 13a30a5b..663e1230 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -265,6 +265,23 @@ describe('public quality evidence CLI', () => { expect(check().status).toBe(0); }); + it('refuses a committed lockfile that disagrees with the installed tool', () => { + const original = readFileSync(resolve(fixture, 'package-lock.json'), 'utf8'); + const lock = JSON.parse(original); + lock.packages['node_modules/vitest'].version = '0.0.0'; + write('package-lock.json', JSON.stringify(lock)); + git('add', '.'); + git('commit', '-qm', 'contradictory tool lock'); + const result = cli('verify', '--base', base); + expect(result.status).toBe(1); + expect(result.text).toContain('installed tool does not match package-lock.json'); + expect(existsSync(resolve(bundle, 'certification.json'))).toBe(false); + write('package-lock.json', original); + git('add', '.'); + git('commit', '-qm', 'restore tool lock'); + head = git('rev-parse', 'HEAD'); + }); + it('preserves new-only required errors while native complexity errors stay advisory', () => { const commit = (source: string) => { write('scripts/fixture-lib.mjs', source); From 88c5dfda3c47160965db2ef46c64951da0a89d2a Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:03:29 +0200 Subject: [PATCH 04/11] fix(quality): identify retries and keep CLI fixtures stable under load --- .github/workflows/ci.yml | 2 +- docs/agents/quality-verification.md | 6 ++- scripts/quality/bundle.mjs | 2 +- tests/scripts/quality-boundaries.test.ts | 50 ++++++++++++++---------- tests/scripts/quality-evidence.test.ts | 4 +- 5 files changed, 39 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 731eb448..5fa8e781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: quality-${{ env.QUALITY_HEAD }} + name: quality-${{ env.QUALITY_HEAD }}-${{ github.run_id }}-${{ github.run_attempt }} path: .fallow/quality include-hidden-files: true if-no-files-found: error diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md index 55376f81..6f6cc5fc 100644 --- a/docs/agents/quality-verification.md +++ b/docs/agents/quality-verification.md @@ -42,6 +42,10 @@ Istanbul-shaped JSON map. Every declared source file must appear, including zero-hit files and empty type-only or barrel maps. The five maps are independent; there is no combined percentage or coverage threshold. +`scripts/**` is outside all five coverage maps. The CLI exercises prove tooling +behavior, but they do not produce measured coverage for these modules. Fallow +function scores for tooling therefore retain estimated provenance. + Absolute includes prevent Vitest 4's substring glob matching from collecting `shared/src` and Worker files in the root profile. The public fixture preserves this regression: its root tests import all three areas, but each profile keeps @@ -140,7 +144,7 @@ Standards/Spec review, Director dispositions, fresh finalizer after `FIXED`, or human merge approval. CI uploads only `.fallow/quality`, including hidden files, under `if: always()`; -missing artifacts are an error. Artifacts are named with the source head and +missing artifacts are an error. Artifacts are named with the source head, run ID and attempt, and retained for 14 days. Record the Actions artifact ID and digest in the handoff. After expiration, check out the same source revision, install its lockfile and Chromium, then regenerate with the same exact base. That creates new evidence diff --git a/scripts/quality/bundle.mjs b/scripts/quality/bundle.mjs index c8ab32ee..4fa2302d 100644 --- a/scripts/quality/bundle.mjs +++ b/scripts/quality/bundle.mjs @@ -20,7 +20,7 @@ export function same(actual, expected, label) { if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`${label} mismatch`); } -export function exactRevision(root, base, head) { +function exactRevision(root, base, head) { if (!/^[a-f0-9]{40}$/.test(base ?? '') || !/^[a-f0-9]{40}$/.test(head ?? '')) throw new Error('base and head must be exact commit SHAs'); same(git(root, 'rev-parse', `${base}^{commit}`), base, 'base'); diff --git a/tests/scripts/quality-boundaries.test.ts b/tests/scripts/quality-boundaries.test.ts index fe3b70a1..b4f98e2f 100644 --- a/tests/scripts/quality-boundaries.test.ts +++ b/tests/scripts/quality-boundaries.test.ts @@ -35,32 +35,42 @@ it('blocks local source crossings even outside a changed-files audit', () => { }); it('rejects static, literal dynamic and computed imports in each production runtime', () => { - for (const [filename, specifier] of [ - ['src/fixture.ts', 'hono'], - ['shared/src/fixture.ts', 'node:fs'], - ['server/src/fixture.ts', 'lit'], - ['workers/telemetry/src/fixture.ts', 'lit'], - ['scripts/fixture.mjs', 'lit'], + const cases = []; + for (const [directory, extension, specifier] of [ + ['src', 'ts', 'hono'], + ['shared/src', 'ts', 'node:fs'], + ['server/src', 'ts', 'lit'], + ['workers/telemetry/src', 'ts', 'lit'], + ['scripts', 'mjs', 'lit'], ]) { - for (const source of [ + mkdirSync(resolve(fixture, directory), { recursive: true }); + const sources = [ `import * as value from '${specifier}'; export { value };`, `export const value = import('${specifier}');`, "const destination = './local.js'; export const value = import(destination);", - ]) { - const result = spawnSync( - resolve(repository, 'node_modules/.bin/eslint'), - ['--stdin', '--stdin-filename', filename, '--format', 'json'], - { cwd: fixture, encoding: 'utf8', input: source } - ); - expect(result.status, result.stderr).toBe(1); - expect( - JSON.parse(result.stdout)[0].messages.some((message: { ruleId: string }) => - ['no-restricted-imports', 'no-restricted-syntax'].includes(message.ruleId) - ) - ).toBe(true); + ]; + for (const [index, source] of sources.entries()) { + const filename = `${directory}/case-${index}.${extension}`; + writeFileSync(resolve(fixture, filename), source); + cases.push(filename); } } -}, 30_000); + const result = spawnSync( + resolve(repository, 'node_modules/.bin/eslint'), + [...cases, '--format', 'json'], + { cwd: fixture, encoding: 'utf8' } + ); + expect(result.status, result.stderr).toBe(1); + const reports = JSON.parse(result.stdout); + expect(reports).toHaveLength(cases.length); + for (const report of reports) + expect( + report.messages.some((message: { ruleId: string }) => + ['no-restricted-imports', 'no-restricted-syntax'].includes(message.ruleId) + ), + report.filePath + ).toBe(true); +}, 60_000); it('keeps a type-accepted wrong-runtime package blocked by the import policy', () => { writeFileSync( diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 663e1230..cf980076 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -142,7 +142,7 @@ beforeAll(() => { .map((name) => readFileSync(resolve(bundle, 'commands', name), 'utf8')) .join('\n'); expect(result, result.text + logs).toMatchObject({ status: 0 }); -}, 120_000); +}, 300_000); afterAll(() => rmSync(fixture, { recursive: true, force: true })); @@ -323,7 +323,7 @@ describe('public quality evidence CLI', () => { (finding: any) => finding.category === 'complexity' ) ).toBe(true); - }, 180_000); + }, 600_000); it('invalidates certification before a failed producer and retains its diagnostics', () => { write( From e0973f537ac126419507a2e4b641cdff621853e8 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:11:39 +0200 Subject: [PATCH 05/11] fix(quality): refuse unclassified non-complexity health errors --- scripts/quality/fallow.mjs | 10 +++++++++- tests/scripts/quality-evidence.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/quality/fallow.mjs b/scripts/quality/fallow.mjs index e0381aba..7b76ce6a 100644 --- a/scripts/quality/fallow.mjs +++ b/scripts/quality/fallow.mjs @@ -65,7 +65,15 @@ function reconcile(audit, sarif, registry, base, head) { } const heuristicRules = new Set( registry.issue_types - .filter((entry) => ['health', 'dupes'].includes(entry.command)) + .filter((entry) => + [ + 'code-duplication', + 'high-complexity', + 'high-cyclomatic-complexity', + 'high-cognitive-complexity', + 'high-crap-score', + ].includes(entry.id) + ) .flatMap((entry) => [entry.rule_id, ...(entry.sarif_rule_ids ?? [])]) ); for (const result of results) diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index cf980076..703fc29b 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -323,6 +323,30 @@ describe('public quality evidence CLI', () => { (finding: any) => finding.category === 'complexity' ) ).toBe(true); + const sarifPath = resolve(bundle, 'audit.sarif.json'); + const sarif = JSON.parse(readFileSync(sarifPath, 'utf8')); + const nativeResult = sarif.runs.flatMap((run: any) => run.results ?? [])[0]; + expect(nativeResult.level).toBe('error'); + nativeResult.ruleId = 'fallow/css-broken-reference'; + const changedSarif = JSON.stringify(sarif); + writeFileSync(sarifPath, changedSarif); + const manifest = JSON.parse(readFileSync(resolve(bundle, 'manifest.json'), 'utf8')); + manifest.artifacts['audit.sarif.json'] = createHash('sha256') + .update(changedSarif) + .digest('hex'); + const changedManifest = JSON.stringify(manifest); + writeFileSync(resolve(bundle, 'manifest.json'), changedManifest); + writeFileSync( + resolve(bundle, 'certification.json'), + JSON.stringify({ + base, + head, + manifest: createHash('sha256').update(changedManifest).digest('hex'), + }) + ); + result = check(); + expect(result.status).toBe(1); + expect(result.text).toContain('unmatched Fallow result: fallow/css-broken-reference'); }, 600_000); it('invalidates certification before a failed producer and retains its diagnostics', () => { From 47b92c491dbe6a44b2db58b50f4db1585cd2188a Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:14:26 +0200 Subject: [PATCH 06/11] fix(quality): accept V8 implicit-else coverage locations --- docs/agents/quality-verification.md | 4 +++- scripts/quality/coverage.mjs | 21 ++++++++++++++++++++- tests/scripts/quality-evidence.test.ts | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md index 6f6cc5fc..aaa40091 100644 --- a/docs/agents/quality-verification.md +++ b/docs/agents/quality-verification.md @@ -51,7 +51,9 @@ Absolute includes prevent Vitest 4's substring glob matching from collecting this regression: its root tests import all three areas, but each profile keeps its declared scope. `excludeAfterRemap` also applies exclusions after source-map translation. V8 can serialize an unbounded end column as `null`; this remains a -valid location, while hit counts must be nonnegative integers. +valid location, while hit counts must be nonnegative integers. The remapper also +emits `{start: {}, end: {}}` for the second location of a two-way `if` branch +with no explicit `else`; only that precise implicit-branch sentinel is accepted. The Worker command runs from the root npm workspace caller, generates its types, and records root `INIT_CWD`. It does not measure workerd or deployed bindings. diff --git a/scripts/quality/coverage.mjs b/scripts/quality/coverage.mjs index 8ee341c3..a6a8a1ff 100644 --- a/scripts/quality/coverage.mjs +++ b/scripts/quality/coverage.mjs @@ -45,6 +45,18 @@ export function validateCoverage(map, profile, producerRoot) { ); } +// The V8 remapper emits this sentinel for the implicit else of an if statement. +function implicitElse(value) { + return ( + object(value) && + Object.keys(value).length === 2 && + object(value.start) && + object(value.end) && + Object.keys(value.start).length === 0 && + Object.keys(value.end).length === 0 + ); +} + function validateFileCounters(file, profileId) { for (const [counters, mappings] of [ ['s', 'statementMap'], @@ -69,7 +81,14 @@ function validateFileCounters(file, profileId) { !location(mapping.loc) || !Array.isArray(mapping.locations) || value.length !== mapping.locations.length || - !mapping.locations.every(location) + !mapping.locations.every( + (value, index) => + location(value) || + (mapping.type === 'if' && + mapping.locations.length === 2 && + index === 1 && + implicitElse(value)) + ) ) throw new Error(`${profileId}: malformed branch coverage`); } else if ( diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 703fc29b..76d23cd0 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -96,7 +96,10 @@ beforeAll(() => { ); write('src/main.ts', 'export const main = 1;\n'); write('src/value.ts', 'export const value = 2;\n'); - write('src/zero.ts', 'export function uncalled() { return 3; }\n'); + write( + 'src/zero.ts', + 'export function uncalled(flag: boolean) { if (flag) return 1; return 3; }\n' + ); write('src/types.ts', 'export interface Value { value: number }\n'); write('shared/src/index.ts', 'export const shared = 3;\n'); write('server/src/index.ts', 'export const server = 4;\n'); @@ -163,6 +166,12 @@ describe('public quality evidence CLI', () => { Object.values(rootMap[resolve(fixture, 'src/zero.ts')].s).every((hits) => hits === 0) ).toBe(true); expect(rootMap[resolve(fixture, 'src/types.ts')].s).toEqual({}); + const branches = Object.values(rootMap[resolve(fixture, 'src/zero.ts')].branchMap) as any[]; + expect( + branches.some( + (branch) => JSON.stringify(branch.locations[1]) === JSON.stringify({ start: {}, end: {} }) + ) + ).toBe(true); const relocated = resolve(fixture, '.fallow/downloaded'); cpSync(bundle, relocated, { recursive: true }); expect(check(relocated).status).toBe(0); @@ -204,12 +213,18 @@ describe('public quality evidence CLI', () => { const map = JSON.parse(original); const incomplete = { ...map }; delete incomplete[resolve(fixture, 'src/zero.ts')]; + const malformedBranch = JSON.parse(original); + const firstBranch = Object.values( + malformedBranch[resolve(fixture, 'src/zero.ts')].branchMap + )[0] as any; + firstBranch.locations[0] = { start: {}, end: {} }; const malformed = JSON.parse(original); malformed[resolve(fixture, 'src/value.ts')].s = { 0: -1 }; for (const [replacement, message] of [ ['{}', 'empty or malformed'], [JSON.stringify(incomplete), 'incomplete or wrong-profile'], [JSON.stringify(malformed), 'malformed'], + [JSON.stringify(malformedBranch), 'malformed branch'], [original.replaceAll(fixture, '/foreign/root'), 'foreign'], [readFileSync(resolve(bundle, 'server/coverage-final.json'), 'utf8'), 'wrong-profile'], ]) { From 084f2acbd16b7fd2e7d6f91f210932248cb99336 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:18:29 +0200 Subject: [PATCH 07/11] test(quality): mutate the native complexity result in policy regression --- tests/scripts/quality-evidence.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 76d23cd0..e6ae3346 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -340,7 +340,9 @@ describe('public quality evidence CLI', () => { ).toBe(true); const sarifPath = resolve(bundle, 'audit.sarif.json'); const sarif = JSON.parse(readFileSync(sarifPath, 'utf8')); - const nativeResult = sarif.runs.flatMap((run: any) => run.results ?? [])[0]; + const nativeResult = sarif.runs + .flatMap((run: any) => run.results ?? []) + .find((result: any) => result.ruleId === 'fallow/high-crap-score'); expect(nativeResult.level).toBe('error'); nativeResult.ruleId = 'fallow/css-broken-reference'; const changedSarif = JSON.stringify(sarif); From aac4582e4a137ab2a4c90782d54095a9a5f22067 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:31:25 +0200 Subject: [PATCH 08/11] fix(quality): bound root coverage worker concurrency --- docs/agents/quality-verification.md | 4 ++++ scripts/quality/config.mjs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md index aaa40091..7904a455 100644 --- a/docs/agents/quality-verification.md +++ b/docs/agents/quality-verification.md @@ -37,6 +37,10 @@ commands, environment, producer root, tools and results. | `server` | All `server/tests/**/*.test.ts` except database and storage suites | `server/src/**/*.ts`, excluding declarations | Node | | `worker` | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding declarations | Node/happy-dom with mocked Worker bindings | +The root coverage command limits Vitest to two workers. This keeps full-suite +coverage and the CLI fixtures from saturating local CPUs; test selection and +timeouts stay unchanged. The limit is part of the recorded and accepted command. + Each profile uses the installed Vitest V8 provider and emits its own Istanbul-shaped JSON map. Every declared source file must appear, including zero-hit files and empty type-only or barrel maps. The five maps are independent; diff --git a/scripts/quality/config.mjs b/scripts/quality/config.mjs index e878c8c3..396fc6dc 100644 --- a/scripts/quality/config.mjs +++ b/scripts/quality/config.mjs @@ -58,7 +58,7 @@ export function profiles(inputs) { id: 'root', runtime: 'Node/happy-dom', cwd: '.', - command: ['node_modules/.bin/vitest', 'run'], + command: ['node_modules/.bin/vitest', 'run', '--maxWorkers=2'], config: ['vitest.config.ts'], include: 'src/**/*.ts', exclude: ['**/*.d.ts', 'src/**/*.test.ts', 'src/main.ts'], From b6526dbf0a4be3b897891b5e46d0bcbf7e9d24ef Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:38:47 +0200 Subject: [PATCH 09/11] fix(quality): isolate root files and budget CLI refusal exercise --- docs/agents/quality-verification.md | 7 ++++--- scripts/quality/config.mjs | 2 +- tests/scripts/quality-evidence.test.ts | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md index 7904a455..7d707f56 100644 --- a/docs/agents/quality-verification.md +++ b/docs/agents/quality-verification.md @@ -37,9 +37,10 @@ commands, environment, producer root, tools and results. | `server` | All `server/tests/**/*.test.ts` except database and storage suites | `server/src/**/*.ts`, excluding declarations | Node | | `worker` | `tests/workers/telemetry-worker.test.ts` | `workers/telemetry/src/**/*.ts`, excluding declarations | Node/happy-dom with mocked Worker bindings | -The root coverage command limits Vitest to two workers. This keeps full-suite -coverage and the CLI fixtures from saturating local CPUs; test selection and -timeouts stay unchanged. The limit is part of the recorded and accepted command. +The root coverage command uses one Vitest worker so its product files and CLI +fixtures do not overlap. This controls this runner’s concurrency; other local +processes can still compete for CPU. Test selection and product timeouts stay +unchanged. The limit is part of the recorded and accepted command. Each profile uses the installed Vitest V8 provider and emits its own Istanbul-shaped JSON map. Every declared source file must appear, including diff --git a/scripts/quality/config.mjs b/scripts/quality/config.mjs index 396fc6dc..705ef3ce 100644 --- a/scripts/quality/config.mjs +++ b/scripts/quality/config.mjs @@ -58,7 +58,7 @@ export function profiles(inputs) { id: 'root', runtime: 'Node/happy-dom', cwd: '.', - command: ['node_modules/.bin/vitest', 'run', '--maxWorkers=2'], + command: ['node_modules/.bin/vitest', 'run', '--maxWorkers=1'], config: ['vitest.config.ts'], include: 'src/**/*.ts', exclude: ['**/*.d.ts', 'src/**/*.test.ts', 'src/main.ts'], diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index e6ae3346..0680cfee 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -278,7 +278,7 @@ describe('public quality evidence CLI', () => { expect(check().status).toBe(1); rmSync(resolve(fixture, 'new-input.txt')); expect(check().status).toBe(0); - }); + }, 30_000); it('refuses a committed lockfile that disagrees with the installed tool', () => { const original = readFileSync(resolve(fixture, 'package-lock.json'), 'utf8'); From 9db4e1e0d093d2b5b69045a69391a148454fef5d Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 01:53:35 +0200 Subject: [PATCH 10/11] fix(quality): confine manifest reads to the selected bundle --- docs/agents/quality-verification.md | 5 +++-- scripts/quality/bundle.mjs | 4 ++-- tests/scripts/quality-evidence.test.ts | 11 +++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/agents/quality-verification.md b/docs/agents/quality-verification.md index 7d707f56..44748f9c 100644 --- a/docs/agents/quality-verification.md +++ b/docs/agents/quality-verification.md @@ -95,8 +95,9 @@ failed, stale, wrong-profile, empty or foreign evidence fails closed. For downloaded evidence, retain the original `producerRoot`. It translates absolute coverage paths; it never permits reading artifacts outside the selected -bundle. The consumer OS need not match the producer OS. Replacing the original -root with the download directory invalidates the evidence. +bundle. Evidence can move between the verified macOS developer host and Linux CI. +Windows producer and consumer paths are unsupported and unverified in this slice. +Replacing the original root with the download directory invalidates the evidence. An input change invalidates acceptance even if HEAD did not change. Restoring the exact revision and inputs can make the same intact historical bundle valid diff --git a/scripts/quality/bundle.mjs b/scripts/quality/bundle.mjs index 4fa2302d..f65ec9cc 100644 --- a/scripts/quality/bundle.mjs +++ b/scripts/quality/bundle.mjs @@ -98,7 +98,7 @@ export function artifactDigests(bundle, paths) { export function validateBundle(root, bundle, base, head, certification = true) { exactRevision(root, base, head); - const manifest = readJson(resolve(bundle, 'manifest.json')); + const manifest = readJson(artifact(bundle, 'manifest.json')); const current = identity(root); if (current.dirty) throw new Error('checkout inputs changed: dirty tree'); const expectedProfiles = profiles(current.inputs); @@ -187,7 +187,7 @@ export function validateBundle(root, bundle, base, head, certification = true) { if (certification) same( readJson(artifact(bundle, 'certification.json')), - { base, head, manifest: digest(readFileSync(resolve(bundle, 'manifest.json'))) }, + { base, head, manifest: digest(readFileSync(artifact(bundle, 'manifest.json'))) }, 'certification' ); return manifest; diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index 0680cfee..a7dd8269 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -180,6 +180,17 @@ describe('public quality evidence CLI', () => { ); }); + it('rejects a manifest symlink outside the selected bundle', () => { + const relocated = resolve(fixture, '.fallow/external-manifest'); + cpSync(bundle, relocated, { recursive: true }); + const manifestPath = resolve(relocated, 'manifest.json'); + rmSync(manifestPath); + symlinkSync(resolve(bundle, 'manifest.json'), manifestPath); + const result = check(relocated); + expect(result.status).toBe(1); + expect(result.text).toContain('artifact path escapes bundle'); + }); + it('rejects absent, malformed, incomplete, foreign, or mixed artifacts', () => { expect(check(resolve(fixture, '.fallow/absent')).status).toBe(1); for (const [path, replacement] of [ From a3260ad1b7eb6ed0a68e6bbfe1a4f6ab1a1e7881 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 10 Sep 2026 12:02:23 +0200 Subject: [PATCH 11/11] fix(quality): include imported app config in CLI fixture --- tests/scripts/quality-evidence.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/quality-evidence.test.ts b/tests/scripts/quality-evidence.test.ts index a7dd8269..ce3fd334 100644 --- a/tests/scripts/quality-evidence.test.ts +++ b/tests/scripts/quality-evidence.test.ts @@ -46,6 +46,7 @@ beforeAll(() => { 'scripts/quality', 'vitest.config.ts', 'vitest.browser.config.ts', + 'vite.config.ts', 'server/vitest.config.ts', 'eslint.config.js', '.fallowrc.json',