diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27a4f141..c9e99e89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,119 @@ concurrency: cancel-in-progress: true jobs: + duplication: + name: Duplication ratchet + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout candidate + uses: actions/checkout@v7 + with: + path: candidate + fetch-depth: 0 + persist-credentials: false + - name: Checkout trusted comparison base + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha || github.event.before }} + path: base + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: candidate/tools/duplication/package-lock.json + - name: Test duplication tooling + run: | + npm ci --prefix candidate/tools/duplication --ignore-scripts --no-audit --no-fund + npm --prefix candidate/tools/duplication test + - name: Measure and compare production duplication + id: duplication-policy + shell: bash + run: | + policy_dir="$GITHUB_WORKSPACE/base/tools/duplication" + if [[ -f "$policy_dir/policy.json" ]]; then + echo "trusted=true" >> "$GITHUB_OUTPUT" + npm ci --prefix "$policy_dir" --ignore-scripts --no-audit --no-fund + else + # One-time bootstrap: the base branch has no duplication policy yet. + policy_dir="$GITHUB_WORKSPACE/candidate/tools/duplication" + fi + node "$policy_dir/check.mjs" --root "$GITHUB_WORKSPACE/candidate" \ + --base "$GITHUB_WORKSPACE/base" --output "$RUNNER_TEMP/duplication-report" + - name: Validate proposed checker and policy against main + if: steps.duplication-policy.outputs.trusted == 'true' + run: | + node "$GITHUB_WORKSPACE/base/tools/duplication/verify.mjs" \ + --root "$GITHUB_WORKSPACE/candidate" --base "$GITHUB_WORKSPACE/base" \ + --output "$RUNNER_TEMP/duplication-report" + - name: Preserve actionable duplication reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: duplication-report + path: ${{ runner.temp }}/duplication-report + retention-days: 30 + if-no-files-found: warn + + duplication-baseline: + name: Update duplication baseline + needs: [duplication, web, test] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + concurrency: + group: duplication-baseline-main + cancel-in-progress: false + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - name: Persist initial baseline or tighter limits + shell: bash + run: | + git fetch --no-tags origin main + if [[ "$(git rev-parse FETCH_HEAD)" != "$GITHUB_SHA" ]]; then + echo "A newer main commit exists; its successful CI run will update the baseline." + exit 0 + fi + npm ci --prefix tools/duplication --ignore-scripts --no-audit --no-fund + node tools/duplication/check.mjs --update-baseline --output "$RUNNER_TEMP/duplication-ratchet" + if git diff --quiet -- tools/duplication/baseline.json; then + echo "No improved limits; baseline left unchanged." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -- tools/duplication/baseline.json + git commit -m "CHORE: ratchet duplication baseline" + # Normal non-force push: never overwrite a concurrently advanced main. + # GITHUB_TOKEN pushes do not recursively start another CI run. + if ! git push origin HEAD:main; then + git fetch --no-tags origin main + if [[ "$(git rev-parse FETCH_HEAD)" != "$GITHUB_SHA" ]]; then + echo "Main advanced during the update; leaving its newer content untouched." + exit 0 + fi + echo "Baseline push failed; check repository Actions permissions and branch rules." + exit 1 + fi + - name: Preserve measured baseline for manual recovery + if: always() + uses: actions/upload-artifact@v7 + with: + name: duplication-baseline-candidate + path: ${{ runner.temp }}/duplication-ratchet + retention-days: 30 + if-no-files-found: ignore + web: name: Experimental web runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6e0ef479..53b7484d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /web/node_modules/ /web/test-results/ /web/playwright-report/ +/tools/duplication/node_modules/ +/tools/duplication/reports/ diff --git a/README.md b/README.md index 25a5c0c4..031b3585 100644 --- a/README.md +++ b/README.md @@ -3788,6 +3788,29 @@ Codexometer uses: - Codex app-server JSON-RPC for authenticated quota data - Local Codex rollout `token_count` records for live Sessions telemetry +### Duplication checks + +The standalone **Duplication ratchet** CI job uses pinned jscpd to keep production +clone counts, duplicated lines and duplicated tokens equal or lower. It runs in +parallel with Go/web tests; test-source duplication is reported but not gated. +There are no LLM calls, embeddings, token charges or application dependencies. + +```sh +npm ci --prefix tools/duplication --ignore-scripts +npm --prefix tools/duplication test +npm --prefix tools/duplication run check +``` + +The first measurement establishes the benchmark without arbitrary limits. +Subsequent checks use the policy and baseline from the PR's main-branch base +commit, with a fresh measurement of that same base. PRs must not edit the +inherited baseline; successful main CI alone commits tighter limits. Ordinary +application PRs use one authoritative comparison; tooling or policy changes get +targeted compatibility/migration validation. CI artifacts contain +metric deltas, both source locations and fragments for actionable refactoring. +See [the duplication tooling guide](tools/duplication/README.md) for baseline +bootstrap, trusted policy, report formats and required branch-rule setup. + ## License Codexometer is available under the [MIT License](LICENSE). diff --git a/tools/duplication/README.md b/tools/duplication/README.md new file mode 100644 index 00000000..5712773d --- /dev/null +++ b/tools/duplication/README.md @@ -0,0 +1,177 @@ +# Duplication ratchet + +The separate **Duplication ratchet** CI job measures tracked, maintained Go and +browser source with pinned jscpd 5.4.2. It runs alongside, not inside, the Go and +web jobs. jscpd is development tooling only: the application does not include or +run it. No LLM, embeddings, network inference, API key or Codex quota is used. +Scanner "tokens" are source-code tokens, not billable model tokens. + +## Local checks + +From the repository root, using Node 24: + +```sh +npm ci --prefix tools/duplication --ignore-scripts +npm --prefix tools/duplication test +npm --prefix tools/duplication run check +``` + +The local check compares with the committed baseline. To reproduce CI's extra +comparison with the actual PR base, check out that commit in another worktree +and use the **base's** checker once it contains this tooling: + +```sh +git worktree add --detach /tmp/codexometer-duplication-base origin/main +npm ci --prefix /tmp/codexometer-duplication-base/tools/duplication --ignore-scripts +node /tmp/codexometer-duplication-base/tools/duplication/check.mjs \ + --root "$PWD" --base /tmp/codexometer-duplication-base \ + --output /tmp/codexometer-duplication-report +``` + +Use an unused worktree path, and replace `origin/main` with the PR's exact base +SHA when diagnosing an older CI run. Only tracked files are scanned: `git add` +new sources before checking them. A scanner/installation/configuration error +fails the job; bootstrap never hides operational failures. + +## What is gated? + +Each production scan must independently stay equal or improve in **all three** +absolute metrics: clone count, duplicated lines and duplicated tokens. + +| Scan | Minimum tokens / lines | Matching | +| ------------- | ---------------------- | ----------------------------------------------------------- | +| `exact-50` | 50 / 5 | Token matches, ignoring comments | +| `renamed-50` | 50 / 5 | Also normalises consistent identifier renaming and literals | +| `exact-100` | 100 / 10 | Larger token matches | +| `renamed-100` | 100 / 10 | Larger normalised matches | + +Percentages are informational: removing non-duplicated code can increase a +percentage without adding duplication. A lower metric cannot offset an increase +in another one. This is a net-metrics gate, not a ban on every new clone pair; +new pairs are still highlighted for review. + +Production means maintained `.go` files excluding `_test.go`, plus browser +source under `web/src` (TypeScript, JavaScript, Svelte and CSS). +Generated files, embedded build output, dependencies and vendored source are +excluded. Go and browser tests are measured separately at the smaller threshold +but are **report-only**, so useful regression fixtures do not fail the gate. +The tooling itself, documentation and other file types are outside this policy. +Missing, oversized, symlinked or outside-repository selected sources fail instead +of silently disappearing from the measurement. + +Normalised matches are refactoring candidates, not proof of identical behaviour. +Palettes, glyph tables and similar data can legitimately resemble one another. +Do not consolidate them automatically or weaken tests just to reduce a metric. + +## Baseline lifecycle + +`baseline.json` is versioned CI policy, unrelated to the application's model +benchmarks or usage history. `policy.json` pins the scanner version, selection +rules, matching settings and thresholds. + +1. The first PR introduces a `pending-bootstrap` sentinel. Because its base has + no duplication policy yet, that first measurement establishes the benchmark + without applying an invented ceiling. The duplication gate passes regardless + of those initial counts, provided the scanner and tooling tests succeed. +2. After all ordinary CI jobs pass on `main`, **Update duplication baseline** + measures again and commits the initial numerical baseline with provenance. +3. Later PRs use the checker, policy and baseline from their main-branch base + commit, not the PR's copy. They compare against both that committed ceiling + and fresh measurements of the same base, taking the lower bound for each + metric. A delayed baseline update therefore cannot hide a regression. The + inherited baseline file must remain byte-for-byte unchanged in PRs; only + successful main CI writes it. Ordinary application PRs use this one + authoritative comparison, not a duplicate candidate run. +4. After another successful main build, the main-only writer commits smaller + limits if any improved. Equality leaves the baseline and provenance unchanged. + Missing, corrupt or incompatible established baselines fail closed unless + an explicit policy migration has passed the old/new checks below. + +PR and fork jobs have read-only repository permissions. Only the main-only +writer receives `contents: write`, waits for Go/web/duplication success, and +stages `baseline.json` alone. It skips an outdated main SHA and never force-pushes +over concurrent work. Its normal `GITHUB_TOKEN` commit does not recursively +start another CI run. + +Repository administrators should require **Duplication ratchet** in branch +rules and require an up-to-date branch (or equivalent merge-queue validation) +to prevent merges that bypass the check. Adding a workflow alone does not enable +branch protection. The PR comparison is pinned to its base SHA, not a moving +`main` fetched halfway through the job. Push CI compares with the pre-push main +commit. Actions and repository rules must permit the main-only writer's baseline +commit; otherwise that job fails and preserves its measured artifact for +diagnosis. Fix the repository configuration before rerunning it; do not edit a +PR's baseline or grant broad protection bypass just to make the check green. + +## Checker and policy maintenance + +When maintained duplication tooling changes (runtime `.mjs`, policy, package or +lock files), the trusted verifier also runs the candidate checker against the +same source/base, using **main's scanner, policy and baseline**. Its metrics, +limits and provenance must match the authoritative result. It also verifies +that the candidate's installed scanner matches its declared version. This +targeted compatibility check catches a broken/no-op/mismeasuring proposed +checker before it becomes main's checker. Documentation/test-only changes do +not trigger additional production scans. + +For an intentional scanner-version or policy change: + +1. Create a worktree at the exact main comparison SHA as shown above. Update the + candidate policy, package and lock as needed, but **do not edit the baseline**. +2. Record an explicit migration with the exact old policy, new policy hash and a + meaningful reason: + + ```sh + node tools/duplication/verify.mjs --prepare-migration \ + --root "$PWD" --base /tmp/codexometer-duplication-base \ + --reason "Explain the scanner or measurement-rule change" + ``` + +3. Commit the resulting `tools/duplication/migration.json` with the tooling + changes. CI still gates production under main's **old** rules, verifies the + candidate checker under those rules, then separately compares head and base + under the **proposed new** rules/scanner. Both comparisons must pass. Preserve + compatibility with the previous checker CLI for this validation. +4. Review both measurements in the summary/artifact. The old baseline remains + untouched throughout the PR. After merge and successful main CI, the writer + migrates it to measurements under the new policy. The migration record may + remain as provenance; it is ignored once the baseline matches the policy. + +A missing/mismatched migration record fails; it cannot silently reset an +ordinary feature PR's ceiling. Do not raise limits to make an application +change green. Any change to the CI machinery itself still needs code review; +the duplication check is not a substitute for protecting workflow changes. + +## Actionable output for humans and Codex + +Download the **duplication-report** artifact from the standalone CI job: + +The GitHub job summary contains the actual/limit statistics table, including +PASS/FAIL/BOOTSTRAP status. The separate test-source table explicitly labels its +statistics REPORT ONLY. CI logs contain a concise result and failure diagnostics, +not a duplicate table. Local checks print the summary directly. + +- `summary.md`: actual/limit table, exact increases and a shortlist of new pairs. +- `result.json`: policy/commit provenance, per-scan ceilings, actual-base metrics, + effective limits, explicit regression deltas, and clone pairs with repository- + relative paths, both line ranges, match kind, source fragments and `isNew`. +- `production//jscpd-report.json`: full production clone reports. +- `base//jscpd-report.json`: comparison-base reports. +- `tests//jscpd-report.json`: report-only test duplication. +- `*//inputs.json` and `config.json`: exact source manifests and scanner + settings, so exclusions and scope are inspectable. +- `baseline.json`: the measured candidate, not permission to loosen a ceiling. +- `verification.json`: whether targeted checker validation was needed, and which + tooling files changed. Tooling-maintenance PRs additionally include + `candidate-validation/` and, for policy changes, `proposed-policy/` reports. + +Paths in `result.json` resolve relative to the artifact root. Source locations +inside clone pairs resolve relative to the repository root. Up to 20 GitHub error +annotations point at candidate source locations on a failing scan. Reports are +retained for 30 days; code fragments are repository source, so respect the same +access restrictions as the repository when sharing them. + +Codex can read `result.json`, identify the regressed metric and both code ranges, +inspect the fragments, apply a behaviour-preserving shared implementation, and +rerun the check plus the relevant application tests. Reports guide refactoring; +they do not authorise automated changes or prove semantic equivalence. diff --git a/tools/duplication/baseline.json b/tools/duplication/baseline.json new file mode 100644 index 00000000..8f43452f --- /dev/null +++ b/tools/duplication/baseline.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "status": "pending-bootstrap" +} diff --git a/tools/duplication/check.mjs b/tools/duplication/check.mjs new file mode 100644 index 00000000..2aeda22f --- /dev/null +++ b/tools/duplication/check.mjs @@ -0,0 +1,400 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + realpathSync, + writeFileSync, + appendFileSync, +} from "node:fs"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { + commandData, + commandProperty, + limits, + metrics, + policyHash, + ratchet, + regressions, + relativeClone, + validateBaseline, + validateMigration, +} from "./metrics.mjs"; + +const directory = dirname(fileURLToPath(import.meta.url)); +const read = (path) => JSON.parse(readFileSync(path, "utf8")); +const write = (path, data) => + writeFileSync(path, JSON.stringify(data, null, 2) + "\n"); +const git = (root, ...args) => + execFileSync("git", ["-C", root, ...args], { encoding: "utf8" }).trim(); + +export function sourceFiles(root, policy, cohort) { + const tracked = execFileSync("git", ["-C", root, "ls-files", "-z"], { + encoding: "utf8", + }) + .split("\0") + .filter(Boolean) + .sort(); + return tracked.filter((file) => { + if ( + !policy[cohort].some((pattern) => new RegExp(pattern).test(file)) || + policy.exclude.some((pattern) => new RegExp(pattern).test(file)) || + (cohort === "production" && file.endsWith("_test.go")) + ) + return false; + const path = resolve(root, file), + within = relative(root, realpathSync(path)); + if ( + isAbsolute(within) || + within.startsWith("..") || + !lstatSync(path).isFile() + ) + throw new Error(`Refusing non-regular/outside source: ${file}`); + if (lstatSync(path).size > policy.maxBytes) + throw new Error(`Source exceeds scanner size limit: ${file}`); + return !/(^|\n)\/\/ Code generated [^\n]*DO NOT EDIT\./.test( + readFileSync(path, "utf8").slice(0, 4096), + ); + }); +} + +function scan( + root, + policy, + cohort, + spec, + artifactRoot, + { + group = cohort, + nativeBaseline, + seed = false, + scanner = resolve(directory, "node_modules/jscpd/run-jscpd.js"), + } = {}, +) { + const destination = resolve(artifactRoot, group, spec.id); + mkdirSync(destination, { recursive: true }); + const files = sourceFiles(root, policy, cohort); + if (!files.length) throw new Error(`No ${cohort} sources selected`); + const config = resolve(destination, "config.json"); + write(config, policy.jscpd); + write(resolve(destination, "inputs.json"), { + commit: git(root, "rev-parse", "HEAD"), + files, + }); + const args = [ + scanner, + "--config", + config, + "--workers", + "1", + "--absolute", + "--no-colors", + "--no-tips", + "--fail-on-empty", + "--min-tokens", + String(spec.minTokens), + "--min-lines", + String(spec.minLines), + "--output", + destination, + ...(spec.renamed + ? ["--ignore-identifiers=consistent", "--ignore-literals"] + : []), + ...(nativeBaseline + ? ["--baseline", nativeBaseline, ...(seed ? ["--update-baseline"] : [])] + : []), + ...files, + ]; + execFileSync(process.execPath, args, { + cwd: root, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + const path = resolve(destination, "jscpd-report.json"), + report = read(path); + report.duplicates = report.duplicates.map((clone) => + relativeClone(clone, root), + ); + write(path, report); + return { + metrics: metrics(report), + report: relative(artifactRoot, path).replaceAll("\\", "/"), + clones: report.duplicates, + }; +} + +export function markdown(result) { + const value = (scan, metric) => + `${scan.metrics[metric]} / ${scan.limits?.[metric] ?? "initial"}`; + const table = (scans, reportOnly = false) => { + const headings = [ + "Scan", + "Clones", + "Duplicated lines", + "Duplicated tokens", + "Result", + ]; + const rows = scans.map((scan) => [ + scan.id, + ...["clones", "duplicatedLines", "duplicatedTokens"].map((metric) => + reportOnly ? String(scan.metrics[metric]) : value(scan, metric), + ), + reportOnly + ? "REPORT ONLY" + : scan.regressions.length + ? "FAIL" + : result.bootstrap + ? "BOOTSTRAP" + : "PASS", + ]); + const line = (row) => `| ${row.join(" | ")} |`; + return [ + line(headings), + "| --- | ---: | ---: | ---: | --- |", + ...rows.map(line), + ]; + }; + const locations = result.scans + .flatMap((scan) => + scan.clones + .filter((clone) => clone.isNew) + .map((clone) => ({ scan, clone })), + ) + .slice(0, 20); + const safe = (text) => + String(text) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("|", "|") + .replaceAll("`", "`") + .replaceAll("\n", " "); + return [ + result.migration + ? "## Duplication ratchet — proposed policy" + : "## Duplication ratchet", + result.bootstrap + ? "Initial measurement: no historical limit applied. The first successful main build persists the baseline." + : "Production must not increase clone counts, duplicated lines or duplicated tokens in any scan.", + "", + "Values below are actual / limit; percentages are informational only.", + "", + ...table(result.scans), + ...(result.tests?.length + ? [ + "", + "### Test-source duplication (report-only)", + "", + ...table(result.tests, true), + ] + : []), + "", + ...result.scans.flatMap((scan) => + scan.regressions.map( + (v) => + `- **${scan.id}: ${v.metric} ${v.limit} → ${v.actual} (+${v.increase})**`, + ), + ), + "", + "### New clone candidates relative to the PR base", + ...locations.map( + ({ scan, clone }) => + `- ${scan.id}: ${safe(clone.firstFile.name)}:${clone.firstFile.start}-${clone.firstFile.end} ↔ ${safe(clone.secondFile.name)}:${clone.secondFile.start}-${clone.secondFile.end}`, + ), + "", + "Download the duplication-report artifact: result.json contains limits/deltas and clone pairs; each jscpd-report.json contains exact source fragments. Tests are report-only. Normalised matches are candidates, not proof of equivalent behaviour.", + "Reproduce: npm ci --prefix tools/duplication --ignore-scripts && npm --prefix tools/duplication run check", + "", + ].join("\n"); +} + +export function run(options = {}) { + const root = realpathSync( + resolve(options.root || resolve(directory, "../..")), + ); + const base = options.base ? realpathSync(resolve(options.base)) : null; + const output = resolve(options.output || resolve(directory, "reports")); + mkdirSync(output, { recursive: true }); + const policy = read( + resolve(options.policy || resolve(directory, "policy.json")), + ); + const baselinePath = + options.baseline || + options.baselinePath || + resolve(directory, "baseline.json"); + const baseline = read(baselinePath); + const migration = + baseline.status === "established" && + baseline.policyHash !== policyHash(policy) + ? validateMigration( + read(resolve(root, "tools/duplication/migration.json")), + baseline, + policy, + ) + : null; + if (!migration) validateBaseline(baseline, policy); + const scanner = resolve( + options.scanner || resolve(directory, "node_modules/jscpd/run-jscpd.js"), + ); + const version = execFileSync(process.execPath, [scanner, "--version"], { + encoding: "utf8", + }).trim(); + if (version !== `jscpd ${policy.toolVersion}`) + throw new Error(`Unexpected scanner version: ${version}`); + const baseHasPolicy = + base && existsSync(resolve(base, "tools/duplication/policy.json")); + const bootstrap = baseline.status === "pending-bootstrap" && !baseHasPolicy; + if ( + baseHasPolicy && + !readFileSync(resolve(root, "tools/duplication/baseline.json")).equals( + readFileSync(resolve(base, "tools/duplication/baseline.json")), + ) + ) { + throw new Error("The baseline is owned by main CI; PRs must not edit it"); + } + if (migration && !base && !options["update-baseline"]) + throw new Error( + "A policy migration requires an actual main-base comparison", + ); + const sourceCommit = git(root, "rev-parse", "HEAD"); + const result = { + schemaVersion: 1, + toolVersion: policy.toolVersion, + policyHash: policyHash(policy), + sourceCommit, + baseCommit: base ? git(base, "rev-parse", "HEAD") : null, + bootstrap, + migration: migration + ? { + fromPolicyHash: policyHash(migration.fromPolicy), + toPolicyHash: policyHash(policy), + reason: migration.reason, + } + : null, + scans: [], + tests: [], + }; + for (const spec of policy.scans) { + const native = resolve(output, `base-${spec.id}.json`); + const previous = base + ? scan(base, policy, "production", spec, output, { + group: "base", + nativeBaseline: native, + seed: true, + scanner, + }) + : null; + const current = scan(root, policy, "production", spec, output, { + nativeBaseline: base ? native : null, + scanner, + }); + const ceiling = + baseline.status === "established" && !migration + ? baseline.scans[spec.id] + : null; + const baseMetrics = bootstrap ? null : previous?.metrics; + result.scans.push({ + id: spec.id, + ...current, + ceiling, + baseMetrics, + limits: limits(ceiling, baseMetrics), + regressions: regressions(current.metrics, ceiling, baseMetrics), + }); + } + for (const spec of policy.scans.filter((spec) => spec.minTokens === 50)) { + if (sourceFiles(root, policy, "tests").length) + result.tests.push({ + id: spec.id, + ...scan(root, policy, "tests", spec, output, { scanner }), + }); + } + const observed = Object.fromEntries( + result.scans.map((scan) => [scan.id, scan.metrics]), + ); + const failed = result.scans.some((scan) => scan.regressions.length); + write(resolve(output, "result.json"), result); + write(resolve(output, "baseline.json"), { + schemaVersion: 1, + status: "established", + toolVersion: policy.toolVersion, + policyHash: policyHash(policy), + sourceCommit, + scans: observed, + }); + const summary = markdown(result); + writeFileSync(resolve(output, "summary.md"), summary); + if ( + !options["no-summary"] && + options.reportToGitHub !== false && + process.env.GITHUB_STEP_SUMMARY + ) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + process.stdout.write( + process.env.GITHUB_ACTIONS || options["no-summary"] + ? `Duplication check ${failed ? "failed" : "passed"}; see the job summary and duplication-report artifact.\n` + : summary + "\n", + ); + let annotations = 0; + for (const scan of result.scans.filter((scan) => scan.regressions.length)) { + const message = `${scan.id}: ${scan.regressions.map((v) => `${v.metric} ${v.limit} → ${v.actual}`).join("; ")}`; + const candidates = scan.clones.filter((clone) => clone.isNew); + for (const clone of (candidates.length ? candidates : scan.clones).slice( + 0, + 5, + )) { + if ( + annotations++ >= 20 || + options.reportToGitHub === false || + !process.env.GITHUB_ACTIONS + ) + break; + process.stdout.write( + `::error file=${commandProperty(clone.firstFile.name)},line=${clone.firstFile.start}::${commandData(message + `; matching code: ${clone.secondFile.name}:${clone.secondFile.start}`)}\n`, + ); + } + } + if (failed) + throw new Error( + "Duplication regressed; see result.json and source-location reports. Do not raise the baseline to make CI pass.", + ); + if (options["update-baseline"]) { + if (base) + throw new Error("Baseline updates are not allowed during PR comparisons"); + const next = ratchet( + migration ? { schemaVersion: 1, status: "pending-bootstrap" } : baseline, + observed, + sourceCommit, + policy, + ); + if (next) write(baselinePath, next); + } + return result; +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const { values } = parseArgs({ + options: { + root: { type: "string" }, + base: { type: "string" }, + output: { type: "string" }, + policy: { type: "string" }, + baseline: { type: "string" }, + scanner: { type: "string" }, + "no-summary": { type: "boolean" }, + "update-baseline": { type: "boolean" }, + }, + }); + run(values); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/tools/duplication/check.test.mjs b/tools/duplication/check.test.mjs new file mode 100644 index 00000000..c80a9c2f --- /dev/null +++ b/tools/duplication/check.test.mjs @@ -0,0 +1,596 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, + mkdirSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { markdown, run as check, sourceFiles } from "./check.mjs"; +import { changedTooling, prepareMigration, verify } from "./verify.mjs"; +import { + commandData, + commandProperty, + metrics, + policyHash, + ratchet, + regressions, + relativeClone, + validateBaseline, + validateMigration, +} from "./metrics.mjs"; + +const policy = JSON.parse( + readFileSync(new URL("./policy.json", import.meta.url), "utf8"), +); +const pending = { schemaVersion: 1, status: "pending-bootstrap" }; +const counts = { clones: 4, duplicatedLines: 40, duplicatedTokens: 400 }; +const observed = () => + Object.fromEntries(policy.scans.map((scan) => [scan.id, { ...counts }])); +const baseline = () => ratchet(pending, observed(), "a".repeat(40), policy); +// Expected regression fixtures must not emit real CI errors or job summaries. +const run = (options) => check({ ...options, reportToGitHub: false }); + +test("first measurement establishes the benchmark without an arbitrary limit", () => { + const current = observed(); + current["exact-50"] = { + clones: 999, + duplicatedLines: 9999, + duplicatedTokens: 99999, + }; + const established = ratchet(pending, current, "a".repeat(40), policy); + assert.equal(established.status, "established"); + assert.deepEqual(established.scans, current); + assert.deepEqual(regressions(current["exact-50"]), []); + assert.equal(validateBaseline(established, policy), established); +}); + +test("every metric must be equal or lower, independently of other improvements", () => { + assert.deepEqual(regressions(counts, counts), []); + for (const metric of Object.keys(counts)) { + const current = { + clones: 0, + duplicatedLines: 0, + duplicatedTokens: 0, + [metric]: counts[metric] + 1, + }; + assert.deepEqual(regressions(current, counts), [ + { + metric, + actual: counts[metric] + 1, + limit: counts[metric], + increase: 1, + }, + ]); + } + assert.equal(regressions(counts, { ...counts, clones: 3 }).length, 1); +}); + +test("actual base metrics prevent stale committed ceilings from hiding regressions", () => { + const stale = { clones: 10, duplicatedLines: 100, duplicatedTokens: 1000 }; + assert.equal( + regressions({ ...counts, duplicatedLines: 41 }, stale, counts)[0].limit, + 40, + ); +}); + +test("ratcheting only tightens and unchanged metrics do not rewrite provenance", () => { + const original = baseline(); + assert.equal(ratchet(original, observed(), "b".repeat(40), policy), null); + const improved = observed(); + improved["exact-50"].duplicatedTokens--; + const tighter = ratchet(original, improved, "b".repeat(40), policy); + assert.equal(tighter.sourceCommit, "b".repeat(40)); + assert.equal(tighter.scans["exact-50"].duplicatedTokens, 399); + improved["renamed-100"].clones++; + assert.throws( + () => ratchet(original, improved, "b".repeat(40), policy), + /Refusing to increase/, + ); +}); + +test("missing, corrupt and incompatible baselines fail rather than rebootstrap", () => { + for (const invalid of [ + null, + {}, + { ...pending, scans: {} }, + { ...baseline(), policyHash: "wrong" }, + { ...baseline(), sourceCommit: "not-a-sha" }, + { ...baseline(), scans: {} }, + ]) { + assert.throws(() => validateBaseline(invalid, policy)); + } + for (const invalid of [null, -1, NaN, Infinity, 1.5, "4"]) { + assert.throws(() => + metrics({ statistics: { total: { ...counts, clones: invalid } } }), + ); + } + assert.notEqual( + policyHash(policy), + policyHash({ ...policy, maxBytes: policy.maxBytes + 1 }), + ); +}); + +test("policy migrations preserve the old main ceiling and require exact provenance", () => { + const next = { ...policy, toolVersion: "5.4.3" }; + const migration = { + schemaVersion: 1, + fromPolicy: policy, + toPolicyHash: policyHash(next), + reason: "Upgrade the pinned scanner", + }; + assert.equal(validateMigration(migration, baseline(), next), migration); + for (const invalid of [ + null, + { ...migration, reason: " " }, + { ...migration, toPolicyHash: "wrong" }, + { ...migration, fromPolicy: next }, + { ...migration, fromPolicy: { ...policy, maxBytes: 1 } }, + ]) { + assert.throws(() => validateMigration(invalid, baseline(), next)); + } +}); + +test("reports retain actionable paths, ranges and exact source payload", () => { + const clone = { + kind: "renamed", + fragment: 'const text = "unchanged";\n', + firstFile: { name: "/repo/a.go", start: 3, end: 9 }, + secondFile: { name: "/repo/b.go", start: 13, end: 19 }, + }; + const relative = relativeClone(clone, "/repo"); + assert.deepEqual(relative, { + ...clone, + firstFile: { ...clone.firstFile, name: "a.go" }, + secondFile: { ...clone.secondFile, name: "b.go" }, + }); + assert.throws(() => + relativeClone( + { ...clone, firstFile: { ...clone.firstFile, name: "/outside/a.go" } }, + "/repo", + ), + ); + const result = { + bootstrap: false, + scans: [ + { + id: "exact-50", + metrics: counts, + regressions: [{ metric: "clones", limit: 3, actual: 4, increase: 1 }], + clones: [{ ...relative, isNew: true }], + }, + ], + }; + const summary = markdown(result); + assert.match(summary, /clones 3 → 4/); + assert.match(summary, /a.go:3-9/); + assert.match(summary, /b.go:13-19/); + assert.match(summary, /source fragments/); + assert.match( + summary, + /\| Scan \| Clones \| Duplicated lines \| Duplicated tokens \| Result \|/, + ); + assert.match( + summary, + /\| exact-50 \| 4 \/ initial \| 40 \/ initial \| 400 \/ initial \| FAIL \|/, + ); + const withTests = markdown({ + ...result, + tests: [{ id: "exact-50", metrics: counts }], + }); + assert.match(withTests, /Test-source duplication \(report-only\)/); + assert.match(withTests, /REPORT ONLY/); +}); + +test("untrusted names cannot inject workflow commands or summary HTML", () => { + assert.equal(commandData("x%\r\n::error::bad"), "x%25%0D%0A::error::bad"); + assert.equal(commandProperty("a,b:1\nx"), "a%2Cb%3A1%0Ax"); + const result = { + bootstrap: false, + scans: [ + { + id: "exact-50", + metrics: counts, + regressions: [], + clones: [ + { + isNew: true, + firstFile: { name: "