From 85753902c81fb44d046c0be27a4620fcd4582778 Mon Sep 17 00:00:00 2001 From: Jose Alekhinne Date: Mon, 5 Oct 2026 19:04:25 -0700 Subject: [PATCH] fix(compliance): restore corrupted em-dashes and detect CP437 mojibake Every em-dash in compliance_test.go's section headers had been double-encoded through CP437 (the Windows console code page) into a six-character garble, 21 times over. TestNoMojibake, which lives in that same file, matched only the Windows-1252 signature (C3 A2 E2), so it passed on its own corruption. Restore the em-dashes and give the detector two more signatures: single CP437 (CE 93 C3) and CP437 applied twice (E2 95 AC C3 B4). Each is the garbled form of UTF-8 lead byte 0xE2, which every dash, quote, ellipsis, and arrow shares, so none occurs in real text. The failure message names the code page. The doc comment spells its examples as code points, since the literal forms would trip the test on its own file. Also: build the failure-context slice with max/min (the old code gave no leading context for a match in the first 20 bytes), and make the allGoFiles doc comment list all five skipped directories instead of only vendor/. Spec: specs/mojibake-cp437-detection.md Signed-off-by: Jose Alekhinne --- internal/compliance/compliance_test.go | 103 ++++++++++++++----------- specs/mojibake-cp437-detection.md | 59 ++++++++++++++ 2 files changed, 119 insertions(+), 43 deletions(-) create mode 100644 specs/mojibake-cp437-detection.md diff --git a/internal/compliance/compliance_test.go b/internal/compliance/compliance_test.go index 7b54336bb..b2efb5698 100644 --- a/internal/compliance/compliance_test.go +++ b/internal/compliance/compliance_test.go @@ -54,7 +54,8 @@ func templateDir(t *testing.T, root string) string { return "" } -// allGoFiles returns all .go files under the project root, excluding vendor/. +// allGoFiles returns all .go files under the project root, excluding +// vendor/, node_modules/, dist/, site/, and .git/. func allGoFiles(t *testing.T, root string) []string { t.Helper() var files []string @@ -96,7 +97,7 @@ func nonTestGoFiles(t *testing.T, root string) []string { } // --------------------------------------------------------------------------- -// 1. License Header ╬ô├ç├╢ every .go file must have the SPDX header +// 1. License Header — every .go file must have the SPDX header // --------------------------------------------------------------------------- // TestLicenseHeader verifies every .go file contains the Apache-2.0 SPDX @@ -128,7 +129,7 @@ func TestLicenseHeader(t *testing.T) { } // --------------------------------------------------------------------------- -// 2. Package doc.go ╬ô├ç├╢ every package under internal/ should have a doc.go +// 2. Package doc.go — every package under internal/ should have a doc.go // --------------------------------------------------------------------------- // TestDocGoExists verifies every Go package under internal/ has a doc.go. @@ -194,7 +195,7 @@ func TestDocGoExists(t *testing.T) { } // --------------------------------------------------------------------------- -// 3. No literal "\n" ╬ô├ç├╢ use config.NewlineLF instead (lint-drift rule 1) +// 3. No literal "\n" — use config.NewlineLF instead (lint-drift rule 1) // --------------------------------------------------------------------------- // TestNoLiteralNewline mirrors lint-drift rule 1: literal "\n" strings @@ -222,7 +223,7 @@ func TestNoLiteralNewline(t *testing.T) { } // --------------------------------------------------------------------------- -// 4. No literal ".md" ╬ô├ç├╢ use config.ExtMarkdown instead (lint-drift rule 4) +// 4. No literal ".md" — use config.ExtMarkdown instead (lint-drift rule 4) // --------------------------------------------------------------------------- // TestNoLiteralMdExtension mirrors lint-drift rule 4: literal ".md" strings @@ -251,7 +252,7 @@ func TestNoLiteralMdExtension(t *testing.T) { } // --------------------------------------------------------------------------- -// 5. No cmd.Printf/cmd.PrintErrf ╬ô├ç├╢ prefer Println (lint-drift rule 2) +// 5. No cmd.Printf/cmd.PrintErrf — prefer Println (lint-drift rule 2) // --------------------------------------------------------------------------- // TestNoCmdPrintf mirrors lint-drift rule 2: cmd.Printf/cmd.PrintErrf should @@ -313,7 +314,7 @@ func TestNoMagicDirectoryStrings(t *testing.T) { } // --------------------------------------------------------------------------- -// 7. No direct fmt.Print* in Cobra command functions ╬ô├ç├╢ use cmd.Print* +// 7. No direct fmt.Print* in Cobra command functions — use cmd.Print* // --------------------------------------------------------------------------- // TestNoDirectFmtPrintInCobraHandlers parses CLI source files and verifies @@ -404,7 +405,7 @@ func TestNoDirectFmtPrintInCobraHandlers(t *testing.T) { if ident.Name == fmtAlias && forbidden[sel.Sel.Name] { pos := fset.Position(call.Pos()) rel, _ := filepath.Rel(root, pos.Filename) - t.Errorf("%s:%d: fmt.%s in Cobra handler ╬ô├ç├╢ use cmd.Print* instead", + t.Errorf("%s:%d: fmt.%s in Cobra handler — use cmd.Print* instead", rel, pos.Line, sel.Sel.Name) } return true @@ -418,7 +419,7 @@ func TestNoDirectFmtPrintInCobraHandlers(t *testing.T) { } // --------------------------------------------------------------------------- -// 8. gofmt compliance ╬ô├ç├╢ all Go files must be properly formatted +// 8. gofmt compliance — all Go files must be properly formatted // --------------------------------------------------------------------------- // TestGofmt verifies all Go files are properly formatted. @@ -457,7 +458,7 @@ func TestGofmt(t *testing.T) { } // --------------------------------------------------------------------------- -// 9. go vet ╬ô├ç├╢ the entire project must pass go vet +// 9. go vet — the entire project must pass go vet // --------------------------------------------------------------------------- // TestGoVet runs go vet across the entire project with CGO disabled. @@ -478,7 +479,7 @@ func TestGoVet(t *testing.T) { } // --------------------------------------------------------------------------- -// 9b. golangci-lint ╬ô├ç├╢ the entire project must pass golangci-lint +// 9b. golangci-lint — the entire project must pass golangci-lint // --------------------------------------------------------------------------- // TestGolangciLint runs golangci-lint across the entire project. @@ -511,7 +512,7 @@ func TestGolangciLint(t *testing.T) { } // --------------------------------------------------------------------------- -// 10. No secrets in .context/ templates ╬ô├ç├╢ no tokens, keys, passwords +// 10. No secrets in .context/ templates — no tokens, keys, passwords // --------------------------------------------------------------------------- // TestNoSecretsInTemplates scans template files for patterns that look like @@ -570,7 +571,7 @@ func TestNoSecretsInTemplates(t *testing.T) { } // --------------------------------------------------------------------------- -// 11. Required context files ╬ô├ç├╢ ctx init must create all required files +// 11. Required context files — ctx init must create all required files // --------------------------------------------------------------------------- // TestRequiredContextFilesInTemplate verifies that all required context file @@ -600,7 +601,7 @@ func TestRequiredContextFilesInTemplate(t *testing.T) { } // --------------------------------------------------------------------------- -// 12. VERSION file ╬ô├ç├╢ must exist and contain a valid semver +// 12. VERSION file — must exist and contain a valid semver // --------------------------------------------------------------------------- // TestVersionFile checks the VERSION file exists and contains valid semver. @@ -626,7 +627,7 @@ func TestVersionFile(t *testing.T) { } // --------------------------------------------------------------------------- -// 13. go.mod ╬ô├ç├╢ module path and Go version check +// 13. go.mod — module path and Go version check // --------------------------------------------------------------------------- // TestGoMod verifies the module path and Go version in go.mod. @@ -657,7 +658,7 @@ func TestGoMod(t *testing.T) { } // --------------------------------------------------------------------------- -// 14. Makefile ╬ô├ç├╢ required targets exist +// 14. Makefile — required targets exist // --------------------------------------------------------------------------- // TestMakefileTargets verifies all expected build targets @@ -693,7 +694,7 @@ func TestMakefileTargets(t *testing.T) { } // --------------------------------------------------------------------------- -// 15. CGO_ENABLED=0 ╬ô├ç├╢ build command must not use CGO +// 15. CGO_ENABLED=0 — build command must not use CGO // --------------------------------------------------------------------------- // TestBuildWithoutCGO verifies that Makefile build and test targets use @@ -739,7 +740,7 @@ func TestBuildWithoutCGO(t *testing.T) { } // --------------------------------------------------------------------------- -// 16. .golangci.yml ╬ô├ç├╢ required linters are configured +// 16. .golangci.yml — required linters are configured // --------------------------------------------------------------------------- // TestGolangciLintConfig verifies that .golangci.yml enables the required @@ -814,7 +815,7 @@ func TestNoNetworkImportsInCore(t *testing.T) { impPath := strings.Trim(imp.Path.Value, `"`) if impPath == "net/http" || impPath == "net" { pos := fset.Position(imp.Pos()) - t.Errorf("%s:%d: %s imports %q ╬ô├ç├╢ ctx core must be local-only", + t.Errorf("%s:%d: %s imports %q — ctx core must be local-only", filepath.Base(pos.Filename), pos.Line, pkg, impPath) } } @@ -825,7 +826,7 @@ func TestNoNetworkImportsInCore(t *testing.T) { } // --------------------------------------------------------------------------- -// 18. Security ╬ô├ç├╢ .gitignore protects sensitive files +// 18. Security — .gitignore protects sensitive files // --------------------------------------------------------------------------- // TestGitignoreProtectsSensitiveFiles ensures .gitignore contains entries for @@ -856,7 +857,7 @@ func TestGitignoreProtectsSensitiveFiles(t *testing.T) { } // --------------------------------------------------------------------------- -// 19. Binary build ╬ô├ç├╢ ensure the project compiles without errors +// 19. Binary build — ensure the project compiles without errors // --------------------------------------------------------------------------- // TestProjectCompiles builds the entire project with CGO disabled to verify @@ -878,7 +879,7 @@ func TestProjectCompiles(t *testing.T) { } // --------------------------------------------------------------------------- -// 20. File permissions ╬ô├ç├╢ config.PermSecret must be 0600 +// 20. File permissions — config.PermSecret must be 0600 // --------------------------------------------------------------------------- // TestPermissionConstants verifies that config.PermSecret and config.PermFile @@ -1185,17 +1186,35 @@ func TestNoUTF8BOM(t *testing.T) { // 24. No mojibake — detect double-encoded UTF-8 (encoding corruption) // --------------------------------------------------------------------------- -// TestNoMojibake catches the classic Windows encoding corruption where UTF-8 -// bytes are misread as Windows-1252/Latin-1 and re-encoded as UTF-8. -// Example: em dash U+2014 becomes a 6-byte garbled sequence starting with -// 0xC3 0xA2. We detect that signature to catch double-encoded files. +// TestNoMojibake catches encoding corruption where UTF-8 bytes are misread +// in a legacy single-byte code page and re-encoded as UTF-8. Two code pages +// show up in practice, so both are covered: +// +// - Windows-1252/Latin-1 (GUI editors): em dash U+2014 (0xE2 0x80 0x94) +// becomes U+00E2 U+20AC U+201D, which starts 0xC3 0xA2 0xE2. +// - CP437 (Windows console, PowerShell pipelines): the same em dash +// becomes U+0393 U+00C7 U+00F6, which starts 0xCE 0x93 0xC3. A second +// CP437 round turns that into U+256C U+00F4 U+251C U+00E7 U+251C +// U+2562, which starts 0xE2 0x95 0xAC 0xC3 0xB4. +// +// Every character from U+2000 to U+2FFF (dashes, curly quotes, ellipsis, +// arrows, box drawing) has UTF-8 lead byte 0xE2, so each signature is the +// garbled form of that lead byte plus the start of what follows it. None +// occurs in legitimate text: Greek capital gamma is never followed by a +// Latin-1 letter, and a box-drawing glyph is never followed by "ô". +// +// The examples are spelled as code points on purpose: written literally, +// they would trip this test on its own source file. func TestNoMojibake(t *testing.T) { root := projectRoot(t) - // 0xC3 0xA2 is UTF-8 for U+00E2 (Latin small letter a with circumflex). - // In mojibake, it always appears followed by 0xE2 as part of a garbled - // multi-byte sequence (e.g., em dash becomes 0xC3 0xA2 0xE2 0x82 ...). - // We match that three-byte signature: 0xC3 0xA2 0xE2. - mojibakePattern := []byte{0xC3, 0xA2, 0xE2} + signatures := []struct { + codePage string + pattern []byte + }{ + {"Windows-1252", []byte{0xC3, 0xA2, 0xE2}}, + {"CP437", []byte{0xCE, 0x93, 0xC3}}, + {"double CP437", []byte{0xE2, 0x95, 0xAC, 0xC3, 0xB4}}, + } for _, p := range allSourceFiles(t, root) { rel, _ := filepath.Rel(root, p) @@ -1204,20 +1223,18 @@ func TestNoMojibake(t *testing.T) { if readErr != nil { t.Fatalf("read: %v", readErr) } - if idx := bytes.Index(data, mojibakePattern); idx >= 0 { - // Show context around the corruption - start := idx - if start > 20 { - start = idx - 20 - } - end := idx + 30 - if end > len(data) { - end = len(data) + for _, sig := range signatures { + idx := bytes.Index(data, sig.pattern) + if idx < 0 { + continue } - t.Errorf("double-encoded UTF-8 (mojibake) detected at byte %d: %q\n"+ - "This usually means a Windows editor re-encoded the file.\n"+ + // Show context around the corruption + start := max(0, idx-20) + end := min(len(data), idx+30) + t.Errorf("double-encoded UTF-8 (%s mojibake) detected at byte %d: %q\n"+ + "This usually means a Windows tool re-encoded the file.\n"+ "Fix: restore from git (git checkout HEAD -- %s) and re-apply changes with a UTF-8-aware editor.", - idx, data[start:end], rel) + sig.codePage, idx, data[start:end], rel) } }) } diff --git a/specs/mojibake-cp437-detection.md b/specs/mojibake-cp437-detection.md new file mode 100644 index 000000000..b786615f8 --- /dev/null +++ b/specs/mojibake-cp437-detection.md @@ -0,0 +1,59 @@ +# Mojibake Detection: Cover CP437 + +`TestNoMojibake` (compliance test 24) exists to keep +double-encoded UTF-8 out of source files. It only knew one +corruption, and the file that holds it carried 21 instances of +another. + +## Problem + +`internal/compliance/compliance_test.go` had every em-dash in its +section headers rendered as `╬ô├ç├╢`: an em dash +(`0xE2 0x80 0x94`) misread as CP437 (`ΓÇö`), re-encoded as UTF-8, +then put through the same round trip a second time. CP437 is the +default code page of the Windows console, so this is what a +PowerShell or `cmd` pipeline produces. + +The detector matched only the Windows-1252 signature +`0xC3 0xA2 0xE2` (the start of `—`), so it passed on its own +corrupted file. + +## Solution + +1. Restore the 21 em-dashes. +2. Teach `TestNoMojibake` three signatures, each the garbled form + of UTF-8 lead byte `0xE2` (shared by everything in + U+2000–U+2FFF: dashes, quotes, ellipsis, arrows) plus the + start of the next character: + + | Code page | Signature | + |---------------------|--------------------------------| + | Windows-1252 | `C3 A2 E2` | + | CP437 | `CE 93 C3` (Γ + Latin-1 lead) | + | CP437, applied twice| `E2 95 AC C3 B4` (╬ô) | + + None of them occurs in legitimate text. The failure message + names the code page. +3. Spell the example corruptions in the test's doc comment as + code points, because the literal forms would trip the test on + its own file. + +Bundled, same file: the context-window slice in the failure +message uses `max`/`min` (the old code showed no leading context +for a match within the first 20 bytes), and the stale +`allGoFiles` doc comment now lists all five directories the +walker skips, not just `vendor/`. + +## Verification + +- `TestNoMojibake` passes on the repaired tree. +- Planting each signature (`╬ô├ç├╢` in `compliance_test.go`, + `ΓÇö` in a `.go` doc file, `—` in another) makes the test fail + once per file with the matching code page named. +- `make lint` and `make test` pass. + +## Non-Goals + +- Scanning Markdown or YAML. `allSourceFiles` covers `.go`, `.ts`, + and `.js`; widening it is a separate decision. A repo-wide grep + for the three signatures found no other occurrences.