From 62802437660cc7e01386b99355d6678ff4f1163e Mon Sep 17 00:00:00 2001 From: Juan Ezquerro LLanes Date: Wed, 21 Oct 2026 02:10:10 +0200 Subject: [PATCH] fix: resolve all 61 SonarQube code quality and security issues --- .github/workflows/sonarcloud.yml | 2 +- gh-pages/javascripts/ecosystem.js | 23 +- gh-pages/stylesheets/extra.css | 122 ++--- internal/config/config_test.go | 111 ++-- internal/engine/chunking_test.go | 60 +-- internal/engine/finalize.go | 807 ++++++++++++++++-------------- internal/engine/orchestrator.go | 66 ++- internal/search/search.go | 297 +++++------ internal/search/search_test.go | 163 +++--- 9 files changed, 857 insertions(+), 794 deletions(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 0d7ebcf..13eb96b 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -31,7 +31,7 @@ jobs: go test -coverprofile=coverage.out -covermode=atomic ./... - name: SonarQube Cloud Scan - uses: SonarSource/sonarqube-scan-action@v8 + uses: SonarSource/sonarqube-scan-action@d209202bc7d53ff1cc128f7f907dac145c9d6ae9 # v8.3.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/gh-pages/javascripts/ecosystem.js b/gh-pages/javascripts/ecosystem.js index 22fbebb..87c710c 100644 --- a/gh-pages/javascripts/ecosystem.js +++ b/gh-pages/javascripts/ecosystem.js @@ -5,13 +5,13 @@ const saved = localStorage.getItem(THEME_KEY); if (saved !== "slate" && saved !== "default") return; - if (document.body.getAttribute("data-md-color-scheme") !== saved) { - document.body.setAttribute("data-md-color-scheme", saved); + if (document.body.dataset.mdColorScheme !== saved) { + document.body.dataset.mdColorScheme = saved; } const inputs = document.querySelectorAll('input[name="__palette"]'); inputs.forEach((input) => { - const scheme = input.getAttribute("data-md-color-scheme"); + const scheme = input.dataset.mdColorScheme; if (scheme === saved && !input.checked) { input.checked = true; } @@ -20,15 +20,12 @@ function bindThemeSync() { const paletteForm = document.querySelector('form[data-md-component="palette"]'); - if (paletteForm && !paletteForm.hasAttribute("data-arrase-bound")) { - paletteForm.setAttribute("data-arrase-bound", "true"); + if (paletteForm && !("arraseBound" in paletteForm.dataset)) { + paletteForm.dataset.arraseBound = "true"; paletteForm.addEventListener("change", (event) => { - const target = event.target; - if (target && target.getAttribute) { - const scheme = target.getAttribute("data-md-color-scheme"); - if (scheme === "slate" || scheme === "default") { - localStorage.setItem(THEME_KEY, scheme); - } + const scheme = event.target?.dataset?.mdColorScheme; + if (scheme === "slate" || scheme === "default") { + localStorage.setItem(THEME_KEY, scheme); } }); } @@ -39,11 +36,11 @@ const isRootPortal = path === "/" || path === "/index.html"; if (isRootPortal) { - document.body.setAttribute("data-arrase-portal", "true"); + document.body.dataset.arrasePortal = "true"; return; } - document.body.removeAttribute("data-arrase-portal"); + delete document.body.dataset.arrasePortal; const headerInner = document.querySelector(".md-header__inner"); if (!headerInner || headerInner.querySelector(".arrase-portal-link")) { diff --git a/gh-pages/stylesheets/extra.css b/gh-pages/stylesheets/extra.css index 1dbb2df..7c4040b 100644 --- a/gh-pages/stylesheets/extra.css +++ b/gh-pages/stylesheets/extra.css @@ -122,8 +122,8 @@ code, pre, kbd, .md-typeset code { } /* Layout & Subtle Top Ambient Light */ -.md-grid { - max-width: 82rem !important; +.md-container .md-grid { + max-width: 82rem; } .md-main { @@ -134,8 +134,8 @@ code, pre, kbd, .md-typeset code { background: radial-gradient(ellipse 80% 40% at 50% -10%, rgba(79, 70, 229, 0.06), transparent 70%); } -.md-content__inner { - padding-top: 1rem !important; +.md-content .md-content__inner { + padding-top: 1rem; } /* Integrated Glassmorphic Header & Navigation Tabs */ @@ -144,63 +144,63 @@ code, pre, kbd, .md-typeset code { backdrop-filter: blur(16px); -webkit-backdrop-filter: blur(16px); border-bottom: 1px solid var(--border-subtle); - box-shadow: none !important; - color: var(--text-main) !important; + box-shadow: none; + color: var(--text-main); } [data-md-color-scheme="slate"] .md-header, [data-md-color-scheme="slate"] .md-tabs { - background: rgba(9, 10, 15, 0.80) !important; + background: rgba(9, 10, 15, 0.80); } [data-md-color-scheme="default"] .md-header, [data-md-color-scheme="default"] .md-tabs { - background: rgba(250, 250, 250, 0.85) !important; + background: rgba(250, 250, 250, 0.85); } -.md-header__button, -.md-header__title, -.md-header__topic, -.md-source, -.md-tabs__link { - color: var(--text-main) !important; +.md-header .md-header__button, +.md-header .md-header__title, +.md-header .md-header__topic, +.md-header .md-source, +.md-tabs .md-tabs__link { + color: var(--text-main); } -.md-tabs__link { +.md-tabs .md-tabs__link { opacity: 0.72; transition: opacity 0.2s ease, color 0.2s ease; } -.md-tabs__link--active, -.md-tabs__link:hover { +.md-tabs .md-tabs__link--active, +.md-tabs .md-tabs__link:hover { opacity: 1; - color: var(--accent-primary) !important; + color: var(--accent-primary); } .md-nav--primary .md-nav__title, .md-nav--secondary .md-nav__title, .md-nav--lifted > .md-nav__list > .md-nav__item--active > .md-nav__link { - background: transparent !important; - box-shadow: none !important; - color: var(--text-muted) !important; + background: transparent; + box-shadow: none; + color: var(--text-muted); font-size: 0.72rem; font-weight: 600; letter-spacing: 0.05em; text-transform: uppercase; } -.md-nav__item .md-nav__link--active { - color: var(--accent-primary) !important; +.md-nav .md-nav__item .md-nav__link--active { + color: var(--accent-primary); font-weight: 600; } -.md-footer-meta { - background: var(--bg-canvas) !important; +.md-footer .md-footer-meta { + background: var(--bg-canvas); border-top: 1px solid var(--border-subtle); } /* Return to Portal Button (Injected on Subprojects) */ -.arrase-portal-link { +.md-header__inner .arrase-portal-link { display: inline-flex; align-items: center; gap: 0.45rem; @@ -209,20 +209,20 @@ code, pre, kbd, .md-typeset code { border-radius: 6px; border: 1px solid var(--border-subtle); background: var(--bg-surface); - color: var(--text-main) !important; + color: var(--text-main); font-size: 0.78rem; font-weight: 500; line-height: 1.4; - text-decoration: none !important; + text-decoration: none; white-space: nowrap; transition: border-color 0.2s ease, background-color 0.2s ease, transform 0.2s ease; } -.arrase-portal-link:hover { +.md-header__inner .arrase-portal-link:hover { border-color: var(--border-hover); background: var(--bg-surface-hover); transform: translateX(-1px); - color: var(--accent-primary) !important; + color: var(--accent-primary); } .arrase-portal-link i { @@ -231,30 +231,30 @@ code, pre, kbd, .md-typeset code { } /* Search Bar — Hidden only on Root Portal, Styled Cleanly on Subprojects */ -body[data-arrase-portal="true"] .md-search { - display: none !important; +body[data-arrase-portal="true"] .md-header .md-search { + display: none; } -.md-search__form { - background-color: var(--bg-surface) !important; +.md-search .md-search__form { + background-color: var(--bg-surface); border: 1px solid var(--border-subtle); - border-radius: 6px !important; + border-radius: 6px; color: var(--text-main); transition: border-color 0.2s ease, background-color 0.2s ease; } -.md-search__form:hover { +.md-search .md-search__form:hover { border-color: var(--border-hover); } -.md-search__input { - color: var(--text-main) !important; +.md-search .md-search__input { + color: var(--text-main); font-size: 0.82rem; } -.md-search__input::placeholder, -.md-search__icon { - color: var(--text-muted) !important; +.md-search .md-search__input::placeholder, +.md-search .md-search__icon { + color: var(--text-muted); } /* Hero Section */ @@ -281,13 +281,13 @@ body[data-arrase-portal="true"] .md-search { margin-bottom: 1rem; } -.hero-section h1, +.md-typeset .hero-section h1, .md-typeset h1#life-is-the-ultimate-side-project { font-size: clamp(2rem, 1.5rem + 2vw, 3rem); font-weight: 800; letter-spacing: -0.03em; line-height: 1.15; - margin: 0 0 0.75rem 0 !important; + margin: 0 0 0.75rem 0; background: linear-gradient(135deg var(--in-oklab), var(--text-main) 20%, var(--accent-primary) 65%, var(--accent-secondary) 100%); -webkit-background-clip: text; background-clip: text; @@ -295,12 +295,12 @@ body[data-arrase-portal="true"] .md-search { text-align: center; } -.hero-subtitle, +.md-typeset .hero-subtitle, .md-typeset h1#life-is-the-ultimate-side-project + p { font-size: clamp(1rem, 0.95rem + 0.25vw, 1.15rem); color: var(--text-muted); max-width: 42rem; - margin: 0 auto 1.25rem auto !important; + margin: 0 auto 1.25rem auto; line-height: 1.6; text-align: center; } @@ -330,11 +330,11 @@ body[data-arrase-portal="true"] .md-search { border-bottom: 1px solid var(--border-subtle); } -.section-header h2 { +.md-typeset .section-header h2 { font-size: 1.15rem; font-weight: 650; letter-spacing: -0.015em; - margin: 0 !important; + margin: 0; color: var(--text-main); } @@ -352,7 +352,7 @@ body[data-arrase-portal="true"] .md-search { } /* Feature Cards — Shared Base & Interactive vs Static Variants */ -.feature-card { +.md-typeset .feature-card { display: flex; flex-direction: column; justify-content: space-between; @@ -360,12 +360,12 @@ body[data-arrase-portal="true"] .md-search { border: 1px solid var(--border-subtle); border-radius: 10px; padding: 1.25rem 1.35rem; - text-decoration: none !important; - color: inherit !important; + text-decoration: none; + color: inherit; position: relative; overflow: hidden; transition: border-color 0.2s ease, background-color 0.2s ease, transform 0.2s ease, box-shadow 0.2s ease; - word-break: break-word; + overflow-wrap: break-word; } .feature-card::before { @@ -380,7 +380,7 @@ body[data-arrase-portal="true"] .md-search { transition: opacity 0.2s ease; } -a.feature-card:hover { +.md-typeset a.feature-card:hover { transform: translateY(-2px); background: var(--bg-surface-hover); border-color: var(--border-hover); @@ -408,9 +408,9 @@ a.feature-card:hover::before { margin-bottom: 0.75rem; } -.feature-icon { +.md-typeset .feature-icon { font-size: 1.25rem; - color: var(--accent-primary) !important; + color: var(--accent-primary); margin-bottom: 0.75rem; display: inline-flex; align-items: center; @@ -444,25 +444,25 @@ a.feature-card:hover .card-badge { } /* Card Content & Tech Tags */ -.feature-card h3 { +.md-typeset .feature-card h3 { font-size: 1.05rem; font-weight: 650; - color: var(--text-main) !important; - margin: 0 0 0.4rem 0 !important; + color: var(--text-main); + margin: 0 0 0.4rem 0; letter-spacing: -0.01em; line-height: 1.35; transition: color 0.2s ease; } -a.feature-card:hover h3 { - color: var(--accent-primary) !important; +.md-typeset a.feature-card:hover h3 { + color: var(--accent-primary); } -.feature-card p { +.md-typeset .feature-card p { font-size: 0.86rem; line-height: 1.5; color: var(--text-muted); - margin: 0 !important; + margin: 0; flex-grow: 1; } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c8ebe66..ef65515 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -52,9 +52,7 @@ func TestLoadConfig(t *testing.T) { func TestResolveConfig(t *testing.T) { t.Run("valid config resolution order", func(t *testing.T) { dir := t.TempDir() - cfg := &Config{OllamaNumCtx: 2048, ModelID: "yaml-model", Language: "yaml-lang"} - err := SaveConfig(dir, cfg) - if err != nil { + if err := SaveConfig(dir, &Config{OllamaNumCtx: 2048, ModelID: "yaml-model", Language: "yaml-lang"}); err != nil { t.Fatalf("failed to save config: %v", err) } @@ -62,104 +60,83 @@ func TestResolveConfig(t *testing.T) { t.Setenv(CodeReducerModelIDEnvKey, "env-model") t.Setenv(CodeReducerLanguageEnvKey, "env-lang") - resolved, err := ResolveConfig(dir, "flag-model", "8192", "flag-lang") - if err != nil { - t.Fatalf("failed to resolve config: %v", err) - } - - if resolved.OllamaNumCtx != 8192 { - t.Errorf("expected 8192, got %d", resolved.OllamaNumCtx) - } - if resolved.ModelID != "flag-model" { - t.Errorf("expected flag-model, got %s", resolved.ModelID) - } - if resolved.Language != "flag-lang" { - t.Errorf("expected flag-lang, got %s", resolved.Language) - } - if resolved.QuickstartPrompt != DefaultQuickstartPrompt { - t.Errorf("expected default quickstart prompt, got %s", resolved.QuickstartPrompt) - } + resolved := mustResolveConfig(t, dir, "flag-model", "8192", "flag-lang") + assertConfigField(t, "OllamaNumCtx", resolved.OllamaNumCtx, 8192) + assertConfigField(t, "ModelID", resolved.ModelID, "flag-model") + assertConfigField(t, "Language", resolved.Language, "flag-lang") + assertConfigField(t, "QuickstartPrompt", resolved.QuickstartPrompt, DefaultQuickstartPrompt) }) t.Run("language 4-tier resolution", func(t *testing.T) { dir := t.TempDir() // Tier 4: Default ("") - resolved, err := ResolveConfig(dir, "", "", "") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resolved.Language != "" { - t.Errorf("expected empty default language, got %q", resolved.Language) - } + assertResolvedLanguage(t, dir, "", "") // Tier 3: YAML config if err := SaveConfig(dir, &Config{Language: "Spanish"}); err != nil { t.Fatalf("failed to save config: %v", err) } - resolved, err = ResolveConfig(dir, "", "", "") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resolved.Language != "Spanish" { - t.Errorf("expected Spanish from YAML, got %q", resolved.Language) - } + assertResolvedLanguage(t, dir, "", "Spanish") // Tier 2: Environment variable overrides YAML t.Setenv(CodeReducerLanguageEnvKey, "French") - resolved, err = ResolveConfig(dir, "", "", "") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resolved.Language != "French" { - t.Errorf("expected French from env, got %q", resolved.Language) - } + assertResolvedLanguage(t, dir, "", "French") // Tier 1: CLI flag overrides Env and YAML - resolved, err = ResolveConfig(dir, "", "", "German") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resolved.Language != "German" { - t.Errorf("expected German from CLI flag, got %q", resolved.Language) - } + assertResolvedLanguage(t, dir, "German", "German") }) t.Run("fail fast on invalid num ctx env", func(t *testing.T) { t.Setenv(OllamaNumCtxEnvKey, "invalid") - _, err := ResolveConfig(t.TempDir(), "", "", "") - if err == nil { - t.Fatal("expected error, got nil") - } + assertResolveError(t, t.TempDir(), "", "", "") }) t.Run("fail fast on zero num ctx env", func(t *testing.T) { t.Setenv(OllamaNumCtxEnvKey, "0") - _, err := ResolveConfig(t.TempDir(), "", "", "") - if err == nil { - t.Fatal("expected error, got nil") - } + assertResolveError(t, t.TempDir(), "", "", "") }) t.Run("fail fast on invalid num ctx flag", func(t *testing.T) { - _, err := ResolveConfig(t.TempDir(), "", "invalid", "") - if err == nil { - t.Fatal("expected error, got nil") - } + assertResolveError(t, t.TempDir(), "", "invalid", "") }) t.Run("resolve base url from env", func(t *testing.T) { t.Setenv(OllamaBaseURLEnvKey, "http://custom-ollama:11434") - resolved, err := ResolveConfig(t.TempDir(), "", "", "") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resolved.OllamaBaseURL != "http://custom-ollama:11434" { - t.Errorf("expected http://custom-ollama:11434, got %s", resolved.OllamaBaseURL) - } + resolved := mustResolveConfig(t, t.TempDir(), "", "", "") + assertConfigField(t, "OllamaBaseURL", resolved.OllamaBaseURL, "http://custom-ollama:11434") }) } +func mustResolveConfig(t *testing.T, dir, flagModel, flagCtx, flagLang string) *Config { + t.Helper() + resolved, err := ResolveConfig(dir, flagModel, flagCtx, flagLang) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return resolved +} + +func assertResolveError(t *testing.T, dir, flagModel, flagCtx, flagLang string) { + t.Helper() + if _, err := ResolveConfig(dir, flagModel, flagCtx, flagLang); err == nil { + t.Fatal("expected error, got nil") + } +} + +func assertConfigField[T comparable](t *testing.T, name string, got, want T) { + t.Helper() + if got != want { + t.Errorf("%s: expected %v, got %v", name, want, got) + } +} + +func assertResolvedLanguage(t *testing.T, dir, flagLang, want string) { + t.Helper() + resolved := mustResolveConfig(t, dir, "", "", flagLang) + assertConfigField(t, "Language", resolved.Language, want) +} + func TestLanguageInstruction(t *testing.T) { if got := LanguageInstruction(""); got != "" { t.Errorf("expected empty string for empty language, got %q", got) diff --git a/internal/engine/chunking_test.go b/internal/engine/chunking_test.go index 287601e..cb38157 100644 --- a/internal/engine/chunking_test.go +++ b/internal/engine/chunking_test.go @@ -2,10 +2,22 @@ package engine import ( "context" + "slices" "strings" "testing" ) +func assertChunks(t *testing.T, text string, maxRunes, overlapRunes int, want ...string) { + t.Helper() + chunks, err := chunkTextWithOverlap(text, maxRunes, overlapRunes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !slices.Equal(chunks, want) { + t.Fatalf("expected %q, got %q", want, chunks) + } +} + func TestChunkTextWithOverlap(t *testing.T) { t.Run("invalid maxRunes", func(t *testing.T) { _, err := chunkTextWithOverlap("test", 0, 0) @@ -24,59 +36,19 @@ func TestChunkTextWithOverlap(t *testing.T) { }) t.Run("text shorter than maxRunes", func(t *testing.T) { - chunks, err := chunkTextWithOverlap("hello", 10, 2) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(chunks) != 1 || chunks[0] != "hello" { - t.Fatalf("expected ['hello'], got %v", chunks) - } + assertChunks(t, "hello", 10, 2, "hello") }) t.Run("text exactly maxRunes", func(t *testing.T) { - chunks, err := chunkTextWithOverlap("hello", 5, 2) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(chunks) != 1 || chunks[0] != "hello" { - t.Fatalf("expected ['hello'], got %v", chunks) - } + assertChunks(t, "hello", 5, 2, "hello") }) t.Run("chunking with overlap", func(t *testing.T) { - chunks, err := chunkTextWithOverlap("abcdefg", 4, 2) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(chunks) != 3 { - t.Fatalf("expected 3 chunks, got %d", len(chunks)) - } - if chunks[0] != "abcd" { - t.Errorf("expected chunk 0 to be 'abcd', got '%s'", chunks[0]) - } - if chunks[1] != "cdef" { - t.Errorf("expected chunk 1 to be 'cdef', got '%s'", chunks[1]) - } - if chunks[2] != "efg" { - t.Errorf("expected chunk 2 to be 'efg', got '%s'", chunks[2]) - } + assertChunks(t, "abcdefg", 4, 2, "abcd", "cdef", "efg") }) t.Run("chunking splits at last newline in tail window", func(t *testing.T) { - text := "line1\nline2\nline3\n" - chunks, err := chunkTextWithOverlap(text, 14, 2) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(chunks) != 2 { - t.Fatalf("expected 2 chunks, got %d: %q", len(chunks), chunks) - } - if chunks[0] != "line1\nline2\n" { - t.Errorf("expected chunk 0 to end after newline 'line1\\nline2\\n', got %q", chunks[0]) - } - if chunks[1] != "2\nline3\n" { - t.Errorf("expected chunk 1 to preserve overlap '2\\nline3\\n', got %q", chunks[1]) - } + assertChunks(t, "line1\nline2\nline3\n", 14, 2, "line1\nline2\n", "2\nline3\n") }) } diff --git a/internal/engine/finalize.go b/internal/engine/finalize.go index 2a996b0..fb34134 100644 --- a/internal/engine/finalize.go +++ b/internal/engine/finalize.go @@ -160,22 +160,10 @@ func sanitizeAndValidateMermaid(markdown string) (string, int) { continue } - infoFields := strings.Fields(info) - isMermaid := len(infoFields) > 0 && strings.EqualFold(infoFields[0], "mermaid") - - var bodyLines []string - closed := false - var closeLine string - for i = i + 1; i < len(lines); i++ { - if tools.IsFenceClose(lines[i], fenceChar, fenceCount) { - closed = true - closeLine = lines[i] - break - } - bodyLines = append(bodyLines, lines[i]) - } + bodyLines, closeLine, closed, nextIdx := collectFencedBlock(lines, i, fenceChar, fenceCount) + i = nextIdx - if !isMermaid { + if !isMermaidFence(info) { out = append(out, line) out = append(out, bodyLines...) if closed { @@ -184,33 +172,56 @@ func sanitizeAndValidateMermaid(markdown string) (string, int) { continue } - indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] - fence := strings.Repeat(string(fenceChar), fenceCount) - rawBody := strings.Join(bodyLines, "\n") - if !closed { + blockLines, degraded := renderMermaidFence(line, fenceChar, fenceCount, bodyLines, closeLine, closed) + out = append(out, blockLines...) + if degraded { degradedCount++ - out = append(out, fmt.Sprintf("%s%s (unclosed mermaid code fence) -->", indent, mermaidFailedPrefix)) - out = append(out, fmt.Sprintf("%s%stext", indent, fence)) - out = append(out, bodyLines...) - continue } + } - sanitizedBody := sanitizeMermaidBody(rawBody) - if errMsg := heuristicMermaidError(sanitizedBody); errMsg != "" { - degradedCount++ - out = append(out, fmt.Sprintf("%s%s (%s) -->", indent, mermaidFailedPrefix, errMsg)) - out = append(out, fmt.Sprintf("%s%stext", indent, fence)) - out = append(out, bodyLines...) - out = append(out, closeLine) - continue + return strings.Join(out, "\n"), degradedCount +} + +func isMermaidFence(info string) bool { + fields := strings.Fields(info) + return len(fields) > 0 && strings.EqualFold(fields[0], "mermaid") +} + +func collectFencedBlock(lines []string, openIdx int, fenceChar byte, fenceCount int) ([]string, string, bool, int) { + var bodyLines []string + for i := openIdx + 1; i < len(lines); i++ { + if tools.IsFenceClose(lines[i], fenceChar, fenceCount) { + return bodyLines, lines[i], true, i + } + bodyLines = append(bodyLines, lines[i]) + } + return bodyLines, "", false, len(lines) +} + +func renderMermaidFence(openLine string, fenceChar byte, fenceCount int, bodyLines []string, closeLine string, closed bool) ([]string, bool) { + indent := openLine[:len(openLine)-len(strings.TrimLeft(openLine, " \t"))] + fence := strings.Repeat(string(fenceChar), fenceCount) + if !closed { + out := []string{ + fmt.Sprintf("%s%s (unclosed mermaid code fence) -->", indent, mermaidFailedPrefix), + fmt.Sprintf("%s%stext", indent, fence), } + return append(out, bodyLines...), true + } - out = append(out, line) - out = append(out, strings.Split(sanitizedBody, "\n")...) - out = append(out, closeLine) + sanitizedBody := sanitizeMermaidBody(strings.Join(bodyLines, "\n")) + if errMsg := heuristicMermaidError(sanitizedBody); errMsg != "" { + out := []string{ + fmt.Sprintf("%s%s (%s) -->", indent, mermaidFailedPrefix, errMsg), + fmt.Sprintf("%s%stext", indent, fence), + } + out = append(out, bodyLines...) + return append(out, closeLine), true } - return strings.Join(out, "\n"), degradedCount + out := []string{openLine} + out = append(out, strings.Split(sanitizedBody, "\n")...) + return append(out, closeLine), false } func firstMermaidHeaderToken(body string) (string, int) { @@ -288,46 +299,16 @@ func sanitizeFlowchartLine(line string) string { for i < n { if line[i] == '"' || line[i] == '|' { - delim := line[i] - b.WriteByte(delim) - i++ - for i < n { - b.WriteByte(line[i]) - if line[i] == delim && line[i-1] != '\\' { - i++ - break - } - i++ - } + end := skipDelimitedSegment(line, i) + b.WriteString(line[i:end]) + i = end continue } if isIdentChar(line[i]) { - start := i - for i < n && isIdentChar(line[i]) { - i++ - } - ident := line[start:i] - - if i < n && (line[i] == '[' || line[i] == '(' || line[i] == '{') { - openChar := line[i] - closeChar := matchingCloseBracket(openChar) - closeIdx := findMatchingDelimiter(line, i, openChar, closeChar) - if closeIdx != -1 { - if ident == "end" { - ident = "endNode" - } - openDelim, inner, closeDelim := splitNodeShape(line[i : closeIdx+1]) - b.WriteString(ident) - b.WriteString(openDelim) - b.WriteString(sanitizeNodeLabel(inner)) - b.WriteString(closeDelim) - i = closeIdx + 1 - continue - } - } - - b.WriteString(ident) + nodeExpr, nextIdx := consumeFlowchartIdentOrNode(line, i) + b.WriteString(nodeExpr) + i = nextIdx continue } @@ -341,6 +322,47 @@ func sanitizeFlowchartLine(line string) string { return res } +func skipDelimitedSegment(s string, start int) int { + delim := s[start] + for i := start + 1; i < len(s); i++ { + if s[i] == delim && s[i-1] != '\\' { + return i + 1 + } + } + return len(s) +} + +func consumeFlowchartIdentOrNode(line string, start int) (string, int) { + i := start + for i < len(line) && isIdentChar(line[i]) { + i++ + } + ident := line[start:i] + + closeChar := matchingCloseBracket(peekByte(line, i)) + if closeChar == 0 { + return ident, i + } + + closeIdx := findMatchingDelimiter(line, i, line[i], closeChar) + if closeIdx == -1 { + return ident, i + } + + if ident == "end" { + ident = "endNode" + } + openDelim, inner, closeDelim := splitNodeShape(line[i : closeIdx+1]) + return ident + openDelim + sanitizeNodeLabel(inner) + closeDelim, closeIdx + 1 +} + +func peekByte(s string, idx int) byte { + if idx < len(s) { + return s[idx] + } + return 0 +} + func findMatchingDelimiter(s string, openIdx int, openChar, closeChar byte) int { depth := 0 inQuotes := false @@ -413,20 +435,8 @@ func heuristicMermaidError(body string) string { continue } contentLines++ - - if countUnescapedQuotes(trimmed)%2 != 0 { - return "unclosed double quote" - } - if !hasBalancedBrackets(trimmed, isFlow) { - return "unbalanced brackets" - } - if isFlow { - if hasFlowchartEndNodeID(trimmed) { - return "reserved keyword 'end' used as node ID" - } - if hasUnquotedSpecialLabel(trimmed) { - return "unquoted special characters in node label" - } + if errMsg := validateMermaidContentLine(trimmed, isFlow); errMsg != "" { + return errMsg } } @@ -436,6 +446,25 @@ func heuristicMermaidError(body string) string { return "" } +func validateMermaidContentLine(trimmed string, isFlow bool) string { + if countUnescapedQuotes(trimmed)%2 != 0 { + return "unclosed double quote" + } + if !hasBalancedBrackets(trimmed, isFlow) { + return "unbalanced brackets" + } + if !isFlow { + return "" + } + if hasFlowchartEndNodeID(trimmed) { + return "reserved keyword 'end' used as node ID" + } + if hasUnquotedSpecialLabel(trimmed) { + return "unquoted special characters in node label" + } + return "" +} + func hasBalancedBrackets(line string, checkBraces bool) bool { var stack []byte inQuotes := false @@ -448,30 +477,31 @@ func hasBalancedBrackets(line string, checkBraces bool) bool { if inQuotes { continue } - switch ch { - case '[', '(': - stack = append(stack, matchingCloseBracket(ch)) - case '{': - if checkBraces { - stack = append(stack, '}') - } - case ']', ')': + if closeCh := expectedCloseBracket(ch, checkBraces); closeCh != 0 { + stack = append(stack, closeCh) + continue + } + if isTrackedCloseBracket(ch, checkBraces) { if len(stack) == 0 || stack[len(stack)-1] != ch { return false } stack = stack[:len(stack)-1] - case '}': - if checkBraces { - if len(stack) == 0 || stack[len(stack)-1] != ch { - return false - } - stack = stack[:len(stack)-1] - } } } return len(stack) == 0 } +func expectedCloseBracket(ch byte, checkBraces bool) byte { + if ch == '{' && !checkBraces { + return 0 + } + return matchingCloseBracket(ch) +} + +func isTrackedCloseBracket(ch byte, checkBraces bool) bool { + return ch == ']' || ch == ')' || (checkBraces && ch == '}') +} + func hasFlowchartEndNodeID(line string) bool { return reArrowToEnd.MatchString(line) || reEndToArrow.MatchString(line) || reEndShapeNode.MatchString(line) } @@ -481,37 +511,15 @@ func hasUnquotedSpecialLabel(line string) bool { n := len(line) for i < n { if line[i] == '"' || line[i] == '|' { - delim := line[i] - i++ - for i < n { - if line[i] == delim && line[i-1] != '\\' { - i++ - break - } - i++ - } + i = skipDelimitedSegment(line, i) continue } if isIdentChar(line[i]) { - for i < n && isIdentChar(line[i]) { - i++ - } - if i < n && (line[i] == '[' || line[i] == '(' || line[i] == '{') { - openChar := line[i] - closeChar := matchingCloseBracket(openChar) - closeIdx := findMatchingDelimiter(line, i, openChar, closeChar) - if closeIdx == -1 { - return true - } - _, inner, _ := splitNodeShape(line[i : closeIdx+1]) - inner = strings.TrimSpace(inner) - isQuoted := len(inner) >= 2 && strings.HasPrefix(inner, "\"") && strings.HasSuffix(inner, "\"") - if !isQuoted && strings.ContainsAny(inner, ";<>") { - return true - } - i = closeIdx + 1 - continue + bad, nextIdx := checkIdentNodeLabel(line, i) + if bad { + return true } + i = nextIdx continue } i++ @@ -519,24 +527,34 @@ func hasUnquotedSpecialLabel(line string) bool { return false } +func checkIdentNodeLabel(line string, start int) (bool, int) { + i := start + for i < len(line) && isIdentChar(line[i]) { + i++ + } + closeChar := matchingCloseBracket(peekByte(line, i)) + if closeChar == 0 { + return false, i + } + closeIdx := findMatchingDelimiter(line, i, line[i], closeChar) + if closeIdx == -1 { + return true, len(line) + } + _, inner, _ := splitNodeShape(line[i : closeIdx+1]) + inner = strings.TrimSpace(inner) + isQuoted := len(inner) >= 2 && strings.HasPrefix(inner, "\"") && strings.HasSuffix(inner, "\"") + if !isQuoted && strings.ContainsAny(inner, ";<>") { + return true, closeIdx + 1 + } + return false, closeIdx + 1 +} + // generateWikiIndex deterministically generates /index.md with links to // architecture.md, quickstart.md, and all documented modules. func generateWikiIndex(repoRoot, docsDir string, tree *DirNode, cache *MetadataCache) error { var b strings.Builder b.WriteString("# Documentation Index\n\n") - - archExists := tools.FileExistsSafely(repoRoot, filepath.Join(docsDir, "architecture.md")) - qsExists := tools.FileExistsSafely(repoRoot, filepath.Join(docsDir, "quickstart.md")) - if archExists || qsExists { - b.WriteString("## Overview\n\n") - if archExists { - b.WriteString("- [Architecture Overview](architecture.md)\n") - } - if qsExists { - b.WriteString("- [Developer Quickstart](quickstart.md)\n") - } - b.WriteString("\n") - } + writeIndexOverview(&b, repoRoot, docsDir) nodesByPath := collectTreeNodes(tree) var dirPaths []string @@ -547,74 +565,78 @@ func generateWikiIndex(repoRoot, docsDir string, tree *DirNode, cache *MetadataC b.WriteString("## Modules\n\n") for _, dirPath := range dirPaths { - relModPath := filepath.ToSlash(filepath.Join("modules", toSafeMarkdownFilename(dirPath))) - fullModPath := filepath.Join(docsDir, relModPath) - - summary := strings.TrimSpace(cache.Modules[dirPath]) - if summary == "" && tools.FileExistsSafely(repoRoot, fullModPath) { - if data, err := tools.ReadFileSafely(repoRoot, fullModPath); err == nil { - summary = strings.TrimSpace(string(data)) - } - } - if summary == "" { - continue + if line := formatModuleIndexLine(repoRoot, docsDir, dirPath, nodesByPath[dirPath], cache); line != "" { + b.WriteString(line + "\n") } + } - label := dirPath - if dirPath == "." { - label = "Root (.)" - } + indexPath := filepath.Join(docsDir, "index.md") + return tools.WriteFileSafely(repoRoot, indexPath, []byte(b.String())) +} - desc := extractSummaryDescription(summary) - filesInfo := formatNodeFiles(nodesByPath[dirPath]) +func writeIndexOverview(b *strings.Builder, repoRoot, docsDir string) { + archExists := tools.FileExistsSafely(repoRoot, filepath.Join(docsDir, "architecture.md")) + qsExists := tools.FileExistsSafely(repoRoot, filepath.Join(docsDir, "quickstart.md")) + if !archExists && !qsExists { + return + } + b.WriteString("## Overview\n\n") + if archExists { + b.WriteString("- [Architecture Overview](architecture.md)\n") + } + if qsExists { + b.WriteString("- [Developer Quickstart](quickstart.md)\n") + } + b.WriteString("\n") +} - line := fmt.Sprintf("- [%s](%s)", label, relModPath) - if desc != "" { - line += " — " + desc - } - if filesInfo != "" { - line += " (" + filesInfo + ")" - } - b.WriteString(line + "\n") +func formatModuleIndexLine(repoRoot, docsDir, dirPath string, node *DirNode, cache *MetadataCache) string { + relModPath := filepath.ToSlash(filepath.Join("modules", toSafeMarkdownFilename(dirPath))) + summary := resolveModuleSummary(repoRoot, filepath.Join(docsDir, relModPath), cache.Modules[dirPath]) + if summary == "" { + return "" } - indexPath := filepath.Join(docsDir, "index.md") - return tools.WriteFileSafely(repoRoot, indexPath, []byte(b.String())) + label := dirPath + if dirPath == "." { + label = "Root (.)" + } + + line := fmt.Sprintf("- [%s](%s)", label, relModPath) + if desc := extractSummaryDescription(summary); desc != "" { + line += " — " + desc + } + if filesInfo := formatNodeFiles(node); filesInfo != "" { + line += " (" + filesInfo + ")" + } + return line +} + +func resolveModuleSummary(repoRoot, fullModPath, cachedSummary string) string { + if summary := strings.TrimSpace(cachedSummary); summary != "" { + return summary + } + if !tools.FileExistsSafely(repoRoot, fullModPath) { + return "" + } + data, err := tools.ReadFileSafely(repoRoot, fullModPath) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) } func extractSummaryDescription(summary string) string { lines := strings.Split(summary, "\n") - inFence := false - var fenceChar byte - var fenceCount int + var fs fenceState for _, rawLine := range lines { - if !inFence { - if ch, count, _, ok := tools.ParseFenceOpen(rawLine); ok { - inFence = true - fenceChar = ch - fenceCount = count - continue - } - } else { - if tools.IsFenceClose(rawLine, fenceChar, fenceCount) { - inFence = false - } + if fs.consume(rawLine) { continue } trimmed := strings.TrimSpace(rawLine) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, "#") || - strings.HasPrefix(trimmed, ">") || - strings.HasPrefix(trimmed, "```") || - strings.HasPrefix(trimmed, "~~~") || - strings.HasPrefix(trimmed, "|") || - strings.HasPrefix(trimmed, "-") || - strings.HasPrefix(trimmed, "*") || - strings.HasPrefix(trimmed, "", brokenLinkCommentPrefix, hrefTarget)) - continue - } - - var targetRelPath string - if linkPath == "" { - targetRelPath = relPath - } else { - joinedRel := filepath.Join(filepath.FromSlash(sourceDir), filepath.FromSlash(linkPath)) - resolvedAbs, resolveErr := security.SafeResolve(repoRoot, joinedRel) - exists := false - if resolveErr == nil { - if info, statErr := os.Stat(resolvedAbs); statErr == nil && !info.IsDir() { - exists = true - if relToRoot, relErr := filepath.Rel(rootAbs, resolvedAbs); relErr == nil { - targetRelPath = filepath.ToSlash(relToRoot) - } - } - } - - if !exists { - if repairedLink, repairedRel, ok := tryAutoRepairLink(repoRoot, docsDir, sourceDir, linkPath); ok { - newHref := repairedLink - if hasHash { - newHref = repairedLink + "#" + anchor - } - updatedLine = updatedLine[:m.hrefStart] + newHref + updatedLine[m.hrefEnd:] - hrefTarget = newHref - linkPath = repairedLink - targetRelPath = repairedRel - exists = true - } - } - - if !exists { - lineStamps = append(lineStamps, fmt.Sprintf("%s [%s] file %q does not exist -->", brokenLinkCommentPrefix, hrefTarget, linkPath)) - continue - } - } - - if anchor != "" && strings.EqualFold(path.Ext(targetRelPath), ".md") && !reLineAnchor.MatchString(anchor) { - targetAnchors, known := anchorsByFile[targetRelPath] - if !known { - if data, readErr := tools.ReadFileSafely(repoRoot, targetRelPath); readErr == nil { - targetAnchors = extractHeadingAnchors(string(data)) - anchorsByFile[targetRelPath] = targetAnchors - } - } - if !targetAnchors[strings.ToLower(anchor)] { - displayPath := linkPath - if displayPath == "" { - displayPath = path.Base(relPath) - } - lineStamps = append(lineStamps, fmt.Sprintf("%s [%s] heading anchor %q does not exist in %q -->", brokenLinkCommentPrefix, hrefTarget, "#"+anchor, displayPath)) - } - } - } +func processFileLinks(repoRoot, docsDir, rootAbs, relPath, cleaned string, anchorsByFile map[string]map[string]bool) (string, int) { + rawLines := strings.Split(cleaned, "\n") + sourceDir := path.Dir(relPath) - for i := len(lineStamps) - 1; i >= 0; i-- { - outLines = append(outLines, lineStamps[i]) - totalBroken++ - } - outLines = append(outLines, updatedLine) + var fs fenceState + var outLines []string + totalBroken := 0 + + for _, rawLine := range rawLines { + if fs.consume(rawLine) { + outLines = append(outLines, rawLine) + continue } - finalContent := strings.Join(outLines, "\n") - if finalContent != rawContents[relPath] { - if err := tools.WriteFileSafely(repoRoot, relPath, []byte(finalContent)); err != nil { - return totalBroken, fmt.Errorf("failed to write %s: %w", relPath, err) + updatedLine, lineStamps := processLineLinks(repoRoot, docsDir, rootAbs, relPath, sourceDir, rawLine, anchorsByFile) + for i := len(lineStamps) - 1; i >= 0; i-- { + outLines = append(outLines, lineStamps[i]) + totalBroken++ + } + outLines = append(outLines, updatedLine) + } + + return strings.Join(outLines, "\n"), totalBroken +} + +func processLineLinks(repoRoot, docsDir, rootAbs, relPath, sourceDir, rawLine string, anchorsByFile map[string]map[string]bool) (string, []string) { + matches := findMarkdownLinks(rawLine) + if len(matches) == 0 { + return rawLine, nil + } + + updatedLine := rawLine + var lineStamps []string + + for i := len(matches) - 1; i >= 0; i-- { + m := matches[i] + rawHref := strings.TrimSpace(rawLine[m.hrefStart:m.hrefEnd]) + newHref, replaced, stamp := validateSingleLink(repoRoot, docsDir, rootAbs, relPath, sourceDir, rawHref, anchorsByFile) + if replaced { + updatedLine = updatedLine[:m.hrefStart] + newHref + updatedLine[m.hrefEnd:] + } + if stamp != "" { + lineStamps = append(lineStamps, stamp) + } + } + + return updatedLine, lineStamps +} + +func validateSingleLink(repoRoot, docsDir, rootAbs, relPath, sourceDir, rawHref string, anchorsByFile map[string]map[string]bool) (string, bool, string) { + hrefTarget := extractLinkTarget(rawHref) + if hrefTarget == "" || isExternalLink(hrefTarget) { + return "", false, "" + } + + linkPath, anchor, hasHash := strings.Cut(hrefTarget, "#") + if strings.HasPrefix(linkPath, "/") { + return "", false, fmt.Sprintf("%s [%s] root-absolute link is not portable; use a relative path -->", brokenLinkCommentPrefix, hrefTarget) + } + + targetRelPath, repairedLink, repaired, ok := resolveLinkPath(repoRoot, docsDir, rootAbs, relPath, sourceDir, linkPath) + if !ok { + return "", false, fmt.Sprintf("%s [%s] file %q does not exist -->", brokenLinkCommentPrefix, hrefTarget, linkPath) + } + + newHref := "" + if repaired { + linkPath = repairedLink + newHref = repairedLink + if hasHash { + newHref = repairedLink + "#" + anchor + } + hrefTarget = newHref + } + + stamp := validateAnchor(repoRoot, relPath, targetRelPath, linkPath, hrefTarget, anchor, anchorsByFile) + return newHref, repaired, stamp +} + +func resolveLinkPath(repoRoot, docsDir, rootAbs, relPath, sourceDir, linkPath string) (string, string, bool, bool) { + if linkPath == "" { + return relPath, "", false, true + } + + joinedRel := filepath.Join(filepath.FromSlash(sourceDir), filepath.FromSlash(linkPath)) + if resolvedAbs, err := security.SafeResolve(repoRoot, joinedRel); err == nil { + if info, statErr := os.Stat(resolvedAbs); statErr == nil && !info.IsDir() { + var targetRelPath string + if relToRoot, relErr := filepath.Rel(rootAbs, resolvedAbs); relErr == nil { + targetRelPath = filepath.ToSlash(relToRoot) } + return targetRelPath, "", false, true } } - return totalBroken, nil + if repairedLink, repairedRel, ok := tryAutoRepairLink(repoRoot, docsDir, sourceDir, linkPath); ok { + return repairedRel, repairedLink, true, true + } + + return "", "", false, false +} + +func validateAnchor(repoRoot, relPath, targetRelPath, linkPath, hrefTarget, anchor string, anchorsByFile map[string]map[string]bool) string { + if anchor == "" || !strings.EqualFold(path.Ext(targetRelPath), ".md") || reLineAnchor.MatchString(anchor) { + return "" + } + + targetAnchors := getFileAnchors(repoRoot, targetRelPath, anchorsByFile) + if targetAnchors[strings.ToLower(anchor)] { + return "" + } + + displayPath := linkPath + if displayPath == "" { + displayPath = path.Base(relPath) + } + return fmt.Sprintf("%s [%s] heading anchor %q does not exist in %q -->", brokenLinkCommentPrefix, hrefTarget, "#"+anchor, displayPath) +} + +func getFileAnchors(repoRoot, targetRelPath string, anchorsByFile map[string]map[string]bool) map[string]bool { + if targetAnchors, known := anchorsByFile[targetRelPath]; known { + return targetAnchors + } + data, err := tools.ReadFileSafely(repoRoot, targetRelPath) + if err != nil { + return nil + } + targetAnchors := extractHeadingAnchors(string(data)) + anchorsByFile[targetRelPath] = targetAnchors + return targetAnchors } func stripBrokenLinkStamps(markdown string) string { @@ -809,22 +884,10 @@ func extractHeadingAnchors(markdown string) map[string]bool { anchors := make(map[string]bool) seenCounts := make(map[string]int) lines := strings.Split(markdown, "\n") - inFence := false - var fenceChar byte - var fenceCount int + var fs fenceState for _, line := range lines { - if !inFence { - if ch, count, _, ok := tools.ParseFenceOpen(line); ok { - inFence = true - fenceChar = ch - fenceCount = count - continue - } - } else { - if tools.IsFenceClose(line, fenceChar, fenceCount) { - inFence = false - } + if fs.consume(line) { continue } @@ -849,68 +912,76 @@ func findMarkdownLinks(line string) []linkMatch { n := len(line) for i := 0; i < n; i++ { if line[i] == '`' { - tickStart := i - for i < n && line[i] == '`' { - i++ - } - tickLen := i - tickStart - j := i - closed := false - for j < n { - if line[j] != '`' { - j++ - continue - } - runStart := j - for j < n && line[j] == '`' { - j++ - } - if j-runStart == tickLen { - i = j - 1 - closed = true - break - } - } - if !closed { - i = tickStart + tickLen - 1 - } + i = skipInlineCodeSpan(line, i) continue } - if line[i] != '[' { + m, ok := matchLinkAt(line, i) + if !ok { continue } - if i > 0 && line[i-1] == '!' { + matches = append(matches, m) + i = m.hrefEnd + } + return matches +} + +func skipInlineCodeSpan(line string, tickStart int) int { + n := len(line) + i := tickStart + for i < n && line[i] == '`' { + i++ + } + tickLen := i - tickStart + + for j := i; j < n; { + if line[j] != '`' { + j++ continue } - - closeBracket := -1 - for j := i + 1; j < n; j++ { - if line[j] == '[' { - break - } - if line[j] == ']' { - closeBracket = j - break - } + runStart := j + for j < n && line[j] == '`' { + j++ } - if closeBracket == -1 || closeBracket+1 >= n || line[closeBracket+1] != '(' { - continue + if j-runStart == tickLen { + return j - 1 } + } + return tickStart + tickLen - 1 +} - openParen := closeBracket + 1 - closeParen := findMatchingDelimiter(line, openParen, '(', ')') - if closeParen == -1 { - continue - } +func matchLinkAt(line string, start int) (linkMatch, bool) { + if line[start] != '[' || (start > 0 && line[start-1] == '!') { + return linkMatch{}, false + } - matches = append(matches, linkMatch{ - hrefStart: openParen + 1, - hrefEnd: closeParen, - }) - i = closeParen + closeBracket := findLinkTextEnd(line, start+1) + if closeBracket == -1 || closeBracket+1 >= len(line) || line[closeBracket+1] != '(' { + return linkMatch{}, false } - return matches + + openParen := closeBracket + 1 + closeParen := findMatchingDelimiter(line, openParen, '(', ')') + if closeParen == -1 { + return linkMatch{}, false + } + + return linkMatch{ + hrefStart: openParen + 1, + hrefEnd: closeParen, + }, true +} + +func findLinkTextEnd(line string, start int) int { + for j := start; j < len(line); j++ { + if line[j] == '[' { + return -1 + } + if line[j] == ']' { + return j + } + } + return -1 } func extractLinkTarget(rawHref string) string { diff --git a/internal/engine/orchestrator.go b/internal/engine/orchestrator.go index f547354..2e9e147 100644 --- a/internal/engine/orchestrator.go +++ b/internal/engine/orchestrator.go @@ -36,7 +36,9 @@ type StatusFunc func(string) // Run executes the documentation pipeline for the specified mode. func Run(ctx context.Context, repoRoot string, cfg *config.Config, mode RunMode, onStatus StatusFunc) error { if onStatus == nil { - onStatus = func(string) {} + onStatus = func(string) { + // No-op when caller does not provide a status callback. + } } if err := tools.EnsureGitignoreHasLockfile(repoRoot); err != nil { @@ -135,21 +137,22 @@ func buildProjectContextItems(rootSum string, tree *DirNode, cache *MetadataCach return items } -func (o *orchestrator) GenerateStandardDocs(ctx context.Context, repoRoot, docsDir, rootSum string, tree *DirNode, cache *MetadataCache, cfg *config.Config, onStatus StatusFunc) error { - contextItems := buildProjectContextItems(rootSum, tree, cache) - langInstr := config.LanguageInstruction(cfg.Language) +func (o *orchestrator) GenerateStandardDocs(ctx context.Context, pState *pipelineState, rootSum string, tree *DirNode) error { + contextItems := buildProjectContextItems(rootSum, tree, pState.cache) + langInstr := config.LanguageInstruction(pState.cfg.Language) + docsDir := pState.cfg.DocsDir - onStatus("Step 3: Global Architecture Synthesis...") + pState.onStatus("Step 3: Global Architecture Synthesis...") archPath := filepath.Join(docsDir, "architecture.md") archHeader := fmt.Sprintf("Write the global architecture overview (%s) based on the project context.", archPath) - if err := o.generateDoc(ctx, repoRoot, archPath, cfg.SystemPrompt+"\n"+cfg.ArchitecturePrompt+langInstr, archHeader, contextItems, onStatus); err != nil { + if err := o.generateDoc(ctx, pState.repoRoot, archPath, pState.cfg.SystemPrompt+"\n"+pState.cfg.ArchitecturePrompt+langInstr, archHeader, contextItems, pState.onStatus); err != nil { return err } - onStatus("Step 4: Generating Quickstart...") + pState.onStatus("Step 4: Generating Quickstart...") qsPath := filepath.Join(docsDir, "quickstart.md") qsHeader := fmt.Sprintf("Write the %s page based on the project context.", qsPath) - return o.generateDoc(ctx, repoRoot, qsPath, cfg.SystemPrompt+"\n"+cfg.QuickstartPrompt+langInstr, qsHeader, contextItems, onStatus) + return o.generateDoc(ctx, pState.repoRoot, qsPath, pState.cfg.SystemPrompt+"\n"+pState.cfg.QuickstartPrompt+langInstr, qsHeader, contextItems, pState.onStatus) } func setupPipeline(repoRoot string, cfg *config.Config) (string, []string, *MetadataCache, error) { @@ -282,7 +285,7 @@ func (o *orchestrator) RunInit(ctx context.Context, repoRoot string, cfg *config return err } - if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, tree, cache, cfg, onStatus); err != nil { + if err := o.GenerateStandardDocs(ctx, pState, rootSum, tree); err != nil { return err } @@ -321,6 +324,10 @@ func pruneStaleCache(repoRoot string, docsDir string, cache *MetadataCache, tree activeDocs[filepath.Join(modulesRoot, toSafeMarkdownFilename(mPath))] = true } + pruneStaleModuleDocs(modulesRoot, activeDocs) +} + +func pruneStaleModuleDocs(modulesRoot string, activeDocs map[string]bool) { var subdirs []string _ = filepath.WalkDir(modulesRoot, func(p string, d os.DirEntry, walkErr error) error { if walkErr != nil { @@ -375,6 +382,26 @@ func detectFileChanges(repoRoot string, ignores []string, cache *MetadataCache, return filteredChanges, currentHashes, validFiles, nil } +func hasModifiedFileFactsChanged(ctx context.Context, pState *pipelineState, nodesByPath map[string]*DirNode, filePath string) (bool, error) { + prevEntry, ok := pState.cache.Files[filePath] + if !ok { + return true, nil + } + var siblingFiles []string + if dirNode := nodesByPath[path.Dir(filePath)]; dirNode != nil { + siblingFiles = dirNode.Files + } + newFacts, err := extractFileFacts(ctx, pState, filePath, siblingFiles) + if err != nil { + return false, err + } + if newFacts == prevEntry.Facts { + pState.onStatus(fmt.Sprintf("➜ Unchanged facts for modified file, skipping module re-synthesis: %s", filePath)) + return false, nil + } + return true, nil +} + func resolveEffectiveChanges(ctx context.Context, pState *pipelineState, tree *DirNode, changes []FileChange) ([]FileChange, bool, error) { nodesByPath := collectTreeNodes(tree) var effective []FileChange @@ -385,19 +412,12 @@ func resolveEffectiveChanges(ctx context.Context, pState *pipelineState, tree *D structureChanged = true } if change.Status == StatusModified { - if prevEntry, ok := pState.cache.Files[change.Path]; ok { - var siblingFiles []string - if dirNode := nodesByPath[path.Dir(change.Path)]; dirNode != nil { - siblingFiles = dirNode.Files - } - newFacts, err := extractFileFacts(ctx, pState, change.Path, siblingFiles) - if err != nil { - return nil, false, err - } - if newFacts == prevEntry.Facts { - pState.onStatus(fmt.Sprintf("➜ Unchanged facts for modified file, skipping module re-synthesis: %s", change.Path)) - continue - } + changed, err := hasModifiedFileFactsChanged(ctx, pState, nodesByPath, change.Path) + if err != nil { + return nil, false, err + } + if !changed { + continue } } effective = append(effective, change) @@ -481,7 +501,7 @@ func (o *orchestrator) RunUpdate(ctx context.Context, repoRoot string, cfg *conf } if missingGlobalDocs || structureChanged || rootSum != prevRootSum || topModulesChanged { - if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, tree, cache, cfg, onStatus); err != nil { + if err := o.GenerateStandardDocs(ctx, pState, rootSum, tree); err != nil { return err } } else { diff --git a/internal/search/search.go b/internal/search/search.go index 3d4dfa3..cb45906 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -123,13 +123,8 @@ func ReadWikiSection(repoRoot, docsDir, target string) (string, error) { } cleanRel := filepath.ToSlash(relToDocs) - for _, seg := range strings.Split(cleanRel, "/") { - if strings.HasPrefix(seg, ".") { - return "", fmt.Errorf("invalid wiki target %q: hidden files and directories are not readable", target) - } - } - if !strings.EqualFold(path.Ext(cleanRel), ".md") { - return "", fmt.Errorf("invalid wiki target %q: only markdown (.md) files are readable", target) + if err := validateWikiMarkdownPath(cleanRel, target); err != nil { + return "", err } data, err := tools.ReadFileSafely(repoRoot, relToRoot) @@ -146,6 +141,18 @@ func ReadWikiSection(repoRoot, docsDir, target string) (string, error) { return extractSectionByAnchor(cleanRel, content, anchor) } +func validateWikiMarkdownPath(cleanRel, rawTarget string) error { + for _, seg := range strings.Split(cleanRel, "/") { + if strings.HasPrefix(seg, ".") { + return fmt.Errorf("invalid wiki target %q: hidden files and directories are not readable", rawTarget) + } + } + if !strings.EqualFold(path.Ext(cleanRel), ".md") { + return fmt.Errorf("invalid wiki target %q: only markdown (.md) files are readable", rawTarget) + } + return nil +} + func discoverWikiMarkdownFiles(docsAbs string) ([]string, error) { var files []string err := filepath.WalkDir(docsAbs, func(p string, d os.DirEntry, walkErr error) error { @@ -235,22 +242,27 @@ func extractBacktickValues(s string) []string { return vals } +func stepFenceState(line string, inFence bool, fenceChar byte, fenceCount int) (bool, byte, int, bool) { + if !inFence { + if ch, count, _, ok := tools.ParseFenceOpen(line); ok { + return true, ch, count, true + } + return false, 0, 0, false + } + if tools.IsFenceClose(line, fenceChar, fenceCount) { + return false, 0, 0, true + } + return true, fenceChar, fenceCount, true +} + func detectSplitHeadingLevel(lines []string) int { - inFence := false + var inFence bool var fenceChar byte var fenceCount int for _, line := range lines { - if !inFence { - if ch, count, _, ok := tools.ParseFenceOpen(line); ok { - inFence = true - fenceChar = ch - fenceCount = count - continue - } - } else { - if tools.IsFenceClose(line, fenceChar, fenceCount) { - inFence = false - } + var isFenced bool + inFence, fenceChar, fenceCount, isFenced = stepFenceState(line, inFence, fenceChar, fenceCount) + if isFenced { continue } if level, _, ok := tools.ParseATXHeading(line); ok && level == 2 { @@ -260,13 +272,29 @@ func detectSplitHeadingLevel(lines []string) int { return 3 } +func isLeadingDocumentH1(level int, hasH1 bool, curAnchor string, sectionCount int, curBody []string) bool { + return level == 1 && !hasH1 && curAnchor == "" && sectionCount == 0 && strings.TrimSpace(strings.Join(curBody, "\n")) == "" +} + +func appendSection(sections []wikiSection, pageRel, anchor, heading string, bodyLines []string, modPath string, sources []string) []wikiSection { + body := strings.TrimSpace(strings.Join(bodyLines, "\n")) + if anchor == "" && body == "" { + return sections + } + target := pageRel + if anchor != "" { + target = pageRel + "#" + anchor + } + return append(sections, buildWikiSection(pageRel, anchor, target, heading, body, modPath, sources)) +} + func parseMarkdownSections(pageRel, markdown string) []wikiSection { modPath, sources := parseProvenance(markdown, pageRel) lines := strings.Split(markdown, "\n") splitLevel := detectSplitHeadingLevel(lines) seenCounts := make(map[string]int) - inFence := false + var inFence bool var fenceChar byte var fenceCount int @@ -276,52 +304,31 @@ func parseMarkdownSections(pageRel, markdown string) []wikiSection { var curBody []string var rawSections []wikiSection - flushSection := func() { - body := strings.TrimSpace(strings.Join(curBody, "\n")) - if curAnchor == "" && body == "" { - return - } - target := pageRel - if curAnchor != "" { - target = pageRel + "#" + curAnchor - } - rawSections = append(rawSections, buildWikiSection(pageRel, curAnchor, target, curHeading, body, modPath, sources)) - } - for _, line := range lines { - if !inFence { - if ch, count, _, ok := tools.ParseFenceOpen(line); ok { - inFence = true - fenceChar = ch - fenceCount = count - curBody = append(curBody, line) - continue - } - } else { - if tools.IsFenceClose(line, fenceChar, fenceCount) { - inFence = false - } + var isFenced bool + inFence, fenceChar, fenceCount, isFenced = stepFenceState(line, inFence, fenceChar, fenceCount) + if isFenced { curBody = append(curBody, line) continue } + if strings.HasPrefix(strings.TrimSpace(line), provenancePrefix) { + continue + } level, headingText, ok := tools.ParseATXHeading(line) if !ok { - if strings.HasPrefix(strings.TrimSpace(line), provenancePrefix) { - continue - } curBody = append(curBody, line) continue } anchor := tools.NextHeadingAnchor(headingText, seenCounts) - if level == 1 && !hasH1 && curAnchor == "" && len(rawSections) == 0 && strings.TrimSpace(strings.Join(curBody, "\n")) == "" { + if isLeadingDocumentH1(level, hasH1, curAnchor, len(rawSections), curBody) { hasH1 = true curHeading = headingText continue } if level == 1 || level == splitLevel { - flushSection() + rawSections = appendSection(rawSections, pageRel, curAnchor, curHeading, curBody, modPath, sources) curHeading = headingText curAnchor = anchor curBody = nil @@ -330,7 +337,7 @@ func parseMarkdownSections(pageRel, markdown string) []wikiSection { curBody = append(curBody, line) } - flushSection() + rawSections = appendSection(rawSections, pageRel, curAnchor, curHeading, curBody, modPath, sources) if len(rawSections) == 0 { rawSections = append(rawSections, buildWikiSection(pageRel, "", pageRel, curHeading, "", modPath, sources)) @@ -366,13 +373,12 @@ func buildWikiSection(page, anchor, target, heading, body, modPath string, sourc } } -func scoreAndRankSections(sections []wikiSection, queryTokens, pathHints []string, limit int) []SearchResult { - nDocs := float64(len(sections)) +func computeCorpusStats(sections []wikiSection, queryTokens []string) (map[string]int, float64) { totalLen := 0.0 for _, s := range sections { totalLen += s.docLen } - avgDL := totalLen / nDocs + avgDL := totalLen / float64(len(sections)) if avgDL <= 0 { avgDL = 1.0 } @@ -385,36 +391,48 @@ func scoreAndRankSections(sections []wikiSection, queryTokens, pathHints []strin } } } + return docFreq, avgDL +} - var results []SearchResult - for _, s := range sections { - score := 0.0 - matchedTerms := 0 - - for _, qt := range queryTokens { - tf := titleWeight*float64(s.titleTerms[qt]) + - sourceWeight*float64(s.sourceTerms[qt]) + - bodyWeight*float64(s.bodyTerms[qt]) - if tf <= 0 { - continue - } - matchedTerms++ - df := float64(docFreq[qt]) - idf := math.Log(1.0 + (nDocs-df+0.5)/(df+0.5)) - norm := tf * (bm25K1 + 1.0) / (tf + bm25K1*(1.0-bm25B+bm25B*(s.docLen/avgDL))) - score += idf * norm - } +func scoreSection(s wikiSection, queryTokens []string, docFreq map[string]int, nDocs, avgDL float64, pathHints []string) float64 { + score := 0.0 + matchedTerms := 0 - if score <= 0 { + for _, qt := range queryTokens { + tf := titleWeight*float64(s.titleTerms[qt]) + + sourceWeight*float64(s.sourceTerms[qt]) + + bodyWeight*float64(s.bodyTerms[qt]) + if tf <= 0 { continue } + matchedTerms++ + df := float64(docFreq[qt]) + idf := math.Log(1.0 + (nDocs-df+0.5)/(df+0.5)) + norm := tf * (bm25K1 + 1.0) / (tf + bm25K1*(1.0-bm25B+bm25B*(s.docLen/avgDL))) + score += idf * norm + } - if matchedTerms > 1 { - score *= 1.0 + 0.25*float64(matchedTerms-1) - } + if score <= 0 { + return 0 + } + if matchedTerms > 1 { + score *= 1.0 + 0.25*float64(matchedTerms-1) + } + if len(pathHints) > 0 && matchesPathHints(s, pathHints) { + score *= pathHintBoost + } + return score +} + +func scoreAndRankSections(sections []wikiSection, queryTokens, pathHints []string, limit int) []SearchResult { + nDocs := float64(len(sections)) + docFreq, avgDL := computeCorpusStats(sections, queryTokens) - if len(pathHints) > 0 && matchesPathHints(s, pathHints) { - score *= pathHintBoost + var results []SearchResult + for _, s := range sections { + score := scoreSection(s, queryTokens, docFreq, nDocs, avgDL, pathHints) + if score <= 0 { + continue } results = append(results, SearchResult{ @@ -445,6 +463,16 @@ func cleanPathHint(p string) string { return strings.TrimPrefix(strings.ToLower(strings.Trim(filepath.ToSlash(strings.TrimSpace(p)), "/")), "./") } +func hintMatchesCandidate(hint, cand string) bool { + if hint == "" || cand == "" { + return false + } + if cand == "." { + return hint == "." + } + return strings.Contains(cand, hint) || strings.Contains(hint, cand) +} + func matchesPathHints(s wikiSection, pathHints []string) bool { candidates := []string{cleanPathHint(s.page), cleanPathHint(s.modPath)} for _, src := range s.sources { @@ -453,20 +481,8 @@ func matchesPathHints(s wikiSection, pathHints []string) bool { for _, rawHint := range pathHints { h := cleanPathHint(rawHint) - if h == "" { - continue - } for _, cand := range candidates { - if cand == "" { - continue - } - if cand == "." { - if h == "." { - return true - } - continue - } - if strings.Contains(cand, h) || strings.Contains(h, cand) { + if hintMatchesCandidate(h, cand) { return true } } @@ -474,46 +490,49 @@ func matchesPathHints(s wikiSection, pathHints []string) bool { return false } -func extractSnippet(body, fallbackHeading string, queryTokens []string) string { - lines := strings.Split(body, "\n") - inFence := false +func isProseLine(trimmed string) bool { + return trimmed != "" && + !strings.HasPrefix(trimmed, "#") && + !strings.HasPrefix(trimmed, provenancePrefix) && + !strings.HasPrefix(trimmed, "