diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c50ad7c..fa625cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,17 +42,19 @@ jobs: - run: cargo test --workspace --all-targets - run: cargo test --workspace --doc + docs-validation: + name: Shared docs validation + permissions: + contents: read + uses: Sigilweaver/ops/.github/workflows/docs.yml@757c6789b96e3ac05bbc7d1a092d01cbeb0cb9e2 + with: + tooling-ref: 757c6789b96e3ac05bbc7d1a092d01cbeb0cb9e2 docs: name: Docs (Docusaurus) + needs: docs-validation + if: ${{ always() }} runs-on: ubuntu-latest + env: + DOCS_RESULT: ${{ needs.docs-validation.result }} steps: - - uses: actions/checkout@v7 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - name: Install - working-directory: docs - run: bun install --frozen-lockfile - - name: Build - working-directory: docs - run: bun run build:cloudflare + - run: test "$DOCS_RESULT" = success diff --git a/docs/bun.lock b/docs/bun.lock index da722d0..2bd1018 100644 --- a/docs/bun.lock +++ b/docs/bun.lock @@ -10,7 +10,6 @@ "@docusaurus/theme-mermaid": "^3.9.2", "@mdx-js/react": "^3.0.0", "clsx": "^2.0.0", - "docusaurus-plugin-llms-txt": "^0.1.3", "prism-react-renderer": "^2.3.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -19,7 +18,7 @@ "@docusaurus/module-type-aliases": "^3.9.2", "@docusaurus/tsconfig": "^3.9.2", "@docusaurus/types": "^3.9.2", - "typescript": "^7.0.2", + "typescript": "~7.0.2", }, }, }, @@ -1168,8 +1167,6 @@ "dns-packet": ["dns-packet@5.6.1", "", { "dependencies": { "@leichtgewicht/ip-codec": "^2.0.1" } }, "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw=="], - "docusaurus-plugin-llms-txt": ["docusaurus-plugin-llms-txt@0.1.3", "", { "dependencies": { "fs-extra": "^11.2.0", "gray-matter": "^4.0.3" }, "peerDependencies": { "@docusaurus/core": ">=2.0.0", "react": ">=16.8.0" } }, "sha512-X/Dqf1/soHFjvELWXtfa9C023498HsWWwNPdVZEu0OoznI2Gzf+VCB/4obwpBSV9OeDeI8D94/dK/ETq534k9A=="], - "dom-converter": ["dom-converter@0.2.0", "", { "dependencies": { "utila": "~0.4" } }, "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA=="], "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index f717c40..8b34175 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -13,7 +13,7 @@ const config: Config = { onBrokenMarkdownLinks: 'warn', }, }, - plugins: ['docusaurus-plugin-llms-txt'], + themes: ['@docusaurus/theme-mermaid'], url: 'https://sigilweaver.app', diff --git a/docs/package.json b/docs/package.json index 4da36e7..1cda3f8 100644 --- a/docs/package.json +++ b/docs/package.json @@ -6,8 +6,8 @@ "docusaurus": "docusaurus", "start": "docusaurus start --port 25814", "dev": "docusaurus start --port 25814", - "build": "docusaurus build", - "build:cloudflare": "docusaurus build --out-dir build/opensqlanywhere/docs", + "build": "docusaurus build && python3 scripts/docs_output.py build --write-index", + "build:cloudflare": "docusaurus build --out-dir build/opensqlanywhere/docs && python3 scripts/docs_output.py build --write-index", "clear": "docusaurus clear", "serve": "docusaurus serve", "typecheck": "tsc" @@ -18,7 +18,6 @@ "@docusaurus/theme-mermaid": "^3.9.2", "@mdx-js/react": "^3.0.0", "clsx": "^2.0.0", - "docusaurus-plugin-llms-txt": "^0.1.3", "prism-react-renderer": "^2.3.0", "react": "^19.0.0", "react-dom": "^19.0.0" @@ -30,7 +29,11 @@ "typescript": "~7.0.2" }, "browserslist": { - "production": [">0.5%", "not dead", "not op_mini all"], + "production": [ + ">0.5%", + "not dead", + "not op_mini all" + ], "development": [ "last 3 chrome version", "last 3 firefox version", diff --git a/docs/scripts/docs_output.py b/docs/scripts/docs_output.py new file mode 100644 index 0000000..eabd63e --- /dev/null +++ b/docs/scripts/docs_output.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Generate and validate llms.txt against a Docusaurus site's built pages. + +Uses only Python's standard library. The homepage's canonical URL and title +identify the site; sitemap.xml supplies the public routes. No source document +names, sidebar assumptions, or external network requests are required. +""" + +from __future__ import annotations + +import argparse +from html.parser import HTMLParser +from pathlib import Path +import re +import sys +from urllib.parse import unquote, urljoin, urlsplit +import xml.etree.ElementTree as ET + + +class Page(HTMLParser): + def __init__(self, path: Path): + super().__init__(convert_charrefs=True) + self.title = "" + self.canonical = "" + self.description = "" + self.ids: set[str] = set() + self.links: list[str] = [] + self.in_title = False + self.feed(path.read_text(encoding="utf-8")) + + def handle_starttag(self, tag, attributes): + attrs = dict(attributes) + if attrs.get("id"): + self.ids.add(attrs["id"]) + if tag == "title": + self.in_title = True + elif tag == "link" and "canonical" in attrs.get("rel", "").split(): + self.canonical = attrs.get("href", "") + elif tag == "meta" and attrs.get("name") == "description": + self.description = attrs.get("content", "") + elif tag == "a" and attrs.get("href"): + self.links.append(attrs["href"]) + if attrs.get("name"): + self.ids.add(attrs["name"]) + + def handle_endtag(self, tag): + if tag == "title": + self.in_title = False + + def handle_data(self, data): + if self.in_title: + self.title += data + + +def page_path(site: Path, base: str, url: str) -> Path: + root = urlsplit(base) + target = urlsplit(url) + if (target.scheme, target.netloc) != (root.scheme, root.netloc): + raise ValueError(f"URL belongs to another origin: {url}") + base_path = unquote(root.path).rstrip("/") + path = unquote(target.path) + if path != base_path and not path.startswith(base_path + "/"): + raise ValueError(f"URL is outside the docs base: {url}") + relative = path[len(base_path):].lstrip("/") + if ".." in Path(relative).parts or "\\" in relative: + raise ValueError(f"Unsafe route: {url}") + candidates = [site / relative, site / relative / "index.html"] + if relative: + candidates.append(site / (relative.rstrip("/") + ".html")) + for candidate in candidates: + if candidate.is_file() and candidate.resolve().is_relative_to(site.resolve()): + return candidate + raise ValueError(f"Missing built page or asset: {url}") + + +def clean_text(text: str) -> str: + return " ".join(text.split()) + + +def markdown_label(text: str) -> str: + return clean_text(text).replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + + +def validate_site(site: Path, write_index: bool = False) -> list[str]: + errors: list[str] = [] + try: + home = Page(site / "index.html") + base = home.canonical.rstrip("/") + "/" + parsed_base = urlsplit(base) + if parsed_base.scheme not in {"https", "http"} or not parsed_base.netloc: + raise ValueError("Homepage needs an absolute HTTP(S) canonical URL") + title = clean_text(home.title.rsplit(" | ", 1)[-1]) + if not title or title.lower() in {"undefined", "null"}: + raise ValueError("Homepage needs a defined site title") + tree = ET.parse(site / "sitemap.xml") + urls = [clean_text(node.text or "") for node in tree.findall(".//{*}loc")] + if not urls or any(not url for url in urls): + raise ValueError("Sitemap must contain nonempty document URLs") + if len(urls) != len(set(urls)): + raise ValueError("Sitemap contains duplicate URLs") + except (OSError, ValueError, ET.ParseError) as exc: + return [f"{site}: {exc}"] + + pages: dict[Path, Page] = {} + entries: list[tuple[str, str]] = [] + for url in sorted(urls): + try: + path = page_path(site, base, url) + page = pages.setdefault(path, Page(path)) + if not page.title.strip(): + raise ValueError(f"Missing page title: {url}") + if page.canonical.rstrip("/") != url.rstrip("/"): + raise ValueError(f"Canonical URL disagrees with sitemap: {url}") + label = page.title.removesuffix(" | " + title) + entries.append((markdown_label(label), url)) + except (OSError, ValueError) as exc: + errors.append(str(exc)) + + # Validate rendered links, including same-page and cross-page fragments. + # Other project sites and external services remain outside this check. + for path, page in list(pages.items()): + for href in page.links: + resolved = urljoin(page.canonical, href) + target = urlsplit(resolved) + root = urlsplit(base) + if (target.scheme, target.netloc) != (root.scheme, root.netloc): + continue + base_path = unquote(root.path).rstrip("/") + target_path = unquote(target.path) + if target_path != base_path and not target_path.startswith(base_path + "/"): + continue + try: + destination = page_path(site, base, resolved) + if target.fragment and destination.suffix == ".html": + target_page = pages.get(destination) + if target_page is None: + target_page = Page(destination) + pages[destination] = target_page + if unquote(target.fragment) not in target_page.ids: + raise ValueError(f"Missing fragment: {resolved}") + except (OSError, ValueError) as exc: + errors.append(f"{path.relative_to(site)} -> {href}: {exc}") + + index = site / "llms.txt" + if write_index and not errors: + description = clean_text(home.description) + header = f"# {title}\n\n" + (f"> {description}\n\n" if description else "") + content = header + "## Documentation\n\n" + content += "".join(f"- [{label}]({url})\n" for label, url in entries) + index.write_text(content, encoding="utf-8") + if not errors: + try: + text = index.read_text(encoding="utf-8") + if text.splitlines()[0] != f"# {title}": + raise ValueError("llms.txt title does not match the built site") + links = re.findall(r"^- \[(?:\\.|[^\]])+\]\(([^)]*)\)$", text, re.MULTILINE) + if len(links) != len(urls) or set(links) != set(urls): + raise ValueError("llms.txt must link exactly once to every sitemap page") + except (OSError, ValueError, IndexError) as exc: + errors.append(f"{index}: {exc}") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("build_dir", type=Path) + parser.add_argument("--write-index", action="store_true") + args = parser.parse_args() + sitemaps = sorted(args.build_dir.rglob("sitemap.xml")) + if not sitemaps: + print(f"No sitemap.xml found under {args.build_dir}", file=sys.stderr) + return 1 + errors = [] + for sitemap in sitemaps: + problems = validate_site(sitemap.parent, args.write_index) + errors.extend(problems) + if not problems: + print(f"Validated docs pages, links, fragments, and llms.txt: {sitemap.parent}") + for error in errors: + print(error, file=sys.stderr) + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main())