From 00ca60e7f137f2ae3842fae0886769fab8ac6888 Mon Sep 17 00:00:00 2001 From: Max Lapin <4691364+mlapin@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:37:56 +0100 Subject: [PATCH] Docs check: every link in every tracked Markdown file, with an opt-in external pass - tools/check_docs.py checks every tracked (or new) Markdown file outside vendor/: inline links, images, reference definitions, HTML href/src and bare URLs, with inline code and comments masked. - Targets must match a tracked file with the exact case; headings follow GitHub's slugs (code spans and links in headings) and Pandoc {#id} attributes. - Links to this repository on GitHub at master or the current version tag resolve in the checkout; blob/main, relative links in the guides rendered on PyPI and crates.io and undefined reference labels fail. - --external requests every external URL (page anchors included); refusals by hosts and a version tag not yet on GitHub are warnings. - CONTRIBUTING.md and tools/README.md describe the check; the release steps run --external. --- CONTRIBUTING.md | 8 +- tests/python/test_build_config.py | 85 ++++++- tools/README.md | 2 +- tools/check_docs.py | 372 +++++++++++++++++++++++++----- 4 files changed, 410 insertions(+), 57 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56d7a34..77f78d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,9 +107,10 @@ Keep API guides in `docs/`. after the usage examples. - **Benchmarks:** Benchmark tables must identify hardware, versions, timing boundaries and their evidence, including slower cases. Do not attribute one binding's timings to another. -- **Checks:** `python tools/check_docs.py` checks local links and heading anchors and - rejects unsupported math commands; `tools/quality.py` runs it in CI. Use `\mathrm{...}` - for named math functions. +- **Checks:** `python tools/check_docs.py` checks every link in the tracked Markdown: the + file (with its exact case), the heading anchor and links to this repository on GitHub. It + also rejects unsupported math commands; `tools/quality.py` runs it in CI. Use + `\mathrm{...}` for named math functions. Add `--external` to request every external URL. - **Absolute links:** The Python and Rust guides also render on PyPI and crates.io, so use absolute links there. - **Rust guide:** Cargo packages `docs/RUST.md` as `RUST.md`; the build script feeds the @@ -120,6 +121,7 @@ Keep API guides in `docs/`. ```bash python tools/bump_version.py 0.3.2 # VERSION and its copies; "## Unreleased" gets the date python tools/quality.py --skip-tidy # checks that every copy agrees +python tools/check_docs.py --external # every external link answers git tag v0.3.2 && git push origin v0.3.2 ``` diff --git a/tests/python/test_build_config.py b/tests/python/test_build_config.py index 11b4a32..6b349bf 100644 --- a/tests/python/test_build_config.py +++ b/tests/python/test_build_config.py @@ -133,7 +133,7 @@ def test_documentation_links_and_math(self, tmp_path: Path): guide.write_text(f"# JAX\n\n## Quick start\n\n[technical report]({report})\n") tool.check_docs(tmp_path) guide.write_text(f"# JAX\n\n## Quick start\n\n[Table 12]({report}#table-12)\n") - with pytest.raises(SystemExit, match=r"docs/JAX.md:5: link to the report without"): + with pytest.raises(SystemExit, match=r"docs/JAX.md:5: link the report as a whole"): tool.check_docs(tmp_path) guide.write_text("# JAX\n\n## Renamed\n") with pytest.raises(SystemExit, match="missing heading"): @@ -157,6 +157,89 @@ def test_documentation_code_links_and_duplicate_headings(self, tmp_path: Path): with pytest.raises(SystemExit, match="missing local target"): tool.check_docs(tmp_path) + def test_documentation_links_in_a_checkout(self, tmp_path: Path): + tool = load_module( + "docs_checkout_test", Path(__file__).resolve().parents[2] / "tools/check_docs.py" + ) + notes = "/".join(("private", "notes.md")) # the public export rejects the literal path + files = { + "VERSION": "1.2.0\n", + "docs/API.md": "# API\n\n### The `solved` certificate\n\n## Options {#opts}\n", + "docs/PYTHON.md": "# Python\n", + "include/sdca/loss.h": "\n", + notes: "[API](../docs/API.md#the-solved-certificate)\n", + } + for name, text in files.items(): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text(text) + git = ["git", "-C", str(tmp_path)] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "add", "."], check=True) + repository = "https://github.com/mlapin/libsdca/blob" + good = [ + "[A](docs/API.md#the-solved-certificate) [B](docs/API.md#opts) [C](include/sdca/)", + "[D](/include/sdca/loss.h) `[code](missing.md)` [E][e] [F]\n\n[e]: docs/API.md\n" + "[f]: ", + f"[G]({repository}/master/docs/API.md#api) [H]({repository}/v1.2.0/include/sdca)", + "[$x$][e] [I](https://example.org/a_(b)) ", + ] + bad = { + "[A](docs/api.md)": "missing local target docs/api.md", + "[A](docs/API.md#solved-certificate)": "missing heading", + "[A][missing]": r"undefined reference link \[A\]\[missing\]", + "![A](figures/a.png)": "missing local target figures/a.png", + 'A': "missing local target docs/NONE.md", + "[A](../outside.md)": "link leaves the repository", + f"[A]({notes})": "public file links outside the public tree", + f"[A]({repository}/main/docs/API.md)": "the public default branch is master", + f"[A]({repository}/master/docs/NONE.md)": "missing repository target", + f"[A]({repository}/v1.2.0/docs/API.md#none)": "missing heading", + "[A](ftp://example.org)": "unsupported link scheme", + } + readme = tmp_path / "README.md" + readme.write_text("\n".join(good) + "\n") + tool.check_docs(tmp_path) + for link, message in bad.items(): + readme.write_text(f"# Guide\n\n{link}\n") + with pytest.raises(SystemExit, match=f"README.md:3: {message}"): + tool.check_docs(tmp_path) + readme.write_text("# Guide\n") + (tmp_path / "docs/PYTHON.md").write_text("[API](API.md)\n") + with pytest.raises(SystemExit, match="relative link in a guide rendered on a package"): + tool.check_docs(tmp_path) + (tmp_path / "docs/PYTHON.md").write_text("[API](#python)\n# Python\n") + tool.check_docs(tmp_path) + + def test_documentation_external_links(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]): + tool = load_module( + "docs_external_test", Path(__file__).resolve().parents[2] / "tools/check_docs.py" + ) + (tmp_path / "README.md").write_text( + "[A](https://example.org/a) [B](https://example.org/b#part) " + "[C](https://example.org/c) [D](https://example.org/$release/d)\n" + ) + (tmp_path / "VERSION").write_text("9.0.0\n") + (tmp_path / "docs").mkdir() + (tmp_path / "docs/RUST.md").write_text( + "[R](https://github.com/mlapin/libsdca/blob/v9.0.0/docs/RUST.md)\n" + ) + answers = { + "https://example.org/a": (200, None), + "https://example.org/b#part": (200, "missing anchor"), + "https://example.org/c": (403, "HTTP 403"), + "https://github.com/mlapin/libsdca/tree/v9.0.0": (404, "HTTP 404"), + } + with patch.object(tool, "request", side_effect=answers.__getitem__) as request: + with pytest.raises(SystemExit) as failure: + tool.check_docs(tmp_path, external=True) + assert sorted(call.args[0] for call in request.call_args_list) == sorted(answers) + assert "README.md:1: missing anchor https://example.org/b#part" in str(failure.value) + assert "example.org/c" not in str(failure.value) # a refusal is only a warning + assert "v9.0.0" not in str(failure.value) # an untagged release is only a warning + warnings = capsys.readouterr().err + assert "the host refuses scripted requests (HTTP 403) https://example.org/c" in warnings + assert "v9.0.0 is not on GitHub yet; 1 links to it (docs/RUST.md:1, ...)" in warnings + def test_quality_versions_cover_every_copy(self, subtests: pytest.Subtests): root = Path(__file__).resolve().parents[2] tool = load_module("quality_tool_test", root / "tools/quality.py") diff --git a/tools/README.md b/tools/README.md index 9cea791..efba7ad 100644 --- a/tools/README.md +++ b/tools/README.md @@ -11,7 +11,7 @@ Scripts used in development and CI ([CONTRIBUTING.md](../CONTRIBUTING.md)): | `check_extension_builds.py` | Compiles the PyTorch CPU extension for AVX-512, AVX2 and generic targets | | `check_install_archives.py` | Checks that an sdist or wheel contains the shared CPU sources and native libraries | | `readme_smoke.py` | Installs a wheel into fresh uv or conda environments and runs the README examples verbatim | -| `check_docs.py` | Checks local Markdown links, heading anchors and GitHub math compatibility; included in `quality.py` | +| `check_docs.py` | Checks the links and heading anchors of every tracked Markdown file and GitHub math compatibility; included in `quality.py`; `--external` also requests every external URL | | `readme_*_smoke.py` | Installed-package probes used by `readme_smoke.py` (precision, CPU and CUDA dispatch, contracts) | | `perf_gate.py` | CUDA speed and memory against `torch.compile`, with per-loss speedup floors | | `apt_mirror.sh` | Adds retries and a fallback Ubuntu mirror before `apt-get` in CI | diff --git a/tools/check_docs.py b/tools/check_docs.py index f21d857..f4e67a8 100644 --- a/tools/check_docs.py +++ b/tools/check_docs.py @@ -1,77 +1,345 @@ -"""Check maintained Markdown for broken local links, unsupported GitHub math and links -into the technical report (the report is linked as a whole: no anchor, no table number). +"""Check the links of every tracked Markdown file, unsupported GitHub math and links into +the technical report (the report is linked as a whole: no anchor, no table number). -python tools/check_docs.py +python tools/check_docs.py # offline: files, headings and repository URLs +python tools/check_docs.py --external # also request every external URL + +Offline, a link must name a tracked file with the exact case (macOS forgives case, GitHub +does not) and, for Markdown, an existing heading. Links to this repository on GitHub are +resolved in the checkout when they name the default branch or the current version tag. +Guides that also render on PyPI and crates.io must use absolute links. """ from __future__ import annotations +import argparse import re -from pathlib import Path +import subprocess +import sys +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path, PurePosixPath from urllib.parse import unquote, urlsplit ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = re.compile( + r"https://github\.com/mlapin/libsdca/(?:blob|tree|raw)/([^/#?]+)/?([^#?]*)(?:\?[^#]*)?(?:#(.*))?$" +) +REPOSITORY_BRANCH = "master" +# Package pages render these guides away from the repository, so relative links break. +PACKAGE_PAGES = ("docs/PYTHON.md", "docs/RUST.md") +# Archived run records keep the links of the tree they were made from. +ARCHIVE = "/".join(("private", "experiments")) + "/" +# Metadata that carries URLs but not Markdown links. +METADATA = ("pyproject.toml", "CITATION.cff", "rust/Cargo.toml") +# Statuses of hosts that refuse scripted requests: a warning, not a failure. +UNVERIFIABLE = {401, 403, 429} -def markdown_paths(root: Path) -> list[Path]: - """Public guides and examples; exclude build products and vendored archives.""" - paths = list(root.glob("*.md")) - for directory in ("docs", "examples"): - paths.extend((root / directory).rglob("*.md")) - for directory in ("python", "rust", "matlab", "tools"): - paths.extend((root / directory).glob("*.md")) - return sorted(paths) +def tracked_files(root: Path) -> set[str] | None: + """Tracked and new (not ignored) paths relative to `root`, or None outside a Git + checkout (an sdist).""" + command = ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"] + try: + listing = subprocess.run(command, cwd=root, capture_output=True, check=True, text=True) + except (OSError, subprocess.CalledProcessError): + return None + paths = {path for path in listing.stdout.split("\0") if path and (root / path).exists()} + return paths or None -def prose(text: str) -> str: - """Mask fenced code while retaining line numbers for link diagnostics.""" - return re.sub( - r"^\s*(`{3,}|~{3,})[^\n]*\n.*?^\s*\1\s*$", - lambda match: "\n" * match[0].count("\n"), - text, - flags=re.M | re.S, +def markdown_paths(root: Path, tracked: set[str] | None) -> list[Path]: + """Every tracked Markdown file outside vendored code and archived experiment records + (snapshots of older trees and report sources with generated anchors).""" + if tracked is None: + found = list(root.glob("*.md")) + for directory in ("docs", "examples", "python", "rust", "matlab", "tools"): + found.extend((root / directory).rglob("*.md")) + return sorted(path for path in found if "build" not in path.relative_to(root).parts) + return sorted( + root / path + for path in tracked + if path.endswith(".md") and not path.startswith(("vendor/", ARCHIVE)) ) +def _blank(match: re.Match) -> str: + return re.sub(r"[^\n]", " ", match[0]) + + +def mask_blocks(text: str) -> str: + """Blank fenced code and HTML comments, keeping offsets and line numbers.""" + fence = r"^ {0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^ {0,3}\1[`~]*[ \t]*$|\Z)" + text = re.sub(fence, _blank, text, flags=re.M | re.S) + return re.sub(r"", _blank, text, flags=re.S) + + +def mask(text: str) -> str: + """Blank fenced code, HTML comments and inline code.""" + return re.sub(r"(`+)(?!`).*?(? str: + """GitHub's anchor for a heading: rendered text, lower case, punctuation dropped.""" + heading = re.sub(r"!?\[([^\]]*)\]\([^)]*\)", r"\1", heading) # links keep their text + heading = re.sub(r"<[^>]+>", "", heading) # inline HTML + return re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-") + + def anchors(text: str) -> set[str]: - """GitHub heading slugs (including duplicate suffixes) and explicit HTML anchors.""" - found = set(re.findall(r'<(?:a|h[1-6])\b[^>]*\b(?:id|name)="([^"]+)"', text)) - for heading in re.findall(r"^#{1,6}\s+(.+?)\s*#*\s*$", prose(text), re.M): - slug = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-") - candidate, suffix = slug, 0 - while candidate in found: - suffix += 1 - candidate = f"{slug}-{suffix}" - found.add(candidate) + """GitHub heading slugs (including duplicate suffixes), explicit HTML anchors and the + Pandoc `{#id}` attributes of documents rendered to HTML.""" + found = set(re.findall(r'<[a-z][a-z0-9]*\b[^>]*\b(?:id|name)="([^"]+)"', text)) + found.update(re.findall(r"\{[^}\n]*#([\w:.-]+)[^}\n]*\}", mask(text))) + counts: dict[str, int] = {} + headings = r"^ {0,3}#{1,6}[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$" + for heading in re.findall(headings, mask_blocks(text), re.M): + base = slug(re.sub(r"\s*\{[^}]*\}$", "", heading)) + count = counts.get(base, 0) + counts[base] = count + 1 + found.add(f"{base}-{count}" if count else base) return found -def check_docs(root: Path = ROOT) -> None: - failures: list[str] = [] - for path in markdown_paths(root): +# Inline links and images, allowing one level of balanced parentheses in the destination. +INLINE = re.compile( + r"\]\(\s*(<[^>\n]*>|[^\s()]*(?:\([^\s()]*\)[^\s()]*)*)(?:\s+(?:\"[^\"]*\"|'[^']*'))?\s*\)" +) +DEFINITION = re.compile(r"^ {0,3}\[([^\]^][^\]]*)\]:[ \t]*(<[^>\n]*>|\S+)", re.M) +REFERENCE = re.compile(r"(?]*?\b(?:href|src)\s*=\s*["']([^"']+)["']""", re.I) +BARE = re.compile(r"""https?://[^\s<>()\[\]"'`|\\]+""") + + +def links(text: str) -> list[tuple[int, str]]: + """(offset, destination) of every link, image, reference definition, HTML link and URL.""" + body = mask(text) + found = [(match.start(1), match[1].strip("<>")) for match in INLINE.finditer(body)] + found += [(match.start(2), match[2].strip("<>")) for match in DEFINITION.finditer(body)] + found += [(match.start(1), match[1]) for match in HTML.finditer(body)] + taken = {offset for offset, _ in found} | {offset + 1 for offset, _ in found} + for match in BARE.finditer(body): # autolinks and bare URLs outside link syntax + if match.start() not in taken: + found.append((match.start(), match[0].rstrip(".,;:!?*_~"))) + return sorted(found) + + +def undefined_references(text: str) -> list[tuple[int, str]]: + """`[text][label]` whose label has no definition (GitHub renders the brackets).""" + math = r"\$\$.*?\$\$|\$[^$\n]+\$" + body = re.sub(math, _blank, mask(text), flags=re.S) + labels = {label.strip().lower() for label, _ in DEFINITION.findall(body)} + return [ + (match.start(), match[0]) + for match in REFERENCE.finditer(body) + if (match[2] or match[1]).strip().lower() not in labels + ] + + +class Checker: + def __init__(self, root: Path) -> None: + self.root = root + self.tracked = tracked_files(root) + self.directories = { + "/".join(path.split("/")[:end]) + for path in self.tracked or () + for end in range(1, path.count("/") + 1) + } + version = root / "VERSION" + self.version = version.read_text().strip() if version.exists() else "" + self.failures: list[str] = [] + self.external: dict[str, list[str]] = {} + self.release: list[str] = [] # links to the tag of the current version + self._anchors: dict[Path, set[str]] = {} + + def fail(self, where: str, message: str) -> None: + self.failures.append(f"{where}: {message}") + + def exists(self, relative: PurePosixPath) -> bool: + """Exact-case membership in the checkout: a tracked file or a directory of them.""" + name = relative.as_posix() + if name == ".": + return True + if self.tracked is None: + return (self.root / name).exists() + return name in self.tracked or name in self.directories + + def heading(self, where: str, relative: PurePosixPath, fragment: str, link: str) -> None: + if not fragment or relative.suffix != ".md": + return + path = self.root / relative + if path not in self._anchors: + self._anchors[path] = anchors(path.read_text()) + if unquote(fragment) not in self._anchors[path]: + self.fail(where, f"missing heading {link}") + + def local(self, where: str, source: PurePosixPath, link: str) -> None: + parts = urlsplit(link) + if parts.path.startswith("/"): # GitHub resolves these from the repository root + target = PurePosixPath(unquote(parts.path).lstrip("/")) + elif parts.path: + target = source.parent / unquote(parts.path) + else: + target = source + resolved: list[str] = [] + for part in target.parts: + if part == "..": + if not resolved: + self.fail(where, f"link leaves the repository {link}") + return + resolved.pop() + elif part != ".": + resolved.append(part) + relative = PurePosixPath(*resolved) if resolved else PurePosixPath(".") + if not self.exists(relative): + self.fail(where, f"missing local target {link}") + elif relative.parts[:1] == ("private",) and source.parts[:1] != ("private",): + self.fail(where, f"public file links outside the public tree {link}") + else: + self.heading(where, relative, parts.fragment, link) + + def repository(self, where: str, link: str, match: re.Match) -> None: + ref, path, fragment = match[1], match[2].rstrip("/"), match[3] or "" + if ref != REPOSITORY_BRANCH and not ref.startswith("v"): + self.fail(where, f"the public default branch is {REPOSITORY_BRANCH}: {link}") + return + if ref not in (REPOSITORY_BRANCH, f"v{self.version}"): + self.external.setdefault(link, []).append(where) # an older release: ask GitHub + return + if ref != REPOSITORY_BRANCH: + self.release.append(where) + relative = PurePosixPath(unquote(path)) if path else PurePosixPath(".") + if relative.parts[:1] == ("private",) or not self.exists(relative): + self.fail(where, f"missing repository target {link}") + else: + self.heading(where, relative, fragment, link) + + def check_markdown(self, path: Path) -> None: + source = PurePosixPath(path.relative_to(self.root).as_posix()) text = path.read_text() - name = path.relative_to(root) - for number, line in enumerate(text.splitlines(), 1): - if re.search(r"\\operatorname\b", line): - failures.append(f"{name}:{number}: unsupported math command; use \\mathrm") - if re.search(r"/libsdca/report/?#", line): - failures.append(f"{name}:{number}: link to the report without an anchor") - body = prose(text) - for match in re.finditer(r"\[[^\]\n]*\]\(([^\s)]+)\)", body): - link = urlsplit(match[1].strip("<>")) - if link.scheme or link.netloc: + + def line(offset: int) -> int: + return text.count("\n", 0, offset) + 1 + + for number, row in enumerate(text.splitlines(), 1): + if re.search(r"\\operatorname\b", row): + self.fail(f"{source}:{number}", "unsupported math command; use \\mathrm") + if re.search(r"/libsdca/report/?#", row): + self.fail(f"{source}:{number}", "link the report as a whole, without an anchor") + for offset, label in undefined_references(text): + self.fail(f"{source}:{line(offset)}", f"undefined reference link {label}") + for offset, link in links(text): + where = f"{source}:{line(offset)}" + scheme = urlsplit(link).scheme.lower() + if scheme in ("mailto", "tel", "data"): continue - target = path.parent / unquote(link.path) if link.path else path - number = body.count("\n", 0, match.start()) + 1 - if not target.exists(): - failures.append(f"{name}:{number}: missing local target {match[1]}") - elif link.fragment and target.suffix == ".md": - if unquote(link.fragment) not in anchors(target.read_text()): - failures.append(f"{name}:{number}: missing heading {match[1]}") - if failures: - raise SystemExit("Documentation check failed:\n" + "\n".join(failures)) + if scheme in ("http", "https"): + match = REPOSITORY.match(link) + if match: + self.repository(where, link, match) + else: + self.external.setdefault(link, []).append(where) + elif scheme or link.startswith("//"): + self.fail(where, f"unsupported link scheme {link}") + elif source.as_posix() in PACKAGE_PAGES and urlsplit(link).path: + self.fail(where, f"relative link in a guide rendered on a package page {link}") + else: + self.local(where, source, link) + + def check_metadata(self, name: str) -> None: + path = self.root / name + if not path.exists(): + return + text = path.read_text() + for match in BARE.finditer(text): + link = match[0].rstrip(".,;:!?") + where = f"{name}:{text.count(chr(10), 0, match.start()) + 1}" + repository = REPOSITORY.match(link) + if repository: + self.repository(where, link, repository) + else: + self.external.setdefault(link, []).append(where) + + def run(self, external: bool) -> list[str]: + for path in markdown_paths(self.root, self.tracked): + self.check_markdown(path) + for name in METADATA: + self.check_metadata(name) + warnings: list[str] = [] + if not external: + return warnings + tag = f"https://github.com/mlapin/libsdca/tree/v{self.version}" + if self.release and request(tag)[0] == 404: + warnings.append( + f"v{self.version} is not on GitHub yet; {len(self.release)} links to it " + f"({', '.join(self.release[:3])}, ...) resolve once the release is tagged" + ) + urls = sorted(url for url in self.external if not re.search(r"[${}<>]", url)) + with ThreadPoolExecutor(16) as pool: + for url, (status, problem) in zip(urls, pool.map(request, urls), strict=True): + if problem is None: + continue + where = ", ".join(self.external[url][:3]) + if len(self.external[url]) > 3: + where += f" (+{len(self.external[url]) - 3} more)" + if status in UNVERIFIABLE: + warnings.append( + f"{where}: the host refuses scripted requests ({problem}) {url}" + ) + else: + self.failures.append(f"{where}: {problem} {url}") + return warnings + + +def request(url: str) -> tuple[int | None, str | None]: + """Fetch `url`; return (status, problem), where problem is None for a working link.""" + address, _, fragment = url.partition("#") + crate = re.match(r"https://crates\.io/crates/([^/?#]+)/?$", address) + if crate: # the crates.io page is a client-side app; ask its API + address = f"https://crates.io/api/v1/crates/{crate[1]}" + # GitHub renders heading anchors with a prefix in client-side code; skip those fragments. + page_anchor = bool(fragment) and urlsplit(address).netloc != "github.com" and not crate + headers = {"User-Agent": "Mozilla/5.0 (compatible; libsdca-docs-check)", "Accept": "*/*"} + status: int | None = None + problem: str | None = None + for method in ("GET",) if page_anchor else ("HEAD", "GET"): + request_ = urllib.request.Request(address, headers=headers, method=method) + try: + with urllib.request.urlopen(request_, timeout=30) as response: + status = response.status + body = response.read(4_000_000) if method == "GET" else b"" + except urllib.error.HTTPError as error: # some servers refuse HEAD: retry with GET + status, problem = error.code, f"HTTP {error.code}" + continue + except (urllib.error.URLError, TimeoutError, OSError) as error: + status, problem = None, f"unreachable ({getattr(error, 'reason', error)})" + continue + if page_anchor: + name = re.escape(unquote(fragment)) + page = body.decode("utf-8", "replace") + if not re.search(rf"""\b(?:id|name)\s*=\s*["']{name}["']""", page): + return status, "missing anchor" + return status, None + return status, problem + + +def check_docs(root: Path = ROOT, external: bool = False) -> None: + checker = Checker(root) + for warning in checker.run(external): + print(f"warning: {warning}", file=sys.stderr) + if checker.failures: + failures = "\n".join(sorted(set(checker.failures))) + raise SystemExit("Documentation check failed:\n" + failures) + + +def main() -> None: + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n\n")[0]) + parser.add_argument("--external", action="store_true", help="also request external URLs") + external = parser.parse_args().external + check_docs(external=external) + print("Documentation links and math: OK" + (" (with external URLs)" if external else "")) if __name__ == "__main__": - check_docs() - print("Documentation links and math: OK") + main()