diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index a2c24db..1a2194f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -13,178 +13,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Manifests valid and in agreement - run: | - test -f .claude-plugin/marketplace.json || { echo "::error::marketplace.json missing"; exit 1; } - test -f plugins/stackwich/.claude-plugin/plugin.json || { echo "::error::plugin.json missing"; exit 1; } - jq -e . .claude-plugin/marketplace.json > /dev/null || { echo "::error::marketplace.json is not valid JSON"; exit 1; } - jq -e . plugins/stackwich/.claude-plugin/plugin.json > /dev/null || { echo "::error::plugin.json is not valid JSON"; exit 1; } - [ "$(jq '.plugins | length' .claude-plugin/marketplace.json)" -eq 1 ] || { echo "::error::marketplace must list exactly one plugin"; exit 1; } - [ "$(jq -r '.plugins[0].source' .claude-plugin/marketplace.json)" = "./plugins/stackwich" ] || { echo "::error::marketplace entry source must be ./plugins/stackwich"; exit 1; } - [ "$(jq -r '.plugins[0].name' .claude-plugin/marketplace.json)" = "$(jq -r '.name' plugins/stackwich/.claude-plugin/plugin.json)" ] || { echo "::error::marketplace entry name and plugin.json name disagree"; exit 1; } - PV="$(jq -r '.version' plugins/stackwich/.claude-plugin/plugin.json)" - SV="$(sed -n 's/^[[:space:]]*version:[[:space:]]*//p' plugins/stackwich/skills/stackwich/SKILL.md | head -1)" - [ "$PV" = "$SV" ] || { echo "::error::plugin.json version ($PV) != SKILL.md version ($SV)"; exit 1; } - cmp -s LICENSE plugins/stackwich/LICENSE || { echo "::error::plugins/stackwich/LICENSE has drifted from the root LICENSE"; exit 1; } - grep -q "^## $PV " CHANGELOG.md || { echo "::error::CHANGELOG.md has no '## $PV' heading — a shipped version with no release note"; exit 1; } - echo "manifests OK" - - - name: Plugin shape invariants - run: | - test ! -e SKILL.md || { echo "::error::a root SKILL.md reappeared — the canonical skill lives at plugins/stackwich/skills/stackwich/SKILL.md and must not be duplicated"; exit 1; } - test ! -e assets || { echo "::error::a root assets/ reappeared — the scaffolding source lives inside the skill directory"; exit 1; } - test ! -e plugins/stackwich/agents || { echo "::error::the plugin must NOT ship agents/ — plugin agents install namespaced (stackwich:advisor) and the policy block references bare names; the skill scaffolds them instead"; exit 1; } - [ "$(ls plugins/stackwich/skills | wc -l)" -eq 1 ] || { echo "::error::plugins/stackwich/skills must contain exactly one skill directory"; exit 1; } - echo "plugin shape OK" - - - name: Frontmatter and structure checks - run: | - S=plugins/stackwich/skills/stackwich/SKILL.md - test -f "$S" || { echo "::error::$S missing"; exit 1; } - head -1 "$S" | grep -qx -- '---' || { echo "::error::missing frontmatter opener"; exit 1; } - grep -q '^name: stackwich' "$S" || { echo "::error::frontmatter name missing"; exit 1; } - grep -q '^description:' "$S" || { echo "::error::frontmatter description missing"; exit 1; } - grep -qE '^[[:space:]]*author:' "$S" || { echo "::error::frontmatter author missing"; exit 1; } - [ "$(grep -cx -- '' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line opening marker"; exit 1; } - [ "$(grep -cx -- '' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line closing marker"; exit 1; } - [ "$(grep -cxE -- '' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line policy-rev comment"; exit 1; } - for h in Prompt Context Harness Loop Graph; do - awk '/^$/,/^$/' "$S" | grep -q "^### $h" || { echo "::error::policy block missing the $h layer"; exit 1; } - done - echo "SKILL.md OK" - - - name: Frontmatter parses as YAML + - name: Repository checks (manifests, SKILL.md, revisions, assets, ports) run: | python3 -m pip install --quiet --disable-pip-version-check pyyaml - python3 - <<'PY' - import sys, yaml, pathlib - targets = { - "plugins/stackwich/skills/stackwich/SKILL.md": "stackwich", - "grok/SKILL.md": "stackwich-grok", - "codex/SKILL.md": "stackwich-codex", - } - bad = False - for path, expect in targets.items(): - text = pathlib.Path(path).read_text() - if not text.startswith("---\n"): - print(f"::error::{path} has no frontmatter block"); bad = True; continue - block = text.split("---\n", 2)[1] - try: - fm = yaml.safe_load(block) - except yaml.YAMLError as e: - print(f"::error::{path} frontmatter is not valid YAML: {e}"); bad = True; continue - if fm.get("name") != expect: - print(f"::error::{path} name is {fm.get('name')!r}, expected {expect!r}"); bad = True - desc = fm.get("description") or "" - if not desc: - print(f"::error::{path} has no description"); bad = True - elif len(desc) > 1024: - print(f"::error::{path} description is {len(desc)} chars, over the 1024 limit"); bad = True - if not fm.get("author") and not (fm.get("metadata") or {}).get("author"): - print(f"::error::{path} has no author — every distributed copy must name one"); bad = True - sys.exit(1 if bad else 0) - PY - echo "frontmatter OK" - - - name: Revision references agree - run: | - python3 - <<'PY' - import json, re, sys, pathlib - bad = [] - def lines(p): return pathlib.Path(p).read_text().splitlines() - def region(ls, start, end, path): - if ls.count(start) != 1 or ls.count(end) != 1: - bad.append(f"{path} needs exactly one whole-line {start} and {end}"); return [] - return ls[ls.index(start) + 1:ls.index(end)] - def one(ls, pattern, what): - hits = [m.group(1) for m in map(re.compile(pattern).fullmatch, ls) if m] - if len(hits) != 1: - bad.append(f"{what}: expected exactly one, found {len(hits)}"); return None - return hits[0] - S = "plugins/stackwich/skills/stackwich/SKILL.md" - s = lines(S) - rev = one(region(s, "", "", S), r"", f"{S} policy-rev comment inside the block") - fm = s[1:s.index("---", 1)] if s[:1] == ["---"] and "---" in s[1:] else [] - md = [k for k, x in enumerate(fm) if x == "metadata:"] - if len(md) != 1: - bad.append(f"{S} frontmatter needs exactly one whole-line metadata: key, found {len(md)}"); meta = None - else: - e = next((k for k in range(md[0] + 1, len(fm)) if fm[k].strip() and not fm[k][0].isspace()), len(fm)) - meta = one(fm[md[0] + 1:e], r" policy-rev: (\d+)", f"{S} frontmatter policy-rev directly under metadata") - h = "## 4. Verify the install" - i = s.index(h) if h in s else len(s) - j = next((k for k in range(i + 1, len(s)) if s[k].startswith("## ")), len(s)) - step4 = one(s[i + 1:j], r"2\. `policy-rev: (\d+)` is present inside the block.*", f"{S} step 4 check 2") - for label, val in (("frontmatter metadata.policy-rev", meta), ("step 4 check 2", step4)): - if None not in (rev, val) and val != rev: - bad.append(f"{S} {label} is {val}, but the block's policy-rev comment is {rev}") - evals = {e["id"]: e for e in json.loads(pathlib.Path("evals/evals.json").read_text())["evals"]} - for cid in (1, 2): - case = evals.get(cid, {}) - found = {n for t in [case.get("expected_output", ""), *case.get("assertions", [])] for n in re.findall(r"policy-rev:? (\d+)", t)} - if not found: - bad.append(f"evals/evals.json case {cid} names no policy-rev number") - for n in sorted(found - {rev}): - bad.append(f"evals/evals.json case {cid} says policy-rev {n}, but the block says {rev}") - for port, path, readme, a, b in ( - ("grok", "grok/SKILL.md", "grok/README.md", "", ""), - ("codex", "codex/references/templates.md", "codex/README.md", "", "")): - prev = one(region(lines(path), a, b, path), rf"", f"{path} stackwich-{port}-rev comment inside the block") - said = re.findall(rf"`stackwich-{port}-rev (\d+)`", pathlib.Path(readme).read_text()) - if len(said) != 1: - bad.append(f"{readme}: expected exactly one `stackwich-{port}-rev N` mention, found {len(said)}") - elif prev is not None and said[0] != prev: - bad.append(f"{readme} says stackwich-{port}-rev {said[0]}, but {path} says {prev}") - for msg in bad: - print(f"::error::{msg}") - sys.exit(1 if bad else 0) - PY - echo "revisions OK" - - - name: Agent assets present and well-formed - run: | - A=plugins/stackwich/skills/stackwich/assets - for a in advisor executor verifier; do - test -f "$A/$a.md" || { echo "::error::$A/$a.md missing"; exit 1; } - grep -q "^name: $a" "$A/$a.md" || { echo "::error::$A/$a.md name wrong"; exit 1; } - grep -q '^model:' "$A/$a.md" || { echo "::error::$A/$a.md model missing"; exit 1; } - grep -q '^tools:' "$A/$a.md" || { echo "::error::$A/$a.md tools missing"; exit 1; } - done - echo "assets OK" - - - name: Grok port well-formed - run: | - test -f grok/SKILL.md || { echo "::error::grok/SKILL.md missing"; exit 1; } - grep -qx -- 'name: stackwich-grok' grok/SKILL.md || { echo "::error::grok port name must be stackwich-grok"; exit 1; } - [ "$(grep -cx -- '' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok needs exactly one whole-line opening marker"; exit 1; } - [ "$(grep -cx -- '' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok needs exactly one whole-line closing marker"; exit 1; } - [ "$(grep -c '^```' grok/SKILL.md)" -eq 2 ] || { echo "::error::grok policy block fence unbalanced"; exit 1; } - [ "$(grep -cxE -- '' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok policy block needs exactly one whole-line rev comment"; exit 1; } - for h in Prompt Context Harness Loop Graph; do - awk '/^$/,/^$/' grok/SKILL.md | grep -q "^### $h" || { echo "::error::grok policy block missing the $h layer"; exit 1; } - done - echo "grok port OK" - - - name: Codex port well-formed - run: | - test -f codex/SKILL.md || { echo "::error::codex/SKILL.md missing"; exit 1; } - grep -qx -- 'name: stackwich-codex' codex/SKILL.md || { echo "::error::codex port name must be stackwich-codex"; exit 1; } - test -f codex/references/templates.md || { echo "::error::codex/references/templates.md missing — SKILL.md depends on it"; exit 1; } - [ "$(grep -cx -- '' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex templates need exactly one whole-line opening marker"; exit 1; } - [ "$(grep -cx -- '' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex templates need exactly one whole-line closing marker"; exit 1; } - [ "$(grep -cxE -- '' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex policy block needs exactly one whole-line rev comment"; exit 1; } - for h in Prompt Context Harness Loop Graph; do - awk '/^$/,/^$/' codex/references/templates.md | grep -q "^### $h" || { echo "::error::codex policy block missing the $h layer"; exit 1; } - done - for a in advisor executor verifier; do - grep -q "^## $a.toml" codex/references/templates.md || { echo "::error::templates missing $a.toml section"; exit 1; } - done - test -f codex/agents/openai.yaml || { echo "::error::codex/agents/openai.yaml missing"; exit 1; } - test -f codex/LICENSE || { echo "::error::codex/LICENSE missing"; exit 1; } - echo "codex port OK" - - - name: README hero image exists - run: test -f docs/sandwich.jpg && echo "docs OK" + python3 tools/validate.py - name: Install checks, diagnostics, uninstall, and first-task demo run: cd tools && python3 -m unittest discover -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7738af2..6efa365 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Issues and pull requests are welcome. This file exists because the repo has five rules that aren't obvious from reading the code, and each one has already caused a real bug. -`.github/workflows/validate.yml` enforces all of them. If CI fails on your PR, it is probably +`tools/validate.py`, which CI runs, enforces all of them. If CI fails on your PR, it is probably one of these. ## 1. Markers are never renamed @@ -41,7 +41,8 @@ opening marker ``" as prose, next to the real marker. A substring check is satisfied by the documentation instead of the thing it documents. That is not hypothetical: a release once shipped with its opening marker destroyed, and CI passed, because the grep matched the prose. Every marker and revision assertion is an exact whole-line -count. Do not relax them to `grep -q`. +count. Do not relax them to substring matches; `tools/test_validate.py` fails if the prose alone +satisfies the opening-marker check. ## 5. `SKILL.md` and `assets/` move together @@ -50,12 +51,16 @@ silently breaks scaffolding for every user, and nothing outside the assets check ## Running the checks locally +Everything the `skill-lint` CI job runs, from the repository root: + ```bash python3 -m pip install pyyaml +python3 tools/validate.py && (cd tools && python3 -m unittest discover) && (cd measured && python3 -m unittest test_measured && python3 approve.py --self-check) ``` -Then run the steps in `.github/workflows/validate.yml`. There is no separate test command — the -workflow is the test suite. +`tools/validate.py` prints one `OK` line per category, or an `::error::` line per problem, and +exits 1 if any category failed. The `recipes` job needs Node and Playwright; see +[docs/recipes.md](docs/recipes.md). ## Versioning diff --git a/README.md b/README.md index 58408e2..a20cac7 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ stackwich/ │ ├── verifier.md │ └── rebound.md # cost/rebound playbook, read only for cost or loop questions ├── evals/evals.json # five install paths + two policy-behavior cases -├── tools/ # check_install, diagnose, uninstall, verify_gate, and handoff +├── tools/ # check_install, diagnose, uninstall, verify_gate, handoff, and validate (CI's checks) ├── examples/first-task/ # the demo project for docs/first-task.md ├── recipes/ # API, browser-flow, responsive, and accessibility checks ├── docs/sandwich.jpg diff --git a/docs/evals.md b/docs/evals.md index 9d670dd..a4d5410 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -104,7 +104,7 @@ and the approval gate's self-check. (cd measured && python3 -m unittest -v test_measured && python3 approve.py --self-check) ``` -`.github/workflows/validate.yml` runs both after its structural checks. +`.github/workflows/validate.yml` runs both after its structural checks, `python3 tools/validate.py`. ## Behavior evals: `evals/evals.json` diff --git a/tools/test_validate.py b/tools/test_validate.py new file mode 100644 index 0000000..97107c4 --- /dev/null +++ b/tools/test_validate.py @@ -0,0 +1,85 @@ +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from validate import run + +ROOT = Path(__file__).resolve().parent.parent +SKILL = Path("plugins/stackwich/skills/stackwich/SKILL.md") + + +class Validate(unittest.TestCase): + """The repository checks CI runs, against a copy of this checkout with one thing broken.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) / "repo" + shutil.copytree(ROOT, self.root, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__")) + + def tearDown(self): + self.tmp.cleanup() + + def edit(self, path, old, new): + full = self.root / path + text = full.read_text(encoding="utf-8") + self.assertIn(old, text, f"fixture drifted: {old!r} not in {path}") + full.write_text(text.replace(old, new, 1), encoding="utf-8") + + def errors(self): + ok, lines = run(self.root) + self.assertFalse(ok, "\n".join(lines)) + return "\n".join(line for line in lines if line.startswith("::error::")) + + def test_this_checkout_passes(self): + ok, lines = run(self.root) + self.assertTrue(ok, "\n".join(lines)) + for done in ("manifests OK", "plugin shape OK", "SKILL.md OK", "frontmatter OK", "revisions OK", + "assets OK", "grok port OK", "codex port OK", "docs OK"): + self.assertIn(done, lines) + + def test_marker_only_in_prose_fails(self): + """CONTRIBUTING rule 4: the prose mention of the marker must not satisfy the check.""" + self.edit(SKILL, "\n\n", "\n") + self.assertIn("``", (self.root / SKILL).read_text(encoding="utf-8")) + self.assertIn("SKILL.md must contain exactly one whole-line opening marker", self.errors()) + + def test_frontmatter_policy_rev_disagrees_with_block(self): + rev = re.search(r"^$", (self.root / SKILL).read_text(), re.M).group(1) + self.edit(SKILL, f" policy-rev: {rev}\n", f" policy-rev: {int(rev) - 1}\n") + self.assertIn(f"frontmatter metadata.policy-rev is {int(rev) - 1}, but the block's policy-rev comment is {rev}", + self.errors()) + + def test_port_readme_disagrees_with_port(self): + readme = (self.root / "grok/README.md").read_text(encoding="utf-8") + rev = re.search(r"`stackwich-grok-rev (\d+)`", readme).group(1) + self.edit("grok/README.md", f"`stackwich-grok-rev {rev}`", f"`stackwich-grok-rev {int(rev) + 1}`") + self.assertIn(f"grok/README.md says stackwich-grok-rev {int(rev) + 1}, but grok/SKILL.md says {rev}", + self.errors()) + + def test_missing_agent_asset(self): + (self.root / SKILL.parent / "assets" / "verifier.md").unlink() + self.assertIn("plugins/stackwich/skills/stackwich/assets/verifier.md missing", self.errors()) + + def test_plugin_must_not_ship_agents(self): + (self.root / "plugins/stackwich/agents").mkdir() + self.assertIn("the plugin must NOT ship agents/", self.errors()) + + def test_version_mismatch(self): + self.edit("plugins/stackwich/.claude-plugin/plugin.json", '"version": "', '"version": "9') + self.assertIn("plugin.json version (9", self.errors()) + + def test_cli_exit_code_and_annotations(self): + self.edit("codex/references/templates.md", "## executor.toml", "## builder.toml") + out = subprocess.run([sys.executable, str(Path(__file__).with_name("validate.py")), "--root", str(self.root)], + capture_output=True, text=True, check=False) + self.assertEqual(out.returncode, 1, out.stdout + out.stderr) + self.assertIn("::error::templates missing executor.toml section", out.stdout) + self.assertIn("grok port OK", out.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py new file mode 100755 index 0000000..160ea70 --- /dev/null +++ b/tools/validate.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Run the repository checks that CI's skill-lint job runs, locally, with one command. + + python3 tools/validate.py # from anywhere in a checkout + python3 tools/validate.py --root DIR # check another copy of the repository + +Each category prints " OK" or one "::error::" line per problem, in the +format GitHub Actions turns into annotations. Every category runs, so one call shows +every failing category; the exit code is 1 if any failed. A category that came from +a shell step stops at its first problem, as the step did. + +Marker and revision checks match whole lines on purpose (CONTRIBUTING.md rule 4): +SKILL.md quotes its own marker in prose, and a substring check would pass on that. +It reads files only. Needs PyYAML: python3 -m pip install pyyaml. +""" + +from __future__ import annotations + +import argparse +import filecmp +import json +import pathlib +import re +import sys + +try: + import yaml +except ImportError: # pragma: no cover - depends on the environment + yaml = None + +ROOT = pathlib.Path(__file__).resolve().parent.parent +S = "plugins/stackwich/skills/stackwich/SKILL.md" +A = "plugins/stackwich/skills/stackwich/assets" +PLUGIN = "plugins/stackwich/.claude-plugin/plugin.json" +MARKET = ".claude-plugin/marketplace.json" +LAYERS = ("Prompt", "Context", "Harness", "Loop", "Graph") + + +class Failed(Exception): + """A shell-derived check stops at its first problem, as `exit 1` did in the workflow.""" + + +def _lines(root: pathlib.Path, path: str) -> list[str]: + return (root / path).read_text(encoding="utf-8").splitlines() + + +def _require(ok: bool, message: str) -> None: + if not ok: + raise Failed(message) + + +def _block(lines: list[str], start: str, end: str) -> list[str]: + """Like awk '/^start$/,/^end$/': from the first whole-line start to the next whole-line end.""" + if start not in lines: + return [] + i = lines.index(start) + j = lines.index(end, i + 1) if end in lines[i + 1:] else len(lines) - 1 + return lines[i:j + 1] + + +def _json(root: pathlib.Path, path: str): + try: + return json.loads((root / path).read_text(encoding="utf-8")) + except ValueError: + return None + + +def manifests(root: pathlib.Path) -> str: + _require((root / MARKET).is_file(), "marketplace.json missing") + _require((root / PLUGIN).is_file(), "plugin.json missing") + market, plugin = _json(root, MARKET), _json(root, PLUGIN) + _require(market is not None, "marketplace.json is not valid JSON") + _require(plugin is not None, "plugin.json is not valid JSON") + plugins = market.get("plugins") or [] + _require(len(plugins) == 1, "marketplace must list exactly one plugin") + _require(plugins[0].get("source") == "./plugins/stackwich", "marketplace entry source must be ./plugins/stackwich") + _require(plugins[0].get("name") == plugin.get("name"), "marketplace entry name and plugin.json name disagree") + pv = str(plugin.get("version")) + sv = next((m.group(1) for m in map(re.compile(r"^\s*version:\s*(.*)$").match, _lines(root, S)) if m), "") + _require(pv == sv, f"plugin.json version ({pv}) != SKILL.md version ({sv})") + _require(filecmp.cmp(root / "LICENSE", root / "plugins/stackwich/LICENSE", shallow=False), + "plugins/stackwich/LICENSE has drifted from the root LICENSE") + _require(any(line.startswith(f"## {pv} ") for line in _lines(root, "CHANGELOG.md")), + f"CHANGELOG.md has no '## {pv}' heading — a shipped version with no release note") + return "manifests OK" + + +def plugin_shape(root: pathlib.Path) -> str: + _require(not (root / "SKILL.md").exists(), "a root SKILL.md reappeared — the canonical skill lives at " + "plugins/stackwich/skills/stackwich/SKILL.md and must not be duplicated") + _require(not (root / "assets").exists(), + "a root assets/ reappeared — the scaffolding source lives inside the skill directory") + _require(not (root / "plugins/stackwich/agents").exists(), "the plugin must NOT ship agents/ — plugin agents " + "install namespaced (stackwich:advisor) and the policy block references bare names; the skill " + "scaffolds them instead") + _require(len(list((root / "plugins/stackwich/skills").iterdir())) == 1, + "plugins/stackwich/skills must contain exactly one skill directory") + return "plugin shape OK" + + +def skill_structure(root: pathlib.Path) -> str: + _require((root / S).is_file(), f"{S} missing") + s = _lines(root, S) + _require(s[:1] == ["---"], "missing frontmatter opener") + _require(any(line.startswith("name: stackwich") for line in s), "frontmatter name missing") + _require(any(line.startswith("description:") for line in s), "frontmatter description missing") + _require(any(re.match(r"\s*author:", line) for line in s), "frontmatter author missing") + _require(s.count("") == 1, "SKILL.md must contain exactly one whole-line opening marker") + _require(s.count("") == 1, "SKILL.md must contain exactly one whole-line closing marker") + _require(sum(bool(re.fullmatch(r"", line)) for line in s) == 1, + "SKILL.md must contain exactly one whole-line policy-rev comment") + block = _block(s, "", "") + for h in LAYERS: + _require(any(line.startswith(f"### {h}") for line in block), f"policy block missing the {h} layer") + return "SKILL.md OK" + + +def frontmatter(root: pathlib.Path) -> list[str]: + targets = {S: "stackwich", "grok/SKILL.md": "stackwich-grok", "codex/SKILL.md": "stackwich-codex"} + bad = [] + for path, expect in targets.items(): + text = (root / path).read_text(encoding="utf-8") + if not text.startswith("---\n"): + bad.append(f"{path} has no frontmatter block"); continue + block = text.split("---\n", 2)[1] + try: + fm = yaml.safe_load(block) + except yaml.YAMLError as e: + bad.append(f"{path} frontmatter is not valid YAML: {e}"); continue + if fm.get("name") != expect: + bad.append(f"{path} name is {fm.get('name')!r}, expected {expect!r}") + desc = fm.get("description") or "" + if not desc: + bad.append(f"{path} has no description") + elif len(desc) > 1024: + bad.append(f"{path} description is {len(desc)} chars, over the 1024 limit") + if not fm.get("author") and not (fm.get("metadata") or {}).get("author"): + bad.append(f"{path} has no author — every distributed copy must name one") + return bad + + +def revisions(root: pathlib.Path) -> list[str]: + bad = [] + def lines(p): return _lines(root, p) + def region(ls, start, end, path): + if ls.count(start) != 1 or ls.count(end) != 1: + bad.append(f"{path} needs exactly one whole-line {start} and {end}"); return [] + return ls[ls.index(start) + 1:ls.index(end)] + def one(ls, pattern, what): + hits = [m.group(1) for m in map(re.compile(pattern).fullmatch, ls) if m] + if len(hits) != 1: + bad.append(f"{what}: expected exactly one, found {len(hits)}"); return None + return hits[0] + s = lines(S) + rev = one(region(s, "", "", S), r"", + f"{S} policy-rev comment inside the block") + fm = s[1:s.index("---", 1)] if s[:1] == ["---"] and "---" in s[1:] else [] + md = [k for k, x in enumerate(fm) if x == "metadata:"] + if len(md) != 1: + bad.append(f"{S} frontmatter needs exactly one whole-line metadata: key, found {len(md)}"); meta = None + else: + e = next((k for k in range(md[0] + 1, len(fm)) if fm[k].strip() and not fm[k][0].isspace()), len(fm)) + meta = one(fm[md[0] + 1:e], r" policy-rev: (\d+)", f"{S} frontmatter policy-rev directly under metadata") + h = "## 4. Verify the install" + i = s.index(h) if h in s else len(s) + j = next((k for k in range(i + 1, len(s)) if s[k].startswith("## ")), len(s)) + step4 = one(s[i + 1:j], r"2\. `policy-rev: (\d+)` is present inside the block.*", f"{S} step 4 check 2") + for label, val in (("frontmatter metadata.policy-rev", meta), ("step 4 check 2", step4)): + if None not in (rev, val) and val != rev: + bad.append(f"{S} {label} is {val}, but the block's policy-rev comment is {rev}") + evals = {e["id"]: e for e in json.loads((root / "evals/evals.json").read_text(encoding="utf-8"))["evals"]} + for cid in (1, 2): + case = evals.get(cid, {}) + found = {n for t in [case.get("expected_output", ""), *case.get("assertions", [])] + for n in re.findall(r"policy-rev:? (\d+)", t)} + if not found: + bad.append(f"evals/evals.json case {cid} names no policy-rev number") + for n in sorted(found - {rev}): + bad.append(f"evals/evals.json case {cid} says policy-rev {n}, but the block says {rev}") + for port, path, readme, a, b in ( + ("grok", "grok/SKILL.md", "grok/README.md", "", ""), + ("codex", "codex/references/templates.md", "codex/README.md", + "", "")): + prev = one(region(lines(path), a, b, path), rf"", + f"{path} stackwich-{port}-rev comment inside the block") + said = re.findall(rf"`stackwich-{port}-rev (\d+)`", (root / readme).read_text(encoding="utf-8")) + if len(said) != 1: + bad.append(f"{readme}: expected exactly one `stackwich-{port}-rev N` mention, found {len(said)}") + elif prev is not None and said[0] != prev: + bad.append(f"{readme} says stackwich-{port}-rev {said[0]}, but {path} says {prev}") + return bad + + +def agent_assets(root: pathlib.Path) -> str: + for a in ("advisor", "executor", "verifier"): + path = f"{A}/{a}.md" + _require((root / path).is_file(), f"{path} missing") + ls = _lines(root, path) + _require(any(line.startswith(f"name: {a}") for line in ls), f"{path} name wrong") + _require(any(line.startswith("model:") for line in ls), f"{path} model missing") + _require(any(line.startswith("tools:") for line in ls), f"{path} tools missing") + return "assets OK" + + +def grok_port(root: pathlib.Path) -> str: + path = "grok/SKILL.md" + _require((root / path).is_file(), "grok/SKILL.md missing") + g = _lines(root, path) + _require("name: stackwich-grok" in g, "grok port name must be stackwich-grok") + _require(g.count("") == 1, "grok needs exactly one whole-line opening marker") + _require(g.count("") == 1, "grok needs exactly one whole-line closing marker") + _require(sum(line.startswith("```") for line in g) == 2, "grok policy block fence unbalanced") + _require(sum(bool(re.fullmatch(r"", line)) for line in g) == 1, + "grok policy block needs exactly one whole-line rev comment") + block = _block(g, "", "") + for h in LAYERS: + _require(any(line.startswith(f"### {h}") for line in block), f"grok policy block missing the {h} layer") + return "grok port OK" + + +def codex_port(root: pathlib.Path) -> str: + _require((root / "codex/SKILL.md").is_file(), "codex/SKILL.md missing") + _require("name: stackwich-codex" in _lines(root, "codex/SKILL.md"), "codex port name must be stackwich-codex") + t = "codex/references/templates.md" + _require((root / t).is_file(), "codex/references/templates.md missing — SKILL.md depends on it") + c = _lines(root, t) + _require(c.count("") == 1, "codex templates need exactly one whole-line opening marker") + _require(c.count("") == 1, + "codex templates need exactly one whole-line closing marker") + _require(sum(bool(re.fullmatch(r"", line)) for line in c) == 1, + "codex policy block needs exactly one whole-line rev comment") + block = _block(c, "", "") + for h in LAYERS: + _require(any(line.startswith(f"### {h}") for line in block), f"codex policy block missing the {h} layer") + for a in ("advisor", "executor", "verifier"): + _require(any(re.match(rf"## {a}.toml", line) for line in c), f"templates missing {a}.toml section") + _require((root / "codex/agents/openai.yaml").is_file(), "codex/agents/openai.yaml missing") + _require((root / "codex/LICENSE").is_file(), "codex/LICENSE missing") + return "codex port OK" + + +def hero_image(root: pathlib.Path) -> str: + _require((root / "docs/sandwich.jpg").is_file(), "docs/sandwich.jpg missing") + return "docs OK" + + +CHECKS = ( + (manifests, None), (plugin_shape, None), (skill_structure, None), + (frontmatter, "frontmatter OK"), (revisions, "revisions OK"), + (agent_assets, None), (grok_port, None), (codex_port, None), (hero_image, None), +) + + +def run(root: pathlib.Path = ROOT) -> tuple[bool, list[str]]: + """Run every category. Returns whether all passed, and the lines to print.""" + out, ok = [], True + for check, done in CHECKS: + try: + result = check(root) + except Failed as exc: + out.append(f"::error::{exc}"); ok = False; continue + except (OSError, ValueError, KeyError, IndexError) as exc: + out.append(f"::error::{check.__name__}: {exc}"); ok = False; continue + if isinstance(result, list): + out.extend(f"::error::{msg}" for msg in result) + ok = ok and not result + if not result: + out.append(done) + else: + out.append(result) + return ok, out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the repository checks CI's skill-lint job runs") + parser.add_argument("--root", default=str(ROOT), help="Repository root to check (default: this checkout)") + args = parser.parse_args(argv) + if yaml is None: + print("::error::PyYAML is not installed; run: python3 -m pip install pyyaml", file=sys.stderr) + return 2 + ok, lines = run(pathlib.Path(args.root).resolve()) + print("\n".join(lines)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main())