Skip to content

Run CI's repository checks with one local command #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ovhirup merged 1 commit into from
Nov 2, 2026
Merged

Run CI's repository checks with one local command #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 2 additions & 170 deletions .github/workflows/validate.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,178 +13,10 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Manifests valid and in agreement
run: |
test -f .claude-plugin/marketplace.json || { echo "::error::marketplace.json missing"; exit 1; }
test -f plugins/stackwich/.claude-plugin/plugin.json || { echo "::error::plugin.json missing"; exit 1; }
jq -e . .claude-plugin/marketplace.json > /dev/null || { echo "::error::marketplace.json is not valid JSON"; exit 1; }
jq -e . plugins/stackwich/.claude-plugin/plugin.json > /dev/null || { echo "::error::plugin.json is not valid JSON"; exit 1; }
[ "$(jq '.plugins | length' .claude-plugin/marketplace.json)" -eq 1 ] || { echo "::error::marketplace must list exactly one plugin"; exit 1; }
[ "$(jq -r '.plugins[0].source' .claude-plugin/marketplace.json)" = "./plugins/stackwich" ] || { echo "::error::marketplace entry source must be ./plugins/stackwich"; exit 1; }
[ "$(jq -r '.plugins[0].name' .claude-plugin/marketplace.json)" = "$(jq -r '.name' plugins/stackwich/.claude-plugin/plugin.json)" ] || { echo "::error::marketplace entry name and plugin.json name disagree"; exit 1; }
PV="$(jq -r '.version' plugins/stackwich/.claude-plugin/plugin.json)"
SV="$(sed -n 's/^[[:space:]]*version:[[:space:]]*//p' plugins/stackwich/skills/stackwich/SKILL.md | head -1)"
[ "$PV" = "$SV" ] || { echo "::error::plugin.json version ($PV) != SKILL.md version ($SV)"; exit 1; }
cmp -s LICENSE plugins/stackwich/LICENSE || { echo "::error::plugins/stackwich/LICENSE has drifted from the root LICENSE"; exit 1; }
grep -q "^## $PV " CHANGELOG.md || { echo "::error::CHANGELOG.md has no '## $PV' heading — a shipped version with no release note"; exit 1; }
echo "manifests OK"

- name: Plugin shape invariants
run: |
test ! -e SKILL.md || { echo "::error::a root SKILL.md reappeared — the canonical skill lives at plugins/stackwich/skills/stackwich/SKILL.md and must not be duplicated"; exit 1; }
test ! -e assets || { echo "::error::a root assets/ reappeared — the scaffolding source lives inside the skill directory"; exit 1; }
test ! -e plugins/stackwich/agents || { echo "::error::the plugin must NOT ship agents/ — plugin agents install namespaced (stackwich:advisor) and the policy block references bare names; the skill scaffolds them instead"; exit 1; }
[ "$(ls plugins/stackwich/skills | wc -l)" -eq 1 ] || { echo "::error::plugins/stackwich/skills must contain exactly one skill directory"; exit 1; }
echo "plugin shape OK"

- name: Frontmatter and structure checks
run: |
S=plugins/stackwich/skills/stackwich/SKILL.md
test -f "$S" || { echo "::error::$S missing"; exit 1; }
head -1 "$S" | grep -qx -- '---' || { echo "::error::missing frontmatter opener"; exit 1; }
grep -q '^name: stackwich' "$S" || { echo "::error::frontmatter name missing"; exit 1; }
grep -q '^description:' "$S" || { echo "::error::frontmatter description missing"; exit 1; }
grep -qE '^[[:space:]]*author:' "$S" || { echo "::error::frontmatter author missing"; exit 1; }
[ "$(grep -cx -- '<!-- stackwich:v1 -->' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line opening marker"; exit 1; }
[ "$(grep -cx -- '<!-- /stackwich:v1 -->' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line closing marker"; exit 1; }
[ "$(grep -cxE -- '<!-- policy-rev: [0-9]+ -->' "$S")" -eq 1 ] || { echo "::error::SKILL.md must contain exactly one whole-line policy-rev comment"; exit 1; }
for h in Prompt Context Harness Loop Graph; do
awk '/^<!-- stackwich:v1 -->$/,/^<!-- \/stackwich:v1 -->$/' "$S" | grep -q "^### $h" || { echo "::error::policy block missing the $h layer"; exit 1; }
done
echo "SKILL.md OK"

- name: Frontmatter parses as YAML
- name: Repository checks (manifests, SKILL.md, revisions, assets, ports)
run: |
python3 -m pip install --quiet --disable-pip-version-check pyyaml
python3 - <<'PY'
import sys, yaml, pathlib
targets = {
"plugins/stackwich/skills/stackwich/SKILL.md": "stackwich",
"grok/SKILL.md": "stackwich-grok",
"codex/SKILL.md": "stackwich-codex",
}
bad = False
for path, expect in targets.items():
text = pathlib.Path(path).read_text()
if not text.startswith("---\n"):
print(f"::error::{path} has no frontmatter block"); bad = True; continue
block = text.split("---\n", 2)[1]
try:
fm = yaml.safe_load(block)
except yaml.YAMLError as e:
print(f"::error::{path} frontmatter is not valid YAML: {e}"); bad = True; continue
if fm.get("name") != expect:
print(f"::error::{path} name is {fm.get('name')!r}, expected {expect!r}"); bad = True
desc = fm.get("description") or ""
if not desc:
print(f"::error::{path} has no description"); bad = True
elif len(desc) > 1024:
print(f"::error::{path} description is {len(desc)} chars, over the 1024 limit"); bad = True
if not fm.get("author") and not (fm.get("metadata") or {}).get("author"):
print(f"::error::{path} has no author — every distributed copy must name one"); bad = True
sys.exit(1 if bad else 0)
PY
echo "frontmatter OK"

- name: Revision references agree
run: |
python3 - <<'PY'
import json, re, sys, pathlib
bad = []
def lines(p): return pathlib.Path(p).read_text().splitlines()
def region(ls, start, end, path):
if ls.count(start) != 1 or ls.count(end) != 1:
bad.append(f"{path} needs exactly one whole-line {start} and {end}"); return []
return ls[ls.index(start) + 1:ls.index(end)]
def one(ls, pattern, what):
hits = [m.group(1) for m in map(re.compile(pattern).fullmatch, ls) if m]
if len(hits) != 1:
bad.append(f"{what}: expected exactly one, found {len(hits)}"); return None
return hits[0]
S = "plugins/stackwich/skills/stackwich/SKILL.md"
s = lines(S)
rev = one(region(s, "<!-- stackwich:v1 -->", "<!-- /stackwich:v1 -->", S), r"<!-- policy-rev: (\d+) -->", f"{S} policy-rev comment inside the block")
fm = s[1:s.index("---", 1)] if s[:1] == ["---"] and "---" in s[1:] else []
md = [k for k, x in enumerate(fm) if x == "metadata:"]
if len(md) != 1:
bad.append(f"{S} frontmatter needs exactly one whole-line metadata: key, found {len(md)}"); meta = None
else:
e = next((k for k in range(md[0] + 1, len(fm)) if fm[k].strip() and not fm[k][0].isspace()), len(fm))
meta = one(fm[md[0] + 1:e], r" policy-rev: (\d+)", f"{S} frontmatter policy-rev directly under metadata")
h = "## 4. Verify the install"
i = s.index(h) if h in s else len(s)
j = next((k for k in range(i + 1, len(s)) if s[k].startswith("## ")), len(s))
step4 = one(s[i + 1:j], r"2\. `policy-rev: (\d+)` is present inside the block.*", f"{S} step 4 check 2")
for label, val in (("frontmatter metadata.policy-rev", meta), ("step 4 check 2", step4)):
if None not in (rev, val) and val != rev:
bad.append(f"{S} {label} is {val}, but the block's policy-rev comment is {rev}")
evals = {e["id"]: e for e in json.loads(pathlib.Path("evals/evals.json").read_text())["evals"]}
for cid in (1, 2):
case = evals.get(cid, {})
found = {n for t in [case.get("expected_output", ""), *case.get("assertions", [])] for n in re.findall(r"policy-rev:? (\d+)", t)}
if not found:
bad.append(f"evals/evals.json case {cid} names no policy-rev number")
for n in sorted(found - {rev}):
bad.append(f"evals/evals.json case {cid} says policy-rev {n}, but the block says {rev}")
for port, path, readme, a, b in (
("grok", "grok/SKILL.md", "grok/README.md", "<!-- stackwich:v1 -->", "<!-- /stackwich:v1 -->"),
("codex", "codex/references/templates.md", "codex/README.md", "<!-- stackwich-codex:v1 -->", "<!-- /stackwich-codex:v1 -->")):
prev = one(region(lines(path), a, b, path), rf"<!-- stackwich-{port}-rev: (\d+) -->", f"{path} stackwich-{port}-rev comment inside the block")
said = re.findall(rf"`stackwich-{port}-rev (\d+)`", pathlib.Path(readme).read_text())
if len(said) != 1:
bad.append(f"{readme}: expected exactly one `stackwich-{port}-rev N` mention, found {len(said)}")
elif prev is not None and said[0] != prev:
bad.append(f"{readme} says stackwich-{port}-rev {said[0]}, but {path} says {prev}")
for msg in bad:
print(f"::error::{msg}")
sys.exit(1 if bad else 0)
PY
echo "revisions OK"

- name: Agent assets present and well-formed
run: |
A=plugins/stackwich/skills/stackwich/assets
for a in advisor executor verifier; do
test -f "$A/$a.md" || { echo "::error::$A/$a.md missing"; exit 1; }
grep -q "^name: $a" "$A/$a.md" || { echo "::error::$A/$a.md name wrong"; exit 1; }
grep -q '^model:' "$A/$a.md" || { echo "::error::$A/$a.md model missing"; exit 1; }
grep -q '^tools:' "$A/$a.md" || { echo "::error::$A/$a.md tools missing"; exit 1; }
done
echo "assets OK"

- name: Grok port well-formed
run: |
test -f grok/SKILL.md || { echo "::error::grok/SKILL.md missing"; exit 1; }
grep -qx -- 'name: stackwich-grok' grok/SKILL.md || { echo "::error::grok port name must be stackwich-grok"; exit 1; }
[ "$(grep -cx -- '<!-- stackwich:v1 -->' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok needs exactly one whole-line opening marker"; exit 1; }
[ "$(grep -cx -- '<!-- /stackwich:v1 -->' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok needs exactly one whole-line closing marker"; exit 1; }
[ "$(grep -c '^```' grok/SKILL.md)" -eq 2 ] || { echo "::error::grok policy block fence unbalanced"; exit 1; }
[ "$(grep -cxE -- '<!-- stackwich-grok-rev: [0-9]+ -->' grok/SKILL.md)" -eq 1 ] || { echo "::error::grok policy block needs exactly one whole-line rev comment"; exit 1; }
for h in Prompt Context Harness Loop Graph; do
awk '/^<!-- stackwich:v1 -->$/,/^<!-- \/stackwich:v1 -->$/' grok/SKILL.md | grep -q "^### $h" || { echo "::error::grok policy block missing the $h layer"; exit 1; }
done
echo "grok port OK"

- name: Codex port well-formed
run: |
test -f codex/SKILL.md || { echo "::error::codex/SKILL.md missing"; exit 1; }
grep -qx -- 'name: stackwich-codex' codex/SKILL.md || { echo "::error::codex port name must be stackwich-codex"; exit 1; }
test -f codex/references/templates.md || { echo "::error::codex/references/templates.md missing — SKILL.md depends on it"; exit 1; }
[ "$(grep -cx -- '<!-- stackwich-codex:v1 -->' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex templates need exactly one whole-line opening marker"; exit 1; }
[ "$(grep -cx -- '<!-- /stackwich-codex:v1 -->' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex templates need exactly one whole-line closing marker"; exit 1; }
[ "$(grep -cxE -- '<!-- stackwich-codex-rev: [0-9]+ -->' codex/references/templates.md)" -eq 1 ] || { echo "::error::codex policy block needs exactly one whole-line rev comment"; exit 1; }
for h in Prompt Context Harness Loop Graph; do
awk '/^<!-- stackwich-codex:v1 -->$/,/^<!-- \/stackwich-codex:v1 -->$/' codex/references/templates.md | grep -q "^### $h" || { echo "::error::codex policy block missing the $h layer"; exit 1; }
done
for a in advisor executor verifier; do
grep -q "^## $a.toml" codex/references/templates.md || { echo "::error::templates missing $a.toml section"; exit 1; }
done
test -f codex/agents/openai.yaml || { echo "::error::codex/agents/openai.yaml missing"; exit 1; }
test -f codex/LICENSE || { echo "::error::codex/LICENSE missing"; exit 1; }
echo "codex port OK"

- name: README hero image exists
run: test -f docs/sandwich.jpg && echo "docs OK"
python3 tools/validate.py

- name: Install checks, diagnostics, uninstall, and first-task demo
run: cd tools && python3 -m unittest discover -v
Expand Down
13 changes: 9 additions & 4 deletions CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Issues and pull requests are welcome. This file exists because the repo has five rules that
aren't obvious from reading the code, and each one has already caused a real bug.

`.github/workflows/validate.yml` enforces all of them. If CI fails on your PR, it is probably
`tools/validate.py`, which CI runs, enforces all of them. If CI fails on your PR, it is probably
one of these.

## 1. Markers are never renamed
Expand Down Expand Up @@ -41,7 +41,8 @@ opening marker `<!-- stackwich:v1 -->`" as prose, next to the real marker.
A substring check is satisfied by the documentation instead of the thing it documents. That is
not hypothetical: a release once shipped with its opening marker destroyed, and CI passed,
because the grep matched the prose. Every marker and revision assertion is an exact whole-line
count. Do not relax them to `grep -q`.
count. Do not relax them to substring matches; `tools/test_validate.py` fails if the prose alone
satisfies the opening-marker check.

## 5. `SKILL.md` and `assets/` move together

Expand All @@ -50,12 +51,16 @@ silently breaks scaffolding for every user, and nothing outside the assets check

## Running the checks locally

Everything the `skill-lint` CI job runs, from the repository root:

```bash
python3 -m pip install pyyaml
python3 tools/validate.py && (cd tools && python3 -m unittest discover) && (cd measured && python3 -m unittest test_measured && python3 approve.py --self-check)
```

Then run the steps in `.github/workflows/validate.yml`. There is no separate test command — the
workflow is the test suite.
`tools/validate.py` prints one `OK` line per category, or an `::error::` line per problem, and
exits 1 if any category failed. The `recipes` job needs Node and Playwright; see
[docs/recipes.md](docs/recipes.md).

## Versioning

Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ stackwich/
│ ├── verifier.md
│ └── rebound.md # cost/rebound playbook, read only for cost or loop questions
├── evals/evals.json # five install paths + two policy-behavior cases
├── tools/ # check_install, diagnose, uninstall, verify_gate, and handoff
├── tools/ # check_install, diagnose, uninstall, verify_gate, handoff, and validate (CI's checks)
├── examples/first-task/ # the demo project for docs/first-task.md
├── recipes/ # API, browser-flow, responsive, and accessibility checks
├── docs/sandwich.jpg
Expand Down
2 changes: 1 addition & 1 deletion docs/evals.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ and the approval gate's self-check.
(cd measured && python3 -m unittest -v test_measured && python3 approve.py --self-check)
```

`.github/workflows/validate.yml` runs both after its structural checks.
`.github/workflows/validate.yml` runs both after its structural checks, `python3 tools/validate.py`.

## Behavior evals: `evals/evals.json`

Expand Down
85 changes: 85 additions & 0 deletions tools/test_validate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

from validate import run

ROOT = Path(__file__).resolve().parent.parent
SKILL = Path("plugins/stackwich/skills/stackwich/SKILL.md")


class Validate(unittest.TestCase):
"""The repository checks CI runs, against a copy of this checkout with one thing broken."""

def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name) / "repo"
shutil.copytree(ROOT, self.root, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__"))

def tearDown(self):
self.tmp.cleanup()

def edit(self, path, old, new):
full = self.root / path
text = full.read_text(encoding="utf-8")
self.assertIn(old, text, f"fixture drifted: {old!r} not in {path}")
full.write_text(text.replace(old, new, 1), encoding="utf-8")

def errors(self):
ok, lines = run(self.root)
self.assertFalse(ok, "\n".join(lines))
return "\n".join(line for line in lines if line.startswith("::error::"))

def test_this_checkout_passes(self):
ok, lines = run(self.root)
self.assertTrue(ok, "\n".join(lines))
for done in ("manifests OK", "plugin shape OK", "SKILL.md OK", "frontmatter OK", "revisions OK",
"assets OK", "grok port OK", "codex port OK", "docs OK"):
self.assertIn(done, lines)

def test_marker_only_in_prose_fails(self):
"""CONTRIBUTING rule 4: the prose mention of the marker must not satisfy the check."""
self.edit(SKILL, "\n<!-- stackwich:v1 -->\n", "\n")
self.assertIn("`<!-- stackwich:v1 -->`", (self.root / SKILL).read_text(encoding="utf-8"))
self.assertIn("SKILL.md must contain exactly one whole-line opening marker", self.errors())

def test_frontmatter_policy_rev_disagrees_with_block(self):
rev = re.search(r"^<!-- policy-rev: (\d+) -->$", (self.root / SKILL).read_text(), re.M).group(1)
self.edit(SKILL, f" policy-rev: {rev}\n", f" policy-rev: {int(rev) - 1}\n")
self.assertIn(f"frontmatter metadata.policy-rev is {int(rev) - 1}, but the block's policy-rev comment is {rev}",
self.errors())

def test_port_readme_disagrees_with_port(self):
readme = (self.root / "grok/README.md").read_text(encoding="utf-8")
rev = re.search(r"`stackwich-grok-rev (\d+)`", readme).group(1)
self.edit("grok/README.md", f"`stackwich-grok-rev {rev}`", f"`stackwich-grok-rev {int(rev) + 1}`")
self.assertIn(f"grok/README.md says stackwich-grok-rev {int(rev) + 1}, but grok/SKILL.md says {rev}",
self.errors())

def test_missing_agent_asset(self):
(self.root / SKILL.parent / "assets" / "verifier.md").unlink()
self.assertIn("plugins/stackwich/skills/stackwich/assets/verifier.md missing", self.errors())

def test_plugin_must_not_ship_agents(self):
(self.root / "plugins/stackwich/agents").mkdir()
self.assertIn("the plugin must NOT ship agents/", self.errors())

def test_version_mismatch(self):
self.edit("plugins/stackwich/.claude-plugin/plugin.json", '"version": "', '"version": "9')
self.assertIn("plugin.json version (9", self.errors())

def test_cli_exit_code_and_annotations(self):
self.edit("codex/references/templates.md", "## executor.toml", "## builder.toml")
out = subprocess.run([sys.executable, str(Path(__file__).with_name("validate.py")), "--root", str(self.root)],
capture_output=True, text=True, check=False)
self.assertEqual(out.returncode, 1, out.stdout + out.stderr)
self.assertIn("::error::templates missing executor.toml section", out.stdout)
self.assertIn("grok port OK", out.stdout)


if __name__ == "__main__":
unittest.main()
Loading
Loading