Release: solace-messaging-skills v0.1.1

.gitattributes

@@ -0,0 +1,4 @@
+# Shell scripts must check out with LF endings on every platform. Bash rejects
+# CRLF scripts, and the output evals compare a generated verify.sh byte for byte
+# against the bundled reference copy.
+*.sh text eol=lf

.github/workflows/ci.yml

@@ -3,10 +3,17 @@ name: ci
 # Parallel quality gates (D-16). No `needs:` between jobs so branch
 # protection on `main` can require each independently:
 #   validate:      CI-01: gh skill publish --dry-run + claude plugin validate (marketplace root)
+#                  + marketplace.json/plugins/ sync + version bumps on release PRs
+#                  + post-merge version backstop on pushes to main
 #   link-check:    CI-02: the tools/check-links.sh sweep
 #   compile-check: EVAL-02 compile the baked JCSMP samples verbatim
 #   trigger-evals: lightweight skill-triggering check; self-skips green without a key
 on:
+  # Deliberately no `edited` type: running on title/body edits either re-runs
+  # the paid eval matrix for a no-code change or, if jobs are gated, lets the
+  # edit supersede a red required check (skipped counts as success). Base
+  # retargeting to main is instead covered by the post-merge backstop below
+  # plus branch protection "require branches to be up to date" on main.
   pull_request:
   push:
     branches: [main]
@@ -18,10 +25,19 @@ on:
 permissions:
   contents: read
 
+# Pin the Claude Code CLI install to an exact version so CI never auto-pulls a
+# fresh npm publish into a job that can hold the ANTHROPIC_API_KEY secret.
+# Defined once so the two install steps cannot drift; bump deliberately.
+env:
+  CLAUDE_CODE_VERSION: 2.1.231
+
 # Supersede stacked runs on the same ref: each trigger-evals leg spends real
-# API budget, so consecutive pushes cancel the previous run instead of queuing.
+# API budget, so consecutive PR pushes cancel the previous run instead of
+# queuing. Pushes to main group by SHA instead, so a backstop run is never
+# cancelled (or pending-replaced) by the next merge, which would leave its
+# pushed range unchecked and grey instead of red.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -37,26 +53,56 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          fetch-depth: 0    # version check: merge-base + `git show <base>:` need real history
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
-          node-version: 20
+          node-version: 24
       - name: Install Claude Code CLI
-        run: npm install -g @anthropic-ai/claude-code
+        run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
       - name: Validate skills against the Agent Skills spec (gh skill publish dry run)
         env:
           GH_TOKEN: ${{ github.token }}
         run: gh skill publish --dry-run
       # Validate the Claude plugin/marketplace manifests from the marketplace root (".").
       - name: claude plugin validate (marketplace root)
         run: claude plugin validate . --strict
+      # Every marketplace entry must resolve to a real plugin dir and every
+      # plugin dir must be listed (pure tree check, runs on every event).
+      - name: Marketplace sync check
+        run: ./tools/check-plugin-versions.sh --sync-only
+      # Version bumps happen only in release PRs: fork dev -> upstream
+      # SolaceProducts main. github.base_ref is the PR's target branch in
+      # whichever repo hosts the PR, so this step fires on those upstream
+      # release PRs (the enforcement point) and on the fork's post-release
+      # main sync (where it passes, the bump already landed). It never runs
+      # on PRs into dev, and base_ref is empty on push and workflow_dispatch,
+      # so it self-skips there too. Claude Code caches installed plugins by
+      # the plugin.json version, so an unbumped release never reaches users.
+      # origin/<base>, not the event's base.sha: the latter goes stale when
+      # the base branch advances after the last PR event, silently widening
+      # the diff to other PRs' commits.
+      - name: Check plugin version bumps (release PRs)
+        if: github.base_ref == 'main'
+        run: ./tools/check-plugin-versions.sh "origin/${{ github.base_ref }}"
+      # Post-merge backstop: merging one release PR does not re-run another's
+      # checks (synchronize fires only on head pushes), so a second PR can
+      # merge on a stale green with a version main already ships. Re-checking
+      # the pushed merge against event.before (the pre-push tip) turns that
+      # red instead of silent. Prevention needs branch protection (up-to-date
+      # branches or a merge queue); `!created` skips the branch-creation push,
+      # where event.before is the all-zeros SHA.
+      - name: Check plugin version bumps (post-merge backstop)
+        if: github.event_name == 'push' && !github.event.created
+        run: ./tools/check-plugin-versions.sh "${{ github.event.before }}"
 
   # CI-02: sweep the plugins/ tree for broken and soft-404 documentation links.
   link-check:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
       - name: Link-check the plugins tree
         run: ./tools/check-links.sh plugins
 
@@ -66,8 +112,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+      - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
         with:
           distribution: temurin
           java-version: 11
@@ -100,14 +146,14 @@ jobs:
       - name: Warn when skipped (no ANTHROPIC_API_KEY)
         if: env.HAS_KEY != 'true'
         run: echo "::warning::trigger-evals self-skipped, the ANTHROPIC_API_KEY secret is not configured; this green check asserted nothing"
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         if: env.HAS_KEY == 'true'
         with:
-          node-version: 20
+          node-version: 24
       - name: Install Claude Code CLI
         if: env.HAS_KEY == 'true'
-        run: npm install -g @anthropic-ai/claude-code
+        run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
       - name: Run trigger evals for ${{ matrix.model }}
         if: env.HAS_KEY == 'true'
         env:
@@ -118,7 +164,7 @@ jobs:
       # runner VM is destroyed, so upload them where they can be inspected.
       - name: Upload transcripts on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: trigger-eval-transcripts-${{ matrix.model }}
           path: ${{ runner.temp }}/trigger-evals

README.md

@@ -16,7 +16,7 @@ A collection of Solace AI agent plugins that package skills for building event-d
 
 ### `skills` CLI
 
-This lets you pick specific skills to install and supports most agents.
+This lets you pick specific skills to install. The skills in this repository follow the open [Agent Skills](https://agentskills.io) standard, and the [`skills` CLI](https://github.com/vercel-labs/skills) discovers skills in this format from any GitHub repository and installs them into a wide range of agents, including Claude Code, Cursor, Codex, GitHub Copilot, and Windsurf.
 
 ```shell
 npx skills add SolaceProducts/agent-plugins
@@ -53,7 +53,7 @@ agent-plugins/
 ├── plugins/                         # Claude plugins, one subdirectory per plugin
 │   └── solace-messaging-skills/     # The messaging skills plugin
 │       ├── .claude-plugin/          # Plugin definition (plugin.json)
-│       ├── evals/                   # Trigger eval corpus (trigger-evals.json) and its README
+│       ├── evals/                   # Trigger and output eval corpora and their README
 │       └── skills/                  # Individual skill definitions (shared by all agents)
 │           ├── solace-application-development/  # Solace application development umbrella skill
 │           │   ├── SKILL.md         # Skill routing and instructions
@@ -62,15 +62,17 @@ agent-plugins/
 │           │   └── SKILL.md         # Navigation-only manifest; reads the live docs page
 │           └── solace-messaging-feedback/       # Feedback formatter skill
 │               └── SKILL.md         # Formats and routes session feedback (Support, Community, or Ideas portal)
-├── tools/                           # Re-runnable scripts (check-links, run-trigger-evals)
+├── tools/                           # Re-runnable scripts (check-links, run-trigger-evals, run-output-evals)
 ├── README.md                        # This file
 └── .claude-plugin/                  # Claude marketplace definition (marketplace.json)
 ```
 
-## Trigger evals
+## Trigger and output evals
 
 Each plugin ships a trigger eval corpus under `plugins/<plugin>/evals/` that checks each skill fires on the prompts it should and stays silent on the prompts it should not. See the [trigger evals README](plugins/solace-messaging-skills/evals/README.md) for the corpus format, how to run the evals locally, and how they run in CI.
 
+Each plugin also ships an output eval corpus in the same directory that checks a skill's output honors the skill contract (design before code, the door question, scrubbed feedback drafts, grounded answers). Output evals run locally only. See the [output evals section](plugins/solace-messaging-skills/evals/README.md#output-evals) of the same README for the corpus format, the graders, and how to run them.
+
 ## License
 
 This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details.

plugins/solace-messaging-skills/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "solace-messaging-skills",
   "description": "Skills for building applications on the Solace platform. Includes solace-application-development (designs and generates Solace messaging code grounded in canonical Solace docs), solace-topic-best-practices (looks up the canonical Topic Architecture Best Practices page online for topic-hierarchy questions), and solace-messaging-feedback (formats a session into copy-paste-ready feedback routed to the Solace Ideas portal, Solace Support, or the Solace Community)",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "license": "Apache-2.0",
   "author": {
     "name": "Solace"