From 14f2201a38a4661586739edbef2fff9cd9db792f Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:35:26 -0800 Subject: [PATCH] chore(deps): update eval remote pin to copilot-evals-v0.24.14 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/backend-defaults.test.mjs | 2 +- .vally.yaml | 4 ++-- README.md | 6 +++--- docs/checks-and-evals.md | 4 ++-- marketplace.json | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/scripts/backend-defaults.test.mjs b/.github/scripts/backend-defaults.test.mjs index a7d395c..e2fdd7b 100644 --- a/.github/scripts/backend-defaults.test.mjs +++ b/.github/scripts/backend-defaults.test.mjs @@ -31,7 +31,7 @@ function assertDocumentedEval(projectRoot, path) { } test("project root fields match the checked Evals readers and inherit native local execution", async () => { - // Root-key contract for marketplace-pinned Evals 0.24.6 (copilot-evals-v0.24.6). + // Root-key contract for marketplace-pinned Evals 0.24.14 (copilot-evals-v0.24.14). const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]); const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); for (const key of Object.keys(configuration)) { diff --git a/.vally.yaml b/.vally.yaml index ac5caad..8a151f7 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -21,11 +21,11 @@ environments: # Vally runs setup in each trial workspace; Copilot discovers these skills natively. commands: - git clone --filter=blob:none --no-checkout --depth 1 --branch - copilot-evals-v0.24.6 --sparse https://github.com/github/evals.git + copilot-evals-v0.24.14 --sparse https://github.com/github/evals.git .vally-eval-source - git -C .vally-eval-source sparse-checkout set packages/plugin/skills - git -C .vally-eval-source checkout --detach - 5c3d5213f88149fe1e5476ef029277fe56fd42e4 + bfae69e479308aaf1320dc2b2bac4315659a006a - >- node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', diff --git a/README.md b/README.md index 2b3c7f6..adb2c9b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Included local maintenance plugins and optional remote plugins: | [plugin-authoring](plugins/plugin-authoring/) | Scaffold, try locally, and prepare contributions to this plugin marketplace. | | [marketplace-maintainer](plugins/marketplace-maintainer/) | Review contributed plugins, maintain remote pins, and prepare marketplace releases. | | [pr-description](plugins/pr-description/) | Draft structured pull request descriptions from supplied change notes or diffs without inventing unshipped work. | -| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. | +| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/copilot-evals-v0.24.14/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. | ## Background @@ -102,7 +102,7 @@ For optional GitHub Copilot Evals, install it explicitly: copilot plugin install eval@plugin-marketplace-starter ``` -See its [upstream setup guidance](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md#install), +See its [upstream setup guidance](https://github.com/github/evals/blob/copilot-evals-v0.24.14/README.md#install), including experimental mode for Copilot CLI commands and Canvas. For remote-entry checks and the local-directory install limitation, see [Remote pin testing](plugins/marketplace-maintainer/skills/update-remote-pins/references/remote-pins.md#testing-a-remote-entry). @@ -231,7 +231,7 @@ This template and its local plugins use the [MIT License](LICENSE). Keep its below it. Remote plugins retain their upstream licenses. GitHub Copilot Evals is source -available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/5c3d5213f88149fe1e5476ef029277fe56fd42e4/LICENSE). +available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/bfae69e479308aaf1320dc2b2bac4315659a006a/LICENSE). It isn't MIT-licensed or open source. ## Maintainers diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 0d0354d..0371702 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -127,8 +127,8 @@ no CLI, eval or project backend is specified. This is an intentional compatible default, not a Docker failure fallback. Explicit eval selections, suites and the standard-model experiment inherit it. -This marketplace pins the GitHub Copilot Evals plugin **0.24.6** (`copilot-evals-v0.24.6`). Its -[full-engine project reader at the pinned commit](https://github.com/github/evals/blob/5c3d5213f88149fe1e5476ef029277fe56fd42e4/packages/plugin/dist/mcp/server.mjs) +This marketplace pins the GitHub Copilot Evals plugin **0.24.14** (`copilot-evals-v0.24.14`). Its +[full-engine project reader at the pinned commit](https://github.com/github/evals/blob/bfae69e479308aaf1320dc2b2bac4315659a006a/packages/plugin/dist/mcp/server.mjs) accepts `environments`, `params`, `metadata`, `paths` and `suites`, and rejects the advanced project `defaults` key. The regression check covers this root-key shape, not an installed-version or end-to-end execution test. Tools still diff --git a/marketplace.json b/marketplace.json index 15957e1..afbfd71 100644 --- a/marketplace.json +++ b/marketplace.json @@ -28,10 +28,10 @@ "source": "github", "repo": "github/evals", "path": "packages/plugin", - "ref": "copilot-evals-v0.24.6", - "sha": "5c3d5213f88149fe1e5476ef029277fe56fd42e4" + "ref": "copilot-evals-v0.24.14", + "sha": "bfae69e479308aaf1320dc2b2bac4315659a006a" }, - "version": "0.24.6", + "version": "0.24.14", "description": "GitHub Copilot Evals (Preview): set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saves use normal host permissions and runs require cost/effect approval.", "homepage": "https://github.com/github/evals" }