diff --git a/.github/actions/setup-pnpm/action.yml b/.github/actions/setup-pnpm/action.yml index df570f1b..0a287ab1 100644 --- a/.github/actions/setup-pnpm/action.yml +++ b/.github/actions/setup-pnpm/action.yml @@ -10,7 +10,7 @@ runs: - name: Setup Node.js uses: actions/setup-node@v7 with: - node-version: '22' + node-version: '24' cache: pnpm cache-dependency-path: pnpm-lock.yaml diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index 3388a0ee..2b29ad88 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v7 with: - node-version: '22' + node-version: '24' cache: pnpm cache-dependency-path: pnpm-lock.yaml diff --git a/.github/workflows/monitor-model-sources.yml b/.github/workflows/monitor-model-sources.yml index 73a8f168..147afa2b 100644 --- a/.github/workflows/monitor-model-sources.yml +++ b/.github/workflows/monitor-model-sources.yml @@ -24,12 +24,11 @@ jobs: - name: Setup pnpm uses: ./.github/actions/setup-pnpm - - name: Check monitored official sources - run: pnpm fetch:model-sources + - name: Update and validate monitored official sources + run: pnpm manifest-data:update -- --only=model-sources - - name: Validate monitoring metadata + - name: Check formatting run: | - pnpm test:validate pnpm format:check pnpm biome:check git diff --check @@ -37,7 +36,7 @@ jobs: - name: Check for changes id: git-check run: | - git diff --exit-code manifests/models || echo "changes=true" >> "$GITHUB_OUTPUT" + git diff --exit-code manifests/models data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" - name: Create review pull request if: steps.git-check.outputs.changes == 'true' diff --git a/.github/workflows/update-benchmarks.yml b/.github/workflows/update-benchmarks.yml index eef7b302..5afe62d2 100644 --- a/.github/workflows/update-benchmarks.yml +++ b/.github/workflows/update-benchmarks.yml @@ -24,16 +24,11 @@ jobs: - name: Setup pnpm uses: ./.github/actions/setup-pnpm - - name: Refresh exact benchmark mappings - run: pnpm fetch:benchmarks + - name: Update and validate exact benchmark mappings + run: pnpm manifest-data:update -- --only=benchmarks - - name: Regenerate manifest modules - run: pnpm generate:manifests - - - name: Validate updated data + - name: Check formatting run: | - pnpm test:validate - pnpm data-health:check pnpm format:check pnpm biome:check git diff --check @@ -41,7 +36,7 @@ jobs: - name: Check for changes id: git-check run: | - git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" + git diff --exit-code manifests data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" - name: Create pull request if: steps.git-check.outputs.changes == 'true' diff --git a/.github/workflows/update-github-stars.yml b/.github/workflows/update-github-stars.yml index 801cbcd4..4354bf6e 100644 --- a/.github/workflows/update-github-stars.yml +++ b/.github/workflows/update-github-stars.yml @@ -24,18 +24,15 @@ jobs: - name: Setup pnpm uses: ./.github/actions/setup-pnpm - - name: Fetch GitHub stars - run: pnpm fetch:github-stars + - name: Update and validate GitHub stars + run: pnpm manifest-data:update -- --only=github-stars env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Generate TypeScript files - run: pnpm generate:manifests - - name: Check for changes id: git-check run: | - git diff --exit-code data/github-stars.json src/lib/generated/github-stars.ts || echo "changes=true" >> $GITHUB_OUTPUT + git diff --exit-code data/github-stars.json data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" - name: Create Pull Request if: steps.git-check.outputs.changes == 'true' @@ -50,7 +47,7 @@ jobs: ### Changes - Updated `data/github-stars.json` with latest star counts - - Regenerated `src/lib/generated/github-stars.ts` + - Regenerated derived data and the data-health snapshot ### Source This PR was automatically generated by the `update-github-stars` workflow. diff --git a/.github/workflows/update-product-versions.yml b/.github/workflows/update-product-versions.yml index 062e3d3e..1f55b0f9 100644 --- a/.github/workflows/update-product-versions.yml +++ b/.github/workflows/update-product-versions.yml @@ -24,19 +24,13 @@ jobs: - name: Setup pnpm uses: ./.github/actions/setup-pnpm - - name: Refresh tracked product versions - run: pnpm fetch:product-versions + - name: Update and validate tracked product versions + run: pnpm manifest-data:update -- --only=product-versions env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Regenerate manifest modules - run: pnpm generate:manifests - - - name: Validate updated data + - name: Check formatting run: | - pnpm test:validate - pnpm validate:i18n - pnpm data-health:check pnpm format:check pnpm biome:check git diff --check @@ -44,7 +38,7 @@ jobs: - name: Check for changes id: git-check run: | - git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" + git diff --exit-code manifests data/data-health.json docs/DATA-HEALTH.md src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT" - name: Create pull request if: steps.git-check.outputs.changes == 'true' diff --git a/.nvmrc b/.nvmrc index 2bd5a0a9..a45fd52c 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 +24 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8c23722..adb4dfbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ Want to contribute code? Great! Please: ### Prerequisites -- **Node.js**: 22.x +- **Node.js**: 24.x - **pnpm**: 11.17.0 (pinned by `packageManager`) - **Git**: For version control diff --git a/data/$schemas/data-health.schema.json b/data/$schemas/data-health.schema.json new file mode 100644 index 00000000..e2fc7309 --- /dev/null +++ b/data/$schemas/data-health.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Manifest Data Health Report", + "description": "Generated manifest provenance, freshness, relationship, and translation health snapshot.", + "type": "object", + "properties": { + "asOf": { "type": "string", "format": "date" }, + "thresholds": { "$ref": "#/$defs/categoryThresholds" }, + "summary": { "$ref": "#/$defs/summary" }, + "byCategory": { "$ref": "#/$defs/categoryHealthMap" }, + "translationsByLocale": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "#/$defs/translationHealth" } + }, + "issues": { + "type": "array", + "items": { "$ref": "#/$defs/issue" } + } + }, + "required": ["asOf", "thresholds", "summary", "byCategory", "translationsByLocale", "issues"], + "additionalProperties": false, + "$defs": { + "categories": { + "type": "string", + "enum": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"] + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "categoryThresholds": { + "type": "object", + "properties": { + "ides": { "$ref": "#/$defs/nonNegativeInteger" }, + "clis": { "$ref": "#/$defs/nonNegativeInteger" }, + "desktops": { "$ref": "#/$defs/nonNegativeInteger" }, + "extensions": { "$ref": "#/$defs/nonNegativeInteger" }, + "models": { "$ref": "#/$defs/nonNegativeInteger" }, + "providers": { "$ref": "#/$defs/nonNegativeInteger" }, + "vendors": { "$ref": "#/$defs/nonNegativeInteger" } + }, + "required": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"], + "additionalProperties": false + }, + "summary": { + "type": "object", + "properties": { + "totalRecords": { "$ref": "#/$defs/nonNegativeInteger" }, + "recordsWithSources": { "$ref": "#/$defs/nonNegativeInteger" }, + "verifiedRecords": { "$ref": "#/$defs/nonNegativeInteger" }, + "provenanceComplete": { "$ref": "#/$defs/nonNegativeInteger" }, + "staleVerifiedRecords": { "$ref": "#/$defs/nonNegativeInteger" }, + "translationPlaceholderValues": { "$ref": "#/$defs/nonNegativeInteger" }, + "danglingRelationships": { "$ref": "#/$defs/nonNegativeInteger" }, + "modelBenchmarkCoverage": { "$ref": "#/$defs/percentage" }, + "productsWithPricing": { "$ref": "#/$defs/nonNegativeInteger" }, + "productRecords": { "$ref": "#/$defs/nonNegativeInteger" }, + "communityUrlsPopulated": { "$ref": "#/$defs/nonNegativeInteger" }, + "communityUrlsWithProvenance": { "$ref": "#/$defs/nonNegativeInteger" }, + "duplicatedVendorCommunityUrls": { "$ref": "#/$defs/nonNegativeInteger" }, + "errors": { "$ref": "#/$defs/nonNegativeInteger" }, + "warnings": { "$ref": "#/$defs/nonNegativeInteger" }, + "info": { "$ref": "#/$defs/nonNegativeInteger" } + }, + "required": [ + "totalRecords", + "recordsWithSources", + "verifiedRecords", + "provenanceComplete", + "staleVerifiedRecords", + "translationPlaceholderValues", + "danglingRelationships", + "modelBenchmarkCoverage", + "productsWithPricing", + "productRecords", + "communityUrlsPopulated", + "communityUrlsWithProvenance", + "duplicatedVendorCommunityUrls", + "errors", + "warnings", + "info" + ], + "additionalProperties": false + }, + "categoryHealth": { + "type": "object", + "properties": { + "total": { "$ref": "#/$defs/nonNegativeInteger" }, + "verified": { "$ref": "#/$defs/nonNegativeInteger" }, + "provenanceComplete": { "$ref": "#/$defs/nonNegativeInteger" }, + "stale": { "$ref": "#/$defs/nonNegativeInteger" } + }, + "required": ["total", "verified", "provenanceComplete", "stale"], + "additionalProperties": false + }, + "categoryHealthMap": { + "type": "object", + "properties": { + "ides": { "$ref": "#/$defs/categoryHealth" }, + "clis": { "$ref": "#/$defs/categoryHealth" }, + "desktops": { "$ref": "#/$defs/categoryHealth" }, + "extensions": { "$ref": "#/$defs/categoryHealth" }, + "models": { "$ref": "#/$defs/categoryHealth" }, + "providers": { "$ref": "#/$defs/categoryHealth" }, + "vendors": { "$ref": "#/$defs/categoryHealth" } + }, + "required": ["ides", "clis", "desktops", "extensions", "models", "providers", "vendors"], + "additionalProperties": false + }, + "translationHealth": { + "type": "object", + "properties": { + "totalStrings": { "$ref": "#/$defs/nonNegativeInteger" }, + "matchingEnglish": { "$ref": "#/$defs/nonNegativeInteger" }, + "matchingEnglishPercent": { "$ref": "#/$defs/percentage" } + }, + "required": ["totalStrings", "matchingEnglish", "matchingEnglishPercent"], + "additionalProperties": false + }, + "issue": { + "type": "object", + "properties": { + "severity": { "type": "string", "enum": ["error", "warning", "info"] }, + "code": { "type": "string", "minLength": 1 }, + "category": { "$ref": "#/$defs/categories" }, + "id": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 } + }, + "required": ["severity", "code", "category", "id", "message"], + "additionalProperties": false + } + } +} diff --git a/data/github-stars.json b/data/github-stars.json index 009bea25..e7b3f14f 100644 --- a/data/github-stars.json +++ b/data/github-stars.json @@ -1,40 +1,40 @@ { - "observedAt": "2026-08-30", + "observedAt": "2026-08-31", "repositories": { - "aaif-goose/goose": 53694, - "anomalyco/opencode": 202526, - "anthropics/claude-code": 143448, + "aaif-goose/goose": 53700, + "anomalyco/opencode": 202538, + "anthropics/claude-code": 143460, "augmentcode/auggie": 276, "aws/amazon-q-developer-cli": 1985, - "can1357/oh-my-pi": 28431, - "cline/cline": 67165, + "can1357/oh-my-pi": 28449, + "cline/cline": 67169, "codota/TabNine": 10774, - "continuedev/continue": 35700, - "cursor/cursor": 33202, - "deepseek-ai/deepseek-harness": 204355, - "earendil-works/pi": 99313, - "esengine/DeepSeek-Reasonix": 35258, + "continuedev/continue": 35701, + "cursor/cursor": 33201, + "deepseek-ai/deepseek-harness": 204470, + "earendil-works/pi": 99340, + "esengine/DeepSeek-Reasonix": 35261, "github/copilot-cli": 11124, "google-antigravity/antigravity-cli": 2081, - "google-gemini/gemini-cli": 106749, - "JetBrains/intellij-community": 20498, + "google-gemini/gemini-cli": 106748, + "JetBrains/intellij-community": 20497, "JetBrains/junie": 414, - "Kilo-Org/kilocode": 27080, + "Kilo-Org/kilocode": 27081, "kirodotdev/Kiro": 4238, - "microsoft/vscode": 190055, + "microsoft/vscode": 190056, "MiniMax-AI/minimax-code": 72, - "mistralai/mistral-vibe": 4895, - "MoonshotAI/kimi-code": 7154, - "openai/codex": 120012, - "QwenLM/qwen-code": 27507, + "mistralai/mistral-vibe": 4896, + "MoonshotAI/kimi-code": 7156, + "openai/codex": 120038, + "QwenLM/qwen-code": 27510, "RooCodeInc/Roo-Code": 24320, "shareAI-lab/Kode-CLI": 5210, "SivanCola/reasonix-vscode": 11, "stagewise-io/stagewise": 6798, "Trae-AI/TRAE": 917, "verdentAI/docs": 4, - "VSCodium/vscodium": 33035, - "xai-org/grok-build": 26250, - "zed-industries/zed": 89451 + "VSCodium/vscodium": 33036, + "xai-org/grok-build": 26251, + "zed-industries/zed": 89459 } } diff --git a/docs/DATA-TRUST.md b/docs/DATA-TRUST.md index 408af8fb..67cce3bf 100644 --- a/docs/DATA-TRUST.md +++ b/docs/DATA-TRUST.md @@ -52,6 +52,10 @@ Every new model catalog entry must have a published Artificial Analysis Intellig ## Freshness policy -Run `pnpm data-health:report` after manifest or translation changes and commit both generated snapshots. CI runs `pnpm data-health:check` to reject invalid or stale snapshots. +`pnpm manifest-data:validate` is the offline validation entry point for schema coverage, semantic relationships, i18n structure, and the committed data-health snapshot. `pnpm manifest-data:check` adds read-only checks against every configured authoritative source. `pnpm manifest-data:update` updates only the explicitly configured safe fields, regenerates derived data and health reports, and reruns the offline validation suite. Source checks can be limited with `--only=`. + +The update harness covers GitHub star snapshots, declared product release versions, exact benchmark mappings, and monitored model-source digests. A pricing or lifecycle source change updates only its digest and observation date; changing the corresponding factual fields still requires review against the cited official source. Any missing, malformed, renamed, or ambiguous upstream result fails the relevant task. + +Run `pnpm data-health:report` after manifest or translation changes made outside the unified update harness and commit both generated snapshots. CI runs `pnpm data-health:check` to reject invalid or stale snapshots. The current review thresholds are 30 days for models and providers, 60 days for IDEs, CLIs, and extensions, and 90 days for vendors. Threshold findings remain warnings so that legacy debt stays visible without blocking unrelated work. Dangling product relationships are errors because they always point to nonexistent records and can break navigation. Network reachability is checked by the separate scheduled URL workflow; source authority and field coverage remain pull-request review responsibilities. diff --git a/docs/GITHUB_SETUP_MANUAL_STEPS.md b/docs/GITHUB_SETUP_MANUAL_STEPS.md index e8a3b030..aef4fdf8 100644 --- a/docs/GITHUB_SETUP_MANUAL_STEPS.md +++ b/docs/GITHUB_SETUP_MANUAL_STEPS.md @@ -172,7 +172,7 @@ This is the simplest method - Cloudflare handles everything automatically. - None required for basic setup 5. **Advanced Settings**: - - Node.js version: `20` + - Node.js version: `24` - Compatibility date: `2025-03-01` (or latest) 6. **Click "Save and Deploy"** diff --git a/manifests/$schemas/github-stars.schema.json b/manifests/$schemas/github-stars.schema.json index af4cceb6..870855ee 100644 --- a/manifests/$schemas/github-stars.schema.json +++ b/manifests/$schemas/github-stars.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://aicodingstack.io/schemas/github-stars.schema.json", "title": "GitHub Repository Stars Data", "description": "Repository-keyed GitHub star snapshots. Product associations and source-code status are derived from product manifests.", diff --git a/manifests/$schemas/mapping.schema.json b/manifests/$schemas/mapping.schema.json new file mode 100644 index 00000000..8d1220d7 --- /dev/null +++ b/manifests/$schemas/mapping.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "External Manifest ID Mapping", + "description": "Explicit vendor and model identifier mappings used by external benchmark comparisons.", + "type": "object", + "properties": { + "vendors": { "$ref": "#/$defs/mapping" }, + "models": { "$ref": "#/$defs/mapping" } + }, + "required": ["vendors", "models"], + "additionalProperties": false, + "$defs": { + "mapping": { + "type": "object", + "propertyNames": { "minLength": 1 }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/manifests/$schemas/ref/product.schema.json b/manifests/$schemas/ref/product.schema.json index e15041b9..1484d058 100644 --- a/manifests/$schemas/ref/product.schema.json +++ b/manifests/$schemas/ref/product.schema.json @@ -149,6 +149,11 @@ "identifier": { "type": "string", "pattern": "^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$" + }, + "tagPrefix": { + "type": "string", + "minLength": 1, + "description": "Exact GitHub release tag prefix removed before updating latestVersion" } }, "required": ["provider", "identifier"], diff --git a/manifests/clis/amp-cli.json b/manifests/clis/amp-cli.json index 4c5c0690..7f05f373 100644 --- a/manifests/clis/amp-cli.json +++ b/manifests/clis/amp-cli.json @@ -45,7 +45,7 @@ "websiteUrl": "https://ampcode.com", "docsUrl": "https://ampcode.com", "vendor": "Sourcegraph", - "latestVersion": "0.0.1785488326-gdfd462", + "latestVersion": "0.0.1788105629-g37d5d1", "releaseTracking": { "provider": "npm", "identifier": "@ampcode/cli" diff --git a/manifests/clis/auggie-cli.json b/manifests/clis/auggie-cli.json index 0a50653d..0d0e3cb9 100644 --- a/manifests/clis/auggie-cli.json +++ b/manifests/clis/auggie-cli.json @@ -45,7 +45,7 @@ "websiteUrl": "https://www.augmentcode.com", "docsUrl": "https://docs.augmentcode.com/cli", "vendor": "Augment", - "latestVersion": "0.34.0", + "latestVersion": "0.36.0", "releaseTracking": { "provider": "npm", "identifier": "@augmentcode/auggie" diff --git a/manifests/clis/claude-code-cli.json b/manifests/clis/claude-code-cli.json index 297193f2..926574c2 100644 --- a/manifests/clis/claude-code-cli.json +++ b/manifests/clis/claude-code-cli.json @@ -108,7 +108,7 @@ "websiteUrl": "https://code.claude.com", "docsUrl": "https://code.claude.com/docs", "vendor": "Anthropic", - "latestVersion": "2.1.220", + "latestVersion": "2.1.251", "releaseTracking": { "provider": "npm", "identifier": "@anthropic-ai/claude-code" diff --git a/manifests/clis/cline-cli.json b/manifests/clis/cline-cli.json index 639fac08..4ec908b4 100644 --- a/manifests/clis/cline-cli.json +++ b/manifests/clis/cline-cli.json @@ -45,7 +45,7 @@ "websiteUrl": "https://cline.bot/cline-cli", "docsUrl": "https://cline.bot/docs", "vendor": "Cline Bot", - "latestVersion": "3.0.46", + "latestVersion": "3.0.60", "releaseTracking": { "provider": "npm", "identifier": "cline" diff --git a/manifests/clis/codebuddy-cli.json b/manifests/clis/codebuddy-cli.json index 6a56cf4b..12b46ae5 100644 --- a/manifests/clis/codebuddy-cli.json +++ b/manifests/clis/codebuddy-cli.json @@ -46,7 +46,7 @@ "websiteUrl": "https://www.codebuddy.ai/cli", "docsUrl": "https://www.codebuddy.ai/docs/cli", "vendor": "Tencent", - "latestVersion": "2.131.0", + "latestVersion": "2.142.0", "releaseTracking": { "provider": "npm", "identifier": "@tencent-ai/codebuddy-code" diff --git a/manifests/clis/codex-cli.json b/manifests/clis/codex-cli.json index 5bdca8df..05e437c5 100644 --- a/manifests/clis/codex-cli.json +++ b/manifests/clis/codex-cli.json @@ -46,7 +46,7 @@ "websiteUrl": "https://openai.com/codex", "docsUrl": "https://developers.openai.com/codex/cli", "vendor": "OpenAI", - "latestVersion": "0.146.0", + "latestVersion": "0.151.0", "releaseTracking": { "provider": "npm", "identifier": "@openai/codex" diff --git a/manifests/clis/cursor-cli.json b/manifests/clis/cursor-cli.json index 7479b83c..558cf26e 100644 --- a/manifests/clis/cursor-cli.json +++ b/manifests/clis/cursor-cli.json @@ -45,7 +45,7 @@ "websiteUrl": "https://cursor.com/cli", "docsUrl": "https://cursor.com/docs/cli/overview", "vendor": "Anysphere", - "latestVersion": "2026.07.23-e383d2b", + "latestVersion": "2026.08.25-3e8eec8", "releaseTracking": { "provider": "homebrew-cask", "identifier": "cursor-cli" diff --git a/manifests/clis/gemini-cli.json b/manifests/clis/gemini-cli.json index 003c9a24..224b29d1 100644 --- a/manifests/clis/gemini-cli.json +++ b/manifests/clis/gemini-cli.json @@ -83,7 +83,7 @@ "websiteUrl": "https://geminicli.com", "docsUrl": "https://geminicli.com/docs", "vendor": "Google", - "latestVersion": "0.52.0", + "latestVersion": "0.57.0", "releaseTracking": { "provider": "npm", "identifier": "@google/gemini-cli" diff --git a/manifests/clis/github-copilot-cli.json b/manifests/clis/github-copilot-cli.json index c3a4ba8c..2e6585a3 100644 --- a/manifests/clis/github-copilot-cli.json +++ b/manifests/clis/github-copilot-cli.json @@ -45,7 +45,7 @@ "websiteUrl": "https://github.com/features/copilot/cli", "docsUrl": "https://docs.github.com/en/copilot/concepts/agents/about-copilot-cli", "vendor": "GitHub", - "latestVersion": "1.0.75", + "latestVersion": "1.0.82", "releaseTracking": { "provider": "npm", "identifier": "@github/copilot" diff --git a/manifests/clis/gitlab-duo-cli.json b/manifests/clis/gitlab-duo-cli.json index 61907a6b..2479ea6a 100644 --- a/manifests/clis/gitlab-duo-cli.json +++ b/manifests/clis/gitlab-duo-cli.json @@ -87,7 +87,7 @@ "websiteUrl": "https://about.gitlab.com/gitlab-duo", "docsUrl": "https://docs.gitlab.com/user/gitlab_duo_cli/", "vendor": "GitLab", - "latestVersion": "9.6.0", + "latestVersion": "9.16.0", "releaseTracking": { "provider": "npm", "identifier": "@gitlab/duo-cli" diff --git a/manifests/clis/goose.json b/manifests/clis/goose.json index 4b6995ad..06b244ef 100644 --- a/manifests/clis/goose.json +++ b/manifests/clis/goose.json @@ -46,7 +46,7 @@ "websiteUrl": "https://goose-docs.ai", "docsUrl": "https://goose-docs.ai/docs/getting-started/installation", "vendor": "Agentic AI Foundation", - "latestVersion": "1.45.0", + "latestVersion": "1.48.0", "releaseTracking": { "provider": "github-release", "identifier": "aaif-goose/goose" diff --git a/manifests/clis/kilo-code-cli.json b/manifests/clis/kilo-code-cli.json index b5bad588..06bdac36 100644 --- a/manifests/clis/kilo-code-cli.json +++ b/manifests/clis/kilo-code-cli.json @@ -42,7 +42,7 @@ "websiteUrl": "https://kilo.ai", "docsUrl": "https://kilo.ai/docs/code-with-ai/platforms/cli", "vendor": "Kilo", - "latestVersion": "7.4.16", + "latestVersion": "7.5.6", "releaseTracking": { "provider": "npm", "identifier": "@kilocode/cli" diff --git a/manifests/clis/kimi-cli.json b/manifests/clis/kimi-cli.json index af78744a..b40aef18 100644 --- a/manifests/clis/kimi-cli.json +++ b/manifests/clis/kimi-cli.json @@ -90,7 +90,7 @@ "websiteUrl": "https://www.kimi.com/code", "docsUrl": "https://www.kimi.com/code/docs/en/kimi-code-cli/guides/getting-started.html", "vendor": "Moonshot AI", - "latestVersion": "0.29.1", + "latestVersion": "0.39.1", "releaseTracking": { "provider": "npm", "identifier": "@moonshot-ai/kimi-code" diff --git a/manifests/clis/omp.json b/manifests/clis/omp.json index 728070a2..f7370d5f 100644 --- a/manifests/clis/omp.json +++ b/manifests/clis/omp.json @@ -73,7 +73,7 @@ "websiteUrl": "https://omp.sh", "docsUrl": "https://omp.sh/docs", "vendor": "Oh My Pi", - "latestVersion": "17.1.7", + "latestVersion": "18.0.11", "releaseTracking": { "provider": "npm", "identifier": "@oh-my-pi/pi-coding-agent" diff --git a/manifests/clis/opencode.json b/manifests/clis/opencode.json index 5b90f08e..8b35da8a 100644 --- a/manifests/clis/opencode.json +++ b/manifests/clis/opencode.json @@ -46,7 +46,7 @@ "websiteUrl": "https://opencode.ai", "docsUrl": "https://opencode.ai/docs", "vendor": "Anomaly", - "latestVersion": "1.18.10", + "latestVersion": "1.18.25", "releaseTracking": { "provider": "npm", "identifier": "opencode-ai" diff --git a/manifests/clis/pi.json b/manifests/clis/pi.json index 60a2111e..89184670 100644 --- a/manifests/clis/pi.json +++ b/manifests/clis/pi.json @@ -75,7 +75,7 @@ "websiteUrl": "https://pi.dev", "docsUrl": "https://pi.dev/docs/latest", "vendor": "Earendil Inc.", - "latestVersion": "0.83.0", + "latestVersion": "0.84.4", "releaseTracking": { "provider": "npm", "identifier": "@earendil-works/pi-coding-agent" diff --git a/manifests/clis/qoder-cli.json b/manifests/clis/qoder-cli.json index ab6346a6..77a8bcc8 100644 --- a/manifests/clis/qoder-cli.json +++ b/manifests/clis/qoder-cli.json @@ -46,7 +46,7 @@ "websiteUrl": "https://qoder.com/cli", "docsUrl": "https://docs.qoder.com/en/cli/quick-start", "vendor": "Alibaba", - "latestVersion": "1.1.7", + "latestVersion": "1.1.37", "releaseTracking": { "provider": "npm", "identifier": "@qoder-ai/qodercli" diff --git a/manifests/clis/qwen-code.json b/manifests/clis/qwen-code.json index cd0120e5..3eec9be4 100644 --- a/manifests/clis/qwen-code.json +++ b/manifests/clis/qwen-code.json @@ -87,7 +87,7 @@ "websiteUrl": "https://qwenlm.github.io/qwen-code-docs/en/", "docsUrl": "https://qwenlm.github.io/qwen-code-docs/en/", "vendor": "Alibaba", - "latestVersion": "0.21.2", + "latestVersion": "0.22.3", "releaseTracking": { "provider": "npm", "identifier": "@qwen-code/qwen-code" diff --git a/manifests/clis/vibe-cli.json b/manifests/clis/vibe-cli.json index bf73fff0..ec46a760 100644 --- a/manifests/clis/vibe-cli.json +++ b/manifests/clis/vibe-cli.json @@ -67,7 +67,7 @@ "websiteUrl": "https://mistral.ai/products/vibe", "docsUrl": "https://docs.mistral.ai/vibe/code/overview", "vendor": "Mistral AI", - "latestVersion": "2.23.0", + "latestVersion": "2.24.5", "releaseTracking": { "provider": "pypi", "identifier": "mistral-vibe" diff --git a/manifests/desktops/goose.json b/manifests/desktops/goose.json index 0cffccb5..9eed56e8 100644 --- a/manifests/desktops/goose.json +++ b/manifests/desktops/goose.json @@ -46,7 +46,7 @@ "websiteUrl": "https://goose-docs.ai", "docsUrl": "https://goose-docs.ai/docs/getting-started/installation", "vendor": "Agentic AI Foundation", - "latestVersion": "1.45.0", + "latestVersion": "1.48.0", "releaseTracking": { "provider": "github-release", "identifier": "aaif-goose/goose" diff --git a/manifests/desktops/opencode-desktop.json b/manifests/desktops/opencode-desktop.json index cafce16e..b8ff088f 100644 --- a/manifests/desktops/opencode-desktop.json +++ b/manifests/desktops/opencode-desktop.json @@ -83,7 +83,7 @@ "websiteUrl": "https://opencode.ai/download", "docsUrl": "https://opencode.ai/docs", "vendor": "Anomaly", - "latestVersion": "1.18.10", + "latestVersion": "1.18.25", "releaseTracking": { "provider": "github-release", "identifier": "anomalyco/opencode" diff --git a/manifests/desktops/reasonix-desktop.json b/manifests/desktops/reasonix-desktop.json index 3b2a6208..144880f0 100644 --- a/manifests/desktops/reasonix-desktop.json +++ b/manifests/desktops/reasonix-desktop.json @@ -81,7 +81,8 @@ "latestVersion": "1.34.0", "releaseTracking": { "provider": "github-release", - "identifier": "esengine/DeepSeek-Reasonix" + "identifier": "esengine/DeepSeek-Reasonix", + "tagPrefix": "desktop-" }, "githubUrl": "https://github.com/esengine/DeepSeek-Reasonix", "license": "MIT", diff --git a/manifests/extensions/claude-code.json b/manifests/extensions/claude-code.json index dcc59945..adfffc40 100644 --- a/manifests/extensions/claude-code.json +++ b/manifests/extensions/claude-code.json @@ -103,7 +103,7 @@ "websiteUrl": "https://code.claude.com", "docsUrl": "https://code.claude.com/docs/en/ide-integrations", "vendor": "Anthropic", - "latestVersion": "2.1.220", + "latestVersion": "2.1.251", "releaseTracking": { "provider": "vscode-marketplace", "identifier": "anthropic.claude-code" diff --git a/manifests/extensions/codex.json b/manifests/extensions/codex.json index 924bac22..a56de547 100644 --- a/manifests/extensions/codex.json +++ b/manifests/extensions/codex.json @@ -73,7 +73,7 @@ "websiteUrl": "https://openai.com/codex", "docsUrl": "https://developers.openai.com/codex/ide", "vendor": "OpenAI", - "latestVersion": "26.5721.30844", + "latestVersion": "26.5825.51511", "releaseTracking": { "provider": "vscode-marketplace", "identifier": "openai.chatgpt" diff --git a/manifests/extensions/kimi-code.json b/manifests/extensions/kimi-code.json index fe2ae87a..7b5bd2a4 100644 --- a/manifests/extensions/kimi-code.json +++ b/manifests/extensions/kimi-code.json @@ -78,7 +78,7 @@ "websiteUrl": "https://www.kimi.com/code", "docsUrl": "https://www.kimi.com/code/docs/en", "vendor": "Moonshot AI", - "latestVersion": "0.6.6", + "latestVersion": "0.7.3", "releaseTracking": { "provider": "vscode-marketplace", "identifier": "moonshot-ai.kimi-code" diff --git a/manifests/extensions/mistral-vibe.json b/manifests/extensions/mistral-vibe.json index b0f0a7ae..acd9bc0e 100644 --- a/manifests/extensions/mistral-vibe.json +++ b/manifests/extensions/mistral-vibe.json @@ -66,7 +66,7 @@ "websiteUrl": "https://mistral.ai/products/vibe", "docsUrl": "https://docs.mistral.ai/vibe/code/overview", "vendor": "Mistral AI", - "latestVersion": "1.14.4", + "latestVersion": "1.18.35", "releaseTracking": { "provider": "vscode-marketplace", "identifier": "mistralai.mistral-vibe-code" diff --git a/manifests/models/claude-haiku-4-5.json b/manifests/models/claude-haiku-4-5.json index 10685e10..a92a118e 100644 --- a/manifests/models/claude-haiku-4-5.json +++ b/manifests/models/claude-haiku-4-5.json @@ -60,8 +60,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:2ff7f541d89a824c7125b0cfab826e4b839e85f2ded2613c746415903266274b", - "observedAt": "2026-07-28" + "digest": "sha256:c5ac65b658f8d597d7d37be868d13b7e7e218f5dfec8db0cf27f8da8b4295d56", + "observedAt": "2026-08-31" } }, { diff --git a/manifests/models/claude-opus-4.json b/manifests/models/claude-opus-4.json index 25636e2e..e5932342 100644 --- a/manifests/models/claude-opus-4.json +++ b/manifests/models/claude-opus-4.json @@ -46,8 +46,8 @@ "fields": ["lifecycle"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:af2e27075e6c7826cf4d7ef23f722bcbf0406d5bc0adaa0facf07e3b92b9c0b0", - "observedAt": "2026-07-28" + "digest": "sha256:57bfce01949cbe5b61945eccc4ee7462e1d812d7b1e810ec4742c50aac0866fc", + "observedAt": "2026-08-31" } } ], diff --git a/manifests/models/deepseek-3-2.json b/manifests/models/deepseek-3-2.json index aeca8186..bfee2e0d 100644 --- a/manifests/models/deepseek-3-2.json +++ b/manifests/models/deepseek-3-2.json @@ -63,8 +63,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:d1c1fc8b58ad75475827f978beac79318391da95b942891f47a8aafe811712ba", - "observedAt": "2026-07-28" + "digest": "sha256:79e449df8dc23f7d70b50540c21fdc130cd423a34782cf25e9b8969fb11b4c57", + "observedAt": "2026-08-31" } } ], diff --git a/manifests/models/gemini-3-5-flash.json b/manifests/models/gemini-3-5-flash.json index a6ba7155..28eb1a06 100644 --- a/manifests/models/gemini-3-5-flash.json +++ b/manifests/models/gemini-3-5-flash.json @@ -59,8 +59,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:aab4d993819344eb390d8e59a37ab5d9712daca3ed395b4271bbebb586bb0c50", - "observedAt": "2026-07-28" + "digest": "sha256:6d77a163cd08ce5c37483ece822e423a881812afc7b6ec62e7c5b1f25f50b382", + "observedAt": "2026-08-31" } }, { @@ -69,8 +69,8 @@ "fields": ["name", "description", "releaseDate", "lifecycle"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:d903a1d48cc08625d5514f5f0d800daed7c207f46cfa33dcef4dcbb98ab16065", - "observedAt": "2026-07-28" + "digest": "sha256:28edf3d7f0b86e889c6af8cc9e348c748b569f693025b2afd8077f2be40ecaa3", + "observedAt": "2026-08-31" } } ], diff --git a/manifests/models/glm-4-6.json b/manifests/models/glm-4-6.json index a165a8e2..36370511 100644 --- a/manifests/models/glm-4-6.json +++ b/manifests/models/glm-4-6.json @@ -92,7 +92,7 @@ "benchmark": "sweBench", "leaderboard": "Verified", "resultId": "20251201_mini-v1.17.1_glm-4.6", - "modelLabel": "mini-SWE-agent + GLM-4.6 (T=1)" + "modelLabel": "mini-SWE-agent + GLM 4.6 (T=1)" } ], "platformUrls": { diff --git a/manifests/models/glm-5-2.json b/manifests/models/glm-5-2.json index 52c73781..966b2c0f 100644 --- a/manifests/models/glm-5-2.json +++ b/manifests/models/glm-5-2.json @@ -60,8 +60,8 @@ "fields": ["releaseDate", "lifecycle"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:5b5ab3620167e1331f3de0aeeefe0cb5df12d59e706e0228bac5dcab6248a6fe", - "observedAt": "2026-07-28" + "digest": "sha256:45d3e8f46ca27e4fdb156a8615eee82a917f3440e4433843de48d4229094b11c", + "observedAt": "2026-08-31" } }, { @@ -70,8 +70,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:3f331fe3d6f5ed696378e4c29c6ce553f463417299d5aa2d222ad8b48fe3ec7a", - "observedAt": "2026-07-28" + "digest": "sha256:7bb2e488e6b0663aff9e62286fa9450c6e3b4c378da94fbeb660fef11e00b52a", + "observedAt": "2026-08-31" } }, { diff --git a/manifests/models/gpt-4-1.json b/manifests/models/gpt-4-1.json index b8911287..393cdda7 100644 --- a/manifests/models/gpt-4-1.json +++ b/manifests/models/gpt-4-1.json @@ -92,7 +92,7 @@ "benchmark": "sweBench", "leaderboard": "Verified", "resultId": "20250726_mini-v1.0.0_gpt-4.1-2025-04-14", - "modelLabel": "mini-SWE-agent + GPT-4.1 (2025-04-14)" + "modelLabel": "mini-SWE-agent + GPT 4.1 (2025-04-14)" } ], "platformUrls": { diff --git a/manifests/models/gpt-4o.json b/manifests/models/gpt-4o.json index 32571d58..b8aa2824 100644 --- a/manifests/models/gpt-4o.json +++ b/manifests/models/gpt-4o.json @@ -92,7 +92,7 @@ "benchmark": "sweBench", "leaderboard": "Verified", "resultId": "20250720_mini-v0.0.0-gpt-4o-2024-11-20", - "modelLabel": "mini-SWE-agent + GPT-4o (2024-11-20)" + "modelLabel": "mini-SWE-agent + GPT 4o (2024-11-20)" } ], "platformUrls": { diff --git a/manifests/models/gpt-5-1.json b/manifests/models/gpt-5-1.json index 43df9d1a..a764fd67 100644 --- a/manifests/models/gpt-5-1.json +++ b/manifests/models/gpt-5-1.json @@ -92,7 +92,7 @@ "benchmark": "sweBench", "leaderboard": "Verified", "resultId": "20251120_mini-v1.15.0_gpt-5.1-2025-11-13", - "modelLabel": "mini-SWE-agent + GPT-5.1 (2025-11-13) (medium reasoning)" + "modelLabel": "mini-SWE-agent + GPT 5.1 (2025-11-13) (medium)" } ], "platformUrls": { diff --git a/manifests/models/gpt-5-6-sol.json b/manifests/models/gpt-5-6-sol.json index 51ffc8f6..20a933a6 100644 --- a/manifests/models/gpt-5-6-sol.json +++ b/manifests/models/gpt-5-6-sol.json @@ -55,8 +55,8 @@ ], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:a8689d63a32710a7b7c1ecb3b7898d57820ce6de6a68f5bec671e606685af349", - "observedAt": "2026-07-28" + "digest": "sha256:0e4bc4ec6f9aee2372dbddd89206a5f051ca5e2730045605abcedbb5ad944e67", + "observedAt": "2026-08-31" } }, { diff --git a/manifests/models/grok-4-20.json b/manifests/models/grok-4-20.json index 4f30ea7b..535127a5 100644 --- a/manifests/models/grok-4-20.json +++ b/manifests/models/grok-4-20.json @@ -60,8 +60,8 @@ ], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:ce3ca747e8bec4e13b038be6f1c3ccfafc8a0ec7d70c12ccada9bb27c4fa9ed6", - "observedAt": "2026-07-28" + "digest": "sha256:121ffa3bfbfddb68360e9b3f963bfdb00d7d993aa140f2192df2c5bafe2c1963", + "observedAt": "2026-08-31" } } ], diff --git a/manifests/models/kimi-k2-thinking.json b/manifests/models/kimi-k2-thinking.json index 96a86022..a07a4136 100644 --- a/manifests/models/kimi-k2-thinking.json +++ b/manifests/models/kimi-k2-thinking.json @@ -46,8 +46,8 @@ "fields": ["lifecycle", "tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:50692812b00a1b51ddd80947258f053d2317be53e34384d7421d2e9034d3b60d", - "observedAt": "2026-07-28" + "digest": "sha256:e7c3efc9e11c627ba1f37cc5cb70599218f12c46753d6340a7d21d4b6a1a6211", + "observedAt": "2026-08-31" } }, { diff --git a/manifests/models/minimax-m2-7.json b/manifests/models/minimax-m2-7.json index c75e02fa..874ffda9 100644 --- a/manifests/models/minimax-m2-7.json +++ b/manifests/models/minimax-m2-7.json @@ -56,8 +56,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:40b4c27c5a485ab71d531141a4f328d3bff0f1c2eb6c9d31103523119aadaae3", - "observedAt": "2026-07-28" + "digest": "sha256:daf9a7fb3784157fac614302966888fa29c4131df05cb40e308d594420aa924f", + "observedAt": "2026-08-31" } }, { diff --git a/manifests/models/minimax-m2.json b/manifests/models/minimax-m2.json index 6189c02c..87c9a11d 100644 --- a/manifests/models/minimax-m2.json +++ b/manifests/models/minimax-m2.json @@ -92,7 +92,7 @@ "benchmark": "sweBench", "leaderboard": "Verified", "resultId": "20251124_mini-v1.17.0_minimax-m2", - "modelLabel": "mini-SWE-agent + Minimax M2" + "modelLabel": "mini-SWE-agent + MiniMax M2" } ], "platformUrls": { diff --git a/manifests/models/mistral-small-4.json b/manifests/models/mistral-small-4.json index 43a7533a..7d3ce89c 100644 --- a/manifests/models/mistral-small-4.json +++ b/manifests/models/mistral-small-4.json @@ -67,8 +67,8 @@ "fields": ["tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:740ecd45540ba01718b7682233cefd2f5b1f4dae7d40a16887e750dc9a0b84f0", - "observedAt": "2026-07-28" + "digest": "sha256:a5d011e375586b5307459881c04ebf3e488f0a8787c952ca957f23ea75bd4e9e", + "observedAt": "2026-08-31" } } ], diff --git a/manifests/models/qwen3-7-max.json b/manifests/models/qwen3-7-max.json index 91d214c8..1f257757 100644 --- a/manifests/models/qwen3-7-max.json +++ b/manifests/models/qwen3-7-max.json @@ -54,8 +54,8 @@ ], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:1d3b2eb8b5a808acf23c0f5183039c4fd5056eca009e35c4a896a3661a82c108", - "observedAt": "2026-07-28" + "digest": "sha256:75d7cba8e521c482e1a8939f0c53461dd83cd8f31fda9bf2ab21c74855223184", + "observedAt": "2026-08-31" } }, { @@ -64,8 +64,8 @@ "fields": ["docsUrl", "contextWindow", "tokenPricing"], "changeTracking": { "method": "normalized-content-sha256", - "digest": "sha256:ea84e2dc7741e6a43646a6c2576e6c78384434821fac0d1774a79993cbc85023", - "observedAt": "2026-07-28" + "digest": "sha256:9aa3ea3ebcecdcca5747c28efc6cd5f8dab9ed564d0f03e6bf8a0cdf2a64ecc7", + "observedAt": "2026-08-31" } }, { diff --git a/package.json b/package.json index 787911a5..39a65837 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "test:ci": "vitest run --reporter=verbose", "test:e2e": "playwright test", "test:e2e:ci": "playwright test --reporter=line", + "manifest-data:validate": "tsx scripts/manifest-data/index.ts validate", + "manifest-data:check": "tsx scripts/manifest-data/index.ts check", + "manifest-data:update": "tsx scripts/manifest-data/index.ts update", "test:validate": "vitest run tests/validate --reporter=verbose", "test:urls": "RUN_URL_TESTS=1 vitest run tests/validate/urls.accessibility.test.ts --reporter=verbose", "validate:i18n": "tsx scripts/validate/validate-i18n.ts", @@ -44,6 +47,7 @@ "refactor:sort-fields": "tsx scripts/refactor/index.ts sort-manifest-fields", "fetch": "tsx scripts/fetch/index.ts", "fetch:github-stars": "tsx scripts/fetch/index.ts github-stars", + "fetch:github-stars:check": "tsx scripts/fetch/fetch-github-stars.ts --check", "fetch:product-versions": "tsx scripts/fetch/fetch-product-versions.ts --write", "fetch:product-versions:check": "tsx scripts/fetch/fetch-product-versions.ts --check", "fetch:benchmarks": "tsx scripts/fetch/fetch-benchmarks.ts --write", @@ -84,7 +88,7 @@ "@playwright/test": "^1.62.1", "@tailwindcss/postcss": "^4.3.3", "@types/mdx": "^2.0.14", - "@types/node": "^22.20.1", + "@types/node": "^24.12.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.5", "@typescript-eslint/parser": "^8.68.0", @@ -116,7 +120,7 @@ }, "homepage": "https://github.com/aicodingstack/aicodingstack.io#readme", "engines": { - "node": "22" + "node": "24" }, "overrides": { "@node-minify/core": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c3bfc055..46569301 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,8 +90,8 @@ importers: specifier: ^2.0.14 version: 2.0.14 '@types/node': - specifier: ^22.20.1 - version: 22.20.1 + specifier: ^24.12.0 + version: 24.13.3 '@types/react': specifier: ^19.2.18 version: 19.2.18 @@ -145,7 +145,7 @@ importers: version: 1.1.675 vitest: specifier: ^4.1.11 - version: 4.1.11(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)) + version: 4.1.11(@types/node@24.13.3)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)) wrangler: specifier: ^4.127.0 version: 4.127.0 @@ -2467,8 +2467,8 @@ packages: '@types/node@18.19.130': resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} - '@types/node@22.20.1': - resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==} + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} '@types/react-dom@19.2.5': resolution: {integrity: sha512-fMPwH9v7r/pp43yUd2/Mbiex5KouJwwR3dzHkhLREUC6764VyDsqxhAxv6OFEYR1RhjOyD1naqba8ECDBe7ZQg==} @@ -4424,8 +4424,8 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.21.0: - resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} undici@7.29.0: resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} @@ -6930,16 +6930,16 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 22.20.1 + '@types/node': 24.13.3 form-data: 4.0.6 '@types/node@18.19.130': dependencies: undici-types: 5.26.5 - '@types/node@22.20.1': + '@types/node@24.13.3': dependencies: - undici-types: 6.21.0 + undici-types: 7.18.2 '@types/react-dom@19.2.5(@types/react@19.2.18)': dependencies: @@ -7018,13 +7018,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.11(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0))': + '@vitest/mocker@4.1.11(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0) + vite: 8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0) '@vitest/pretty-format@4.1.11': dependencies: @@ -9360,7 +9360,7 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.21.0: {} + undici-types@7.18.2: {} undici@7.29.0: {} @@ -9476,7 +9476,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0): + vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.7 @@ -9484,7 +9484,7 @@ snapshots: rolldown: 1.1.5 tinyglobby: 0.2.17 optionalDependencies: - '@types/node': 22.20.1 + '@types/node': 24.13.3 esbuild: 0.28.2 fsevents: 2.3.3 jiti: 2.7.0 @@ -9492,10 +9492,10 @@ snapshots: tsx: 4.23.12 yaml: 2.9.0 - vitest@4.1.11(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)): + vitest@4.1.11(@types/node@24.13.3)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)) + '@vitest/mocker': 4.1.11(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -9512,10 +9512,10 @@ snapshots: tinyexec: 1.3.0 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.1.5(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0) + vite: 8.1.5(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.16.9)(tsx@4.23.12)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 22.20.1 + '@types/node': 24.13.3 transitivePeerDependencies: - msw diff --git a/scripts/README.md b/scripts/README.md index e4df1510..7d21a6ec 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -11,13 +11,17 @@ Repository automation is written in TypeScript and executed with `tsx`. Prefer t | Generate content metadata | `pnpm generate:metadata` | Rebuilds article, docs, FAQ, and manifesto metadata | | Sort manifest fields | `pnpm refactor:sort-fields` | Reorders manifest JSON using schema order | | Fetch GitHub stars | `pnpm fetch:github-stars` | Updates `data/github-stars.json` | +| Check GitHub stars | `pnpm fetch:github-stars:check` | Fails when a tracked repository's star count changed, without writing | | Refresh product versions | `pnpm fetch:product-versions` | Updates manifest `latestVersion` values from configured package registries | | Check product versions | `pnpm fetch:product-versions:check` | Fails when a configured registry reports a newer stable version | | Refresh benchmark scores | `pnpm fetch:benchmarks` | Updates exact configured entries from official structured leaderboards | | Check benchmark scores | `pnpm fetch:benchmarks:check` | Fails when an exact configured leaderboard score changed | | Record model source changes | `pnpm fetch:model-sources` | Updates only monitored official-source content digests | | Check model sources | `pnpm fetch:model-sources:check` | Fails when monitored pricing or lifecycle source content changed | -| Validate manifests and data | `pnpm test:validate` | Runs the validation test suite | +| Validate manifest data locally | `pnpm manifest-data:validate` | Runs schema coverage, semantic validation, i18n validation, and the data-health snapshot check | +| Check manifest data and sources | `pnpm manifest-data:check` | Runs local validation and read-only drift checks against every configured authoritative source | +| Update manifest data | `pnpm manifest-data:update` | Refreshes safe source-backed fields, regenerates derived data and health reports, then validates the result | +| Validate manifests and data | `pnpm test:validate` | Runs the underlying semantic validation test suite | | Run browser smoke tests | `pnpm test:e2e` | Exercises core journeys in Chromium against a local Next.js server | | Audit dependencies | `pnpm security:audit` | Fails on high or critical npm advisories in the full dependency tree | | Check data health | `pnpm data-health:check` | Fails on invalid health-report data | @@ -28,7 +32,18 @@ Repository automation is written in TypeScript and executed with `tsx`. Prefer t | Check representative URLs | `pnpm validate:urls:quick` | Runs the default network URL check | | Check every URL | `pnpm validate:urls:all` | Checks all locales and slugs | -Networked commands can fail because an external service is unavailable. CI keeps URL validation separate from deterministic manifest, type, and build checks. +`manifest-data:validate` is deterministic and offline. `manifest-data:check` and +`manifest-data:update` access external sources and can fail when a source is unavailable. Limit a +networked run with `--only`, for example: + +```bash +pnpm manifest-data:check -- --only=product-versions,benchmarks +pnpm manifest-data:update -- --only=github-stars +``` + +Add `--json` for a compact machine-readable report. Successful subprocess output is suppressed; +full output is included only when a validation step fails. This keeps routine agent and CI runs +concise while retaining diagnostic evidence. ## Layout @@ -37,6 +52,7 @@ scripts/ ├── _shared/ shared runner utilities ├── fetch/ external data refreshes and comparison helpers ├── generate/ derived TypeScript source generation +├── manifest-data/ unified validation, source-check, and update harness ├── refactor/ mechanical manifest and locale maintenance ├── validate/ i18n and URL validation └── temp/ ignored experimental workspace @@ -65,6 +81,12 @@ Benchmark synchronization also keeps model-to-result mappings in JSON manifests. Pricing and lifecycle monitoring fingerprints only explicitly configured official source pages. A source-content change updates the source digest in a review pull request; it never modifies `tokenPricing`, `lifecycle`, or other model facts automatically. +The unified maintenance harness includes four source-backed tasks: GitHub star snapshots, declared +product release versions, exact configured benchmark mappings, and monitored official model-source +digests. Each source task fails closed when upstream data is missing or ambiguous. Updates preserve +curated relationships, translations, rankings, pricing, and lifecycle facts; the model-source task +records a review signal instead of changing those facts automatically. + ## Adding a script 1. Add a TypeScript file to the relevant category. diff --git a/scripts/fetch/compare-models.ts b/scripts/fetch/compare-models.ts index cd2ffbfb..38b00dc0 100644 --- a/scripts/fetch/compare-models.ts +++ b/scripts/fetch/compare-models.ts @@ -3,6 +3,7 @@ import { readdir, readFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' +import type { ManifestMapping } from '../../src/types/manifests' const __dirname = dirname(fileURLToPath(import.meta.url)) const manifestsDir = join(__dirname, '../../manifests/models') @@ -75,11 +76,6 @@ interface ApiData { } } -interface MappingData { - vendors: Record - models: Record -} - function getPrimaryManifestRates(manifest: { tokenPricing?: { status?: string @@ -176,7 +172,7 @@ async function main(): Promise { const apiData = JSON.parse(await readFile(apiDataFile, 'utf-8')) as ApiData const { vendors: vendorMapping, models: modelMapping } = JSON.parse( await readFile(mappingFile, 'utf-8') - ) as MappingData + ) as ManifestMapping // Read all model manifests const files = await readdir(manifestsDir) diff --git a/scripts/fetch/fetch-github-stars.ts b/scripts/fetch/fetch-github-stars.ts index a855b089..b09c361f 100644 --- a/scripts/fetch/fetch-github-stars.ts +++ b/scripts/fetch/fetch-github-stars.ts @@ -1,84 +1,48 @@ -import fs from 'node:fs' -import https from 'node:https' -import path, { dirname } from 'node:path' -import { fileURLToPath } from 'node:url' +#!/usr/bin/env node -const __filename = fileURLToPath(import.meta.url) -const __dirname = dirname(__filename) -const GITHUB_TOKEN = process.env.GITHUB_TOKEN -const GITHUB_STARS_FILE = path.join(__dirname, '..', '..', 'data', 'github-stars.json') +import { syncGithubStars } from './lib/github-stars-sync' -type StarsData = { - observedAt: string - repositories: Record +function hasFlag(flag: string): boolean { + return process.argv.slice(2).includes(flag) } -function fetchStars(repositoryId: string): Promise { - return new Promise((resolve, reject) => { - const options: https.RequestOptions = { - hostname: 'api.github.com', - path: `/repos/${repositoryId}`, - method: 'GET', - headers: { - 'User-Agent': 'aicodingstack-stars-fetcher', - Accept: 'application/vnd.github+json', - ...(GITHUB_TOKEN ? { Authorization: `Bearer ${GITHUB_TOKEN}` } : {}), - }, - } - - const request = https.request(options, response => { - let body = '' - response.on('data', chunk => { - body += chunk - }) - response.on('end', () => { - if (response.statusCode !== 200) { - reject(new Error(`GitHub API returned status ${String(response.statusCode)}`)) - return - } - try { - const payload = JSON.parse(body) as { stargazers_count?: unknown } - if (!Number.isInteger(payload.stargazers_count)) { - reject(new Error('GitHub response did not include an integer stargazer count')) - return - } - resolve(payload.stargazers_count as number) - } catch (error) { - reject(new Error(`Failed to parse GitHub response: ${(error as Error).message}`)) - } - }) - }) - - request.on('error', reject) - request.end() - }) +function currentShanghaiDate(): string { + return new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date()) } async function main(): Promise { - const current = JSON.parse(fs.readFileSync(GITHUB_STARS_FILE, 'utf8')) as StarsData - let updated = 0 - let failed = 0 + const check = hasFlag('--check') + const write = hasFlag('--write') || !check + if (write && check) throw new Error('Use either --write or --check, not both.') + + const result = await syncGithubStars({ + rootDir: process.cwd(), + write, + observedAt: currentShanghaiDate(), + }) - for (const repositoryId of Object.keys(current.repositories).sort()) { - try { - const stars = await fetchStars(repositoryId) - current.repositories[repositoryId] = stars - updated += 1 - console.log(`✓ ${repositoryId}: ${stars.toLocaleString('en-US')}`) - } catch (error) { - failed += 1 - console.error(`✗ ${repositoryId}: ${(error as Error).message}`) - } + console.log(`Checked ${result.checked} GitHub repositories.`) + if (result.changes.length === 0) { + console.log( + write ? 'Refreshed the GitHub stars observation date.' : 'All star counts are current.' + ) + return } - current.observedAt = new Date().toISOString().slice(0, 10) - fs.writeFileSync(GITHUB_STARS_FILE, `${JSON.stringify(current, null, 2)}\n`, 'utf8') - console.log(`Updated ${updated} repositories; ${failed} retained their previous values.`) - - if (failed > 0) process.exitCode = 1 + for (const change of result.changes) { + console.log( + `${write ? 'Updated' : 'Would update'} ${change.repositoryId}: ${String(change.previousStars)} -> ${change.nextStars}` + ) + } + if (check) process.exitCode = 1 } main().catch(error => { - console.error(error) + console.error(error instanceof Error ? error.message : error) process.exit(1) }) diff --git a/scripts/fetch/lib/benchmark-sync.ts b/scripts/fetch/lib/benchmark-sync.ts index 36596b46..b394c576 100644 --- a/scripts/fetch/lib/benchmark-sync.ts +++ b/scripts/fetch/lib/benchmark-sync.ts @@ -134,32 +134,38 @@ export async function syncBenchmarks(options: { const leaderboards = trackingEntries.length ? await fetchSweBenchLeaderboards(fetchImpl) : [] const changes: BenchmarkChange[] = [] + const failures: string[] = [] for (const { entry, tracking } of trackingEntries) { const leaderboard = leaderboards.find(item => item.name === tracking.leaderboard) if (!leaderboard) { - throw new Error( + failures.push( `models/${entry.manifest.id}: SWE-bench leaderboard ${tracking.leaderboard} was not found` ) + continue } const matches = leaderboard.results.filter(result => result.folder === tracking.resultId) if (matches.length !== 1) { - throw new Error( + failures.push( `models/${entry.manifest.id}: expected exactly one SWE-bench result ${tracking.resultId}, found ${matches.length}` ) + continue } const result = matches[0] if (!result) { - throw new Error(`models/${entry.manifest.id}: SWE-bench result disappeared after validation`) + failures.push(`models/${entry.manifest.id}: SWE-bench result disappeared after validation`) + continue } if (result.name !== tracking.modelLabel) { - throw new Error( + failures.push( `models/${entry.manifest.id}: SWE-bench label changed from "${tracking.modelLabel}" to "${result.name}"` ) + continue } const previousScore = entry.manifest.benchmarks[tracking.benchmark] if (typeof previousScore !== 'number' && previousScore !== null) { - throw new Error(`models/${entry.manifest.id}: ${tracking.benchmark} must be a number or null`) + failures.push(`models/${entry.manifest.id}: ${tracking.benchmark} must be a number or null`) + continue } if (previousScore === result.resolved) continue changes.push({ @@ -173,6 +179,12 @@ export async function syncBenchmarks(options: { }) } + if (failures.length > 0) { + throw new Error( + `Benchmark synchronization failed without writing changes:\n${failures.join('\n')}` + ) + } + if (options.write) { const changesByFile = Map.groupBy(changes, change => change.filePath) for (const [filePath, fileChanges] of changesByFile) { diff --git a/scripts/fetch/lib/github-stars-sync.ts b/scripts/fetch/lib/github-stars-sync.ts new file mode 100644 index 00000000..6359c777 --- /dev/null +++ b/scripts/fetch/lib/github-stars-sync.ts @@ -0,0 +1,124 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +interface JsonResponse { + ok: boolean + status: number + statusText: string + json(): Promise +} + +export type GithubStarsFetch = ( + input: string | URL, + init?: { headers?: Record } +) => Promise + +interface StarsData { + observedAt: string + repositories: Record +} + +export interface GithubStarsChange { + repositoryId: string + previousStars: number | null + nextStars: number +} + +export interface GithubStarsSyncResult { + checked: number + changes: GithubStarsChange[] + observedAt: string + previousObservedAt: string + wroteSnapshot: boolean +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} + +function githubHeaders(): Record { + const headers: Record = { + Accept: 'application/vnd.github+json', + 'User-Agent': 'aicodingstack-stars-fetcher', + 'X-GitHub-Api-Version': '2022-11-28', + } + if (process.env.GITHUB_TOKEN) headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}` + return headers +} + +async function fetchStars(repositoryId: string, fetchImpl: GithubStarsFetch): Promise { + const url = `https://api.github.com/repos/${repositoryId}` + const response = await fetchImpl(url, { headers: githubHeaders() }) + if (!response.ok) { + throw new Error(`${url} returned ${response.status} ${response.statusText}`) + } + const payload = await response.json() + const stars = isRecord(payload) ? payload.stargazers_count : undefined + if (!Number.isInteger(stars) || (stars as number) < 0) { + throw new Error(`${url} returned no usable stargazers_count`) + } + return stars as number +} + +function parseStarsData(value: unknown): StarsData { + if (!isRecord(value) || typeof value.observedAt !== 'string' || !isRecord(value.repositories)) { + throw new Error('data/github-stars.json has an invalid shape') + } + const repositories: Record = {} + for (const [repositoryId, stars] of Object.entries(value.repositories)) { + if (stars !== null && (!Number.isInteger(stars) || (stars as number) < 0)) { + throw new Error(`data/github-stars.json has an invalid count for ${repositoryId}`) + } + repositories[repositoryId] = stars as number | null + } + return { observedAt: value.observedAt, repositories } +} + +export async function syncGithubStars(options: { + rootDir: string + write: boolean + observedAt: string + fetchImpl?: GithubStarsFetch +}): Promise { + const filePath = path.join(options.rootDir, 'data', 'github-stars.json') + const current = parseStarsData(JSON.parse(await fs.readFile(filePath, 'utf8')) as unknown) + const repositoryIds = Object.keys(current.repositories).sort() + const fetchImpl = options.fetchImpl ?? (fetch as GithubStarsFetch) + const observations: Array<{ repositoryId: string; stars: number }> = [] + const failures: string[] = [] + for (const repositoryId of repositoryIds) { + try { + observations.push({ repositoryId, stars: await fetchStars(repositoryId, fetchImpl) }) + } catch (error) { + failures.push(`${repositoryId}: ${String(error)}`) + } + } + if (failures.length > 0) { + throw new Error( + `GitHub stars synchronization failed without writing changes:\n${failures.join('\n')}` + ) + } + + const changes: GithubStarsChange[] = [] + const repositories = { ...current.repositories } + for (const { repositoryId, stars } of observations) { + const previousStars = current.repositories[repositoryId] ?? null + repositories[repositoryId] = stars + if (previousStars !== stars) { + changes.push({ repositoryId, previousStars, nextStars: stars }) + } + } + + if (options.write) { + const next: StarsData = { observedAt: options.observedAt, repositories } + await fs.writeFile(filePath, `${JSON.stringify(next, null, 2)}\n`, 'utf8') + } + + return { + checked: repositoryIds.length, + changes, + observedAt: options.observedAt, + previousObservedAt: current.observedAt, + wroteSnapshot: options.write, + } +} diff --git a/scripts/fetch/lib/product-version-sync.ts b/scripts/fetch/lib/product-version-sync.ts index 21551526..ef0683a2 100644 --- a/scripts/fetch/lib/product-version-sync.ts +++ b/scripts/fetch/lib/product-version-sync.ts @@ -188,7 +188,7 @@ export async function fetchTrackedVersion( } case 'github-release': { const url = `https://api.github.com/repos/${tracking.identifier}/releases/latest` - const data = await fetchJson(fetchImpl, url, githubHeaders()) + const data = await fetchJson(fetchImpl, url, { headers: githubHeaders() }) return { version: getRequiredString( isRecord(data) ? data.tag_name : null, @@ -219,8 +219,20 @@ function compareNumericVersions(left: number[], right: number[]): number { return 0 } -function formatObservedVersion(observed: string, current: string): string { - const normalized = observed.trim() +function formatObservedVersion( + observed: string, + current: string, + tracking: ManifestReleaseTracking +): string { + let normalized = observed.trim() + if (tracking.provider === 'github-release' && tracking.tagPrefix) { + if (!normalized.startsWith(tracking.tagPrefix)) { + throw new Error( + `github-release reported ${observed}, which does not start with configured tagPrefix ${tracking.tagPrefix}` + ) + } + normalized = normalized.slice(tracking.tagPrefix.length) + } if (/^v\d/.test(current) && !/^v\d/.test(normalized)) return `v${normalized}` if (!/^v\d/.test(current) && /^v\d/.test(normalized)) return normalized.slice(1) return normalized @@ -284,7 +296,11 @@ export async function syncProductVersions(options: { const tracking = entry.manifest.releaseTracking if (!tracking) throw new Error(`${entry.manifest.id} has no releaseTracking configuration`) const observation = await fetchTrackedVersion(tracking, options.fetchImpl) - const nextVersion = formatObservedVersion(observation.version, entry.manifest.latestVersion) + const nextVersion = formatObservedVersion( + observation.version, + entry.manifest.latestVersion, + tracking + ) assertNotDowngrade(entry.manifest.latestVersion, nextVersion, tracking) return { entry, observation, nextVersion } }) diff --git a/scripts/manifest-data/index.ts b/scripts/manifest-data/index.ts new file mode 100644 index 00000000..3088f3e7 --- /dev/null +++ b/scripts/manifest-data/index.ts @@ -0,0 +1,104 @@ +#!/usr/bin/env node + +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { + type MaintenanceEvent, + type MaintenanceMode, + ManifestDataMaintenanceError, + runManifestDataMaintenance, + SOURCE_NAMES, + type SourceName, +} from './lib/maintenance' + +function usage(): string { + return [ + 'Usage: pnpm manifest-data: [--only=source,...] [--json]', + `Sources: ${SOURCE_NAMES.join(', ')}`, + '', + 'validate Run deterministic local schema, semantic, i18n, and health checks.', + 'check Run local validation, then read-only checks against configured sources.', + 'update Update source-backed fields, regenerate derived data, then validate.', + ].join('\n') +} + +function parseArguments(arguments_: string[]): { + mode: MaintenanceMode + sources?: SourceName[] + json: boolean +} { + const [modeArgument, ...rawFlags] = arguments_ + const flags = rawFlags.filter(flag => flag !== '--') + if (!['validate', 'check', 'update'].includes(modeArgument ?? '')) { + throw new Error(usage()) + } + const unknownFlags = flags.filter(flag => flag !== '--json' && !flag.startsWith('--only=')) + if (unknownFlags.length > 0) + throw new Error(`Unknown arguments: ${unknownFlags.join(', ')}\n\n${usage()}`) + + const onlyArguments = flags.filter(flag => flag.startsWith('--only=')) + if (onlyArguments.length > 1) throw new Error('Use --only at most once.') + const requestedSources = onlyArguments[0] + ?.slice('--only='.length) + .split(',') + .map(source => source.trim()) + .filter(Boolean) + const invalidSources = requestedSources?.filter( + source => !SOURCE_NAMES.includes(source as SourceName) + ) + if (invalidSources?.length) throw new Error(`Unknown sources: ${invalidSources.join(', ')}`) + + return { + mode: modeArgument as MaintenanceMode, + ...(requestedSources ? { sources: [...new Set(requestedSources)] as SourceName[] } : {}), + json: flags.includes('--json'), + } +} + +function renderEvent(event: MaintenanceEvent): void { + if (event.phase === 'start') { + process.stdout.write(`… ${event.name}\n`) + return + } + const step = event.step + if (!step) return + const marker = + step.status === 'failed' + ? '✗' + : step.status === 'drift' + ? '!' + : step.status === 'current' + ? '·' + : '✓' + process.stdout.write(`${marker} ${step.name}: ${step.summary}\n`) +} + +async function main(): Promise { + const arguments_ = parseArguments(process.argv.slice(2)) + const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') + try { + const report = await runManifestDataMaintenance({ + rootDir, + mode: arguments_.mode, + ...(arguments_.sources ? { sources: arguments_.sources } : {}), + ...(!arguments_.json ? { onEvent: renderEvent } : {}), + }) + if (arguments_.json) { + console.log(JSON.stringify(report, null, 2)) + } else { + console.log( + `Manifest data ${report.mode} passed: ${report.steps.length} steps, ${report.changes} source changes.` + ) + } + } catch (error) { + if (arguments_.json && error instanceof ManifestDataMaintenanceError) { + console.error(JSON.stringify(error.report, null, 2)) + } else { + console.error(error instanceof Error ? error.message : error) + } + process.exitCode = 1 + } +} + +main() diff --git a/scripts/manifest-data/lib/maintenance.ts b/scripts/manifest-data/lib/maintenance.ts new file mode 100644 index 00000000..64d9e876 --- /dev/null +++ b/scripts/manifest-data/lib/maintenance.ts @@ -0,0 +1,342 @@ +import { spawn } from 'node:child_process' + +import { syncBenchmarks } from '../../fetch/lib/benchmark-sync' +import { syncGithubStars } from '../../fetch/lib/github-stars-sync' +import { syncModelSourceDigests } from '../../fetch/lib/model-source-sync' +import { syncProductVersions } from '../../fetch/lib/product-version-sync' +import { runManifestDataHarness } from '../../validate/lib/manifest-data-harness' + +export const SOURCE_NAMES = [ + 'product-versions', + 'benchmarks', + 'model-sources', + 'github-stars', +] as const + +export type SourceName = (typeof SOURCE_NAMES)[number] +export type MaintenanceMode = 'validate' | 'check' | 'update' +export type StepStatus = 'passed' | 'current' | 'drift' | 'updated' | 'failed' + +export interface MaintenanceStep { + name: string + kind: 'validation' | 'source' | 'derived' + status: StepStatus + summary: string + details?: string[] +} + +export interface MaintenanceReport { + mode: MaintenanceMode + sources: SourceName[] + steps: MaintenanceStep[] + changes: number + passed: boolean +} + +export interface SourceTaskResult { + checked: number + changes: number + wrote: boolean + details: string[] +} + +export interface SourceTask { + name: SourceName + run(options: { rootDir: string; write: boolean; observedAt: string }): Promise +} + +export interface MaintenanceEvent { + phase: 'start' | 'finish' + name: string + step?: MaintenanceStep +} + +export interface MaintenanceDependencies { + runCommand(rootDir: string, script: string): Promise + sourceTasks: readonly SourceTask[] + validateSchemas(rootDir: string): { + documentsChecked: number + schemasChecked: number + failures: string[] + } +} + +export class ManifestDataMaintenanceError extends Error { + readonly report: MaintenanceReport + + constructor(message: string, report: MaintenanceReport) { + super(message) + this.name = 'ManifestDataMaintenanceError' + this.report = report + } +} + +function currentShanghaiDate(): string { + return new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date()) +} + +function runPnpmScript(rootDir: string, script: string): Promise { + return new Promise((resolve, reject) => { + const executable = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' + const child = spawn(executable, [script], { + cwd: rootDir, + env: process.env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }) + let stdout = '' + let stderr = '' + child.stdout?.on('data', chunk => { + stdout += String(chunk) + }) + child.stderr?.on('data', chunk => { + stderr += String(chunk) + }) + child.on('error', reject) + child.on('close', code => { + if (code === 0) { + resolve() + return + } + const output = [stdout.trim(), stderr.trim()].filter(Boolean).join('\n') + reject(new Error(`${script} exited with code ${String(code)}${output ? `\n${output}` : ''}`)) + }) + }) +} + +const defaultSourceTasks: readonly SourceTask[] = [ + { + name: 'github-stars', + async run({ rootDir, write, observedAt }) { + const result = await syncGithubStars({ rootDir, write, observedAt }) + return { + checked: result.checked, + changes: result.changes.length, + wrote: + write && (result.changes.length > 0 || result.previousObservedAt !== result.observedAt), + details: result.changes.map( + change => `${change.repositoryId}: ${String(change.previousStars)} -> ${change.nextStars}` + ), + } + }, + }, + { + name: 'product-versions', + async run({ rootDir, write }) { + const result = await syncProductVersions({ rootDir, write }) + return { + checked: result.checked, + changes: result.changes.length, + wrote: write && result.changes.length > 0, + details: result.changes.map( + change => + `${change.category}/${change.id}: ${change.previousVersion} -> ${change.nextVersion}` + ), + } + }, + }, + { + name: 'benchmarks', + async run({ rootDir, write }) { + const result = await syncBenchmarks({ rootDir, write }) + return { + checked: result.checked, + changes: result.changes.length, + wrote: write && result.changes.length > 0, + details: result.changes.map( + change => + `models/${change.modelId} ${change.benchmark}: ${String(change.previousScore)} -> ${change.nextScore}` + ), + } + }, + }, + { + name: 'model-sources', + async run({ rootDir, write, observedAt }) { + const result = await syncModelSourceDigests({ rootDir, write, observedAt }) + return { + checked: result.checked, + changes: result.changes.length, + wrote: write && result.changes.length > 0, + details: result.changes.map( + change => `models/${change.modelId}: ${change.fields.join(', ')} (${change.url})` + ), + } + }, + }, +] + +export const defaultMaintenanceDependencies: MaintenanceDependencies = { + runCommand: runPnpmScript, + sourceTasks: defaultSourceTasks, + validateSchemas: runManifestDataHarness, +} + +const validationScripts = [ + ['semantic validation', 'test:validate'], + ['i18n validation', 'validate:i18n'], + ['data-health snapshot', 'data-health:check'], +] as const + +function notify( + listener: ((event: MaintenanceEvent) => void) | undefined, + event: MaintenanceEvent +): void { + listener?.(event) +} + +export async function runManifestDataMaintenance(options: { + rootDir: string + mode: MaintenanceMode + sources?: SourceName[] + observedAt?: string + dependencies?: MaintenanceDependencies + onEvent?: (event: MaintenanceEvent) => void +}): Promise { + const dependencies = options.dependencies ?? defaultMaintenanceDependencies + if (options.mode === 'validate' && options.sources) { + throw new Error('--only can be used only with check or update') + } + const selectedSources = options.mode === 'validate' ? [] : (options.sources ?? [...SOURCE_NAMES]) + const unknownSources = selectedSources.filter( + source => !dependencies.sourceTasks.some(task => task.name === source) + ) + if (unknownSources.length > 0) { + throw new Error(`Unknown manifest-data sources: ${unknownSources.join(', ')}`) + } + + const report: MaintenanceReport = { + mode: options.mode, + sources: options.mode === 'validate' ? [] : selectedSources, + steps: [], + changes: 0, + passed: false, + } + + const addStep = (step: MaintenanceStep): void => { + report.steps.push(step) + notify(options.onEvent, { phase: 'finish', name: step.name, step }) + } + const fail = (name: string, kind: MaintenanceStep['kind'], error: unknown): never => { + const message = error instanceof Error ? error.message : String(error) + addStep({ name, kind, status: 'failed', summary: message }) + throw new ManifestDataMaintenanceError(`${name} failed`, report) + } + + notify(options.onEvent, { phase: 'start', name: 'schema coverage' }) + const schemaReport = dependencies.validateSchemas(options.rootDir) + if (schemaReport.failures.length > 0) { + fail('schema coverage', 'validation', new Error(schemaReport.failures.join('\n'))) + } + addStep({ + name: 'schema coverage', + kind: 'validation', + status: 'passed', + summary: `${schemaReport.documentsChecked} documents, ${schemaReport.schemasChecked} schemas`, + }) + + if (options.mode !== 'update') { + for (const [name, script] of validationScripts) { + notify(options.onEvent, { phase: 'start', name }) + try { + await dependencies.runCommand(options.rootDir, script) + addStep({ name, kind: 'validation', status: 'passed', summary: script }) + } catch (error) { + fail(name, 'validation', error) + } + } + } + + if (options.mode !== 'validate') { + let driftDetected = false + let sourceFailureDetected = false + for (const sourceName of selectedSources) { + const task = dependencies.sourceTasks.find(candidate => candidate.name === sourceName) + if (!task) throw new Error(`Missing manifest-data source task: ${sourceName}`) + notify(options.onEvent, { phase: 'start', name: sourceName }) + try { + const result = await task.run({ + rootDir: options.rootDir, + write: options.mode === 'update', + observedAt: options.observedAt ?? currentShanghaiDate(), + }) + report.changes += result.changes + const status: StepStatus = + options.mode === 'check' + ? result.changes > 0 + ? 'drift' + : 'current' + : result.wrote + ? 'updated' + : 'current' + if (status === 'drift') driftDetected = true + addStep({ + name: sourceName, + kind: 'source', + status, + summary: `${result.checked} checked, ${result.changes} changes`, + ...(result.details.length > 0 ? { details: result.details } : {}), + }) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + addStep({ name: sourceName, kind: 'source', status: 'failed', summary: message }) + sourceFailureDetected = true + } + } + + if (options.mode === 'check' && (driftDetected || sourceFailureDetected)) { + const failures = report.steps.filter(step => step.status === 'failed').map(step => step.name) + const drift = report.steps.filter(step => step.status === 'drift').map(step => step.name) + throw new ManifestDataMaintenanceError( + [ + ...(failures.length ? [`Source checks failed: ${failures.join(', ')}`] : []), + ...(drift.length ? [`Source drift detected: ${drift.join(', ')}`] : []), + ].join('; '), + report + ) + } + } + + if (options.mode === 'update') { + for (const [name, script] of [ + ['generated data', 'generate'], + ['data-health report', 'data-health:report'], + ] as const) { + notify(options.onEvent, { phase: 'start', name }) + try { + await dependencies.runCommand(options.rootDir, script) + addStep({ name, kind: 'derived', status: 'updated', summary: script }) + } catch (error) { + fail(name, 'derived', error) + } + } + + for (const [name, script] of validationScripts) { + notify(options.onEvent, { phase: 'start', name }) + try { + await dependencies.runCommand(options.rootDir, script) + addStep({ name, kind: 'validation', status: 'passed', summary: script }) + } catch (error) { + fail(name, 'validation', error) + } + } + + const failedSources = report.steps + .filter(step => step.kind === 'source' && step.status === 'failed') + .map(step => step.name) + if (failedSources.length > 0) { + throw new ManifestDataMaintenanceError( + `Source updates failed: ${failedSources.join(', ')}`, + report + ) + } + } + + report.passed = true + return report +} diff --git a/scripts/validate/lib/manifest-data-harness.ts b/scripts/validate/lib/manifest-data-harness.ts new file mode 100644 index 00000000..1d67cc69 --- /dev/null +++ b/scripts/validate/lib/manifest-data-harness.ts @@ -0,0 +1,260 @@ +import fs from 'node:fs' +import path from 'node:path' +import { pathToFileURL } from 'node:url' +import type { ErrorObject, ValidateFunction } from 'ajv' +import Ajv2020 from 'ajv/dist/2020.js' +import addFormats from 'ajv-formats' + +export interface ManifestDataDocument { + filePath: string + relativePath: string + schemaPath: string | null + category: string | null + data: unknown +} + +export interface ManifestDataHarnessReport { + documentsChecked: number + schemasChecked: number + documentsByRoot: Record + failures: string[] +} + +function listJsonFiles(directory: string, skipSchemaDirectories = false): string[] { + if (!fs.existsSync(directory)) return [] + + return fs + .readdirSync(directory, { withFileTypes: true }) + .flatMap(entry => { + if (skipSchemaDirectories && entry.isDirectory() && entry.name === '$schemas') return [] + const entryPath = path.join(directory, entry.name) + if (entry.isDirectory()) return listJsonFiles(entryPath, skipSchemaDirectories) + return entry.isFile() && entry.name.endsWith('.json') ? [entryPath] : [] + }) + .sort() +} + +function readJson(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown +} + +function getLocalSchemaReference(data: unknown): string | null { + if (!data || typeof data !== 'object' || Array.isArray(data)) return null + const schema = (data as Record).$schema + return typeof schema === 'string' && schema.startsWith('.') ? schema : null +} + +function resolveConventionSchema( + rootDir: string, + filePath: string +): { + category: string | null + schemaPath: string | null +} { + const relativePath = path.relative(rootDir, filePath) + const segments = relativePath.split(path.sep) + const basename = path.basename(filePath, '.json') + + if (segments[0] === 'manifests' && segments.length === 3) { + const category = segments[1] ?? '' + const singular = category.endsWith('s') ? category.slice(0, -1) : category + return { + category, + schemaPath: path.join(rootDir, 'manifests', '$schemas', `${singular}.schema.json`), + } + } + + const schemaRoots = + segments[0] === 'manifests' + ? [path.join(rootDir, 'manifests', '$schemas')] + : [path.join(rootDir, 'data', '$schemas'), path.join(rootDir, 'manifests', '$schemas')] + const schemaPath = schemaRoots + .map(schemaRoot => path.join(schemaRoot, `${basename}.schema.json`)) + .find(candidate => fs.existsSync(candidate)) + + return { category: null, schemaPath: schemaPath ?? null } +} + +export function discoverManifestData(rootDir: string): ManifestDataDocument[] { + const files = [ + ...listJsonFiles(path.join(rootDir, 'manifests'), true), + ...listJsonFiles(path.join(rootDir, 'data'), true), + ].sort() + + return files.map(filePath => { + let data: unknown = null + try { + data = readJson(filePath) + } catch { + // Parsing failures are reported by the validation pass. + } + + const conventional = resolveConventionSchema(rootDir, filePath) + const localSchemaReference = getLocalSchemaReference(data) + const declaredSchemaPath = localSchemaReference + ? path.resolve(path.dirname(filePath), localSchemaReference) + : null + + return { + filePath, + relativePath: path.relative(rootDir, filePath).replaceAll(path.sep, '/'), + schemaPath: conventional.schemaPath ?? declaredSchemaPath, + category: conventional.category, + data, + } + }) +} + +function formatAjvErrors(errors: ErrorObject[] | null | undefined): string { + if (!errors?.length) return 'unknown schema error' + return errors + .map(error => { + const location = error.instancePath || '(root)' + return `${location}: ${error.message ?? error.keyword}` + }) + .join('; ') +} + +function createSchemaRegistry(schemaPaths: string[]): { + ajv: Ajv2020 + schemaKey: (schemaPath: string) => string +} { + const ajv = new Ajv2020({ + allErrors: true, + strict: true, + strictTypes: true, + allowUnionTypes: true, + }) + addFormats(ajv) + + const registered = new Set() + const schemaKey = (schemaPath: string) => pathToFileURL(schemaPath).href + + const registerSchema = (schemaPath: string): void => { + const resolvedPath = path.resolve(schemaPath) + if (registered.has(resolvedPath)) return + registered.add(resolvedPath) + + const schema = structuredClone(readJson(resolvedPath)) as Record + schema.$id = schemaKey(resolvedPath) + + const rewriteReferences = (value: unknown): void => { + if (!value || typeof value !== 'object') return + if (Array.isArray(value)) { + for (const item of value) rewriteReferences(item) + return + } + + const record = value as Record + if (typeof record.$ref === 'string' && record.$ref.startsWith('.')) { + const [referencePath, fragment] = record.$ref.split('#', 2) + const referencedSchemaPath = path.resolve(path.dirname(resolvedPath), referencePath ?? '') + registerSchema(referencedSchemaPath) + record.$ref = `${schemaKey(referencedSchemaPath)}${fragment ? `#${fragment}` : ''}` + } + + for (const child of Object.values(record)) rewriteReferences(child) + } + + rewriteReferences(schema) + ajv.addSchema(schema) + } + + for (const schemaPath of schemaPaths) registerSchema(schemaPath) + return { ajv, schemaKey } +} + +export function runManifestDataHarness(rootDir: string): ManifestDataHarnessReport { + const failures: string[] = [] + const documents = discoverManifestData(rootDir) + const allSchemaPaths = [ + ...listJsonFiles(path.join(rootDir, 'manifests', '$schemas')), + ...listJsonFiles(path.join(rootDir, 'data', '$schemas')), + ] + const documentsByRoot = { manifests: 0, data: 0 } + + for (const document of documents) { + const root = document.relativePath.split('/')[0] + if (root === 'manifests' || root === 'data') documentsByRoot[root]++ + + try { + document.data = readJson(document.filePath) + } catch (error) { + failures.push(`${document.relativePath}: invalid JSON: ${(error as Error).message}`) + continue + } + + if (!document.schemaPath) { + failures.push(`${document.relativePath}: no schema discovered`) + continue + } + if (!fs.existsSync(document.schemaPath)) { + failures.push( + `${document.relativePath}: schema does not exist: ${path.relative(rootDir, document.schemaPath)}` + ) + continue + } + + const declaredReference = getLocalSchemaReference(document.data) + if (declaredReference) { + const declaredPath = path.resolve(path.dirname(document.filePath), declaredReference) + if (declaredPath !== path.resolve(document.schemaPath)) { + failures.push( + `${document.relativePath}: declared schema does not match discovered schema ${path.relative(rootDir, document.schemaPath)}` + ) + } + } + } + + let registry: ReturnType | null = null + try { + registry = createSchemaRegistry(allSchemaPaths) + for (const schemaPath of allSchemaPaths) { + registry.ajv.getSchema(registry.schemaKey(schemaPath)) + } + } catch (error) { + failures.push(`schema registry: ${(error as Error).message}`) + } + + if (registry) { + for (const document of documents) { + if (!document.schemaPath || !fs.existsSync(document.schemaPath)) continue + let validate: ValidateFunction | undefined + try { + validate = registry.ajv.getSchema(registry.schemaKey(document.schemaPath)) + } catch (error) { + failures.push( + `${document.relativePath}: schema compilation failed: ${(error as Error).message}` + ) + continue + } + if (!validate) { + failures.push(`${document.relativePath}: schema was not registered`) + continue + } + if (!validate(document.data)) { + failures.push(`${document.relativePath}: ${formatAjvErrors(validate.errors)}`) + } + + if (document.category) { + const expectedId = path.basename(document.filePath, '.json') + const actualId = + document.data && typeof document.data === 'object' && !Array.isArray(document.data) + ? (document.data as Record).id + : undefined + if (actualId !== expectedId) { + failures.push( + `${document.relativePath}: filename/id mismatch (expected ${expectedId}, received ${String(actualId)})` + ) + } + } + } + } + + return { + documentsChecked: documents.length, + schemasChecked: allSchemaPaths.length, + documentsByRoot, + failures: failures.sort(), + } +} diff --git a/src/types/manifests.ts b/src/types/manifests.ts index 4e7ff840..4bbb930e 100644 --- a/src/types/manifests.ts +++ b/src/types/manifests.ts @@ -176,6 +176,7 @@ export type ManifestReleaseTracking = | { provider: 'github-release' identifier: string + tagPrefix?: string } /** @@ -493,7 +494,20 @@ export interface ManifestGitHubStars { } // ============================================================================= -// SECTION 6: Manifest Array Types (for JSON file imports) +// SECTION 6: External ID Mapping +// ============================================================================= + +/** + * External vendor and model identifiers used by comparison data sources + * Based on: /manifests/$schemas/mapping.schema.json + */ +export interface ManifestMapping { + vendors: Record + models: Record +} + +// ============================================================================= +// SECTION 7: Manifest Array Types (for JSON file imports) // ============================================================================= /** @@ -508,7 +522,7 @@ export type ManifestProviderArray = ManifestProvider[] export type ManifestVendorArray = ManifestVendor[] // ============================================================================= -// SECTION 7: Utility Types +// SECTION 8: Utility Types // ============================================================================= /** diff --git a/tests/github-stars-sync.test.ts b/tests/github-stars-sync.test.ts new file mode 100644 index 00000000..b46411a2 --- /dev/null +++ b/tests/github-stars-sync.test.ts @@ -0,0 +1,109 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { type GithubStarsFetch, syncGithubStars } from '../scripts/fetch/lib/github-stars-sync' + +const temporaryDirectories: string[] = [] + +async function createStarsRoot(repositories: Record): Promise { + const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), 'github-stars-sync-')) + temporaryDirectories.push(rootDir) + await fs.mkdir(path.join(rootDir, 'data'), { recursive: true }) + await fs.writeFile( + path.join(rootDir, 'data', 'github-stars.json'), + `${JSON.stringify({ observedAt: '2026-08-01', repositories }, null, 2)}\n` + ) + return rootDir +} + +function starsFetch(counts: Record): GithubStarsFetch { + return async input => { + const repositoryId = String(input).replace('https://api.github.com/repos/', '') + const count = counts[repositoryId] + if (count === undefined) { + return { + ok: false, + status: 404, + statusText: 'Not Found', + async json() { + return {} + }, + } + } + return { + ok: true, + status: 200, + statusText: 'OK', + async json() { + return { stargazers_count: count } + }, + } + } +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map(directory => fs.rm(directory, { force: true, recursive: true })) + ) +}) + +describe('GitHub stars synchronization', () => { + it('reports source drift without writing in check mode', async () => { + const rootDir = await createStarsRoot({ 'example/one': 10 }) + const filePath = path.join(rootDir, 'data', 'github-stars.json') + const before = await fs.readFile(filePath, 'utf8') + + const result = await syncGithubStars({ + rootDir, + write: false, + observedAt: '2026-08-31', + fetchImpl: starsFetch({ 'example/one': 12 }), + }) + + expect(result.changes).toEqual([ + { repositoryId: 'example/one', previousStars: 10, nextStars: 12 }, + ]) + expect(await fs.readFile(filePath, 'utf8')).toBe(before) + }) + + it('writes counts and the observation date after all sources succeed', async () => { + const rootDir = await createStarsRoot({ 'example/one': 10, 'example/two': null }) + + const result = await syncGithubStars({ + rootDir, + write: true, + observedAt: '2026-08-31', + fetchImpl: starsFetch({ 'example/one': 12, 'example/two': 3 }), + }) + const snapshot = JSON.parse( + await fs.readFile(path.join(rootDir, 'data', 'github-stars.json'), 'utf8') + ) as Record + + expect(result.checked).toBe(2) + expect(snapshot).toEqual({ + observedAt: '2026-08-31', + repositories: { 'example/one': 12, 'example/two': 3 }, + }) + }) + + it('does not write a partial snapshot when one source fails', async () => { + const rootDir = await createStarsRoot({ 'example/one': 10, 'example/two': 20 }) + const filePath = path.join(rootDir, 'data', 'github-stars.json') + const before = await fs.readFile(filePath, 'utf8') + + await expect( + syncGithubStars({ + rootDir, + write: true, + observedAt: '2026-08-31', + fetchImpl: starsFetch({ 'example/one': 12 }), + }) + ).rejects.toThrow('failed without writing changes') + expect(await fs.readFile(filePath, 'utf8')).toBe(before) + }) +}) diff --git a/tests/manifest-data-maintenance.test.ts b/tests/manifest-data-maintenance.test.ts new file mode 100644 index 00000000..a2191735 --- /dev/null +++ b/tests/manifest-data-maintenance.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from 'vitest' + +import { + type MaintenanceDependencies, + ManifestDataMaintenanceError, + runManifestDataMaintenance, + type SourceTask, +} from '../scripts/manifest-data/lib/maintenance' + +function sourceTask(name: SourceTask['name'], changes: number, events: string[]): SourceTask { + return { + name, + async run({ write }) { + events.push(`${name}:${write ? 'write' : 'check'}`) + return { + checked: 1, + changes, + wrote: write && changes > 0, + details: changes ? [`${name} changed`] : [], + } + }, + } +} + +function dependencies(events: string[], sourceTasks: SourceTask[]): MaintenanceDependencies { + return { + validateSchemas() { + events.push('schema') + return { documentsChecked: 10, schemasChecked: 2, failures: [] } + }, + async runCommand(_rootDir, script) { + events.push(script) + }, + sourceTasks, + } +} + +describe('manifest-data maintenance harness', () => { + it('runs the complete deterministic validation suite without source access', async () => { + const events: string[] = [] + const report = await runManifestDataMaintenance({ + rootDir: '/fixture', + mode: 'validate', + dependencies: dependencies(events, []), + }) + + expect(events).toEqual(['schema', 'test:validate', 'validate:i18n', 'data-health:check']) + expect(report.passed).toBe(true) + expect(report.sources).toEqual([]) + }) + + it('checks selected sources without writing and reports every detected drift', async () => { + const events: string[] = [] + const taskDependencies = dependencies(events, [ + sourceTask('github-stars', 2, events), + sourceTask('benchmarks', 1, events), + ]) + + let failure: ManifestDataMaintenanceError | undefined + try { + await runManifestDataMaintenance({ + rootDir: '/fixture', + mode: 'check', + sources: ['github-stars', 'benchmarks'], + dependencies: taskDependencies, + }) + } catch (error) { + failure = error as ManifestDataMaintenanceError + } + + expect(events).toEqual([ + 'schema', + 'test:validate', + 'validate:i18n', + 'data-health:check', + 'github-stars:check', + 'benchmarks:check', + ]) + expect(failure).toBeInstanceOf(ManifestDataMaintenanceError) + expect(failure?.report.changes).toBe(3) + expect(failure?.report.steps.filter(step => step.status === 'drift')).toHaveLength(2) + }) + + it('updates sources before regenerating and validating derived data', async () => { + const events: string[] = [] + const report = await runManifestDataMaintenance({ + rootDir: '/fixture', + mode: 'update', + sources: ['product-versions'], + observedAt: '2026-08-31', + dependencies: dependencies(events, [sourceTask('product-versions', 1, events)]), + }) + + expect(events).toEqual([ + 'schema', + 'product-versions:write', + 'generate', + 'data-health:report', + 'test:validate', + 'validate:i18n', + 'data-health:check', + ]) + expect(report.passed).toBe(true) + expect(report.changes).toBe(1) + }) + + it('continues read-only source checks after one source fails', async () => { + const events: string[] = [] + const failingTask: SourceTask = { + name: 'benchmarks', + async run() { + events.push('benchmarks:check') + throw new Error('upstream mapping changed') + }, + } + const taskDependencies = dependencies(events, [ + failingTask, + sourceTask('model-sources', 0, events), + ]) + + let failure: ManifestDataMaintenanceError | undefined + try { + await runManifestDataMaintenance({ + rootDir: '/fixture', + mode: 'check', + sources: ['benchmarks', 'model-sources'], + dependencies: taskDependencies, + }) + } catch (error) { + failure = error as ManifestDataMaintenanceError + } + + expect(events.at(-2)).toBe('benchmarks:check') + expect(events.at(-1)).toBe('model-sources:check') + expect(failure?.report.steps.at(-1)).toMatchObject({ + name: 'model-sources', + status: 'current', + }) + }) + + it('regenerates and validates successful writes when another update source fails', async () => { + const events: string[] = [] + const failingTask: SourceTask = { + name: 'product-versions', + async run() { + events.push('product-versions:write') + throw new Error('registry unavailable') + }, + } + const taskDependencies = dependencies(events, [ + failingTask, + sourceTask('github-stars', 1, events), + ]) + + let failure: ManifestDataMaintenanceError | undefined + try { + await runManifestDataMaintenance({ + rootDir: '/fixture', + mode: 'update', + sources: ['product-versions', 'github-stars'], + dependencies: taskDependencies, + }) + } catch (error) { + failure = error as ManifestDataMaintenanceError + } + + expect(events).toContain('github-stars:write') + expect(events).toContain('generate') + expect(events.at(-1)).toBe('data-health:check') + expect(failure?.report.steps).toContainEqual( + expect.objectContaining({ name: 'product-versions', status: 'failed' }) + ) + }) +}) diff --git a/tests/product-version-sync.test.ts b/tests/product-version-sync.test.ts index 613b3f2d..4a597c40 100644 --- a/tests/product-version-sync.test.ts +++ b/tests/product-version-sync.test.ts @@ -93,6 +93,35 @@ describe('product version sources', () => { }) }) + it('sends GitHub API headers through RequestInit', async () => { + const previousToken = process.env.GITHUB_TOKEN + process.env.GITHUB_TOKEN = 'test-token' + let headers: Record | undefined + const fetchImpl: VersionFetch = async (_input, init) => { + headers = init?.headers + return jsonFetch({ + tag_name: 'v1.0.0', + html_url: 'https://github.com/example/tool/releases/tag/v1.0.0', + })(_input, init) + } + + try { + await fetchTrackedVersion( + { provider: 'github-release', identifier: 'example/tool' }, + fetchImpl + ) + } finally { + if (previousToken === undefined) delete process.env.GITHUB_TOKEN + else process.env.GITHUB_TOKEN = previousToken + } + + expect(headers).toMatchObject({ + Accept: 'application/vnd.github+json', + Authorization: 'Bearer test-token', + 'User-Agent': 'aicodingstack-version-sync', + }) + }) + it('reads PyPI and Visual Studio Marketplace versions', async () => { const pypi = await fetchTrackedVersion( { provider: 'pypi', identifier: 'example-tool' }, @@ -167,6 +196,56 @@ describe('product version synchronization', () => { ) }) + it('removes an explicitly configured GitHub release tag prefix', async () => { + const rootDir = await createManifestRoot([ + { + id: 'desktop-example', + latestVersion: '1.2.0', + releaseTracking: { + provider: 'github-release', + identifier: 'example/tool', + tagPrefix: 'desktop-', + }, + }, + ]) + + const result = await syncProductVersions({ + rootDir, + write: false, + fetchImpl: jsonFetch({ + tag_name: 'desktop-v1.3.0', + html_url: 'https://github.com/example/tool/releases/tag/desktop-v1.3.0', + }), + }) + + expect(result.changes[0]?.nextVersion).toBe('1.3.0') + }) + + it('fails closed when a GitHub release tag does not match its configured prefix', async () => { + const rootDir = await createManifestRoot([ + { + id: 'desktop-example', + latestVersion: '1.2.0', + releaseTracking: { + provider: 'github-release', + identifier: 'example/tool', + tagPrefix: 'desktop-', + }, + }, + ]) + + await expect( + syncProductVersions({ + rootDir, + write: true, + fetchImpl: jsonFetch({ + tag_name: 'cli-v1.3.0', + html_url: 'https://github.com/example/tool/releases/tag/cli-v1.3.0', + }), + }) + ).rejects.toThrow('does not start with configured tagPrefix') + }) + it('does not write partial changes when a source fails', async () => { const rootDir = await createManifestRoot([ { diff --git a/tests/validate/githubStars.consistency.test.ts b/tests/validate/githubStars.consistency.test.ts index 15437825..88325e04 100644 --- a/tests/validate/githubStars.consistency.test.ts +++ b/tests/validate/githubStars.consistency.test.ts @@ -1,7 +1,7 @@ import fs from 'node:fs' import path from 'node:path' -import Ajv from 'ajv' +import Ajv2020 from 'ajv/dist/2020.js' import addFormats from 'ajv-formats' import { describe, expect, it } from 'vitest' @@ -75,7 +75,7 @@ describe('validate: github-stars consistency', () => { path.join(rootDir, 'manifests', '$schemas', 'github-stars.schema.json') ) const githubStars = readJsonFile(path.join(rootDir, 'data', 'github-stars.json')) - const ajv = new Ajv({ allErrors: true }) + const ajv = new Ajv2020({ allErrors: true }) addFormats(ajv) const validate = ajv.compile(schema as object) diff --git a/tests/validate/manifests.schema.test.ts b/tests/validate/manifests.schema.test.ts index 20f211c4..bb90113d 100644 --- a/tests/validate/manifests.schema.test.ts +++ b/tests/validate/manifests.schema.test.ts @@ -1,235 +1,31 @@ import fs from 'node:fs' +import os from 'node:os' import path from 'node:path' -import type { ErrorObject } from 'ajv' -import Ajv2020 from 'ajv/dist/2020.js' -import addFormats from 'ajv-formats' -import { describe, it } from 'vitest' +import { describe, expect, it } from 'vitest' +import { runManifestDataHarness } from '../../scripts/validate/lib/manifest-data-harness' -/** - * Create an Ajv instance aligned with the repository's manifest validation behavior. - */ -function createAjv() { - const ajv = new Ajv2020({ - allErrors: true, - verbose: true, - strict: true, - strictTypes: true, - allowUnionTypes: true, - }) - addFormats(ajv) - return ajv -} - -/** - * Read and parse JSON from disk. - */ -function readJsonFile(filePath: string): unknown { - const content = fs.readFileSync(filePath, 'utf8') - return JSON.parse(content) as unknown -} - -/** - * Format Ajv errors into stable, human-readable lines. - */ -function formatAjvErrors(errors: ErrorObject[] | null | undefined): string[] { - if (!errors || errors.length === 0) { - return [] - } +describe('validate: manifest data harness', () => { + it('schema-validates every manifest and data JSON document', () => { + const report = runManifestDataHarness(process.cwd()) - return errors.map(err => { - const instancePath = err.instancePath || '(root)' - if (err.keyword === 'required') { - return `${instancePath}: missing required field ${(err.params as { missingProperty: string }).missingProperty}` - } - if (err.keyword === 'additionalProperties') { - return `${instancePath}: additional property not allowed ${(err.params as { additionalProperty: string }).additionalProperty}` - } - if (err.keyword === 'enum') { - const allowed = (err.params as { allowedValues: unknown[] }).allowedValues - return `${instancePath}: ${err.message} (allowed: ${allowed.join(', ')})` - } - if (err.keyword === 'format') { - const format = (err.params as { format: string }).format - return `${instancePath}: ${err.message} (expected format: ${format})` - } - return `${instancePath}: ${err.message ?? 'validation error'}` + expect(report.documentsChecked).toBeGreaterThan(0) + expect(report.documentsByRoot.manifests).toBeGreaterThan(0) + expect(report.documentsByRoot.data).toBeGreaterThan(0) + expect(report.failures, report.failures.join('\n')).toEqual([]) }) -} - -/** - * Add base ref schemas into Ajv to support $ref resolution used by top-level schemas. - */ -function loadBaseSchemas(ajv: ReturnType, schemasDir: string) { - const refDir = path.join(schemasDir, 'ref') - const toAdd = [ - 'translations.schema.json', - 'community-urls.schema.json', - 'platform-urls.schema.json', - 'entity.schema.json', - 'vendor-entity.schema.json', - 'product.schema.json', - 'app.schema.json', - ] - - for (const name of toAdd) { - const schemaPath = path.join(refDir, name) - if (!fs.existsSync(schemaPath)) { - continue - } - const schema = readJsonFile(schemaPath) - ajv.addSchema(schema as object, name) - } -} - -/** - * Resolve relative $ref entries of the form "./ref/foo.schema.json" to Ajv schema IDs (basenames). - */ -function resolveRelativeRefs(obj: unknown, baseDir: string, ajv: ReturnType) { - if (obj === null || typeof obj !== 'object') return - if (Array.isArray(obj)) { - for (const item of obj) resolveRelativeRefs(item, baseDir, ajv) - return - } - - const record = obj as Record - const ref = record.$ref - if (typeof ref === 'string' && ref.startsWith('./')) { - const refPath = path.resolve(baseDir, ref) - if (fs.existsSync(refPath)) { - const refSchema = readJsonFile(refPath) - const refId = path.basename(refPath) - if (!ajv.getSchema(refId)) { - ajv.addSchema(refSchema as object, refId) - } - record.$ref = refId - } - } - - for (const value of Object.values(record)) { - resolveRelativeRefs(value, baseDir, ajv) - } -} - -/** - * Load a top-level schema and prepare it for validation by resolving relative $ref. - */ -function loadSchema(schemaPath: string, ajv: ReturnType): unknown { - const schema = readJsonFile(schemaPath) - resolveRelativeRefs(schema, path.dirname(schemaPath), ajv) - return schema -} - -/** - * Validate all manifests on disk against their schemas. - */ -function validateAllManifests(rootDir: string): string[] { - const failures: string[] = [] - const manifestsDir = path.join(rootDir, 'manifests') - const schemasDir = path.join(manifestsDir, '$schemas') - - const ajv = createAjv() - loadBaseSchemas(ajv, schemasDir) - const manifestSchemaMap: Record = { - clis: 'cli.schema.json', - desktops: 'desktop.schema.json', - ides: 'ide.schema.json', - extensions: 'extension.schema.json', - providers: 'provider.schema.json', - models: 'model.schema.json', - vendors: 'vendor.schema.json', - } - - const singleFileSchemas: Record = { - 'collections.json': 'collections.schema.json', - } - - for (const [dirName, schemaFile] of Object.entries(manifestSchemaMap)) { - const dirPath = path.join(manifestsDir, dirName) - const schemaPath = path.join(schemasDir, schemaFile) - if (!fs.existsSync(dirPath) || !fs.existsSync(schemaPath)) { - continue - } - - const fullSchema = loadSchema(schemaPath, ajv) as { - type?: unknown - items?: unknown - $defs?: unknown - } - const schemaToUse = - fullSchema.type === 'array' && fullSchema.items - ? ({ - ...(fullSchema.items as object), - ...(fullSchema.$defs ? { $defs: fullSchema.$defs } : {}), - } as object) - : (fullSchema as object) - - const validate = ajv.compile(schemaToUse) - const files = fs.readdirSync(dirPath).filter(f => f.endsWith('.json')) - - for (const file of files) { - const filePath = path.join(dirPath, file) - const expectedId = path.basename(file, '.json') - let data: unknown - try { - data = readJsonFile(filePath) - } catch (error) { - failures.push(`[${dirName}/${file}] JSON parse error: ${(error as Error).message}`) - continue - } - - const ok = validate(data) - if (!ok) { - const lines = formatAjvErrors(validate.errors) - failures.push(`[${dirName}/${file}] schema errors:\n${lines.map(l => `- ${l}`).join('\n')}`) - continue - } - - if (typeof data === 'object' && data !== null && 'id' in (data as Record)) { - const actualId = (data as Record).id - if (actualId !== expectedId) { - failures.push( - `[${dirName}/${file}] filename/id mismatch: expected "${expectedId}" but got "${String(actualId)}"` - ) - } - } else { - failures.push(`[${dirName}/${file}] missing 'id' field for filename check`) - } - } - } - - for (const [manifestFile, schemaFile] of Object.entries(singleFileSchemas)) { - const manifestPath = path.join(manifestsDir, manifestFile) - const schemaPath = path.join(schemasDir, schemaFile) - if (!fs.existsSync(manifestPath) || !fs.existsSync(schemaPath)) { - continue - } - - let data: unknown + it('rejects a newly added data document without a schema', () => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'manifest-data-harness-')) try { - data = readJsonFile(manifestPath) - } catch (error) { - failures.push(`[${manifestFile}] JSON parse error: ${(error as Error).message}`) - continue - } - - const schema = loadSchema(schemaPath, ajv) - const validate = ajv.compile(schema as object) - const ok = validate(data) - if (!ok) { - const lines = formatAjvErrors(validate.errors) - failures.push(`[${manifestFile}] schema errors:\n${lines.map(l => `- ${l}`).join('\n')}`) - } - } + fs.mkdirSync(path.join(fixtureRoot, 'manifests'), { recursive: true }) + fs.mkdirSync(path.join(fixtureRoot, 'data'), { recursive: true }) + fs.writeFileSync(path.join(fixtureRoot, 'data', 'uncovered.json'), '{}\n') - return failures -} + const report = runManifestDataHarness(fixtureRoot) -describe('validate: manifests schema', () => { - it('all manifest JSON files match their schemas', () => { - const failures = validateAllManifests(process.cwd()) - if (failures.length > 0) { - throw new Error(`Manifest schema validation failed:\n\n${failures.join('\n\n')}`) + expect(report.failures).toContain('data/uncovered.json: no schema discovered') + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }) } }) })