From da6eab94836b77ebe4a2c62d5eec8dc9d181f0b5 Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Mon, 14 Sep 2026 23:10:55 +0800 Subject: [PATCH 01/24] docs: accuracy and contract redesign spec + implementation plan --- ...26-09-14-accuracy-and-contract-redesign.md | 1972 +++++++++++++++++ ...4-accuracy-and-contract-redesign-design.md | 153 ++ 2 files changed, 2125 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-14-accuracy-and-contract-redesign.md create mode 100644 docs/superpowers/specs/2026-09-14-accuracy-and-contract-redesign-design.md diff --git a/docs/superpowers/plans/2026-09-14-accuracy-and-contract-redesign.md b/docs/superpowers/plans/2026-09-14-accuracy-and-contract-redesign.md new file mode 100644 index 0000000..d319e11 --- /dev/null +++ b/docs/superpowers/plans/2026-09-14-accuracy-and-contract-redesign.md @@ -0,0 +1,1972 @@ +# Accuracy and Contract Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make refkit's default search accurate across keyless sources and move the core contract to its first principles (facts-driven gate, core-completed references, one control registry, one search channel). + +**Architecture:** `@refkit/core` stays zero-network and zod-only; providers return `EmittedReference` and core stamps provenance; the orchestrator becomes `select → pipeline stages → cursor loop → meta`; merge takes per-source confidence weights and the lexical reranker runs by default over title + description + tags. + +**Tech Stack:** TypeScript strict, pnpm workspace, vitest (`pnpm test:run`), zod 4, tsup. + +**Spec:** `docs/superpowers/specs/2026-09-14-accuracy-and-contract-redesign-design.md` + +## Global Constraints + +- `@refkit/core` depends only on `zod`; no `fetch(` call and no `http(s)://` literal in `packages/core/src` (test-enforced by `no-network.test.ts`). +- Every `@refkit/provider-*` package depends only on `@refkit/core`. +- No compatibility shims, no deprecated aliases: removed surfaces are removed. +- At the end of every task: `pnpm typecheck && pnpm lint && pnpm test:run` all green. +- Commits: conventional prefixes, no attribution / Co-authored-by trailers. +- Type shapes in spec sections D1, D2, D4–D7 are binding (copied into the tasks below). +- Work only inside this worktree; never `git stash`; never push. +- Test files live in `src/__tests__/*.test.ts` per package; run one package with `pnpm --filter @refkit/core test`. + +--- + +### Task 1: License facts drive the gate, merge and reranker + +**Files:** +- Modify: `packages/core/src/license.ts` +- Modify: `packages/core/src/rights.ts` +- Modify: `packages/core/src/evaluate-use.ts` +- Modify: `packages/core/src/attribution.ts` +- Modify: `packages/core/src/merge.ts` +- Modify: `packages/core/src/rerank.ts` +- Modify: `packages/core/src/provider-helpers.ts` (remove `CC_VERSIONED_FAMILIES`, `ccVersionFor`; keep the rest) +- Modify: `packages/core/src/client.ts` (pass `facts` into `buildAttribution`) +- Modify: `packages/core/src/index.ts` +- Modify: `packages/provider-testkit/src/index.ts` (import `CC_VERSIONED_FAMILIES` still works — it is re-exported from index; no change needed unless typecheck says so) +- Test: `packages/core/src/__tests__/license.test.ts`, `rights.test.ts`, `evaluate-use.test.ts`, `attribution.test.ts`, `merge.test.ts`, `rerank.test.ts` + +**Interfaces:** +- Produces (exported from `@refkit/core`): + - `type KnownLicenseId`, `type LicenseId = KnownLicenseId | (string & {})`, `isKnownLicenseId(id: string): id is KnownLicenseId` + - `factsFor(license: LicenseId): LicenseFacts` (unknown id → `LICENSE_FACTS.unknown`) + - `factsOf(r: Pick): LicenseFacts` + - `isIndeterminate(f: LicenseFacts): boolean` + - `compareRestrictiveness(a: LicenseFacts, b: LicenseFacts): 'a' | 'b' | 'equal' | 'incomparable'` + - `permissivenessScore(f: LicenseFacts): number` + - `CC_VERSIONED_FAMILIES`, `ccVersionFor` (now from `license.ts`) + - `licenseFactsSchema`, `RightsRecord.facts?: LicenseFacts`, `AttributionInput.facts?: LicenseFacts` +- Removed: `stricterLicense`, `LICENSE_PERMISSIVENESS`. + +- [ ] **Step 1: Write failing tests for the facts API** + +Append to `packages/core/src/__tests__/license.test.ts`: + +```ts +import { compareRestrictiveness, factsFor, isIndeterminate, isKnownLicenseId, permissivenessScore, LICENSE_FACTS } from '../license' + +describe('facts API', () => { + it('factsFor falls back to unknown for an id outside the table', () => { + expect(factsFor('acme-stock')).toEqual(LICENSE_FACTS.unknown) + expect(isKnownLicenseId('acme-stock')).toBe(false) + expect(isKnownLicenseId('CC-BY')).toBe(true) + }) + it('isIndeterminate is true only when all three tri axes are unknown', () => { + expect(isIndeterminate(LICENSE_FACTS.unknown)).toBe(true) + expect(isIndeterminate(LICENSE_FACTS['CC-BY-NC'])).toBe(false) + }) + it('compareRestrictiveness orders by dominance and reports incomparable pairs', () => { + expect(compareRestrictiveness(LICENSE_FACTS['CC-BY'], LICENSE_FACTS['CC0-1.0'])).toBe('a') + expect(compareRestrictiveness(LICENSE_FACTS['CC0-1.0'], LICENSE_FACTS['CC-BY'])).toBe('b') + expect(compareRestrictiveness(LICENSE_FACTS['CC0-1.0'], LICENSE_FACTS.PD)).toBe('equal') + // unsplash forbids redistribution but needs no attribution; CC-BY is the reverse + expect(compareRestrictiveness(LICENSE_FACTS.unsplash, LICENSE_FACTS['CC-BY'])).toBe('incomparable') + }) + it('permissivenessScore is 1 for CC0 and treats unknown as not granted', () => { + expect(permissivenessScore(LICENSE_FACTS['CC0-1.0'])).toBe(1) + expect(permissivenessScore(LICENSE_FACTS['CC-BY'])).toBe(0.875) + expect(permissivenessScore(LICENSE_FACTS.unknown)).toBe(0.25) + expect(permissivenessScore(LICENSE_FACTS['CC-BY-NC-ND'])).toBe(0.125) + }) +}) +``` + +Append to `packages/core/src/__tests__/rights.test.ts`: + +```ts +import { factsOf, rightsRecordSchema } from '../rights' + +describe('RightsRecord facts', () => { + const base = { rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: 'u' } } as const + it('accepts a custom license id when facts are supplied', () => { + const r = rightsRecordSchema.parse({ ...base, license: 'acme-stock', facts: { commercialUse: true, derivatives: false, redistribution: false, attributionRequired: true, shareAlike: false } }) + expect(factsOf(r).derivatives).toBe(false) + }) + it('a custom id without facts resolves to the unknown row', () => { + const r = rightsRecordSchema.parse({ ...base, license: 'acme-stock' }) + expect(factsOf(r).commercialUse).toBe('unknown') + }) + it('facts override the table for a known id', () => { + const r = rightsRecordSchema.parse({ ...base, license: 'CC-BY', facts: { commercialUse: false, derivatives: true, redistribution: true, attributionRequired: true, shareAlike: false } }) + expect(factsOf(r).commercialUse).toBe(false) + }) + it('rejects licenseVersion on a non-CC-family license', () => { + expect(() => rightsRecordSchema.parse({ ...base, license: 'unsplash', licenseVersion: '4.0' })).toThrow() + expect(() => rightsRecordSchema.parse({ ...base, license: 'CC-BY', licenseVersion: '4.0' })).not.toThrow() + }) +}) +``` + +Append to `packages/core/src/__tests__/evaluate-use.test.ts`: + +```ts +describe('facts-driven gate', () => { + const base = { rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: 'u' } } as const + it('gates a custom id by its supplied facts', () => { + const r = { ...base, license: 'acme-stock', facts: { commercialUse: true, derivatives: false, redistribution: false, attributionRequired: false, shareAlike: false } } + expect(evaluateUse(r, 'commercial-product').decision).toBe('allowed') + expect(evaluateUse(r, 'ai-generation-input').decision).toBe('denied') + }) + it('a custom id without facts is needs-review with low confidence', () => { + const v = evaluateUse({ ...base, license: 'acme-stock' }, 'internal-moodboard') + expect(v.decision).toBe('needs-review') + expect(v.confidence).toBe('low') + }) +}) +``` + +Append to `packages/core/src/__tests__/merge.test.ts` (replace any existing `stricterLicense` tests with these): + +```ts +import { compareRestrictiveness, LICENSE_FACTS } from '../license' + +describe('cross-source rights resolution (facts)', () => { + const ref = (providerId: string, license: string, facts?: LicenseFacts): Reference => ({ + id: `${providerId}:1`, modality: 'image', source: { providerId, sourceUrl: 'https://x.test/a' }, canonicalUrl: 'https://x.test/a', + rights: { license, ...(facts ? { facts } : {}), rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: 'https://x.test/a' } }, + verifiedAt: new Date().toISOString(), relevance: 0, + }) + it('the stricter facts win regardless of id spelling', () => { + const custom: LicenseFacts = { commercialUse: true, derivatives: true, redistribution: true, attributionRequired: true, shareAlike: true } + const out = mergeReferences([[ref('a', 'CC0-1.0')], [ref('b', 'acme-sa', custom)]]) + expect(out[0].rights.license).toBe('acme-sa') + }) + it('an indeterminate side collapses the conflict to unknown', () => { + const out = mergeReferences([[ref('a', 'proprietary')], [ref('b', 'unknown')]]) + expect(out[0].rights.license).toBe('unknown') + }) + it('incomparable facts collapse to unknown', () => { + const out = mergeReferences([[ref('a', 'unsplash')], [ref('b', 'CC-BY')]]) + expect(out[0].rights.license).toBe('unknown') + expect(compareRestrictiveness(LICENSE_FACTS.unsplash, LICENSE_FACTS['CC-BY'])).toBe('incomparable') + }) +}) +``` + +(Import `LicenseFacts` and `Reference` types from `../license` / `../reference` at the top of the file.) + +Append to `packages/core/src/__tests__/rerank.test.ts`: + +```ts +it('license boost is derived from facts: CC0 outranks CC-BY outranks unknown', () => { + const mk = (id: string, license: string): Reference => ({ + id, modality: 'image', title: 'same', source: { providerId: 'p', sourceUrl: `https://x.test/${id}` }, canonicalUrl: `https://x.test/${id}`, + rights: { license, rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: 'u' } }, verifiedAt: new Date().toISOString(), relevance: 0, + }) + const out = lexicalReranker({ lexicalWeight: 0, qualityWeight: 0, licenseWeight: 1, sourceDiversity: 0 })({ query: 'same', refs: [mk('u', 'unknown'), mk('b', 'CC-BY'), mk('z', 'CC0-1.0')] }) as Reference[] + expect(out.map(r => r.id)).toEqual(['z', 'b', 'u']) +}) +``` + +- [ ] **Step 2: Run the new tests to confirm they fail** + +Run: `pnpm --filter @refkit/core test` +Expected: FAIL — `compareRestrictiveness`, `factsOf`, `isKnownLicenseId` not exported; refine not present. + +- [ ] **Step 3: Rewrite `license.ts`** + +Replace the file with: + +```ts +export const LICENSE_IDS = [ + 'CC0-1.0', 'CC-BY', 'CC-BY-SA', 'CC-BY-NC', 'CC-BY-NC-SA', 'CC-BY-NC-ND', 'CC-BY-ND', 'PD', + 'unsplash', 'pexels', 'pixabay', 'proprietary', 'unknown', +] as const + +export type KnownLicenseId = (typeof LICENSE_IDS)[number] +/** Open id: known ids resolve to LICENSE_FACTS; any other id must ship its own + * `facts` on the RightsRecord, else it is treated as `unknown` (strict-deny). */ +export type LicenseId = KnownLicenseId | (string & {}) + +export function isKnownLicenseId(id: string): id is KnownLicenseId { + return (LICENSE_IDS as readonly string[]).includes(id) +} + +/** Three-state: known-true / known-false / not-determinable. Drives strict-deny. */ +export type Tri = true | false | 'unknown' + +export interface LicenseFacts { + commercialUse: Tri + derivatives: Tri + redistribution: Tri + attributionRequired: boolean + shareAlike: boolean +} + +// Canonical, auditable license facts for the known ids. Conservative by design: +// anything not clearly granted is false/unknown. +export const LICENSE_FACTS: Record = { + 'CC0-1.0': { commercialUse: true, derivatives: true, redistribution: true, attributionRequired: false, shareAlike: false }, + 'PD': { commercialUse: true, derivatives: true, redistribution: true, attributionRequired: false, shareAlike: false }, + 'CC-BY': { commercialUse: true, derivatives: true, redistribution: true, attributionRequired: true, shareAlike: false }, + 'CC-BY-SA': { commercialUse: true, derivatives: true, redistribution: true, attributionRequired: true, shareAlike: true }, + // NC family: sharing/derivatives are granted only NON-commercially. The + // 'redistribution' intent doesn't model commercial vs non-commercial, so the + // honest tri-state is 'unknown' (→ needs-review). + 'CC-BY-NC': { commercialUse: false, derivatives: true, redistribution: 'unknown', attributionRequired: true, shareAlike: false }, + 'CC-BY-NC-SA': { commercialUse: false, derivatives: true, redistribution: 'unknown', attributionRequired: true, shareAlike: true }, + 'CC-BY-NC-ND': { commercialUse: false, derivatives: false, redistribution: 'unknown', attributionRequired: true, shareAlike: false }, + // ND: verbatim reuse (incl. commercial) is granted; derivatives are not. + 'CC-BY-ND': { commercialUse: true, derivatives: false, redistribution: true, attributionRequired: true, shareAlike: false }, + // Stock-platform licenses: free incl. commercial, no attribution legally required, + // but NOT redistributable as-is. + 'unsplash': { commercialUse: true, derivatives: true, redistribution: false, attributionRequired: false, shareAlike: false }, + 'pexels': { commercialUse: true, derivatives: true, redistribution: false, attributionRequired: false, shareAlike: false }, + 'pixabay': { commercialUse: true, derivatives: true, redistribution: false, attributionRequired: false, shareAlike: false }, + 'proprietary': { commercialUse: false, derivatives: false, redistribution: false, attributionRequired: false, shareAlike: false }, + 'unknown': { commercialUse: 'unknown', derivatives: 'unknown', redistribution: 'unknown', attributionRequired: false, shareAlike: false }, +} + +/** Resolve facts for an id; unrecognized → `unknown` (strict-deny fallback). */ +export function factsFor(license: LicenseId): LicenseFacts { + return (LICENSE_FACTS as Record)[license] ?? LICENSE_FACTS.unknown +} + +/** All three permission axes undeterminable — nothing can be granted or denied. */ +export function isIndeterminate(f: LicenseFacts): boolean { + return f.commercialUse === 'unknown' && f.derivatives === 'unknown' && f.redistribution === 'unknown' +} + +// — restrictiveness partial order (used by cross-source conflict resolution) — +// Each axis ranks smaller = stricter. One facts row is "no more permissive" than +// another when it is ≤ on EVERY axis; pairs that each grant something the other +// doesn't are incomparable. +const triRank = (t: Tri): number => (t === true ? 2 : t === 'unknown' ? 1 : 0) + +function permissivenessVector(f: LicenseFacts): number[] { + return [ + triRank(f.commercialUse), + triRank(f.derivatives), + triRank(f.redistribution), + f.attributionRequired ? 0 : 1, // carrying the obligation is stricter + f.shareAlike ? 0 : 1, + ] +} + +export function compareRestrictiveness(a: LicenseFacts, b: LicenseFacts): 'a' | 'b' | 'equal' | 'incomparable' { + const va = permissivenessVector(a) + const vb = permissivenessVector(b) + let aNoMorePermissive = true + let bNoMorePermissive = true + for (let i = 0; i < va.length; i++) { + if (va[i] > vb[i]) aNoMorePermissive = false + if (vb[i] > va[i]) bNoMorePermissive = false + } + if (aNoMorePermissive && bNoMorePermissive) return 'equal' + if (aNoMorePermissive) return 'a' + if (bNoMorePermissive) return 'b' + return 'incomparable' +} + +/** Scalar permissiveness in 0..1 for ranking boosts. Grants weigh 2, obligations 1; + * an 'unknown' axis counts as NOT granted, mirroring the strict-deny gate. */ +export function permissivenessScore(f: LicenseFacts): number { + const granted = (t: Tri): number => (t === true ? 1 : 0) + return ( + 2 * granted(f.commercialUse) + 2 * granted(f.derivatives) + 2 * granted(f.redistribution) + + (f.attributionRequired ? 0 : 1) + (f.shareAlike ? 0 : 1) + ) / 8 +} + +// — CC version metadata (attribution/audit only; never read by the gate) — + +/** The six versioned CC families — the only ids allowed to carry licenseVersion. */ +export const CC_VERSIONED_FAMILIES: ReadonlySet = new Set([ + 'CC-BY', 'CC-BY-SA', 'CC-BY-NC', 'CC-BY-NC-SA', 'CC-BY-NC-ND', 'CC-BY-ND', +]) + +/** `version` when `license` is a versioned CC family, else undefined. */ +export function ccVersionFor(license: LicenseId, version: string | undefined): string | undefined { + return version !== undefined && CC_VERSIONED_FAMILIES.has(license) ? version : undefined +} +``` + +- [ ] **Step 4: Update `rights.ts`** + +```ts +import { z } from 'zod' +import { CC_VERSIONED_FAMILIES, factsFor, type LicenseFacts, type LicenseId, type Tri } from './license' + +export type RehostPolicy = 'hotlink-required' | 'cache-allowed' | 'thumbnail-only' | 'no-store' + +// What a satellite emits per result. Permissions are derived from `license` via +// factsFor() unless the record ships its own `facts` (required for ids outside +// LICENSE_FACTS; an override for known ids whose source terms are narrower). +export interface RightsRecord { + license: LicenseId + /** Permission facts for this record. Read via factsOf(); never duplicated elsewhere. */ + facts?: LicenseFacts + /** Precise CC version ("4.0", "3.0", …) for the six CC families only. + * Attribution/audit only — never read by evaluateUse. */ + licenseVersion?: string + author?: string + rehostPolicy: RehostPolicy + /** Source-declared jurisdiction of the PD/copyright status (e.g. 'US'). */ + jurisdiction?: string + editorialOnly?: boolean + /** Auditable anchor back to the source's stated terms. */ + raw: { sourceTerms: string; sourceUrl: string } +} + +/** The facts that govern a record: its own row when supplied, else the table row. */ +export function factsOf(r: Pick): LicenseFacts { + return r.facts ?? factsFor(r.license) +} + +const triSchema: z.ZodType = z.union([z.literal(true), z.literal(false), z.literal('unknown')]) + +export const licenseFactsSchema: z.ZodType = z.object({ + commercialUse: triSchema, + derivatives: triSchema, + redistribution: triSchema, + attributionRequired: z.boolean(), + shareAlike: z.boolean(), +}) + +export const rightsRecordSchema: z.ZodType = z.object({ + license: z.string().min(1), + facts: licenseFactsSchema.optional(), + licenseVersion: z.string().optional(), + author: z.string().optional(), + rehostPolicy: z.enum(['hotlink-required', 'cache-allowed', 'thumbnail-only', 'no-store']), + jurisdiction: z.string().optional(), + editorialOnly: z.boolean().optional(), + raw: z.object({ sourceTerms: z.string(), sourceUrl: z.string() }), +}).refine( + r => r.licenseVersion === undefined || CC_VERSIONED_FAMILIES.has(r.license), + { message: 'licenseVersion is only valid on a versioned CC family license' }, +) +``` + +- [ ] **Step 5: Update `evaluate-use.ts`, `attribution.ts`, `merge.ts`, `rerank.ts`, `provider-helpers.ts`, `client.ts`, `index.ts`** + +`evaluate-use.ts` — replace the imports and the head of `evaluatePermissions`: + +```ts +import { isIndeterminate, type Tri } from './license' +import { factsOf, type RightsRecord } from './rights' +// … + const facts = factsOf(r) + const reasons: string[] = [] + const indeterminate = isIndeterminate(facts) + const confidence: 'high' | 'low' = indeterminate ? 'low' : 'high' + const base = { reasons, confidence, disclaimer: NOT_LEGAL_ADVICE } + + // Indeterminate facts: never allowed — needs-review regardless of required permissions. + if (indeterminate) { + reasons.push('license could not be determined (strict-deny)') + return { decision: 'needs-review', ...base } + } +``` +Everything else in the function is unchanged (it already reads `facts[perm]`). + +`attribution.ts`: + +```ts +import { type LicenseFacts, type LicenseId } from './license' +import { factsOf } from './rights' + +export interface AttributionInput { + license: LicenseId + facts?: LicenseFacts + licenseVersion?: string + canonicalUrl: string + author?: string + title?: string +} +// in buildAttribution: + const facts = factsOf(input) +``` + +`merge.ts` — delete `triRank`, `permissivenessVector`, `stricterLicense`; replace `resolveRightsConflict`: + +```ts +import { compareRestrictiveness, isIndeterminate, type LicenseId } from './license' +import { factsOf, type RightsRecord } from './rights' + +function unknownRecord(anchor: RightsRecord): RightsRecord { + // No honest single license exists for the conflict: strict-deny to 'unknown'. + // Keep the anchor's per-item data as the audit trail; drop facts/version that + // only made sense for the original id. + return { ...anchor, license: 'unknown', licenseVersion: undefined, facts: undefined } +} + +function resolveRightsConflict(current: RightsRecord, incoming: RightsRecord): RightsRecord { + const fa = factsOf(current) + const fb = factsOf(incoming) + // An indeterminate side grants nothing determinable — the conflict can only + // resolve to unknown. + if (isIndeterminate(fa) || isIndeterminate(fb)) return unknownRecord(current) + const cmp = compareRestrictiveness(fa, fb) + if (cmp === 'a' || cmp === 'equal') return current + if (cmp === 'b') return incoming + return unknownRecord(current) +} +``` +`RightsConflict.licenses` and `resolvedLicense` keep type `LicenseId`. + +`rerank.ts` — delete `LICENSE_PERMISSIVENESS`; import `permissivenessScore` from `./license` and `factsOf` from `./rights`; in the scoring line use `licW * permissivenessScore(factsOf(ref.rights))`. + +`provider-helpers.ts` — delete the `CC_VERSIONED_FAMILIES` and `ccVersionFor` definitions (lines under "Canonical membership set…"); keep `mapCcDeedUrl`, `mapRightsUrl`, `CC_FAMILY_BY_TOKEN` etc. `mapCcDeedUrl` needs no version helper. + +`client.ts` — in the returned `buildAttribution`, add `facts: ref.rights.facts,`. + +`index.ts` — replace the license/rights/merge/helpers export lines with: + +```ts +export { LICENSE_FACTS, LICENSE_IDS, factsFor, isKnownLicenseId, isIndeterminate, compareRestrictiveness, permissivenessScore, CC_VERSIONED_FAMILIES, ccVersionFor } from './license' +export type { LicenseId, KnownLicenseId, LicenseFacts, Tri } from './license' +export type { RehostPolicy, RightsRecord } from './rights' +export { rightsRecordSchema, licenseFactsSchema, factsOf } from './rights' +// … +export { mergeReferences } from './merge' +// … and drop CC_VERSIONED_FAMILIES / ccVersionFor from the provider-helpers export line +``` + +- [ ] **Step 6: Fix remaining compile errors and old tests** + +Run `pnpm typecheck`. Update any test that imported `stricterLicense` (replace with the facts tests from Step 1) or asserted the exact old permissiveness numbers. MCP `evaluate_use` / `build_attribution` keep `z.enum(LICENSE_IDS)` — no MCP change. + +- [ ] **Step 7: Verify** + +Run: `pnpm typecheck && pnpm lint && pnpm test:run` +Expected: all green. + +- [ ] **Step 8: Commit** + +```bash +git add -A packages/core packages/provider-testkit +git commit -m "refactor(core): license facts drive the gate, merge and reranker" +``` + +--- + +### Task 2: Delete the legacy filters / queryFeatures channels + +**Files:** +- Modify: `packages/core/src/provider.ts`, `packages/core/src/query.ts`, `packages/core/src/client.ts`, `packages/core/src/index.ts` +- Modify: `packages/mcp/src/index.ts` +- Test: `packages/core/src/__tests__/query.test.ts`, `client.test.ts`, `provider.test.ts`; `packages/mcp/src/__tests__/mcp.test.ts`; `packages/provider-unsplash/src/__tests__/unsplash.test.ts`, `packages/provider-pexels/src/__tests__/pexels.test.ts`, `packages/provider-pixabay/src/__tests__/pixabay.test.ts` + +**Interfaces:** +- Removed: `QueryFeature`, `SearchFilters`, `ReferenceProvider.queryFeatures`, `NormalizedQuery.filters`, `SearchInput.filters`, `SearchMeta.appliedFilters`, `mergeSearchControls`, MCP `filters` parameter and `appliedFilters` in its meta schema. +- Produces: `normalizeQuery(input: { query, modalities, controls?, providerOptions?, limit? }, provider)`. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/core/src/__tests__/query.test.ts`: + +```ts +it('routes only the controls a provider declares in capabilities.controls; a provider without capabilities gets none', () => { + const p = defineProvider({ id: 'p', modalities: ['image'], capabilities: { controls: ['color'] }, search: async () => [] }) + const q = normalizeQuery({ query: 'x', modalities: ['image'], controls: { color: 'red', orientation: 'landscape' } }, p) + expect(q.controls).toEqual({ color: 'red' }) + expect('filters' in q).toBe(false) + const bare = defineProvider({ id: 'b', modalities: ['image'], search: async () => [] }) + expect(normalizeQuery({ query: 'x', modalities: ['image'], controls: { color: 'red' } }, bare).controls).toBeUndefined() +}) +``` + +- [ ] **Step 2: Run to confirm it fails** + +Run: `pnpm --filter @refkit/core test` +Expected: FAIL on `'filters' in q` (the mirror is still emitted). + +- [ ] **Step 3: Remove the legacy surface in core** + +`provider.ts`: delete `QueryFeature`, `SearchFilters`, the `queryFeatures` member and the `filters` member of `NormalizedQuery` (with their doc comments). + +`query.ts`: delete `LEGACY_FEATURE_CONTROLS`, `controlsFromFilters`, `mergeSearchControls`; replace `effectiveControlCaps` with: + +```ts +function effectiveControlCaps(provider: ReferenceProvider): readonly SearchControlKey[] { + return provider.capabilities?.controls ?? [] +} +``` + +Replace `normalizeControlsForProvider` and `normalizeQuery`: + +```ts +export function normalizeControlsForProvider(controls: SearchControls | undefined, provider: ReferenceProvider): SearchControls | undefined { + if (!controls) return undefined + const supported = supportedControlKeys(provider, controls) + if (supported.length === 0) return undefined + const out: SearchControls = {} + for (const key of supported) setControl(out, key, controls) + return out +} + +export function normalizeQuery( + input: { query: string; modalities: Modality[]; controls?: SearchControls; providerOptions?: ProviderOptionsById; limit?: number }, + provider: ReferenceProvider, +): NormalizedQuery { + const controls = normalizeControlsForProvider(input.controls, provider) + return { + text: input.query, + modalities: input.modalities.filter(m => provider.modalities.includes(m)), + ...(controls ? { controls } : {}), + ...(input.providerOptions?.[provider.id] ? { providerOptions: input.providerOptions[provider.id] } : {}), + ...(input.limit !== undefined ? { limit: input.limit } : {}), + } +} +``` + +`client.ts`: remove `SearchFilters` import and `mergeSearchControls` import; delete `SearchInput.filters` and `SearchMeta.appliedFilters`; in `runPass` use `const requestedControls = requestedControlKeys(controls ?? {})` and pass `controls ?? {}` to `supportedControlKeys`/`unsupportedControlKeys`; drop `filters: input.filters` from the `normalizeQuery` call; drop the `appliedFilters` spread from meta. + +`index.ts`: remove `QueryFeature` and `SearchFilters` from the type export list. + +- [ ] **Step 4: Remove the MCP filters parameter** + +In `packages/mcp/src/index.ts`: delete `filtersSchema`, the `filters` entry of `inputSchema`, the `filters` destructure and `filters:` line in `searchInput`, the `appliedFilters` line in `searchMetaSchema`, and the `SearchFilters` type import. + +- [ ] **Step 5: Update tests** + +- `query.test.ts`: delete the cases titled "routes legacy filters…", "legacy compat…", "capabilities, once declared, win over queryFeatures", "omits filters entirely…", "maps legacy filters into controls…", "prefers primary controls over conflicting legacy filters…". Keep the rest. +- `provider.test.ts`: replace `queryFeatures: [...]` in fixtures with `capabilities: { controls: [...] }` (map `orientation` → `'orientation'`, `keyword` → nothing). +- `client.test.ts`: delete cases whose subject is `filters` / `appliedFilters`; where a case merely passes `filters: { … }` incidentally, rewrite it as `controls: { … }`. +- `mcp.test.ts`: same rule — delete filters-subject cases, convert incidental uses. +- `unsplash.test.ts`, `pexels.test.ts`, `pixabay.test.ts`: delete the "keeps primary controls ahead of conflicting legacy filters…" cases (and any other case passing `filters:`). + +- [ ] **Step 6: Verify** + +Run: `pnpm typecheck && pnpm lint && pnpm test:run` +Expected: green; `grep -rn "filters\|queryFeatures" packages/*/src --include='*.ts'` returns nothing. + +- [ ] **Step 7: Commit** + +```bash +git add -A packages +git commit -m "refactor: remove the legacy filters and queryFeatures channels" +``` + +--- + +### Task 3: One control registry; zod schemas exported from core and reused by MCP + +**Files:** +- Create: `packages/core/src/controls.ts` +- Create: `packages/core/src/schemas.ts` +- Modify: `packages/core/src/modality.ts`, `provider.ts`, `query.ts`, `reference.ts`, `client.ts`, `index.ts` +- Modify: `packages/mcp/src/index.ts` +- Test: `packages/core/src/__tests__/controls.test.ts` (new), `packages/mcp/src/__tests__/mcp.test.ts` + +**Interfaces:** +- Produces (from `@refkit/core`): `MODALITIES` tuple; `CONTROL_PATHS`, `SEARCH_CONTROL_KEYS`, `getControl`, `setControl`, `hasControl`, `buildSearchControlsSchema(kinds?)`, `searchControlsSchema`; `PROVIDER_SKIP_REASONS`, `ProviderSkipReason`; `searchMetaSchema`, `providerSearchStatusSchema`, `searchControlKeySchema`. +- `SearchControlKey` is now `keyof typeof CONTROL_PATHS`. + +- [ ] **Step 1: Write the failing test** + +Create `packages/core/src/__tests__/controls.test.ts`: + +```ts +import { describe, expect, it } from 'vitest' +import { CONTROL_PATHS, SEARCH_CONTROL_KEYS, buildSearchControlsSchema, getControl, hasControl, setControl, type SearchControls } from '../controls' + +describe('control registry', () => { + it('lists every key exactly once and in registry order', () => { + expect(SEARCH_CONTROL_KEYS).toEqual(Object.keys(CONTROL_PATHS)) + expect(new Set(SEARCH_CONTROL_KEYS).size).toBe(SEARCH_CONTROL_KEYS.length) + expect(SEARCH_CONTROL_KEYS).toContain('license.commercial') + expect(SEARCH_CONTROL_KEYS).toContain('page') + }) + it('get/set/has walk nested paths', () => { + const c: SearchControls = { license: { commercial: true }, page: 2 } + expect(getControl(c, 'license.commercial')).toBe(true) + expect(getControl(c, 'media.kind')).toBeUndefined() + expect(hasControl(c, 'page')).toBe(true) + const out: SearchControls = {} + setControl(out, 'media.kind', 'photo') + setControl(out, 'media.minWidth', 100) + setControl(out, 'sort', 'latest') + expect(out).toEqual({ media: { kind: 'photo', minWidth: 100 }, sort: 'latest' }) + }) + it('schema accepts any kind by default and only the given kinds when restricted', () => { + expect(buildSearchControlsSchema().safeParse({ media: { kind: 'anything' } }).success).toBe(true) + expect(buildSearchControlsSchema(['photo']).safeParse({ media: { kind: 'texture' } }).success).toBe(false) + expect(buildSearchControlsSchema(['photo']).safeParse({ media: { kind: 'photo' }, page: 1 }).success).toBe(true) + }) +}) +``` + +- [ ] **Step 2: Run to confirm it fails** + +Run: `pnpm --filter @refkit/core test` +Expected: FAIL — module `../controls` not found. + +- [ ] **Step 3: Create `controls.ts`** + +```ts +import { z } from 'zod' + +export type SearchSort = 'relevance' | 'latest' | 'popular' | 'interesting' +export type SearchSafety = 'strict' | 'moderate' | 'off' + +/** Fine-grained resource kind. Open vocabulary: well-known values get + * autocomplete; any other string is a valid custom kind. */ +export type WellKnownKind = + | 'photo' | 'illustration' | 'vector' | 'icon' | 'artwork' + | 'texture' | 'hdri' | '3d-model' + | 'film' | 'animation' + | 'music' | 'sound-effect' + | 'ebook' | 'poem' +export type ResourceKind = WellKnownKind | (string & {}) + +export interface SearchLicenseControls { + commercial?: boolean + modification?: boolean + allowUnknown?: boolean +} +export interface SearchMediaControls { + kind?: ResourceKind + size?: 'small' | 'medium' | 'large' + minWidth?: number + minHeight?: number + duration?: 'short' | 'medium' | 'long' +} +export interface SearchCreatorControls { id?: string; name?: string } +export interface SearchTextControls { copyright?: 'public-domain' | 'copyrighted' | 'any' } + +export interface SearchControls { + orientation?: 'landscape' | 'portrait' | 'square' + color?: string + language?: string + sort?: SearchSort + safety?: SearchSafety + license?: SearchLicenseControls + media?: SearchMediaControls + creator?: SearchCreatorControls + text?: SearchTextControls + /** Provider-local page (1-based): each provider paginates its own stream. */ + page?: number +} + +/** The single control registry: every routable control as its path inside + * SearchControls. The key union, key list, accessors and zod schema all derive + * from this table — adding a control means adding one row here (and one field + * in SearchControls + the schema builder below). */ +export const CONTROL_PATHS = { + orientation: ['orientation'], + color: ['color'], + language: ['language'], + sort: ['sort'], + safety: ['safety'], + 'license.commercial': ['license', 'commercial'], + 'license.modification': ['license', 'modification'], + 'license.allowUnknown': ['license', 'allowUnknown'], + 'media.kind': ['media', 'kind'], + 'media.size': ['media', 'size'], + 'media.minWidth': ['media', 'minWidth'], + 'media.minHeight': ['media', 'minHeight'], + 'media.duration': ['media', 'duration'], + 'creator.id': ['creator', 'id'], + 'creator.name': ['creator', 'name'], + 'text.copyright': ['text', 'copyright'], + page: ['page'], +} as const satisfies Record + +export type SearchControlKey = keyof typeof CONTROL_PATHS +export const SEARCH_CONTROL_KEYS = Object.keys(CONTROL_PATHS) as SearchControlKey[] + +type Path = readonly [keyof SearchControls, string?] + +export function getControl(controls: SearchControls, key: SearchControlKey): unknown { + const [head, tail] = CONTROL_PATHS[key] as Path + const value = controls[head] + return tail === undefined ? value : (value as Record | undefined)?.[tail] +} + +export function hasControl(controls: SearchControls, key: SearchControlKey): boolean { + return getControl(controls, key) !== undefined +} + +/** Write `value` at the key's path (creating the nested group as needed). */ +export function setControl(out: SearchControls, key: SearchControlKey, value: unknown): void { + const [head, tail] = CONTROL_PATHS[key] as Path + const target = out as Record + if (tail === undefined) { target[head] = value; return } + target[head] = { ...((target[head] as Record | undefined) ?? {}), [tail]: value } +} + +/** Zod schema for SearchControls. `kinds` restricts media.kind to a closed enum + * (the MCP server passes the union of registered providers' kinds). */ +export function buildSearchControlsSchema(kinds?: readonly string[]): z.ZodType { + const kind = kinds && kinds.length > 0 ? z.enum(kinds as [string, ...string[]]) : z.string() + return z.object({ + orientation: z.enum(['landscape', 'portrait', 'square']).optional(), + color: z.string().optional(), + language: z.string().optional(), + sort: z.enum(['relevance', 'latest', 'popular', 'interesting']).optional(), + safety: z.enum(['strict', 'moderate', 'off']).optional(), + license: z.object({ + commercial: z.boolean().optional(), + modification: z.boolean().optional(), + allowUnknown: z.boolean().optional(), + }).optional(), + media: z.object({ + kind: kind.optional(), + size: z.enum(['small', 'medium', 'large']).optional(), + minWidth: z.number().int().nonnegative().optional(), + minHeight: z.number().int().nonnegative().optional(), + duration: z.enum(['short', 'medium', 'long']).optional(), + }).optional(), + creator: z.object({ id: z.string().optional(), name: z.string().optional() }).optional(), + text: z.object({ copyright: z.enum(['public-domain', 'copyrighted', 'any']).optional() }).optional(), + page: z.number().int().positive().optional(), + }) +} + +export const searchControlsSchema: z.ZodType = buildSearchControlsSchema() +``` + +- [ ] **Step 4: Rewire `provider.ts`, `query.ts`, `modality.ts`, `reference.ts`, `client.ts`** + +`modality.ts`: +```ts +export const MODALITIES = ['image', 'video', 'audio', 'text'] as const +export type Modality = (typeof MODALITIES)[number] +``` +`reference.ts`: `const modalitySchema: z.ZodType = z.enum(MODALITIES)` (import `MODALITIES`). + +`provider.ts`: delete the moved type definitions (`SearchSort` … `SearchControlKey`) and re-export them for internal consumers: `export type { SearchControls, SearchControlKey, SearchSort, SearchSafety, WellKnownKind, ResourceKind, SearchLicenseControls, SearchMediaControls, SearchCreatorControls, SearchTextControls } from './controls'`; keep `ProviderCapabilities { controls: readonly SearchControlKey[] }`, `ProviderOptions*`, `NormalizedQuery`, `KeyValueCache`, `ProviderContext`, `ReferenceProvider`, `defineProvider`. + +`query.ts`: delete the local `hasControl`, `setControl`, and the `allControlKeys` array; import `{ SEARCH_CONTROL_KEYS, getControl, hasControl, setControl }` from `./controls`; + +```ts +export function requestedControlKeys(controls: SearchControls): SearchControlKey[] { + return SEARCH_CONTROL_KEYS.filter(key => hasControl(controls, key)) +} +// … in normalizeControlsForProvider: + for (const key of supported) setControl(out, key, getControl(controls, key)) +``` + +`client.ts`: add near the top +```ts +export const PROVIDER_SKIP_REASONS = ['unsupported-modality', 'unsupported-kind', 'not-selected'] as const +export type ProviderSkipReason = (typeof PROVIDER_SKIP_REASONS)[number] +``` +and use `reason?: ProviderSkipReason` in `ProviderSearchStatus`; replace the inline `NonNullable` usages with `ProviderSkipReason`. + +- [ ] **Step 5: Create `schemas.ts`** + +```ts +import { z } from 'zod' +import { MODALITIES } from './modality' +import { SEARCH_CONTROL_KEYS } from './controls' +import { INTENTS } from './evaluate-use' +import { PROVIDER_SKIP_REASONS, type ProviderSearchStatus, type SearchMeta } from './client' + +export const searchControlKeySchema = z.enum(SEARCH_CONTROL_KEYS as [string, ...string[]]) + +export const providerSearchStatusSchema: z.ZodType = z.object({ + providerId: z.string(), + status: z.enum(['fulfilled', 'failed', 'skipped']), + returned: z.number().optional(), + accepted: z.number().optional(), + rejected: z.number().optional(), + reason: z.enum(PROVIDER_SKIP_REASONS).optional(), + error: z.string().optional(), + latencyMs: z.number().optional(), + cached: z.boolean().optional(), +}) + +export const searchMetaSchema: z.ZodType = z.object({ + query: z.string(), + modalities: z.array(z.enum(MODALITIES)), + limit: z.number(), + poolFactor: z.number(), + fetchLimit: z.number(), + controls: z.object({ + requested: z.array(searchControlKeySchema), + appliedByProvider: z.record(z.string(), z.array(searchControlKeySchema)), + ignoredByProvider: z.record(z.string(), z.array(searchControlKeySchema)), + }).optional(), + providerOptions: z.array(z.string()).optional(), + providers: z.array(providerSearchStatusSchema), + gate: z.object({ intent: z.enum(INTENTS), before: z.number(), after: z.number(), dropped: z.number() }).optional(), + nextCursor: z.string().optional(), + warnings: z.array(z.string()), +}) as z.ZodType +``` +(If zod's inferred type for `controls.requested` is `string[]` and TypeScript rejects the `z.ZodType` annotation, keep the trailing `as z.ZodType` cast — the runtime enum is the exact key list.) + +`index.ts` additions: +```ts +export { MODALITIES } from './modality' +export { CONTROL_PATHS, SEARCH_CONTROL_KEYS, getControl, setControl, hasControl, buildSearchControlsSchema, searchControlsSchema } from './controls' +export { searchMetaSchema, providerSearchStatusSchema, searchControlKeySchema } from './schemas' +export { PROVIDER_SKIP_REASONS } from './client' +export type { ProviderSkipReason } from './client' +``` +(Keep exporting the control types via the existing `from './provider'` type block or move them to `from './controls'` — either is fine, but each name must be exported exactly once.) + +- [ ] **Step 6: MCP reuses the core schemas** + +In `packages/mcp/src/index.ts`: delete `MODALITIES`, `ORIENTATIONS`, `SEARCH_CONTROL_KEYS`, `searchControlKeySchema`, `buildSearchControlsSchema`, `searchMetaSchema`. Import `{ buildSearchControlsSchema, searchMetaSchema, MODALITIES }` from `@refkit/core` (plus what it already imports). Keep `BASE_MEDIA_KINDS` and build `const searchControlsSchema = buildSearchControlsSchema(kindValues)`. `outputSchema.meta: searchMetaSchema.optional()` stays. Delete the `SearchControlKey`/`SearchMeta` type imports if unused. + +- [ ] **Step 7: Verify and commit** + +Run: `pnpm typecheck && pnpm lint && pnpm test:run` — green. Also `grep -n "'orientation'" packages/mcp/src/index.ts` must show no hand-written control key list. + +```bash +git add -A packages/core packages/mcp +git commit -m "refactor(core,mcp): single control registry and core-exported search schemas" +``` + +--- + +### Task 4: Providers emit, core completes — EmittedReference contract, okJson, default User-Agent, provider migration + +**Files:** +- Modify: `packages/core/src/reference.ts`, `provider.ts`, `provider-run.ts`, `provider-helpers.ts`, `resilience.ts`, `client.ts`, `index.ts` +- Modify: `packages/provider-testkit/src/index.ts` +- Modify: every `packages/provider-*/src/index.ts` (19 packages) and their tests +- Modify: `packages/mcp/src/__tests__/mcp.test.ts`, `packages/core/src/__tests__/*.test.ts` fakes that return `Reference` from `search` +- Test: `packages/core/src/__tests__/reference.test.ts`, `provider-run.test.ts`, `resilience.test.ts`, `client.test.ts` + +**Interfaces:** +- Produces (from `@refkit/core`): + ```ts + interface EmittedReference { + modality: Modality; kind?: string; title?: string; description?: string; tags?: string[] + sourceUrl: string; canonicalUrl?: string; rights: RightsRecord + thumbnail?: ReferenceMedia; preview?: MediaPreview; perceptualHash?: string + visual?: VisualMeta; text?: TextMeta; sourceScore?: number; raw?: unknown + } + interface Reference { id: string; modality; kind?; title?; description?; tags?; source: { providerId; sourceUrl }; canonicalUrl: string; rights; verifiedAt: string; thumbnail?; preview?; perceptualHash?; visual?; text?; relevance: number; sourceScore?; raw? } + emittedReferenceSchema; parseEmitted(input): EmittedReference + completeReference(providerId: string, e: EmittedReference, now: string): Reference + okJson(res: Response, label: string): Promise + withDefaultUserAgent(fetchImpl: typeof fetch, ua: string): typeof fetch + RefkitOptions.userAgent?: string | false // default 'refkit-client/1' + ReferenceProvider.search(query, ctx): Promise + ``` +- `runProviderSearch` parses each raw item with `parseEmitted`, completes it, and truncates the batch to `query.limit`. + +- [ ] **Step 1: Write failing core tests** + +Append to `packages/core/src/__tests__/reference.test.ts`: + +```ts +import { completeReference, parseEmitted } from '../reference' + +describe('EmittedReference → Reference', () => { + const emitted = { + modality: 'image', title: 'T', sourceUrl: 'https://X.test/a/', tags: ['t1'], + rights: { license: 'CC0-1.0', rehostPolicy: 'cache-allowed', raw: { sourceTerms: 't', sourceUrl: 'https://x.test/a' } }, + sourceScore: 12.5, + } + it('completeReference stamps id, source, canonicalUrl, verifiedAt and relevance', () => { + const r = completeReference('p', parseEmitted(emitted), '2026-01-01T00:00:00.000Z') + expect(r.id).toMatch(/^p:[0-9a-z]+$/) + expect(r.source).toEqual({ providerId: 'p', sourceUrl: 'https://X.test/a/' }) + expect(r.canonicalUrl).toBe('https://X.test/a/') + expect(r.verifiedAt).toBe('2026-01-01T00:00:00.000Z') + expect(r.relevance).toBe(0) + expect(r.tags).toEqual(['t1']) + expect(r.sourceScore).toBe(12.5) + expect('sourceUrl' in r).toBe(false) + }) + it('an explicit canonicalUrl is kept', () => { + const r = completeReference('p', parseEmitted({ ...emitted, canonicalUrl: 'https://x.test/canon' }), '2026-01-01T00:00:00.000Z') + expect(r.canonicalUrl).toBe('https://x.test/canon') + expect(r.source.sourceUrl).toBe('https://X.test/a/') + }) + it('parseEmitted rejects a missing sourceUrl', () => { + expect(() => parseEmitted({ ...emitted, sourceUrl: undefined })).toThrow() + }) +}) +``` + +Append to `packages/core/src/__tests__/provider-run.test.ts`: + +```ts +it('completes emitted items and truncates to query.limit', async () => { + const provider = defineProvider({ + id: 'p', modalities: ['image'], + search: async () => Array.from({ length: 5 }, (_, i) => ({ + modality: 'image' as const, sourceUrl: `https://x.test/${i}`, + rights: { license: 'CC0-1.0', rehostPolicy: 'cache-allowed' as const, raw: { sourceTerms: 't', sourceUrl: 'u' } }, + })), + }) + const run = await runProviderSearch(provider, { text: 'q', modalities: ['image'], limit: 3 }, { fetch: (async () => new Response('')) as typeof fetch, cacheTtlMs: 0, cacheRaw: true }) + expect(run.ok && run.valid.length).toBe(3) + expect(run.ok && run.returned).toBe(5) + expect(run.ok && run.valid[0].source.providerId).toBe('p') + expect(run.ok && run.valid[0].id.startsWith('p:')).toBe(true) +}) +``` + +Append to `packages/core/src/__tests__/resilience.test.ts`: + +```ts +import { withDefaultUserAgent } from '../resilience' + +it('withDefaultUserAgent adds a UA only when the request has none', async () => { + const seen: string[] = [] + const inner = (async (_i: unknown, init?: RequestInit) => { seen.push(new Headers(init?.headers).get('user-agent') ?? '(none)'); return new Response('') }) as typeof fetch + const f = withDefaultUserAgent(inner, 'refkit-client/1') + await f('https://x.test/') + await f('https://x.test/', { headers: { 'User-Agent': 'custom/2' } }) + expect(seen).toEqual(['refkit-client/1', 'custom/2']) +}) +``` + +- [ ] **Step 2: Run to confirm they fail** + +Run: `pnpm --filter @refkit/core test` +Expected: FAIL — `completeReference`, `parseEmitted`, `withDefaultUserAgent` missing. + +- [ ] **Step 3: Rewrite `reference.ts`** + +```ts +import { z } from 'zod' +import { MODALITIES, type Modality } from './modality' +import { rightsRecordSchema, type RightsRecord } from './rights' +import { referenceId } from './dedup-key' + +export interface ReferenceMedia { url: string; width?: number; height?: number } +export interface MediaPreview { url: string; mediaType: string; width?: number; height?: number } +export interface VisualMeta { width: number; height: number; dominantColors?: string[] } +export interface TextMeta { + excerpt: string + excerptKind: 'passage' | 'structure' | 'quote' + locator?: string +} + +/** What a provider emits for one result: everything the SOURCE knows. Core stamps + * id, source, verifiedAt and relevance (see completeReference) — providers never + * write those, never post-truncate, and never compute ids. */ +export interface EmittedReference { + modality: Modality + /** Fine-grained kind (open vocabulary, see ResourceKind), e.g. 'photo', 'texture'. */ + kind?: string + title?: string + /** Free-text description from the source (caption, medium, synopsis…). Feeds ranking. */ + description?: string + /** Source tags / subjects / categories. Feeds ranking. */ + tags?: string[] + /** Landing page at the source. Also the canonical URL unless canonicalUrl is set. */ + sourceUrl: string + canonicalUrl?: string + rights: RightsRecord + thumbnail?: ReferenceMedia + preview?: MediaPreview + /** Computed by the satellite (pHash/blockhash); core only compares it. */ + perceptualHash?: string + visual?: VisualMeta + text?: TextMeta + /** Upstream relevance score in the source's own scale; only the order within one + * source is meaningful. */ + sourceScore?: number + raw?: unknown +} + +export interface Reference extends Omit { + /** Content-addressed: `${providerId}:${hash(sourceUrl)}`; stable within a result set. */ + id: string + source: { providerId: string; sourceUrl: string } + canonicalUrl: string + /** ISO; the moment the satellite's output was parsed. */ + verifiedAt: string + /** 0..1, meaningful only after merge (RRF) or rerank; providers never set it. */ + relevance: number +} + +const modalitySchema: z.ZodType = z.enum(MODALITIES) +const mediaSchema = z.object({ url: z.string(), width: z.number().optional(), height: z.number().optional() }) +const previewSchema = z.object({ url: z.string(), mediaType: z.string(), width: z.number().optional(), height: z.number().optional() }) +const visualSchema = z.object({ width: z.number(), height: z.number(), dominantColors: z.array(z.string()).optional() }) +const textSchema = z.object({ excerpt: z.string(), excerptKind: z.enum(['passage', 'structure', 'quote']), locator: z.string().optional() }) + +const emittedFields = { + modality: modalitySchema, + kind: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + tags: z.array(z.string()).optional(), + rights: rightsRecordSchema, + thumbnail: mediaSchema.optional(), + preview: previewSchema.optional(), + perceptualHash: z.string().optional(), + visual: visualSchema.optional(), + text: textSchema.optional(), + sourceScore: z.number().optional(), + raw: z.unknown().optional(), +} + +export const emittedReferenceSchema: z.ZodType = z.object({ + ...emittedFields, + sourceUrl: z.string().min(1), + canonicalUrl: z.string().min(1).optional(), +}) + +export const referenceSchema: z.ZodType = z.object({ + ...emittedFields, + id: z.string().min(1), + source: z.object({ providerId: z.string().min(1), sourceUrl: z.string().min(1) }), + canonicalUrl: z.string().min(1), + verifiedAt: z.string().datetime(), + relevance: z.number().min(0).max(1), +}) + +/** Validate a provider-emitted item at the core boundary. Throws on malformed input. */ +export function parseEmitted(input: unknown): EmittedReference { + return emittedReferenceSchema.parse(input) +} + +/** Validate a complete reference (cache hits, host-supplied refs). */ +export function parseReference(input: unknown): Reference { + return referenceSchema.parse(input) +} + +/** Stamp the fields that are core's concern onto an emitted item. */ +export function completeReference(providerId: string, e: EmittedReference, now: string): Reference { + const { sourceUrl, canonicalUrl, ...rest } = e + return { + ...rest, + id: referenceId(providerId, sourceUrl), + source: { providerId, sourceUrl }, + canonicalUrl: canonicalUrl ?? sourceUrl, + verifiedAt: now, + relevance: 0, + } +} +``` + +- [ ] **Step 4: Core plumbing** + +`provider.ts`: `search(query: NormalizedQuery, ctx: ProviderContext): Promise` (import the type). + +`provider-run.ts`: import `{ completeReference, parseEmitted, parseReference }`; replace `parseItems` with two helpers and apply the limit: + +```ts + const parseCached = (raw: unknown[]): Reference[] => { + const valid: Reference[] = [] + for (const item of raw) { + try { valid.push(parseReference(item)) } catch (error) { deps.onError?.(error) } + } + return valid + } + const completeEmitted = (raw: unknown[]): Reference[] => { + const now = new Date().toISOString() + const valid: Reference[] = [] + for (const item of raw) { + try { valid.push(completeReference(provider.id, parseEmitted(item), now)) } catch (error) { deps.onError?.(error) } + } + return valid + } + const truncate = (refs: Reference[]): Reference[] => + typeof query.limit === 'number' && query.limit > 0 ? refs.slice(0, query.limit) : refs +``` +Cache-hit path: `const valid = truncate(parseCached(payload.refs))`. Live path: `const valid = truncate(completeEmitted(raw))`. Cache write stores `valid` (already completed). `returned: raw.length` unchanged. + +`provider-helpers.ts` append: +```ts +/** Throw `${label} failed: ${status}` on a non-2xx response, else parse the JSON body. */ +export async function okJson(res: Response, label: string): Promise { + if (!res.ok) throw new Error(`${label} failed: ${res.status}`) + return (await res.json()) as T +} +``` + +`resilience.ts` append: +```ts +/** Add a User-Agent to requests that carry none (Node's default UA is rejected by + * some source edges). Browsers ignore the header silently. */ +export function withDefaultUserAgent(fetchImpl: typeof fetch, ua: string): typeof fetch { + const wrapped = (input: Parameters[0], init?: Parameters[1]): Promise => { + const fromRequest = typeof Request !== 'undefined' && input instanceof Request ? input.headers : undefined + const headers = new Headers(init?.headers ?? fromRequest) + if (!headers.has('user-agent')) headers.set('user-agent', ua) + return fetchImpl(input, { ...init, headers }) + } + return wrapped as typeof fetch +} +``` + +`client.ts`: `RefkitOptions.userAgent?: string | false` with doc "Default 'refkit-client/1'; false disables"; `const DEFAULT_USER_AGENT = 'refkit-client/1'`; when building `sharedFetch`, wrap: `const withRetry = …; const sharedFetch = options.userAgent === false ? withRetry : withDefaultUserAgent(withRetry, options.userAgent ?? DEFAULT_USER_AGENT)`. + +`index.ts`: export `EmittedReference` type, `emittedReferenceSchema`, `parseEmitted`, `completeReference`, `okJson`, `withDefaultUserAgent`. + +- [ ] **Step 5: Testkit** + +In `packages/provider-testkit/src/index.ts` replace the body of `searchConformant` after the `provider.search` call: + +```ts + const raw = await provider.search(query, ctx) + const enforceImages = opts.enforceImageUrls ?? provider.modalities.includes('image') + const now = new Date().toISOString() + return raw.map((item, i) => { + let ref: Reference + try { + ref = completeReference(provider.id, parseEmitted(item), now) + } catch (e) { + throw new Error(`[${provider.id}] result #${i} failed emittedReferenceSchema: ${(e as Error).message}`) + } + if (provider.kinds && provider.kinds.length > 0 && ref.kind !== undefined && !provider.kinds.includes(ref.kind)) { + throw new Error(`[${provider.id}] result #${i} kind "${ref.kind}" is not in the provider's declared kinds [${provider.kinds.join(', ')}]`) + } + if (enforceImages) { + // … keep the two D8 checks exactly as they are today … + } + return ref + }) +``` +Delete the `VERSIONED` constant, the id-prefix check, the `source.providerId` check and the licenseVersion check. Update imports (`completeReference`, `parseEmitted`, drop `parseReference`, `CC_VERSIONED_FAMILIES`, `LicenseId` if unused). Update `packages/provider-testkit/src/__tests__/testkit.test.ts` accordingly (fake providers now emit `sourceUrl`, no `id`/`source`). + +- [ ] **Step 6: Migrate every provider package (mechanical)** + +Apply to each `packages/provider-*/src/index.ts` (artic, brave, europeana, flickr, freesound, gutendex, internet-archive, jamendo, met, nailbook, openverse, pexels, pixabay, poetrydb, polyhaven, rijksmuseum, smithsonian, unsplash, wikimedia-commons — every factory in the file, including the audio/video/second factories): + +1. Import `type EmittedReference` instead of `type Reference`; import `okJson`; drop `referenceId` from the import. +2. In each `toReference`: return type `EmittedReference` (or `EmittedReference | null`); delete the `id:`, `source:`, `verifiedAt:`, `relevance: 0` lines; rename `canonicalUrl: X` to `sourceUrl: X` (every current provider uses the same URL for both; if you find one where `canonicalUrl` and `source.sourceUrl` differ, keep `sourceUrl` = the old `source.sourceUrl` and add `canonicalUrl` = the old `canonicalUrl`). +3. Replace each `if (!res.ok) throw new Error(\`