diff --git a/docs/api.md b/docs/api.md index 27aa4c9..a515aa6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -20,7 +20,7 @@ see [architecture.md](architecture.md#information-flow) for that flow. | `buildSpaceGraph(nodes, hierarchy?)` | Build a navigable `SpaceGraph` directly from an array of `SpaceNode`s and a `Hierarchy` definition. | | `exportSpaceGraph(nodes, metadata)` | Turn nodes and the schema's hierarchies into plain JSON for another process. See [Space graph format](#space-graph-format). | | `spaceGraphFrom(data, options?)` | Build a `SpaceGraph` from that JSON. `options.hierarchy` names the hierarchy that structures it (default: the main hierarchy). | -| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. | +| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. Semantic rules run only when `semantic` is passed; it takes the same values as `sctx validate --semantic`, with the provider coming from config and the environment. | | `validateFile(filePath, config)` | Validate a single file within its space. | | `listSchemas(options?)` | List the schemas available to a config: bundled, plugin-contributed and configured. | | `formatWikilink(target, alias?)` | Format a target into an Obsidian wikilink (`[[target]]` or `[[target|alias]]`). | diff --git a/src/api.ts b/src/api.ts index f1b632c..34f4c48 100644 --- a/src/api.ts +++ b/src/api.ts @@ -25,6 +25,7 @@ export type { Config, SpaceConfig } from './config'; export { loadConfig, resolveConfigPath, resolveSchema, resolveSchemaPath, setConfigPath } from './config'; export { addSpace } from './config/write'; export { LOCAL_CONFIG_FILE, PLUGIN_PREFIX } from './constants'; +export { SctxError, type SctxErrorCode } from './errors'; export { assembleSpaceGraph, loadSpaceGraph } from './load-space-graph'; export type { ParseHook, diff --git a/src/commands/validate-file.ts b/src/commands/validate-file.ts index c7f5d27..59a5408 100644 --- a/src/commands/validate-file.ts +++ b/src/commands/validate-file.ts @@ -1,9 +1,7 @@ import { loadConfig } from '../config'; -import { judgeForRun } from '../judge/factory'; -import { parseRuleFilter } from '../schema/rule-filter'; import { createSpaceContext } from '../space-context'; import type { FileValidationIssue, FileValidationResult, ValidateFileOutput } from '../types'; -import { fileNotInSpace, resolveFileSpace, validateResolvedFile } from '../validate'; +import { fileNotInSpace, resolveFileSpace, type ValidateFileOptions, validateResolvedFile } from '../validate'; export { validateFile } from '../validate'; @@ -14,18 +12,18 @@ export { validateFile } from '../validate'; */ export async function validateFileCommand( filePath: string, - options: { json?: boolean; includeGated?: boolean; semantic?: boolean | string; ruleFilter?: string[] } = {}, + options: { json?: boolean } & Omit = {}, ): Promise { const config = loadConfig(); const resolved = resolveFileSpace(filePath, config); let result: ValidateFileOutput; if (resolved) { - const judge = judgeForRun(config, resolved.space, options.semantic); - result = await validateResolvedFile(filePath, resolved.label, createSpaceContext(resolved.space.name, config), { - includeGated: options.includeGated, - judge, - ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), - }); + result = await validateResolvedFile( + filePath, + resolved.label, + createSpaceContext(resolved.space.name, config), + options, + ); } else { result = fileNotInSpace(filePath); } diff --git a/src/commands/validate.ts b/src/commands/validate.ts index 08a70c0..389359d 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -1,29 +1,16 @@ import { dirname } from 'node:path'; import chokidar from 'chokidar'; import { getConfigSourceFiles } from '../config'; -import { judgeForRun } from '../judge/factory'; import { resolveSpacePluginSchemas } from '../plugins/util'; -import { parseRuleFilter } from '../schema/rule-filter'; import { bundledSchemasDir } from '../schema/schema'; import type { FileValidationIssue, RuleSeverity, SpaceContext } from '../types'; -import { formatSource, ruleViolationMessage, validateSpace } from '../validate'; +import { formatSource, ruleViolationMessage, type ValidateSpaceOptions, validateSpace } from '../validate'; export async function validate( context: SpaceContext, - options: { - json?: boolean; - depth?: number; - includeGated?: boolean; - semantic?: boolean | string; - ruleFilter?: string[]; - } = {}, + options: { json?: boolean } & ValidateSpaceOptions = {}, ): Promise { - const result = await validateSpace(context, { - depth: options.depth, - includeGated: options.includeGated, - judge: judgeForRun(context.config, context.space, options.semantic), - ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), - }); + const result = await validateSpace(context, options); const ruleCounts: Record = { error: 0, warning: 0, info: 0 }; for (const v of result.ruleViolations) ruleCounts[v.severity]++; diff --git a/src/errors.ts b/src/errors.ts index 9b6b27e..e149daa 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -13,7 +13,8 @@ export type SctxErrorCode = | 'E_SCHEMA_INVALID' | 'E_SPACE_EXISTS' | 'E_TARGET_REQUIRED' - | 'E_CONFIG_NEEDS_MANUAL_EDIT'; + | 'E_CONFIG_NEEDS_MANUAL_EDIT' + | 'E_MODEL_PROVIDER_INVALID'; export class SctxError extends Error { constructor( diff --git a/src/index.ts b/src/index.ts index 3c9adbe..33d4cdb 100755 --- a/src/index.ts +++ b/src/index.ts @@ -17,7 +17,6 @@ import { validateFileCommand } from './commands/validate-file'; import { loadConfig, setConfigPath } from './config'; import { CLI_NAME } from './constants'; import { SctxError } from './errors'; -import { JudgeConfigError } from './judge/factory'; import { createSpaceContext, SpaceNotFoundError } from './space-context'; import type { SpaceContext } from './types'; @@ -39,17 +38,6 @@ export function buildSpaceContext(spaceName: string): SpaceContext { } } -/** Run a validate command, reporting a misconfigured model provider as a plain error rather than a stack trace. */ -async function reportJudgeConfigErrors(run: () => Promise): Promise { - try { - return await run(); - } catch (err) { - if (!(err instanceof JudgeConfigError)) throw err; - console.error(`Error: ${err.message}`); - return 1; - } -} - const require = createRequire(import.meta.url); const packageJson = require('../package.json'); @@ -81,14 +69,12 @@ program ) .option('--semantic [provider]', 'Evaluate semantic rules, with the configured model provider or the one named') .action(async (filePath, options) => { - process.exitCode = await reportJudgeConfigErrors(() => - validateFileCommand(filePath, { - json: options.json, - includeGated: options.includeGated, - semantic: options.semantic, - ruleFilter: options.ruleFilter, - }), - ); + process.exitCode = await validateFileCommand(filePath, { + json: options.json, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, + }); }); program @@ -118,15 +104,13 @@ program await watchValidate(context); } else { // exitCode rather than process.exit(): exiting straight away can cut off large piped output - process.exitCode = await reportJudgeConfigErrors(() => - validate(context, { - json: options.json, - depth: options.depth, - includeGated: options.includeGated, - semantic: options.semantic, - ruleFilter: options.ruleFilter, - }), - ); + process.exitCode = await validate(context, { + json: options.json, + depth: options.depth, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, + }); } }); diff --git a/src/judge/factory.ts b/src/judge/factory.ts index dd00628..680a9d4 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,3 +1,4 @@ +import { SctxError } from '../errors'; import type { AnthropicJudgeProviderOptions } from './provider/anthropic'; import { AnthropicJudgeProvider, RECOMMENDED_ANTHROPIC_MODEL } from './provider/anthropic'; import { FakeJudgeProvider } from './provider/fake'; @@ -83,7 +84,6 @@ const DEFAULT_KEY_ENV: Record = { } as const; /** Semantic evaluation is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ -export class JudgeConfigError extends Error {} /** * Hydrate a provider definition into concrete values, in one place. Per field: the explicit value @@ -92,7 +92,8 @@ export class JudgeConfigError extends Error {} */ export function hydrateProvider(name: string, def: ModelProviderConfig, env: Env = process.env): HydratedProvider { if (!(MODEL_PROVIDER_APIS as readonly string[]).includes(def.api)) { - throw new JudgeConfigError( + throw new SctxError( + 'E_MODEL_PROVIDER_INVALID', `Model provider "${name}" has unknown api "${def.api}". Use one of: ${MODEL_PROVIDER_APIS.join(', ')}.`, ); } @@ -101,7 +102,8 @@ export function hydrateProvider(name: string, def: ModelProviderConfig, env: Env if (!envName) return undefined; const envValue = env[envName]; if (envValue !== undefined && envValue.trim() === '') { - throw new JudgeConfigError( + throw new SctxError( + 'E_MODEL_PROVIDER_INVALID', `Model provider "${name}" has ${envName} set but empty; set a value or unset the variable.`, ); } @@ -128,7 +130,8 @@ export function createModelProvider( // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key; the others must resolve one. // A key never borrows another provider's variable: only the named apiKeyEnv (or literal apiKey) counts. if (!h.apiKey && h.api !== 'openai-completions' && h.api !== 'fake') { - throw new JudgeConfigError( + throw new SctxError( + 'E_MODEL_PROVIDER_INVALID', `Model provider "${name}" (${h.api}) has no API key. Set ${h.keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, ); } @@ -163,7 +166,7 @@ export function judgeForRun( /** * The provider for a run: the one named by `name` (the `--semantic ` override), else the space's * `semantic.provider`, else the top-level one. The name resolves against `modelProviders` first, then - * the bundled providers. Throws {@link JudgeConfigError} when it cannot be built. + * the bundled providers. Throws {@link SctxError} `E_MODEL_PROVIDER_INVALID` when it cannot be built. */ export function createJudge( config: { modelProviders?: Record; semantic?: { provider?: string } }, @@ -173,7 +176,8 @@ export function createJudge( ): SemanticJudgeProvider { const chosen = name ?? space.semantic?.provider ?? config.semantic?.provider; if (!chosen) { - throw new JudgeConfigError( + throw new SctxError( + 'E_MODEL_PROVIDER_INVALID', '--semantic needs a model provider. Set "semantic.provider" in config.json, or pass --semantic .', ); } @@ -181,7 +185,8 @@ export function createJudge( if (!def) { const known = [...Object.keys(BUNDLED_PROVIDERS), ...Object.keys(config.modelProviders ?? {})]; - throw new JudgeConfigError( + throw new SctxError( + 'E_MODEL_PROVIDER_INVALID', `Model provider "${chosen}" is not configured.${known.length ? ` Known providers: ${known.join(', ')}.` : ''}`, ); } diff --git a/src/judge/index.ts b/src/judge/index.ts deleted file mode 100644 index 2068eb3..0000000 --- a/src/judge/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { createJudge, createModelProvider, JudgeConfigError } from './factory'; -export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; -export type { AnthropicJudgeProviderOptions } from './provider/anthropic'; -export { AnthropicJudgeProvider } from './provider/anthropic'; -export type { FakeAnswer, FakeJudgeHandler } from './provider/fake'; -export { FakeJudgeProvider } from './provider/fake'; -export type { JevJudgeProviderOptions } from './provider/jev'; -export { JevJudgeProvider } from './provider/jev'; -export type { OpenAIJudgeProviderOptions } from './provider/openai'; -export { OpenAIJudgeProvider } from './provider/openai'; -export { PerRuleJudgeProvider } from './provider/per-rule'; -export type { - JudgeConfig, - ModelProviderConfig, - SemanticJudgeInput, - SemanticJudgeProvider, - SemanticJudgeResult, - SemanticJudgeVerdict, - SemanticRuleSpec, -} from './types'; -export { DEFAULT_VERDICTS, isJudgeError, semanticRuleSpec } from './verdicts'; diff --git a/src/validate.ts b/src/validate.ts index 5ccb0df..b3bdb88 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,11 +1,12 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; +import { judgeForRun } from './judge/factory'; import type { TrimReport } from './judge/trim'; import type { SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; -import { filterRules, type RuleFilter } from './schema/rule-filter'; +import { filterRules, parseRuleFilter, type RuleFilter } from './schema/rule-filter'; import { extractEntityInfo, typeValidatorFor } from './schema/schema'; import { validateGraph } from './schema/validate-graph'; import { validateRules } from './schema/validate-rules'; @@ -172,16 +173,40 @@ export function formatErrors( return formatted; } -export interface ValidateSpaceOptions { - readResult?: ReadSpaceResult; - /** Limit validation to hierarchy nodes within N breadth-first levels from the top. */ - depth?: number; +interface CommonValidateOptions { /** Evaluate rules on gated nodes anyway (gates are still reported). */ includeGated?: boolean; - /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ + /** Evaluate only rules matching these `dimension=values` filters (category, severity, type, id). */ + ruleFilter?: string[]; + /** + * Evaluate `semantic` rules with the model provider named here, or with the configured one for `true`. + * Without it, a rule runs only its `expression`, and a semantic-only rule is skipped. + */ + semantic?: boolean | string; +} + +/** What a run evaluates with: the public options after the filter is parsed and the judge is built. */ +export interface RunOptions { + includeGated?: boolean; ruleFilter?: RuleFilter; - /** Evaluates `semantic` rules. Without one, a rule runs only its `expression`, and a semantic-only rule is skipped. */ judge?: SemanticJudgeProvider; + /** Limit judging to one file (and the ancestors that can gate it). */ + judgeTarget?: string; +} + +/** Throws `SctxError` `E_MODEL_PROVIDER_INVALID` when `semantic` names a provider that cannot be built. */ +function runOptionsFrom(context: SpaceContext, options: CommonValidateOptions): RunOptions { + return { + includeGated: options.includeGated, + ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), + judge: judgeForRun(context.config, context.space, options.semantic), + }; +} + +export interface ValidateSpaceOptions extends CommonValidateOptions { + readResult?: ReadSpaceResult; + /** Limit validation to hierarchy nodes within N breadth-first levels from the top. */ + depth?: number; } /** `file`, `file:line` or `file:line:column`, for messages. */ @@ -251,7 +276,7 @@ interface NodeValidationFailure { } /** The outcome of validating a set of nodes, before either view shapes it. */ -interface SpaceValidationResult { +export interface SpaceValidationResult { validCount: number; nodeFailures: NodeValidationFailure[]; refErrors: ValidationResult['refErrors']; @@ -287,16 +312,11 @@ export function findDuplicateIds(nodes: SpaceNode[]): ValidationResult['duplicat * and `validateFile` are views over it, so a file's results match its entry in a full run. * `judgeTarget` limits judging to one file (and the ancestors that can gate it). */ -async function validateNodes( +export async function validateNodes( context: SpaceContext, nodes: SpaceNode[], unresolvedRefs: ReadSpaceResult['unresolvedRefs'], - options: { - includeGated?: boolean; - ruleFilter?: RuleFilter; - judge?: SemanticJudgeProvider; - judgeTarget?: string; - } = {}, + options: RunOptions = {}, ): Promise { const { schema, schemaRefRegistry, schemaValidator } = context; const metadata = schema.metadata; @@ -405,6 +425,7 @@ export async function validateSpace( ): Promise { const metadata = context.schema.metadata; + const runOptions = runOptionsFrom(context, options); const readResult = options.readResult ?? (await readSpace(context)); let nodes = readResult.nodes; @@ -426,11 +447,7 @@ export async function validateSpace( nodes = included; } - const result = await validateNodes(context, nodes, readResult.unresolvedRefs, { - includeGated: options.includeGated, - ruleFilter: options.ruleFilter, - judge: options.judge, - }); + const result = await validateNodes(context, nodes, readResult.unresolvedRefs, runOptions); return { validCount: result.validCount, @@ -469,14 +486,8 @@ export function resolveFileSpace(filePath: string, config: Config): { space: Spa return null; } -export interface ValidateFileOptions { +export interface ValidateFileOptions extends CommonValidateOptions { configDir?: string; - /** Evaluate rules on gated nodes anyway (gates are still reported). */ - includeGated?: boolean; - /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ - ruleFilter?: RuleFilter; - /** Evaluates `semantic` rules. Without one, a rule runs only its `expression`, and a semantic-only rule is skipped. */ - judge?: SemanticJudgeProvider; } /** @@ -571,20 +582,28 @@ export function fileNotInSpace(filePath: string): FileNotInSpaceResult { * Validate a file in an already-resolved space. Checks the whole space (required for graph * correctness), judging only the file, then selects the file's issues from the result. */ -export async function validateResolvedFile( +export function validateResolvedFile( filePath: string, label: string, context: SpaceContext, options: Omit = {}, +): Promise { + return validateResolvedFileWith(filePath, label, context, runOptionsFrom(context, options)); +} + +/** `validateResolvedFile` with the options already resolved, so a caller can supply its own judge. */ +export async function validateResolvedFileWith( + filePath: string, + label: string, + context: SpaceContext, + runOptions: RunOptions, ): Promise { const readResult = await readSpace(context); const result = await validateNodes(context, readResult.nodes, readResult.unresolvedRefs, { - includeGated: options.includeGated, - ruleFilter: options.ruleFilter, - judge: options.judge, + ...runOptions, judgeTarget: label, }); - const { errors, warnings } = issuesForFile(result, label, options.includeGated); + const { errors, warnings } = issuesForFile(result, label, runOptions.includeGated); return { file: isAbsolute(filePath) ? filePath : resolve(process.cwd(), filePath), diff --git a/tests/helpers/validate-run.ts b/tests/helpers/validate-run.ts new file mode 100644 index 0000000..b863514 --- /dev/null +++ b/tests/helpers/validate-run.ts @@ -0,0 +1,29 @@ +import type { Config } from '../../src/config'; +import { readSpace } from '../../src/read/read-space'; +import { createSpaceContext } from '../../src/space-context'; +import type { SpaceContext } from '../../src/types'; +import { + fileNotInSpace, + type RunOptions, + resolveFileSpace, + validateNodes, + validateResolvedFileWith, +} from '../../src/validate'; + +/** + * `validateSpace` with the judge and rule filter already built, so a test can supply its own + * `FakeJudgeProvider` (and read its call history). The public options only accept config-derived ones. + */ +export async function validateSpaceWith(context: SpaceContext, runOptions: RunOptions = {}) { + const read = await readSpace(context); + return validateNodes(context, read.nodes, read.unresolvedRefs, runOptions); +} + +/** `validateFile` with the options already built; see {@link validateSpaceWith}. */ +export async function validateFileWith(filePath: string, config: Config, options: RunOptions & { configDir?: string }) { + const { configDir, ...runOptions } = options; + const resolved = resolveFileSpace(filePath, config); + if (!resolved) return fileNotInSpace(filePath); + const context = createSpaceContext(resolved.space.name, config, { configDir }); + return validateResolvedFileWith(filePath, resolved.label, context, runOptions); +} diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 906624c..3e5018e 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,11 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { - createJudge, - createModelProvider, - hydrateProvider, - JudgeConfigError, - judgeForRun, -} from '../../src/judge/factory'; +import { createJudge, createModelProvider, hydrateProvider, judgeForRun } from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; @@ -148,7 +142,9 @@ describe('createJudge', () => { it('never infers a provider from API keys in the environment', () => { const keys = { ANTHROPIC_API_KEY: 'a', OPENAI_API_KEY: 'o', OPENROUTER_API_KEY: 'r' }; - expect(() => createJudge({}, {}, undefined, keys)).toThrow(JudgeConfigError); + expect(() => createJudge({}, {}, undefined, keys)).toThrow( + expect.objectContaining({ code: 'E_MODEL_PROVIDER_INVALID' }), + ); expect(() => createJudge({ modelProviders: config.modelProviders }, {}, undefined, keys)).toThrow( '--semantic needs a model provider', ); diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts index 601b9fa..f316760 100644 --- a/tests/validate/semantic-rule-shape.test.ts +++ b/tests/validate/semantic-rule-shape.test.ts @@ -2,8 +2,9 @@ import { afterAll, describe, expect, it } from 'bun:test'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult, NodeId } from '../../src/types'; -import { validateFile, validateSpace } from '../../src/validate'; +import { validateSpace } from '../../src/validate'; import { makeTempSpace } from '../helpers/temp-space'; +import { validateFileWith, validateSpaceWith } from '../helpers/validate-run'; const spaces: ReturnType[] = []; afterAll(() => { @@ -39,7 +40,7 @@ function setup(rules: object[], options?: Parameters[1]) { describe('semantic rules: verdict severity, ignore and guidance', () => { it('reports each verdict at its mapped severity with its guidance', async () => { const { context } = setup([stepRule]); - const minor = await validateSpace(context(), { + const minor = await validateSpaceWith(context(), { judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), }); expect(minor.ruleViolations).toEqual([ @@ -51,7 +52,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { }), ]); - const major = await validateSpace(context(), { + const major = await validateSpaceWith(context(), { judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'major' } }), }); expect(major.ruleViolations[0]).toMatchObject({ severity: 'error', verdict: 'major' }); @@ -60,7 +61,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('judges an ignored verdict without reporting it', async () => { const { context } = setup([stepRule]); const fake = new FakeJudgeProvider({ 'step-fit': { verdict: 'unsure' } }); - const result = await validateSpace(context(), { judge: fake }); + const result = await validateSpaceWith(context(), { judge: fake }); expect(fake.callHistory).toHaveLength(1); expect(result.ruleViolations).toEqual([]); }); @@ -70,19 +71,19 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { const { context } = setup([taskRule, { ...stepRule, id: 'step-fit' }]); const minor = new FakeJudgeProvider({ 'task-fit': { verdict: 'minor' } }); - const ungated = await validateSpace(context(), { judge: minor }); + const ungated = await validateSpaceWith(context(), { judge: minor }); expect(ungated.coverageGates).toEqual([]); expect(minor.callHistory.some((c) => c.node.id === 'S.md')).toBe(true); const major = new FakeJudgeProvider({ 'task-fit': { verdict: 'major' } }); - const gated = await validateSpace(context(), { judge: major }); + const gated = await validateSpaceWith(context(), { judge: major }); expect(gated.coverageGates[0]).toMatchObject({ reason: 'rule', ruleId: 'task-fit' }); expect(major.callHistory.some((c) => c.node.id === 'S.md')).toBe(false); }); it('shows guidance in validate-file output', async () => { const { space } = setup([stepRule]); - const result = (await validateFile(space.file('S.md'), space.config, { + const result = (await validateFileWith(space.file('S.md'), space.config, { judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), configDir: space.dir, })) as FileValidationResult; @@ -92,10 +93,10 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('--rule-filter severity matches any severity a verdict map can report', async () => { const { context } = setup([stepRule]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - await validateSpace(context(), { judge: judge, ruleFilter: { severity: ['warning'] } }); + await validateSpaceWith(context(), { judge: judge, ruleFilter: { severity: ['warning'] } }); expect(judge.callHistory.length).toBeGreaterThan(0); const none = new FakeJudgeProvider(); - await validateSpace(context(), { judge: none, ruleFilter: { severity: ['info'] } }); + await validateSpaceWith(context(), { judge: none, ruleFilter: { severity: ['info'] } }); expect(none.callHistory).toHaveLength(0); }); }); @@ -120,12 +121,12 @@ describe('rule `when` precondition', () => { const rule = { ...stepRule, id: 'task-fit', type: 'task', when }; const { context } = setup([rule], { frontmatter: { T: 'status: ready' } }); const judge = new FakeJudgeProvider(); - await validateSpace(context(), { judge: judge }); + await validateSpaceWith(context(), { judge: judge }); expect(judge.callHistory.map((c) => c.node.id as string)).toEqual(['T.md']); const none = setup([rule]); const idle = new FakeJudgeProvider(); - const result = await validateSpace(none.context(), { judge: idle }); + const result = await validateSpaceWith(none.context(), { judge: idle }); expect(idle.callHistory).toHaveLength(0); expect(result.coverageGates).toEqual([]); // not applicable is not gated }); @@ -141,7 +142,7 @@ describe('judge option', () => { it('judges with the judge it is given', async () => { const { context } = setup([stepRule]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judge.callHistory.length).toBeGreaterThan(0); expect(result.ruleViolations[0]).toMatchObject({ verdict: 'minor' }); }); @@ -184,7 +185,7 @@ describe('semantic rule check precondition', () => { it('judges when the check passes', async () => { const { context } = setup([gated], { frontmatter: { S: 'owner: sam' } }); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'fail' } }); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judgedLabels(judge)).toEqual(['S.md']); expect(result.ruleViolations).toEqual([expect.objectContaining({ ruleId: 'step-fit', verdict: 'fail' })]); }); @@ -192,7 +193,7 @@ describe('semantic rule check precondition', () => { it('reports a failing check at the rule severity without calling the judge', async () => { const { context } = setup([gated]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'fail' } }); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judge.callHistory).toHaveLength(0); expect(result.ruleViolations).toEqual([ { @@ -211,7 +212,7 @@ describe('semantic rule check precondition', () => { const run = async (r: object) => { const { context } = setup([r]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judge.callHistory).toHaveLength(0); return result.ruleViolations; }; @@ -225,7 +226,7 @@ describe('semantic rule check precondition', () => { const taskRule = { ...gated, id: 'task-fit', type: 'task', gatesDescendants: undefined }; const run = async (rule: Record) => { const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - const result = await validateSpace(setup([rule, stepRule]).context(), { judge }); + const result = await validateSpaceWith(setup([rule, stepRule]).context(), { judge }); return { result, judged: judgedLabels(judge) }; }; @@ -255,7 +256,7 @@ describe('semantic rule check precondition', () => { it('gates the file when an ancestor check fails (single-file validation)', async () => { const { space } = setup([{ ...gated, id: 'task-fit', type: 'task' }, stepRule]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - const result = (await validateFile(space.file('S.md'), space.config, { + const result = (await validateFileWith(space.file('S.md'), space.config, { judge, configDir: space.dir, })) as FileValidationResult; @@ -291,9 +292,9 @@ describe('semantic rule check precondition', () => { it('honours --rule-filter', async () => { const { context } = setup([gated]); - const kept = await validateSpace(context(), { ruleFilter: { id: ['step-fit'] } }); + const kept = await validateSpaceWith(context(), { ruleFilter: { id: ['step-fit'] } }); expect(kept.ruleViolations).toHaveLength(1); - const dropped = await validateSpace(context(), { ruleFilter: { id: ['other'] } }); + const dropped = await validateSpaceWith(context(), { ruleFilter: { id: ['other'] } }); expect(dropped.ruleViolations).toEqual([]); }); }); @@ -312,7 +313,7 @@ describe('semantic rule check precondition', () => { it('evaluates when before check', async () => { const { context } = setup([{ ...gated, when: 'false' }]); const judge = new FakeJudgeProvider(); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judge.callHistory).toHaveLength(0); expect(result.ruleViolations).toEqual([]); }); @@ -320,7 +321,7 @@ describe('semantic rule check precondition', () => { it('reports a check that cannot be evaluated as an error without judging', async () => { const { context } = setup([{ ...gated, expression: '$notAFunction(' }]); const judge = new FakeJudgeProvider(); - const result = await validateSpace(context(), { judge }); + const result = await validateSpaceWith(context(), { judge }); expect(judge.callHistory).toHaveLength(0); expect(result.ruleViolations[0]).toMatchObject({ nodeId: 'S.md', severity: 'error' }); expect(result.ruleViolations[0]?.description).toContain('Rule error'); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index c06a397..94d9d08 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -9,6 +9,7 @@ import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; import { validateFile, validateSpace } from '../../src/validate'; import { makeTempSpace, type TempSpace } from '../helpers/temp-space'; +import { validateFileWith, validateSpaceWith } from '../helpers/validate-run'; const STRICT_SCHEMA_PATH = join(bundledSchemasDir, 'strict_ost.json'); const VALID_DIR = join(import.meta.dir, '../fixtures/strict_ost/valid-directory'); @@ -39,7 +40,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', evidence: 'Opportunity is framed in clear customer language describing difficulty with sign-up.', }); - const result = await validateSpace(validContext, { + const result = await validateSpaceWith(validContext, { judge: fakeProvider, }); @@ -57,7 +58,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', evidence: 'The opportunity reads like a feature request rather than a problem statement.', }); - const result = await validateSpace(validContext, { + const result = await validateSpaceWith(validContext, { judge: fakeProvider, }); @@ -81,12 +82,12 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', const invalidContext = createSpaceContext('strict-invalid', invalidConfig, { configDir: import.meta.dir }); const fakeProvider = new FakeJudgeProvider(); - const result = await validateSpace(invalidContext, { + const result = await validateSpaceWith(invalidContext, { judge: fakeProvider, }); // opportunity-no-source.md fails schema validation (missing required 'source' field) - expect(result.nodeErrors.some((e) => e.nodeId === 'opportunity-no-source.md')).toBe(true); + expect(result.nodeFailures.some((e) => e.node.id === 'opportunity-no-source.md')).toBe(true); // Therefore it must be gated and never evaluated by the judge expect(result.coverageGates.some((g) => g.nodeId === 'opportunity-no-source.md' && g.reason === 'schema')).toBe( @@ -115,7 +116,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', evidence: 'Opportunity is framed in solution space.', }); - const result = (await validateFile(targetFile, validConfig, { + const result = (await validateFileWith(targetFile, validConfig, { judge: failingProvider, configDir: import.meta.dir, })) as FileValidationResult; @@ -151,7 +152,7 @@ describe('validateFile judges only the target and its gate-capable ancestor rule afterAll(() => space.cleanup()); const run = (judge: FakeJudgeProvider, file: string, extra = {}) => - validateFile(space.file(file), space.config, { + validateFileWith(space.file(file), space.config, { judge: judge, configDir: space.dir, ...extra, @@ -194,7 +195,7 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { it('judges each node once with all of its semantic rules', async () => { const fake = new FakeJudgeProvider(); - await validateSpace(context, { judge: fake }); + await validateSpaceWith(context, { judge: fake }); const labels = fake.callHistory.map((c) => c.node.id); expect(new Set(labels).size).toBe(labels.length); expect(fake.callHistory.every((c) => c.rules.length >= 1)).toBe(true); @@ -202,28 +203,28 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { it('reports a verdict that has a severity as a violation carrying the verdict and evidence', async () => { const fake = new FakeJudgeProvider({ verdict: 'fail', confidence: 0.7, evidence: 'solution in disguise' }); - const result = await validateSpace(context, { judge: fake }); + const result = await validateSpaceWith(context, { judge: fake }); const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); expect(violation).toMatchObject({ verdict: 'fail', evidence: 'solution in disguise' }); }); it('reports a per-rule judge error as a violation without gating the subtree', async () => { const fake = new FakeJudgeProvider({ error: 'rate limited' }); - const result = await validateSpace(context, { judge: fake }); + const result = await validateSpaceWith(context, { judge: fake }); const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); expect(violation?.description).toContain('Semantic evaluation error: rate limited'); expect(result.coverageGates.filter((g) => g.ruleId === 'opportunity-customer-voice')).toEqual([]); }); it('--rule-filter narrows evaluation: filtering to the semantic rule runs no expression rules, filtering it out runs no judge', async () => { - const semanticOnly = await validateSpace(context, { + const semanticOnly = await validateSpaceWith(context, { judge: new FakeJudgeProvider({ pass: false }), ruleFilter: { id: ['opportunity-customer-voice'] }, }); expect(semanticOnly.ruleViolations.every((v) => v.ruleId === 'opportunity-customer-voice')).toBe(true); const fake = new FakeJudgeProvider(); - const expressionOnly = await validateSpace(context, { + const expressionOnly = await validateSpaceWith(context, { judge: fake, ruleFilter: { category: ['workflow'] }, }); @@ -249,7 +250,7 @@ describe('context trimming in a run', () => { it('reports how many nodes were trimmed and for which rules', async () => { const context = createSpaceContext('ts', space.config, { configDir: space.dir }); - const result = await validateSpace(context, { judge: new FakeJudgeProvider({ pass: true }) }); + const result = await validateSpaceWith(context, { judge: new FakeJudgeProvider({ pass: true }) }); expect(result.contextTrimmed).toEqual({ nodeCount: 1, ruleIds: ['task-fits-goal'] }); }); diff --git a/tests/validate/validate-views.test.ts b/tests/validate/validate-views.test.ts index 2f1fd60..d5fee55 100644 --- a/tests/validate/validate-views.test.ts +++ b/tests/validate/validate-views.test.ts @@ -8,6 +8,7 @@ import { bundledSchemasDir } from '../../src/schema/schema'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; import { validateFile, validateSpace } from '../../src/validate'; +import { validateFileWith } from '../helpers/validate-run'; const META = 'https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json'; @@ -120,7 +121,7 @@ describe('judging a node with an expression rule violation', () => { it(`skips the node's semantic rules after a gating violation, ${label} a hierarchy`, async () => { const { config, dir, file } = setup(hierarchy, 'error'); const judge = new FakeJudgeProvider({ pass: true }); - const result = (await validateFile(file, config, { judge, configDir: dir })) as FileValidationResult; + const result = (await validateFileWith(file, config, { judge, configDir: dir })) as FileValidationResult; expect(result.errors['rule:never-holds']).toBeDefined(); expect(judgedRules(judge)).toEqual([]); }); @@ -128,7 +129,7 @@ describe('judging a node with an expression rule violation', () => { it(`still judges the node after a non-gating violation, ${label} a hierarchy`, async () => { const { config, dir, file } = setup(hierarchy, 'info'); const judge = new FakeJudgeProvider({ pass: true }); - const result = (await validateFile(file, config, { judge, configDir: dir })) as FileValidationResult; + const result = (await validateFileWith(file, config, { judge, configDir: dir })) as FileValidationResult; expect(result.warnings['rule:never-holds']).toBeDefined(); expect(judgedRules(judge)).toEqual(['sound']); });