diff --git a/.githooks/leak-patterns.sh b/.githooks/leak-patterns.sh new file mode 100644 index 0000000000..cb03f7c8fe --- /dev/null +++ b/.githooks/leak-patterns.sh @@ -0,0 +1,15 @@ +# Shared leak-scan patterns, sourced by .githooks/pre-commit and .githooks/leak-scan.sh so the +# local hook and the hosted `leak-scan` status can never disagree about what a leak is. +# Names NO private project; private names come only from the untracked denylist. +# +# /Users/ is the macOS home shape. It was missing from leak-scan.sh until a rename-with-edits +# slipped past both (audit finding 12). + +# Generic, non-sensitive structural patterns (safe to commit): +LEAK_PATTERNS='/home/[A-Za-z0-9._-]+/|/Users/[A-Za-z0-9._-]+/|~/Development' + +# Legitimate matches to ignore (the published npm author email). +LEAK_ALLOW='doug@dougmealing\.com' + +# Diff filter: Added, Copied, Modified and Renamed (a `git mv` plus an edit is a rename). +LEAK_DIFF_FILTER='ACMR' diff --git a/.githooks/leak-scan.sh b/.githooks/leak-scan.sh index 255a5b8d11..a858e9bc13 100755 --- a/.githooks/leak-scan.sh +++ b/.githooks/leak-scan.sh @@ -15,13 +15,13 @@ # set -uo pipefail -# Generic, non-sensitive structural patterns (safe to commit — name no project): -PATTERNS='/home/[A-Za-z0-9._-]+/|~/Development' - -# Legitimate matches to ignore (the published npm author email). -ALLOW='doug@dougmealing\.com' - ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" +# Patterns, allow-list and diff filter are shared with .githooks/pre-commit. +# shellcheck source=leak-patterns.sh +. "$ROOT/.githooks/leak-patterns.sh" +PATTERNS="$LEAK_PATTERNS" +ALLOW="$LEAK_ALLOW" + DENY_FILE="$(git config --get hooks.denyListPath 2>/dev/null || true)" [ -z "$DENY_FILE" ] && DENY_FILE="$ROOT/.githooks/deny-list.local" if [ -f "$DENY_FILE" ]; then @@ -30,9 +30,9 @@ if [ -f "$DENY_FILE" ]; then fi if [ -n "${1:-}" ]; then - DIFF=$(git diff "$1...HEAD" -U0 --no-color --diff-filter=ACM 2>/dev/null) + DIFF=$(git diff "$1...HEAD" -U0 --no-color --diff-filter="$LEAK_DIFF_FILTER" 2>/dev/null) else - DIFF=$(git diff --cached --diff-filter=ACM -U0 --no-color 2>/dev/null) + DIFF=$(git diff --cached --diff-filter="$LEAK_DIFF_FILTER" -U0 --no-color 2>/dev/null) fi # Scan only newly-ADDED lines, attributed to their file; skip the hooks dir itself. diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 3fe220ff29..6eaae8e1c0 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -17,19 +17,17 @@ # set -uo pipefail -# Generic, non-sensitive structural patterns — safe to commit (name no project): -# /Users/ is the macOS home shape. It was missing until a plan doc authored on a Mac -# committed an absolute developer home path that this hook scanned and passed. -PATTERNS='/home/[A-Za-z0-9._-]+/|/Users/[A-Za-z0-9._-]+/|~/Development' - -# Legitimate matches to ignore (e.g. the published npm author email). -ALLOW='doug@dougmealing\.com' +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" +# Patterns, allow-list and diff filter are shared with .githooks/leak-scan.sh. +# shellcheck source=leak-patterns.sh +. "$ROOT/.githooks/leak-patterns.sh" +PATTERNS="$LEAK_PATTERNS" +ALLOW="$LEAK_ALLOW" # Merge in private-name patterns from a denylist. Source priority: # 1) git config hooks.denyListPath -> point at your private denylist file (kept in a private repo) # 2) .githooks/deny-list.local -> untracked local file or symlink # If neither resolves, only the generic structural patterns above are enforced. -ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" DENY_FILE="$(git config --get hooks.denyListPath 2>/dev/null || true)" [ -z "$DENY_FILE" ] && DENY_FILE="$ROOT/.githooks/deny-list.local" if [ -f "$DENY_FILE" ]; then @@ -38,7 +36,7 @@ if [ -f "$DENY_FILE" ]; then fi # Scan only newly-ADDED staged lines, attributed to their file; skip the hooks dir. -hits=$(git diff --cached --diff-filter=ACM -U0 --no-color 2>/dev/null | awk ' +hits=$(git diff --cached --diff-filter="$LEAK_DIFF_FILTER" -U0 --no-color 2>/dev/null | awk ' /^\+\+\+ / { f=$2; sub(/^b\//,"",f); next } /^\+/ && !/^\+\+\+/ { if (f !~ /^\.githooks\//) print f "\t" substr($0,2) } ' | grep -Ei "$PATTERNS" | grep -vEi "$ALLOW") diff --git a/AGENTS.md b/AGENTS.md index f66787ae7e..afc42a1358 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ PyPI has had no product change since `0.25.0` — nothing is broken. - **Kotlin** — `codegen-kotlin` (KotlinPoet on JVM): entity + Exposed table + Spring controller + payload + relations + filter allowlist + validator + stored-proc + output-parser generators. `integration-tests-kotlin` runs the persistence-conformance corpus through Exposed against Testcontainers Postgres. **Cross-port conformance corpora** (every port runs the shared corpus): -- Metamodel: `fixtures/conformance/` (413 fixtures; 34 shared corpora in total — per-corpus counts + the corpus x port matrix live in `docs/CONFORMANCE.md`). TS / C# / Java / Python all green. +- Metamodel: `fixtures/conformance/` (421 fixtures; 34 shared corpora in total — per-corpus counts + the corpus x port matrix live in `docs/CONFORMANCE.md`). TS / C# / Java / Python all green. - Render: `fixtures/render-conformance/`. TS / C# / Java / Kotlin / Python byte-identical. - Persistence: `fixtures/persistence-conformance/`. **Query** scenarios run on every port (TS / C# / Java / Kotlin / Python), each provisioning its test DB by executing the committed, TS-produced `canonical/schema.postgres.sql` (Postgres only — Derby dropped for the cross-port query corpus, ADR-0015; the FR-044 `report-*` scenarios also run on SQLite, MySQL and D1 where a port has the engine, see [docs/CONFORMANCE.md](docs/CONFORMANCE.md)). The **migration** scenarios are exercised by **TS only** (TS owns schema migrations). **The corpus now gates WRITES, not just reads (SP-H):** an `op: roundtrip` scenario type INSERTs through each port's runtime/ORM write codec (NOT raw SQL), reads the row back, and asserts the wire-normalized value. The `AllTypes` entity (`roundtrip-all-types.yaml`) carries one field of **every** persistable `field.*` subtype — string/int/long/double/float/decimal/boolean/date/time/timestamp(+tz)/currency/enum/uuid/object — plus an **array-of-VO** `field.object @isArray @storage:jsonb` column (`labels`, written as 2-element / empty-`[]` / single-element arrays across the three rows) — so every subtype write+read (incl. the array-of-value-object jsonb codec) round-trips through every port against Testcontainers PG. (`field.byte`/`field.short`/`field.class` were cut as non-functional registration-only stubs — the matrix tracks only genuinely-supported subtypes; see `fixtures/registry-conformance/README.md` → "Per-subtype write-round-trip matrix".) - API-contract: `fixtures/api-contract-conformance/`. TS / C# / Java / Kotlin / Python all green — each port runs **two lanes**: a hand-rolled reference server AND its **generated** API artifact booted over HTTP (the deployed controller/routes; TS+C# full-stack vs Testcontainers PG, Java/Kotlin/Python generated controller + in-memory repo behind the consumer seam). The generated fan-out found 10 real deployment bugs golden snapshots missed. Three sub-corpora run the **generated lane only, on all five ports** — `write-through/`, `projection/` (F22: a view-only `object.projection` serves GET list + GET by id and answers every write verb with `405 {"error": "method_not_allowed"}`) and `report/` (FR-044: a view-backed `object.report` serves GET list, answers `POST` with that same `405`, and mounts no `/{id}` route). That is deliberate, not a gap: what is under test is whether a port's GENERATOR emits those routes, and a hand-rolled reference server would answer every scenario by construction. The `m2m/` sub-corpus also gates **TPH x M:N together** (base-declared, subtype-declared, abstract-mid-declared, a non-subtype source onto a subtype TARGET, and the cross-subtype source id answering `200 []`) — the two corpora were originally built disjoint (`tph/` had no relationships, `m2m/` no discriminators), which is precisely how that defect class survived. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7494cde3f7..76f4c81bfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,18 @@ here.** is unchanged and the default output is byte-identical. The `@metaobjectsdev/docs-site` README lists the five files and how to serve them. Copying them into the output is not done for you. +- **`ERR_ENUM_NORMALIZE_COLLISION`.** Loading refuses an enum, in all five ports, when two declared + members fold to the same key under the default `@normalize: strip` (`READ_ONLY` and `READONLY`, + `X_1` and `X1`), because a reply naming one could not be told from the other. Set + `@normalize: collapse` or `none`, or rename a member. Collapse and none are unaffected and no + accepted reply changes. Fixtures `error-enum-normalize-collision`, `enum-normalize-no-collision` + and `enum-normalize-collision-collapse-ok`. + +- **`ERR_COLUMN_NAME_COLLISION`.** Two fields of one stored object that resolve to the same + physical column (`userId` and `userID` under snake_case, or two fields sharing `@column`) are a + load error in every port, naming both fields. A model that loaded before with a duplicate column + now fails at load. Fixtures `error-column-name-collision` and `column-name-no-collision`. + - **Jinja templates in Python and TypeScript, held to the same payload check (ADR-0061, closes #483).** A `template.prompt` or `template.output` body may be a Jinja file — `/.jinja`, `.j2` or `.jinja2` — instead of a `.mustache` one. Nothing in the @@ -232,6 +244,32 @@ here.** ### Changed +- **C# no longer reads source code to work out its configuration.** `dotnet meta verify + --codegen` regenerates under the namespace `gen` resolves instead of reading it from the + committed output, so output generated under another namespace is drift. A C# namespace comes + from the package bindings (`GenConfig.TypeOverrides`, then `PackageNamespaces`, then the + `Convention` rule, then `UnmappedStrategy`, each keyed by an object's own metadata package); + the default `Flatten` strategy places every type in one namespace, `--namespace` or else the + new `csharp.namespace` key in `.metaobjects/config.json`. There is no default value: the + constant `Generated` is gone, and a run that emits a file needing the namespace without one + set stops and asks for it (a run that emits nothing never does). A namespace that is not a + dotted path of C# identifiers fails generation. **Upgrade step:** set + `"csharp": { "namespace": "" }` to the namespace your committed output uses (`Generated` + if you never passed `--namespace`), so `gen` and a bare `dotnet meta verify` both use it. `dotnet meta gen`, `verify --codegen`, `init` + and `eject` no longer check whether `codegen/Program.cs` runs `Generators.All`. + **Upgrade step:** if `codegen/Program.cs` builds its own generator list, replace that list + with `Generators.All`. Such a project still runs its own list and ignores + `codegen/Generators.cs`, and nothing warns about it. Move each entry into `Generators.All` + in `codegen/Generators.cs`, then change `Program.cs`: + + ```csharp + // before + IReadOnlyList generators = [new MetaObjects.Codegen.Generators.EntityGenerator()]; + return CodegenCli.Run(args, generators); + // after + return CodegenCli.Run(args, Generators.All); + ``` + - **On Postgres, `meta verify --db` and `meta migrate` ask Postgres whether a CHECK or an index expression is the one stored, instead of comparing rewritten text.** For each table the plan creates a temporary table with the live columns inside a transaction it rolls back (a savepoint @@ -260,6 +298,20 @@ here.** TypeScript-owned, so no other port carries this comparison. MySQL, SQL Server and Oracle are not migrate dialects, so they have no probe. +- **Java and Kotlin `` name globs now use the same grammar as `scope`.** `*` matches + within one `::` segment and no longer crosses `::`; a whole segment `**` matches one or more + segments; `@` and `?` are ordinary characters, no longer wildcards. `!` still excludes and the + `.[attr:...]` predicate is unchanged. A filter that now selects fewer objects than before fails + generation, naming the pattern, an object it no longer selects and the `**` replacement; before, + those objects vanished from the output silently. To update, in every `` list in your + pom (and every generator entry that repeats them, such as `entity` and `extractor`): + 1. Run `mvn metaobjects:generate`; the error lists each affected pattern with its replacement. + 2. Replace a segment that is only `*` with `**` (`acme::*` becomes `acme::**`). + 3. Replace a mixed segment such as `*Store` for nested packages with `**::*Store`: keep + `acme::*Store` for direct children and add `acme::**::*Store` as a second entry. + 4. Replace `@` (one segment) with `*`; replace each `?` with an explicit name or a `*`. + 5. Run `mvn metaobjects:generate` again; the generated file set should be unchanged. + - **A bare `verify` runs the same default gate set in every port: codegen drift, prompt templates and the requirement ledger.** Until now a bare run covered one gate: `meta verify` and `dotnet meta verify` ran only templates, so "run `verify` in CI" did not catch generated @@ -347,6 +399,92 @@ here.** Separately, the rename rewrite no longer touches a cast type, a function name or the inside of an `E'..'` literal when a column of that word is renamed. +- **Regex and hand-rewrite audit follow-ups.** Places that decided a match by regex or string + rewriting now ask the authoritative source: + - `parseCurrency` reads the locale's separators and sign forms from `Intl.NumberFormat`, returns + `null` on junk, and reads `$-5.00`, `(5.00)` and `5.00-` as negative. A long fraction rounds + exactly (`1.005` is 101, not 100). + - The migration SQL splitter keeps a SQLite `CREATE TRIGGER … BEGIN … END` as one statement and + honours Postgres `E'…'` backslash escapes; a bare `END` is dropped as transaction control. + - SQLite/D1 view comparison is case-sensitive and whitespace-exact inside quoted literals (as is + the Postgres fingerprint), so literal-only view changes are detected. A Postgres view whose + body has a whitespace-bearing literal re-stamps once. SQLite/D1 `AUTOINCREMENT` detection + ignores comments, literals and quoted identifiers. + - Python loads YAML with YAML 1.2 core-schema scalars, matching the TypeScript loader: `off`, + `no`, `1:30` and `2024-01-02` stay strings; a duplicate mapping key is a load error. The + Python codegen drift gate compares bytes, so CRLF edits are no longer hidden; `@pattern` uses + ASCII `\d`/`\w`/`\s`; a shared enum whose values differ only by case fails generation. + - Template verify and render agree with the Mustache engine in C#, Java, Python and TypeScript: + a partial tag in a comment is not expanded, `{{=<% %>=}}` raises no false + `ERR_VAR_NOT_ON_PAYLOAD`, `@requiredTags` is not satisfied inside a comment, and an unbalanced + section is `ERR_TEMPLATE_SYNTAX`. Render rejects a delimiter change in a template that uses + partials, or in a partial body, and verify reports the same as `ERR_TEMPLATE_SYNTAX` at the + first partial or the partial's ref (fixtures `verify-set-delimiter-with-partial`, + `verify-set-delimiter-in-partial`). The Python renderer rejects every `{{=`, so Python verify + reports every one (`expected-drift.python.json` overrides). + - The forgiving XML extract reader (all five ports) decodes entities, reads CDATA as text, + ignores comments, keeps literal `List` prose inside closed elements, and locates the real + root. An unclosed open tag after `>`, whitespace or the start of the body is still a child + (`text`). Seven new `extract-conformance` cases. + - `meta upgrade` ignores JSONC comments when scanning JSON metadata, so a brace or trailing + comment no longer skips a retirement silently. + - The hosted `leak-scan` checks renamed files and `/Users//` paths and shares its patterns + with the local pre-commit hook. + - The Java canonical serializer and `source.rdb` name resolution no longer treat an authored + name like `required7` or `rdb1` as auto-generated (new fixture + `authored-name-looks-autogenerated`). + - Workspace discovery parses `pnpm-workspace.yaml` as YAML and honours negations and `**`. + - Python generated Pydantic models match `validator.regex` `@pattern` (and the hostname format) + as an ASCII full match: `\d`/`\w`/`\s` no longer accept non-ASCII, lookahead works, a trailing + newline is rejected. A pattern Python's `re` cannot compile, such as `\p{L}`, now fails at `gen` + with a clear error instead of at validation time. **Upgrade step:** generated models now need + Pydantic 2.8 or later. They pass a compiled `re.ASCII` pattern to `Field(pattern=...)`, which + Pydantic accepts from 2.7.0 and keeps the flags of from 2.8.0; on an older 2.x the ASCII rule + is lost. Raise your `pydantic` requirement to `>=2.8`. + - Codegen refuses, with `ERR_COLLECTION_NAME_COLLISION`, two served entities whose snake-cased, + pluralized REST segments are equal (`ApiKey` and `APIKey`, both `/api_keys`); TS, C# and Python + compared only the pluralized PascalCase name. Java and Kotlin already refused it. + - The Spring and Kotlin filter-allowlist generators import their support class explicitly, so an + ejected, package-renamed copy compiles again (broken since #503). + - The CSV and spreadsheet formula-injection guard no longer prefixes `'` to strict numeric + literals (`-5`, `-5.25`) in all four render ports; every other value starting with + `= + - @ TAB CR` is still guarded. Fixture `render-csv-numeric-exempt`. + - Generated Pydantic models publish an ECMA-262-valid `pattern` in JSON Schema/OpenAPI while + still enforcing an ASCII full match. `parseCurrency` caches the locale shape per + (currency, locale). Workspace patterns find the members pnpm finds: `**` tolerates symlink cycles and + includes its base directory, a wildcard skips dot-directories, and only `node_modules`, + `bower_components`, `test` and `tests` are never entered (`dist` and `build` are ordinary directories). `meta eject --list` says `, reordered` for a reorder-only + copy; trailing whitespace and blank lines alone still read `identical` in Python and C#. + `parseCurrency` reads the decimal separator for zero-decimal currencies (JPY in `de-DE`). C# `ConstraintErrors` falls back to message text when a driver code is not a constraint + code. The migration splitter treats `E'...'` as backslash-escaped on Postgres only. + - Smaller instances: Java `` glob no longer throws on metacharacters; Java and C# + generated string literals escape newlines and control characters; `library.json` is read with + Gson; the Kotlin relations helper imports its primary-key type; `eject` reads `pom.xml` with + Maven's model reader; C# constraint-violation classification uses SQLSTATE / SQLite extended + codes before message text; numeric `@discriminatorValue` rejects non-ASCII digits and a + trailing newline and treats `"1"` and `"01"` as duplicates (new fixtures); `meta deps sync` + no longer warns that Python `1.0` differs from `1.0.0`; `DirectorySource` excludes containing + `/` match the relative path. + +- **The reply parser no longer corrupts escaped JSON strings or cuts a fenced reply short, in + every port (Kotlin through the Java render module).** The forgiving JSON reader decoded only + `\n`, `\t` and `\r`, so `"caf\u00e9"` came back as `cafu00e9` and was reported `EXTRACTED`; it + now reads the full JSON escape grammar (`\uXXXX` with surrogate pairs, `\b`, `\f`, `\/`). The + fence matcher closed at the first triple backtick inside the payload, so + `{"summary": "run ```make``` first"}` extracted as `run `; a fence now closes only at a line + that starts with a backtick run at least as long as the opening one. Python `LONG` coercion + also no longer rounds a value above 2^53 through `float`. Three new `extract-conformance` + cases (`json-string-escapes`, `json-fence-inner-backticks`, `json-fence-inner-block`) gate all + five ports. + +- **`eject --undo` no longer deletes a copy whose only edit is a reordering (TypeScript, Python, + C#).** "Untouched" was decided from a sorted multiset of lines, so swapping two statements read + as the stock file and `--undo` removed it without `--force`. A copy is now stock only when its + text matches the reference in order (TypeScript keeps formatter-insensitivity, and import runs + stay order-insensitive; Python and C# compare exact text, line endings aside). The "N behind, + M of your own" counts are unchanged. #505 is unreleased, so this ships with the first release + of `--undo`. + - **`@default: "now"` is the current time on Postgres, not the table's creation time.** The expected schema stored it as the literal `'now'`, and Postgres converts that literal to a timestamp once, when it creates the column's `DEFAULT`, so every row got the same value. It diff --git a/bun.lock b/bun.lock index 1fa3f2c880..0ea7786deb 100644 --- a/bun.lock +++ b/bun.lock @@ -422,6 +422,7 @@ "version": "1.1.4", "dependencies": { "@metaobjectsdev/metadata": "workspace:*", + "yaml": "^2.9.0", "zod": "^4.6.6", }, "devDependencies": { diff --git a/client/web/packages/react/src/currency-input.tsx b/client/web/packages/react/src/currency-input.tsx index 5a68839f24..c537f867d8 100644 --- a/client/web/packages/react/src/currency-input.tsx +++ b/client/web/packages/react/src/currency-input.tsx @@ -27,7 +27,7 @@ export interface CurrencyInputProps { * - Empty input + !required → emits 0 * - Empty input + required → reverts to last valid; no onChange * - Unparseable input on blur → reverts to last valid format; no onChange - * - Float drift avoided via Math.round in parseCurrency + * - Float drift avoided: parseCurrency uses decimal-string arithmetic */ export function CurrencyInput(props: CurrencyInputProps): ReactElement { const currency = props.currency ?? "USD"; diff --git a/client/web/packages/runtime-web/src/currency.ts b/client/web/packages/runtime-web/src/currency.ts index 4260da58d3..539ee9fd0f 100644 --- a/client/web/packages/runtime-web/src/currency.ts +++ b/client/web/packages/runtime-web/src/currency.ts @@ -1,11 +1,20 @@ /** * Currency helpers — read-side display, write-side parsing, minor-units lookup. - * All operations are integer-cents in / integer-cents out; float arithmetic is - * confined to a single Math.round to avoid drift (e.g., 15.99 * 100 = 1598.999...). + * All operations are integer-cents in / integer-cents out. parseCurrency does + * exact decimal-string arithmetic, so it never multiplies a float (e.g., + * 15.99 * 100 = 1598.999...). */ +const intlCache = new Map(); + function intlFor(currency: string, locale: string): Intl.NumberFormat { - return new Intl.NumberFormat(locale, { style: "currency", currency }); + const key = `${locale}\u0000${currency}`; + let fmt = intlCache.get(key); + if (fmt === undefined) { + fmt = new Intl.NumberFormat(locale, { style: "currency", currency }); + intlCache.set(key, fmt); + } + return fmt; } export function minorUnitsFor(currency: string, locale = "en-US"): number { @@ -25,34 +34,151 @@ export function formatCurrency( return intlFor(currency, locale).format(cents / factor); } +/** Whitespace-like group separators (fr-FR uses U+202F) accept any ordinary space typed. */ +const SPACE_LIKE = /^[\s  ]+$/; +const SPACE_CLASS = "[\\s\\u00a0\\u202f]"; + +function escapeRe(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** What the locale writes: separators, digit grouping sizes, and the currency's spellings. */ +interface LocaleShape { + readonly group: string; + readonly decimal: string; + /** Size of the last integer group and of the ones before it (3/3, or 3/2 for en-IN). */ + readonly primary: number; + readonly secondary: number; + readonly tokens: readonly string[]; +} + +/** What `parseCurrency` derives per (currency, locale): the shape and its compiled regexes. */ +interface ParseSpec { + readonly shape: LocaleShape; + readonly grammar: RegExp; + readonly groupStrip: RegExp; +} + +const parseSpecCache = new Map(); + +function parseSpecFor(currency: string, locale: string): ParseSpec { + const key = `${locale}\u0000${currency}`; + let spec = parseSpecCache.get(key); + if (spec === undefined) { + const shape = localeShape(currency, locale); + spec = { + shape, + grammar: numberGrammar(shape), + groupStrip: new RegExp(groupClass(shape), "g"), + }; + parseSpecCache.set(key, spec); + } + return spec; +} + +function localeShape(currency: string, locale: string): LocaleShape { + let group = ","; + let decimal = "."; + let primary = 3; + let secondary = 3; + const tokens = new Set(); + try { + const parts = new Intl.NumberFormat(locale, { + style: "currency", + currency, + useGrouping: true, + minimumFractionDigits: 2, + }).formatToParts(123456789.12); + group = parts.find((p) => p.type === "group")?.value ?? group; + decimal = parts.find((p) => p.type === "decimal")?.value ?? decimal; + const groups = parts.filter((p) => p.type === "integer").map((p) => p.value.length); + primary = groups[groups.length - 1] ?? primary; + secondary = groups[groups.length - 2] ?? primary; + for (const display of ["symbol", "narrowSymbol", "code"] as const) { + const token = new Intl.NumberFormat(locale, { style: "currency", currency, currencyDisplay: display }) + .formatToParts(1) + .find((p) => p.type === "currency")?.value; + if (token) tokens.add(token); + } + } catch { + // unknown currency or locale: fall back to the en-US shape, with the code as the token + tokens.add(currency); + } + // Longest first so "US$" is peeled before "$". + return { group, decimal, primary, secondary, tokens: [...tokens].sort((x, y) => y.length - x.length) }; +} + +function groupClass(shape: LocaleShape): string { + return SPACE_LIKE.test(shape.group) ? SPACE_CLASS : escapeRe(shape.group); +} + +/** The unsigned number grammar for a locale: grouped or plain integer, optional fraction. */ +function numberGrammar(shape: LocaleShape): RegExp { + const g = groupClass(shape); + const d = escapeRe(shape.decimal); + const grouped = `\\d{1,${shape.secondary}}(?:${g}\\d{${shape.secondary}})*${g}\\d{${shape.primary}}`; + return new RegExp(`^(${grouped}|\\d+)?(?:${d}(\\d*))?$`); +} + +/** + * Parse text a person typed or pasted into integer minor units, or `null` when it is not a + * money amount in this locale. The locale's own separators, grouping and currency spellings + * come from `Intl.NumberFormat#formatToParts`; the accepted forms are a strict grammar, not + * "strip everything that is not a digit". One negative marker is allowed, in any of the + * forms `-5.00`, `$-5.00`, `-$5.00`, `5.00-` or `(5.00)`; two, or anything unrecognised + * (`1e3`, `12.50 (2 items)`, a separator the locale does not use), is `null`. + */ export function parseCurrency( text: string, currency = "USD", locale = "en-US", ): number | null { - if (typeof text !== "string" || text.trim() === "") return null; - - // Detect locale's decimal separator via Intl. - const sample = (0.1).toLocaleString(locale); - const decimalSep = sample.includes(",") ? "," : "."; - - // Determine sign from leading minus. - const sign = text.trim().startsWith("-") ? -1 : 1; - - // Remove all non-numeric chars except the decimal separator. - const escapedSep = decimalSep === "." ? "\\." : decimalSep; - const digitsAndDecimal = text.replace(new RegExp(`[^0-9${escapedSep}]`, "g"), ""); - if (digitsAndDecimal === "" || digitsAndDecimal === decimalSep) return null; + if (typeof text !== "string") return null; + const { shape, grammar, groupStrip } = parseSpecFor(currency, locale); + let rest = text.replace(/[−‒-—]/g, "-").trim(); + if (rest === "") return null; - // Normalize to "." for parseFloat. - const normalized = - decimalSep === "." ? digitsAndDecimal : digitsAndDecimal.replace(decimalSep, "."); - - if ((normalized.match(/\./g) ?? []).length > 1) return null; + let markers = 0; // negative markers seen: sign or parentheses + let positive = false; + let tokenSeen = false; + for (let peeled = true; peeled; ) { + peeled = false; + rest = rest.trim(); + if (rest.startsWith("(") && rest.endsWith(")")) { + rest = rest.slice(1, -1); + markers++; + peeled = true; + } else if (rest.startsWith("-") || rest.startsWith("+")) { + if (rest[0] === "-") markers++; + else positive = true; + rest = rest.slice(1); + peeled = true; + } else if (rest.endsWith("-")) { + markers++; + rest = rest.slice(0, -1); + peeled = true; + } else if (!tokenSeen) { + const token = shape.tokens.find((t) => rest.startsWith(t) || rest.endsWith(t)); + if (token !== undefined) { + rest = rest.startsWith(token) ? rest.slice(token.length) : rest.slice(0, -token.length); + tokenSeen = true; + peeled = true; + } + } + } + if (markers > 1 || (markers === 1 && positive)) return null; - const parsed = Number.parseFloat(normalized); - if (!Number.isFinite(parsed)) return null; + const match = grammar.exec(rest.trim()); + if (match === null) return null; + const whole = (match[1] ?? "").replace(groupStrip, ""); + const fraction = match[2] ?? ""; + if (whole === "" && fraction === "") return null; - const factor = Math.pow(10, minorUnitsFor(currency, locale)); - return Math.round(sign * parsed * factor); + // Exact decimal-string arithmetic: no float multiply, half rounds away from zero. + const minor = minorUnitsFor(currency, locale); + const kept = fraction.slice(0, minor).padEnd(minor, "0"); + let units = Number(`${whole === "" ? "0" : whole}${kept}`); + if ((fraction[minor] ?? "0") >= "5") units += 1; + if (!Number.isSafeInteger(units)) return null; + return markers === 1 && units !== 0 ? -units : units; } diff --git a/client/web/packages/runtime-web/test/currency.test.ts b/client/web/packages/runtime-web/test/currency.test.ts index a6448ffdfb..25945cf17a 100644 --- a/client/web/packages/runtime-web/test/currency.test.ts +++ b/client/web/packages/runtime-web/test/currency.test.ts @@ -74,3 +74,72 @@ describe("parseCurrency", () => { expect(parseCurrency("15.9.9.9")).toBeNull(); }); }); + +describe("parseCurrency — strict grammar and sign forms (audit finding 5)", () => { + test.each([ + ["$-5.00", -500], + ["-$5.00", -500], + ["(5.00)", -500], + ["($5.00)", -500], + ["5.00-", -500], + ["−5.00", -500], + ["+5.00", 500], + ["USD 1,234.50", 123450], + ["1.005", 101], + [".5", 50], + ["5.", 500], + ["0.00", 0], + ["-0.00", 0], + ])("%s → %p", (input, cents) => { + expect(parseCurrency(input)).toBe(cents); + }); + + test.each(["1e3", "12.50 (2 items)", "--5", "(-5.00)", "-5.00-", "$$5", "1,23.45", "1,2345", "5 apples", "-", "$", "()", "1.5.2"])( + "%s → null", + (input) => { + expect(parseCurrency(input)).toBeNull(); + }, + ); + + test("de-DE uses , as decimal and . as grouping; '1.5' is not an amount", () => { + expect(parseCurrency("1.234,56 €", "EUR", "de-DE")).toBe(123456); + expect(parseCurrency("-5,00 €", "EUR", "de-DE")).toBe(-500); + expect(parseCurrency("1.5", "EUR", "de-DE")).toBeNull(); + }); + + test("fr-FR accepts an ordinary space for its narrow no-break group separator", () => { + expect(parseCurrency("1 234,50 €", "EUR", "fr-FR")).toBe(123450); + }); + + test("en-IN groups 2,2,3", () => { + expect(parseCurrency("₹12,34,567.00", "INR", "en-IN")).toBe(123456700); + }); + + test("de-DE JPY keeps the locale's , decimal although yen has no minor units", () => { + expect(parseCurrency("1.5", "JPY", "de-DE")).toBeNull(); + expect(parseCurrency("1,5", "JPY", "de-DE")).toBe(2); + expect(parseCurrency("1.000 ¥", "JPY", "de-DE")).toBe(1000); + }); + + test("JPY has no minor units; a fraction rounds", () => { + expect(parseCurrency("¥1,000", "JPY")).toBe(1000); + }); +}); + +describe("parseCurrency memoisation", () => { + test("repeat parses for one (currency, locale) build no new Intl.NumberFormat", () => { + expect(parseCurrency("1.234,56", "EUR", "de-DE")).toBe(123456); // warm the cache + const Original = Intl.NumberFormat; + let constructed = 0; + (Intl as unknown as { NumberFormat: unknown }).NumberFormat = function (...args: ConstructorParameters) { + constructed++; + return new Original(...args); + }; + try { + for (let i = 0; i < 5; i++) expect(parseCurrency("1.234,56", "EUR", "de-DE")).toBe(123456); + } finally { + Intl.NumberFormat = Original; + } + expect(constructed).toBe(0); + }); +}); diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index c6d41380bb..36834aadbc 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -26,14 +26,14 @@ regenerate with `ls -d fixtures//*/ | wc -l` for directory-shaped corpor | Corpus | Fixtures | TS | Java | Kotlin | C# | Python | |---|---|---|---|---|---|---| -| [`fixtures/conformance/`](../fixtures/conformance/) (metamodel) | 413 | ✓ | ✓ | inherits via `metadata-ktx` | ✓ | ✓ | +| [`fixtures/conformance/`](../fixtures/conformance/) (metamodel) | 421 | ✓ | ✓ | inherits via `metadata-ktx` | ✓ | ✓ | | [`fixtures/yaml-conformance/`](../fixtures/yaml-conformance/) | 16 | 16 / 16 | 15 / 16 (1 ledgered: `yaml-quoted-leading-zero` — Java pipeline strips quotes off `"007"`) | inherits via Java | 15 / 16 (1 ledgered: `error-yaml-coerced-hex-in-string` — YamlDotNet doesn't coerce `0xFF`) | 16 / 16 | -| [`fixtures/verify-conformance/`](../fixtures/verify-conformance/) | 37 | ✓ | ✓ | inherits via Java | ✓ | ✓ | +| [`fixtures/verify-conformance/`](../fixtures/verify-conformance/) | 45 | ✓ | ✓ | inherits via Java | ✓ | ✓ | | [`fixtures/verify-strict-conformance/`](../fixtures/verify-strict-conformance/) | 1 | ✓ | — | — | — | ✓ | -| [`fixtures/render-conformance/`](../fixtures/render-conformance/) | 16 | ✓ | ✓ | inherits via Java | ✓ | ✓ | +| [`fixtures/render-conformance/`](../fixtures/render-conformance/) | 18 | ✓ | ✓ | inherits via Java | ✓ | ✓ | | [`fixtures/jinja-render-conformance/`](../fixtures/jinja-render-conformance/) (ADR-0061 — Jinja bodies render the same in jinja2 and nunjucks) | 15 | ✓ (nunjucks) | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | refuses, via Java | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | ✓ (jinja2) | | [`fixtures/jinja-verify-conformance/`](../fixtures/jinja-verify-conformance/) (ADR-0061 — the payload check over a Jinja body) | 31 | ✓ | refuses Jinja | refuses, via Java | refuses Jinja | ✓ | -| [`fixtures/extract-conformance/`](../fixtures/extract-conformance/) | 48 | ✓ | ✓ | inherits the shared JVM engine | ✓ | ✓ | +| [`fixtures/extract-conformance/`](../fixtures/extract-conformance/) | 58 | ✓ | ✓ | inherits the shared JVM engine | ✓ | ✓ | | [`fixtures/output-prompt-conformance/`](../fixtures/output-prompt-conformance/) | 17 | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/persistence-conformance/`](../fixtures/persistence-conformance/) | 59 (53 query + 6 migration) | all 59; the 24 `report-*` and the `-15` decimal scenarios also on SQLite, MySQL and D1, `decimal-exact` also on MySQL (a scenario's `engines:` key is exact) | 53 query (migrations TS-only, ADR-0015); the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query (via Exposed); the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query; the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query (pg8000); the `report-*` and `engines:` scenarios also on SQLite (`sqlite3`) and MySQL (PyMySQL) | | [`fixtures/api-contract-conformance/`](../fixtures/api-contract-conformance/) | 119 (31 core + 10 tph + 9 m2m + 5 jsonb + 2 write-through + 2 secret-fields + 12 projection + 37 report + 11 decimal) | ✓ (Fastify reference + generated lane); `report/` and `decimal/` also on SQLite, MySQL and D1 | ✓ (embedded HTTP + JDBC); `report/` and `decimal/` generated controllers over OMDB on Postgres, SQLite and MySQL | ✓ (embedded HTTP + Exposed); `report/` and `decimal/` also on SQLite and MySQL | ✓ (Kestrel + Npgsql); `report/` and `decimal/` also on SQLite and MySQL | ✓ (FastAPI + pg8000); `report/` and `decimal/` also on SQLite and MySQL | @@ -284,7 +284,7 @@ unit-test runners (`bun test`, `dotnet test`, `pytest`, `mvn test`) pull Docker. ## Fixture-to-doc mapping -### `fixtures/conformance/` — metamodel loader + canonical serializer (413) +### `fixtures/conformance/` — metamodel loader + canonical serializer (421) | Fixture prefix | Feature doc | |---|---| @@ -319,14 +319,15 @@ before its base in one file, etc.) and 7 `error-yaml-*` fixtures that pin the YAML 1.1 coercion guards (bool / null / hex in string contexts; numeric in enum contexts; reserved-as-attr). -### `fixtures/render-conformance/` (16) +### `fixtures/render-conformance/` (18) -All 16 fixtures → [features/templates-and-payloads.md](features/templates-and-payloads.md) -(render engine output section). 4 are end-to-end shape examples (prompt / email / -spreadsheet / CSV-injection escape); 11 pin Mustache-engine semantics — dotted-path -lookup, parent-context fallthrough, falsy/empty-array section behavior, inverted -sections, nested partials, standalone-tag whitespace stripping, raw-HTML bypass, -trailing-newline preservation, unicode multibyte handling, and the `-last` list-separator accessor. +All 18 fixtures → [features/templates-and-payloads.md](features/templates-and-payloads.md) +(render engine output section). 5 are end-to-end shape and escape examples (prompt / email / +spreadsheet / CSV-injection escape / CSV numeric-literal exemption); 13 pin Mustache-engine +semantics — dotted-path lookup, parent-context fallthrough, falsy/empty-array section behavior, +inverted sections, nested partials, a comment tag beside a partial, standalone-tag whitespace +stripping, raw-HTML bypass, trailing-newline preservation, unicode multibyte handling, the derived +`has` accessor, and the `-last` list-separator accessor. ### `fixtures/jinja-render-conformance/` (15) and `fixtures/jinja-verify-conformance/` (31) @@ -609,8 +610,8 @@ files into a real Cube. ## Orphaned fixtures (tested but not yet documented) -The fixtures in the nine corpora mapped above (metamodel 413 + yaml 16 + verify 37 -+ render 16 + persistence 59 + api-contract 119 + source-resolution 25 + scope 10 + +The fixtures in the nine corpora mapped above (metamodel 421 + yaml 16 + verify 45 ++ render 18 + persistence 59 + api-contract 119 + source-resolution 25 + scope 10 + dependency 37) each map to a feature doc, and so do the two requirement corpora, `shared-model-conformance` and the `cube-model` corpus, whose case lists live in their own READMEs. None are orphaned today. The remaining corpora in the totals table gate tooling contracts (registry manifests, provider diff --git a/docs/features/cli.md b/docs/features/cli.md index 545aaf6ae7..52c6cd7344 100644 --- a/docs/features/cli.md +++ b/docs/features/cli.md @@ -216,7 +216,10 @@ the committed `--out` tree, never touching it), and a **clean `--db` rejection (exit 2)** — bare `dotnet meta verify` runs `--templates` and, when an output directory and `codegen/Generators.cs` are present, `--codegen`. The output directory is `csharp.outDir` in `.metaobjects/config.json` (the same setting `gen` writes to); `--out ` -wins over it. With neither, the codegen gate is skipped with a note that names the setting. A bare run prints the subverb note. The **Java/Kotlin `mvn metaobjects:verify`** port expresses the same vocabulary as a +wins over it. With neither, the codegen gate is skipped with a note that names the setting. The +namespace for types no package binding places is `--namespace`, else `csharp.namespace` there, +read the same way by `gen` and `verify`; a run that emits such a type with neither set stops and +asks for one. A bare run prints the subverb note. The **Java/Kotlin `mvn metaobjects:verify`** port expresses the same vocabulary as a `mode` parameter (Maven goals are parameter-driven, not flag-driven). With `mode` unset the goal runs both gates below, skipping the template gate with a note when no `templateRoot` is set. `-Dmeta.verify.mode=codegen` diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 4c61d71ab6..6dfda56930 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -84,17 +84,20 @@ become a four-port concern. The Node CLI remains the file's only writer; `meta init --config-only` writes it into a Maven- or pip-rooted project without adding a TypeScript scaffold. -**`scope` and `migrate.scope` remain Node-CLI-only.** Java's shipped `` -grammar uses `*` to cross the `::` package separator and `@` to match one -segment — respectively `scope`'s `**` and `*`, **inverted** — plus `!`-prefix -exclusion and a `.[attr]` predicate that `scope` cannot express at all -(`GeneratorUtil.createRegexFromGlob` even carries a `TODO` conceding its own -separator handling is wrong). Both are output filters over the same resolved -file set, so they compete rather than layer, and reconciling them changes -behavior for existing Java consumers — a separate, adopter-affecting decision -rather than a mechanical port. No cross-port behavior depends on `scope`. (The pattern -grammar itself is ported and corpus-gated in Java, Kotlin and C# as a library, ahead of the -cross-repo dependency work; no command in those ports reads `scope` from the config yet.) +**`scope` and `migrate.scope` remain Node-CLI-only, but Java's `` name glob is +the same grammar.** Since 8.2.0 the Maven plugin's `` patterns use `scope`'s +pattern grammar: `*` inside a segment never crosses `::`, a segment that is exactly `**` +spans one or more segments, and every other character (`@` and `?` included) is literal. +The pre-8.2.0 rule let `*` cross `::` and gave `@` and `?` wildcard meaning. A filter that +the old rule matched and the new one does not fails generation with an error naming the +pattern and its `**` replacement (for example `acme::*` becomes `acme::**`), so output +never drops silently. `` keeps what `scope` cannot express: `!`-prefix exclusion +and a `.[attr]` predicate. Both are output filters over the same resolved file set, so +they compete rather than layer. No cross-port behavior depends on `scope`; the TypeScript, +C#, Python and Kotlin ports have no separate generator-filter glob (Kotlin runs through the +same Maven plugin and so the same ``). (The pattern grammar itself is ported and +corpus-gated in Java, Kotlin and C# as a library, ahead of the cross-repo dependency work; +no command in those ports reads `scope` from the config yet.) **Two things are deliberately not cross-port contracts**, because the corpus that gates this feature says so explicitly: diff --git a/docs/features/migrations/docs-are-generators.md b/docs/features/migrations/docs-are-generators.md index ef3ff7f10b..de92e9b139 100644 --- a/docs/features/migrations/docs-are-generators.md +++ b/docs/features/migrations/docs-are-generators.md @@ -324,14 +324,11 @@ A project that already has `codegen/Program.cs` with an inline list: move each e `new Codegen.Generators.()`, a packaged one `new MetaObjects.Codegen.Generators.()`), and replace `Program.cs` with the two lines above. `dotnet meta init` writes `Generators.cs` beside an existing `Program.cs` and leaves `Program.cs` alone. Until -`Program.cs` runs `Generators.All`, `dotnet meta gen` prints this and exits 2, writing nothing -(`dotnet meta verify --codegen` in place of `dotnet meta gen` for the gate): +`Program.cs` runs `Generators.All`, `dotnet meta gen` and `dotnet meta verify --codegen` run +the list `Program.cs` builds and ignore `Generators.cs`; nothing warns about it, so replace +`Program.cs` as part of the upgrade. -``` -dotnet meta gen: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and `return CodegenCli.Run(args, Generators.All);`. See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section "C#". -``` - -`init` and `eject` say the same, and warn when `codegen/Codegen.csproj` still pins an older +`init` and `eject` warn when `codegen/Codegen.csproj` still pins an older `MetaObjects.Codegen`: set that version to the tool's, or the 1.0 codegen runs. **A template spec.** Through 1.0, `dotnet meta gen` and `verify --codegen` also read a diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index c6d10a9d2f..1ae5208fa7 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -160,10 +160,10 @@ falls behind the reference. To drop one: - **`identical to the reference` is the only mark that proves a copy unmodified.** Such a copy can be replaced without looking further. - **Every other mark needs a look first.** `same content as the reference, your - formatting` comes from a comparison that ignores line order, so a copy whose - statements you reordered reads the same way. `DIFFERS` with `0 line(s) of your own` - rules out lines you added or edited, not lines you removed or reordered: a line you - deleted is counted as "behind", exactly like a line upstream added. Before dropping + formatting` means the lines match in order. A copy whose statements you reordered + reads `DIFFERS` with `, reordered` appended. `DIFFERS` with `0 line(s) of your own` + rules out lines you added or edited, not lines you removed: a line you deleted is + counted as "behind", exactly like a line upstream added. Before dropping a copy with either mark, confirm it from the file's history (`git log -p -- codegen/generators/.ts` shows no edit after the eject) or from the diff against the reference diff --git a/docs/features/reporting.md b/docs/features/reporting.md index ab2140965b..87e3edb98b 100644 --- a/docs/features/reporting.md +++ b/docs/features/reporting.md @@ -795,9 +795,10 @@ refused with `400 export_with_count`. The route fetches is materialised up to the cap; there is no streaming in this version. **Formula injection.** A CSV cell that is a string, an enum symbol or a uri and starts with `=`, -`+`, `-`, `@`, TAB or CR gets a leading `'`, so a spreadsheet reads it as text. This is the -render escapers' rule, one rule in every port. A number or a date is never changed, so `-5` -stays a number. +`+`, `-`, `@`, TAB or CR gets a leading `'`, so a spreadsheet reads it as text, unless the whole +cell is a strict numeric literal (an optional `-`, ASCII digits, an optional fraction such as +`-5` or `12.50`), which is left alone. This is the render escapers' rule, one rule in every +port. A number or a date is never changed. **Per-port XML null default.** The default follows the port's standard XML serializer (ADR-0019 Amendment 2); the corpus pins `"nil"` in every lane. diff --git a/docs/ports/csharp.md b/docs/ports/csharp.md index 16c7ee5844..dc128fc15f 100644 --- a/docs/ports/csharp.md +++ b/docs/ports/csharp.md @@ -181,10 +181,10 @@ in `.metaobjects/config.json`, below). ```bash # Run the list: EF Core entities + AppDbContext + CRUD minimal-API routes -dotnet meta gen ./metaobjects --out ./Generated --namespace Acme.Blog +dotnet meta gen ./metaobjects --out ./Generated # The drift gate: regenerate into a temp tree and fail (exit 1, naming the files) if the -# committed ./Generated differs. Run it in CI; the namespace is inferred from ./Generated. +# committed ./Generated differs. Run it in CI. dotnet meta verify ./metaobjects --codegen --out ./Generated ``` @@ -196,13 +196,24 @@ against `prompts/` when it exists, else `templates/`, and fails the same way whe exists. It also runs the codegen gate when `--out ` and `codegen/Generators.cs` are present, and skips it with a note otherwise; name both subverbs to require both. -The output directory is a project setting, so `gen` and `verify` cannot disagree about it. -Declare it once in `.metaobjects/config.json`, relative to the directory holding `.metaobjects/`: +The output directory and the namespace are project settings, so `gen` and `verify` cannot +disagree about them. Declare them once in `.metaobjects/config.json`; `outDir` is relative to the +directory holding `.metaobjects/`: ```json -{ "schema_version": 1, "csharp": { "outDir": "Generated" } } +{ "schema_version": 1, "csharp": { "outDir": "Generated", "namespace": "Acme.Blog" } } ``` +Each generated type's namespace comes from the package bindings, keyed by that type's own +metadata package: a type-level override (`GenConfig.TypeOverrides`), then a package-level one +(`PackageNamespaces`), then the `Convention` rule (strip a package prefix, prepend a root, case +each `::` segment), then `UnmappedStrategy`. The default strategy, `Flatten`, puts every type in +one namespace: `--namespace` when given, else `csharp.namespace`. There is no built-in value, so +a run that emits a file needing it with neither set stops and names both; a run that emits +nothing never asks. The namespace must be a dotted path of C# identifiers. `dotnet meta gen` and +`verify` set only that flat namespace; the per-package bindings are `GenConfig` properties, set +when you run codegen from your own code with `CodegenRunner`. + With that, `dotnet meta gen` writes there, `dotnet meta verify --codegen` diffs against it, and a plain `dotnet meta verify` in CI runs the codegen gate with no flags. `--out ` still works and wins over the config. A project that sets neither gets the skip note, which names the setting. diff --git a/docs/ports/java.md b/docs/ports/java.md index ba156e209d..e4f14b9f02 100644 --- a/docs/ports/java.md +++ b/docs/ports/java.md @@ -425,6 +425,15 @@ extractor lands beside its class. A pom with more than one `entity` entry lists `extractor` was listed; that was output no entry declared, and 8.1.0 removed it (ADR-0034 Amendment 9). +**`` name globs.** A generator's `` take the `scope` pattern grammar: +`*` stays inside one `::` segment, a whole segment `**` spans packages, anything else is +literal (`@` and `?` were wildcards through 8.1.x and no longer are). A `!` prefix excludes. +A pattern that selected objects before 8.2.0 and now selects fewer fails generation naming +the pattern and its `**` replacement, so output cannot vanish silently. Upgrade: +`acme::*` becomes `acme::**` wherever it must reach nested packages. `acme::*Store` keeps +matching direct children as `acme::*Store`; nested children need `acme::**::*Store` as a +second entry. `@` becomes `*`. + ### Shared and `@provided` enums A package-level abstract `field.enum` that fields extend is one top-level Java `enum`, diff --git a/docs/ports/python.md b/docs/ports/python.md index a1fae38a40..3891af36b3 100644 --- a/docs/ports/python.md +++ b/docs/ports/python.md @@ -3,8 +3,9 @@ Targets Python 3.11+ on the FastAPI / Pydantic stack. Ships the metadata loader (canonical JSON + sigil-free YAML), conformance runner over the shared corpora, the FR-004 render engine, and the entity / payload / router -codegen. **Keep your ORM:** the `entity` generator writes Pydantic v2 models with no ORM of any -kind, so your SQLAlchemy or Django models stay hand-written beside them. Schema migrations are owned by the Node `meta` CLI (ADR-0015) — the +codegen. **Keep your ORM:** the `entity` generator writes Pydantic v2 models (Pydantic 2.8 or later: +a `validator.regex` field passes a compiled `re.ASCII` pattern, whose flags Pydantic keeps from 2.8) +with no ORM of any kind, so your SQLAlchemy or Django models stay hand-written beside them. Schema migrations are owned by the Node `meta` CLI (ADR-0015) — the Python port has no `migrate` command by design; it consumes the canonical `schema.postgres.sql` artifact verbatim. diff --git a/docs/ports/typescript-client.md b/docs/ports/typescript-client.md index 86604f0d97..72dfbeffa5 100644 --- a/docs/ports/typescript-client.md +++ b/docs/ports/typescript-client.md @@ -343,8 +343,9 @@ on `Author`): the locale-formatted string. Server never formats. 4. **Form input**: `` from `@metaobjectsdev/react` is a controlled bidirectional input — focus strips symbol + grouping for - editing, blur re-formats and emits cents to `onChange`. Float drift - is bounded by a single `Math.round` in `parseCurrency`. + editing, blur re-formats and emits cents to `onChange`. `parseCurrency` + does exact decimal-string arithmetic (no float multiply) and rounds the + first dropped digit half away from zero. Metadata: diff --git a/examples/showcase/site-payload.json b/examples/showcase/site-payload.json index 2e96740fde..a3c26fd067 100644 --- a/examples/showcase/site-payload.json +++ b/examples/showcase/site-payload.json @@ -7,7 +7,7 @@ "metamodel": "1.2" }, "counts": { - "fixtures": 413, + "fixtures": 421, "corpora": 34, "baseTypes": 17 }, diff --git a/fixtures/conformance/ERROR-CODES.json b/fixtures/conformance/ERROR-CODES.json index f736a92d2e..d2c931bbad 100644 --- a/fixtures/conformance/ERROR-CODES.json +++ b/fixtures/conformance/ERROR-CODES.json @@ -38,6 +38,7 @@ "ERR_EXTENDS_ORIGIN_MISMATCH": "FR-024 (ADR-0029 decision 7): a field declares both an entity-nested extends (shape lineage) and an origin.passthrough @from (data lineage) and they disagree — the resolved @from target is not the field's resolved extends target (nor anywhere on its extends chain). Host-agnostic (projections, entities, values). origin.aggregate is never judged (it computes something new); a top-level abstract extends target is never judged (shape-only reuse makes no lineage claim).", "ERR_PASSTHROUGH_TYPE_MISMATCH": "#185: a field carrying origin.passthrough @from: \"Entity.field\" declares a field. or array-ness that differs from its resolved source field — a passthrough forwards the value unchanged, so its type must be identical. Host-agnostic (projections, entities, values, stored-proc parameter refs). Nullability is NOT judged (a view over an outer join legitimately widens NOT NULL → nullable). Opt out of a deliberate type change with @convert: true on the origin.passthrough (an acknowledgement — it does not generate a cast); real type-converting projections are origin.expression's job (#159).", "ERR_DERIVED_FIELD_NO_READ_SOURCE": "FR-024 (spec §7): an object.entity field carrying an origin.* child is derived (read-only) and must be providable — the entity must declare at least one source with a read-only @kind (view/materializedView/storedProc/tableFunction, e.g. a @role replica view beside the table primary). Table-only or source-less entities with origin-bearing fields error on the field. Projections and object.value hosts are exempt. Until the Phase-E B4b cutover removes view-primary entities, a read-only-kind PRIMARY source also counts as providable (legacy spelling).", + "ERR_COLUMN_NAME_COLLISION": "Two fields of one stored object (a concrete object declaring a source) resolve to the same physical column name: snake_case folds `userId` and `userID` into `user_id`, or two fields share an explicit @column. Only RESOLVED physical names are compared (explicit @column honoured, inherited fields included, derived and source-less objects exempt). The message names both fields and the column.", "ERR_INVALID_TEMPLATE": "A template (prompt/output) declares a @payloadRef that does not resolve, or @requiredSlots that are not fields on its payload.", "ERR_INVALID_RELATIONSHIP": "FR-017: a M:N relationship's slim vocabulary is invalid — @through does not resolve to a junction declaring two identity.reference children (ADR-0042: @through resolves package-locally when bare, exactly when FQN — a bare cross-package @through no longer binds elsewhere), @sourceRefField does not match one of them, or a M:N-only attr (@through/@sourceRefField/@symmetric) is set on a non-M:N (1:N / @cardinality:one) relationship; OR a relationship's @objectRef does not resolve to any object in the loaded tree (a dangling target).", "ERR_INVALID_REFERENCE": "An identity.reference @references names an FK target object that does not resolve to any object in the loaded tree (a dangling cross-reference between metadata).", @@ -48,7 +49,7 @@ "ERR_REQUIRED_SLOT_UNUSED": "Build-time verify (warning): a template's declared @requiredSlots slot is never referenced by the template text.", "ERR_OUTPUT_TAG_MISSING": "Build-time verify: a template declares @requiredTags but its text (including provider-resolved partials) omits a required output tag's opening () form. path = the tag name.", "ERR_TEMPLATE_NOT_STATIC": "Build-time verify (ADR-0061, Jinja bodies, Python and TypeScript): a template reference the check cannot settle statically — a computed attribute or include name, a dynamic subscript, a call, a tag outside the supported set. Reported rather than passed; path = the expression or tag.", - "ERR_TEMPLATE_SYNTAX": "Build-time verify (ADR-0061, Jinja bodies, Python and TypeScript): the template body, or a template it includes or imports, does not parse. path = the parser's message, which is engine-specific.", + "ERR_TEMPLATE_SYNTAX": "Build-time verify: the template body, or a template it includes or imports, does not parse. Jinja bodies (ADR-0061, Python and TypeScript): path = the parser's message, which is engine-specific. Mustache bodies (every port): an unbalanced section (a close with no open, a close naming another section, a section never closed); path = the section's name.", "ERR_TEMPLATE_ENGINE_UNSUPPORTED": "Build-time verify and render (ADR-0061): a template body is written for an engine this port does not ship — a Jinja body (.jinja / .j2 / .jinja2) in the Java, Kotlin or C# port, which render Mustache only. path = the template ref.", "ERR_BAD_ATTR_FILTER": "An attr.filter references a field that is not addressable in that context or uses a disallowed op: a dataGrid @filter over a non-filterable field or an op disallowed for the field's subtype, or a projection row-scope @filter over a non-declared (dangling) or aggregate-derived field (#207). FR-044: a segment / measure.aggregate / object.report @filter names a field its entity (or @from) lacks or an op illegal for the field (S1); a relative-date value { \"now\": \"\" } appears outside the @filter of a segment, measure.aggregate or object.report (F1); or a relative-date value sits on a field that is not field.date/field.timestamp, under an op other than gt/gte/lt/lte, carries a value that is not an ISO-8601 duration, or has a startOf other than day, week or month, or a key besides now and startOf (F2).", "ERR_STORAGE_FLATTENED_ARRAY": "@storage \"flattened\" cannot be combined with isArray=true.", @@ -95,6 +96,7 @@ "ERR_ORIGIN_UNDER_SQL_BODY": "#208: a host object whose read source carries @sql also declares an origin.*-bearing (derived) field, or (on object.projection) an @filter (#207) — two sources of truth for the same body (the synthesized derivation/outer-WHERE vs. the author's verbatim SQL). Fail-closed.", "ERR_ENUM_EXTENDS_VALUES_CONFLICT": "A field.enum both extends a shared package-level abstract enum and declares its own @values. One shared enum type has one member set — the own @values would be silently dropped in codegen. Remove the own @values to inherit the shared set, or extend a concrete (non-shared) enum instead.", "ERR_ENUM_INT_VALUE_MAP_ARRAY": "A field.enum carries @intValueMap together with isArray=true. Int-backing is a persistence-layer codec and no port implements it element-wise over an array column, so the combination would silently persist member SYMBOLS into an integer array. An array-of-enum stays string-backed: drop @intValueMap, or make the field scalar.", + "ERR_ENUM_NORMALIZE_COLLISION": "Two declared members of one field.enum normalize to the same key under the default `@normalize: strip` (upper-case, delete everything outside A-Z0-9), e.g. READ_ONLY and READONLY. A reply naming one of them is indistinguishable from the other, so the extract engine would coerce it to the wrong member. Rename a member so the two differ after that fold, or set `@normalize: collapse` or `none` on the field. Not raised under collapse or none.", "ERR_REQUIREMENT_RETIRED_HAS_IMPLEMENTORS": "A requirement.* with @status: retired declares @implementedBy. Refused rather than exempted (FR-039): a retired capability has no implementation by definition, so forbidding the attribute makes the dangling-reference class unreachable instead of silently tolerated.", "ERR_REQUIREMENT_SUPERSEDED_BY_NOT_RETIRED": "@supersededBy on a requirement whose @status is not `retired`. The attribute names what REPLACED a withdrawn capability; on a live one there is nothing to have replaced it.", "ERR_UNKNOWN_LIBRARY": "FR-043: .metaobjects/config.json's `libraries` names a shipped library or layer this build does not have. Refused with the available tokens rather than skipped — skipped, it resurfaces as ERR_UNRESOLVED_SUPER against the adopter's own metadata.", diff --git a/fixtures/conformance/authored-name-looks-autogenerated/expected.json b/fixtures/conformance/authored-name-looks-autogenerated/expected.json new file mode 100644 index 0000000000..4a058e20e3 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/expected.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "acme::shop", + "children": [ + { + "object.entity": { + "name": "Item", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "sku", + "children": [ + { + "validator.required": { + "name": "required7" + } + }, + { + "validator.length": { + "@max": 10 + } + } + ] + } + }, + { + "identity.primary": { + "name": "pk", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json b/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json new file mode 100644 index 0000000000..4a058e20e3 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "acme::shop", + "children": [ + { + "object.entity": { + "name": "Item", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "sku", + "children": [ + { + "validator.required": { + "name": "required7" + } + }, + { + "validator.length": { + "@max": 10 + } + } + ] + } + }, + { + "identity.primary": { + "name": "pk", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/authored-name-looks-autogenerated/providers.json b/fixtures/conformance/authored-name-looks-autogenerated/providers.json new file mode 100644 index 0000000000..8ec88fc091 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types"] diff --git a/fixtures/conformance/column-name-no-collision/expected.json b/fixtures/conformance/column-name-no-collision/expected.json new file mode 100644 index 0000000000..66921351cb --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/expected.json @@ -0,0 +1,60 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Account", + "children": [ + { + "source.rdb": { + "@kind": "table" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "userId" + } + }, + { + "field.string": { + "name": "userID", + "@column": "user_id_legacy" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": [ + "id" + ] + } + } + ] + } + }, + { + "object.value": { + "name": "Shape", + "children": [ + { + "field.string": { + "name": "userId" + } + }, + { + "field.string": { + "name": "userID" + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/column-name-no-collision/input/meta.demo.json b/fixtures/conformance/column-name-no-collision/input/meta.demo.json new file mode 100644 index 0000000000..ea776f41a2 --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/input/meta.demo.json @@ -0,0 +1,24 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { "object.entity": { + "name": "Account", + "children": [ + { "source.rdb": { "@kind": "table" } }, + { "field.long": { "name": "id" } }, + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID", "@column": "user_id_legacy" } }, + { "identity.primary": { "name": "id", "@fields": ["id"] } } + ] + }}, + { "object.value": { + "name": "Shape", + "children": [ + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID" } } + ] + }} + ] + } +} diff --git a/fixtures/conformance/column-name-no-collision/providers.json b/fixtures/conformance/column-name-no-collision/providers.json new file mode 100644 index 0000000000..aefa027b6a --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types", "metaobjects-db"] diff --git a/fixtures/conformance/enum-normalize-collision-collapse-ok/expected.json b/fixtures/conformance/enum-normalize-collision-collapse-ok/expected.json new file mode 100644 index 0000000000..b2ba036b84 --- /dev/null +++ b/fixtures/conformance/enum-normalize-collision-collapse-ok/expected.json @@ -0,0 +1,38 @@ +{ + "metadata.root": { + "package": "acme", + "children": [ + { + "object.entity": { + "name": "Order", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.enum": { + "name": "status", + "@normalize": "collapse", + "@values": [ + "READ_ONLY", + "READONLY", + "WRITE" + ] + } + }, + { + "identity.primary": { + "name": "id", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/enum-normalize-collision-collapse-ok/input/meta.enums.json b/fixtures/conformance/enum-normalize-collision-collapse-ok/input/meta.enums.json new file mode 100644 index 0000000000..9e4b649918 --- /dev/null +++ b/fixtures/conformance/enum-normalize-collision-collapse-ok/input/meta.enums.json @@ -0,0 +1,36 @@ +{ + "metadata.root": { + "package": "acme", + "children": [ + { + "object.entity": { + "name": "Order", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.enum": { + "name": "status", + "@values": [ + "READ_ONLY", + "READONLY", + "WRITE" + ], + "@normalize": "collapse" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/enum-normalize-no-collision/expected.json b/fixtures/conformance/enum-normalize-no-collision/expected.json new file mode 100644 index 0000000000..c2cae094c4 --- /dev/null +++ b/fixtures/conformance/enum-normalize-no-collision/expected.json @@ -0,0 +1,37 @@ +{ + "metadata.root": { + "package": "acme", + "children": [ + { + "object.entity": { + "name": "Order", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.enum": { + "name": "status", + "@values": [ + "READ_ONLY", + "WRITE", + "ADMIN" + ] + } + }, + { + "identity.primary": { + "name": "id", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/enum-normalize-no-collision/input/meta.enums.json b/fixtures/conformance/enum-normalize-no-collision/input/meta.enums.json new file mode 100644 index 0000000000..dfc355e944 --- /dev/null +++ b/fixtures/conformance/enum-normalize-no-collision/input/meta.enums.json @@ -0,0 +1,35 @@ +{ + "metadata.root": { + "package": "acme", + "children": [ + { + "object.entity": { + "name": "Order", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.enum": { + "name": "status", + "@values": [ + "READ_ONLY", + "WRITE", + "ADMIN" + ] + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/error-column-name-collision/expected-errors.json b/fixtures/conformance/error-column-name-collision/expected-errors.json new file mode 100644 index 0000000000..1ce5056dc8 --- /dev/null +++ b/fixtures/conformance/error-column-name-collision/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_COLUMN_NAME_COLLISION", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[0]['object.entity'].children[3]['field.string']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-column-name-collision/input/meta.demo.json b/fixtures/conformance/error-column-name-collision/input/meta.demo.json new file mode 100644 index 0000000000..e2ed439a7b --- /dev/null +++ b/fixtures/conformance/error-column-name-collision/input/meta.demo.json @@ -0,0 +1,17 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { "object.entity": { + "name": "Account", + "children": [ + { "source.rdb": { "@kind": "table" } }, + { "field.long": { "name": "id" } }, + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID" } }, + { "identity.primary": { "name": "id", "@fields": ["id"] } } + ] + }} + ] + } +} diff --git a/fixtures/conformance/error-enum-normalize-collision/expected-errors.json b/fixtures/conformance/error-enum-normalize-collision/expected-errors.json new file mode 100644 index 0000000000..d6893c55e1 --- /dev/null +++ b/fixtures/conformance/error-enum-normalize-collision/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_ENUM_NORMALIZE_COLLISION", + "source": { + "format": "json", + "files": [ + "meta.enums.json" + ], + "jsonPath": "$['metadata.root'].children[0]['object.entity'].children[1]['field.enum']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-enum-normalize-collision/input/meta.enums.json b/fixtures/conformance/error-enum-normalize-collision/input/meta.enums.json new file mode 100644 index 0000000000..de8c40f86b --- /dev/null +++ b/fixtures/conformance/error-enum-normalize-collision/input/meta.enums.json @@ -0,0 +1,35 @@ +{ + "metadata.root": { + "package": "acme", + "children": [ + { + "object.entity": { + "name": "Order", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.enum": { + "name": "status", + "@values": [ + "READ_ONLY", + "READONLY", + "WRITE" + ] + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json new file mode 100644 index 0000000000..ec9d869b77 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_DISCRIMINATOR_VALUE_DUPLICATE", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[2]['object.entity']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json new file mode 100644 index 0000000000..4f3c5bf927 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json @@ -0,0 +1,52 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Auth", + "@discriminator": "kind", + "children": [ + { + "source.rdb": { + "@table": "auths" + } + }, + { + "field.int": { + "name": "kind" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + }, + { + "object.entity": { + "name": "One", + "extends": "demo::Auth", + "@discriminatorValue": "1", + "children": [] + } + }, + { + "object.entity": { + "name": "AlsoOne", + "extends": "demo::Auth", + "@discriminatorValue": "01", + "children": [] + } + } + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json new file mode 100644 index 0000000000..f9bb435a37 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types","metaobjects-db"] diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json new file mode 100644 index 0000000000..4b916f01a2 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[1]['object.entity']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json new file mode 100644 index 0000000000..2932beb01b --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Auth", + "@discriminator": "kind", + "children": [ + { + "source.rdb": { + "@table": "auths" + } + }, + { + "field.int": { + "name": "kind" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + }, + { + "object.entity": { + "name": "Five", + "extends": "demo::Auth", + "@discriminatorValue": "5\n", + "children": [] + } + } + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json new file mode 100644 index 0000000000..f9bb435a37 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types","metaobjects-db"] diff --git a/fixtures/extract-conformance/README.md b/fixtures/extract-conformance/README.md index 22c5958f97..f460780668 100644 --- a/fixtures/extract-conformance/README.md +++ b/fixtures/extract-conformance/README.md @@ -82,3 +82,10 @@ strict parser does not change: only the tolerant recovery reads past comments. The three `json-array-*-items-*` cases pin `validator.array` bounds: too many elements truncated under NORMAL, too many rejected under STRICT, too few rejected (and listed in `malformedRequired`) under any tolerance. + +The `json-string-escapes` and `json-fence-inner-*` cases pin two reply-parser defects found by +the regex audit. `json-string-escapes` holds every escape of the JSON grammar (`\uXXXX` including +a surrogate pair, `\b`, `\f`, `\/`, `\"`, `\\`) and a digit string above 2^53 that must survive +verbatim. `json-fence-inner-backticks` has triple backticks inside a JSON string in a three-tick +fence, and `json-fence-inner-block` a nested fenced sample inside a four-tick fence: a fence +closes only at a line that starts with a backtick run at least as long as the one that opened it. diff --git a/fixtures/extract-conformance/json-fence-inner-backticks/expected.json b/fixtures/extract-conformance/json-fence-inner-backticks/expected.json new file mode 100644 index 0000000000..67d4742c1e --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-backticks/expected.json @@ -0,0 +1 @@ +{"empty": false, "states": {"summary": "EXTRACTED", "score": "EXTRACTED", "note": "EXTRACTED"}, "data": {"summary": "run ```make``` first", "score": 7, "note": "kept"}} \ No newline at end of file diff --git a/fixtures/extract-conformance/json-fence-inner-backticks/input.txt b/fixtures/extract-conformance/json-fence-inner-backticks/input.txt new file mode 100644 index 0000000000..a7e59da0c3 --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-backticks/input.txt @@ -0,0 +1,5 @@ +Here you go: +```json +{"summary": "run ```make``` first", "score": 7, "note": "kept"} +``` +Hope that helps! diff --git a/fixtures/extract-conformance/json-fence-inner-backticks/schema.json b/fixtures/extract-conformance/json-fence-inner-backticks/schema.json new file mode 100644 index 0000000000..ae1f6a8b51 --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-backticks/schema.json @@ -0,0 +1,4 @@ +{ "format": "JSON", "rootName": "answer", "fields": [ + { "name": "summary", "kind": "STRING", "required": true }, + { "name": "score", "kind": "INT", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } diff --git a/fixtures/extract-conformance/json-fence-inner-block/expected.json b/fixtures/extract-conformance/json-fence-inner-block/expected.json new file mode 100644 index 0000000000..3038a9df5c --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-block/expected.json @@ -0,0 +1 @@ +{"empty": false, "states": {"summary": "EXTRACTED", "score": "EXTRACTED", "note": "EXTRACTED"}, "data": {"summary": "Use:\n```bash\nmake\n```\nthen test", "score": 7, "note": "kept"}} \ No newline at end of file diff --git a/fixtures/extract-conformance/json-fence-inner-block/input.txt b/fixtures/extract-conformance/json-fence-inner-block/input.txt new file mode 100644 index 0000000000..cca5ba4112 --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-block/input.txt @@ -0,0 +1,4 @@ +Here you go: +````json +{"summary": "Use:\n```bash\nmake\n```\nthen test", "score": 7, "note": "kept"} +```` diff --git a/fixtures/extract-conformance/json-fence-inner-block/schema.json b/fixtures/extract-conformance/json-fence-inner-block/schema.json new file mode 100644 index 0000000000..ae1f6a8b51 --- /dev/null +++ b/fixtures/extract-conformance/json-fence-inner-block/schema.json @@ -0,0 +1,4 @@ +{ "format": "JSON", "rootName": "answer", "fields": [ + { "name": "summary", "kind": "STRING", "required": true }, + { "name": "score", "kind": "INT", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } diff --git a/fixtures/extract-conformance/json-string-escapes/expected.json b/fixtures/extract-conformance/json-string-escapes/expected.json new file mode 100644 index 0000000000..2d87282b04 --- /dev/null +++ b/fixtures/extract-conformance/json-string-escapes/expected.json @@ -0,0 +1 @@ +{"empty": false, "states": {"name": "EXTRACTED", "count": "EXTRACTED", "tail": "EXTRACTED"}, "data": {"name": "caf\u00e9 \ud83d\ude00 a/b\bx\fy \"q\" \\ done", "count": "9007199254740993", "tail": "ok"}} \ No newline at end of file diff --git a/fixtures/extract-conformance/json-string-escapes/input.txt b/fixtures/extract-conformance/json-string-escapes/input.txt new file mode 100644 index 0000000000..2b96081f7e --- /dev/null +++ b/fixtures/extract-conformance/json-string-escapes/input.txt @@ -0,0 +1 @@ +{"name":"café 😀 a\/b\bx\fy \"q\" \\ done","count":"9007199254740993","tail":"ok"} \ No newline at end of file diff --git a/fixtures/extract-conformance/json-string-escapes/schema.json b/fixtures/extract-conformance/json-string-escapes/schema.json new file mode 100644 index 0000000000..d7187d8a05 --- /dev/null +++ b/fixtures/extract-conformance/json-string-escapes/schema.json @@ -0,0 +1,4 @@ +{ "format": "JSON", "rootName": "answer", "fields": [ + { "name": "name", "kind": "STRING", "required": true }, + { "name": "count", "kind": "STRING", "required": true }, + { "name": "tail", "kind": "STRING", "required": false } ] } diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/expected.json b/fixtures/extract-conformance/xml-cdata-with-tags/expected.json new file mode 100644 index 0000000000..07fc491b40 --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "x bold & ", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/input.txt b/fixtures/extract-conformance/xml-cdata-with-tags/input.txt new file mode 100644 index 0000000000..a282ddfbdd --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/input.txt @@ -0,0 +1 @@ +<![CDATA[x <b>bold</b> & <!-- not a comment -->]]>n \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/schema.json b/fixtures/extract-conformance/xml-cdata-with-tags/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-comment-ignored/expected.json b/fixtures/extract-conformance/xml-comment-ignored/expected.json new file mode 100644 index 0000000000..4ea7b4495e --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "final" } } diff --git a/fixtures/extract-conformance/xml-comment-ignored/input.txt b/fixtures/extract-conformance/xml-comment-ignored/input.txt new file mode 100644 index 0000000000..74217348ec --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/input.txt @@ -0,0 +1 @@ +realfinal \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-comment-ignored/schema.json b/fixtures/extract-conformance/xml-comment-ignored/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-entities-decoded/expected.json b/fixtures/extract-conformance/xml-entities-decoded/expected.json new file mode 100644 index 0000000000..dff6155d70 --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "A & B AB", "note": "< stays" } } diff --git a/fixtures/extract-conformance/xml-entities-decoded/input.txt b/fixtures/extract-conformance/xml-entities-decoded/input.txt new file mode 100644 index 0000000000..f8e502488f --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/input.txt @@ -0,0 +1 @@ +A & B <ok> AB&lt; stays \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-entities-decoded/schema.json b/fixtures/extract-conformance/xml-entities-decoded/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json b/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json new file mode 100644 index 0000000000..2acfc669fa --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "use List and Map here", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt b/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt new file mode 100644 index 0000000000..08e26f30e7 --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt @@ -0,0 +1 @@ +use List<int> and Map<string, int> heren \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json b/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-in-comment/expected.json b/fixtures/extract-conformance/xml-root-in-comment/expected.json new file mode 100644 index 0000000000..c907092dc6 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-root-in-comment/input.txt b/fixtures/extract-conformance/xml-root-in-comment/input.txt new file mode 100644 index 0000000000..136fd46f24 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/input.txt @@ -0,0 +1,2 @@ + +realn \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-in-comment/schema.json b/fixtures/extract-conformance/xml-root-in-comment/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json b/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json new file mode 100644 index 0000000000..c907092dc6 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt b/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt new file mode 100644 index 0000000000..1b5224f227 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt @@ -0,0 +1,2 @@ +Wrap the answer in a element like so: +realn \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json b/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/expected.json b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/expected.json new file mode 100644 index 0000000000..86a3b9b7da --- /dev/null +++ b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/expected.json @@ -0,0 +1,3 @@ +{ "empty": false, + "states": { "check": "EXTRACTED", "check.id": "EXTRACTED", "check.resolved": "EXTRACTED", "check.payoff": "EXTRACTED", "note": "EXTRACTED" }, + "data": { "check.id": "A", "check.resolved": "yes", "check.payoff": "The gate swings wide.", "note": "use List here" } } diff --git a/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/input.txt b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/input.txt new file mode 100644 index 0000000000..01eb397c5e --- /dev/null +++ b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/input.txt @@ -0,0 +1 @@ +The gate swings wide.use List here \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/schema.json b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/schema.json new file mode 100644 index 0000000000..89a500e8dc --- /dev/null +++ b/fixtures/extract-conformance/xml-unclosed-leaf-in-closed-parent/schema.json @@ -0,0 +1,6 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "check", "kind": "OBJECT", "required": true, "fields": [ + { "name": "id", "kind": "STRING", "required": true }, + { "name": "resolved", "kind": "STRING", "required": true }, + { "name": "payoff", "kind": "STRING", "required": true } ] }, + { "name": "note", "kind": "STRING", "required": false } ] } diff --git a/fixtures/naming-conformance/already-plural-pluralize.json b/fixtures/naming-conformance/already-plural-pluralize.json index 396d37439a..e52a9a434d 100644 --- a/fixtures/naming-conformance/already-plural-pluralize.json +++ b/fixtures/naming-conformance/already-plural-pluralize.json @@ -63,6 +63,12 @@ "entityB": "Orders", "collidesOn": "Orders", "note": "same shape as Address/Addresses — Order legacy-pluralizes to Orders; Orders is already-plural and stays Orders" + }, + { + "entityA": "ApiKey", + "entityB": "APIKey", + "collidesOn": "api_keys", + "note": "route-segment twins — the pluralized PascalCase names differ (ApiKeys / APIKeys) but both snake_case to api_key and serve /api_keys" } ] } diff --git a/fixtures/render-conformance/render-comment-with-partial/expected.txt b/fixtures/render-conformance/render-comment-with-partial/expected.txt new file mode 100644 index 0000000000..abfd547468 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/expected.txt @@ -0,0 +1 @@ +Hi Ada }} \ No newline at end of file diff --git a/fixtures/render-conformance/render-comment-with-partial/meta.json b/fixtures/render-conformance/render-comment-with-partial/meta.json new file mode 100644 index 0000000000..740389bd32 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/meta.json @@ -0,0 +1 @@ +{ "format": "text", "note": "a comment ends at the first closing delimiter, so a partial tag written inside one is never a partial (no unresolved-partial error)" } diff --git a/fixtures/render-conformance/render-comment-with-partial/payload.json b/fixtures/render-conformance/render-comment-with-partial/payload.json new file mode 100644 index 0000000000..8f842fdc28 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/payload.json @@ -0,0 +1 @@ +{ "name": "Ada" } diff --git a/fixtures/render-conformance/render-comment-with-partial/template.mustache b/fixtures/render-conformance/render-comment-with-partial/template.mustache new file mode 100644 index 0000000000..e1a2d1a2d0 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/template.mustache @@ -0,0 +1 @@ +Hi {{name}}{{! {{> missing}} }} \ No newline at end of file diff --git a/fixtures/render-conformance/render-csv-numeric-exempt/expected.txt b/fixtures/render-conformance/render-csv-numeric-exempt/expected.txt new file mode 100644 index 0000000000..3b75a745bc --- /dev/null +++ b/fixtures/render-conformance/render-csv-numeric-exempt/expected.txt @@ -0,0 +1,6 @@ +id,value +1,-5 +2,-5.25 +3,'=1+1 +4,'-cmd +5,'@x diff --git a/fixtures/render-conformance/render-csv-numeric-exempt/meta.json b/fixtures/render-conformance/render-csv-numeric-exempt/meta.json new file mode 100644 index 0000000000..8d25c4bbca --- /dev/null +++ b/fixtures/render-conformance/render-csv-numeric-exempt/meta.json @@ -0,0 +1 @@ +{ "format": "csv", "note": "CSV formula guard exempts strict numeric literals (-5, -5.25); other leading = + - @ values stay guarded" } diff --git a/fixtures/render-conformance/render-csv-numeric-exempt/payload.json b/fixtures/render-conformance/render-csv-numeric-exempt/payload.json new file mode 100644 index 0000000000..ad01c81a71 --- /dev/null +++ b/fixtures/render-conformance/render-csv-numeric-exempt/payload.json @@ -0,0 +1,24 @@ +{ + "rows": [ + { + "id": "1", + "v": "-5" + }, + { + "id": "2", + "v": "-5.25" + }, + { + "id": "3", + "v": "=1+1" + }, + { + "id": "4", + "v": "-cmd" + }, + { + "id": "5", + "v": "@x" + } + ] +} \ No newline at end of file diff --git a/fixtures/render-conformance/render-csv-numeric-exempt/template.mustache b/fixtures/render-conformance/render-csv-numeric-exempt/template.mustache new file mode 100644 index 0000000000..3686e66181 --- /dev/null +++ b/fixtures/render-conformance/render-csv-numeric-exempt/template.mustache @@ -0,0 +1,3 @@ +id,value +{{#rows}}{{id}},{{v}} +{{/rows}} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json @@ -0,0 +1 @@ +[] diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json new file mode 100644 index 0000000000..d2d8a2fbe6 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json @@ -0,0 +1 @@ +{ "provider": "with" } diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache b/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache new file mode 100644 index 0000000000..e1a2d1a2d0 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache @@ -0,0 +1 @@ +Hi {{name}}{{! {{> missing}} }} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json b/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json new file mode 100644 index 0000000000..11cb5ec64f --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "name" } ] diff --git a/fixtures/verify-conformance/verify-mismatched-close/payload.json b/fixtures/verify-conformance/verify-mismatched-close/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-mismatched-close/template.mustache b/fixtures/verify-conformance/verify-mismatched-close/template.mustache new file mode 100644 index 0000000000..2fbef13473 --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/template.mustache @@ -0,0 +1 @@ +{{#name}}x{{/zzz}} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json b/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json new file mode 100644 index 0000000000..10adfc23b8 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_OUTPUT_TAG_MISSING", "path": "answer" } ] diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/options.json b/fixtures/verify-conformance/verify-required-tag-in-comment/options.json new file mode 100644 index 0000000000..4f863d00b1 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/options.json @@ -0,0 +1 @@ +{ "requiredTags": ["answer"] } diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json b/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache b/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache new file mode 100644 index 0000000000..fc54efbef8 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache @@ -0,0 +1 @@ +{{name}}{{! }} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json @@ -0,0 +1 @@ +[] diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json new file mode 100644 index 0000000000..751ee879de --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "<% %>" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json b/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache new file mode 100644 index 0000000000..205b12f067 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache @@ -0,0 +1 @@ +{{=<% %>=}}<% name %> {{literal}} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json new file mode 100644 index 0000000000..e9119e2212 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_VAR_NOT_ON_PAYLOAD", "path": "nope" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json new file mode 100644 index 0000000000..8782129305 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json @@ -0,0 +1 @@ +[ { "code": "ERR_VAR_NOT_ON_PAYLOAD", "path": "nope" }, { "code": "ERR_TEMPLATE_SYNTAX", "path": "<% %>" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json b/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache new file mode 100644 index 0000000000..8373c1983c --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache @@ -0,0 +1 @@ +{{=<% %>=}}<% name %> <% nope %> \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json new file mode 100644 index 0000000000..9787342aa4 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "partials/leaf" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache b/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache new file mode 100644 index 0000000000..27c658ae77 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache @@ -0,0 +1 @@ +{{=<% %>=}}hi diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json b/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache new file mode 100644 index 0000000000..cd616c89f3 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache @@ -0,0 +1 @@ +{{> partials/leaf}} {{name}} diff --git a/fixtures/verify-conformance/verify-set-delimiter-with-partial/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-with-partial/expected-drift.json new file mode 100644 index 0000000000..793222a09b --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-with-partial/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "partials/greet" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-with-partial/partials/greet.mustache b/fixtures/verify-conformance/verify-set-delimiter-with-partial/partials/greet.mustache new file mode 100644 index 0000000000..7e66132938 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-with-partial/partials/greet.mustache @@ -0,0 +1 @@ +{{name}} diff --git a/fixtures/verify-conformance/verify-set-delimiter-with-partial/payload.json b/fixtures/verify-conformance/verify-set-delimiter-with-partial/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-with-partial/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-with-partial/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-with-partial/template.mustache new file mode 100644 index 0000000000..c2997f79a0 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-with-partial/template.mustache @@ -0,0 +1 @@ +{{=<% %>=}}<% name %> <%> partials/greet %> diff --git a/fixtures/verify-conformance/verify-unopened-close/expected-drift.json b/fixtures/verify-conformance/verify-unopened-close/expected-drift.json new file mode 100644 index 0000000000..1e9ea3a114 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "zzz" } ] diff --git a/fixtures/verify-conformance/verify-unopened-close/payload.json b/fixtures/verify-conformance/verify-unopened-close/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-unopened-close/template.mustache b/fixtures/verify-conformance/verify-unopened-close/template.mustache new file mode 100644 index 0000000000..4e67e9feb5 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/template.mustache @@ -0,0 +1 @@ +x{{/zzz}} \ No newline at end of file diff --git a/scripts/leak-scan.test.ts b/scripts/leak-scan.test.ts new file mode 100644 index 0000000000..d0d19225a5 --- /dev/null +++ b/scripts/leak-scan.test.ts @@ -0,0 +1,88 @@ +// The public-repo leak scan, exercised against a throwaway repository. +// +// `.githooks/leak-scan.sh` is the hosted `leak-scan` status and `.githooks/pre-commit` is the +// local hook; both read `.githooks/leak-patterns.sh`. Two defects (audit finding 12) let a +// leak through: `--diff-filter=ACM` skipped renamed-with-edits files, and the hosted scan had +// no `/Users//` pattern while the local hook did. +import { afterAll, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const REPO = resolve(import.meta.dirname, ".."); +const tmp = mkdtempSync(join(tmpdir(), "leak-scan-")); +afterAll(() => rmSync(tmp, { recursive: true, force: true })); + +function git(cwd: string, ...args: string[]): void { + const r = spawnSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd }); + if (r.status !== 0) throw new Error(`git ${args.join(" ")}: ${r.stderr}`); +} + +function fixture(): string { + const dir = mkdtempSync(join(tmp, "repo-")); + git(dir, "init", "-q", "-b", "main"); + mkdirSync(join(dir, ".githooks")); + for (const f of ["leak-scan.sh", "leak-patterns.sh", "pre-commit"]) { + cpSync(join(REPO, ".githooks", f), join(dir, ".githooks", f)); + } + // Long enough that an edit stays above git's rename-similarity threshold. + writeFileSync(join(dir, "a.md"), Array.from({ length: 20 }, (_, i) => `line ${i}`).join("\n") + "\n"); + git(dir, "add", "-A"); + git(dir, "commit", "-q", "-m", "base"); + return dir; +} + +function scan(dir: string, script = "leak-scan.sh") { + const r = spawnSync("bash", [join(dir, ".githooks", script)], { cwd: dir, encoding: "utf8" }); + return { status: r.status, out: `${r.stdout}${r.stderr}` }; +} + +// Built from parts so this test file never trips the scan itself. +const mac = "/" + "Users" + "/someone/project"; +const linux = "/" + "home" + "/someone/project"; + +describe("leak-scan", () => { + test("clean staged change passes", () => { + const dir = fixture(); + writeFileSync(join(dir, "a.md"), "line 0\nnothing here\n"); + git(dir, "add", "-A"); + expect(scan(dir).status).toBe(0); + }); + + test("an added /home path is blocked (existing behaviour)", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${linux}\n`); + git(dir, "add", "-A"); + expect(scan(dir).status).toBe(1); + }); + + test("an added macOS home path is blocked (previously only the local hook caught it)", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${mac}\n`); + git(dir, "add", "-A"); + const r = scan(dir); + expect(r.status).toBe(1); + expect(r.out).toContain("someone"); + }); + + test("a leak added while RENAMING a file is blocked", () => { + const dir = fixture(); + git(dir, "mv", "a.md", "b.md"); + writeFileSync( + join(dir, "b.md"), + Array.from({ length: 20 }, (_, i) => `line ${i}`).join("\n") + `\nsee ${linux}\n`, + ); + git(dir, "add", "-A"); + const names = spawnSync("git", ["diff", "--cached", "--name-status"], { cwd: dir, encoding: "utf8" }).stdout; + expect(names).toMatch(/^R/); + expect(scan(dir).status).toBe(1); + }); + + test("the local pre-commit hook agrees with the hosted scan", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${mac}\n`); + git(dir, "add", "-A"); + expect(scan(dir, "pre-commit").status).toBe(1); + }); +}); diff --git a/server/csharp/MetaObjects.Cli.Tests/ApiDocsNamespaceTests.cs b/server/csharp/MetaObjects.Cli.Tests/ApiDocsNamespaceTests.cs index e2dc560e3e..d5686c150e 100644 --- a/server/csharp/MetaObjects.Cli.Tests/ApiDocsNamespaceTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/ApiDocsNamespaceTests.cs @@ -56,8 +56,8 @@ private CodegenCli.GenOutcome RunDocs(string ns) => [Fact] public void Code_blocks_are_using_directives_never_the_bare_namespace_placeholder() { - // Default namespace = gen's default. - var outcome = RunDocs(GenCommand.DefaultNamespace); + // Any namespace: the docs must import it, never name it bare. + var outcome = RunDocs("Generated"); Assert.True(outcome.Ok, string.Join("; ", outcome.LoadErrors)); var pages = Directory.EnumerateFiles(OutDir, "*.md", SearchOption.AllDirectories) @@ -87,7 +87,7 @@ public void Code_blocks_are_using_directives_never_the_bare_namespace_placeholde [Fact] public void Documented_namespace_defaults_to_the_gen_default_and_appears_in_blocks() { - var outcome = RunDocs(GenCommand.DefaultNamespace); + var outcome = RunDocs("Generated"); Assert.True(outcome.Ok); // acme::shop binds to the default namespace + package → ".Acme.Shop"-style. @@ -96,7 +96,7 @@ public void Documented_namespace_defaults_to_the_gen_default_and_appears_in_bloc var orderPage = Directory.EnumerateFiles(OutDir, "Order.md", SearchOption.AllDirectories).Single(); var text = File.ReadAllText(orderPage); Assert.Contains("```csharp\nusing ", text); - Assert.Contains(GenCommand.DefaultNamespace, text); + Assert.Contains("Generated", text); } [Fact] diff --git a/server/csharp/MetaObjects.Cli.Tests/CliSurfaceTests.cs b/server/csharp/MetaObjects.Cli.Tests/CliSurfaceTests.cs index 68892521e2..4ad0315e11 100644 --- a/server/csharp/MetaObjects.Cli.Tests/CliSurfaceTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/CliSurfaceTests.cs @@ -35,7 +35,7 @@ public void Help_is_output_on_stdout_and_exits_0() var (exit, stdout, _) = CliProcess.Run(_tmp, "--help"); Assert.Equal(0, exit); Assert.Contains("usage: dotnet meta ", stdout); - // --namespace defaults (GenCommand.DefaultNamespace); the banner must not show it as required. + // --namespace has a default (config, else the metadata package); the banner must not show it as required. Assert.Contains("--out [--namespace ]", stdout); } diff --git a/server/csharp/MetaObjects.Cli.Tests/CodegenNamespaceTests.cs b/server/csharp/MetaObjects.Cli.Tests/CodegenNamespaceTests.cs new file mode 100644 index 0000000000..f1b2892977 --- /dev/null +++ b/server/csharp/MetaObjects.Cli.Tests/CodegenNamespaceTests.cs @@ -0,0 +1,119 @@ +using Xunit; + +namespace MetaObjects.Cli.Tests; + +/// +/// The flat namespace gen and verify --codegen generate under: --namespace, else +/// csharp.namespace in .metaobjects/config.json; with neither, only a run that emits +/// a file needing one stops. Driven through the entry point a project's +/// codegen/Program.cs calls (); the oracle is the +/// namespace ...; declaration in the generated file, the output contract an adopter's code +/// compiles against. +/// +[Collection(OwnedRunnerCollection.Name)] +public sealed class CodegenNamespaceTests : IDisposable +{ + private readonly string _tmp = Path.Combine(Path.GetTempPath(), "meta-cli-ns-" + Guid.NewGuid().ToString("N")); + + private const string Metadata = """ + { "metadata.root": { "package": "acme::shop", "children": [ + { "object.entity": { "name": "Subscriber", "children": [ + { "source.rdb": { "@table": "subscribers" } }, + { "field.long": { "name": "id" } }, + { "identity.primary": { "@fields": "id" } } + ]}} + ]}} + """; + + private string MetaDir => Path.Combine(_tmp, "metaobjects"); + private string OutDir => Path.Combine(_tmp, "Generated"); + + public CodegenNamespaceTests() + { + Directory.CreateDirectory(MetaDir); + File.WriteAllText(Path.Combine(MetaDir, "meta.shop.json"), Metadata); + } + + public void Dispose() { try { Directory.Delete(_tmp, recursive: true); } catch { } } + + private void WriteConfig(string csharpBlock) + { + var dir = Path.Combine(_tmp, ".metaobjects"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "config.json"), $$"""{ "schema_version": 1, "csharp": { {{csharpBlock}} } }"""); + } + + private (int Exit, string Output) Run(params string[] args) + { + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity"], args); + return (exit, stdout + stderr); + } + + private string EntityFile() => File.ReadAllText(Path.Combine(OutDir, "Subscriber.g.cs")); + + [Fact] + public void A_config_namespace_is_used_by_gen_and_a_bare_verify_is_clean() + { + WriteConfig("\"outDir\": \"Generated\", \"namespace\": \"Acme.Blog\""); + + var gen = Run("gen"); + Assert.True(gen.Exit == 0, gen.Output); + Assert.Contains("namespace Acme.Blog;", EntityFile()); + + var verify = Run("verify", "--codegen"); + Assert.True(verify.Exit == 0, verify.Output); + } + + [Fact] + public void With_no_namespace_a_run_that_emits_code_stops_and_asks_for_one() + { + WriteConfig("\"outDir\": \"Generated\""); + + var gen = Run("gen"); + Assert.NotEqual(0, gen.Exit); + Assert.Contains("\"csharp\": { \"namespace\": \"\" }", gen.Output); + Assert.False(File.Exists(Path.Combine(OutDir, "Subscriber.g.cs"))); + + var verify = Run("verify", "--codegen"); + Assert.Equal(2, verify.Exit); + Assert.Contains("\"csharp\": { \"namespace\": \"\" }", verify.Output); + } + + [Fact] + public void With_no_namespace_a_project_with_nothing_to_emit_generates_cleanly() + { + WriteConfig("\"outDir\": \"Generated\""); + File.WriteAllText(Path.Combine(MetaDir, "meta.shop.json"), + """{ "metadata.root": { "package": "acme::shop", "children": [] } }"""); + + var gen = Run("gen"); + Assert.True(gen.Exit == 0, gen.Output); + var verify = Run("verify", "--codegen"); + Assert.True(verify.Exit == 0, verify.Output); + } + + [Fact] + public void The_flag_wins_over_config_and_output_under_another_namespace_is_drift() + { + WriteConfig("\"outDir\": \"Generated\", \"namespace\": \"Acme.Blog\""); + + Assert.Equal(0, Run("gen", "--namespace", "Other.Ns").Exit); + Assert.Contains("namespace Other.Ns;", EntityFile()); + Assert.Equal(1, Run("verify", "--codegen").Exit); + } + + [Theory] + [InlineData("My-app.Shop")] + [InlineData("Acme.2fa")] + [InlineData("Acme..Shop")] + [InlineData("Acme.public")] + public void A_namespace_that_is_not_a_dotted_identifier_path_fails_generation(string ns) + { + WriteConfig($"\"outDir\": \"Generated\", \"namespace\": \"{ns}\""); + + var gen = Run("gen"); + Assert.NotEqual(0, gen.Exit); + Assert.Contains($"\"{ns}\" is not a C# namespace", gen.Output); + Assert.False(File.Exists(Path.Combine(OutDir, "Subscriber.g.cs"))); + } +} diff --git a/server/csharp/MetaObjects.Cli.Tests/ColumnNamingFlagTests.cs b/server/csharp/MetaObjects.Cli.Tests/ColumnNamingFlagTests.cs index cf5793a865..9328411512 100644 --- a/server/csharp/MetaObjects.Cli.Tests/ColumnNamingFlagTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/ColumnNamingFlagTests.cs @@ -126,7 +126,7 @@ public void The_removed_flag_is_refused_on_gen_naming_the_value_given() { var outDir = Path.Combine(_tmp, "generated-bad"); var (exit, stdout, stderr) = OwnedRunner.Run( - _tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--column-naming", "snake_case"); + _tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--column-naming", "snake_case", "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(2, exit); Assert.Contains("--column-naming was removed in 1.1.0", output); @@ -140,10 +140,10 @@ public void The_removed_flag_is_refused_on_gen_naming_the_value_given() public void The_removed_flag_is_refused_on_verify_codegen() { var outDir = Path.Combine(_tmp, "generated-ok"); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); var (exit, stdout, stderr) = OwnedRunner.Run( - _tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--column-naming", "literal"); + _tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--column-naming", "literal", "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(2, exit); Assert.Contains("--column-naming was removed in 1.1.0", output); @@ -159,10 +159,10 @@ public void Verify_with_the_wrong_naming_says_the_naming_differs_instead_of_rege // wrong column names against the live database). var outDir = Path.Combine(_tmp, "generated-snake"); WriteConfig("\"columns\": \"snake_case\""); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); WriteConfig("\"columns\": \"literal\""); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(1, exit); Assert.Contains( @@ -184,11 +184,11 @@ public void Verify_with_another_table_rule_says_the_naming_differs_instead_of_re // a table the database does not have, so verify names the table rule that reproduces it. var outDir = Path.Combine(_tmp, "generated-literal-tables"); WriteConfig("\"tables\": \"literal\""); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); Assert.Contains("\"OrderItems\"", File.ReadAllText(Path.Combine(outDir, "OrderItemNames.g.cs"))); WriteConfig(null); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(1, exit); Assert.Contains( @@ -208,11 +208,11 @@ public void Verify_finds_another_table_rule_when_only_the_entity_generator_runs( // through the loaded tree (which caches it). The probe must still see the other rule. var outDir = Path.Combine(_tmp, "generated-entity-only"); WriteConfig("\"tables\": \"literal\""); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); Assert.Contains("[Table(\"OrderItems\")]", File.ReadAllText(Path.Combine(outDir, "OrderItem.g.cs"))); WriteConfig(null); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity"], "verify", "--codegen", MetaDir, "--out", outDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity"], "verify", "--codegen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(1, exit); Assert.Contains( @@ -228,10 +228,10 @@ public void Verify_finds_a_column_and_table_rule_that_differ_together() // No single rule change reproduces the committed tree; the two together do. var outDir = Path.Combine(_tmp, "generated-literal-both"); WriteConfig("\"columns\": \"literal\", \"tables\": \"literal\""); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); WriteConfig(null); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(1, exit); Assert.Contains( @@ -245,10 +245,10 @@ public void Real_drift_still_says_regenerate() { var outDir = Path.Combine(_tmp, "generated-edited"); WriteConfig("\"columns\": \"snake_case\""); - Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, ["entity", "names"], "gen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated").ExitCode); File.AppendAllText(Path.Combine(outDir, "SubscriberNames.g.cs"), "// edited\n"); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity", "names"], "verify", "--codegen", MetaDir, "--out", outDir, "--namespace", "Acme.Generated"); var output = stdout + stderr; Assert.Equal(1, exit); Assert.Contains("Run 'dotnet meta gen' to regenerate", output); diff --git a/server/csharp/MetaObjects.Cli.Tests/EjectUndoTests.cs b/server/csharp/MetaObjects.Cli.Tests/EjectUndoTests.cs index 791b5344c4..5f69449ed0 100644 --- a/server/csharp/MetaObjects.Cli.Tests/EjectUndoTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/EjectUndoTests.cs @@ -35,6 +35,26 @@ public void With_no_names_removes_every_identical_copy_and_keeps_the_edited_one( Assert.DoesNotContain(result.Out, l => l.Contains("removed codegen/generators/EntityGenerator.cs")); } + [Fact] + public void A_copy_whose_only_edit_is_a_reordering_is_customised_and_needs_force() + { + Eject("filter-allowlist"); + var path = Owned("codegen/generators/FilterAllowlistGenerator.cs"); + var lines = File.ReadAllLines(path).ToList(); + // Swap two adjacent, different, non-blank lines: the same lines in another order. + var i = Enumerable.Range(0, lines.Count - 1).First(k => + lines[k].Trim().Length > 0 && lines[k + 1].Trim().Length > 0 && lines[k].Trim() != lines[k + 1].Trim()); + (lines[i], lines[i + 1]) = (lines[i + 1], lines[i]); + File.WriteAllLines(path, lines); + + Assert.Equal(0, EjectCommand.Undo([], _root, force: false).ExitCode); + Assert.True(File.Exists(path)); + Assert.Equal(2, EjectCommand.Undo(["filter-allowlist"], _root, force: false).ExitCode); + Assert.True(File.Exists(path)); + Assert.Equal(0, EjectCommand.Undo(["filter-allowlist"], _root, force: true).ExitCode); + Assert.False(File.Exists(path)); + } + [Fact] public void It_prints_the_edit_that_points_the_entry_back_and_never_edits_Generators_cs() { diff --git a/server/csharp/MetaObjects.Cli.Tests/LibrariesConfigTests.cs b/server/csharp/MetaObjects.Cli.Tests/LibrariesConfigTests.cs index 0e0546f6a8..1474cd5756 100644 --- a/server/csharp/MetaObjects.Cli.Tests/LibrariesConfigTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/LibrariesConfigTests.cs @@ -123,7 +123,7 @@ public void The_api_reference_with_an_explicit_positional_metadataDir_also_resol var outDir = Path.Combine(_tmp, "apidocs"); var (exitCode, stdout, stderr) = OwnedRunner.Run( _tmp, [new MetaObjects.Codegen.Generators.ApiDocsGenerator { OutDir = outDir }], - "gen", modelDir, "--out", Path.Combine(_tmp, "generated")); + "gen", modelDir, "--out", Path.Combine(_tmp, "generated"), "--namespace", "Acme.Generated"); Assert.True(exitCode == 0, $"exit={exitCode}\nstdout={stdout}\nstderr={stderr}"); Assert.NotEmpty(Directory.GetFiles(outDir, "AgentCall.md", SearchOption.AllDirectories)); diff --git a/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs b/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs index 9d55499853..22666e80e2 100644 --- a/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs @@ -62,7 +62,7 @@ public void A_list_that_declares_nothing_writes_zero_files_and_says_so() { var before = ProjectFiles(); - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, Array.Empty(), "gen", MetaDir, "--out", OutDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, Array.Empty(), "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); Assert.Contains("dotnet meta gen: codegen/Generators.cs declares no generators; nothing written", stdout); @@ -76,7 +76,7 @@ public void A_list_that_declares_nothing_writes_zero_files_and_says_so() public void One_listed_docs_generator_writes_only_its_pages_in_its_own_directory() { var (exit, stdout, stderr) = OwnedRunner.Run( - _tmp, [new ApiDocsGenerator { OutDir = DocsDir, ProjectName = "acme" }], "gen", MetaDir, "--out", OutDir); + _tmp, [new ApiDocsGenerator { OutDir = DocsDir, ProjectName = "acme" }], "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); var pages = FilesUnder(DocsDir); @@ -90,7 +90,7 @@ public void One_listed_docs_generator_writes_only_its_pages_in_its_own_directory [Fact] public void One_listed_code_generator_writes_only_its_own_files() { - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); Assert.Equal(["Subscriber.g.cs"], FilesUnder(OutDir)); @@ -103,7 +103,7 @@ public void One_listed_code_generator_writes_only_its_own_files() public void A_selection_naming_an_undeclared_generator_is_refused_and_writes_nothing() { var (exit, _, stderr) = OwnedRunner.Run( - _tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--generators", "entity,names"); + _tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--generators", "entity,names", "--namespace", "Acme.Generated"); Assert.Equal(2, exit); Assert.Contains("--generators names \"names\", which codegen/Generators.cs does not declare", stderr); @@ -115,7 +115,7 @@ public void A_selection_runs_only_the_declared_entries_it_names() { var (exit, stdout, stderr) = OwnedRunner.Run( _tmp, [new EntityGenerator(), new NamesGenerator(), new ApiDocsGenerator { OutDir = DocsDir }], - "gen", MetaDir, "--out", OutDir, "--generators", "names"); + "gen", MetaDir, "--out", OutDir, "--generators", "names", "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); Assert.Equal(["SubscriberNames.g.cs"], FilesUnder(OutDir)); @@ -126,16 +126,16 @@ public void A_selection_runs_only_the_declared_entries_it_names() public void A_selection_writes_its_entries_as_the_whole_list_does() { IReadOnlyList declared = OwnedRunner.Declared(_tmp, "entity", "names"); - Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); var entity = Path.Combine(OutDir, "Subscriber.g.cs"); var whole = File.ReadAllText(entity); Assert.Contains("SubscriberNames.", whole); - var narrowed = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--generators", "entity"); + var narrowed = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--generators", "entity", "--namespace", "Acme.Generated"); Assert.True(narrowed.ExitCode == 0, $"exit={narrowed.ExitCode}\nstdout={narrowed.Stdout}\nstderr={narrowed.Stderr}"); Assert.Equal(whole, File.ReadAllText(entity)); - var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "entity"); + var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "entity", "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); } @@ -203,7 +203,7 @@ public void Gen_given_1_0_docs_flags_prints_the_amendment_line_and_writes_nothin public void The_codegen_project_refuses_1_0_docs_flags_the_same_way() { var (exit, _, stderr) = OwnedRunner.Run( - _tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--project", "p", "--model-base-url", "https://x.example/model"); + _tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--project", "p", "--model-base-url", "https://x.example/model", "--namespace", "Acme.Generated"); Assert.Equal(2, exit); Assert.Equal(LegacyLine, stderr.Trim()); @@ -300,8 +300,6 @@ public void Eject_into_an_empty_project_lists_the_ejected_generator() Assert.Equal(CodegenScaffold.ProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); } - // ---- a 1.0 Program.cs, which never runs Generators.All, is refused -------------------------- - private const string LegacyProgramCs = "using MetaObjects.Codegen;\n" + "IReadOnlyList generators = [new MetaObjects.Codegen.Generators.EntityGenerator()];\n" + @@ -317,67 +315,6 @@ private void WriteLegacyCodegenProject(bool withGeneratorList) File.WriteAllText(Path.Combine(_tmp, "codegen", "Generators.cs"), CodegenScaffold.GeneratorsCs([])); } - private static string LegacyProgramLine(string command) => - $"{command}: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in " + - "codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and " + - "`return CodegenCli.Run(args, Generators.All);`. See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\"."; - - [Fact] - public void Gen_with_a_1_0_program_refuses_with_the_migration_line_and_writes_nothing() - { - WriteLegacyCodegenProject(withGeneratorList: true); - - var (exit, _, stderr) = CliProcess.Run(_tmp, "gen", MetaDir, "--out", OutDir); - - Assert.Equal(2, exit); - Assert.Equal(LegacyProgramLine("dotnet meta gen"), stderr.Trim()); - Assert.Equal(CodegenCli.LegacyProgram("dotnet meta gen"), stderr.Trim()); - Assert.Empty(FilesUnder(OutDir)); - } - - [Fact] - public void Verify_codegen_with_a_1_0_program_refuses_with_the_migration_line() - { - WriteLegacyCodegenProject(withGeneratorList: true); - - var (exit, _, stderr) = CliProcess.Run(_tmp, "verify", MetaDir, "--codegen", "--out", OutDir); - - Assert.Equal(2, exit); - Assert.Contains(LegacyProgramLine("dotnet meta verify --codegen"), stderr); - } - - [Fact] - public void Init_beside_a_1_0_program_says_to_replace_it_instead_of_claiming_gen_runs_the_list() - { - WriteLegacyCodegenProject(withGeneratorList: false); - - var lines = InitCommand.Run(_tmp); - - Assert.Contains(" wrote codegen/Generators.cs", lines); - Assert.Contains(" kept codegen/Program.cs (already yours)", lines); - Assert.DoesNotContain(lines, l => l.Contains("is the generator list `dotnet meta gen` runs", StringComparison.Ordinal)); - Assert.Contains(lines, l => l.StartsWith("codegen/Program.cs does not run Generators.All", StringComparison.Ordinal)); - Assert.Contains(" return CodegenCli.Run(args, Generators.All);", lines); - Assert.Contains("See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\".", lines); - Assert.Equal(LegacyProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); - } - - [Fact] - public void Eject_beside_a_1_0_program_says_to_replace_it_instead_of_claiming_gen_runs_the_list() - { - WriteLegacyCodegenProject(withGeneratorList: false); - - var outcome = EjectCommand.Run(["entity"], _tmp, force: false); - - Assert.Equal(0, outcome.ExitCode); - Assert.Contains(outcome.Out, l => l.StartsWith("codegen/Generators.cs lists the generator(s) you just ejected", StringComparison.Ordinal)); - Assert.DoesNotContain(outcome.Out, l => l.Contains("run that list", StringComparison.Ordinal)); - Assert.Contains(outcome.Out, l => l.StartsWith("codegen/Program.cs does not run Generators.All", StringComparison.Ordinal)); - Assert.Contains(" return CodegenCli.Run(args, Generators.All);", outcome.Out); - Assert.Contains("See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\".", outcome.Out); - Assert.Equal(LegacyProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); - } - [Fact] public void Init_in_a_project_pinned_to_an_older_package_warns_and_names_the_line_to_change() { @@ -419,17 +356,17 @@ public void A_requirement_tests_entry_with_its_own_directory_is_written_and_chec IReadOnlyList declared = [new EntityGenerator(), new RequirementTestsGenerator { OutDir = testsDir, WarnUncovered = false }]; - var gen = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir); + var gen = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(gen.ExitCode == 0, $"exit={gen.ExitCode}\nstdout={gen.Stdout}\nstderr={gen.Stderr}"); Assert.Equal(["Subscriber.g.cs"], FilesUnder(OutDir)); Assert.Equal(["Requirements_acme_Tests.g.cs", "Requirements_acme_Witnesses.g.cs"], FilesUnder(testsDir)); - var clean = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir); + var clean = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(clean.ExitCode == 0, $"exit={clean.ExitCode}\nstdout={clean.Stdout}\nstderr={clean.Stderr}"); File.AppendAllText(Path.Combine(testsDir, "Requirements_acme_Tests.g.cs"), "// edited\n"); - var drift = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir); + var drift = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.Equal(1, drift.ExitCode); Assert.Contains("Requirements_acme_Tests.g.cs", drift.Stderr); } @@ -444,31 +381,31 @@ public void A_configured_out_dir_is_where_gen_writes_and_verify_checks_and_out_o """{ "schema_version": 1, "csharp": { "outDir": "generated" } }"""); IReadOnlyList declared = [new EntityGenerator()]; - var gen = OwnedRunner.Run(_tmp, declared, "gen"); + var gen = OwnedRunner.Run(_tmp, declared, "gen", "--namespace", "Acme.Generated"); Assert.True(gen.ExitCode == 0, $"exit={gen.ExitCode}\nstdout={gen.Stdout}\nstderr={gen.Stderr}"); Assert.Equal(["Subscriber.g.cs"], FilesUnder(OutDir)); - var clean = OwnedRunner.Run(_tmp, declared, "verify", "--codegen"); + var clean = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", "--namespace", "Acme.Generated"); Assert.True(clean.ExitCode == 0, $"exit={clean.ExitCode}\nstdout={clean.Stdout}\nstderr={clean.Stderr}"); File.AppendAllText(Path.Combine(OutDir, "Subscriber.g.cs"), "// edited\n"); - var drift = OwnedRunner.Run(_tmp, declared, "verify", "--codegen"); + var drift = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", "--namespace", "Acme.Generated"); Assert.Equal(1, drift.ExitCode); Assert.Contains("Subscriber.g.cs", drift.Stderr); // --out beats the config: gen writes there and leaves the configured directory alone. var other = Path.Combine(_tmp, "elsewhere"); - var genOther = OwnedRunner.Run(_tmp, declared, "gen", "--out", other); + var genOther = OwnedRunner.Run(_tmp, declared, "gen", "--out", other, "--namespace", "Acme.Generated"); Assert.True(genOther.ExitCode == 0, $"exit={genOther.ExitCode}\n{genOther.Stdout}{genOther.Stderr}"); Assert.Equal(["Subscriber.g.cs"], FilesUnder(other)); - var cleanOther = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", "--out", other); + var cleanOther = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", "--out", other, "--namespace", "Acme.Generated"); Assert.True(cleanOther.ExitCode == 0, $"exit={cleanOther.ExitCode}\n{cleanOther.Stdout}{cleanOther.Stderr}"); } [Fact] public void Without_a_configured_out_dir_or_out_flag_gen_asks_for_the_setting() { - var gen = OwnedRunner.Run(_tmp, [new EntityGenerator()], "gen"); + var gen = OwnedRunner.Run(_tmp, [new EntityGenerator()], "gen", "--namespace", "Acme.Generated"); Assert.Equal(2, gen.ExitCode); Assert.Contains("csharp", gen.Stderr); Assert.Contains("outDir", gen.Stderr); @@ -480,17 +417,17 @@ public void Without_a_configured_out_dir_or_out_flag_gen_asks_for_the_setting() public void Verify_narrowed_to_an_entry_with_its_own_directory_does_not_convict_the_rest_of_the_list() { IReadOnlyList declared = [new EntityGenerator(), new ApiDocsGenerator { OutDir = DocsDir }]; - Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); - var docsOnly = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "api-docs"); + var docsOnly = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "api-docs", "--namespace", "Acme.Generated"); Assert.True(docsOnly.ExitCode == 0, $"exit={docsOnly.ExitCode}\nstdout={docsOnly.Stdout}\nstderr={docsOnly.Stderr}"); - var codeOnly = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "entity"); + var codeOnly = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "entity", "--namespace", "Acme.Generated"); Assert.True(codeOnly.ExitCode == 0, $"exit={codeOnly.ExitCode}\nstdout={codeOnly.Stdout}\nstderr={codeOnly.Stderr}"); // What the narrowed run does produce is still checked. File.AppendAllText(Path.Combine(DocsDir, "api", "csharp", "README.md"), "edited\n"); - var drift = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "api-docs"); + var drift = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--generators", "api-docs", "--namespace", "Acme.Generated"); Assert.Equal(1, drift.ExitCode); Assert.Contains("api/csharp/README.md", drift.Stderr); } @@ -500,9 +437,9 @@ public void An_entry_directory_nested_in_out_is_compared_once_as_its_own() { var nestedDocs = Path.Combine(OutDir, "docs"); IReadOnlyList declared = [new EntityGenerator(), new ApiDocsGenerator { OutDir = nestedDocs }]; - Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); - var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir); + var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); } @@ -529,12 +466,12 @@ public void A_conventional_template_spec_no_entry_declares_is_refused_and_nothin WriteSpec(); var before = ProjectFiles(); - var (exit, _, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir); + var (exit, _, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.Equal(2, exit); Assert.Equal(CodegenCli.LegacyTemplateSpec("dotnet meta gen", "template-spec.json"), stderr.Trim()); Assert.Equal(before, ProjectFiles()); - var (verifyExit, _, verifyErr) = OwnedRunner.Run(_tmp, ["entity"], "verify", "--codegen", MetaDir, "--out", OutDir); + var (verifyExit, _, verifyErr) = OwnedRunner.Run(_tmp, ["entity"], "verify", "--codegen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.Equal(2, verifyExit); Assert.Equal(CodegenCli.LegacyTemplateSpec("dotnet meta verify --codegen", "template-spec.json"), verifyErr.Trim()); } @@ -545,7 +482,7 @@ public void The_template_spec_flag_is_refused_with_the_line_that_says_where_it_w var spec = WriteSpec("other-spec.json"); var before = ProjectFiles(); - var (exit, _, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--template-spec", spec); + var (exit, _, stderr) = OwnedRunner.Run(_tmp, ["entity"], "gen", MetaDir, "--out", OutDir, "--template-spec", spec, "--namespace", "Acme.Generated"); Assert.Equal(2, exit); Assert.Equal(CodegenCli.LegacyTemplateSpec("dotnet meta gen", "--template-spec"), stderr.Trim()); @@ -559,12 +496,12 @@ public void A_declared_template_spec_entry_renders_its_templates_and_verifies_cl IReadOnlyList declared = [new TemplateSpecGenerator { Spec = spec, TemplateRoot = Path.Combine(_tmp, "templates") }]; - var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir); + var (exit, stdout, stderr) = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); Assert.Equal(["Subscriber.summary.txt"], FilesUnder(OutDir)); Assert.Equal("name=Subscriber\n", File.ReadAllText(Path.Combine(OutDir, "Subscriber.summary.txt"))); - var (verifyExit, verifyOut, verifyErr) = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir); + var (verifyExit, verifyOut, verifyErr) = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(verifyExit == 0, $"exit={verifyExit}\nstdout={verifyOut}\nstderr={verifyErr}"); Assert.Contains("OK (generated output is in sync)", verifyOut); } @@ -575,7 +512,7 @@ public void A_declared_template_spec_that_cannot_be_read_fails_cleanly_before_wr IReadOnlyList declared = [new EntityGenerator(), new TemplateSpecGenerator { Spec = Path.Combine(_tmp, "missing.json") }]; - var (exit, _, stderr) = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir); + var (exit, _, stderr) = OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated"); Assert.Equal(1, exit); Assert.Contains("template spec", stderr); @@ -592,13 +529,13 @@ public void The_runs_template_root_reaches_a_declared_spec_that_names_none() IReadOnlyList declared = [new TemplateSpecGenerator { Spec = spec }]; var (exit, stdout, stderr) = OwnedRunner.Run( - _tmp, declared, "gen", MetaDir, "--out", OutDir, "--template-root", "templates"); + _tmp, declared, "gen", MetaDir, "--out", OutDir, "--template-root", "templates", "--namespace", "Acme.Generated"); Assert.True(exit == 0, $"exit={exit}\nstdout={stdout}\nstderr={stderr}"); Assert.Equal("name=Subscriber\n", File.ReadAllText(Path.Combine(OutDir, "Subscriber.summary.txt"))); - var verify = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir, "--template-root", "templates"); + var verify = OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir, "--template-root", "templates", "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); - Assert.Equal(1, OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(1, OwnedRunner.Run(_tmp, declared, "verify", "--codegen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); } // ---- a directory gen never had to create is an empty tree, not a missing one --------------- @@ -610,10 +547,10 @@ public void An_entry_directory_that_got_no_files_verifies_clean() IReadOnlyList declared = [new EntityGenerator(), new RequirementTestsGenerator { OutDir = testsDir, WarnUncovered = false }]; - Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); Assert.False(Directory.Exists(testsDir)); - var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir); + var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); } @@ -626,12 +563,12 @@ public void A_list_of_entries_with_their_own_directories_verifies_clean_in_a_fre new ApiDocsGenerator { OutDir = DocsDir }, new RequirementTestsGenerator { OutDir = testsDir, WarnUncovered = false }, ]; - Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, declared, "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); Assert.Empty(FilesUnder(OutDir)); Directory.Delete(OutDir); Assert.False(Directory.Exists(testsDir)); - var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir); + var verify = OwnedRunner.Run(_tmp, declared, "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); } @@ -639,9 +576,9 @@ public void A_list_of_entries_with_their_own_directories_verifies_clean_in_a_fre [Fact] public void Verify_codegen_of_a_list_that_declares_nothing_is_clean() { - Assert.Equal(0, OwnedRunner.Run(_tmp, Array.Empty(), "gen", MetaDir, "--out", OutDir).ExitCode); + Assert.Equal(0, OwnedRunner.Run(_tmp, Array.Empty(), "gen", MetaDir, "--out", OutDir, "--namespace", "Acme.Generated").ExitCode); - var verify = OwnedRunner.Run(_tmp, Array.Empty(), "verify", MetaDir, "--codegen", "--out", OutDir); + var verify = OwnedRunner.Run(_tmp, Array.Empty(), "verify", MetaDir, "--codegen", "--out", OutDir, "--namespace", "Acme.Generated"); Assert.True(verify.ExitCode == 0, $"exit={verify.ExitCode}\nstdout={verify.Stdout}\nstderr={verify.Stderr}"); Assert.Contains("codegen/Generators.cs declares no generators; nothing to check", verify.Stdout); diff --git a/server/csharp/MetaObjects.Cli.Tests/OwnedCopyTests.cs b/server/csharp/MetaObjects.Cli.Tests/OwnedCopyTests.cs index 91c965c1bb..eef7bf4dbc 100644 --- a/server/csharp/MetaObjects.Cli.Tests/OwnedCopyTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/OwnedCopyTests.cs @@ -28,14 +28,30 @@ public void Compare_identical_text_is_identical() } [Fact] - public void Compare_is_blind_to_blank_lines_and_line_order() + public void Compare_is_blind_to_line_endings_only() { - // Line-multiset, not a sequence diff — matches the Python port exactly (it is - // NOT formatter-canonicalized like TS's Biome-based comparison). - var cmp = OwnedCopy.Compare("b\n\na\n\nc\n", "a\nb\nc\n"); + Assert.Equal(OwnedCopy.Verdict.Identical, OwnedCopy.Compare("a\r\nb\r\nc\r\n", "a\nb\nc\n").Verdict); + } + + [Fact] + public void Compare_is_blind_to_trailing_whitespace_and_blank_lines() + { + // An editor's save-time cleanup reorders nothing, so it must not read as an edit. + var cmp = OwnedCopy.Compare("a \n\nb\t\nc\n\n", "a\nb\nc\n"); Assert.Equal(OwnedCopy.Verdict.Identical, cmp.Verdict); } + [Fact] + public void Compare_reads_a_reordering_as_an_edit() + { + // The same lines in another order are a different program; `eject --undo` deletes an + // identical copy, so this must not read identical. The counts stay order-blind. + var cmp = OwnedCopy.Compare("b\na\nc\n", "a\nb\nc\n"); + Assert.Equal(OwnedCopy.Verdict.Differs, cmp.Verdict); + Assert.Equal(0, cmp.Behind); + Assert.Equal(0, cmp.OfYourOwn); + } + [Fact] public void Compare_counts_behind_and_of_your_own_separately() { diff --git a/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs b/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs index 4da708f582..c773dce5ac 100644 --- a/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs @@ -253,70 +253,31 @@ public void Codegen_with_metadata_that_does_not_load_is_exit1_not_the_usage_exit Assert.Contains("did not load", r.Codegen!.Error); } - // -------------------- the namespace-inference footgun -------------------- - // gen used a CUSTOM namespace; verify --codegen WITHOUT --namespace must infer - // it from the committed output (else every file would spuriously drift on the - // embedded `namespace {ns};`). Build the Options with NO explicit namespace. + // -------------------- the namespace is gen's, never read from the output -------------------- + // verify --codegen regenerates under the namespace gen resolves (--namespace, else + // csharp.namespace). Output generated under another namespace is drift, not something to adopt. - /// Codegen opts with NO explicit namespace (the footgun scenario). - private VerifyCommand.Options CodegenOptsNoNamespace() => - new() - { - MetadataDir = MetaDir, - // verify --codegen re-runs the SELECTION; there is no default suite to fall - // back on, so the mechanics tests name the one their fixtures were written for. - Generators = GenSuite.Names, - OutDir = OutDir, - Codegen = true, - // Namespace intentionally NOT set + NamespaceExplicit defaults to false. - }; + private VerifyCommand.Options CodegenOpts(string ns) => + new() { MetadataDir = MetaDir, Generators = GenSuite.Names, OutDir = OutDir, Codegen = true, Namespace = ns }; [Fact] - public void Codegen_infers_custom_namespace_from_committed_output_no_flag() + public void Codegen_output_under_another_namespace_is_drift() { File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); - // Committed output generated with a CUSTOM namespace. GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // verify --codegen WITHOUT --namespace → must infer "Acme.Generated" from the - // committed files and produce a byte-identical regen → exit 0 (no spurious drift). - var r = VerifyCommand.RunSubverbs(CodegenOptsNoNamespace()); - Assert.Equal(0, r.ExitCode); - Assert.True(r.Codegen!.Clean, string.Join("; ", r.Codegen!.Lines)); - } - - [Fact] - public void Codegen_inference_still_detects_real_drift_with_custom_namespace() - { - File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); - GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // A real hand-edit on top of the custom namespace must still be drift. - File.AppendAllText(Path.Combine(OutDir, "Subscriber.g.cs"), "\n// real drift\n"); - - var r = VerifyCommand.RunSubverbs(CodegenOptsNoNamespace()); + var r = VerifyCommand.RunSubverbs(CodegenOpts("Other.Ns")); Assert.NotEqual(0, r.ExitCode); Assert.Contains(r.Codegen!.DriftedFiles, f => f.EndsWith("Subscriber.g.cs")); } [Fact] - public void Codegen_explicit_namespace_still_wins_over_inference() + public void Codegen_with_the_namespace_gen_used_is_clean() { File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // Explicit namespace set (matching) → wins, byte-identical regen → exit 0. - var opts = new VerifyCommand.Options - { - MetadataDir = MetaDir, - // verify --codegen re-runs the SELECTION; there is no default suite to fall - // back on, so the mechanics tests name the one their fixtures were written for. - Generators = GenSuite.Names, - OutDir = OutDir, - Namespace = "Acme.Generated", - NamespaceExplicit = true, - Codegen = true, - }; - var r = VerifyCommand.RunSubverbs(opts); + var r = VerifyCommand.RunSubverbs(CodegenOpts("Acme.Generated")); Assert.Equal(0, r.ExitCode); Assert.True(r.Codegen!.Clean, string.Join("; ", r.Codegen!.Lines)); } @@ -371,7 +332,6 @@ private void WriteNamingConfig(string columns) Generators = GenSuite.Names, OutDir = OutDir, Namespace = "Acme.Generated", - NamespaceExplicit = true, Codegen = true, Naming = new PhysicalNaming(verifyStrategy, TableNamingRule.SnakeCase), }; diff --git a/server/csharp/MetaObjects.Cli.Tests/VerifyTemplateSpecTests.cs b/server/csharp/MetaObjects.Cli.Tests/VerifyTemplateSpecTests.cs index 48e68240b3..647ac9af5c 100644 --- a/server/csharp/MetaObjects.Cli.Tests/VerifyTemplateSpecTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/VerifyTemplateSpecTests.cs @@ -62,6 +62,7 @@ public VerifyTemplateSpecTests() TemplatesRoot = TemplateRoot, TemplateRoot = TemplateRoot, OutDir = OutDir, + Namespace = "Acme.Generated", Templates = false, Codegen = true, Db = false, diff --git a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs index 2488b79c48..2fb7aeecad 100644 --- a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs +++ b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs @@ -22,14 +22,6 @@ public static bool HasOwnedCodegen(string projectRoot) => public static bool HasGeneratorList(string projectRoot) => File.Exists(Path.Combine(projectRoot, "codegen", "Generators.cs")); - /// True iff has a codegen/Program.cs that does not run - /// Generators.All: a 1.0 entry point, which declares its own list and ignores codegen/Generators.cs. - public static bool ProgramIgnoresList(string projectRoot) - { - var program = Path.Combine(projectRoot, "codegen", "Program.cs"); - return File.Exists(program) && !File.ReadAllText(program).Contains("Generators.All", StringComparison.Ordinal); - } - /// /// When has an ejected codegen project, run `dotnet /// run --project codegen -- <args>` (inheriting this process's console) and set diff --git a/server/csharp/MetaObjects.Cli/EjectCommand.cs b/server/csharp/MetaObjects.Cli/EjectCommand.cs index c6be8be129..8cd3d7e2af 100644 --- a/server/csharp/MetaObjects.Cli/EjectCommand.cs +++ b/server/csharp/MetaObjects.Cli/EjectCommand.cs @@ -221,12 +221,11 @@ public static Outcome Run(IReadOnlyList names, string root, bool force) outLines.Add(""); outLines.Add($"Also wrote {string.Join(", ", scaffolded)}."); } - var programIgnoresList = CodegenHandoff.ProgramIgnoresList(root); if (scaffolded.Contains("codegen/Generators.cs")) { outLines.Add( - $"codegen/Generators.cs lists the generator(s) you just ejected ({string.Join(", ", entries)})." + - (programIgnoresList ? "" : " `dotnet meta gen` and `dotnet meta verify --codegen` run that list.")); + $"codegen/Generators.cs lists the generator(s) you just ejected ({string.Join(", ", entries)}). " + + "`dotnet meta gen` and `dotnet meta verify --codegen` run that list."); } else { @@ -237,11 +236,6 @@ public static Outcome Run(IReadOnlyList names, string root, bool force) outLines.Add("to the list there, in place of the packaged entry if it lists one " + "(MetaObjects.Codegen.Generators.)."); } - if (programIgnoresList) - { - outLines.Add(""); - outLines.AddRange(CodegenScaffold.ReplaceProgram()); - } if (csprojExisted) outLines.AddRange(OlderPinWarning(csprojPath, "eject")); return new Outcome(0, outLines, []); diff --git a/server/csharp/MetaObjects.Cli/GenCommand.cs b/server/csharp/MetaObjects.Cli/GenCommand.cs index 7e9f6faa55..11da661015 100644 --- a/server/csharp/MetaObjects.Cli/GenCommand.cs +++ b/server/csharp/MetaObjects.Cli/GenCommand.cs @@ -19,13 +19,6 @@ public sealed record Outcome(IReadOnlyList LoadErrors, CodegenRunner.Run public bool Ok => LoadErrors.Count == 0 && Result is not null; } - /// - /// The default C# namespace generated code lands in when --namespace is omitted. - /// The api reference documents the same namespace, so its using <ns>; import - /// lines match what an adopter writes against generated code. - /// - public const string DefaultNamespace = "Generated"; - /// /// The error a run with no generator selection reports. /// diff --git a/server/csharp/MetaObjects.Cli/InitCommand.cs b/server/csharp/MetaObjects.Cli/InitCommand.cs index 4b1483eb29..3caf2eebc9 100644 --- a/server/csharp/MetaObjects.Cli/InitCommand.cs +++ b/server/csharp/MetaObjects.Cli/InitCommand.cs @@ -49,15 +49,6 @@ public static class CodegenScaffold "using MetaObjects.Codegen;\n" + "return CodegenCli.Run(args, Generators.All);\n"; - /// What init and eject say in place of "gen runs the list" when the project's own - /// codegen/Program.cs does not run it (). - public static IReadOnlyList ReplaceProgram() => - [ - "codegen/Program.cs does not run Generators.All, so `dotnet meta gen` and `dotnet meta verify --codegen` " + - "refuse until it does. It is yours, so init and eject never edit it: replace it with the two-line entry", - .. ProgramCs.TrimEnd('\n').Split('\n').Select(l => " " + l), - $"See {CodegenCli.DocsMigrationGuide}.", - ]; /// /// codegen/Generators.cs, the project's generator list, holding : @@ -140,12 +131,9 @@ public static IReadOnlyList Run(string root, bool withDocs = false) foreach (var rel in CodegenScaffold.Files) lines.Add(written.Contains(rel) ? $" wrote {rel}" : $" kept {rel} (already yours)"); lines.Add(""); - if (CodegenHandoff.ProgramIgnoresList(root)) - lines.AddRange(CodegenScaffold.ReplaceProgram()); - else - lines.Add( - "codegen/Generators.cs is the generator list `dotnet meta gen` runs. It declares nothing yet, " + - "so gen writes nothing until you add entries: `dotnet meta gen --list` is the catalog."); + lines.Add( + "codegen/Generators.cs is the generator list `dotnet meta gen` runs. It declares nothing yet, " + + "so gen writes nothing until you add entries: `dotnet meta gen --list` is the catalog."); if (withDocs) { if (listExisted) diff --git a/server/csharp/MetaObjects.Cli/OwnedCopy.cs b/server/csharp/MetaObjects.Cli/OwnedCopy.cs index 309ab59d3b..598f8d86ed 100644 --- a/server/csharp/MetaObjects.Cli/OwnedCopy.cs +++ b/server/csharp/MetaObjects.Cli/OwnedCopy.cs @@ -1,8 +1,10 @@ // ADR-0034 Amendment 3 — eject. How an OWNED generator copy (codegen/generators/) // compares to the reference `dotnet meta eject` would write today. Line-multiset -// comparison, the same shape as the Python port's owned_status (Counter subtraction) — +// counts, the same shape as the Python port's owned_status (Counter subtraction) — // NOT the formatter-canonicalized comparison TS's owned-copy.ts does, since this port -// has no bundled C# formatter to normalise through. "behind" counts reference lines the +// has no bundled C# formatter to normalise through. The VERDICT is the ordered line +// sequence (trailing whitespace and blank lines aside): `eject --undo` deletes an identical +// copy, so an edit that only reorders lines must read as an edit. "behind" counts reference lines the // copy lacks (upstream moved); "of your own" counts copy lines the reference lacks // (your edit). The comparison is against the REWRITTEN reference (EjectableGenerators. // RewriteForEject applied) — the namespace/using lines eject itself changes are not @@ -21,27 +23,32 @@ public sealed record Comparison(Verdict Verdict, int Behind, int OfYourOwn); public static string OwnedPath(string root, string sourceFileName) => Path.Combine(root, "codegen", "generators", sourceFileName); - /// The non-blank line multiset of , trailing + /// The non-blank lines of in order, trailing /// whitespace and CRLF stripped per line so a checkout's line-ending style never /// reads as drift. - private static Dictionary LineCounts(string text) + private static List OrderedLines(string text) => + text.Split('\n').Select(raw => raw.TrimEnd('\r', ' ', '\t')).Where(line => line.Length > 0).ToList(); + + /// The multiset of . + private static Dictionary LineCounts(IEnumerable lines) { var counts = new Dictionary(StringComparer.Ordinal); - foreach (var raw in text.Split('\n')) - { - var line = raw.TrimEnd('\r', ' ', '\t'); - if (line.Length == 0) continue; + foreach (var line in lines) counts[line] = counts.GetValueOrDefault(line) + 1; - } return counts; } - /// Line-multiset compare against . + /// Compare against : + /// Identical only for the same lines in the same order (trailing whitespace and blank + /// lines aside); the line counts are order-blind. public static Comparison Compare(string owned, string reference) { - var mine = LineCounts(owned); - var theirs = LineCounts(reference); + var ownedLines = OrderedLines(owned); + var referenceLines = OrderedLines(reference); + if (ownedLines.SequenceEqual(referenceLines, StringComparer.Ordinal)) + return new Comparison(Verdict.Identical, 0, 0); + var mine = LineCounts(ownedLines); + var theirs = LineCounts(referenceLines); int behind = 0, ofYourOwn = 0; foreach (var line in mine.Keys.Concat(theirs.Keys).Distinct(StringComparer.Ordinal)) { @@ -50,9 +57,16 @@ public static Comparison Compare(string owned, string reference) if (r > m) behind += r - m; if (m > r) ofYourOwn += m - r; } - return new Comparison(behind == 0 && ofYourOwn == 0 ? Verdict.Identical : Verdict.Differs, behind, ofYourOwn); + return new Comparison(Verdict.Differs, behind, ofYourOwn); } + /// "N behind, M of your own"; a copy with the same lines in another order + /// (0 and 0) says so. + private static string Counts(Comparison c) => + c.Behind == 0 && c.OfYourOwn == 0 + ? "0 behind, 0 of your own, reordered" + : $"{c.Behind} behind, {c.OfYourOwn} of your own"; + /// /// "identical" / "DIFFERS: N behind, M of your own", or null when /// is not ejectable or nothing is owned at (); if (cmp.Verdict == Verdict.Differs) - clauses.Add($"DIFFERS: {cmp.Behind} behind, {cmp.OfYourOwn} of your own"); + clauses.Add($"DIFFERS: {Counts(cmp)}"); foreach (var sourcePath in entry.Companions) { @@ -94,7 +108,7 @@ public static Comparison Compare(string owned, string reference) MetaObjects.Codegen.EjectableGenerators.RewriteForEject( MetaObjects.Codegen.EjectableGenerators.ReadSourcePath(sourcePath))); if (c.Verdict == Verdict.Differs) - clauses.Add($"{name} DIFFERS: {c.Behind} behind, {c.OfYourOwn} of your own"); + clauses.Add($"{name} DIFFERS: {Counts(c)}"); } return clauses.Count == 0 ? "identical" : string.Join("; ", clauses); } @@ -123,7 +137,7 @@ public static Comparison Compare(string owned, string reference) } var cmp = Compare(File.ReadAllText(path), MetaObjects.Codegen.HelperRuntime.OwnedReference(file)); if (cmp.Verdict == Verdict.Differs) - clauses.Add($"{file} DIFFERS: {cmp.Behind} behind, {cmp.OfYourOwn} of your own"); + clauses.Add($"{file} DIFFERS: {Counts(cmp)}"); } return clauses.Count == 0 ? "identical" : string.Join("; ", clauses); } diff --git a/server/csharp/MetaObjects.Cli/Program.cs b/server/csharp/MetaObjects.Cli/Program.cs index 541f5cf41f..236ddd5f3c 100644 --- a/server/csharp/MetaObjects.Cli/Program.cs +++ b/server/csharp/MetaObjects.Cli/Program.cs @@ -46,8 +46,8 @@ " --codegen regen-to-temp vs committed --out\n" + " (--out defaults to csharp.outDir in\n" + " .metaobjects/config.json, as for gen)\n" + - " --namespace codegen regen namespace; inferred\n" + - " from the committed --out when omitted\n" + + " --namespace codegen regen namespace (default: csharp.namespace\n" + + " in .metaobjects/config.json, else the metadata package)\n" + " --db NOT supported in C# (migrate engine)\n" + " --lax load lax (legacy); strict-by-default\n" + " rejects an unregistered @attr (ADR-0023)\n" + @@ -173,11 +173,6 @@ static int RunGen(string[] rest) "`dotnet meta init` writes the missing files and keeps the ones you have."); return 2; } - if (CodegenHandoff.ProgramIgnoresList(projectRoot)) - { - Console.Error.WriteLine(CodegenCli.LegacyProgram(command)); - return 2; - } return null; } @@ -385,8 +380,7 @@ static int RunVerify(string[] rest) string? metadataDir = null; string? templatesRoot = null; string? outDir = null; - string ns = "Generated"; - bool nsExplicit = false; + string? ns = null; string? generatorsCsv = null; string? templateRoot = null; string? columnNamingRaw = null; @@ -422,7 +416,7 @@ static int RunVerify(string[] rest) // ledger has caught up with its links. META_REQUIRE_IMPLEMENTERS=1 does the same. else if (a == "--require-implementers") requireImplementers = true; else if (a == "--out" && i + 1 < rest.Length) outDir = rest[++i]; - else if (a == "--namespace" && i + 1 < rest.Length) { ns = rest[++i]; nsExplicit = true; } + else if (a == "--namespace" && i + 1 < rest.Length) { ns = rest[++i]; } else if (a == "--generators" && i + 1 < rest.Length) generatorsCsv = rest[++i]; else if (a == "--template-root" && i + 1 < rest.Length) templateRoot = rest[++i]; // ADR-0058 — the flag is removed: column and table naming is one project setting @@ -513,7 +507,7 @@ static int RunVerify(string[] rest) // --generators narrows the declared list there exactly as it narrows gen's. List codegenArgs = ["verify", "--codegen", "--out", outDir]; if (metadataDir is not null) codegenArgs.Insert(2, metadataDir); - if (nsExplicit) codegenArgs.AddRange(["--namespace", ns]); + if (ns is not null) codegenArgs.AddRange(["--namespace", ns]); if (generatorsCsv is not null) codegenArgs.AddRange(["--generators", generatorsCsv]); if (templateRoot is not null) codegenArgs.AddRange(["--template-root", templateRoot]); if (lax) codegenArgs.Add("--lax"); @@ -540,7 +534,7 @@ static int RunVerify(string[] rest) TemplatesRoot = templatesRoot, OutDir = outDir, Namespace = ns, - NamespaceExplicit = nsExplicit, + ConfiguredNamespace = resolvedMeta.Namespace, Generators = generators, TemplateRoot = templateRoot, Templates = templates, diff --git a/server/csharp/MetaObjects.Cli/VerifyCommand.cs b/server/csharp/MetaObjects.Cli/VerifyCommand.cs index 7a8a578ce0..bc85d1e049 100644 --- a/server/csharp/MetaObjects.Cli/VerifyCommand.cs +++ b/server/csharp/MetaObjects.Cli/VerifyCommand.cs @@ -123,12 +123,11 @@ public sealed record Options public string? TemplatesRoot { get; init; } /// The committed output dir for the --codegen gate (--out <dir>). public string? OutDir { get; init; } - /// Code-gen namespace (only affects the temp regen; default mirrors `gen`). - public string Namespace { get; init; } = "Generated"; - /// True when the user explicitly passed --namespace. When false and - /// --codegen runs, the namespace is inferred from the committed output so a - /// regen matches output generated with any namespace (avoids spurious drift). - public bool NamespaceExplicit { get; init; } + /// --namespace for the codegen regen; null falls back to + /// , as gen does. + public string? Namespace { get; init; } + /// csharp.namespace from .metaobjects/config.json, or null. + public string? ConfiguredNamespace { get; init; } /// Generator selection for the --codegen regen. There is no default suite: name the ones gen ran. public IReadOnlyList? Generators { get; init; } /// Template root for render-helper/template generators in the codegen regen. @@ -432,15 +431,6 @@ private static Codegen.CodegenDrift.Result RunCodegenDrift(Options opts, out boo ], }; - // The C# namespace is embedded in every generated file (`namespace {ns};`). - // If the user did NOT pass --namespace, infer it from the committed output so - // the regen matches output produced with any namespace (otherwise every file - // would spuriously drift). An explicit --namespace always wins (back-compat / - // override); inference falling back to null leaves the default in place. - var ns = opts.Namespace; - if (!opts.NamespaceExplicit) - ns = Codegen.CodegenDrift.InferNamespace(opts.OutDir) ?? opts.Namespace; - // C1 — same presence gate `gen` computes (GeneratorRegistry.IncludesNames over // the SAME resolved `names` list above), or a regen under a `--generators` // selection that excludes `names` would reference `Names.*` while `gen` @@ -448,7 +438,7 @@ private static Codegen.CodegenDrift.Result RunCodegenDrift(Options opts, out boo // direction, a false-clean pass in the other. var config = new GenConfig { - OutDir = opts.OutDir, Namespace = ns, + OutDir = opts.OutDir, Namespace = opts.Namespace ?? opts.ConfiguredNamespace, IncludeNames = GeneratorRegistry.IncludesNames(names), // ADR-0058 — one project setting, read from the loaded root: `gen` produced // --out reading the SAME root's naming.columns, so this regen agrees with it diff --git a/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs b/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs index 39e06fc052..0d083ab4b7 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs @@ -266,4 +266,16 @@ public void NamingConformance_collisionCasesAreRefused(CollisionCase c) Assert.Throws( () => CSharpNaming.AssertNoCollectionNameCollisions(entities)); } + + [Fact] + public void AssertNoCollectionNameCollisions_throws_forRouteSegmentTwins() + { + // ApiKey / APIKey pluralize to different names but both snake_case to /api_keys. + var entities = MultiEntityRoot(("ApiKey", false), ("APIKey", false)); + var ex = Assert.Throws( + () => CSharpNaming.AssertNoCollectionNameCollisions(entities)); + Assert.Contains("\"ApiKey\"", ex.Message); + Assert.Contains("\"APIKey\"", ex.Message); + Assert.Contains("api_keys", ex.Message); + } } diff --git a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs index 1ae5bacbf0..6fa7a1e2ae 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs @@ -151,48 +151,6 @@ public void Drift_run_never_writes_to_the_real_out_dir() Assert.NotEqual(before, afterMutation); } - // -------------------- namespace inference from committed output ---------- - - [Fact] - public void InferNamespace_reads_file_scoped_namespace() - { - Gen(); // writes files with `namespace Acme.Generated;` - Assert.Equal("Acme.Generated", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_reads_block_scoped_namespace() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "Block.g.cs"), - "// \nnamespace Acme.Block\n{\n public class Foo { }\n}\n"); - Assert.Equal("Acme.Block", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_picks_the_first_sorted_file() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "Zeta.g.cs"), "namespace Zeta.Ns;\n"); - File.WriteAllText(Path.Combine(OutDir, "Alpha.g.cs"), "namespace Alpha.Ns;\n"); - Assert.Equal("Alpha.Ns", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_returns_null_for_missing_dir() - { - Assert.Null(CodegenDrift.InferNamespace(Path.Combine(_tmp, "does-not-exist"))); - } - - [Fact] - public void InferNamespace_returns_null_when_no_cs_has_a_namespace() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "NoNs.g.cs"), "// top-level statements only\nvar x = 1;\n"); - File.WriteAllText(Path.Combine(OutDir, "notes.txt"), "namespace NotCSharp;\n"); - Assert.Null(CodegenDrift.InferNamespace(OutDir)); - } - private static Dictionary SnapshotDir(string dir) { var snap = new Dictionary(StringComparer.Ordinal); diff --git a/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs b/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs index d0c01080b9..72fa0d7e5e 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs @@ -98,4 +98,62 @@ public void The_chain_walk_is_depth_bounded() Assert.Null(ConstraintErrors.Classify(deep)); } + + private sealed class CodedDbException(string message, string? sqlState) : System.Data.Common.DbException(message) + { + public override string? SqlState { get; } = sqlState; + } + + private sealed class FakeSqliteException(string message, int extended) : Exception(message) + { + public int SqliteExtendedErrorCode { get; } = extended; + } + + [Theory] + [InlineData("23505", 409, "unique")] + [InlineData("23503", 409, "foreign_key")] + [InlineData("23514", 400, "check")] + [InlineData("23502", 400, "not_null")] + public void A_sqlstate_decides_even_when_the_message_names_another_constraint( + string sqlState, int status, string constraint) + { + // The message would classify as a foreign key / not null by text; the code wins. + var failure = ConstraintErrors.Classify(new InvalidOperationException(EfWrapper, + new CodedDbException("duplicate value 'FOREIGN KEY' NOT NULL", sqlState))); + + Assert.NotNull(failure); + Assert.Equal(status, failure!.Status); + Assert.Equal(constraint, failure.Constraint); + } + + [Fact] + public void An_unrecognised_sqlstate_falls_through_to_message_text() + { + // 40001 is not a constraint code, so it is no verdict: the message is still read. + var failure = ConstraintErrors.Classify(new CodedDbException("23505: duplicate key value", "40001")); + Assert.NotNull(failure); + Assert.Equal("unique", failure!.Constraint); + } + + [Fact] + public void An_unrecognised_sqlstate_with_no_constraint_text_is_not_classified() + { + Assert.Null(ConstraintErrors.Classify(new CodedDbException("could not serialize access", "40001"))); + } + + [Theory] + [InlineData(787, 409, "foreign_key")] + [InlineData(2067, 409, "unique")] + [InlineData(1555, 409, "unique")] + [InlineData(275, 400, "check")] + [InlineData(1299, 400, "not_null")] + public void A_sqlite_extended_code_decides_over_the_message(int extended, int status, string constraint) + { + var failure = ConstraintErrors.Classify(new InvalidOperationException(EfWrapper, + new FakeSqliteException("misleading UNIQUE CONSTRAINT text", extended))); + + Assert.NotNull(failure); + Assert.Equal(status, failure!.Status); + Assert.Equal(constraint, failure.Constraint); + } } diff --git a/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs b/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs new file mode 100644 index 0000000000..837fe29b04 --- /dev/null +++ b/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs @@ -0,0 +1,38 @@ +using MetaObjects.Codegen.Generators; +using Xunit; + +namespace MetaObjects.Codegen.Tests; + +public class EntityGeneratorEscapeTests +{ + private sealed class Probe : EntityGenerator + { + public static string Escape(string s) => EscapeAttrString(s); + } + + [Fact] + public void Escapes_what_would_end_the_literal() + { + Assert.Equal("a\\\\b\\\"c", Probe.Escape("a\\b\"c")); + Assert.Equal("l1\\nl2\\r\\t", Probe.Escape("l1\nl2\r\t")); + Assert.Equal("\\u0001\\u2028\\u0085", Probe.Escape("\u0001" + (char)0x2028 + (char)0x85)); + } + + [Fact] + public void Leaves_ordinary_text_alone() + { + Assert.Equal("^[a-z]+é$", Probe.Escape("^[a-z]+é$")); + } + + [Fact] + public void An_escaped_value_survives_a_round_trip_through_the_compiler_literal_rules() + { + var original = "line1\nline2\t\"q\"\\ " + (char)0x2029; + var src = "class C { public const string V = \"" + Probe.Escape(original) + "\"; }"; + var tree = Microsoft.CodeAnalysis.CSharp.CSharpSyntaxTree.ParseText(src); + Assert.Empty(tree.GetDiagnostics()); + var literal = tree.GetRoot().DescendantNodes() + .OfType().Single(); + Assert.Equal(original, literal.Token.ValueText); + } +} diff --git a/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs b/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs index ddb0be54cc..cdc8435b58 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs @@ -243,7 +243,7 @@ public void A_child_field_colliding_with_an_INHERITED_one_is_refused_naming_the_ { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "CreatedAt" } }, + { "field.timestamp": { "name": "CreatedAt", "@column": "created_at_alt" } }, { "identity.primary": { "name": "pk", "@fields": "id", "@generation": "increment" } } ]}} ]}} diff --git a/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs b/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs index 03b614dd97..5ae9bb5e7c 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs @@ -343,7 +343,9 @@ public void The_type_prefix_keeps_an_unnamed_primary_key_off_the_primary_ROLE_me [Fact] public void Two_fields_colliding_on_their_Pascal_form_is_refused_naming_the_model() { - // userId and UserId both Pascalize to "UserId" — two duplicate const members. + // userId and UserId both Pascalize to "UserId" — two duplicate const members. (The + // explicit @column keeps their PHYSICAL columns apart: equal physical columns are + // refused earlier, by the loader, as ERR_COLUMN_NAME_COLLISION.) // C# would refuse to compile the generated file, but the error would name a // generated .g.cs and read as a codegen bug rather than a model one. Fail here // instead, naming the entity and both offending field names. @@ -353,7 +355,7 @@ public void Two_fields_colliding_on_their_Pascal_form_is_refused_naming_the_mode { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": "id" } } ]}} ]}} diff --git a/server/csharp/MetaObjects.Codegen.Tests/OrphanSweepTests.cs b/server/csharp/MetaObjects.Codegen.Tests/OrphanSweepTests.cs index 00e60621a6..0dbbca4ddf 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/OrphanSweepTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/OrphanSweepTests.cs @@ -76,8 +76,8 @@ private void Model(string package, params string[] values) => finally { Console.SetOut(prevOut); Console.SetError(prevErr); } } - private (int Code, string Out, string Err) Gen(params string[] extra) => Run(["gen", Meta, "--out", Out, .. extra]); - private (int Code, string Out, string Err) Verify() => Run("verify", "--codegen", Meta, "--out", Out); + private (int Code, string Out, string Err) Gen(params string[] extra) => Run(["gen", Meta, "--out", Out, "--namespace", "Acme.Generated", .. extra]); + private (int Code, string Out, string Err) Verify() => Run("verify", "--codegen", Meta, "--out", Out, "--namespace", "Acme.Generated"); private Dictionary Manifest() => JsonSerializer.Deserialize>(File.ReadAllText(ManifestPath))!; diff --git a/server/csharp/MetaObjects.Codegen.Tests/PackageBindingResolverTests.cs b/server/csharp/MetaObjects.Codegen.Tests/PackageBindingResolverTests.cs index 9c616fb153..45e64cc9e5 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/PackageBindingResolverTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/PackageBindingResolverTests.cs @@ -112,10 +112,9 @@ public void Unmapped_strategy_Error_throws_with_useful_diagnostic() [Theory] [InlineData(PackageCase.PascalCase, "users-access", "UsersAccess")] [InlineData(PackageCase.CamelCase, "users-access", "usersAccess")] - [InlineData(PackageCase.KebabCase, "users-access", "users-access")] [InlineData(PackageCase.SnakeCase, "users-access", "users_access")] [InlineData(PackageCase.Lowercase, "users-access", "usersaccess")] - [InlineData(PackageCase.Preserve, "users-access", "users-access")] + [InlineData(PackageCase.Preserve, "users_access", "users_access")] public void Case_transformations(PackageCase casing, string input, string expected) { var cfg = CfgWithDefaults() with @@ -133,6 +132,54 @@ public void Case_transformations(PackageCase casing, string input, string expect Assert.Equal($"Root.{expected}", result); } + private static GenConfig WithConvention(PackageCase casing) => CfgWithDefaults() with + { + Convention = new PackageBindingConvention { Strip = "acme::", Prepend = "Root", Case = casing }, + }; + + [Theory] + [InlineData(PackageCase.KebabCase, "users-access", "users-access")] + [InlineData(PackageCase.Preserve, "my-app", "my-app")] + [InlineData(PackageCase.PascalCase, "2fa", "2fa")] + [InlineData(PackageCase.Lowercase, "event", "event")] + [InlineData(PackageCase.Preserve, "default", "default")] + public void A_convention_that_yields_a_non_identifier_segment_stops_generation( + PackageCase casing, string input, string badSegment) + { + var cfg = WithConvention(casing); + + var ex = Assert.Throws(() => PackageBindingResolver.Resolve(cfg, $"acme::{input}", "X")); + Assert.Contains($"\"{badSegment}\" is not an identifier", ex.Message); + Assert.Contains($"metadata package \"acme::{input}\"", ex.Message); + Assert.Throws(() => PackageBindingResolver.TryResolve(cfg, $"acme::{input}", "X")); + Assert.Throws(() => PackageBindingResolver.ResolveForProvidedEnum( + cfg with { Convention = cfg.Convention! with { ProvidedEnumPrepend = "Root" } }, $"acme::{input}", "X")); + } + + [Theory] + [InlineData("My-app.Shop", "My-app")] + [InlineData("Acme.2fa", "2fa")] + [InlineData("Acme.object", "object")] + public void An_override_that_is_not_an_identifier_path_stops_generation(string target, string badSegment) + { + var typeOverride = CfgWithDefaults() with + { + TypeOverrides = new Dictionary { ["acme::cases::Case"] = target }, + }; + var packageOverride = CfgWithDefaults() with + { + PackageNamespaces = new Dictionary { ["acme::cases"] = target }, + }; + + foreach (var cfg in new[] { typeOverride, packageOverride }) + { + var ex = Assert.Throws(() => PackageBindingResolver.Resolve(cfg, "acme::cases", "Case")); + Assert.Contains($"\"{badSegment}\" is not an identifier", ex.Message); + Assert.Contains("metadata package \"acme::cases\"", ex.Message); + Assert.Throws(() => PackageBindingResolver.TryResolve(cfg, "acme::cases", "Case")); + } + } + [Fact] public void ProvidedEnumPrepend_routes_provided_enums_to_a_different_namespace_tree() { diff --git a/server/csharp/MetaObjects.Codegen/CSharpNaming.cs b/server/csharp/MetaObjects.Codegen/CSharpNaming.cs index cbbebe0be4..2d39b5ef6e 100644 --- a/server/csharp/MetaObjects.Codegen/CSharpNaming.cs +++ b/server/csharp/MetaObjects.Codegen/CSharpNaming.cs @@ -12,6 +12,7 @@ using static MetaObjects.Core.Field.FieldConstants; using static MetaObjects.Core.Identity.IdentityConstants; using static MetaObjects.Core.Index.IndexConstants; +using static MetaObjects.Core.Object.ObjectConstants; using static MetaObjects.Persistence.Source.SourceConstants; using static MetaObjects.Shared.BaseTypes; @@ -1177,6 +1178,38 @@ public static void AssertNoCollectionNameCollisions(IReadOnlyList en ownerOfPlural[plural] = obj; } } + AssertNoRouteSegmentCollisions(entities); + } + + /// + /// The route half. is the entity NAME snake_cased then + /// pluralized, so "ApiKey" and "APIKey" both serve "/api_keys" while their pluralized + /// PascalCase names differ and pass the DbSet-name check above. Counts only objects that + /// actually get a route: not a value object, not abstract, not sourceless, and not a + /// TPH subtype (folded into its base's table and served under the base's route). + /// + private static void AssertNoRouteSegmentCollisions(IReadOnlyList entities) + { + var ownerOfSegment = new Dictionary(StringComparer.Ordinal); + foreach (var obj in entities) + { + if (obj.IsValue() || !InstanceArtifacts.EmitsInstanceArtifacts(obj)) continue; + // ADR-0039: OwnAttr is REQUIRED — a TPH subtype declares @discriminatorValue itself. + if (obj.OwnAttr(OBJECT_ATTR_DISCRIMINATOR_VALUE) is string { Length: > 0 }) continue; + var segment = RoutePath(obj); + if (ownerOfSegment.TryGetValue(segment, out var existing)) + { + if (existing.ResolutionKey() != obj.ResolutionKey()) + throw new InvalidOperationException( + $"\"{existing.Name}\" and \"{obj.Name}\" both resolve " + + $"to the REST collection segment \"{segment}\" — rename one entity so its " + + $"snake_cased, pluralized route segment is distinct."); + } + else + { + ownerOfSegment[segment] = obj; + } + } } private static readonly char[] CSharpEscapeChars = { '\\', '"', '\t', '\n', '\r' }; diff --git a/server/csharp/MetaObjects.Codegen/CodegenCli.cs b/server/csharp/MetaObjects.Codegen/CodegenCli.cs index d4eb86f360..8bb057b73a 100644 --- a/server/csharp/MetaObjects.Codegen/CodegenCli.cs +++ b/server/csharp/MetaObjects.Codegen/CodegenCli.cs @@ -191,7 +191,7 @@ public sealed record GenOutcome(IReadOnlyList LoadErrors, CodegenRunner. /// directly. /// public static GenOutcome RunGen( - string metadataDir, string outDir, string ns, bool emitAbstractShapes, + string metadataDir, string outDir, string? ns, bool emitAbstractShapes, IReadOnlyList generators, string baseline = "default") => RunGen(MetadataLocation.Resolve(metadataDir, Directory.GetCurrentDirectory()).Load(), outDir, ns, emitAbstractShapes, generators, ProjectRootFor(metadataDir), baseline); @@ -226,7 +226,7 @@ public static GenOutcome RunGen( /// them emitted is stale output and is deleted when untouched (). /// public static GenOutcome RunGen( - LoadResult load, string outDir, string ns, bool emitAbstractShapes, + LoadResult load, string outDir, string? ns, bool emitAbstractShapes, IReadOnlyList generators, string? projectRoot = null, string baseline = "default", bool? includeNames = null, @@ -316,14 +316,6 @@ public static string NoGeneratorList(string command) => $"{command}: no codegen/Generators.cs. The generator list lives there in 1.1.0, documentation included; " + "`dotnet meta init` writes it. See " + DocsMigrationGuide + "."; - /// What a gen or verify --codegen run prints, exit 2, when codegen/Program.cs does not - /// run Generators.All: a 1.0 entry point declares its generators inline and would ignore the list. - /// The command's own name: dotnet meta gen or dotnet meta verify --codegen. - public static string LegacyProgram(string command) => - $"{command}: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in " + - "codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and " + - "`return CodegenCli.Run(args, Generators.All);`. See " + DocsMigrationGuide + "."; - /// What a gen run prints, exit 2, when it finds 1.0 template-spec configuration: a /// project-root template-spec.json no entry declares, or --template-spec. Through /// 1.0 the gen command read either; a spec now runs only as a @@ -546,9 +538,9 @@ public static IReadOnlyList ComposeGenerators( private sealed class OwnedArgs { public string? MetadataDir, OutDir, TemplateRoot, TemplateSpecPath, ColumnNamingRaw, GeneratorsCsv; - public string Namespace = "Generated"; + public string? Namespace; public string Baseline = "default"; - public bool NamespaceExplicit, EmitAbstractShapes, Codegen, Lax, Strict; + public bool EmitAbstractShapes, Codegen, Lax, Strict; /// 1.0 docs flags given (), in order. public List LegacyDocs = []; /// The usage error the arguments earn, or null. @@ -584,7 +576,7 @@ public static OwnedArgs Parse(string[] rest, string command, string[] valueFlags switch (flag) { case "--out": a.OutDir = value; break; - case "--namespace": a.Namespace = value; a.NamespaceExplicit = true; break; + case "--namespace": a.Namespace = value; break; case "--column-naming": a.ColumnNamingRaw = value; break; case "--baseline": a.Baseline = value; break; case "--template-root": a.TemplateRoot = value; break; @@ -761,7 +753,7 @@ private static int RunGenArgs(string[] rest, IReadOnlyList declared) WarnUnknownAttrs(meta); genLoad = meta.Load(); } - var outcome = RunGen(genLoad, a.OutDir, a.Namespace, a.EmitAbstractShapes, suite, projectRoot, a.Baseline, + var outcome = RunGen(genLoad, a.OutDir, a.Namespace ?? meta.Namespace, a.EmitAbstractShapes, suite, projectRoot, a.Baseline, includeNames: declared.Any(g => g.Name == "names"), // Only a run of the whole declared list can tell a stale file from one the // narrowing left out. @@ -954,10 +946,6 @@ private static int RunVerifyArgs(string[] rest, IReadOnlyList declar return 0; } - // Same inference rule `dotnet meta verify --codegen` has always used: an explicit - // --namespace always wins; otherwise infer from the committed output so a regen - // matches output produced with any namespace (avoids spurious drift). - var effectiveNs = a.NamespaceExplicit ? a.Namespace : (CodegenDrift.InferNamespace(a.OutDir) ?? a.Namespace); // ADR-0058 — the loaded root carries the project's column rule; `gen` read it the // same way producing the committed output, so this regen agrees with it by // construction instead of needing a flag told to match. @@ -965,7 +953,7 @@ private static int RunVerifyArgs(string[] rest, IReadOnlyList declar var config = new GenConfig { OutDir = a.OutDir, - Namespace = effectiveNs, + Namespace = a.Namespace ?? meta.Namespace, ColumnNamingStrategy = columnNaming, IncludeNames = declared.Any(g => g.Name == "names"), // The manifest gen keeps: an extra file is classified by what gen recorded writing. diff --git a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs index 1d60595dd7..29711710e4 100644 --- a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs +++ b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs @@ -11,7 +11,6 @@ // SAFETY: the regen is written ONLY under a fresh temp dir; the real OutDir is // read but never written. On any outcome the temp tree is removed. -using System.Text.RegularExpressions; using MetaObjects.Meta; namespace MetaObjects.Codegen; @@ -19,41 +18,6 @@ namespace MetaObjects.Codegen; /// Regenerate-to-temp + diff vs committed output (ADR-0021 D2). public static class CodegenDrift { - // Matches the FIRST top-level namespace declaration in a C# source file: - // file-scoped `namespace X.Y;` or block `namespace X.Y {` / `namespace X.Y`. - // Multiline so `^` anchors each line; we scan line-leading declarations only. - private static readonly Regex NamespaceDecl = - new(@"^\s*namespace\s+([A-Za-z_][\w.]*)\s*;?", RegexOptions.Multiline | RegexOptions.Compiled); - - /// - /// Infer the C# namespace embedded in the committed generated output under - /// . Scans the FIRST .cs file (ordinal-sorted) - /// that carries a top-level namespace declaration and returns it. - /// Returns null when the dir is missing or no .cs file declares a - /// namespace — the caller then falls back to its default (there is nothing - /// meaningful to diff against in that case anyway). Used by - /// verify --codegen so a regen matches output produced with ANY namespace - /// when --namespace is omitted. - /// - public static string? InferNamespace(string outDir) - { - if (!Directory.Exists(outDir)) return null; - - var csFiles = Directory.EnumerateFiles(outDir, "*.cs", SearchOption.AllDirectories) - .OrderBy(p => p.Replace(Path.DirectorySeparatorChar, '/'), StringComparer.Ordinal); - - foreach (var file in csFiles) - { - string text; - try { text = File.ReadAllText(file); } - catch { continue; } - - var m = NamespaceDecl.Match(text); - if (m.Success) return m.Groups[1].Value; - } - return null; - } - /// The outcome of a codegen-drift computation (pure; no console I/O). public sealed record Result { @@ -128,8 +92,16 @@ public static Result Compute(GenConfig config, MetaRoot root, IReadOnlyList temps[roots.FindIndex(r => r.Committed == Path.GetFullPath(realRootFor(g)))]); + CodegenRunner.RunResult regen; + try + { + regen = CodegenRunner.Run(tempConfig, root, generators, + g => temps[roots.FindIndex(r => r.Committed == Path.GetFullPath(realRootFor(g)))]); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return new Result { Clean = false, Error = $"verify --codegen: {ex.Message}" }; + } var overwriteDirs = CodegenRunner.OverwriteDirs(generators, realRootFor, config.OutDir); var drifted = new SortedSet(StringComparer.Ordinal); diff --git a/server/csharp/MetaObjects.Codegen/Generator.cs b/server/csharp/MetaObjects.Codegen/Generator.cs index 853fcb910e..20f8d38c07 100644 --- a/server/csharp/MetaObjects.Codegen/Generator.cs +++ b/server/csharp/MetaObjects.Codegen/Generator.cs @@ -68,8 +68,23 @@ public sealed record GenConfig /// /// public string Baseline { get; init; } = "default"; - /// C# namespace for generated types. - public required string Namespace { get; init; } + /// + /// C# namespace for generated types that no package binding places — the + /// namespace (--namespace, else + /// csharp.namespace in .metaobjects/config.json). Unset is fine until a generated + /// file needs it; reading it then throws naming both + /// settings, so a run that emits nothing never asks for one. + /// + [System.Diagnostics.CodeAnalysis.AllowNull] + public string Namespace + { + get => PackageBindingResolver.ValidNamespace(_namespace ?? throw new InvalidOperationException( + "no C# namespace for the generated code: set \"csharp\": { \"namespace\": \"\" } in " + + ".metaobjects/config.json or pass --namespace .")); + init => _namespace = value; + } + private readonly string? _namespace; + /// /// Strategy applied to field names with no @column override. ADR-0058 — one /// project setting (naming.columns in .metaobjects/config.json), read by diff --git a/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs b/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs index cbf1848f5f..810c3c78e7 100644 --- a/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs +++ b/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs @@ -1842,9 +1842,31 @@ protected static string DefaultInitializer(MetaField field, string baseType) System.Globalization.CultureInfo.InvariantCulture, out _) ? s : null; } - // Escapes a regex pattern for embedding in a C# string-literal attribute argument. - protected static string EscapeAttrString(string s) => - s.Replace("\\", "\\\\").Replace("\"", "\\\""); + // Escapes a regex pattern or default for embedding in a C# string-literal attribute argument. + // A raw newline, any other control character, or a Unicode line separator would end the + // literal and emit source that does not compile. + protected static string EscapeAttrString(string s) + { + var sb = new System.Text.StringBuilder(s.Length + 8); + foreach (var c in s) + { + switch (c) + { + case '\\': sb.Append("\\\\"); break; + case '"': sb.Append("\\\""); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (char.IsControl(c) || c is '\u2028' or '\u2029') + sb.Append("\\u").Append(((int)c).ToString("x4", System.Globalization.CultureInfo.InvariantCulture)); + else + sb.Append(c); + break; + } + } + return sb.ToString(); + } // An owned-type navigation property for an object-typed field. Returns null (with // a warning) when the @objectRef can't be resolved. diff --git a/server/csharp/MetaObjects.Codegen/Generators/Fr019SharedEnum.cs b/server/csharp/MetaObjects.Codegen/Generators/Fr019SharedEnum.cs index 97106df981..bfe6ca73df 100644 --- a/server/csharp/MetaObjects.Codegen/Generators/Fr019SharedEnum.cs +++ b/server/csharp/MetaObjects.Codegen/Generators/Fr019SharedEnum.cs @@ -134,6 +134,6 @@ public static string SharedEnumTypeReference(SharedEnum shared, GenConfig config $"@provided but no C# namespace is configured to reference it from. Set " + $"Convention.ProvidedEnumPrepend, map its package in PackageNamespaces, " + $"or set the legacy ProvidedEnumNamespace fallback."); - return $"{ns}.{shared.Name}"; + return $"{PackageBindingResolver.ValidNamespace(ns, shared.Package)}.{shared.Name}"; } } diff --git a/server/csharp/MetaObjects.Codegen/PackageBindingResolver.cs b/server/csharp/MetaObjects.Codegen/PackageBindingResolver.cs index d2b394b76e..485f78af2b 100644 --- a/server/csharp/MetaObjects.Codegen/PackageBindingResolver.cs +++ b/server/csharp/MetaObjects.Codegen/PackageBindingResolver.cs @@ -88,7 +88,7 @@ public static string Resolve( string? package, string? typeName = null, string? fallbackContext = null) - => ResolveImpl(config, package, typeName, useProvidedEnumPrepend: false, fallbackContext); + => ValidNamespace(ResolveImpl(config, package, typeName, useProvidedEnumPrepend: false, fallbackContext), package); /// Resolve for an abstract field.enum with @provided: true /// (FR-019 shared-enum reference case). Layer order is the same as , @@ -99,7 +99,7 @@ public static string ResolveForProvidedEnum( string? package, string? typeName = null, string? fallbackContext = null) - => ResolveImpl(config, package, typeName, useProvidedEnumPrepend: true, fallbackContext); + => ValidNamespace(ResolveImpl(config, package, typeName, useProvidedEnumPrepend: true, fallbackContext), package); /// Resolve via the layered map (TypeOverrides → PackageNamespaces → Convention) /// WITHOUT applying the unmapped-strategy fallback. Returns null when none of the layers @@ -111,6 +111,44 @@ public static string ResolveForProvidedEnum( string? package, string? typeName = null, bool useProvidedEnumPrepend = false) + => TryResolveUnchecked(config, package, typeName, useProvidedEnumPrepend) is { } ns + ? ValidNamespace(ns, package) + : null; + + /// when it is a dotted path of C# identifiers (Acme.Shop); + /// otherwise an naming the bad part and, when given, the + /// metadata it came from, so a namespace the compiler would reject + /// fails generation instead of the adopter's build. Every namespace a binding layer or the flat + /// fallback produces passes through here once, at the resolver. + public static string ValidNamespace(string ns, string? package = null) + { + var bad = ns.Split('.').FirstOrDefault(s => + s.Length == 0 || !(char.IsLetter(s[0]) || s[0] == '_') || !s.All(c => char.IsLetterOrDigit(c) || c == '_') + || ReservedKeywords.Contains(s)); + if (bad is null) return ns; + var from = string.IsNullOrEmpty(package) ? "" : $" (from metadata package \"{package}\")"; + var why = ReservedKeywords.Contains(bad) ? " (a reserved C# keyword)" : ""; + throw new InvalidOperationException( + $"\"{ns}\" is not a C# namespace{from}: \"{bad}\" is not an identifier{why}."); + } + + private static readonly HashSet ReservedKeywords = new(StringComparer.Ordinal) + { + "abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", "class", "const", + "continue", "decimal", "default", "delegate", "do", "double", "else", "enum", "event", "explicit", + "extern", "false", "finally", "fixed", "float", "for", "foreach", "goto", "if", "implicit", "in", "int", + "interface", "internal", "is", "lock", "long", "namespace", "new", "null", "object", "operator", "out", + "override", "params", "private", "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", + "short", "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try", + "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "virtual", "void", "volatile", + "while", + }; + + private static string? TryResolveUnchecked( + GenConfig config, + string? package, + string? typeName, + bool useProvidedEnumPrepend) { // 1. Type-level override — wins absolutely for both paths. if (!string.IsNullOrEmpty(package) && !string.IsNullOrEmpty(typeName)) diff --git a/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs b/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs index f4d49fddcb..3cdeb1818f 100644 --- a/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs +++ b/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs @@ -20,7 +20,15 @@ // KIND — matching TypeScript byte for byte rather than inventing a taxonomy the other // ports would then have to follow. // -// WHY TEXT AND NOT A TYPED DRIVER CHECK: this assembly must not take a dependency on any +// STRUCTURED CODES FIRST: when the chain holds a DbException that reports a SQLSTATE +// (DbException.SqlState, which Npgsql fills), or an exception exposing a SQLite extended +// result code (Microsoft.Data.Sqlite's SqliteExtendedErrorCode, read by property name so no +// provider is referenced), that code is the verdict and message text is never consulted: +// a unique-violation whose offending value happens to read "NOT NULL" is still a unique +// violation. Message matching survives only as the fallback for exceptions that carry no +// code at all, such as an in-memory double. +// +// WHY NOT A TYPED DRIVER CHECK: this assembly must not take a dependency on any // specific ADO.NET provider — a consumer may be on Npgsql, SQL Server, SQLite or an // in-memory double, and binding to one of them here would make the generated routes // unusable on the others. Npgsql puts the SQLSTATE at the front of PostgresException's @@ -79,6 +87,11 @@ public static class ConstraintErrors /// public static ConstraintFailure? Classify(Exception? error) { + var coded = CodedVerdict(error, out var hasCode); + // A code that is not one of the four constraint classes (a serialization failure, + // say) is NOT a verdict: fall through to the message text, as for a code-less double. + if (hasCode && coded is not null) return Failure(coded.Value); + var haystack = ChainText(error); if (haystack.Length == 0) return null; @@ -87,8 +100,71 @@ public static class ConstraintErrors // 409 for referential/uniqueness conflicts with EXISTING state; 400 for a value // the request itself got wrong. Both are client errors — neither is a 500. + return Failure(kind.Value); + } + + private static ConstraintFailure Failure(ConstraintKind kind) + { var status = kind is ConstraintKind.ForeignKey or ConstraintKind.Unique ? 409 : 400; - return new ConstraintFailure(status, kind.Value); + return new ConstraintFailure(status, kind); + } + + private const int MaxDepth = 8; + + // SQLite extended result codes (https://www.sqlite.org/rescode.html). + private const int SqliteConstraintCheck = 275; + private const int SqliteConstraintForeignKey = 787; + private const int SqliteConstraintNotNull = 1299; + private const int SqliteConstraintPrimaryKey = 1555; + private const int SqliteConstraintUnique = 2067; + + /// + /// The verdict of the first exception in the chain that carries a structured code: + /// says whether one was found, and the return is its kind (null + /// for a code that is not one of the four constraint classes, which callers treat as no verdict). + /// + // The SQLite code is read by property name on purpose (no provider reference). The + // property is public on the provider's exception type, which the application roots + // by using it; a trimmer that removed it would only lose this fast path, and the + // message-text fallback below still classifies the violation. + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage( + "Trimming", "IL2075", Justification = "Optional provider property; message-text fallback covers its absence.")] + private static ConstraintKind? CodedVerdict(Exception? error, out bool hasCode) + { + var seen = 0; + for (var e = error; e is not null && seen < MaxDepth; e = e.InnerException, seen++) + { + foreach (var candidate in e is AggregateException agg ? agg.InnerExceptions.Prepend(e) : new[] { e }) + { + if (candidate is System.Data.Common.DbException { SqlState: { Length: > 0 } state }) + { + hasCode = true; + return state switch + { + "23503" => ConstraintKind.ForeignKey, + "23505" => ConstraintKind.Unique, + "23514" => ConstraintKind.Check, + "23502" => ConstraintKind.NotNull, + _ => null, + }; + } + var extended = candidate.GetType().GetProperty("SqliteExtendedErrorCode")?.GetValue(candidate); + if (extended is int code) + { + hasCode = true; + return code switch + { + SqliteConstraintForeignKey => ConstraintKind.ForeignKey, + SqliteConstraintUnique or SqliteConstraintPrimaryKey => ConstraintKind.Unique, + SqliteConstraintCheck => ConstraintKind.Check, + SqliteConstraintNotNull => ConstraintKind.NotNull, + _ => null, + }; + } + } + } + hasCode = false; + return null; } private static ConstraintKind? KindOf(string haystack) @@ -132,9 +208,8 @@ private static bool Contains(string haystack, string needle) => /// private static string ChainText(Exception? error) { - const int maxDepth = 8; - var parts = new List(maxDepth); - for (var e = error; e is not null && parts.Count < maxDepth; e = e.InnerException) + var parts = new List(MaxDepth); + for (var e = error; e is not null && parts.Count < MaxDepth; e = e.InnerException) { if (!string.IsNullOrEmpty(e.Message)) parts.Add(e.Message); // An AggregateException holds its failures beside InnerException, so the chain diff --git a/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs b/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs index 81bc7ffe1a..de439c6e53 100644 --- a/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs @@ -46,7 +46,7 @@ public void Discovers_all_extract_conformance_cases() { // FR-011: lock the corpus size so a deleted fixture fails CI rather than // silently reducing coverage. Mirrors the TS / Java / Python count guards. - Assert.Equal(48, Cases().Count()); + Assert.Equal(58, Cases().Count()); } [Theory] diff --git a/server/csharp/MetaObjects.Render.Tests/Extract/JsonForgivingReaderTests.cs b/server/csharp/MetaObjects.Render.Tests/Extract/JsonForgivingReaderTests.cs index 324d09f0b0..c5be090670 100644 --- a/server/csharp/MetaObjects.Render.Tests/Extract/JsonForgivingReaderTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/Extract/JsonForgivingReaderTests.cs @@ -113,4 +113,11 @@ public void EmptyValueThenMoreKeysContinues() Assert.Same(JsonForgivingReader.Truncated, m["a"]); Assert.Equal("2", m["b"]); } + + [Fact] + public void FullJsonEscapeGrammar() + { + var m = Read("{\"a\":\"caf\\u00e9 \\ud83d\\ude00 a\\/b\\bx\\fy \\\"q\\\" \\\\ \\u12\"}"); + Assert.Equal("caf\u00e9 \ud83d\ude00 a/b\bx\fy \"q\" \\ u12", m["a"]); + } } diff --git a/server/csharp/MetaObjects.Render.Tests/Extract/StripTests.cs b/server/csharp/MetaObjects.Render.Tests/Extract/StripTests.cs index c43b40f147..d7ed7095fe 100644 --- a/server/csharp/MetaObjects.Render.Tests/Extract/StripTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/Extract/StripTests.cs @@ -43,4 +43,30 @@ public void NullSafe() { Assert.Equal("", Strip.Apply(null)); } + + [Fact] + public void BackticksInsideAJsonStringDoNotCloseTheFence() + { + Assert.Equal("{\"summary\": \"run ```make``` first\", \"score\": 7}", + Strip.Apply("```json\n{\"summary\": \"run ```make``` first\", \"score\": 7}\n```")); + } + + [Fact] + public void LongerOpeningFenceClosesOnlyOnAnEqualOrLongerRun() + { + Assert.Equal("before\n```\ninner\n```\nafter\ntail", + Strip.Apply("````md\nbefore\n```\ninner\n```\nafter\n````\ntail")); + } + + [Fact] + public void ClosingFenceOnThePayloadLineIsAcceptedAtTheEnd() + { + Assert.Equal("{\"a\":1}", Strip.Apply("```json\n{\"a\":1}```")); + } + + [Fact] + public void UnclosedFenceIsNotAFence() + { + Assert.Equal("```json\n{\"a\":1}", Strip.Apply("```json\n{\"a\":1}")); + } } diff --git a/server/csharp/MetaObjects.Render.Tests/RenderTests.cs b/server/csharp/MetaObjects.Render.Tests/RenderTests.cs index a4a82393f7..ddc7239ea7 100644 --- a/server/csharp/MetaObjects.Render.Tests/RenderTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/RenderTests.cs @@ -40,6 +40,16 @@ public void Resolves_a_template_by_2layer_ref() => Assert.Equal("Hi Ada.", Render("text", Obj(("name", "Ada")), P(new() { ["g/main"] = "Hi {{name}}." }), @ref: "g/main")); + [Fact] + public void A_delimiter_change_in_a_root_template_resolved_by_ref_with_no_partials_renders() => + Assert.Equal("Hi Ada.", Render("text", Obj(("name", "Ada")), + P(new() { ["g/main"] = "{{=<% %>=}}Hi <% name %>." }), @ref: "g/main")); + + [Fact] + public void A_delimiter_change_in_a_partial_body_is_refused() => + Assert.Throws(() => Render("text", Obj(("name", "Ada")), + P(new() { ["g/main"] = "{{> g/leaf }} {{name}}", ["g/leaf"] = "{{=<% %>=}}hi" }), @ref: "g/main")); + [Fact] public void Iterates_arrays_in_order() => Assert.Equal("a,b,c,", Render("text", Obj(("xs", new List { "a", "b", "c" })), P(), diff --git a/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs b/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs index 22a0f04d23..9526cb4407 100644 --- a/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs @@ -25,6 +25,7 @@ public class VerifyConformanceTests Verify.ERR_PARTIAL_UNRESOLVED, Verify.ERR_REQUIRED_SLOT_UNUSED, Verify.ERR_OUTPUT_TAG_MISSING, + Verify.ERR_TEMPLATE_SYNTAX, }; private static string CorpusRoot() diff --git a/server/csharp/MetaObjects.Render.Tests/VerifyTests.cs b/server/csharp/MetaObjects.Render.Tests/VerifyTests.cs index fbef950463..34e6dc6dbd 100644 --- a/server/csharp/MetaObjects.Render.Tests/VerifyTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/VerifyTests.cs @@ -33,6 +33,9 @@ private static InMemoryProvider P(Dictionary? m = null) => [Fact] public void Known_scalar_var_no_drift() => Assert.Empty(Verify.Check("Hi {{displayName}}.", AuthorBrief)); + [Fact] public void Empty_set_delimiter_tag_is_ignored_not_a_crash() => + Assert.Empty(Verify.Check("{{=}}Hi {{displayName}}.", AuthorBrief)); + [Fact] public void Unknown_var_is_drift_with_path() { var errs = Verify.Check("Hi {{notARealField}}.", AuthorBrief); diff --git a/server/csharp/MetaObjects.Render/Escapers.cs b/server/csharp/MetaObjects.Render/Escapers.cs index 39163ed609..5dcb07ea87 100644 --- a/server/csharp/MetaObjects.Render/Escapers.cs +++ b/server/csharp/MetaObjects.Render/Escapers.cs @@ -34,7 +34,21 @@ private static string Xml(string s) => // OWASP CSV/Excel formula-injection guard: neutralize a leading active char. private static string InjectionGuard(string s) => - s.Length > 0 && "=+-@\t\r".IndexOf(s[0]) >= 0 ? "'" + s : s; + s.Length > 0 && "=+-@\t\r".IndexOf(s[0]) >= 0 && !IsStrictNumeric(s) ? "'" + s : s; + + // ^-?[0-9]+(\.[0-9]+)?$ over ASCII digits, whole string (hand-rolled: .NET `$` accepts a trailing \n). + private static bool IsStrictNumeric(string s) + { + var i = s.Length > 0 && s[0] == '-' ? 1 : 0; + var intStart = i; + while (i < s.Length && s[i] >= '0' && s[i] <= '9') i++; + if (i == intStart) return false; + if (i == s.Length) return true; + if (s[i] != '.') return false; + var fracStart = ++i; + while (i < s.Length && s[i] >= '0' && s[i] <= '9') i++; + return i > fracStart && i == s.Length; + } private static string Csv(string s) { diff --git a/server/csharp/MetaObjects.Render/Extract/JsonForgivingReader.cs b/server/csharp/MetaObjects.Render/Extract/JsonForgivingReader.cs index 8b562dad34..2488f8651b 100644 --- a/server/csharp/MetaObjects.Render/Extract/JsonForgivingReader.cs +++ b/server/csharp/MetaObjects.Render/Extract/JsonForgivingReader.cs @@ -123,25 +123,54 @@ private string ReadString(char quote) { _i++; // opening quote var sb = new System.Text.StringBuilder(); - bool esc = false; while (_i < _s.Length) { char c = _s[_i++]; - if (esc) { sb.Append(Unescape(c)); esc = false; } - else if (c == '\\') esc = true; + if (c == '\\') sb.Append(ReadEscape()); else if (c == quote) return sb.ToString(); else sb.Append(c); } return sb.ToString(); // unterminated string → return what we have } - private static char Unescape(char c) => c switch + /// The escape after a consumed backslash: the full JSON grammar. A \uXXXX escape + /// is one UTF-16 unit, so a surrogate pair is two escapes appended in order. An unknown escape + /// keeps its character and a truncated \u keeps its text, so a cut-off reply still + /// yields what it has. + private string ReadEscape() { - 'n' => '\n', - 't' => '\t', - 'r' => '\r', - _ => c - }; + if (_i >= _s.Length) return ""; + char c = _s[_i++]; + switch (c) + { + case 'n': return "\n"; + case 't': return "\t"; + case 'r': return "\r"; + case 'b': return "\b"; + case 'f': return "\f"; + case 'u': + return TryHex4(out int unit) ? ((char)unit).ToString() : "u"; + default: return c.ToString(); // \" \\ \/ and any unknown escape + } + } + + private bool TryHex4(out int unit) + { + unit = 0; + if (_i + 4 > _s.Length) return false; + for (int k = 0; k < 4; k++) + { + char h = _s[_i + k]; + int d = h is >= '0' and <= '9' ? h - '0' + : h is >= 'a' and <= 'f' ? h - 'a' + 10 + : h is >= 'A' and <= 'F' ? h - 'A' + 10 + : -1; + if (d < 0) return false; + unit = unit * 16 + d; + } + _i += 4; + return true; + } private object? ReadBareScalar() { diff --git a/server/csharp/MetaObjects.Render/Extract/Locate.cs b/server/csharp/MetaObjects.Render/Extract/Locate.cs index 0609f777a5..92fd0b4913 100644 --- a/server/csharp/MetaObjects.Render/Extract/Locate.cs +++ b/server/csharp/MetaObjects.Render/Extract/Locate.cs @@ -104,18 +104,28 @@ private static int ScanBalanced(string s, int open) var flags = caseInsensitive ? RegexOptions.IgnoreCase : RegexOptions.None; + // Comments and CDATA are not searched; the view has the same length as the text. + string view = XmlForgivingReader.MaskNonMarkup(text); var openPattern = new Regex($"<{Regex.Escape(rootName)}(\\s[^>]*)?>", flags); - var openMatch = openPattern.Match(text); + var openMatch = openPattern.Match(view); if (!openMatch.Success) return null; int start = openMatch.Index; int gt = openMatch.Index + openMatch.Length - 1; // position of the '>' closing the open tag var closePattern = new Regex($"", flags); - var closeMatch = closePattern.Match(text, openMatch.Index + openMatch.Length); + var closeMatch = closePattern.Match(view, openMatch.Index + openMatch.Length); // Guard: if close found but its end position <= gt, treat as no-close (shouldn't // normally occur, but protects against malformed input such as a bare close tag). if (closeMatch.Success && closeMatch.Index + closeMatch.Length > gt) { + // A root does not nest in itself: another opener between the first one and its + // close is the real element (the tag was mentioned in prose first). + var next = openPattern.Match(view, openMatch.Index + openMatch.Length); + while (next.Success && next.Index < closeMatch.Index) + { + start = next.Index; + next = openPattern.Match(view, next.Index + next.Length); + } return text[start..(closeMatch.Index + closeMatch.Length)]; } return text[start..]; diff --git a/server/csharp/MetaObjects.Render/Extract/Strip.cs b/server/csharp/MetaObjects.Render/Extract/Strip.cs index 2c42bb9660..45925c3301 100644 --- a/server/csharp/MetaObjects.Render/Extract/Strip.cs +++ b/server/csharp/MetaObjects.Render/Extract/Strip.cs @@ -7,32 +7,94 @@ namespace MetaObjects.Render.Extract; /// public static class Strip { - // Captures the body inside a fenced block; optional language tag (json/xml/etc) is dropped. - private static readonly Regex Fence = new( - @"```[a-zA-Z0-9_\-]*\s*\r?\n(.*?)\r?\n?```", - RegexOptions.Singleline | RegexOptions.Compiled); + // An opening fence: a run of 3+ backticks, an optional language tag (json/xml/etc, dropped), a line end. + private static readonly Regex Open = new( + @"(`{3,})[a-zA-Z0-9_\-]*\s*\r?\n", + RegexOptions.Compiled); + + private readonly record struct Fence(int Start, int End, string Body); /// - /// Removes any markdown code-fence wrapper from . - /// Returns the fence body (with surrounding prose) trimmed, or the trimmed raw input if no fence. - /// Returns "" for null input. + /// Every fenced block, in order. A fence closes at a line that starts with a backtick run at + /// least as long as the opening one and carries nothing else, so backticks inside the payload + /// (a JSON string holding `make` or a nested code sample) do not truncate it. A reply that + /// closes on the payload's own line is accepted when that run ends the text. /// + private static List Fences(string raw) + { + var output = new List(); + int pos = 0; + while (pos <= raw.Length) + { + var m = Open.Match(raw, pos); + if (!m.Success) break; + int bodyStart = m.Index + m.Length; + var close = FindClose(raw, bodyStart, m.Groups[1].Length); + if (close is not var (bodyEnd, end)) + { + pos = bodyStart; + continue; + } + output.Add(new Fence(m.Index, end, raw[bodyStart..bodyEnd])); + pos = end; + } + return output; + } + + private static (int BodyEnd, int End)? FindClose(string raw, int from, int width) + { + for (int ls = from; ls <= raw.Length;) + { + int i = ls; + while (i < raw.Length && (raw[i] == ' ' || raw[i] == '\t') && i - ls < 4) i++; + int j = i; + while (j < raw.Length && raw[j] == '`') j++; + if (j - i >= width) + { + int k = j; + while (k < raw.Length && (raw[k] == ' ' || raw[k] == '\t')) k++; + if (k >= raw.Length || raw[k] == '\n' || raw[k] == '\r') return (BodyEnd(raw, from, ls), j); + } + int nl = raw.IndexOf('\n', ls); + if (nl < 0) break; + ls = nl + 1; + } + // No closing line: accept a final backtick run that ends the text on the payload's own line. + int tail = raw.TrimEnd().Length; + int t = tail; + while (t > from && raw[t - 1] == '`') t--; + return tail - t >= width && t > from ? (t, tail) : null; + } + + private static int BodyEnd(string raw, int from, int lineStart) + { + int e = lineStart; + if (e > from && raw[e - 1] == '\n') e--; + if (e > from && raw[e - 1] == '\r') e--; + return e; + } + /// The body of every fenced block, in order. Locate searches these before the /// whole text, because a model told to fence its answer puts the answer there. public static IReadOnlyList FencedBodies(string? raw) { if (raw == null) return Array.Empty(); - return Fence.Matches(raw).Select(m => m.Groups[1].Value).ToList(); + return Fences(raw).Select(f => f.Body).ToList(); } + /// + /// Removes any markdown code-fence wrapper from . + /// Returns the fence body (with surrounding prose) trimmed, or the trimmed raw input if no fence. + /// Returns "" for null input. + /// public static string Apply(string? raw) { if (raw == null) return ""; - var m = Fence.Match(raw); - if (m.Success) + var fences = Fences(raw); + if (fences.Count > 0) { - int end = m.Index + m.Length; - return (raw[..m.Index] + m.Groups[1].Value + raw[end..]).Trim(); + var f = fences[0]; + return (raw[..f.Start] + f.Body + raw[f.End..]).Trim(); } return raw.Trim(); } diff --git a/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs b/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs index 6f88b9dec3..4d4c6be023 100644 --- a/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs +++ b/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs @@ -34,10 +34,105 @@ public sealed class XmlForgivingReader "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))", RegexOptions.Compiled); - public Dictionary Read(string? span, bool caseInsensitive) + // a character an identifier can end with: '<' right after one is a type argument, not a tag. + private static bool IsIdentChar(char c) => char.IsAsciiLetterOrDigit(c) || c == '_'; + + private const string CommentOpen = ""; + private const string CdataOpen = ""; + // the five predefined XML entities plus numeric character references. + private static readonly Regex EntityRegex = new( + "&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);", RegexOptions.Compiled); + + private enum RegionKind { Comment, Cdata } + + /// + /// Rewrite every XML comment and CDATA section in through + /// , scanning left to right so a comment marker inside CDATA (or + /// the reverse) is never mistaken for markup. An unclosed comment is left as text; an + /// unclosed CDATA section runs to the end. + /// + private static string RewriteRegions(string text, Func replace) + { + var sb = new System.Text.StringBuilder(); + int pos = 0, from = 0; + while (true) + { + int c = text.IndexOf(CommentOpen, from, StringComparison.Ordinal); + int d = text.IndexOf(CdataOpen, from, StringComparison.Ordinal); + bool isComment = c >= 0 && (d < 0 || c < d); + int start = isComment ? c : d; + if (start < 0) break; + int end; + if (isComment) + { + int e = text.IndexOf(CommentClose, start + CommentOpen.Length, StringComparison.Ordinal); + if (e < 0) { from = start + CommentOpen.Length; continue; } + end = e + CommentClose.Length; + } + else + { + int e = text.IndexOf(CdataClose, start + CdataOpen.Length, StringComparison.Ordinal); + end = e < 0 ? text.Length : e + CdataClose.Length; + } + sb.Append(text, pos, start - pos) + .Append(replace(isComment ? RegionKind.Comment : RegionKind.Cdata, text.Substring(start, end - start))); + pos = end; + from = end; + } + return sb.Append(text, pos, text.Length - pos).ToString(); + } + + /// with comments and CDATA sections blanked to spaces (same + /// length, so indices hold): the view the tag locators search, so markup inside them is + /// never located. + internal static string MaskNonMarkup(string text) => + RewriteRegions(text, (_, region) => new string(' ', region.Length)); + + /// Drop comments; turn each CDATA section into its text with & < > + /// entity-escaped, so the tag scanner cannot split it and the entity decode restores it + /// exactly. + private static string Prepare(string text) => RewriteRegions(text, (kind, region) => + { + if (kind == RegionKind.Comment) return ""; + string body = region.EndsWith(CdataClose, StringComparison.Ordinal) + ? region.Substring(CdataOpen.Length, region.Length - CdataOpen.Length - CdataClose.Length) + : region.Substring(CdataOpen.Length); + return body.Replace("&", "&").Replace("<", "<").Replace(">", ">"); + }); + + /// Decode predefined entities and numeric references in one pass (so + /// &amp;lt; is &lt;). An unknown or out-of-range reference stays as + /// written. + private static string DecodeEntities(string s) + { + if (!s.Contains('&')) return s; + return EntityRegex.Replace(s, m => + { + string body = m.Groups[1].Value; + switch (body) + { + case "lt": return "<"; + case "gt": return ">"; + case "amp": return "&"; + case "quot": return "\""; + case "apos": return "'"; + } + bool hex = body[1] == 'x'; + string digits = body.Substring(hex ? 2 : 1); + if (!int.TryParse(digits, hex ? System.Globalization.NumberStyles.HexNumber : System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, out int cp)) return m.Value; + bool surrogate = cp >= 0xD800 && cp <= 0xDFFF; + return cp >= 1 && cp <= 0x10FFFF && !surrogate ? char.ConvertFromUtf32(cp) : m.Value; + }); + } + + public Dictionary Read(string? rawSpan, bool caseInsensitive) { var out_ = new Dictionary(); - if (string.IsNullOrWhiteSpace(span)) return out_; + if (string.IsNullOrWhiteSpace(rawSpan)) return out_; + string span = Prepare(rawSpan); int gt = span.IndexOf('>'); if (gt < 0) return out_; @@ -46,7 +141,7 @@ public sealed class XmlForgivingReader int innerEnd = (rootEnd < 0 || rootEnd <= gt) ? span.Length : rootEnd; string inner = span.Substring(gt + 1, innerEnd - (gt + 1)); - ParseChildren(inner, caseInsensitive, out_, null); + ParseChildren(inner, caseInsensitive, out_, null, false); return out_; } @@ -60,7 +155,7 @@ public sealed class XmlForgivingReader { var out_ = new Dictionary(); if (string.IsNullOrWhiteSpace(text)) return out_; - ParseChildren(text!, caseInsensitive, out_, null); + ParseChildren(Prepare(text!), caseInsensitive, out_, null, false); return out_; } @@ -68,15 +163,32 @@ public sealed class XmlForgivingReader /// Parse 's elements into . When /// is non-null, the text BETWEEN those elements (an element's own /// text in mixed content) is appended to it, one segment per gap. + /// is set for the body of an element that DID close: an + /// open tag in there that directly follows an identifier character and has no close tag + /// anywhere after it (List<int> and ...) is prose, not an element. One after + /// >, whitespace or the start of the body is a child whose close the model forgot. /// - private static void ParseChildren(string inner, bool ci, Dictionary out_, List? loose) + private static void ParseChildren(string inner, bool ci, Dictionary out_, List? loose, + bool literalUnclosed) { - var openTag = new Regex(OpenTagPattern, ci ? RegexOptions.IgnoreCase : RegexOptions.None); + var flags = ci ? RegexOptions.IgnoreCase : RegexOptions.None; + var openTag = new Regex(OpenTagPattern, flags); + var anyClose = new Regex(CloseTagPattern, flags); int pos = 0; - Match m = openTag.Match(inner, pos); + int scan = 0; + Match m = openTag.Match(inner, scan); while (m.Success) { + if (literalUnclosed && m.Index > 0 && IsIdentChar(inner[m.Index - 1]) + && !m.Groups[2].Value.TrimEnd().EndsWith('/') + && !anyClose.Match(inner, m.Index + m.Length).Success) + { + scan = m.Index + m.Length; + if (scan >= inner.Length) break; + m = openTag.Match(inner, scan); + continue; + } loose?.Add(inner.Substring(pos, m.Index - pos)); string tag = m.Groups[1].Value; @@ -91,6 +203,7 @@ private static void ParseChildren(string inner, bool ci, Dictionary= inner.Length) break; m = openTag.Match(inner, pos); continue; @@ -152,8 +265,9 @@ private static void ParseChildren(string inner, bool ci, Dictionary= inner.Length) break; m = openTag.Match(inner, pos); } @@ -188,13 +302,13 @@ private static (int Start, int End)? FindStrayClose(string inner, int from, int /// content keeps BOTH: the children, and the element's own text under , /// so a scalar or @xmlText consumer still reads the prose around a child element. /// - private static object? Combine(Dictionary attrs, string content, bool ci) + private static object? Combine(Dictionary attrs, string content, bool ci, bool closed) { if (content.Contains('<')) { var nested = new Dictionary(); var loose = new List(); - ParseChildren(content, ci, nested, loose); + ParseChildren(content, ci, nested, loose, closed); if (nested.Count > 0) { // attributes first; a child element wins a name collision @@ -216,7 +330,7 @@ private static string MixedText(List segments, bool ci) var parts = new List(); foreach (string seg in segments) { - string t = closeRegex.Replace(seg, "").Trim(); + string t = DecodeEntities(closeRegex.Replace(seg, "").Trim()); if (t.Length > 0) parts.Add(t); } return string.Join(" ", parts); @@ -224,7 +338,7 @@ private static string MixedText(List segments, bool ci) private static object? TextValue(Dictionary attrs, string content) { - string text = content.Trim(); + string text = DecodeEntities(content.Trim()); if (attrs.Count == 0) return text; var m = new Dictionary(attrs) { [TextKey] = text }; return m; @@ -240,7 +354,7 @@ private static string MixedText(List segments, bool ci) string val = a.Groups[2].Success ? a.Groups[2].Value : a.Groups[3].Success ? a.Groups[3].Value : a.Groups[4].Success ? a.Groups[4].Value : ""; - if (!attrs.ContainsKey(name)) attrs[name] = val; + if (!attrs.ContainsKey(name)) attrs[name] = DecodeEntities(val); } return attrs; } diff --git a/server/csharp/MetaObjects.Render/MustacheTags.cs b/server/csharp/MetaObjects.Render/MustacheTags.cs new file mode 100644 index 0000000000..c9933ceeb7 --- /dev/null +++ b/server/csharp/MetaObjects.Render/MustacheTags.cs @@ -0,0 +1,124 @@ +// One Mustache tag scanner shared by Verify and the renderer's partial expansion. +// +// Both used to find tags on their own (Verify with a "{{" search, Renderer with a +// "{{> x}}" regex), and neither knew what the engine knows: a comment ends at the +// first closing delimiter, "{{=<% %>=}}" changes the delimiters, and a section close +// must name the section it closes. Scan follows the Mustache spec for all three, so the +// two callers cannot disagree about where a tag is. Mirrors Mustache.parse in the +// TypeScript port and the Java and Python scanners. + +namespace MetaObjects.Render; + +/// One tag: its kind, trimmed name, and [Start, End) span in the text. +internal sealed record MustacheTag(MustacheTagKind Kind, string Value, int Start, int End); + +internal enum MustacheTagKind { Name, Triple, Amp, Section, Inverted, Close, Partial, Comment, Delimiters } + +/// An unbalanced section; is the section the finding is about. +internal sealed class MustacheSyntaxException(string message, string section) : Exception(message) +{ + public string Section { get; } = section; +} + +internal static class MustacheTags +{ + private const string DefaultOpen = "{{"; + private const string DefaultClose = "}}"; + + /// + /// Every tag in , in order, honouring comments and delimiter changes. + /// Throws for a close with no open, a close naming a + /// different section than the one open, or a section never closed. An unterminated tag ends + /// the scan (the rest is text), as it always did here. + /// + public static List Scan(string text) + { + var tags = new List(); + string open = DefaultOpen, close = DefaultClose; + var openSections = new Stack(); + int pos = 0; + while (pos < text.Length) + { + int start = text.IndexOf(open, pos, StringComparison.Ordinal); + if (start < 0) break; + int contentStart = start + open.Length; + bool triple = contentStart < text.Length && text[contentStart] == '{'; + string endDelim = triple ? "}" + close : close; + if (triple) contentStart++; + int closeIdx = text.IndexOf(endDelim, contentStart, StringComparison.Ordinal); + if (closeIdx < 0) break; + string content = text[contentStart..closeIdx].Trim(); + int end = closeIdx + endDelim.Length; + pos = end; + + if (triple) { tags.Add(new(MustacheTagKind.Triple, content, start, end)); continue; } + if (content.Length == 0) continue; + char sigil = content[0]; + string name = content.Length > 1 ? content[1..].Trim() : ""; + switch (sigil) + { + case '!': + tags.Add(new(MustacheTagKind.Comment, name, start, end)); + break; + case '=': + if (content.Length > 1 && content.EndsWith('=')) + { + var parts = content[1..^1].Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) { open = parts[0]; close = parts[1]; } + } + tags.Add(new(MustacheTagKind.Delimiters, name, start, end)); + break; + case '#': + case '^': + openSections.Push(name); + tags.Add(new(sigil == '#' ? MustacheTagKind.Section : MustacheTagKind.Inverted, name, start, end)); + break; + case '/': + if (openSections.Count == 0) + throw new MustacheSyntaxException($"Unopened section \"{name}\"", name); + string opened = openSections.Pop(); + if (opened != name) + throw new MustacheSyntaxException($"Unclosed section \"{opened}\"", opened); + tags.Add(new(MustacheTagKind.Close, name, start, end)); + break; + case '>': + tags.Add(new(MustacheTagKind.Partial, name, start, end)); + break; + case '&': + tags.Add(new(MustacheTagKind.Amp, name, start, end)); + break; + default: + tags.Add(new(MustacheTagKind.Name, content, start, end)); + break; + } + } + if (openSections.Count > 0) + { + string opened = openSections.Peek(); + throw new MustacheSyntaxException($"Unclosed section \"{opened}\"", opened); + } + return tags; + } + + /// Where a text that changes delimiters cannot render, or null when it can: + /// inlined text is parsed under the surrounding delimiters. A partial body + /// ( is its ref) that changes them is reported at that ref; a text + /// that changes them and inlines a partial is reported at its first partial. + public static string? DelimiterConflict(IReadOnlyList tags, string? inlinedAs) => + !tags.Any(t => t.Kind == MustacheTagKind.Delimiters) + ? null + : inlinedAs ?? tags.FirstOrDefault(t => t.Kind == MustacheTagKind.Partial)?.Value; + + /// with every comment cut out: what a tag-presence check may scan. + public static string WithoutComments(string text, IReadOnlyList tags) + { + var sb = new System.Text.StringBuilder(); + int pos = 0; + foreach (var tag in tags.Where(t => t.Kind == MustacheTagKind.Comment)) + { + sb.Append(text, pos, tag.Start - pos); + pos = tag.End; + } + return sb.Append(text, pos, text.Length - pos).ToString(); + } +} diff --git a/server/csharp/MetaObjects.Render/Renderer.cs b/server/csharp/MetaObjects.Render/Renderer.cs index 42d4bf0b73..5423aa6f07 100644 --- a/server/csharp/MetaObjects.Render/Renderer.cs +++ b/server/csharp/MetaObjects.Render/Renderer.cs @@ -1,14 +1,13 @@ // Deterministic, logic-less render: (template + payload + provider) -> string. // // Ported from typescript/packages/render/src/render.ts. Partials are inlined by -// regex BEFORE the Mustache pass (deterministic whitespace + path-based cycle/ +// tag scan BEFORE the Mustache pass (deterministic whitespace + path-based cycle/ // depth detection, independent of any lib partial loader); escaping is engine- // owned (the format escaper is installed as Stubble's encoding function, the // equivalent of overriding mustache.js's Mustache.escape). `{{{x}}}` bypasses it. // // The render-time `verify` guard (FR-004 Plan #3) is intentionally not wired yet. -using System.Text.RegularExpressions; using Stubble.Core.Builders; using Stubble.Core.Settings; @@ -45,21 +44,28 @@ public sealed record RenderRequest } /// The logic-less, deterministic MetaObjects render engine. -public static partial class Renderer +public static class Renderer { private const int MaxDepth = 32; - [GeneratedRegex(@"\{\{>\s*([^}\s]+)\s*\}\}")] - private static partial Regex PartialPattern(); - // Resolve `{{> group/source }}` partials by inlining their text, recursively, // BEFORE Mustache parses — giving deterministic whitespace and true, path-based - // cycle/depth detection. - private static string Expand(string text, IProvider provider, IReadOnlyList path) + // cycle/depth detection. Partial tags are located by the shared scanner + // (MustacheTags), so one written inside a comment is not a partial. + private static string Expand(string text, IProvider provider, IReadOnlyList path, string? inlinedAs = null) { - return PartialPattern().Replace(text, match => + List tags; + try { tags = MustacheTags.Scan(text); } + catch (MustacheSyntaxException e) { throw new RenderException($"template syntax error: {e.Message}"); } + if (MustacheTags.DelimiterConflict(tags, inlinedAs) is not null) + throw new RenderException("a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + var partials = tags.Where(t => t.Kind == MustacheTagKind.Partial).ToList(); + if (partials.Count == 0) return text; + var sb = new System.Text.StringBuilder(); + int pos = 0; + foreach (var tag in partials) { - string reference = match.Groups[1].Value; + string reference = tag.Value; if (path.Contains(reference)) throw new RenderException($"partial cycle: {string.Join(" -> ", path.Append(reference))}"); if (path.Count >= MaxDepth) @@ -68,8 +74,10 @@ private static string Expand(string text, IProvider provider, IReadOnlyList(path) { reference }; - return Expand(t, provider, next); - }); + sb.Append(text, pos, tag.Start - pos).Append(Expand(t, provider, next, reference)); + pos = tag.End; + } + return sb.Append(text, pos, text.Length - pos).ToString(); } /// Deterministic, logic-less render of a template against a payload. diff --git a/server/csharp/MetaObjects.Render/Verify.cs b/server/csharp/MetaObjects.Render/Verify.cs index ffa1d399db..8b6d9e8b76 100644 --- a/server/csharp/MetaObjects.Render/Verify.cs +++ b/server/csharp/MetaObjects.Render/Verify.cs @@ -54,6 +54,9 @@ public static class Verify /// A declared @requiredTags output tag is absent from the template text. public const string ERR_OUTPUT_TAG_MISSING = "ERR_OUTPUT_TAG_MISSING"; + /// The template text does not parse: an unbalanced Mustache section. path = the section's name. + public const string ERR_TEMPLATE_SYNTAX = "ERR_TEMPLATE_SYNTAX"; + private const int MaxDepth = 32; // --- Mustache tag token model (rendering-agnostic; only tag structure) --- @@ -62,61 +65,50 @@ private sealed record VarTok(string Value) : Tok; private sealed record SectionTok(string Value, bool Inverted, IReadOnlyList Children) : Tok; private sealed record PartialTok(string Value) : Tok; + // The tag tree of `text`, built from the shared scanner (MustacheTags). Throws + // MustacheSyntaxException on an unbalanced section. private static List Tokenize(string text) { - int pos = 0; - return ParseTokens(text, ref pos, null); - } - - // Parse tokens until EOF or a matching {{/closeName}} (which is consumed). - private static List ParseTokens(string text, ref int pos, string? closeName) - { - var tokens = new List(); - while (pos < text.Length) + var root = new List(); + var levels = new Stack>(); + levels.Push(root); + foreach (var tag in MustacheTags.Scan(text)) { - int open = text.IndexOf("{{", pos, StringComparison.Ordinal); - if (open < 0) break; // remainder is plain text - bool triple = open + 2 < text.Length && text[open + 2] == '{'; - string closeDelim = triple ? "}}}" : "}}"; - int contentStart = open + (triple ? 3 : 2); - int close = text.IndexOf(closeDelim, contentStart, StringComparison.Ordinal); - if (close < 0) break; // malformed; stop - string content = text[contentStart..close].Trim(); - pos = close + closeDelim.Length; - - if (triple) + switch (tag.Kind) { - tokens.Add(new VarTok(content)); - continue; - } - if (content.Length == 0) continue; - - char sigil = content[0]; - string name = content.Length > 1 ? content[1..].Trim() : ""; - switch (sigil) - { - case '!': break; // comment - case '#': - case '^': - var children = ParseTokens(text, ref pos, name); - tokens.Add(new SectionTok(name, sigil == '^', children)); + case MustacheTagKind.Section: + case MustacheTagKind.Inverted: + var children = new List(); + levels.Peek().Add(new SectionTok(tag.Value, tag.Kind == MustacheTagKind.Inverted, children)); + levels.Push(children); break; - case '/': - return tokens; // close (matched or mismatched) ends this level - case '>': - tokens.Add(new PartialTok(name)); + case MustacheTagKind.Close: + levels.Pop(); // the scanner already matched it to its open break; - case '&': - tokens.Add(new VarTok(name)); + case MustacheTagKind.Partial: + levels.Peek().Add(new PartialTok(tag.Value)); break; - default: - tokens.Add(new VarTok(content)); // {{x}}, {{x.y}}, {{.}} + case MustacheTagKind.Name: + case MustacheTagKind.Triple: + case MustacheTagKind.Amp: + levels.Peek().Add(new VarTok(tag.Value)); // {{x}}, {{x.y}}, {{.}}, {{{x}}}, {{&x}} break; + // comments and delimiter changes carry no reference } } - return tokens; + return root; } + /// A delimiter change the renderer refuses (). + private static void ReportDelimiterConflict(string text, string? inlinedAs, List errors) + { + if (MustacheTags.DelimiterConflict(MustacheTags.Scan(text), inlinedAs) is { } path) + errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, path)); + } + + // `text` minus its comments, for the output-tag scan. + private static string StaticText(string text) => MustacheTags.WithoutComments(text, MustacheTags.Scan(text)); + private static PayloadField? Find(IReadOnlyList fields, string name) => fields.FirstOrDefault(f => f.Name == name); @@ -234,14 +226,30 @@ void Walk(IReadOnlyList tokens, List> stack, Li errors.Add(new VerifyError(ERR_PARTIAL_UNRESOLVED, p.Value)); break; } - staticTexts.Add(text); - Walk(Tokenize(text), stack, new List(seen) { p.Value }, inElement); + List partialToks; + try { partialToks = Tokenize(text); } + catch (MustacheSyntaxException e) + { + errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)); + break; + } + ReportDelimiterConflict(text, p.Value, errors); + staticTexts.Add(StaticText(text)); + Walk(partialToks, stack, new List(seen) { p.Value }, inElement); break; } } } - Walk(Tokenize(templateText), [root], []); + List bodyToks; + try { bodyToks = Tokenize(templateText); } + catch (MustacheSyntaxException e) + { + return [new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)]; + } + ReportDelimiterConflict(templateText, null, errors); + staticTexts[0] = StaticText(templateText); + Walk(bodyToks, [root], []); foreach (var slot in options?.RequiredSlots ?? []) if (!referencedAtRoot.Contains(slot)) diff --git a/server/csharp/MetaObjects/Config/MetadataLocation.cs b/server/csharp/MetaObjects/Config/MetadataLocation.cs index 0b28c0d008..a86dcbe822 100644 --- a/server/csharp/MetaObjects/Config/MetadataLocation.cs +++ b/server/csharp/MetaObjects/Config/MetadataLocation.cs @@ -42,7 +42,7 @@ namespace MetaObjects.Config; /// public sealed record ResolvedMetadata( string Directory, IReadOnlyList? Files, IReadOnlyList Libraries, PhysicalNaming Naming, - WireFormat? Wire = null, string? OutDir = null, ImportScope? Imports = null) + WireFormat? Wire = null, string? OutDir = null, ImportScope? Imports = null, string? Namespace = null) { /// Load this metadata, with the project's libraries, and set the loaded /// root's from (ADR-0058) and its @@ -117,7 +117,7 @@ public static ResolvedMetadata Resolve(string? metadataDir, string cwd) // No config found means no project to read a lock from: nothing is imported. var explicitImports = explicitCfg is null ? ImportScope.None : ImportScope.Resolve(explicitCfg.ConfigDir, explicitCfg); return new ResolvedMetadata( - metadataDir, null, explicitLibs, explicitNaming, explicitCfg?.Wire, explicitCfg?.OutDir, explicitImports); + metadataDir, null, explicitLibs, explicitNaming, explicitCfg?.Wire, explicitCfg?.OutDir, explicitImports, explicitCfg?.Namespace); } var cfg = NeutralConfig.Read(cwd); @@ -132,7 +132,7 @@ public static ResolvedMetadata Resolve(string? metadataDir, string cwd) // No declared sources: resolve the DEFAULT directory through the same ladder // the shared conformance corpus gates (ERR_COLLECTION_NOT_FOUND when absent). var defaultFiles = SourceResolver.ResolveCollection(cwd); - return new ResolvedMetadata(Path.Combine(cwd, NeutralConfig.DefaultMetadataDir), defaultFiles, libraries, naming, wire, cfg?.OutDir, imports); + return new ResolvedMetadata(Path.Combine(cwd, NeutralConfig.DefaultMetadataDir), defaultFiles, libraries, naming, wire, cfg?.OutDir, imports, cfg?.Namespace); } if (specs.Count > 1) @@ -162,6 +162,6 @@ public static ResolvedMetadata Resolve(string? metadataDir, string cwd) "\"sources\" at the file's containing directory."); } - return new ResolvedMetadata(resolved, files, libraries, naming, wire, cfg?.OutDir, imports); + return new ResolvedMetadata(resolved, files, libraries, naming, wire, cfg?.OutDir, imports, cfg?.Namespace); } } diff --git a/server/csharp/MetaObjects/Config/NeutralConfig.cs b/server/csharp/MetaObjects/Config/NeutralConfig.cs index e6e98ea1d3..4ec962d0e3 100644 --- a/server/csharp/MetaObjects/Config/NeutralConfig.cs +++ b/server/csharp/MetaObjects/Config/NeutralConfig.cs @@ -25,7 +25,8 @@ // or a non-object `naming` is refused (ERR_BAD_ATTR_VALUE), the same "present but // wrong-shaped raises" rule `sources`/`libraries` already follow. // -// `csharp.outDir` is the C#-scoped generated-code directory; the other ports ignore the block. +// `csharp.outDir` is the C#-scoped generated-code directory and `csharp.namespace` the C# namespace +// generated code lands in; the other ports ignore the block. // // `wire` (ADR-0019 Amendment 2) is read here too — the output form of a generated REST surface // (`decimals`, `nullMembers`), refused exactly as `naming` is. @@ -91,9 +92,15 @@ public sealed class NeutralConfig /// directory. Null when the key is absent. `--out` on the command line wins over it. public string? OutDir { get; } + /// The C# namespace generated code lands in (`csharp.namespace` in config.json), or null when + /// the key is absent. `--namespace` on the command line wins over it. It is the namespace for + /// generated types no package binding places (GenConfig.Namespace); with neither, a run that + /// emits such a type stops and asks for one. + public string? Namespace { get; } + private NeutralConfig( IReadOnlyList> sources, IReadOnlyList libraries, - PhysicalNaming naming, WireFormat wire, string? outDir, + PhysicalNaming naming, WireFormat wire, (string? OutDir, string? Namespace) csharp, IReadOnlyList dependencies, IReadOnlyList scopeInclude, IReadOnlyList? migrateScope, string configDir) { @@ -101,7 +108,8 @@ private NeutralConfig( ScopeInclude = scopeInclude; MigrateScope = migrateScope; ConfigDir = configDir; - OutDir = outDir; + OutDir = csharp.OutDir; + Namespace = csharp.Namespace; Sources = sources; Libraries = libraries; Naming = naming; @@ -263,7 +271,7 @@ private NeutralConfig( // Unknown top-level keys, and every key of `scope` / `migrate` but the two pattern // lists read above, are IGNORED by design — see the file header. return new NeutralConfig(specs, libraries, new PhysicalNaming(columns, tables), ReadWire(root, path), - ReadOutDir(root, path, configDir), dependencies, scopeInclude, migrateScope, + ReadCSharp(root, path, configDir), dependencies, scopeInclude, migrateScope, Path.GetFullPath(configDir)); } } @@ -299,29 +307,32 @@ private static IReadOnlyList ReadDependencies(JsonElement root, return list.EnumerateArray().Select(v => v.GetString()!).ToList(); } - /// The `csharp` block: the one C#-scoped setting, `outDir`, where `dotnet meta gen` writes - /// and `dotnet meta verify` diffs. The other ports ignore the block, as they ignore any key - /// they do not own. An unknown key, a non-string or empty `outDir`, or a non-object block is - /// refused, the same rule as `naming`. - private static string? ReadOutDir(JsonElement root, string path, string configDir) + /// The `csharp` block: the C#-scoped settings `outDir`, where `dotnet meta gen` writes and + /// `dotnet meta verify` diffs, and `namespace`, the namespace both generate under. The other + /// ports ignore the block, as they ignore any key they do not own. An unknown key, a + /// non-string or empty value, or a non-object block is refused, the same rule as `naming`. + private static (string? OutDir, string? Namespace) ReadCSharp(JsonElement root, string path, string configDir) { - if (!root.TryGetProperty("csharp", out var block)) return null; + if (!root.TryGetProperty("csharp", out var block)) return (null, null); if (block.ValueKind != JsonValueKind.Object) throw new MetaModelException($"{path}: \"csharp\" must be an object", ErrorCode.ERR_BAD_ATTR_VALUE); - string? outDir = null; + string? outDir = null, ns = null; foreach (var p in block.EnumerateObject()) { - if (p.Name != "outDir") + if (p.Name is not ("outDir" or "namespace")) throw new MetaModelException( - $"{path}: \"csharp\" has an unknown key \"{p.Name}\" (expected \"outDir\")", + $"{path}: \"csharp\" has an unknown key \"{p.Name}\" (expected \"outDir\" and/or \"namespace\")", ErrorCode.ERR_BAD_ATTR_VALUE); if (p.Value.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(p.Value.GetString())) throw new MetaModelException( - $"{path}: \"csharp.outDir\" must be a non-empty string", ErrorCode.ERR_BAD_ATTR_VALUE); - outDir = Path.GetFullPath(p.Value.GetString()!, Path.GetFullPath(configDir)); + $"{path}: \"csharp.{p.Name}\" must be a non-empty string", ErrorCode.ERR_BAD_ATTR_VALUE); + if (p.Name == "outDir") + outDir = Path.GetFullPath(p.Value.GetString()!, Path.GetFullPath(configDir)); + else + ns = p.Value.GetString()!; } - return outDir; + return (outDir, ns); } /// The `wire` block (ADR-0019 Amendment 2): an unknown key, an unknown value, or a diff --git a/server/csharp/MetaObjects/Errors.cs b/server/csharp/MetaObjects/Errors.cs index d2498740bc..da46527c5a 100644 --- a/server/csharp/MetaObjects/Errors.cs +++ b/server/csharp/MetaObjects/Errors.cs @@ -247,6 +247,9 @@ public enum ErrorCode // @fields), whichever is declared supplies no key, or a named field does not exist // in the entity's effective field set. ERR_INVALID_INDEX, + // Two fields of one stored object resolve to the same PHYSICAL column (snake_case folds + // `userId` and `userID` together, or two fields share an @column). + ERR_COLUMN_NAME_COLLISION, // FR-044 — reporting vocabulary (ValidationPasses.ValidateReporting). Rule ids // D1-D4 / M1-M6 / R1-R7 are the cross-port rule table; S1/F1/F2 reuse // ERR_BAD_ATTR_FILTER. @@ -296,6 +299,9 @@ public enum ErrorCode // an integer array. An array-of-enum stays string-backed: drop @intValueMap, // or make the field scalar. ERR_ENUM_INT_VALUE_MAP_ARRAY, + // Two declared members of one field.enum normalize to the same key under the + // default @normalize: strip; rename a member or set collapse / none. + ERR_ENUM_NORMALIZE_COLLISION, ERR_UNKNOWN, } diff --git a/server/csharp/MetaObjects/Loader/MetaDataLoader.cs b/server/csharp/MetaObjects/Loader/MetaDataLoader.cs index 41ef2e75e1..f5796338fa 100644 --- a/server/csharp/MetaObjects/Loader/MetaDataLoader.cs +++ b/server/csharp/MetaObjects/Loader/MetaDataLoader.cs @@ -606,6 +606,9 @@ public LoadResult Load(IReadOnlyList sources) } } + // Two members folding to one key under `strip` — ERR_ENUM_NORMALIZE_COLLISION. + errors.AddRange(ValidationPasses.ValidateEnumNormalizeCollision(root)); + // FR-014: TPH discriminator cross-attribute rules. errors.AddRange(ValidationPasses.ValidateDiscriminator(root)); @@ -642,6 +645,9 @@ public LoadResult Load(IReadOnlyList sources) // and each must exist in the entity's effective field set (ERR_INVALID_INDEX). errors.AddRange(ValidationPasses.ValidateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + errors.AddRange(ValidationPasses.ValidateColumnNameCollisions(root)); + // FR-039 — a retired requirement carries no @implementedBy (refused, not // exempted, so the dangling-ref class is unreachable) and @supersededBy is // legal only there. diff --git a/server/csharp/MetaObjects/Loader/ValidationPasses.cs b/server/csharp/MetaObjects/Loader/ValidationPasses.cs index a49fbe06e0..13f06857b2 100644 --- a/server/csharp/MetaObjects/Loader/ValidationPasses.cs +++ b/server/csharp/MetaObjects/Loader/ValidationPasses.cs @@ -3842,6 +3842,70 @@ private static void VisitEnumNormalizeAmbiguity(MetaData node, List + /// ERR_ENUM_NORMALIZE_COLLISION — two declared members of one enum fold to the SAME key + /// under @normalize: strip (e.g. READ_ONLY and READONLY), so one is unreachable and a + /// reply naming it coerces to the other. An ERROR (the vocabulary itself is ambiguous); + /// collapse / none are not gated. Mirrors the TS reference. + /// + public static IReadOnlyList ValidateEnumNormalizeCollision(MetaData root) + { + var errors = new List(); + VisitEnumNormalizeCollision(root, errors); + return errors; + } + + private static void VisitEnumNormalizeCollision(MetaData node, List errors) + { + if (node.Type == TYPE_FIELD && node.SubType == FIELD_SUBTYPE_ENUM) + { + // ADR-0039 sanctioned own: the vocabulary DECLARED here; an inheriting enum + // shares the super's set, already checked there. + var ownValues = node.OwnAttr(FIELD_ATTR_VALUES); + if (ownValues is System.Collections.IEnumerable rawEnum && ownValues is not string) + { + var members = new List(); + foreach (var o in rawEnum) members.Add(o?.ToString() ?? string.Empty); + var normalize = EffectiveNormalizeMode(node); + if (members.Count > 1 && normalize.Mode == NORMALIZE_DEFAULT) + { + var byKey = new Dictionary>(); + var order = new List(); + foreach (var m in members) + { + var key = StripNormalize(m); + if (key.Length == 0) continue; + if (!byKey.TryGetValue(key, out var list)) + { + byKey[key] = list = new List(); + order.Add(key); + } + // Distinct members only: an exact duplicate is ERR_BAD_ATTR_VALUE's job. + if (!list.Contains(m)) list.Add(m); + } + foreach (var key in order) + { + var distinct = byKey[key]; + if (distinct.Count < 2) continue; + var listed = string.Join(", ", distinct.Select(d => $"'{d}'")); + errors.Add(new MetaError( + $"field.enum \"{node.Name}\" members {listed} all normalize to '{key}' under " + + $"@{FIELD_ATTR_NORMALIZE}: '{NORMALIZE_DEFAULT}'{(normalize.Explicit ? "" : " (the default)")}, " + + "which upper-cases and deletes everything outside A-Z0-9, so a reply naming one of " + + "them is indistinguishable from the others. Rename the members so they differ after " + + $"that fold, or set @{FIELD_ATTR_NORMALIZE}: 'collapse' (keeps '_' distinct) or " + + "'none' (exact match) on this field.", + ErrorCode.ERR_ENUM_NORMALIZE_COLLISION, + Envelope: node.Source)); + break; // one error per declaring node + } + } + } + } + // ADR-0039 sanctioned own: structural walk of what each node declares. + foreach (var child in node.OwnChildren()) VisitEnumNormalizeCollision(child, errors); + } + // ========================================================================= // FR-037 R1 — field-level @mutability cross-attribute rules. // ERR_MUTABILITY_AUTOSET_CONFLICT / ERR_MUTABILITY_DOWNGRADE / @@ -4105,7 +4169,7 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) } else if (NumericDiscriminatorSubtypes.Contains(field.SubType)) { - if (!Regex.IsMatch(value, "^-?\\d+$")) + if (!IsAsciiInteger(value)) { errors.Add(new MetaError( $"object.entity \"{obj.Name}\" @discriminatorValue: \"{value}\" does not coerce to " + @@ -4129,9 +4193,14 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) foreach (var discRoot in order) { var seen = new Dictionary(); + var rootField = discRoot.OwnAttr(OBJECT_ATTR_DISCRIMINATOR) is string rootFieldName + ? FindFieldOnEntity(discRoot, rootFieldName) : null; + var numericRoot = rootField != null && NumericDiscriminatorSubtypes.Contains(rootField.SubType); foreach (var (subtype, value) in bindingsByRoot[discRoot]) { - if (seen.TryGetValue(value, out var prev)) + // "1" and "01" are one value to a numeric column, so they are one claim. + var key = numericRoot && IsAsciiInteger(value) ? CanonicalInteger(value) : value; + if (seen.TryGetValue(key, out var prev)) { errors.Add(new MetaError( $"object.entity \"{subtype.Name}\" @discriminatorValue: \"{value}\" duplicates the " + @@ -4141,7 +4210,7 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) } else { - seen[value] = subtype; + seen[key] = subtype; } } } @@ -4164,6 +4233,21 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) return errors.AsReadOnly(); } + /// An optional minus sign and ASCII digits only. Not \d (which matches every Unicode + /// decimal digit) and not $ (which matches before a trailing newline). + private static bool IsAsciiInteger(string value) + { + var start = value.Length > 0 && value[0] == '-' ? 1 : 0; + if (value.Length == start) return false; + for (var i = start; i < value.Length; i++) + if (value[i] < '0' || value[i] > '9') return false; + return true; + } + + /// The integer's canonical decimal spelling: no leading zeros, and no sign on zero. + private static string CanonicalInteger(string value) => + System.Numerics.BigInteger.Parse(value, System.Globalization.CultureInfo.InvariantCulture).ToString(System.Globalization.CultureInfo.InvariantCulture); + private static MetaField? FindFieldOnEntity(MetaData entity, string name) { var f = entity.OwnChildren().FirstOrDefault(c => c.Type == TYPE_FIELD && c.Name == name) as MetaField; @@ -4353,6 +4437,42 @@ void Walk(MetaData node) return errors; } + // ValidateColumnNameCollisions — ERR_COLUMN_NAME_COLLISION: two fields of one stored + // object resolving to the SAME physical column (userId + userID both snake_case to + // user_id, or two fields sharing an explicit @column). Only the RESOLVED name is + // compared (PhysicalNaming.ResolveColumnName), so an explicit @column, an inherited + // field and the project's column strategy are honoured. Scope: a concrete object + // declaring at least one source (a source-less object generates nothing); derived + // fields and field.object (may flatten to several columns) are skipped. + // ADR-0039: effective children, so an inherited field collides like an own one. + public static IReadOnlyList ValidateColumnNameCollisions(MetaData root) + { + var errors = new List(); + foreach (var obj in root.OwnChildren().Where(c => c.Type == TYPE_OBJECT)) + { + if (obj.IsAbstract) continue; + if (!obj.Children().Any(c => c.Type == TYPE_SOURCE)) continue; + var seen = new Dictionary(StringComparer.Ordinal); + foreach (var field in obj.Children().OfType()) + { + if (field.SubType == FIELD_SUBTYPE_OBJECT || field.IsDerived()) continue; + var column = PhysicalNaming.ResolveColumnName(field); + if (!seen.TryGetValue(column, out var first)) + { + seen[column] = field; + continue; + } + errors.Add(new MetaError( + $"fields \"{first.Name}\" and \"{field.Name}\" on \"{obj.Name}\" both resolve to " + + $"physical column \"{column}\". Rename one field, or give one an explicit @column " + + "so the two columns differ.", + ErrorCode.ERR_COLUMN_NAME_COLLISION, + Envelope: field.Source)); + } + } + return errors; + } + public static IReadOnlyList ValidateIndexLookupFields(MetaData root) { var errors = new List(); diff --git a/server/csharp/MetaObjects/Meta/PhysicalNaming.cs b/server/csharp/MetaObjects/Meta/PhysicalNaming.cs index e30f5f80d4..0844b9eb1a 100644 --- a/server/csharp/MetaObjects/Meta/PhysicalNaming.cs +++ b/server/csharp/MetaObjects/Meta/PhysicalNaming.cs @@ -61,6 +61,20 @@ public sealed record PhysicalNaming(ColumnNamingStrategy Columns, TableNamingRul public static PhysicalNaming Of(MetaData node) => node.Root() is MetaRoot root ? root.PhysicalNaming : Default; + /// A field's physical column name: its explicit @column (resolving, so an + /// inherited override counts), else the field name through the project's column strategy. + /// The loader's collision check compares THESE resolved names, not the raw field names. + public static string ResolveColumnName(MetaField field) + { + if (field.DbColumn is { Length: > 0 } explicitColumn) return explicitColumn; + return PhysicalNaming.Of(field).Columns switch + { + ColumnNamingStrategy.SnakeCase => SourceNaming.ToSnakeCase(field.Name), + ColumnNamingStrategy.KebabCase => SourceNaming.ToSnakeCase(field.Name).Replace('_', '-'), + _ => field.Name, + }; + } + /// cased by the table rule — no pluralization (the /// caller decides whether to pluralize; see ). public static string ApplyTableNamingRule(string name, TableNamingRule rule) => diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java index 9feb1f34f3..aaa48bb669 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java @@ -224,6 +224,6 @@ private static boolean isBlank(String s) { } private static String escape(String s) { - return s.replace("\\", "\\\\").replace("\"", "\\\""); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(s); } } diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java index cf7aa81d39..8c1f25cfce 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java @@ -210,7 +210,7 @@ private void emitServicesRegistration() throws IOException { } private static String escape(String s) { - return s.replace("\\", "\\\\").replace("\"", "\\\""); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(s); } @Override diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java index 56e74ffb59..3c6dd2ff95 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java @@ -1,6 +1,8 @@ package com.metaobjects.generator.util; import com.metaobjects.MetaData; +import com.metaobjects.MetaDataException; +import com.metaobjects.config.Scope; import com.metaobjects.generator.MetaDataFilters; import com.metaobjects.loader.MetaDataLoader; import com.metaobjects.object.MetaObject; @@ -168,29 +170,91 @@ public static boolean filterByMetaData(MetaData md, String metaDataFilter) { return false; } + /** + * Whether {@code metaDataName} matches the {@code } name glob. The grammar is + * {@link Scope#compilePattern}'s, the one every port's {@code scope} uses: {@code *} inside a + * segment never crosses {@code ::}, a segment that is exactly {@code **} spans one or more + * segments, and every other character (including {@code ?}, {@code @} and regex + * metacharacters) is literal. + * + *

Before 8.2.0 {@code *} crossed {@code ::}, {@code @} matched one segment and {@code ?} one + * character. A pattern the old rule matched but this one does not would silently drop objects + * from the output, so it fails instead: the error names the pattern and its {@code **} + * replacement. + * + * @throws MetaDataException when the legacy rule matches this name and the current one does not + */ public static boolean filterByName( String metaDataName, String nameFilter) { - return Pattern.compile(createRegexFromGlob(nameFilter)).matcher( metaDataName ).matches(); + boolean matches = Scope.compilePattern(nameFilter).matcher( metaDataName ).matches(); + if (!matches && Pattern.compile(createLegacyRegexFromGlob(nameFilter)).matcher( metaDataName ).matches()) { + throw new MetaDataException( vanishingFilterMessage( metaDataName, nameFilter )); + } + return matches; + } + + private static String vanishingFilterMessage( String metaDataName, String nameFilter ) { + StringBuilder msg = new StringBuilder("The pattern \"").append(nameFilter) + .append("\" no longer selects \"").append(metaDataName) + .append("\": since 8.2.0 a filter uses the scope grammar, where '*' stops at '::', '**' spans") + .append(" packages, and '@' and '?' are ordinary characters. Without this error the object would") + .append(" silently drop out of the generated output."); + String suggestion = suggestReplacement( metaDataName, nameFilter ); + if (suggestion != null) { + msg.append(" Replace it with \"").append(suggestion).append("\"."); + } else { + msg.append(" Rewrite it with '*' (within a package) and '**' (across packages)."); + } + return msg.toString(); + } + + /** The legacy pattern rewritten in the current grammar, or null when no rewrite selects {@code name}. */ + private static String suggestReplacement( String name, String glob ) { + List out = new ArrayList<>(); + for (String raw : glob.split( Pattern.quote(MetaData.PKG_SEPARATOR), -1 )) { + boolean wild = raw.contains("*"); + boolean bare = wild && raw.chars().allMatch( c -> c == '*' ); + String segment = raw.replace('@', '*'); // the old one-segment wildcard is this grammar's '*' + if (wild && bare) { + if (out.isEmpty() || !out.get(out.size()-1).equals("**")) out.add("**"); + } else if (wild) { + if (out.isEmpty() || !out.get(out.size()-1).equals("**")) out.add("**"); + out.add(segment); + } else { + out.add(segment); + } + } + String candidate = String.join( MetaData.PKG_SEPARATOR, out ); + try { + return Scope.compilePattern(candidate).matcher(name).matches() ? candidate : null; + } catch (MetaDataException e) { + return null; + } } - private static String createRegexFromGlob(String glob) + /** + * The pre-8.2.0 {@code } glob, kept only to detect a filter whose selection shrank: + * {@code *} any run (crossing {@code ::}), {@code @} one segment, {@code ?} one character, + * every other character quoted. + */ + private static String createLegacyRegexFromGlob(String glob) { - String out = "^"; + StringBuilder out = new StringBuilder("^"); + StringBuilder literal = new StringBuilder(); for(int i = 0; i < glob.length(); ++i) { final char c = glob.charAt(i); - switch(c) - { - case '*': out += ".*"; break; - case '@': out += "[^:]+"; break; - case '?': out += "."; break; - case '.': out += "\\."; break; - case ':': out += "\\:"; break; // TODO: This doesn't seem to work on enforcing the ::'s as a separator for * - case '\\': out += "\\\\"; break; - default: out += c; - } + String wildcard = switch(c) { + case '*' -> ".*"; + case '@' -> "[^:]+"; + case '?' -> "."; + default -> null; + }; + if (wildcard == null) { literal.append(c); continue; } + if (literal.length() > 0) { out.append(Pattern.quote(literal.toString())); literal.setLength(0); } + out.append(wildcard); } - out += '$'; - return out; + if (literal.length() > 0) out.append(Pattern.quote(literal.toString())); + return out.append('$').toString(); } public static String toRelativePackage( String p1, String p2 ) { diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java index a461aa1897..0976f921b9 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java @@ -94,4 +94,29 @@ public static String escapeMember(String name) { public static String escapeIdentifier(String name) { return isKeyword(name) ? name + "_" : name; } + + /** + * {@code s} escaped for the inside of a Java double-quoted string literal. Beyond the + * backslash and quote, a raw newline, carriage return or other control character would end + * the literal (or the file) and generate source that does not compile. + */ + public static String escapeStringLiteral(String s) { + StringBuilder out = new StringBuilder(s.length() + 8); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': out.append("\\\\"); break; + case '"': out.append("\\\""); break; + case '\n': out.append("\\n"); break; + case '\r': out.append("\\r"); break; + case '\t': out.append("\\t"); break; + case '\b': out.append("\\b"); break; + case '\f': out.append("\\f"); break; + default: + if (c < 0x20 || c == 0x7f) out.append(String.format("\\u%04x", (int) c)); + else out.append(c); + } + } + return out.toString(); + } } diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java index e3088a4768..8e57904e93 100644 --- a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java @@ -3,32 +3,63 @@ import static org.junit.Assert.*; import com.metaobjects.generator.util.GeneratorUtil; +import com.metaobjects.MetaDataException; import org.junit.Test; import static com.metaobjects.generator.util.GeneratorUtil.*; public class GeneratorUtilTest { + @Test + public void testFilterTreatsRegexMetacharactersAsLiterals() { + assertTrue( filterByName( "a::B(", "a::B(" )); + assertTrue( filterByName( "a::B[1]", "a::B[1]" )); + assertFalse( filterByName( "a::B1", "a::B[1]" )); + assertFalse( filterByName( "a::Bxc", "a::B.c" )); + assertTrue( filterByName( "a::B+c", "a::B+*" )); + } + @Test public void testFilterMatching() { - assertTrue( filterByName( "acme::store::PrimaryStore", "acme::*" )); + assertTrue( filterByName( "acme::store::PrimaryStore", "acme::**" )); + assertTrue( filterByName( "acme::PrimaryStore", "acme::*" )); assertTrue( filterByName( "acme::store::PrimaryStore", "acme::store::PrimaryStore" )); - assertTrue( filterByName( "acme::store::PrimaryStore", "acme::store::???????Store" )); - assertTrue( filterByName( "acme::store::PrimaryStore", "acme::store::@Store" )); - assertFalse( filterByName( "acme::store::PrimaryStore", "acme::store::??????Store" )); assertTrue( filterByName( "acme::store::PrimaryStore", "acme::*::*Store" )); - assertTrue( filterByName( "acme::store::good::PrimaryStore", "acme::*Store" )); - assertTrue( filterByName( "acme::store::good::PrimaryStore", "acme::*::*Store" )); - assertTrue( filterByName( "acme::store::PrimaryStore", "acme::@::@Store" )); - assertFalse( filterByName( "acme::store::bad::PrimaryStore", "acme::@::@Store" )); - assertFalse( filterByName( "acme::store::bad::PrimaryStore", "acme::store::@Store" )); - assertTrue( filterByName( "acme::store::PrimaryStore", "acme::store::Pri@Store" )); assertTrue( filterByName( "acme::store::PrimaryStore", "acme::store::Pri*Store" )); - assertFalse( filterByName( "acme::store::PrimaryStore", "acme::*::*StoreZ" )); + assertTrue( filterByName( "acme::store::good::PrimaryStore", "acme::**::*Store" )); assertTrue( filterByName( "acme::store::PrimaryStore", "acme::*::*Store*" )); - assertFalse( filterByName( "acme::store::PrimaryStore", "acme::*::*Store@" )); - assertFalse( filterByName( "acme::store::PrimaryStore", "acme::*::*Store?" )); + assertFalse( filterByName( "acme::store::PrimaryStore", "acme::*::*StoreZ" )); assertFalse( filterByName( "acme::store::PrimaryStore", "bobs::*" )); + assertFalse( filterByName( "acme::store::PrimaryStore", "bobs::**" )); + } + + @Test + public void testAtAndQuestionMarkAreLiteralInTheScopeGrammar() { + assertTrue( filterByName( "acme::a@b", "acme::a@b" )); + assertTrue( filterByName( "acme::a?b", "acme::a?b" )); + assertFalse( filterByName( "acme::store::Pri@Store", "acme::store::PriXStore" )); + } + + @Test + public void testFilterThatWouldSilentlyDropObjectsFailsNamingTheDoubleStarReplacement() { + MetaDataException e = assertThrows( MetaDataException.class, + () -> filterByName( "acme::store::PrimaryStore", "acme::*" )); + assertTrue( e.getMessage(), e.getMessage().contains( "\"acme::*\"" )); + assertTrue( e.getMessage(), e.getMessage().contains( "\"acme::**\"" )); + + e = assertThrows( MetaDataException.class, + () -> filterByName( "acme::store::good::PrimaryStore", "acme::*Store" )); + assertTrue( e.getMessage(), e.getMessage().contains( "\"acme::**::*Store\"" )); + } + + @Test + public void testRemovedWildcardsFailInsteadOfVanishing() { + MetaDataException e = assertThrows( MetaDataException.class, + () -> filterByName( "acme::store::PrimaryStore", "acme::store::@Store" )); + assertTrue( e.getMessage(), e.getMessage().contains( "\"acme::store::*Store\"" )); + e = assertThrows( MetaDataException.class, + () -> filterByName( "acme::store::PrimaryStore", "acme::store::???????Store" )); + assertTrue( e.getMessage(), e.getMessage().contains( "Rewrite it" )); } @Test diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java new file mode 100644 index 0000000000..037dab4c5e --- /dev/null +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java @@ -0,0 +1,20 @@ +package com.metaobjects.generator.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class JavaIdentifiersEscapeTest { + + @Test + public void escapesWhatWouldEndTheLiteral() { + assertEquals("a\\\\b\\\"c", JavaIdentifiers.escapeStringLiteral("a\\b\"c")); + assertEquals("line1\\nline2\\r\\t", JavaIdentifiers.escapeStringLiteral("line1\nline2\r\t")); + assertEquals("\\u0001x\\u007f", JavaIdentifiers.escapeStringLiteral("\u0001x\u007f")); + } + + @Test + public void leavesOrdinaryTextAlone() { + assertEquals("^[a-z]+é$", JavaIdentifiers.escapeStringLiteral("^[a-z]+é$")); + } +} diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/RouteNamingTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/RouteNamingTest.java index d50db9d2ba..99a8597fbd 100644 --- a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/RouteNamingTest.java +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/RouteNamingTest.java @@ -179,6 +179,16 @@ public void assertNoCollectionNameCollisionsThrowsForOrderAndOrders() { () -> RouteNaming.assertNoCollectionNameCollisions(List.of(entity("Order"), entity("Orders")))); } + @Test + public void assertNoCollectionNameCollisionsThrowsForRouteSegmentTwins() { + // ApiKey / APIKey both snake_case to api_key and serve /api_keys. + GeneratorException ex = assertThrows(GeneratorException.class, + () -> RouteNaming.assertNoCollectionNameCollisions(List.of(entity("ApiKey"), entity("APIKey")))); + assertTrue(ex.getMessage().contains("\"ApiKey\"")); + assertTrue(ex.getMessage().contains("\"APIKey\"")); + assertTrue(ex.getMessage().contains("api_keys")); + } + @Test public void assertNoCollectionNameCollisionsExcludesValueObjects() { // "Address" (entity) resolves to /addresses; an unrelated object.value named diff --git a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt index baddea1d3e..f3a12ea074 100644 --- a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt +++ b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt @@ -4,6 +4,7 @@ package com.metaobjects.generator.kotlin // com.metaobjects.generator.kotlin when this generator is copied out via // `mvn metaobjects:eject` and its own package is renamed — an explicit import, not // same-package bare-name resolution, is what keeps the ejected copy compiling. +import com.metaobjects.generator.kotlin.KotlinGenUtil import com.metaobjects.generator.kotlin.KotlinNaming import com.metaobjects.generator.kotlin.KotlinTphPlan import com.metaobjects.generator.kotlin.PackageMapping diff --git a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt index 0fa26f99b0..07fbf9de42 100644 --- a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt +++ b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt @@ -23,6 +23,7 @@ import com.metaobjects.loader.MetaDataLoader import com.metaobjects.`object`.MetaObject import com.metaobjects.relationship.CompositionRelationship import com.metaobjects.relationship.MetaRelationship +import com.squareup.kotlinpoet.ClassName import com.squareup.kotlinpoet.LONG import com.squareup.kotlinpoet.TypeName import java.io.OutputStream @@ -143,9 +144,10 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() val api = exposedApi() val (pkg, ownerShort) = PackageMapping.splitFqn(entity.name) val ownerTable = ownerShort + "Table" - // TypeName.toString() yields the FQN (`kotlin.Long`, `java.util.UUID`, ...); - // grab the trailing simple name for the generated source. - val pkParamSimpleName = primaryKeyKotlinType(entity).toString().substringAfterLast('.') + // The generated source is text, so the PK type is rendered as a simple name plus + // the import it needs (a ClassName), not by slicing TypeName.toString(). + val pkRendered = renderType(primaryKeyKotlinType(entity)) + val pkParamSimpleName = pkRendered.name // FK column matches KotlinExposedTableGenerator.buildInverseFkSpec: Id. val fkColName = ownerShort.replaceFirstChar { it.lowercaseChar() } + "Id" @@ -205,9 +207,9 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() // reverse-FK value type) is uuid — the param types were already derived, but // without this import the generated file had an unresolved reference. Emitted // conditionally so uuid-free files stay byte-identical. - if (pkParamSimpleName == "UUID" || reverseFks.any { it.valueType == "UUID" }) { - append("import java.util.UUID\n") - } + val typeImports = (listOf(pkRendered.import) + reverseFks.map { it.valueImport }) + .filterNotNull().toSortedSet() + for (line in typeImports) append("import $line\n") val crossPackageTables = KotlinM2mSupport.importLines(m2mNavs.map { it.nav }, pkg, junction = true, dataClass = false) + helpers.filter { it.targetPackage != pkg } .map { if (it.targetPackage.isEmpty()) it.targetTable else "${it.targetPackage}.${it.targetTable}" } @@ -342,7 +344,26 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() * the FK field name, the target entity short name, and the Kotlin value type of the * FK (the type a `findBy` arg / element takes — the FK column's own Kotlin type). */ - protected data class ReverseFk(val fkField: String, val targetShortName: String, val valueType: String) + protected data class ReverseFk( + val fkField: String, + val targetShortName: String, + val valueType: String, + val valueImport: String? = null, + ) + + /** A Kotlin type as it is written into generated source: the name to print and the import it needs, if any. */ + protected data class RenderedType(val name: String, val import: String?) + + /** + * Render [type] for text emission. A [ClassName] prints its simple name and imports its + * canonical name (nothing for the auto-imported `kotlin` / `java.lang` packages). Any other + * shape (parameterized, nullable, ...) prints fully qualified, which is always valid. + */ + protected fun renderType(type: TypeName): RenderedType { + if (type !is ClassName || type.isNullable) return RenderedType(type.toString(), null) + val autoImported = type.packageName == "kotlin" || type.packageName == "java.lang" + return RenderedType(type.simpleNames.joinToString("."), if (autoImported) null else type.topLevelClassName().canonicalName) + } /** * Collect every reverse FK [entity] declares, in declaration order — one per @@ -360,11 +381,10 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() if (fields.size != 1) continue // single-column FKs only val fkField = fields.first() val targetShort = child.targetEntity?.let { PackageMapping.splitFqn(it).second } ?: "" - val valueType = entity.metaFields.firstOrNull { it.name == fkField }?.let { f -> - runCatching { KotlinTypeMapper.kotlinTypeName(f).toString().substringAfterLast('.') } - .getOrDefault("Long") - } ?: "Long" - out.add(ReverseFk(fkField, targetShort, valueType)) + val rendered = entity.metaFields.firstOrNull { it.name == fkField }?.let { f -> + runCatching { renderType(KotlinTypeMapper.kotlinTypeName(f)) }.getOrNull() + } ?: renderType(LONG) + out.add(ReverseFk(fkField, targetShort, rendered.name, rendered.import)) } return out } diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt index 6f6a860ad5..ebc1a0dc83 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt @@ -343,7 +343,7 @@ class KotlinNamesGeneratorTest { { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } @@ -514,7 +514,7 @@ class KotlinNamesGeneratorTest { { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "created_at" } }, + { "field.timestamp": { "name": "created_at", "@column": "created_at_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt index 4900d90f21..b8430107f9 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt @@ -139,4 +139,35 @@ class KotlinRelationsGeneratorTest { outDir.toFile().deleteRecursively() } } + + @Test fun `a non-uuid library-typed primary key imports its own type`() { + val fixture = """{ + "metadata.root": { "package": "acme::blog", "children": [ + { "object.entity": { "name": "Day", "children": [ + { "field.date": { "name": "id" } }, + { "relationship.composition": { + "name": "posts", "@objectRef": "Post", "@cardinality": "many" + } }, + { "source.rdb": { "@table": "days" } }, + { "identity.primary": { "name": "pk", "@fields": ["id"] } } + ] } }, + { "object.entity": { "name": "Post", "children": [ + { "field.long": { "name": "id" } }, + { "source.rdb": { "@table": "posts" } }, + { "identity.primary": { "name": "pk", "@fields": ["id"], "@generation": "increment" } } + ] } } + ] } + }""".trimIndent() + val outDir = Files.createTempDirectory("krel-") + try { + val gen = KotlinRelationsGenerator() + gen.setArgs(mapOf("outputDir" to outDir.toString())) + gen.execute(loadString("rel-date-pk", fixture)) + val src = Files.readString(outDir.resolve("acme/blog/DayRelations.kt")) + assertTrue("import java.time.LocalDate" in src, src) + assertTrue("dayId: LocalDate): Query" in src, src) + } finally { + outDir.toFile().deleteRecursively() + } + } } diff --git a/server/java/codegen-plantuml/src/test/java/com/metaobjects/generator/plantuml/PlantUMLTest.java b/server/java/codegen-plantuml/src/test/java/com/metaobjects/generator/plantuml/PlantUMLTest.java index 04990c3b50..8b353fda56 100644 --- a/server/java/codegen-plantuml/src/test/java/com/metaobjects/generator/plantuml/PlantUMLTest.java +++ b/server/java/codegen-plantuml/src/test/java/com/metaobjects/generator/plantuml/PlantUMLTest.java @@ -102,7 +102,7 @@ public void testPlantUMLGenerator5() { .outputDir(getGeneratedTestSourcesPath()) .withFilters(Arrays.asList(new String[] { "produce::v1::fruit::*", - "produce::v1::container::@" + "produce::v1::container::*" })) .build(); @@ -121,7 +121,7 @@ public void testPlantUMLGenerator6() { drawUML( args, Arrays.asList( new String[] { "produce::v1::fruit::*", "produce::v1::vegetable::*", - "produce::v1::container::@" + "produce::v1::container::*" } )); } @@ -137,7 +137,7 @@ public void testPlantUMLGenerator7() { drawUML( args, Arrays.asList( new String[] { "produce::v1::fruit::*", "produce::v1::vegetable::*", - "produce::v1::container::@" + "produce::v1::container::*" } )); } diff --git a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java index ee88b7fbf9..03af95a822 100644 --- a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java +++ b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java @@ -4,6 +4,7 @@ // com.metaobjects.generator.spring when this generator is copied out via // `mvn metaobjects:eject` and its own package is renamed — an explicit import, not // same-package bare-name resolution, is what keeps the ejected copy compiling. +import com.metaobjects.generator.spring.SpringGeneratorSupport; import com.metaobjects.generator.spring.SpringNaming; import com.metaobjects.generator.spring.TphPlan; diff --git a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java index 8d5bdf62dd..cccb653d7a 100644 --- a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java +++ b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java @@ -635,7 +635,7 @@ public static boolean hasValidator(MetaField field, Class used, /** Escape a value for embedding inside a Java double-quoted string literal. */ private static String escapeJava(String value) { - StringBuilder sb = new StringBuilder(value.length() + 4); - for (int i = 0; i < value.length(); i++) { - char c = value.charAt(i); - switch (c) { - case '\\': sb.append("\\\\"); break; - case '"': sb.append("\\\""); break; - case '\t': sb.append("\\t"); break; - case '\n': sb.append("\\n"); break; - case '\r': sb.append("\\r"); break; - default: sb.append(c); - } - } - return sb.toString(); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(value); } /** diff --git a/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java b/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java index f09f761385..efde171a77 100644 --- a/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java +++ b/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java @@ -322,7 +322,7 @@ public void aChildFieldCollidingWithAnInheritedOneIsRefusedNamingTheModel() thro { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "created_at" } }, + { "field.timestamp": { "name": "created_at", "@column": "created_at_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } @@ -435,7 +435,7 @@ public void collidingScreamingSnakeMembersAreRefusedNamingTheModel() throws IOEx { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } diff --git a/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java b/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java index a3bf34d194..aab627e11d 100644 --- a/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java +++ b/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java @@ -1,7 +1,12 @@ package com.metaobjects.mojo; import com.metaobjects.generator.GeneratorRegistry; +import org.apache.maven.model.Build; import org.apache.maven.model.Dependency; +import org.apache.maven.model.Model; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -11,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -21,8 +27,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -456,7 +460,7 @@ private Path resolveRuntimeRoot() throws MojoFailureException { Path childPom = base.resolve(module).resolve("pom.xml"); try { if (Files.isRegularFile(childPom) - && Files.readString(childPom, StandardCharsets.UTF_8).contains("metaobjects-maven-plugin")) { + && declares(Files.readString(childPom, StandardCharsets.UTF_8), "metaobjects-maven-plugin")) { candidates.add(module); } } catch (IOException ex) { @@ -710,11 +714,40 @@ private String codegenPom(String artifactId, Set ports) { + "\n"; } - private static final Pattern SOURCE_DIRECTORY = - Pattern.compile("\\s*([^<]*?)\\s*"); + /** The pom parsed by Maven's own reader, or null when it is not well-formed. */ + static Model parsePom(String pom) { + try { + return new MavenXpp3Reader().read(new StringReader(pom), false); + } catch (IOException | XmlPullParserException ex) { + return null; + } + } + + /** + * Whether the pom declares {@code artifactId} as a dependency or a build plugin (managed or + * not). Read from the parsed model, so the name inside a comment or a property does not count. + */ + static boolean declares(String pom, String artifactId) { + Model model = parsePom(pom); + if (model == null) return false; + for (Dependency d : model.getDependencies()) { + if (artifactId.equals(d.getArtifactId())) return true; + } + Build build = model.getBuild(); + if (build == null) return false; + List plugins = new ArrayList<>(build.getPlugins()); + if (build.getPluginManagement() != null) plugins.addAll(build.getPluginManagement().getPlugins()); + for (Plugin plugin : plugins) { + if (artifactId.equals(plugin.getArtifactId())) return true; + } + return false; + } - private static boolean declares(String pom, String artifactId) { - return pom.contains("" + artifactId + ""); + /** {@code } as written (trimmed), or null when absent. */ + static String declaredSourceDirectory(String pom) { + Model model = parsePom(pom); + if (model == null || model.getBuild() == null || model.getBuild().getSourceDirectory() == null) return null; + return model.getBuild().getSourceDirectory().trim(); } /** @@ -741,9 +774,9 @@ private void reportWhatThePomLacks(Path codegenRoot) throws MojoExecutionExcepti changes.add(" Add to :\n" + kotlinPlugin(KOTLIN_SOURCE_DIRS)); } if (port == EjectSupport.Port.JAVA) { - Matcher sourceDirectory = SOURCE_DIRECTORY.matcher(content); - if (sourceDirectory.find() && !sourceDirectory.group(1).endsWith(sourceRoot)) { - changes.add(" Remove " + sourceDirectory.group() + " from , so that Maven compiles " + String sourceDirectory = declaredSourceDirectory(content); + if (sourceDirectory != null && !sourceDirectory.endsWith(sourceRoot)) { + changes.add(" Remove " + sourceDirectory + " from , so that Maven compiles " + sourceRoot + ", and add to the of " + KOTLIN_PLUGIN + ":\n" + KOTLIN_SOURCE_DIRS); } diff --git a/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java b/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java new file mode 100644 index 0000000000..a1127dbc04 --- /dev/null +++ b/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java @@ -0,0 +1,41 @@ +package com.metaobjects.mojo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** Eject reads a pom with Maven's model reader: a name in a comment declares nothing. */ +public class EjectPomReadingTest { + + private static final String POM = "\n" + + " \n" + + " 4.0.0\n" + + " greal-dep\n" + + " \n" + + " \n" + + " \n src/main/java\n \n" + + " real-plugin\n" + + " \n" + + ""; + + @Test + public void commentedArtifactIsNotDeclared() { + assertFalse(MetaDataEjectMojo.declares(POM, "metaobjects-maven-plugin")); + assertTrue(MetaDataEjectMojo.declares(POM, "real-dep")); + assertTrue(MetaDataEjectMojo.declares(POM, "real-plugin")); + } + + @Test + public void sourceDirectoryIgnoresCommentsAndWhitespace() { + assertEquals("src/main/java", MetaDataEjectMojo.declaredSourceDirectory(POM)); + assertNull(MetaDataEjectMojo.declaredSourceDirectory("")); + } + + @Test + public void malformedPomDeclaresNothing() { + assertFalse(MetaDataEjectMojo.declares("", "real-dep")); + } +} diff --git a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java index 91a9221b49..01355a0b6b 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java +++ b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java @@ -492,6 +492,9 @@ public enum ErrorCode { * entity's effective (resolved, via {@code extends:}) field set. */ ERR_INVALID_INDEX, + /** Two fields of one stored object resolve to the same PHYSICAL column (snake_case folds + * {@code userId} and {@code userID} together, or two fields share an @column). */ + ERR_COLUMN_NAME_COLLISION, /** FR-039 — @status: retired declares @implementedBy. Refused rather than exempted: * a retired capability has no implementation BY DEFINITION, so forbidding the * attribute makes the dangling-reference class unreachable instead of tolerated. */ @@ -561,6 +564,12 @@ public enum ErrorCode { */ ERR_ENUM_INT_VALUE_MAP_ARRAY, + /** + * Two declared members of one {@code field.enum} normalize to the same key under the + * default {@code @normalize: strip}; rename a member or set {@code collapse} / {@code none}. + */ + ERR_ENUM_NORMALIZE_COLLISION, + /** * FR-044: a {@code dimension.*} is invalid: {@code @of} is not Entity.field on the * owning entity (or, with {@code @via}, on the {@code @via} terminal) or names no diff --git a/server/java/metadata/src/main/java/com/metaobjects/config/Scope.java b/server/java/metadata/src/main/java/com/metaobjects/config/Scope.java index 1673fe3dec..205acadc12 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/config/Scope.java +++ b/server/java/metadata/src/main/java/com/metaobjects/config/Scope.java @@ -42,9 +42,12 @@ * after it. Matching is case-sensitive. * * - *

This is deliberately NOT the {@code } glob of the Maven plugin - * ({@code GeneratorUtil.createRegexFromGlob}), whose {@code *} and {@code @} are the - * inverse of {@code **} and {@code *} here. The two are separate surfaces. + *

The Maven plugin's {@code } name glob uses this same grammar + * ({@code GeneratorUtil.filterByName} compiles with {@link #compilePattern}), so a name that + * is in {@code scope} reads the same in a filter. A filter pattern that the pre-8.2.0 rule + * ({@code *} crossing {@code ::}, {@code @} one segment, {@code ?} one character) matched + * and this one does not fails generation with an error naming its {@code **} replacement, + * rather than dropping objects from the output. */ public final class Scope { diff --git a/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java b/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java index 0a4dc9d013..ed4e91473a 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java +++ b/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java @@ -673,43 +673,24 @@ private static boolean getNativeIsArray(MetaData node, boolean effective) { // --------------------------------------------------------------------------- /** - * Returns {@code true} if the node's short name was auto-generated by the parser. + * Returns {@code true} if the node's short name was synthesised by the parser. * - *

Auto-naming applies only to {@code validator} and {@code view} type nodes. - * {@link com.metaobjects.loader.parser.BaseMetaDataParser} generates names of the - * exact form {@code } where {@code prefix} is the node's subType - * lowercased (or the type lowercased when there is no subType), and {@code N} is - * a positive integer. Detection therefore uses the exact expected prefix - * derived from this node's own type/subType — not a blanket pattern.

- * - *

This prevents silently dropping an explicitly authored name that happens to - * end in a digit (e.g. a validator named {@code length2} whose subType is - * {@code regex} would only be auto-detected as {@code regex\d+}, so {@code length2} - * is correctly treated as authored).

- * - *

All other type nodes require explicit names; if the short name is present - * it was authored and must be emitted.

+ *

Decided by the flag the parser sets at the moment it generates a name + * ({@link MetaData#isAutoNamed()}), never by the shape of the name: an authored + * name that merely looks generated (a validator authored as {@code required7}) + * is an authored name and must survive a canonical round trip. Only the + * auto-naming types ({@link BaseMetaDataParser#isAutoNamingType}) can carry a + * synthesised name; an empty name is the degenerate "no name" case.

*/ private static boolean isAutoGeneratedName(MetaData node) { - String typeName = node.getType(); - // Only the types in isAutoNamingType (validator, view, source, identity) use - // auto-naming — delegate to the single source of truth. - if (!BaseMetaDataParser.isAutoNamingType(typeName)) { + if (!BaseMetaDataParser.isAutoNamingType(node.getType())) { return false; } String shortName = node.getShortName(); if (shortName == null || shortName.isEmpty()) { - return true; // empty = auto (degenerate case) - } - // Derive the exact prefix the parser would have used: - // subType lowercased if non-null/non-empty, otherwise type lowercased. - String subType = node.getSubType(); - String expectedPrefix = (subType != null && !subType.isEmpty()) - ? subType.toLowerCase() - : typeName.toLowerCase(); - // Auto-generated name matches EXACTLY . - // Using Pattern.quote so any regex-special chars in the prefix are treated literally. - return shortName.matches("^" + Pattern.quote(expectedPrefix) + "\\d+$"); + return true; + } + return node.isAutoNamed(); } // --------------------------------------------------------------------------- diff --git a/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java b/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java index e898fd3346..80ce804d74 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java +++ b/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java @@ -1,5 +1,8 @@ package com.metaobjects.library; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.metaobjects.loader.FileSource; import com.metaobjects.loader.InMemoryStringSource; import com.metaobjects.loader.MetaDataSource; @@ -48,30 +51,32 @@ private LibrarySources() {} private static Map>> buildLayers() { Map>> map = new LinkedHashMap<>(); for (String name : new TreeSet<>(EmbeddedLibrary.MANIFESTS.keySet())) { - // Hand-parsed rather than pulled through Jackson: this module is the metadata - // core and does not depend on a JSON binder, and the shape read here is four - // keys deep in a file this repo generates. A binder would be a dependency - // added for a manifest we also write. map.put(name, parseLayers(EmbeddedLibrary.MANIFESTS.get(name))); } return map; } + private static JsonObject manifestRoot(String manifestJson) { + return JsonParser.parseString(manifestJson).getAsJsonObject(); + } + + /** A string array member as a list; absent or not an array gives empty. */ + private static List stringArray(JsonObject obj, String key) { + List out = new ArrayList<>(); + JsonElement el = obj.get(key); + if (el == null || !el.isJsonArray()) return out; + for (JsonElement item : el.getAsJsonArray()) out.add(item.getAsString()); + return out; + } + /** The {@code "layers"} object of a manifest: token to refs, in declaration order. */ - private static Map> parseLayers(String manifestJson) { + static Map> parseLayers(String manifestJson) { Map> layers = new LinkedHashMap<>(); - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"layers\"\\s*:\\s*\\{(.*?)\\n \\}", java.util.regex.Pattern.DOTALL) - .matcher(manifestJson); - if (!block.find()) return layers; - java.util.regex.Matcher entry = java.util.regex.Pattern - .compile("\"([^\"]*)\"\\s*:\\s*\\{[^}]*?\"refs\"\\s*:\\s*\\[([^\\]]*)\\]", java.util.regex.Pattern.DOTALL) - .matcher(block.group(1)); - while (entry.find()) { - List refs = new ArrayList<>(); - java.util.regex.Matcher ref = java.util.regex.Pattern.compile("\"([^\"]+)\"").matcher(entry.group(2)); - while (ref.find()) refs.add(ref.group(1)); - layers.put(entry.group(1), refs); + JsonElement block = manifestRoot(manifestJson).get("layers"); + if (block == null || !block.isJsonObject()) return layers; + for (Map.Entry entry : block.getAsJsonObject().entrySet()) { + if (!entry.getValue().isJsonObject()) continue; + layers.put(entry.getKey(), stringArray(entry.getValue().getAsJsonObject(), "refs")); } return layers; } @@ -96,19 +101,21 @@ private static Map> parseLayers(String manifestJson) { public static List generatorAnchors(String generatorName) { List out = new ArrayList<>(); for (String name : new TreeSet<>(EmbeddedLibrary.MANIFESTS.keySet())) { - String manifest = EmbeddedLibrary.MANIFESTS.get(name); - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"generators\"\\s*:\\s*\\[(.*?)\\]", java.util.regex.Pattern.DOTALL) - .matcher(manifest); - if (!block.find()) continue; - java.util.regex.Matcher obj = java.util.regex.Pattern - .compile("\\{([^}]*)\\}").matcher(block.group(1)); - while (obj.find()) { - String body = obj.group(1); - String declared = manifestField(body, "name"); - String anchor = manifestField(body, "anchor"); - if (generatorName.equals(declared) && anchor != null) out.add(anchor); - } + out.addAll(generatorAnchors(EmbeddedLibrary.MANIFESTS.get(name), generatorName)); + } + return out; + } + + static List generatorAnchors(String manifest, String generatorName) { + List out = new ArrayList<>(); + JsonElement block = manifestRoot(manifest).get("generators"); + if (block == null || !block.isJsonArray()) return out; + for (JsonElement el : block.getAsJsonArray()) { + if (!el.isJsonObject()) continue; + JsonObject gen = el.getAsJsonObject(); + String declared = manifestField(gen, "name"); + String anchor = manifestField(gen, "anchor"); + if (generatorName.equals(declared) && anchor != null) out.add(anchor); } return out; } @@ -119,27 +126,22 @@ public static List generatorAnchors(String generatorName) { * *

The provenance key for requirement-coverage activation (FR-043 §5.4): a * requirement whose package is NOT in this set was authored by the adopter. Read from the - * manifests, hand-parsed for the reason {@link #parseLayers} is, and never regenerated.

+ * manifests and never regenerated.

* * @return the declared package names, sorted */ public static java.util.Set libraryPackages() { java.util.Set out = new TreeSet<>(); for (String manifest : EmbeddedLibrary.MANIFESTS.values()) { - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"packages\"\\s*:\\s*\\[([^\\]]*)\\]").matcher(manifest); - if (!block.find()) continue; - java.util.regex.Matcher pkg = java.util.regex.Pattern.compile("\"([^\"]+)\"").matcher(block.group(1)); - while (pkg.find()) out.add(pkg.group(1)); + out.addAll(stringArray(manifestRoot(manifest), "packages")); } return out; } - /** One {@code "key": "value"} string field out of a flat JSON object body. */ - private static String manifestField(String objectBody, String key) { - java.util.regex.Matcher m = java.util.regex.Pattern - .compile("\"" + key + "\"\\s*:\\s*\"([^\"]*)\"").matcher(objectBody); - return m.find() ? m.group(1) : null; + /** One string member of a JSON object, or null when absent or not a string. */ + private static String manifestField(JsonObject obj, String key) { + JsonElement el = obj.get(key); + return el != null && el.isJsonPrimitive() ? el.getAsString() : null; } /** The prefix every library source id carries. */ diff --git a/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java b/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java index d252e72e37..1ddc61efbc 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java +++ b/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java @@ -191,6 +191,8 @@ public static void run(MetaRoot root, MetaDataLoader loader) { // Authoring guard — a field.enum vocabulary ambiguous under the default // @normalize: strip. WARN_ENUM_NORMALIZE_AMBIGUOUS. pass(collected, () -> validateEnumNormalizeAmbiguity(root, loader)); + // Two members folding to one key under `strip` -- ERR_ENUM_NORMALIZE_COLLISION. + validateEnumNormalizeCollision(root, collected); // FR-014 — TPH discriminator cross-attribute rules. pass(collected, () -> validateDiscriminator(root)); // FR-015 — source.rdb @parameterRef typed-input rules. @@ -287,6 +289,8 @@ public static void run(MetaRoot root, MetaDataLoader loader) { pass(collected, () -> validateFilterableHasSupportedOps(root)); pass(collected, () -> validateSortableHasSupportedSubtype(root)); pass(collected, () -> validateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + pass(collected, () -> validateColumnNameCollisions(root)); // FR-039 — a retired requirement carries no @implementedBy (refused, not exempted, // so the dangling-ref class is unreachable) and @supersededBy is legal only there. pass(collected, () -> validateRetiredRequirementLinks(root)); @@ -589,6 +593,64 @@ static void validateEnumNormalizeAmbiguity(MetaRoot root, MetaDataLoader loader) walkEnumNormalizeAmbiguity(root, loader); } + /** + * ERR_ENUM_NORMALIZE_COLLISION -- two declared members of one enum fold to the SAME key + * under {@code @normalize: strip} (e.g. READ_ONLY and READONLY), so one is unreachable + * and a reply naming it coerces to the other. An ERROR (the vocabulary itself is + * ambiguous); {@code collapse} / {@code none} are not gated. Mirrors the TS reference. + */ + static void validateEnumNormalizeCollision(MetaData node, List out) { + checkEnumNormalizeCollision(node, out); + // Own children only -- inherited children are checked on their declaring nodes. + for (MetaData child : node.getChildren(MetaData.class, false)) { + validateEnumNormalizeCollision(child, out); + } + } + + private static void checkEnumNormalizeCollision(MetaData node, List out) { + if (!EnumField.TYPE_FIELD.equals(node.getType()) + || !EnumField.SUBTYPE_ENUM.equals(node.getSubType())) { + return; + } + // Own-attrs-only (ADR-0039): the vocabulary DECLARED here; an inheriting enum + // shares the super's set, already checked there. + if (!node.hasMetaAttr(EnumField.ATTR_VALUES, false)) return; + Object raw = node.getMetaAttr(EnumField.ATTR_VALUES, false).getValue(); + if (!(raw instanceof List) || ((List) raw).size() < 2) return; + String[] normalize = effectiveNormalizeMode(node); + if (!EnumField.NORMALIZE_DEFAULT.equals(normalize[0])) return; + boolean modeIsExplicit = "true".equals(normalize[1]); + + java.util.Map> byKey = new java.util.LinkedHashMap<>(); + for (Object o : (List) raw) { + String member = String.valueOf(o); + String key = stripNormalize(member); + if (key.isEmpty()) continue; + byKey.computeIfAbsent(key, k -> new java.util.LinkedHashSet<>()).add(member); + } + for (java.util.Map.Entry> e : byKey.entrySet()) { + // Distinct members only: an exact duplicate is ERR_BAD_ATTR_VALUE's job. + if (e.getValue().size() < 2) continue; + StringBuilder listed = new StringBuilder(); + for (String m : e.getValue()) { + if (listed.length() > 0) listed.append(", "); + listed.append('\'').append(m).append('\''); + } + out.add(new MetaDataException( + ErrorMessageConstants.ERR_ENUM_NORMALIZE_COLLISION + + ": field.enum \"" + shortNameOf(node) + "\" members " + listed + + " all normalize to '" + e.getKey() + "' under @" + EnumField.ATTR_NORMALIZE + + ": '" + EnumField.NORMALIZE_DEFAULT + "'" + + (modeIsExplicit ? "" : " (the default)") + + ", which upper-cases and deletes everything outside A-Z0-9, so a reply" + + " naming one of them is indistinguishable from the others. Rename the" + + " members so they differ after that fold, or set @" + EnumField.ATTR_NORMALIZE + + ": 'collapse' (keeps '_' distinct) or 'none' (exact match) on this field.", + ErrorCode.ERR_ENUM_NORMALIZE_COLLISION, node.getSource())); + return; // one error per declaring node + } + } + private static void walkEnumNormalizeAmbiguity(MetaData node, MetaDataLoader loader) { checkEnumNormalizeAmbiguity(node, loader); // Own children only — inherited children are checked on their declaring nodes. @@ -5043,6 +5105,9 @@ private static Set primaryAssignedFieldNames(MetaObject obj) { private static final Set NUMERIC_DISCRIMINATOR_SUBTYPES = Set.of( IntegerField.SUBTYPE_INT, LongField.SUBTYPE_LONG); + /** An optional minus sign and ASCII digits; matched whole, so no newline slips past. */ + private static final java.util.regex.Pattern INTEGER_LITERAL = java.util.regex.Pattern.compile("-?[0-9]+"); + static void validateDiscriminator(MetaRoot root) { List entities = new java.util.ArrayList<>(); for (MetaData rc : root.getChildren(MetaData.class, false)) { @@ -5089,7 +5154,7 @@ static void validateDiscriminator(MetaRoot root) { ErrorCode.ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH, obj.getSource()); } } else if (NUMERIC_DISCRIMINATOR_SUBTYPES.contains(st)) { - if (!value.matches("-?\\d+")) { + if (!INTEGER_LITERAL.matcher(value).matches()) { throw new MetaDataException( "ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH" + ": object.entity \"" + shortNameOf(obj) + "\" @discriminatorValue: \"" @@ -5107,10 +5172,16 @@ static void validateDiscriminator(MetaRoot root) { // Pass 3: ERR_DISCRIMINATOR_VALUE_DUPLICATE within each root's subtypes. for (Map.Entry> e : bindingsByRoot.entrySet()) { Map seen = new java.util.HashMap<>(); + String rootFieldName = ownAttrString(e.getKey(), MetaObject.ATTR_DISCRIMINATOR); + MetaField rootField = rootFieldName == null ? null : findFieldOnEntity(e.getKey(), rootFieldName); + boolean numericRoot = rootField != null && NUMERIC_DISCRIMINATOR_SUBTYPES.contains(rootField.getSubType()); for (Object[] b : e.getValue()) { MetaObject sub = (MetaObject) b[0]; String value = (String) b[1]; - MetaObject prev = seen.get(value); + // "1" and "01" are one value to a numeric column, so they are one claim. + String key = numericRoot && INTEGER_LITERAL.matcher(value).matches() + ? new java.math.BigInteger(value).toString() : value; + MetaObject prev = seen.get(key); if (prev != null) { throw new MetaDataException( "ERR_DISCRIMINATOR_VALUE_DUPLICATE" @@ -5119,7 +5190,7 @@ static void validateDiscriminator(MetaRoot root) { + shortNameOf(prev) + "\"", ErrorCode.ERR_DISCRIMINATOR_VALUE_DUPLICATE, sub.getSource()); } - seen.put(value, sub); + seen.put(key, sub); } } @@ -5327,6 +5398,39 @@ private static void walkRetiredRequirementLinks(MetaData node) { } } + /** + * ERR_COLUMN_NAME_COLLISION: two fields of one stored object resolving to the SAME + * physical column ({@code userId} + {@code userID} both snake_case to {@code user_id}, + * or two fields sharing an explicit {@code @column}). + * + *

Only the RESOLVED name is compared, through {@link com.metaobjects.database.ColumnNaming#resolve}, + * so an explicit {@code @column}, an inherited field and the project's column strategy are + * honoured. Scope: a concrete object declaring at least one source (a source-less object + * generates nothing); derived fields and {@code field.object} (may flatten to several + * columns) are skipped. ADR-0039: resolving accessors, so an inherited field collides like + * an own one.

+ */ + static void validateColumnNameCollisions(MetaRoot root) { + for (MetaObject obj : root.getChildren(MetaObject.class, false)) { + if (isAbstract(obj)) continue; + if (obj.getChildren(MetaSource.class, true).isEmpty()) continue; + java.util.Map> seen = new java.util.HashMap<>(); + for (MetaField field : obj.getChildren(MetaField.class, true)) { + if (ObjectField.SUBTYPE_OBJECT.equals(field.getSubType()) || field.isDerived()) continue; + String column = com.metaobjects.database.ColumnNaming.resolve(field); + MetaField first = seen.putIfAbsent(column, field); + if (first == null) continue; + throw new MetaDataException( + ErrorMessageConstants.ERR_COLUMN_NAME_COLLISION + + ": fields \"" + first.getName() + "\" and \"" + field.getName() + + "\" on \"" + obj.getName() + "\" both resolve to physical column \"" + + column + "\". Rename one field, or give one an explicit @column so the" + + " two columns differ.", + ErrorCode.ERR_COLUMN_NAME_COLLISION, field.getSource()); + } + } + } + static void validateIndexLookupFields(MetaRoot root) { for (MetaData rootChild : root.getChildren(MetaData.class, false)) { walkIndexLookupFields(rootChild); diff --git a/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java b/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java index 1613893814..92d0edc772 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java +++ b/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java @@ -303,7 +303,7 @@ && hasMetaAttr(ATTR_TABLE)) { // auto-generated N names so a nameless authored source falls // through to Step 4 (matches TS, where unnamed sources have name === ""). String shortName = getShortName(); - if (shortName != null && !shortName.isEmpty() && !isAutoGeneratedSourceName(shortName)) { + if (shortName != null && !shortName.isEmpty() && !isAutoNamed()) { return com.metaobjects.database.PhysicalNaming.applyTableRule(shortName, tableRule); } @@ -322,28 +322,6 @@ && hasMetaAttr(ATTR_TABLE)) { return ""; } - /** - * Returns {@code true} when the given short name matches the parser's - * auto-naming pattern for source nodes ({@code N} — e.g. {@code rdb1}). - * Mirrors {@link com.metaobjects.io.json.CanonicalJsonSerializer}'s - * suppression of auto-names on emit so the FR-016 four-step rule treats - * authored vs auto-generated source names consistently across ports. - */ - private boolean isAutoGeneratedSourceName(String shortName) { - String subType = getSubType(); - String expectedPrefix = (subType != null && !subType.isEmpty()) - ? subType.toLowerCase() - : TYPE_SOURCE.toLowerCase(); - // Auto-generated name matches EXACTLY . - if (!shortName.startsWith(expectedPrefix)) return false; - String tail = shortName.substring(expectedPrefix.length()); - if (tail.isEmpty()) return false; - for (int i = 0; i < tail.length(); i++) { - if (!Character.isDigit(tail.charAt(i))) return false; - } - return true; - } - /** * FROZEN — the DEFAULT PHYSICAL name derivation only (step 4 above, reached only * when metadata declares no explicit physical name). Tiny inflector mirroring the diff --git a/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java b/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java index 6af6112fdd..41402721ff 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java +++ b/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java @@ -256,6 +256,9 @@ private ErrorMessageConstants() { */ public static final String ERR_INVALID_INDEX = "ERR_INVALID_INDEX"; + /** Cross-language contract: {@code ERR_COLUMN_NAME_COLLISION}. */ + public static final String ERR_COLUMN_NAME_COLLISION = "ERR_COLUMN_NAME_COLLISION"; + /** FR-039 — a retired requirement may not declare @implementedBy. */ public static final String ERR_REQUIREMENT_RETIRED_HAS_IMPLEMENTORS = "ERR_REQUIREMENT_RETIRED_HAS_IMPLEMENTORS"; @@ -306,6 +309,9 @@ private ErrorMessageConstants() { */ public static final String ERR_ENUM_INT_VALUE_MAP_ARRAY = "ERR_ENUM_INT_VALUE_MAP_ARRAY"; + /** Cross-language contract: {@code ERR_ENUM_NORMALIZE_COLLISION}. */ + public static final String ERR_ENUM_NORMALIZE_COLLISION = "ERR_ENUM_NORMALIZE_COLLISION"; + /** * #208 (design doc §5 R6) warning: an {@code origin.*}-bearing (derived) own * field lives under a host object that declares an {@code @unmanaged} source. diff --git a/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java b/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java index ff616ce08f..2388c84395 100644 --- a/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java +++ b/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java @@ -429,4 +429,21 @@ public void testAuthoredValidatorNameEndingInDigitIsEmitted() { assertTrue("authored validator name 'length2' must be emitted (not suppressed as auto-generated), got: " + json, json.contains("\"name\": \"length2\"")); } + + // An authored name that merely LOOKS auto-generated ("required7" on a required + // validator) is an authored name: suppression is decided by the parser's + // auto-named flag, never by the shape of the name. + @Test + public void testAuthoredNameMatchingAutoPatternIsEmitted() { + MetaRoot root = new MetaRoot("test::pkg"); + ValueMetaObject obj = ValueMetaObject.create("test::pkg::Widget"); + RegexValidator validator = new RegexValidator("regex7"); + obj.addChild(validator); + root.addChild(obj); + + String json = CanonicalJsonSerializer.canonicalSerialize(root); + + assertTrue("authored 'regex7' must survive canonical serialization, got: " + json, + json.contains("\"name\": \"regex7\"")); + } } diff --git a/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java b/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java new file mode 100644 index 0000000000..6e772fe8ec --- /dev/null +++ b/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java @@ -0,0 +1,33 @@ +package com.metaobjects.library; + +import static org.junit.Assert.assertEquals; + +import java.util.List; +import java.util.Map; +import org.junit.Test; + +/** The manifest reader is a JSON parser, not a pattern: braces and quotes inside strings are data. */ +public class LibrarySourcesManifestTest { + + private static final String MANIFEST = "{\n" + + " \"description\": \"has } and ] and \\\"layers\\\" inside\",\n" + + " \"generators\": [ { \"anchor\": \"a::B\", \"name\": \"g\", \"note\": \"x}y\" } ],\n" + + " \"layers\": {\n" + + " \"\": { \"description\": \"core } here\", \"refs\": [\"x/model\", \"x/req\"] },\n" + + " \"db\": { \"refs\": [\"x/db\"] }\n" + + " }\n" + + "}"; + + @Test + public void layersSurviveBracesInsideStrings() { + Map> layers = LibrarySources.parseLayers(MANIFEST); + assertEquals(List.of("x/model", "x/req"), layers.get("")); + assertEquals(List.of("x/db"), layers.get("db")); + } + + @Test + public void generatorAnchorReadsKeysInAnyOrder() { + assertEquals(List.of("a::B"), LibrarySources.generatorAnchors(MANIFEST, "g")); + assertEquals(List.of(), LibrarySources.generatorAnchors(MANIFEST, "other")); + } +} diff --git a/server/java/render/src/main/java/com/metaobjects/render/Escapers.java b/server/java/render/src/main/java/com/metaobjects/render/Escapers.java index 1876f275a3..69f924006c 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/Escapers.java +++ b/server/java/render/src/main/java/com/metaobjects/render/Escapers.java @@ -72,11 +72,24 @@ static String injectionGuard(String s) { char first = s.charAt(0); if (first == '=' || first == '+' || first == '-' || first == '@' || first == '\t' || first == '\r') { - return "'" + s; + return isStrictNumeric(s) ? s : "'" + s; } return s; } + // ^-?[0-9]+(\.[0-9]+)?$ over ASCII digits, whole string. + private static boolean isStrictNumeric(String s) { + int i = !s.isEmpty() && s.charAt(0) == '-' ? 1 : 0; + int intStart = i; + while (i < s.length() && s.charAt(i) >= '0' && s.charAt(i) <= '9') i++; + if (i == intStart) return false; + if (i == s.length()) return true; + if (s.charAt(i) != '.') return false; + int fracStart = ++i; + while (i < s.length() && s.charAt(i) >= '0' && s.charAt(i) <= '9') i++; + return i > fracStart && i == s.length(); + } + static String escapeCsv(String s) { String guarded = injectionGuard(s); boolean needsQuote = guarded.indexOf(',') >= 0 diff --git a/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java new file mode 100644 index 0000000000..2f726ca9c4 --- /dev/null +++ b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java @@ -0,0 +1,139 @@ +package com.metaobjects.render; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; + +/** + * One Mustache tag scanner shared by {@link Verify} and the renderer's partial + * expansion. + * + *

Both used to find tags on their own ({@code Verify} with a {@code {{} search, + * {@code Renderer} with a {@code {{> x}}} regex), and neither knew what the engine + * knows: a comment ends at the first closing delimiter, {@code {{=<% %>=}}} changes + * the delimiters, and a section close must name the section it closes. {@link #scan} + * follows the Mustache spec for all three, so the two callers cannot disagree about + * where a tag is. Mirrors {@code Mustache.parse} in the TypeScript port and the C# and + * Python scanners. + */ +final class MustacheTags { + + enum Kind { NAME, TRIPLE, AMP, SECTION, INVERTED, CLOSE, PARTIAL, COMMENT, DELIMITERS } + + /** One tag: its kind, trimmed name, and {@code [start, end)} span in the text. */ + record Tag(Kind kind, String value, int start, int end) {} + + /** An unbalanced section; {@link #section()} is the section the finding is about. */ + static final class SyntaxException extends RuntimeException { + private final String section; + + SyntaxException(String message, String section) { + super(message); + this.section = section; + } + + String section() { + return section; + } + } + + private static final String DEFAULT_OPEN = "{{"; + private static final String DEFAULT_CLOSE = "}}"; + + private MustacheTags() {} + + /** + * Every tag in {@code text}, in order, honouring comments and delimiter changes. + * Throws {@link SyntaxException} for a close with no open, a close naming a different + * section than the one open, or a section never closed. An unterminated tag ends the + * scan (the rest is text), as it always did here. + */ + static List scan(String text) { + List tags = new ArrayList<>(); + String open = DEFAULT_OPEN; + String close = DEFAULT_CLOSE; + Deque openSections = new ArrayDeque<>(); + int pos = 0; + while (pos < text.length()) { + int start = text.indexOf(open, pos); + if (start < 0) break; + int contentStart = start + open.length(); + boolean triple = contentStart < text.length() && text.charAt(contentStart) == '{'; + String endDelim = triple ? "}" + close : close; + if (triple) contentStart++; + int closeIdx = text.indexOf(endDelim, contentStart); + if (closeIdx < 0) break; + String content = text.substring(contentStart, closeIdx).trim(); + int end = closeIdx + endDelim.length(); + pos = end; + + if (triple) { + tags.add(new Tag(Kind.TRIPLE, content, start, end)); + continue; + } + if (content.isEmpty()) continue; + char sigil = content.charAt(0); + String name = content.length() > 1 ? content.substring(1).trim() : ""; + switch (sigil) { + case '!' -> tags.add(new Tag(Kind.COMMENT, name, start, end)); + case '=' -> { + if (content.length() > 1 && content.endsWith("=")) { + String[] parts = content.substring(1, content.length() - 1).trim().split("\\s+"); + if (parts.length == 2) { + open = parts[0]; + close = parts[1]; + } + } + tags.add(new Tag(Kind.DELIMITERS, name, start, end)); + } + case '#', '^' -> { + openSections.push(name); + tags.add(new Tag(sigil == '#' ? Kind.SECTION : Kind.INVERTED, name, start, end)); + } + case '/' -> { + if (openSections.isEmpty()) { + throw new SyntaxException("Unopened section \"" + name + "\"", name); + } + String opened = openSections.pop(); + if (!opened.equals(name)) { + throw new SyntaxException("Unclosed section \"" + opened + "\"", opened); + } + tags.add(new Tag(Kind.CLOSE, name, start, end)); + } + case '>' -> tags.add(new Tag(Kind.PARTIAL, name, start, end)); + case '&' -> tags.add(new Tag(Kind.AMP, name, start, end)); + default -> tags.add(new Tag(Kind.NAME, content, start, end)); + } + } + if (!openSections.isEmpty()) { + String opened = openSections.peek(); + throw new SyntaxException("Unclosed section \"" + opened + "\"", opened); + } + return tags; + } + + /** + * Where a text that changes delimiters cannot render, or {@code null} when it can: inlined + * text is parsed under the surrounding delimiters. A partial body ({@code inlinedAs} is its + * ref) that changes them is reported at that ref; a text that changes them and inlines a + * partial is reported at its first partial. + */ + static String delimiterConflict(List tags, String inlinedAs) { + if (tags.stream().noneMatch(t -> t.kind() == Kind.DELIMITERS)) return null; + if (inlinedAs != null) return inlinedAs; + return tags.stream().filter(t -> t.kind() == Kind.PARTIAL).map(Tag::value).findFirst().orElse(null); + } + + /** {@code text} with every comment cut out: what a tag-presence check may scan. */ + static String withoutComments(String text, List tags) { + StringBuilder sb = new StringBuilder(text.length()); + int pos = 0; + for (Tag tag : tags) { + if (tag.kind() != Kind.COMMENT) continue; + sb.append(text, pos, tag.start()); + pos = tag.end(); + } + return sb.append(text, pos, text.length()).toString(); + } +} diff --git a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java index 53d8825390..baf1f4b674 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java +++ b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java @@ -10,9 +10,8 @@ import java.io.Writer; import java.util.ArrayDeque; import java.util.Deque; +import java.util.List; import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Mustache-driven render pipeline for the {@code template.*} metatype (FR-004). @@ -35,10 +34,6 @@ public final class Renderer { static final int MAX_DEPTH = 32; - // {{> name }} — captures the reference between the marker and the closing }}. - private static final Pattern PARTIAL_PATTERN = - Pattern.compile("\\{\\{>\\s*([^\\s}]+)\\s*\\}\\}"); - public String render(RenderRequest req) { validate(req); @@ -122,14 +117,26 @@ private static String preExpandPartials(String text, Provider provider, Deque ", stack) + ")"); } - Matcher m = PARTIAL_PATTERN.matcher(text); - if (!m.find()) return text; - m.reset(); + // Partial tags are located by the shared scanner, so one written inside a comment + // is not a partial and a {{=<% %>=}} delimiter change is honoured. + List tags; + try { + tags = MustacheTags.scan(text); + } catch (MustacheTags.SyntaxException e) { + throw new RenderException("template syntax error: " + e.getMessage()); + } + if (MustacheTags.delimiterConflict(tags, stack.peek()) != null) { + throw new RenderException( + "a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + } + List partials = tags.stream() + .filter(t -> t.kind() == MustacheTags.Kind.PARTIAL).toList(); + if (partials.isEmpty()) return text; StringBuilder out = new StringBuilder(text.length()); int last = 0; - while (m.find()) { - String partialRef = m.group(1); + for (MustacheTags.Tag tag : partials) { + String partialRef = tag.value(); if (stack.contains(partialRef)) { throw new RenderException("partial cycle: " + String.join(" -> ", stack) + " -> " + partialRef); @@ -144,9 +151,9 @@ private static String preExpandPartials(String text, Provider provider, Deque check( // The static text the output-tag check scans: the body plus every // provider-resolved partial body, collected during the single walk below. List staticTexts = new ArrayList<>(); - staticTexts.add(templateText); + + List bodyToks; + try { + bodyToks = tokenize(templateText); + } catch (MustacheTags.SyntaxException e) { + errors.add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); + return errors; + } + reportDelimiterConflict(templateText, null, errors); + staticTexts.add(staticText(templateText)); WalkContext ctx = new WalkContext(provider, root, errors, referencedAtRoot, staticTexts); - walk(tokenize(templateText), List.of(root), List.of(), ctx, false); + walk(bodyToks, List.of(root), List.of(), ctx, false); if (opts.requiredSlots() != null) { for (String slot : opts.requiredSlots()) { @@ -217,10 +230,18 @@ private static void walk( ctx.errors().add(new VerifyError(ERR_PARTIAL_UNRESOLVED, p.value())); break; } - ctx.staticTexts().add(text); + List partialToks; + try { + partialToks = tokenize(text); + } catch (MustacheTags.SyntaxException e) { + ctx.errors().add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); + break; + } + reportDelimiterConflict(text, p.value(), ctx.errors()); + ctx.staticTexts().add(staticText(text)); List nextSeen = new ArrayList<>(seen); nextSeen.add(p.value()); - walk(tokenize(text), stack, Collections.unmodifiableList(nextSeen), ctx, inElement); + walk(partialToks, stack, Collections.unmodifiableList(nextSeen), ctx, inElement); } } } @@ -288,61 +309,38 @@ private static boolean isBooleanAccessor(List> stack, String * cares about: variables (incl. {@code {{&x}}} and {@code {{{x}}}}), * sections + inverted sections (with nested children), and partials. * Comments and set-delimiter changes are dropped; plain text is skipped. + * Tags come from the shared scanner ({@link MustacheTags}); an unbalanced + * section throws {@link MustacheTags.SyntaxException}. */ static List tokenize(String text) { - int[] pos = {0}; - return parseTokens(text, pos); - } - - /** - * Parse tokens until EOF or a matching {@code {{/...}}} (which is consumed). - * Mirrors C# {@code Verify.cs ParseTokens()} — note that any close tag - * (matched or mismatched) ends the current level, which is intentionally - * lenient (templates that drift to imbalanced sections still tokenize). - */ - private static List parseTokens(String text, int[] pos) { - List tokens = new ArrayList<>(); - while (pos[0] < text.length()) { - int open = text.indexOf("{{", pos[0]); - if (open < 0) break; // remainder is plain text - boolean triple = open + 2 < text.length() && text.charAt(open + 2) == '{'; - String closeDelim = triple ? "}}}" : "}}"; - int contentStart = open + (triple ? 3 : 2); - int close = text.indexOf(closeDelim, contentStart); - if (close < 0) break; // malformed; stop - String content = text.substring(contentStart, close).trim(); - pos[0] = close + closeDelim.length(); - - if (triple) { - tokens.add(new VarTok(content)); - continue; - } - if (content.isEmpty()) continue; - - char sigil = content.charAt(0); - String name = content.length() > 1 ? content.substring(1).trim() : ""; - switch (sigil) { - case '!': - break; // comment - case '#': - case '^': - List children = parseTokens(text, pos); - tokens.add(new SectionTok(name, sigil == '^', children)); - break; - case '/': - return tokens; // close (matched or mismatched) ends this level - case '>': - tokens.add(new PartialTok(name)); - break; - case '&': - tokens.add(new VarTok(name)); - break; - default: - tokens.add(new VarTok(content)); // {{x}}, {{x.y}}, {{.}} - break; + List root = new ArrayList<>(); + Deque> levels = new ArrayDeque<>(); + levels.push(root); + for (MustacheTags.Tag tag : MustacheTags.scan(text)) { + switch (tag.kind()) { + case SECTION, INVERTED -> { + List children = new ArrayList<>(); + levels.peek().add(new SectionTok(tag.value(), tag.kind() == MustacheTags.Kind.INVERTED, children)); + levels.push(children); + } + case CLOSE -> levels.pop(); // the scanner already matched it to its open + case PARTIAL -> levels.peek().add(new PartialTok(tag.value())); + case NAME, TRIPLE, AMP -> levels.peek().add(new VarTok(tag.value())); // {{x}}, {{x.y}}, {{.}} + default -> { } // comments and delimiter changes carry no reference } } - return tokens; + return root; + } + + /** A delimiter change the renderer refuses ({@link MustacheTags#delimiterConflict}). */ + private static void reportDelimiterConflict(String text, String inlinedAs, List errors) { + String path = MustacheTags.delimiterConflict(MustacheTags.scan(text), inlinedAs); + if (path != null) errors.add(new VerifyError(ERR_TEMPLATE_SYNTAX, path)); + } + + /** {@code text} minus its comments, for the output-tag scan. */ + private static String staticText(String text) { + return MustacheTags.withoutComments(text, MustacheTags.scan(text)); } // ---------------- Required-tag matching ---------------- diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/JsonForgivingReader.java b/server/java/render/src/main/java/com/metaobjects/render/extract/JsonForgivingReader.java index b8bca97d5f..0aa10083eb 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/JsonForgivingReader.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/JsonForgivingReader.java @@ -107,19 +107,45 @@ private String readKey() { private Object readString(char quote) { i++; // opening quote StringBuilder sb = new StringBuilder(); - boolean esc = false; while (i < s.length()) { char c = s.charAt(i++); - if (esc) { sb.append(unescape(c)); esc = false; } - else if (c == '\\') esc = true; + if (c == '\\') sb.append(readEscape()); else if (c == quote) return sb.toString(); else sb.append(c); } return sb.toString(); // unterminated string → return what we have } - private static char unescape(char c) { - return switch (c) { case 'n' -> '\n'; case 't' -> '\t'; case 'r' -> '\r'; default -> c; }; + /** The escape after a consumed backslash: the full JSON grammar. A {@code \\uXXXX} escape is one + * UTF-16 unit, so a surrogate pair is two escapes appended in order. An unknown escape keeps its + * character and a truncated {@code \\u} keeps its text, so a cut-off reply still yields what it has. */ + private String readEscape() { + if (i >= s.length()) return ""; + char c = s.charAt(i++); + return switch (c) { + case 'n' -> "\n"; + case 't' -> "\t"; + case 'r' -> "\r"; + case 'b' -> "\b"; + case 'f' -> "\f"; + case 'u' -> { + int unit = hex4(); + yield unit < 0 ? "u" : String.valueOf((char) unit); + } + default -> String.valueOf(c); // \" \\ \/ and any unknown escape + }; + } + + private int hex4() { + if (i + 4 > s.length()) return -1; + int v = 0; + for (int k = 0; k < 4; k++) { + int d = Character.digit(s.charAt(i + k), 16); + if (d < 0 || s.charAt(i + k) > 'f') return -1; + v = v * 16 + d; + } + i += 4; + return v; } private Object readBareScalar() { diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java b/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java index 1d98e9dda8..6f7692d165 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java @@ -77,15 +77,26 @@ private static int scanBalanced(String s, int open) { return -1; } - /** Span of ...; if close absent, opener to end; null if no opener. */ + /** + * Span of ...; if close absent, opener to end; null if no opener. Comments and + * CDATA are not searched, and when another opener of the same root sits between the first + * one and its close (the tag mentioned in prose, then the real element) the span starts at + * the LAST such opener: a root does not nest in itself. + */ public static String xml(String text, String rootName, boolean caseInsensitive) { if (text == null || rootName == null) return null; int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0; - Matcher open = Pattern.compile("<" + Pattern.quote(rootName) + "(\\s[^>]*)?>", flags).matcher(text); + String view = XmlForgivingReader.maskNonMarkup(text); + Matcher open = Pattern.compile("<" + Pattern.quote(rootName) + "(\\s[^>]*)?>", flags).matcher(view); if (!open.find()) return null; int start = open.start(); - Matcher close = Pattern.compile("", flags).matcher(text); - if (close.find(open.end())) return text.substring(start, close.end()); - return text.substring(start); + Matcher close = Pattern.compile("", flags).matcher(view); + if (!close.find(open.end())) return text.substring(start); + int at = open.end(); + while (open.find(at) && open.start() < close.start()) { + start = open.start(); + at = open.end(); + } + return text.substring(start, close.end()); } } diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/Strip.java b/server/java/render/src/main/java/com/metaobjects/render/extract/Strip.java index 6121cc011d..5d009de042 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/Strip.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/Strip.java @@ -9,26 +9,79 @@ public final class Strip { private Strip() {} - // Captures the body inside a fenced block; optional language tag (json/xml/etc) is dropped. - private static final Pattern FENCE = Pattern.compile( - "```[a-zA-Z0-9_-]*\\s*\\r?\\n(.*?)\\r?\\n?```", - Pattern.DOTALL); + // An opening fence: a run of 3+ backticks, an optional language tag (json/xml/etc, dropped), a line end. + private static final Pattern OPEN = Pattern.compile("(`{3,})[a-zA-Z0-9_-]*\\s*\\r?\\n"); + + /** One fenced block: where it starts and ends in the reply, and its body. */ + private record Fence(int start, int end, String body) {} + + /** + * Every fenced block, in order. A fence closes at a line that starts with a backtick run at + * least as long as the opening one and carries nothing else, so backticks inside the payload + * (a JSON string holding `make` or a nested code sample) do not truncate it. A reply that + * closes on the payload's own line is accepted when that run ends the text. + */ + private static List fences(String raw) { + List out = new ArrayList<>(); + Matcher m = OPEN.matcher(raw); + int pos = 0; + while (pos <= raw.length() && m.find(pos)) { + int[] close = findClose(raw, m.end(), m.group(1).length()); + if (close == null) { pos = m.end(); continue; } + out.add(new Fence(m.start(), close[1], raw.substring(m.end(), close[0]))); + pos = close[1]; + } + return out; + } + + /** {bodyEnd, end} of the closing fence, or null when the block never closes. */ + private static int[] findClose(String raw, int from, int width) { + for (int ls = from; ls <= raw.length(); ) { + int i = ls; + while (i < raw.length() && (raw.charAt(i) == ' ' || raw.charAt(i) == '\t') && i - ls < 4) i++; + int j = i; + while (j < raw.length() && raw.charAt(j) == '`') j++; + if (j - i >= width) { + int k = j; + while (k < raw.length() && (raw.charAt(k) == ' ' || raw.charAt(k) == '\t')) k++; + if (k >= raw.length() || raw.charAt(k) == '\n' || raw.charAt(k) == '\r') { + return new int[] {bodyEnd(raw, from, ls), j}; + } + } + int nl = raw.indexOf('\n', ls); + if (nl < 0) break; + ls = nl + 1; + } + // No closing line: accept a final backtick run that ends the text on the payload's own line. + int tail = raw.length(); + while (tail > from && Character.isWhitespace(raw.charAt(tail - 1))) tail--; + int k = tail; + while (k > from && raw.charAt(k - 1) == '`') k--; + return tail - k >= width && k > from ? new int[] {k, tail} : null; + } + + private static int bodyEnd(String raw, int from, int lineStart) { + int e = lineStart; + if (e > from && raw.charAt(e - 1) == '\n') e--; + if (e > from && raw.charAt(e - 1) == '\r') e--; + return e; + } /** The body of every fenced block, in order. Locate searches these before the whole text, * because a model told to fence its answer puts the answer there. */ public static List fencedBodies(String raw) { List out = new ArrayList<>(); if (raw == null) return out; - Matcher m = FENCE.matcher(raw); - while (m.find()) out.add(m.group(1)); + for (Fence f : fences(raw)) out.add(f.body()); return out; } public static String strip(String raw) { if (raw == null) return ""; - Matcher m = FENCE.matcher(raw); - if (m.find()) { - return (raw.substring(0, m.start()) + m.group(1) + raw.substring(m.end())).trim(); + List fences = fences(raw); + if (!fences.isEmpty()) { + Fence f = fences.get(0); + return (raw.substring(0, f.start()) + f.body() + raw.substring(f.end())).trim(); } return raw.trim(); } diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java index 3d7f285123..4b1110957e 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java @@ -4,6 +4,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,14 +51,118 @@ public final class XmlForgivingReader { private static final Pattern ATTR = Pattern.compile( "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"); - public Map read(String span, boolean caseInsensitive) { + // a character an identifier can end with: '<' right after one is a type argument, not a tag. + private static boolean isIdentChar(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_'; + } + + private static final String COMMENT_OPEN = ""; + private static final String CDATA_OPEN = ""; + // the five predefined XML entities plus numeric character references. + private static final Pattern ENTITY = Pattern.compile("&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);"); + + private static final String KIND_COMMENT = "comment"; + private static final String KIND_CDATA = "cdata"; + + /** + * Rewrite every XML comment and CDATA section in {@code text} through {@code replace} + * (kind, region), scanning left to right so a comment marker inside CDATA (or the reverse) + * is never mistaken for markup. An unclosed comment is left as text; an unclosed CDATA + * section runs to the end. + */ + private static String rewriteRegions(String text, BiFunction replace) { + StringBuilder out = new StringBuilder(); + int pos = 0; + int from = 0; + while (true) { + int c = text.indexOf(COMMENT_OPEN, from); + int d = text.indexOf(CDATA_OPEN, from); + boolean isComment = c >= 0 && (d < 0 || c < d); + int start = isComment ? c : d; + if (start < 0) break; + int end; + if (isComment) { + int e = text.indexOf(COMMENT_CLOSE, start + COMMENT_OPEN.length()); + if (e < 0) { + from = start + COMMENT_OPEN.length(); + continue; + } + end = e + COMMENT_CLOSE.length(); + } else { + int e = text.indexOf(CDATA_CLOSE, start + CDATA_OPEN.length()); + end = e < 0 ? text.length() : e + CDATA_CLOSE.length(); + } + out.append(text, pos, start) + .append(replace.apply(isComment ? KIND_COMMENT : KIND_CDATA, text.substring(start, end))); + pos = end; + from = end; + } + return out.append(text.substring(pos)).toString(); + } + + /** {@code text} with comments and CDATA sections blanked to spaces (same length, so + * indices hold): the view the tag locators search, so markup inside them is never located. */ + static String maskNonMarkup(String text) { + return rewriteRegions(text, (kind, region) -> " ".repeat(region.length())); + } + + /** Drop comments; turn each CDATA section into its text with {@code & < >} entity-escaped, + * so the tag scanner cannot split it and the entity decode restores it exactly. */ + private static String prepare(String text) { + return rewriteRegions(text, (kind, region) -> { + if (kind.equals(KIND_COMMENT)) return ""; + String body = region.endsWith(CDATA_CLOSE) + ? region.substring(CDATA_OPEN.length(), region.length() - CDATA_CLOSE.length()) + : region.substring(CDATA_OPEN.length()); + return body.replace("&", "&").replace("<", "<").replace(">", ">"); + }); + } + + /** Decode predefined entities and numeric references in one pass (so {@code &lt;} is + * {@code <}). An unknown or out-of-range reference stays as written. */ + private static String decodeEntities(String s) { + if (s.indexOf('&') < 0) return s; + Matcher m = ENTITY.matcher(s); + StringBuilder sb = new StringBuilder(); + int last = 0; + while (m.find()) { + String body = m.group(1); + String rep = m.group(); + switch (body) { + case "lt" -> rep = "<"; + case "gt" -> rep = ">"; + case "amp" -> rep = "&"; + case "quot" -> rep = "\""; + case "apos" -> rep = "'"; + default -> { + try { + int cp = body.charAt(1) == 'x' + ? Integer.parseInt(body.substring(2), 16) + : Integer.parseInt(body.substring(1), 10); + boolean surrogate = cp >= 0xD800 && cp <= 0xDFFF; + if (cp >= 1 && cp <= 0x10FFFF && !surrogate) rep = new String(Character.toChars(cp)); + } catch (NumberFormatException ignored) { + // out-of-range reference: stays as written + } + } + } + sb.append(s, last, m.start()).append(rep); + last = m.end(); + } + return sb.append(s.substring(last)).toString(); + } + + public Map read(String rawSpan, boolean caseInsensitive) { Map out = new LinkedHashMap<>(); - if (span == null || span.isBlank()) return out; + if (rawSpan == null || rawSpan.isBlank()) return out; + String span = prepare(rawSpan); int gt = span.indexOf('>'); if (gt < 0) return out; int rootEnd = span.lastIndexOf(" read(String span, boolean caseInsensitive) { public Map readRootless(String text, boolean caseInsensitive) { Map out = new LinkedHashMap<>(); if (text == null || text.isBlank()) return out; - parseChildren(text, caseInsensitive, out, null); + parseChildren(prepare(text), caseInsensitive, out, null, false); return out; } /** * Parse {@code inner}'s elements into {@code out}. When {@code loose} is non-null, the text * BETWEEN those elements (an element's own text in mixed content) is appended to it, one - * segment per gap. + * segment per gap. {@code literalUnclosed} is set for the body of an element that DID + * close: an open tag in there that directly follows an identifier character and has no close + * tag anywhere after it ({@code List and ...}) is prose, not an element. One after + * {@code >}, whitespace or the start of the body is a child whose close the model forgot. */ - private void parseChildren(String inner, boolean ci, Map out, List loose) { + private void parseChildren(String inner, boolean ci, Map out, List loose, + boolean literalUnclosed) { Pattern openTag = Pattern.compile(OPEN_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); + Pattern closeTag = Pattern.compile(CLOSE_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); Matcher m = openTag.matcher(inner); int pos = 0; - while (m.find(pos)) { + int scan = 0; + while (m.find(scan)) { + if (literalUnclosed && m.start() > 0 && isIdentChar(inner.charAt(m.start() - 1)) + && !m.group(2).stripTrailing().endsWith("/") + && !closeTag.matcher(inner).region(m.end(), inner.length()).find()) { + scan = m.end(); + continue; + } if (loose != null) loose.add(inner.substring(pos, m.start())); String tag = m.group(1); String key = ci ? tag.toLowerCase() : tag; @@ -96,6 +213,7 @@ private void parseChildren(String inner, boolean ci, Map out, Li if (selfClosing) { accumulate(out, key, attrs.isEmpty() ? "" : attrs); pos = m.end(); + scan = pos; continue; } @@ -103,7 +221,8 @@ private void parseChildren(String inner, boolean ci, Map out, Li String closeRe = ""; Matcher close = Pattern.compile(closeRe, ci ? Pattern.CASE_INSENSITIVE : 0).matcher(inner); int contentEnd, next; - if (close.find(contentStart)) { + boolean closed = close.find(contentStart); + if (closed) { contentEnd = close.start(); next = close.end(); } else { @@ -141,8 +260,9 @@ private void parseChildren(String inner, boolean ci, Map out, Li } } String content = inner.substring(contentStart, contentEnd); - accumulate(out, key, combine(attrs, content, ci)); + accumulate(out, key, combine(attrs, content, ci, closed)); pos = next; + scan = next; } if (loose != null) loose.add(inner.substring(pos)); } @@ -171,11 +291,11 @@ private static int[] findStrayClose(String inner, int from, int to, boolean ci) * content keeps BOTH: the children, and the element's own text under {@link #TEXT_KEY}, so * a scalar or {@code @xmlText} consumer still reads the prose around a child element. */ - private Object combine(Map attrs, String content, boolean ci) { + private Object combine(Map attrs, String content, boolean ci, boolean closed) { if (content.contains("<")) { Map nested = new LinkedHashMap<>(); List loose = new ArrayList<>(); - parseChildren(content, ci, nested, loose); + parseChildren(content, ci, nested, loose, closed); if (!nested.isEmpty()) { // attributes first; a child element takes precedence on a name collision Map merged = new LinkedHashMap<>(attrs); @@ -194,7 +314,7 @@ private static String mixedText(List segments, boolean ci) { Pattern close = Pattern.compile(CLOSE_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); List parts = new ArrayList<>(); for (String seg : segments) { - String t = close.matcher(seg).replaceAll("").trim(); + String t = decodeEntities(close.matcher(seg).replaceAll("").trim()); if (!t.isEmpty()) parts.add(t); } return String.join(" ", parts); @@ -203,7 +323,7 @@ private static String mixedText(List segments, boolean ci) { /** A text body: bare {@code String} when there are no attributes, else a map carrying the * attributes plus the text under {@link #TEXT_KEY}. */ private Object textValue(Map attrs, String content) { - String text = content.trim(); + String text = decodeEntities(content.trim()); if (attrs.isEmpty()) return text; Map m = new LinkedHashMap<>(attrs); m.put(TEXT_KEY, text); @@ -219,7 +339,7 @@ private Map parseAttrs(String rawAttrs, boolean ci) { String val = a.group(2) != null ? a.group(2) : a.group(3) != null ? a.group(3) : a.group(4) != null ? a.group(4) : ""; - attrs.putIfAbsent(name, val); + attrs.putIfAbsent(name, decodeEntities(val)); } return attrs; } diff --git a/server/java/render/src/test/java/com/metaobjects/render/EscapersTest.java b/server/java/render/src/test/java/com/metaobjects/render/EscapersTest.java index 54773b8194..dde024eea8 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/EscapersTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/EscapersTest.java @@ -25,7 +25,8 @@ public class EscapersTest { @Test public void csvFormulaInjectionGuard() { assertEquals("'=SUM(A1:A5)", Escapers.escape("csv", "=SUM(A1:A5)")); assertEquals("'+1+1", Escapers.escape("csv", "+1+1")); - assertEquals("'-1", Escapers.escape("csv", "-1")); + assertEquals("-1", Escapers.escape("csv", "-1")); + assertEquals("'-1a", Escapers.escape("csv", "-1a")); assertEquals("'@cmd", Escapers.escape("csv", "@cmd")); assertEquals("'\tTAB", Escapers.escape("csv", "\tTAB")); } diff --git a/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java b/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java index 09e59143ab..730876e30c 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java @@ -58,7 +58,8 @@ public class VerifyConformanceTest { Verify.ERR_VAR_NOT_ON_PAYLOAD, Verify.ERR_PARTIAL_UNRESOLVED, Verify.ERR_REQUIRED_SLOT_UNUSED, - Verify.ERR_OUTPUT_TAG_MISSING + Verify.ERR_OUTPUT_TAG_MISSING, + Verify.ERR_TEMPLATE_SYNTAX ); static { diff --git a/server/java/render/src/test/java/com/metaobjects/render/VerifyTest.java b/server/java/render/src/test/java/com/metaobjects/render/VerifyTest.java index a312e54300..094e34a4d6 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/VerifyTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/VerifyTest.java @@ -18,6 +18,14 @@ public void scalarVariableOnPayloadOk() { assertTrue(errors.isEmpty()); } + @Test + public void emptySetDelimiterTagIsIgnoredNotACrash() { + var errors = Verify.check("{{=}}Hello {{name}}", + List.of(PayloadField.scalar("name")), + VerifyOptions.empty()); + assertTrue(errors.isEmpty()); + } + @Test public void variableNotOnPayloadFlagged() { var errors = Verify.check("Hello {{missing}}", diff --git a/server/java/render/src/test/java/com/metaobjects/render/extract/JsonForgivingReaderTest.java b/server/java/render/src/test/java/com/metaobjects/render/extract/JsonForgivingReaderTest.java index 6d8782f1a1..5057b0036a 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/extract/JsonForgivingReaderTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/extract/JsonForgivingReaderTest.java @@ -115,4 +115,10 @@ public void emptyValueThenMoreKeysContinues() { assertSame(JsonForgivingReader.TRUNCATED, m.get("a")); assertEquals("2", m.get("b")); } + + @Test + public void fullJsonEscapeGrammar() { + Map m = read("{\"a\":\"caf\\u00e9 \\ud83d\\ude00 a\\/b\\bx\\fy \\\"q\\\" \\\\ \\u12\"}"); + org.junit.Assert.assertEquals("caf\u00e9 \ud83d\ude00 a/b\bx\fy \"q\" \\ u12", m.get("a")); + } } diff --git a/server/java/render/src/test/java/com/metaobjects/render/extract/StripTest.java b/server/java/render/src/test/java/com/metaobjects/render/extract/StripTest.java index 2d3fcd5df0..5f53628238 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/extract/StripTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/extract/StripTest.java @@ -34,4 +34,26 @@ public void noFenceReturnsTrimmedInput() { public void nullSafe() { assertEquals("", Strip.strip(null)); } + + @Test + public void backticksInsideAJsonStringDoNotCloseTheFence() { + assertEquals("{\"summary\": \"run ```make``` first\", \"score\": 7}", + Strip.strip("```json\n{\"summary\": \"run ```make``` first\", \"score\": 7}\n```")); + } + + @Test + public void longerOpeningFenceClosesOnlyOnAnEqualOrLongerRun() { + assertEquals("before\n```\ninner\n```\nafter\ntail", + Strip.strip("````md\nbefore\n```\ninner\n```\nafter\n````\ntail")); + } + + @Test + public void closingFenceOnThePayloadLineIsAcceptedAtTheEnd() { + assertEquals("{\"a\":1}", Strip.strip("```json\n{\"a\":1}```")); + } + + @Test + public void unclosedFenceIsNotAFence() { + assertEquals("```json\n{\"a\":1}", Strip.strip("```json\n{\"a\":1}")); + } } diff --git a/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt b/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt new file mode 100644 index 0000000000..abfd547468 --- /dev/null +++ b/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt @@ -0,0 +1 @@ +Hi Ada }} \ No newline at end of file diff --git a/server/java/render/src/test/resources/snapshots/render-csv-numeric-exempt.txt b/server/java/render/src/test/resources/snapshots/render-csv-numeric-exempt.txt new file mode 100644 index 0000000000..3b75a745bc --- /dev/null +++ b/server/java/render/src/test/resources/snapshots/render-csv-numeric-exempt.txt @@ -0,0 +1,6 @@ +id,value +1,-5 +2,-5.25 +3,'=1+1 +4,'-cmd +5,'@x diff --git a/server/python/pyproject.toml b/server/python/pyproject.toml index 6750dd1c32..8963592002 100644 --- a/server/python/pyproject.toml +++ b/server/python/pyproject.toml @@ -50,7 +50,7 @@ Issues = "https://github.com/metaobjectsdev/metaobjects/issues" Changelog = "https://github.com/metaobjectsdev/metaobjects/blob/main/CHANGELOG.md" [project.optional-dependencies] -dev = ["pytest>=8", "pytest-xdist>=3.6", "mypy>=1.10", "ruff>=0.6", "pydantic>=2"] +dev = ["pytest>=8", "pytest-xdist>=3.6", "mypy>=1.10", "ruff>=0.6", "pydantic>=2.8"] # On-demand cross-language conformance suites — persistence + api-contract. # pg8000 is pure-python so this extra installs cleanly without native deps; # the persistence harness shells out to the docker CLI directly (matches the @@ -114,7 +114,7 @@ dev = [ # The validator-parity corpus (tests/codegen/test_validation_conformance.py) # imports pydantic to exercise the generated Pydantic models, so it must be here # or the CI `uv run pytest` collection fails with ModuleNotFoundError. - "pydantic>=2", + "pydantic>=2.8", # ruff is the Python codegen format pass (src/metaobjects/codegen/format.py); the # cli-registry corpus runs codegen, so without ruff in the group `uv run pytest` # fails at format time with "No module named ruff". diff --git a/server/python/src/metaobjects/cli.py b/server/python/src/metaobjects/cli.py index 79204d894d..06817f3b69 100644 --- a/server/python/src/metaobjects/cli.py +++ b/server/python/src/metaobjects/cli.py @@ -1877,7 +1877,9 @@ def _relative_set(root: Path, nested: list[Path] | None = None) -> dict[str, str if p.suffix in _UNCOMPARED_SUFFIXES: continue try: - files[str(rel)] = p.read_text(encoding="utf-8") + # Bytes, not read_text(): text mode translates \r\n and lone \r to \n, so a + # committed CRLF file would compare equal to the LF the generator wrote. + files[str(rel)] = p.read_bytes().decode("utf-8") except (UnicodeDecodeError, OSError): continue return files diff --git a/server/python/src/metaobjects/codegen/collection_name_collision.py b/server/python/src/metaobjects/codegen/collection_name_collision.py index f0abe1aad4..8958016b6d 100644 --- a/server/python/src/metaobjects/codegen/collection_name_collision.py +++ b/server/python/src/metaobjects/codegen/collection_name_collision.py @@ -16,10 +16,14 @@ """ from __future__ import annotations -from metaobjects.apidocs.naming import pluralize +from metaobjects.apidocs.naming import pluralize, route_path from metaobjects.errors import ErrorCode from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.core.object.object_constants import OBJECT_SUBTYPE_VALUE +from metaobjects.meta.persistence.source.meta_source import MetaSource +from metaobjects.runtime.tph import tph_subtype_of + +from .instance_artifacts import is_abstract ERR_COLLECTION_NAME_COLLISION = ErrorCode.ERR_COLLECTION_NAME_COLLISION.value @@ -56,3 +60,31 @@ def assert_no_collection_name_collisions(entities: list[MetaObject]) -> None: ) else: owner_of_plural[plural] = obj + _assert_no_route_segment_collisions(entities) + + +def _assert_no_route_segment_collisions(entities: list[MetaObject]) -> None: + """The route half. The REST segment is the entity NAME snake_cased and then + pluralized (:func:`route_path`), so ``ApiKey`` and ``APIKey`` both serve + ``/api_keys`` while their pluralized names differ and pass the check above. + Counts only objects that actually get a route: not a value object, not abstract, + not sourceless, not a TPH subtype (folded into its base and served under it).""" + owner_of_segment: dict[str, MetaObject] = {} + for obj in entities: + if obj.sub_type == OBJECT_SUBTYPE_VALUE or is_abstract(obj): + continue + # ADR-0039: children() resolves — an inherited source still gets a route. + if not any(isinstance(c, MetaSource) for c in obj.children()): + continue + if tph_subtype_of(obj) is not None: + continue + segment = route_path(obj.name) + existing = owner_of_segment.get(segment) + if existing is None: + owner_of_segment[segment] = obj + elif existing.resolution_key() != obj.resolution_key(): + raise ValueError( + f'{ERR_COLLECTION_NAME_COLLISION}: "{existing.name}" and "{obj.name}" both resolve ' + f'to the REST collection segment "{segment}" — rename one entity so its ' + "snake_cased, pluralized route segment is distinct." + ) diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index c135099a2d..39f613418a 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -155,8 +155,9 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: packaged = packaged_runtime_path(module) reference = packaged.read_text(encoding="utf-8") if rt.exists(): - behind, own = _line_diff(rt.read_text(encoding="utf-8"), reference) - out.append(f" runtime {rel}: kept your copy — {_status(behind, own)} " + mine = rt.read_text(encoding="utf-8") + behind, _own = _line_diff(mine, reference) + out.append(f" runtime {rel}: kept your copy — {_compare(mine, reference)} " "(eject never overwrites runtime)") if behind: out.append(f' {n} may need the newer copy: diff -u "{rt}" "{packaged}"; ' @@ -259,15 +260,22 @@ def packaged_module_path(entry: GeneratorEntry) -> Path: return Path(module.__file__) +def _ordered(text: str) -> list[str]: + return [line.rstrip() for line in text.splitlines() if line.strip()] + + def _normalised(text: str) -> Counter[str]: - return Counter(line.rstrip() for line in text.splitlines() if line.strip()) + return Counter(_ordered(text)) def owned_status(root: Path, entry: GeneratorEntry) -> str | None: """``identical`` / ``DIFFERS: N behind, M of your own``, or ``None`` if not owned. - Line-multiset comparison, the same shape as TS ``owned-copy.ts``: *behind* counts - reference lines the copy lacks, *of your own* counts copy lines the reference lacks. + ``identical`` means the reference lines in the reference order (trailing whitespace and + blank lines aside): an edit that only reorders + lines is a customisation, because ``--undo`` deletes an ``identical`` copy. *behind* + counts reference lines the copy lacks, *of your own* counts copy lines the reference + lacks, as in TS ``owned-copy.ts``. """ path = owned_path(root, entry.name) if entry.source is None or not path.exists(): @@ -312,13 +320,18 @@ def _line_diff(mine_text: str, ref_text: str) -> tuple[int, int]: def _status(behind: int, own: int) -> str: - if behind == 0 and own == 0: - return "identical" return f"DIFFERS: {behind} behind, {own} of your own" def _compare(mine_text: str, ref_text: str) -> str: - return _status(*_line_diff(mine_text, ref_text)) + """``identical`` only for the reference lines in the reference order, trailing whitespace and + blank lines aside. The line counts ignore order, so a reorder reads ``DIFFERS: 0 behind, 0 of + your own, reordered``.""" + if _ordered(mine_text) == _ordered(ref_text): + return "identical" + behind, own = _line_diff(mine_text, ref_text) + status = _status(behind, own) + return f"{status}, reordered" if behind == 0 and own == 0 else status def runtime_status(root: Path) -> list[tuple[Path, str]]: diff --git a/server/python/src/metaobjects/codegen/generators/entity_model.py b/server/python/src/metaobjects/codegen/generators/entity_model.py index 42587b2eec..885c1ae8b7 100644 --- a/server/python/src/metaobjects/codegen/generators/entity_model.py +++ b/server/python/src/metaobjects/codegen/generators/entity_model.py @@ -8,6 +8,7 @@ another object shares its short name).""" from __future__ import annotations +import re from metaobjects.meta.core.field.meta_field import MetaField from metaobjects.meta.core.object.meta_object import MetaObject @@ -94,6 +95,42 @@ def _first_attr(field: MetaField, sub_type: str, attr_name: str) -> object | Non return None +class _AsciiPattern(str): + """A ``validator.regex`` / hostname expression emitted as ``re.compile(..., re.ASCII)``. + + Pydantic's default ``pattern=`` string runs on pydantic-core's Rust regex, where ``\\d`` + ``\\w`` ``\\s`` are Unicode (Arabic-Indic digits pass) and lookaround is unsupported. A + COMPILED pattern is run on Python ``re`` instead, so ``re.ASCII`` can pin the classes to + ASCII like the TS and Java ports and like the runtime validator (validator_runner.py).""" + + schema_pattern: str = "" + + +def _full_match_pattern(raw: str, field: MetaField) -> _AsciiPattern: + """FR-036 Pin 2 — a validator.regex is a FULL match. A compiled pydantic pattern is a + ``search``, so anchor the authored expression in a non-capturing group (alternation + still spans the whole value). ``\\Z`` not ``$``: ``$`` would also accept a trailing + newline. + + Fails at generation when Python ``re`` cannot compile the expression (for example a + Java/ICU ``\\p{L}`` property class, which Python lacks), instead of emitting a module + that raises at import. The runtime validator reports the same pattern as invalid.""" + anchored = f"^(?:{raw})\\Z" + try: + re.compile(anchored, re.ASCII) + except re.error as exc: + raise ValueError( + f"field '{field.name}': validator pattern {raw!r} is not valid in Python's re " + f"({exc}); Unicode property classes such as \\p{{L}} are not supported by the " + "Python port, spell the character class out" + ) from exc + pat = _AsciiPattern(anchored) + # The published JSON-Schema / OpenAPI pattern must be ECMA-262 valid: ``\\Z`` is a + # literal ``Z`` there, so schema consumers get the portable ``$`` form instead. + pat.schema_pattern = f"^(?:{raw})$" + return pat + + def _validator_constraints(field: MetaField, *, wire: bool) -> dict[str, object]: """Map the field's ``validator.*`` children + field attrs to Pydantic ``Field`` kwargs. Cross-port-canonical semantics (TS is the reference): @@ -160,7 +197,7 @@ def _validator_constraints(field: MetaField, *, wire: bool) -> dict[str, object] # is search/partial (un-anchored), so wrap the authored expression in an # anchored non-capturing group so alternation still spans the whole value # (a redundant anchor on an already-anchored pattern matches identically). - kwargs["pattern"] = f"^(?:{pattern})$" + kwargs["pattern"] = _full_match_pattern(pattern, field) break # ADR-0036/0037 Wave 3 — @stringFormat: hostname -> a codegen-owned canonical @@ -168,7 +205,7 @@ def _validator_constraints(field: MetaField, *, wire: bool) -> dict[str, object] # every port replicates the byte-identical form). @stringFormat: email is bound # as EmailStr in _field_line (a type, not a pattern), so it is not handled here. if field.attrs().get(fc.FIELD_ATTR_STRING_FORMAT) == fc.STRING_FORMAT_HOSTNAME: - kwargs["pattern"] = _HOSTNAME_REGEX + kwargs["pattern"] = _full_match_pattern(_HOSTNAME_REGEX, field) # FR-036 Pin 1 (#224 — WIRE TIER ONLY) — a @required non-array string is # non-empty by default: reject null / "" but ACCEPT whitespace-only. Emit an @@ -201,7 +238,18 @@ def _validator_constraints(field: MetaField, *, wire: bool) -> dict[str, object] def _constraint_parts(constraints: dict[str, object]) -> list[str]: """The ``Field(...)`` constraint kwargs rendered in the stable, deterministic order.""" - return [f"{k}={constraints[k]!r}" for k in _CONSTRAINT_ORDER if k in constraints] + return [_constraint_source(k, constraints[k]) for k in _CONSTRAINT_ORDER if k in constraints] + + +def _constraint_source(key: str, value: object) -> str: + if isinstance(value, _AsciiPattern): + # json_schema_extra overrides the schema ``pattern`` (the compiled one carries the + # Python-only ``\\Z`` anchor) with an ECMA-262-valid form for OpenAPI consumers. + return ( + f"{key}=re.compile({str(value)!r}, re.ASCII), " + f"json_schema_extra={{'pattern': {value.schema_pattern!r}}}" + ) + return f"{key}={value!r}" def _type_expr_for_field( @@ -750,6 +798,8 @@ def render_entity_model( # The base-class import is subtracted back out: when an entity BOTH extends a # target and navigates to it, the base class is needed at class-creation time and # deferring it would break the module for a different reason. + if any("re.compile(" in line for line in (*body, *create_lines, *patch_lines)): + imports.add("import re") deferred = self._deferred_nav_imports( entity, object_index, config if config is not None else GenConfig(out_dir="") ) - { @@ -830,6 +880,16 @@ def _render_shared_enums_module(self, entities: list[MetaObject]) -> EmittedFile # Member names follow the Python convention (UPPER_CASE constants); the # value preserves the wire form, so ``Enum("statistical")`` and # ``member == "statistical"`` still work (str-enum). + names: dict[str, str] = {} + for m in e.values: + # Two values folding to one member name ('Open' / 'open') would make the + # second an alias of the first at import time, silently dropping a value. + if m.upper() in names: + raise ValueError( + f"shared enum '{e.name}': values '{names[m.upper()]}' and '{m}' both map " + f"to the Python member {m.upper()}; rename one so the members are distinct" + ) + names[m.upper()] = m parts += [f' {m.upper()} = "{m}"' for m in e.values] parts.append("") return EmittedFile(path="enums.py", content=ruff_format("\n".join(parts))) diff --git a/server/python/src/metaobjects/codegen/project_config.py b/server/python/src/metaobjects/codegen/project_config.py index 3f98021eed..8f57b78150 100644 --- a/server/python/src/metaobjects/codegen/project_config.py +++ b/server/python/src/metaobjects/codegen/project_config.py @@ -30,6 +30,7 @@ from metaobjects.codegen.config import DEFAULT_API_PREFIX from metaobjects.naming import removed_naming_source_message +from metaobjects.source.yaml_core_schema import safe_load_core #: Config filename looked up in the cwd when ``--config`` is not given. CONFIG_FILENAME = "metaobjects.config.yaml" @@ -374,7 +375,7 @@ def load_project_config(path: Path) -> ProjectConfig: except (OSError, UnicodeDecodeError) as exc: raise ConfigError(f"{path}: cannot read config file: {exc}") from exc try: - raw = yaml.safe_load(text) + raw = safe_load_core(text) except yaml.YAMLError as exc: raise ConfigError(f"{path}: invalid YAML: {exc}") from exc diff --git a/server/python/src/metaobjects/errors.py b/server/python/src/metaobjects/errors.py index 0e27deaa5c..fc8a3b55e5 100644 --- a/server/python/src/metaobjects/errors.py +++ b/server/python/src/metaobjects/errors.py @@ -201,6 +201,9 @@ class ErrorCode(str, Enum): # Cross-language vocabulary for features other ports added (FR-003 storage, FR-004 template); # the Python loader does not emit these yet, but the enum tracks the shared corpus codes. ERR_INVALID_TEMPLATE = "ERR_INVALID_TEMPLATE" + # Two fields of one stored object resolve to the same PHYSICAL column (snake_case + # folds `userId` and `userID` together, or two fields share an @column). + ERR_COLUMN_NAME_COLLISION = "ERR_COLUMN_NAME_COLLISION" ERR_STORAGE_FLATTENED_ARRAY = "ERR_STORAGE_FLATTENED_ARRAY" ERR_STORAGE_WITHOUT_OBJECT_REF = "ERR_STORAGE_WITHOUT_OBJECT_REF" # ADR-0013: a field.object REQUIRES @objectRef (open/untyped JSON uses the @@ -290,6 +293,9 @@ class ErrorCode(str, Enum): # an integer array. An array-of-enum stays string-backed: drop @intValueMap, # or make the field scalar. ERR_ENUM_INT_VALUE_MAP_ARRAY = "ERR_ENUM_INT_VALUE_MAP_ARRAY" + # Two declared members of one field.enum normalize to the same key under the + # default `@normalize: strip`; rename a member or set `collapse` / `none`. + ERR_ENUM_NORMALIZE_COLLISION = "ERR_ENUM_NORMALIZE_COLLISION" ERR_UNKNOWN = "ERR_UNKNOWN" diff --git a/server/python/src/metaobjects/loader/validate_discriminator.py b/server/python/src/metaobjects/loader/validate_discriminator.py index 055ffbd79c..08adbd58f3 100644 --- a/server/python/src/metaobjects/loader/validate_discriminator.py +++ b/server/python/src/metaobjects/loader/validate_discriminator.py @@ -38,7 +38,8 @@ _NUMERIC_DISCRIMINATOR_SUBTYPES = frozenset( {FIELD_SUBTYPE_INT, FIELD_SUBTYPE_LONG} ) -_INT_RE = re.compile(r"^-?\d+$") +# ASCII digits only and `\Z`: `\d` matches every Unicode digit and `$` matches before a trailing newline. +_INT_RE = re.compile(r"-?[0-9]+\Z") def _find_field_on_entity(entity: MetaData, name: str) -> MetaData | None: @@ -154,8 +155,13 @@ def validate_discriminator(root: MetaData, errors: list[MetaError]) -> None: # Pass 3: ERR_DISCRIMINATOR_VALUE_DUPLICATE within each root's subtypes. for _disc_root, bindings in bindings_by_root: seen: dict[str, MetaData] = {} + root_field_name = _disc_root.attr(OBJECT_ATTR_DISCRIMINATOR) # Python attr() is own-only: the root declares it + root_field = _find_field_on_entity(_disc_root, root_field_name) if isinstance(root_field_name, str) else None + numeric_root = root_field is not None and root_field.sub_type in _NUMERIC_DISCRIMINATOR_SUBTYPES for subtype, value in bindings: - prev = seen.get(value) + # "1" and "01" are one value to a numeric column, so they are one claim. + key = str(int(value)) if numeric_root and _INT_RE.match(value) else value + prev = seen.get(key) if prev is not None: errors.append( MetaError( @@ -167,7 +173,7 @@ def validate_discriminator(root: MetaData, errors: list[MetaError]) -> None: ) ) else: - seen[value] = subtype + seen[key] = subtype # Pass 4: ERR_DISCRIMINATOR_VALUE_MISSING — every concrete entity that extends # a @discriminator-bearing root must declare a value. diff --git a/server/python/src/metaobjects/loader/validate_enum_normalize_ambiguity.py b/server/python/src/metaobjects/loader/validate_enum_normalize_ambiguity.py index ebc3f836c4..a537626af2 100644 --- a/server/python/src/metaobjects/loader/validate_enum_normalize_ambiguity.py +++ b/server/python/src/metaobjects/loader/validate_enum_normalize_ambiguity.py @@ -34,6 +34,7 @@ NORMALIZE_DEFAULT, ) from ..shared.base_types import TYPE_FIELD, TYPE_OBJECT +from ..errors import ErrorCode, MetaError from ..source.error_source import LoaderWarning WARN_ENUM_NORMALIZE_AMBIGUOUS = "WARN_ENUM_NORMALIZE_AMBIGUOUS" @@ -161,3 +162,53 @@ def visit(node: MetaData) -> None: visit(child) visit(root) + + +def validate_enum_normalize_collision(root: MetaData, errors: list[MetaError]) -> None: + """``ERR_ENUM_NORMALIZE_COLLISION`` — two declared members of one enum fold to + the SAME key under ``@normalize: strip`` (e.g. ``READ_ONLY`` and ``READONLY``). + + One of them is then unreachable and a reply naming it coerces to the other + (wrong-and-green). An ERROR, unlike the concatenation advisory above: the + vocabulary itself is ambiguous. ``collapse`` / ``none`` are not gated. + Mirrors the TS reference.""" + + def visit(node: MetaData) -> None: + if node.type == TYPE_FIELD and node.sub_type == FIELD_SUBTYPE_ENUM: + # ADR-0039 sanctioned own: the vocabulary DECLARED here; an inheriting + # enum shares the super's set, already checked there. + raw = node.own_attrs().get(FIELD_ATTR_VALUES) + if isinstance(raw, list) and len(raw) > 1: + mode, explicit = _effective_normalize(node) + if mode == NORMALIZE_DEFAULT: + by_key: dict[str, list[str]] = {} + for m in raw: + member = str(m) + key = _strip_normalize(member) + if key: + by_key.setdefault(key, []).append(member) + for key, members in by_key.items(): + # Distinct members only: an exact duplicate is ERR_BAD_ATTR_VALUE's job. + distinct = list(dict.fromkeys(members)) + if len(distinct) < 2: + continue + listed = ", ".join(f"'{d}'" for d in distinct) + errors.append( + MetaError( + f'field.enum "{node.name}" members {listed} all normalize to ' + f"'{key}' under @{FIELD_ATTR_NORMALIZE}: '{NORMALIZE_DEFAULT}'" + f"{'' if explicit else ' (the default)'}, which upper-cases and " + f"deletes everything outside A-Z0-9, so a reply naming one of " + f"them is indistinguishable from the others. Rename the members " + f"so they differ after that fold, or set " + f"@{FIELD_ATTR_NORMALIZE}: 'collapse' (keeps '_' distinct) or " + f"'none' (exact match) on this field.", + ErrorCode.ERR_ENUM_NORMALIZE_COLLISION, + envelope=node.source, + ) + ) + break # one error per declaring node + for child in node.own_children(): + visit(child) + + visit(root) diff --git a/server/python/src/metaobjects/loader/validation_passes.py b/server/python/src/metaobjects/loader/validation_passes.py index 47c02628b5..34e1886d77 100644 --- a/server/python/src/metaobjects/loader/validation_passes.py +++ b/server/python/src/metaobjects/loader/validation_passes.py @@ -14,7 +14,10 @@ from ..errors import ErrorCode, MetaError from ..source.error_source import LoaderWarning from .validate_source_physical_names import validate_source_physical_names -from .validate_enum_normalize_ambiguity import validate_enum_normalize_ambiguity +from .validate_enum_normalize_ambiguity import ( + validate_enum_normalize_ambiguity, + validate_enum_normalize_collision, +) from .validate_field_mutability import validate_field_mutability from .validate_discriminator import validate_discriminator from .validate_source_parameter_ref import validate_source_parameter_ref @@ -236,6 +239,8 @@ def run_validations( # Authoring guard — a field.enum vocabulary ambiguous under the default # @normalize: strip. WARN_ENUM_NORMALIZE_AMBIGUOUS. validate_enum_normalize_ambiguity(root, envelope_warnings, warnings) + # Two members folding to one key under `strip` — ERR_ENUM_NORMALIZE_COLLISION. + validate_enum_normalize_collision(root, errors) # FR-014 — TPH discriminator cross-attribute rules. validate_discriminator(root, errors) # FR-015 — source.rdb @parameterRef typed-input rules. @@ -261,6 +266,8 @@ def run_validations( # #335 Half B — @sortable on an array field, or a subtype with no operator band → error. _validate_sortable_has_supported_subtype(root, errors) _validate_index_lookup_fields(root, errors) + # Two fields of one stored object resolving to the same physical column. + _validate_column_name_collisions(root, errors) # FR-039 -- a retired requirement carries no @implementedBy (refused, not exempted, # so the dangling-ref class is unreachable) and @supersededBy is legal only there. _validate_retired_requirement_links(root, errors) @@ -4191,6 +4198,45 @@ def _validate_index_lookup_fields(root: MetaData, errors: list[MetaError]) -> No # --------------------------------------------------------------------------- +def _validate_column_name_collisions(root: MetaData, errors: list[MetaError]) -> None: + """ERR_COLUMN_NAME_COLLISION: two fields of one stored object resolving to the SAME + physical column (``userId`` + ``userID`` both snake_case to ``user_id``, or two + fields sharing an explicit ``@column``). + + Only the RESOLVED name is compared, through :func:`resolve_column_name`, so an + explicit ``@column``, an inherited field and the project's column strategy are + honoured. Scope: a concrete object declaring at least one source (a source-less + object generates nothing); derived fields and ``field.object`` (may flatten to + several columns) are skipped. ADR-0039: effective children (inherited fields count). + """ + from ..naming import resolve_column_name + from ..meta.core.field.meta_field import MetaField + for obj in (c for c in root.children() if c.type == TYPE_OBJECT): + if obj.is_abstract: + continue + if not any(c.type == TYPE_SOURCE for c in obj.children()): + continue + seen: dict[str, MetaData] = {} + for field in obj.children(): + if not isinstance(field, MetaField) or field.sub_type == FIELD_SUBTYPE_OBJECT \ + or field.is_derived(): + continue + column = resolve_column_name(field) + first = seen.get(column) + if first is None: + seen[column] = field + continue + errors.append(MetaError( + code=ErrorCode.ERR_COLUMN_NAME_COLLISION, + message=( + f'fields "{first.name}" and "{field.name}" on "{obj.name}" both resolve ' + f'to physical column "{column}". Rename one field, or give one an ' + f"explicit @column so the two columns differ." + ), + envelope=field.source, + )) + + def _validate_retired_requirement_links(root: MetaData, errors: list[MetaError]) -> None: from ..meta.core.requirement.requirement_constants import ( REQUIREMENT_ATTR_IMPLEMENTED_BY, diff --git a/server/python/src/metaobjects/render/escapers.py b/server/python/src/metaobjects/render/escapers.py index 187cc0ab9b..379e6c8a89 100644 --- a/server/python/src/metaobjects/render/escapers.py +++ b/server/python/src/metaobjects/render/escapers.py @@ -10,6 +10,7 @@ "triple-stache" raw form is preserved by the renderer). """ from __future__ import annotations +import re FORMAT_TEXT = "text" FORMAT_HTML = "html" @@ -38,13 +39,17 @@ def _escape_xml(s: str) -> str: return "".join(out) +# Strict numeric literal (ASCII digits only, whole string) is exempt from the guard. +_STRICT_NUMERIC = re.compile(r"-?[0-9]+(\.[0-9]+)?", re.ASCII) + + def _injection_guard(s: str) -> str: """OWASP CSV/Excel formula-injection guard: prefix a literal apostrophe when the cell starts with an active char (``=``, ``+``, ``-``, ``@``, tab, CR).""" if not s: return s first = s[0] - if first in ("=", "+", "-", "@", "\t", "\r"): + if first in ("=", "+", "-", "@", "\t", "\r") and not _STRICT_NUMERIC.fullmatch(s): return "'" + s return s diff --git a/server/python/src/metaobjects/render/extract/coerce.py b/server/python/src/metaobjects/render/extract/coerce.py index 8c45740341..66affce920 100644 --- a/server/python/src/metaobjects/render/extract/coerce.py +++ b/server/python/src/metaobjects/render/extract/coerce.py @@ -216,8 +216,9 @@ def _coerce_int( return MALFORMED # Integer parse first (matches Java Long.parseLong / C# long.TryParse), then a # float fallback (matches Java's Double.parseDouble fallback). + # The int stays an int: float() would round a long above 2**53. try: - return _clamp(float(int(trimmed)), spec, path, report, as_long=True, lenient=lenient) + return _clamp(int(trimmed), spec, path, report, as_long=True, lenient=lenient) except ValueError: pass try: @@ -241,7 +242,7 @@ def _coerce_double( def _clamp( - n: float, spec: FieldSpec, path: str, report: ExtractionReport, as_long: bool, lenient: bool + n: float | int, spec: FieldSpec, path: str, report: ExtractionReport, as_long: bool, lenient: bool ) -> object: """Apply the field's @min/@max range (sourced from its numeric validator). Under LENIENT tolerance an out-of-range value is CLAMPED to the bound (recorded as a "clamp" coercion); diff --git a/server/python/src/metaobjects/render/extract/json_forgiving_reader.py b/server/python/src/metaobjects/render/extract/json_forgiving_reader.py index 9da057fbf8..7bbf784201 100644 --- a/server/python/src/metaobjects/render/extract/json_forgiving_reader.py +++ b/server/python/src/metaobjects/render/extract/json_forgiving_reader.py @@ -162,21 +162,50 @@ def _read_key(self) -> str | None: def _read_string(self, quote: str) -> str: self._i += 1 # opening quote out: list[str] = [] - esc = False while self._i < len(self._s): c = self._s[self._i] self._i += 1 - if esc: - out.append(_unescape(c)) - esc = False - elif c == "\\": - esc = True + if c == "\\": + out.append(self._read_escape()) elif c == quote: return "".join(out) else: out.append(c) return "".join(out) # unterminated string → return what we have + def _read_escape(self) -> str: + """The escape after a consumed backslash: the full JSON grammar. A ``\\uXXXX`` high + surrogate followed by a ``\\uXXXX`` low surrogate is one code point; a lone surrogate + or an unknown escape keeps its character, and a truncated ``\\u`` keeps its text, so a + cut-off reply still yields what it has.""" + if self._i >= len(self._s): + return "" + c = self._s[self._i] + self._i += 1 + simple = {"n": "\n", "t": "\t", "r": "\r", "b": "\b", "f": "\f"} + if c in simple: + return simple[c] + if c != "u": + return c # \" \\ \/ and any unknown escape + unit = self._hex4() + if unit is None: + return "u" + if 0xD800 <= unit < 0xDC00 and self._s.startswith("\\u", self._i): + save = self._i + self._i += 2 + low = self._hex4() + if low is not None and 0xDC00 <= low < 0xE000: + return chr(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)) + self._i = save + return chr(unit) + + def _hex4(self) -> int | None: + digits = self._s[self._i : self._i + 4] + if len(digits) == 4 and all(d in "0123456789abcdefABCDEF" for d in digits): + self._i += 4 + return int(digits, 16) + return None + def _read_bare_scalar(self) -> object | None: start = self._i # A comment after an unquoted value (`7 // good`) ends it; `http://x` does not. @@ -227,13 +256,3 @@ def comment_end(s: str, i: int) -> int: return j close = s.find("*/", i + 2) return len(s) if close < 0 else close + 2 - - -def _unescape(c: str) -> str: - if c == "n": - return "\n" - if c == "t": - return "\t" - if c == "r": - return "\r" - return c diff --git a/server/python/src/metaobjects/render/extract/locate.py b/server/python/src/metaobjects/render/extract/locate.py index b5d6c024e2..f791f61ea6 100644 --- a/server/python/src/metaobjects/render/extract/locate.py +++ b/server/python/src/metaobjects/render/extract/locate.py @@ -8,6 +8,7 @@ import re from metaobjects.render.extract.json_forgiving_reader import comment_end +from metaobjects.render.extract.xml_forgiving_reader import mask_non_markup def json(text: str | None) -> str | None: @@ -84,17 +85,29 @@ def _scan_balanced(s: str, open_idx: int) -> int: def xml(text: str | None, root_name: str | None, case_insensitive: bool) -> str | None: - """Span of ``...``; if close absent, opener to end; ``None`` if no opener.""" + """Span of ``...``; if close absent, opener to end; ``None`` if no opener. + + Comments and CDATA are not searched, and when another opener of the same root sits between + the first one and its close (the tag mentioned in prose, then the real element) the span + starts at the LAST such opener: a root does not nest in itself.""" if text is None or root_name is None: return None flags = re.IGNORECASE if case_insensitive else 0 + view = mask_non_markup(text) open_re = re.compile("<" + re.escape(root_name) + r"(\s[^>]*)?>", flags) - open_m = open_re.search(text) + open_m = open_re.search(view) if open_m is None: return None start = open_m.start() close_re = re.compile("", flags) - close_m = close_re.search(text, open_m.end()) - if close_m is not None: - return text[start : close_m.end()] - return text[start:] + close_m = close_re.search(view, open_m.end()) + if close_m is None: + return text[start:] + at = open_m.end() + while True: + nxt = open_re.search(view, at) + if nxt is None or nxt.start() >= close_m.start(): + break + start = nxt.start() + at = nxt.end() + return text[start : close_m.end()] diff --git a/server/python/src/metaobjects/render/extract/strip.py b/server/python/src/metaobjects/render/extract/strip.py index 1755a72bc3..44c9b612d2 100644 --- a/server/python/src/metaobjects/render/extract/strip.py +++ b/server/python/src/metaobjects/render/extract/strip.py @@ -5,18 +5,84 @@ from __future__ import annotations import re +from typing import NamedTuple -# Captures the body inside a fenced block; optional language tag (json/xml/etc) is dropped. -_FENCE = re.compile(r"```[a-zA-Z0-9_-]*[ \t]*\r?\n(.*?)\r?\n?```", re.DOTALL) +# An opening fence: a run of 3+ backticks, an optional language tag (json/xml/etc, dropped), a line end. +_OPEN = re.compile(r"(`{3,})[a-zA-Z0-9_-]*\s*\r?\n") + + +class _Fence(NamedTuple): + start: int # index of the opening backticks + end: int # index just past the closing backticks + body: str + + +def _fences(raw: str) -> list[_Fence]: + """Every fenced block, in order. A fence closes at a line that starts with a backtick run at + least as long as the opening one and carries nothing else, so backticks inside the payload (a + JSON string holding ``make`` or a nested code sample) do not truncate it. A reply that closes + on the payload's own line is accepted when that run ends the text.""" + out: list[_Fence] = [] + pos = 0 + while True: + m = _OPEN.search(raw, pos) + if m is None: + return out + close = _find_close(raw, m.end(), len(m.group(1))) + if close is None: + pos = m.end() + continue + body_end, end = close + out.append(_Fence(m.start(), end, raw[m.end() : body_end])) + pos = end + + +def _find_close(raw: str, start: int, width: int) -> tuple[int, int] | None: + ls = start + while ls <= len(raw): + i = ls + while i < len(raw) and raw[i] in " \t" and i - ls < 4: + i += 1 + j = i + while j < len(raw) and raw[j] == "`": + j += 1 + if j - i >= width: + k = j + while k < len(raw) and raw[k] in " \t": + k += 1 + if k >= len(raw) or raw[k] in "\r\n": + return _body_end(raw, start, ls), j + nl = raw.find("\n", ls) + if nl < 0: + break + ls = nl + 1 + # No closing line: accept a final backtick run that ends the text on the payload's own line. + tail = raw.rstrip() + k = len(tail) + while k > start and tail[k - 1] == "`": + k -= 1 + if len(tail) - k >= width and k > start: + return k, len(tail) + return None + + +def _body_end(raw: str, start: int, line_start: int) -> int: + e = line_start + if e > start and raw[e - 1] == "\n": + e -= 1 + if e > start and raw[e - 1] == "\r": + e -= 1 + return e def strip(raw: str | None) -> str: """Strip the first markdown code fence, splicing its body back in place; else trim.""" if raw is None: return "" - m = _FENCE.search(raw) - if m: - return (raw[: m.start()] + m.group(1) + raw[m.end():]).strip() + fences = _fences(raw) + if fences: + f = fences[0] + return (raw[: f.start] + f.body + raw[f.end :]).strip() return raw.strip() @@ -25,4 +91,4 @@ def fenced_bodies(raw: str | None) -> list[str]: text, because a model told to fence its answer puts the answer there.""" if raw is None: return [] - return [m.group(1) for m in _FENCE.finditer(raw)] + return [f.body for f in _fences(raw)] diff --git a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py index 468be5a8e0..4ee86c899c 100644 --- a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py +++ b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py @@ -21,6 +21,7 @@ from __future__ import annotations import re +from collections.abc import Callable #: Reserved key holding an element's own text content when the element is represented as a #: dict (because it also carries attributes). ``#`` is not a legal XML name char, so it never @@ -35,22 +36,106 @@ # the body) — LLMs commonly end a block with the wrong tag. _CLOSE_TAG = re.compile(r"") _CLOSE_TAG_CI = re.compile(r"", re.IGNORECASE) +# a character an identifier can end with: '<' right after one is a type argument, not a tag. +_IDENT_CHAR = re.compile(r"[A-Za-z0-9_]") # one attribute: name = "double" | 'single' | bareword. _ATTR = re.compile(r"""([A-Za-z_:][A-Za-z0-9_:.\-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s/>]+))""") +_COMMENT_OPEN = "" +_CDATA_OPEN = "" +# the five predefined XML entities plus numeric character references. +_ENTITY = re.compile(r"&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);") +_NAMED_ENTITIES = {"lt": "<", "gt": ">", "amp": "&", "quot": '"', "apos": "'"} +_KIND_COMMENT = "comment" +_KIND_CDATA = "cdata" + + +def _rewrite_regions(text: str, replace: Callable[[str, str], str]) -> str: + """Rewrite every XML comment and CDATA section in ``text`` through ``replace(kind, region)``, + scanning left to right so a comment marker inside CDATA (or the reverse) is never mistaken + for markup. An unclosed comment is left as text; an unclosed CDATA section runs to the end.""" + out: list[str] = [] + pos = 0 + frm = 0 + while True: + c = text.find(_COMMENT_OPEN, frm) + d = text.find(_CDATA_OPEN, frm) + is_comment = c >= 0 and (d < 0 or c < d) + start = c if is_comment else d + if start < 0: + break + if is_comment: + e = text.find(_COMMENT_CLOSE, start + len(_COMMENT_OPEN)) + if e < 0: + frm = start + len(_COMMENT_OPEN) + continue + end = e + len(_COMMENT_CLOSE) + else: + e = text.find(_CDATA_CLOSE, start + len(_CDATA_OPEN)) + end = len(text) if e < 0 else e + len(_CDATA_CLOSE) + out.append(text[pos:start]) + out.append(replace(_KIND_COMMENT if is_comment else _KIND_CDATA, text[start:end])) + pos = end + frm = end + out.append(text[pos:]) + return "".join(out) + + +def mask_non_markup(text: str) -> str: + """``text`` with comments and CDATA sections blanked to spaces (same length, so indices + hold): the view the tag locators search, so markup inside them is never located.""" + return _rewrite_regions(text, lambda _kind, region: " " * len(region)) + + +def _prepare(text: str) -> str: + """Drop comments; turn each CDATA section into its text with ``& < >`` entity-escaped, so + the tag scanner cannot split it and the entity decode restores it exactly.""" + + def replace(kind: str, region: str) -> str: + if kind == _KIND_COMMENT: + return "" + body = ( + region[len(_CDATA_OPEN) : -len(_CDATA_CLOSE)] + if region.endswith(_CDATA_CLOSE) + else region[len(_CDATA_OPEN) :] + ) + return body.replace("&", "&").replace("<", "<").replace(">", ">") + + return _rewrite_regions(text, replace) + + +def _decode_entities(s: str) -> str: + """Decode predefined entities and numeric references in one pass (so ``&lt;`` is + ``<``). An unknown or out-of-range reference stays as written.""" + if "&" not in s: + return s + + def sub(m: re.Match[str]) -> str: + body = m.group(1) + if not body.startswith("#"): + return _NAMED_ENTITIES[body] + cp = int(body[2:], 16) if body[1] == "x" else int(body[1:], 10) + surrogate = 0xD800 <= cp <= 0xDFFF + return chr(cp) if 1 <= cp <= 0x10FFFF and not surrogate else m.group(0) + + return _ENTITY.sub(sub, s) + class XmlForgivingReader: - def read(self, span: str | None, case_insensitive: bool) -> dict[str, object]: + def read(self, raw_span: str | None, case_insensitive: bool) -> dict[str, object]: out: dict[str, object] = {} - if span is None or span.strip() == "": + if raw_span is None or raw_span.strip() == "": return out + span = _prepare(raw_span) gt = span.find(">") if gt < 0: return out root_end = span.rfind(" dict[str, object]: @@ -61,21 +146,40 @@ def read_rootless(self, text: str | None, case_insensitive: bool) -> dict[str, o out: dict[str, object] = {} if text is None or text.strip() == "": return out - self._parse_children(text, case_insensitive, out, None) + self._parse_children(_prepare(text), case_insensitive, out, None, False) return out def _parse_children( - self, inner: str, ci: bool, out: dict[str, object], loose: list[str] | None + self, + inner: str, + ci: bool, + out: dict[str, object], + loose: list[str] | None, + literal_unclosed: bool, ) -> None: """Parse ``inner``'s elements into ``out``. When ``loose`` is not ``None``, the text BETWEEN those elements (an element's own text in mixed content) is appended to it, - one segment per gap.""" + one segment per gap. ``literal_unclosed`` is set for the body of an element that DID + close: an open tag in there that directly follows an identifier character and has no + close tag anywhere after it (``List and ...``) is prose, not an element. One after + ``>``, whitespace or the start of the body is a child whose close the model forgot.""" open_tag = _OPEN_TAG_CI if ci else _OPEN_TAG + any_close = _CLOSE_TAG_CI if ci else _CLOSE_TAG pos = 0 + scan = 0 while True: - m = open_tag.search(inner, pos) + m = open_tag.search(inner, scan) if m is None: break + if ( + literal_unclosed + and m.start() > 0 + and _IDENT_CHAR.match(inner[m.start() - 1]) is not None + and not (m.group(2) or "").rstrip().endswith("/") + and any_close.search(inner, m.end()) is None + ): + scan = m.end() + continue if loose is not None: loose.append(inner[pos : m.start()]) tag = m.group(1) @@ -90,6 +194,7 @@ def _parse_children( if self_closing: self._accumulate(out, key, "" if not attrs else attrs) pos = m.end() + scan = pos continue content_start = m.end() @@ -130,8 +235,9 @@ def _parse_children( if stray is not None: content_end, nxt = stray content = inner[content_start:content_end] - self._accumulate(out, key, self._combine(attrs, content, ci)) + self._accumulate(out, key, self._combine(attrs, content, ci, close_m is not None)) pos = nxt + scan = nxt if loose is not None: loose.append(inner[pos:]) @@ -154,7 +260,9 @@ def _find_stray_close( return (m.start(), m.end()) at = m.end() - def _combine(self, attrs: dict[str, object], content: str, ci: bool) -> object: + def _combine( + self, attrs: dict[str, object], content: str, ci: bool, closed: bool + ) -> object: """Combine an element's attributes with its body (nested children or plain text). Mixed content keeps BOTH: the children, and the element's own text under :data:`TEXT_KEY`, so a scalar or ``@xmlText`` consumer still reads the prose @@ -162,7 +270,7 @@ def _combine(self, attrs: dict[str, object], content: str, ci: bool) -> object: if "<" in content: nested: dict[str, object] = {} loose: list[str] = [] - self._parse_children(content, ci, nested, loose) + self._parse_children(content, ci, nested, loose, closed) if nested: # attributes first; a child element wins a name collision merged: dict[str, object] = dict(attrs) @@ -179,13 +287,13 @@ def _mixed_text(self, segments: list[str], ci: bool) -> str: close_re = _CLOSE_TAG_CI if ci else _CLOSE_TAG parts: list[str] = [] for seg in segments: - t = close_re.sub("", seg).strip() + t = _decode_entities(close_re.sub("", seg).strip()) if t: parts.append(t) return " ".join(parts) def _text_value(self, attrs: dict[str, object], content: str) -> object: - text = content.strip() + text = _decode_entities(content.strip()) if not attrs: return text m: dict[str, object] = dict(attrs) @@ -204,7 +312,7 @@ def _parse_attrs(self, raw_attrs: str, ci: bool) -> dict[str, object]: ) ) if name not in attrs: - attrs[name] = val + attrs[name] = _decode_entities(val) return attrs def _accumulate(self, out: dict[str, object], key: str, value: object) -> None: diff --git a/server/python/src/metaobjects/render/mustache_tags.py b/server/python/src/metaobjects/render/mustache_tags.py new file mode 100644 index 0000000000..8df7d34896 --- /dev/null +++ b/server/python/src/metaobjects/render/mustache_tags.py @@ -0,0 +1,128 @@ +"""One Mustache tag scanner shared by ``verify`` and the renderer's partial expansion. + +Both used to find tags with their own ``{{`` search (verify) or a ``{{> x}}`` regex +(render), and neither knew what the engine knows: a comment ends at the first +closing delimiter, ``{{=<% %>=}}`` changes the delimiters, and a section close must +name the section it closes. :func:`scan` follows the Mustache spec for all three, so +the two callers cannot disagree about where a tag is. + +Mirrors ``Mustache.parse`` in the TypeScript port (``mustache.js``), ``MustacheTags`` in +the C# and Java ports. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +DEFAULT_OPEN = "{{" +DEFAULT_CLOSE = "}}" + +#: Tag kinds. ``name`` is a plain ``{{x}}``; ``{`` is ``{{{x}}}``; the rest are the sigil. +KIND_NAME = "name" +KIND_TRIPLE = "{" +KIND_AMP = "&" +KIND_SECTION = "#" +KIND_INVERTED = "^" +KIND_CLOSE = "/" +KIND_PARTIAL = ">" +KIND_COMMENT = "!" +KIND_DELIMITERS = "=" + + +@dataclass(frozen=True) +class Tag: + """One tag: its kind, its trimmed name, and its ``[start, end)`` span in the text.""" + + kind: str + value: str + start: int + end: int + + +class MustacheSyntaxError(ValueError): + """An unbalanced section. ``section`` is the section name the finding is about: + the unopened close's name, or the section left open.""" + + def __init__(self, message: str, section: str) -> None: + super().__init__(message) + self.section = section + + +def scan(text: str) -> list[Tag]: + """Every tag in ``text``, in order, honouring comments and delimiter changes. + + Raises :class:`MustacheSyntaxError` for a close with no open, a close that names a + different section than the one open, or a section never closed. An unterminated tag + ends the scan (the rest is text), as it always did here. + """ + tags: list[Tag] = [] + open_delim, close_delim = DEFAULT_OPEN, DEFAULT_CLOSE + open_sections: list[str] = [] + pos = 0 + while pos < len(text): + start = text.find(open_delim, pos) + if start < 0: + break + content_start = start + len(open_delim) + triple = text.startswith("{", content_start) + end_delim = "}" + close_delim if triple else close_delim + if triple: + content_start += 1 + close = text.find(end_delim, content_start) + if close < 0: + break + content = text[content_start:close].strip() + end = close + len(end_delim) + pos = end + + if triple: + tags.append(Tag(KIND_TRIPLE, content, start, end)) + continue + if not content: + continue + sigil = content[0] + name = content[1:].strip() if len(content) > 1 else "" + if sigil == KIND_COMMENT: + tags.append(Tag(KIND_COMMENT, name, start, end)) + elif sigil == KIND_DELIMITERS: + parts = content[1:].removesuffix("=").split() if content.endswith("=") else [] + if len(parts) == 2: + open_delim, close_delim = parts + tags.append(Tag(KIND_DELIMITERS, name, start, end)) + elif sigil in (KIND_SECTION, KIND_INVERTED): + open_sections.append(name) + tags.append(Tag(sigil, name, start, end)) + elif sigil == KIND_CLOSE: + if not open_sections: + raise MustacheSyntaxError(f'Unopened section "{name}"', name) + opened = open_sections.pop() + if opened != name: + raise MustacheSyntaxError(f'Unclosed section "{opened}"', opened) + tags.append(Tag(KIND_CLOSE, name, start, end)) + elif sigil in (KIND_PARTIAL, KIND_AMP): + tags.append(Tag(sigil, name, start, end)) + else: + tags.append(Tag(KIND_NAME, content, start, end)) + if open_sections: + opened = open_sections[-1] + raise MustacheSyntaxError(f'Unclosed section "{opened}"', opened) + return tags + + +def delimiter_change(tags: list[Tag]) -> str | None: + """The delimiters of the first ``{{=<% %>=}}`` change in ``tags`` (``"<% %>"``), or ``None``. + The Python renderer refuses every such change.""" + change = next((t for t in tags if t.kind == KIND_DELIMITERS), None) + return None if change is None else " ".join(change.value.removesuffix("=").split()) + + +def without_comments(text: str, tags: list[Tag]) -> str: + """``text`` with every comment tag cut out — what a tag-presence check may scan.""" + out: list[str] = [] + pos = 0 + for tag in tags: + if tag.kind == KIND_COMMENT: + out.append(text[pos : tag.start]) + pos = tag.end + out.append(text[pos:]) + return "".join(out) diff --git a/server/python/src/metaobjects/render/renderer.py b/server/python/src/metaobjects/render/renderer.py index feb23860d5..a230dcf9ba 100644 --- a/server/python/src/metaobjects/render/renderer.py +++ b/server/python/src/metaobjects/render/renderer.py @@ -10,15 +10,13 @@ cross-port whitespace and lets the engine own cycle detection. The Mustache subset implemented (interpolation, raw triple-stache, sections, inverted sections, partials, comments) is the corpus's vocabulary — set-delimiter -directives are intentionally NOT supported (matches the verify side and the -TS/C#/Java engines). +directives are rejected loudly (verify understands them; this interpreter does not). Mirrors ``server/java/render/.../Renderer.java`` and ``server/csharp/MetaObjects.Render/Renderer.cs``. """ from __future__ import annotations -import re from dataclasses import dataclass, field from typing import Any @@ -30,6 +28,7 @@ partial_engine_mismatch, template_engine_of, ) +from .mustache_tags import KIND_PARTIAL, MustacheSyntaxError, delimiter_change, scan from .payload_accessors import with_derived_accessors from .verify import ( ERR_REQUIRED_SLOT_UNUSED, @@ -40,10 +39,6 @@ ) from .verify import verify as verify_template -# {{> name }} — captures the reference between the marker and the closing }}. -_PARTIAL_PATTERN = re.compile(r"\{\{>\s*([^\s}]+)\s*\}\}") - - class RenderError(Exception): """Anything that goes wrong during render (unresolved partial, cycle, etc.).""" @@ -145,9 +140,20 @@ def _pre_expand_partials(text: str, provider: Provider, stack: list[str]) -> str raise RenderError( f"partial depth exceeded {MAX_DEPTH} (chain: {' -> '.join(stack)})" ) - - def replace(match: re.Match[str]) -> str: - ref = match.group(1) + try: + tags = scan(text) + except MustacheSyntaxError as e: + raise RenderError(f"template syntax error: {e}") from e + if delimiter_change(tags) is not None: + raise RenderError( + "set-delimiter tags ({{=<% %>=}}) are not supported by the Python renderer" + ) + out: list[str] = [] + pos = 0 + for tag in tags: + if tag.kind != KIND_PARTIAL: + continue + ref = tag.value if ref in stack: raise RenderError(f"partial cycle: {' -> '.join(stack)} -> {ref}") body = provider.resolve(ref) @@ -157,11 +163,14 @@ def replace(match: re.Match[str]) -> str: raise RenderError(partial_engine_message(ref, TEMPLATE_ENGINE_MUSTACHE)) stack.append(ref) try: - return _pre_expand_partials(body, provider, stack) + expanded = _pre_expand_partials(body, provider, stack) finally: stack.pop() - - return _PARTIAL_PATTERN.sub(replace, text) + out.append(text[pos : tag.start]) + out.append(expanded) + pos = tag.end + out.append(text[pos:]) + return "".join(out) # --------------------------------------------------------------------------- diff --git a/server/python/src/metaobjects/render/verify.py b/server/python/src/metaobjects/render/verify.py index ed6fb9cbda..d0b276b656 100644 --- a/server/python/src/metaobjects/render/verify.py +++ b/server/python/src/metaobjects/render/verify.py @@ -11,11 +11,10 @@ ``csharp/MetaObjects.Render/Verify.cs``; the three share the ``fixtures/verify-conformance/`` corpus that proves they agree. -The tokenizer models only plain interpolation, sections, inverted sections, and -partials (matching the C# port). It deliberately does NOT model Mustache -set-delimiter directives (``{{=<% %>=}}``); the TS engine parses with the real -``mustache`` library, so a fixture using set-delimiters would diverge across -ports. The corpus uses only plain tags — keep it that way. +Tags come from :mod:`metaobjects.render.mustache_tags`, which follows the Mustache +spec (a comment ends at the first closing delimiter, ``{{=<% %>=}}`` changes the +delimiters, a close must name its open) and is shared with the renderer's partial +expansion, so verify and render cannot disagree about where a tag is. """ from __future__ import annotations @@ -24,6 +23,19 @@ from typing import Protocol from .engine import TEMPLATE_ENGINE_JINJA, TEMPLATE_ENGINE_MUSTACHE, partial_engine_mismatch +from .mustache_tags import ( + KIND_AMP, + KIND_CLOSE, + KIND_INVERTED, + KIND_NAME, + KIND_PARTIAL, + KIND_SECTION, + KIND_TRIPLE, + MustacheSyntaxError, + delimiter_change, + scan, + without_comments, +) from .payload_accessors import is_boolean_accessor, is_positional_accessor #: A ``{{var}}`` references a field the (contextual) payload does not declare. @@ -39,7 +51,8 @@ #: dynamic key on an object (``user[key]``), a computed ``attribute=`` argument, or a #: template name that is an expression (``{% include name %}``). Reported, never passed. ERR_TEMPLATE_NOT_STATIC = "ERR_TEMPLATE_NOT_STATIC" -#: Jinja: the template text does not parse. path = the parser's message. +#: The template text does not parse. Jinja: path = the parser's message. Mustache: an +#: unbalanced section, path = the section's name. ERR_TEMPLATE_SYNTAX = "ERR_TEMPLATE_SYNTAX" #: A body is written for an engine this port does not run (Java, Kotlin and C# run #: Mustache only, so a ``.jinja`` template there is refused with this code). @@ -134,48 +147,42 @@ class _Partial: def _tokenize(text: str) -> list[_Token]: - tokens, _ = _parse_tokens(text, 0) - return tokens - - -def _parse_tokens(text: str, pos: int) -> tuple[list[_Token], int]: - """Parse tokens until EOF or a matching ``{{/close}}`` (which is consumed).""" - tokens: list[_Token] = [] - while pos < len(text): - open_idx = text.find("{{", pos) - if open_idx < 0: - break # remainder is plain text - triple = open_idx + 2 < len(text) and text[open_idx + 2] == "{" - close_delim = "}}}" if triple else "}}" - content_start = open_idx + (3 if triple else 2) - close_idx = text.find(close_delim, content_start) - if close_idx < 0: - break # malformed; stop - content = text[content_start:close_idx].strip() - pos = close_idx + len(close_delim) - - if triple: - tokens.append(_Var(content)) - continue - if not content: - continue - - sigil = content[0] - name = content[1:].strip() if len(content) > 1 else "" - if sigil == "!": - continue # comment - if sigil in ("#", "^"): - children, pos = _parse_tokens(text, pos) - tokens.append(_Section(name, sigil == "^", children)) - elif sigil == "/": - return tokens, pos # close (matched or mismatched) ends this level - elif sigil == ">": - tokens.append(_Partial(name)) - elif sigil == "&": - tokens.append(_Var(name)) - else: - tokens.append(_Var(content)) # {{x}}, {{x.y}}, {{.}} - return tokens, pos + """The tag tree of ``text``, built from the shared scanner (:mod:`mustache_tags`). + + Raises :class:`MustacheSyntaxError` on an unbalanced section.""" + root: list[_Token] = [] + open_levels: list[list[_Token]] = [root] + for tag in scan(text): + level = open_levels[-1] + if tag.kind in (KIND_SECTION, KIND_INVERTED): + section = _Section(tag.value, tag.kind == KIND_INVERTED, []) + level.append(section) + open_levels.append(section.children) + elif tag.kind == KIND_CLOSE: + open_levels.pop() # the scanner already matched it to its open + elif tag.kind == KIND_PARTIAL: + level.append(_Partial(tag.value)) + elif tag.kind in (KIND_NAME, KIND_TRIPLE, KIND_AMP): + level.append(_Var(tag.value)) # {{x}}, {{x.y}}, {{.}}, {{{x}}}, {{&x}} + # comments and delimiter changes carry no reference + return root + + +def _delimiter_errors(text: str, inlined_as: str | None) -> list[VerifyError]: + """The Python renderer refuses every delimiter change. Reported where the other ports report + the changes their renderers refuse: at the partial ref for a partial body, at the first + partial for a text that inlines one; elsewhere at the delimiters themselves.""" + tags = scan(text) + change = delimiter_change(tags) + if change is None: + return [] + first_partial = next((t.value for t in tags if t.kind == KIND_PARTIAL), None) + return [VerifyError(ERR_TEMPLATE_SYNTAX, inlined_as or first_partial or change)] + + +def _static_text(text: str) -> str: + """``text`` minus its comments, for the output-tag scan.""" + return without_comments(text, scan(text)) def _find(fields: list[PayloadField], name: str) -> PayloadField | None: @@ -321,10 +328,22 @@ def walk( if partial_engine_mismatch(provider, tok.value, TEMPLATE_ENGINE_MUSTACHE): errors.append(VerifyError(ERR_TEMPLATE_ENGINE_UNSUPPORTED, tok.value)) continue - static_texts.append(body) - walk(_tokenize(body), stack, [*seen, tok.value], in_element) - - walk(_tokenize(template_text), [root], []) + try: + partial_tokens = _tokenize(body) + except MustacheSyntaxError as e: + errors.append(VerifyError(ERR_TEMPLATE_SYNTAX, e.section)) + continue + errors.extend(_delimiter_errors(body, tok.value)) + static_texts.append(_static_text(body)) + walk(partial_tokens, stack, [*seen, tok.value], in_element) + + try: + body_tokens = _tokenize(template_text) + except MustacheSyntaxError as e: + return [VerifyError(ERR_TEMPLATE_SYNTAX, e.section)] + errors.extend(_delimiter_errors(template_text, None)) + static_texts[0] = _static_text(template_text) + walk(body_tokens, [root], []) errors.extend(slot_and_tag_findings(referenced_at_root, static_texts, required_slots, required_tags)) return errors diff --git a/server/python/src/metaobjects/runtime/validator_runner.py b/server/python/src/metaobjects/runtime/validator_runner.py index 306c64deca..1f0577243c 100644 --- a/server/python/src/metaobjects/runtime/validator_runner.py +++ b/server/python/src/metaobjects/runtime/validator_runner.py @@ -343,8 +343,10 @@ def _scalar_errors(field: MetaData, value: object, required: bool, label: str) - if not isinstance(pattern, str): continue try: - # @pattern is FULL-MATCH: the whole value must match. - matched = re.fullmatch(f"(?:{pattern})", value) is not None + # @pattern is FULL-MATCH: the whole value must match. re.ASCII keeps \d, \w + # and \s to ASCII like the TS and Java ports; str patterns default to + # Unicode, so \d would otherwise accept Arabic-Indic digits. + matched = re.fullmatch(f"(?:{pattern})", value, re.ASCII) is not None except re.error: errors.append(ValidationFailure( label, "regex", f"'{label}' has an invalid validator pattern: {pattern}", pattern, diff --git a/server/python/src/metaobjects/source/yaml_core_schema.py b/server/python/src/metaobjects/source/yaml_core_schema.py new file mode 100644 index 0000000000..ac6f7149da --- /dev/null +++ b/server/python/src/metaobjects/source/yaml_core_schema.py @@ -0,0 +1,122 @@ +"""YAML 1.2 core-schema loader. + +PyYAML resolves plain scalars as YAML 1.1, where ``off``/``no``/``y`` are +booleans, ``1:30`` is a sexagesimal int, ``0777`` is octal, ``1_000`` is an int +and ``2024-01-02`` is a ``date``. The TypeScript loader uses the ``yaml`` +package, which resolves the 1.2 core schema: only ``true``/``false`` are +booleans, only ``[-+]?[0-9]+`` / ``0o`` / ``0x`` are ints, and there is no +timestamp type. This loader gives Python the same scalar resolution, so one +YAML file loads to the same model in every port. +""" +from __future__ import annotations + +import math +import re +from typing import Any + +import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs + +_TAG_NULL = "tag:yaml.org,2002:null" +_TAG_BOOL = "tag:yaml.org,2002:bool" +_TAG_INT = "tag:yaml.org,2002:int" +_TAG_FLOAT = "tag:yaml.org,2002:float" + +# YAML 1.2 section 10.3.2 (core schema) tag resolution. +_NULL_RE = re.compile(r"^(?:~|null|Null|NULL|)$") +_BOOL_RE = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +_INT_RE = re.compile(r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$") +_FLOAT_RE = re.compile( + r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$" +) + + +class CoreSchemaLoader(yaml.SafeLoader): # type: ignore[misc] + """``SafeLoader`` whose implicit scalar resolvers are the YAML 1.2 core schema.""" + + # Own table: ``add_implicit_resolver`` copies the inherited one on first use, + # so reset it here to drop every 1.1 resolver (timestamp, merge, value, ...). + yaml_implicit_resolvers: dict[str, list[tuple[str, re.Pattern[str]]]] = {} + + +CoreSchemaLoader.add_implicit_resolver(_TAG_NULL, _NULL_RE, ["~", "n", "N", ""]) +CoreSchemaLoader.add_implicit_resolver(_TAG_BOOL, _BOOL_RE, list("tTfF")) +CoreSchemaLoader.add_implicit_resolver(_TAG_INT, _INT_RE, list("-+0123456789")) +CoreSchemaLoader.add_implicit_resolver(_TAG_FLOAT, _FLOAT_RE, list("-+0123456789.")) + + +def _construct_null(loader: yaml.SafeLoader, node: yaml.Node) -> None: + loader.construct_scalar(node) + return None + + +def _construct_bool(loader: yaml.SafeLoader, node: yaml.Node) -> bool: + return str(loader.construct_scalar(node)).lower() == "true" + + +def _construct_int(loader: yaml.SafeLoader, node: yaml.Node) -> int: + text = str(loader.construct_scalar(node)) + if text.startswith("0o"): + return int(text[2:], 8) + if text.startswith("0x"): + return int(text[2:], 16) + return int(text, 10) + + +def _construct_float(loader: yaml.SafeLoader, node: yaml.Node) -> float: + text = str(loader.construct_scalar(node)).lower() + if text.endswith(".nan"): + return math.nan + if text.endswith(".inf"): + return -math.inf if text.startswith("-") else math.inf + return float(text) + + +CoreSchemaLoader.add_constructor(_TAG_NULL, _construct_null) +CoreSchemaLoader.add_constructor(_TAG_BOOL, _construct_bool) +CoreSchemaLoader.add_constructor(_TAG_INT, _construct_int) +CoreSchemaLoader.add_constructor(_TAG_FLOAT, _construct_float) + + +def safe_load_core(text: str) -> Any: + """``yaml.safe_load`` with YAML 1.2 core-schema scalars and unique mapping keys.""" + loader = CoreSchemaLoader(text) + try: + node = loader.get_single_node() + if node is None: + return None + _reject_duplicate_keys(node) + return loader.construct_document(node) + finally: + loader.dispose() + + +def _reject_duplicate_keys(node: yaml.Node) -> None: + """Raise on a repeated scalar key in any mapping (the ``yaml`` package does too).""" + seen_nodes: set[int] = set() + stack = [node] + while stack: + current = stack.pop() + if id(current) in seen_nodes: + continue + seen_nodes.add(id(current)) + if isinstance(current, yaml.SequenceNode): + stack.extend(current.value) + elif isinstance(current, yaml.MappingNode): + check_unique_keys(current) + for key_node, value_node in current.value: + stack.append(key_node) + stack.append(value_node) + + +def check_unique_keys(node: yaml.MappingNode) -> None: + seen: set[str] = set() + for key_node, _ in node.value: + if not isinstance(key_node, yaml.ScalarNode): + continue + key = str(key_node.value) + if key in seen: + raise yaml.constructor.ConstructorError( + None, None, f"duplicate mapping key {key!r}", key_node.start_mark + ) + seen.add(key) diff --git a/server/python/src/metaobjects/source/yaml_positions.py b/server/python/src/metaobjects/source/yaml_positions.py index 5a65e70f55..653a2b79d5 100644 --- a/server/python/src/metaobjects/source/yaml_positions.py +++ b/server/python/src/metaobjects/source/yaml_positions.py @@ -36,6 +36,7 @@ import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs from .error_source import YamlPosition +from .yaml_core_schema import CoreSchemaLoader, check_unique_keys class YamlPositionMap(dict): @@ -100,8 +101,9 @@ def set_position_map( def parse_yaml_with_positions(text: str) -> Any: """Parse YAML text and return a Python structure with positions attached. - Mirrors the contract of :func:`yaml.safe_load` for the shapes the - metaobjects authoring grammar uses (mappings, sequences, scalars). Aliases + Mirrors the contract of the TS ``yaml.parse`` (YAML 1.2 core schema, unique + mapping keys) for the shapes the metaobjects authoring grammar uses + (mappings, sequences, scalars). Aliases are resolved via the underlying SafeLoader pipeline — i.e. they resolve as the library normally would. @@ -110,10 +112,10 @@ def parse_yaml_with_positions(text: str) -> Any: the position of the KEY token in the YAML source. Raises: - :class:`yaml.YAMLError`: on YAML syntax errors (same as - ``yaml.safe_load``). + :class:`yaml.YAMLError`: on YAML syntax errors and on a duplicate + mapping key. """ - loader = yaml.SafeLoader(text) + loader = CoreSchemaLoader(text) try: node = loader.get_single_node() if node is None: @@ -130,16 +132,17 @@ def _yaml_node_to_py(node: Any, loader: Any) -> Any: :class:`YamlPositionMap`) onto the resulting dict — the position of each key is the (line, col) of the KEY token in the YAML source. - Scalar nodes are constructed via the SafeLoader's constructor pipeline, - so YAML 1.1's typed scalars (null, bool, int, float) coerce exactly the - way :func:`yaml.safe_load` produces them. The desugar's D2 coercion - guard fires on those coerced values, matching TS / Java / C# behavior. + Scalar nodes are constructed via :class:`CoreSchemaLoader`, so typed scalars + (null, bool, int, float) resolve under the YAML 1.2 core schema, matching the + TS loader: ``off``/``no``/``1:30`` stay strings and ``0777`` is the decimal int 777. The desugar's D2 + coercion guard fires on the values that do coerce (``true``, ``42``). """ if isinstance(node, yaml.ScalarNode): return loader.construct_object(node, deep=True) if isinstance(node, yaml.SequenceNode): return [_yaml_node_to_py(item, loader) for item in node.value] if isinstance(node, yaml.MappingNode): + check_unique_keys(node) out = YamlPositionMap() positions: dict[str, YamlPosition] = {} for key_node, value_node in node.value: diff --git a/server/python/tests/codegen/test_collection_name_collision.py b/server/python/tests/codegen/test_collection_name_collision.py index 815c5348cd..40107a3ebb 100644 --- a/server/python/tests/codegen/test_collection_name_collision.py +++ b/server/python/tests/codegen/test_collection_name_collision.py @@ -107,3 +107,12 @@ def test_naming_conformance_collision_cases_are_refused(tmp_path: Path) -> None: assert_no_collection_name_collisions( _entities(tmp_path / str(i), case["entityA"], case["entityB"]) ) + + +def test_raises_for_route_segment_twins(tmp_path: Path) -> None: + # ApiKey / APIKey pluralize to different names but both snake_case to /api_keys. + with pytest.raises(ValueError) as exc_info: + assert_no_collection_name_collisions(_entities(tmp_path, "ApiKey", "APIKey")) + msg = str(exc_info.value) + assert ERR_COLLECTION_NAME_COLLISION in msg + assert '"ApiKey"' in msg and '"APIKey"' in msg and "api_keys" in msg diff --git a/server/python/tests/codegen/test_eject_undo.py b/server/python/tests/codegen/test_eject_undo.py index b2ea79b399..2488b783c9 100644 --- a/server/python/tests/codegen/test_eject_undo.py +++ b/server/python/tests/codegen/test_eject_undo.py @@ -143,3 +143,34 @@ def test_undo_with_a_repeated_name_removes_it_once(tmp_path: Path, monkeypatch, capsys.readouterr() assert main(["eject", "--undo", "entity", "entity"]) == 0 assert not (tmp_path / GEN / "entity.py").exists() + + +def test_undo_keeps_a_copy_whose_only_edit_is_a_reordering(tmp_path: Path, monkeypatch, capsys) -> None: + monkeypatch.chdir(tmp_path) + assert main(["eject", "entity", "names"]) == 0 + path = tmp_path / GEN / "names.py" + lines = path.read_text().splitlines() + # Swap two adjacent, different, non-blank lines: the same lines in another order. + i = next(i for i in range(len(lines) - 1) + if lines[i].strip() and lines[i + 1].strip() and lines[i].strip() != lines[i + 1].strip()) + lines[i], lines[i + 1] = lines[i + 1], lines[i] + path.write_text("\n".join(lines) + "\n") + capsys.readouterr() + assert main(["eject", "--undo"]) == 0 + assert not (tmp_path / GEN / "entity.py").exists() + assert path.exists() + assert main(["eject", "--undo", "names"]) != 0 + assert path.exists() + assert main(["eject", "--undo", "--force", "names"]) == 0 + assert not path.exists() + + +def test_undo_removes_a_copy_whose_only_edit_is_whitespace(tmp_path: Path, monkeypatch, capsys) -> None: + monkeypatch.chdir(tmp_path) + assert main(["eject", "names"]) == 0 + path = tmp_path / GEN / "names.py" + # An editor's save-time cleanup: trailing whitespace and an extra blank line, nothing reordered. + path.write_text("\n".join(line + " " for line in path.read_text().splitlines()) + "\n\n\n") + capsys.readouterr() + assert main(["eject", "--undo", "names"]) == 0 + assert not path.exists() diff --git a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py index 05ba7d5a8e..0914caf1d6 100644 --- a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py +++ b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py @@ -292,3 +292,12 @@ def test_narrowed_run_on_only_the_projection_still_materializes_the_shared_enum( ticket_summary_src = (out_dir / "TicketSummary.py").read_text() assert "level: Priority" in ticket_summary_src assert "from .enums import Priority" in ticket_summary_src + + +def test_shared_enum_values_colliding_after_upper_are_a_codegen_error() -> None: + with pytest.raises(ValueError, match="both map to the Python member OPEN"): + _emit_doc({"metadata.root": {"package": "a::b", "children": [ + {"field.enum": {"name": "State", "abstract": True, "@values": ["Open", "open"]}}, + {"object.value": {"name": "M", "children": [ + {"field.enum": {"name": "st", "extends": "a::b::State"}}]}}, + ]}}) diff --git a/server/python/tests/codegen/test_names_extends_chain.py b/server/python/tests/codegen/test_names_extends_chain.py index c59864590a..866ab7a400 100644 --- a/server/python/tests/codegen/test_names_extends_chain.py +++ b/server/python/tests/codegen/test_names_extends_chain.py @@ -296,7 +296,7 @@ def test_a_child_field_colliding_with_an_inherited_one_is_refused_naming_the_mod "children": [ {"source.rdb": {"@table": "rows"}}, {"field.long": {"name": "id"}}, - {"field.timestamp": {"name": "created_at"}}, + {"field.timestamp": {"name": "created_at", "@column": "created_at_alt"}}, {"identity.primary": {"name": "pk", "@fields": "id", "@generation": "increment"}}, ], } diff --git a/server/python/tests/codegen/test_pattern_ascii.py b/server/python/tests/codegen/test_pattern_ascii.py new file mode 100644 index 0000000000..2678d64681 --- /dev/null +++ b/server/python/tests/codegen/test_pattern_ascii.py @@ -0,0 +1,99 @@ +"""Generated Pydantic ``validator.regex`` / hostname patterns are ASCII full matches. + +Pydantic's string ``pattern=`` runs on the Rust regex engine, where ``\\d`` matches +non-ASCII digits and lookaround is unsupported. The generator emits +``re.compile(..., re.ASCII)`` so the model agrees with the TS and Java ports and with the +runtime validator (``validator_runner``).""" +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import metaobjects.core_types # noqa: F401 — registers core types +from metaobjects import MetaDataLoader +from metaobjects.codegen.generators.entity_model import render_entity_model +from metaobjects.shared.base_types import TYPE_OBJECT + +ARABIC_INDIC = "١٢" + + +def _meta(pattern: str | None, string_format: str | None = None) -> str: + field: dict = {"name": "code", "@required": True} + if string_format: + field["@stringFormat"] = string_format + children: list = [] + if pattern is not None: + children = [{"validator.regex": {"@pattern": pattern}}] + field["children"] = children + return json.dumps({"metadata.root": {"package": "t", "children": [ + {"object.entity": {"name": "Doc", "children": [ + {"field.string": field}, + ]}}, + ]}}) + + +def _entity(meta: str): + result = MetaDataLoader.from_string(meta) + assert not result.errors, [str(e) for e in result.errors] + return next(c for c in result.root.children() if c.type == TYPE_OBJECT) + + +def _model(meta: str): + source = render_entity_model(_entity(meta)) + mod_dir = tempfile.mkdtemp() + name = f"gen_pattern_{abs(hash(meta))}" + path = Path(mod_dir) / f"{name}.py" + path.write_text(source) + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return source, module.DocCreate + + +def test_digit_class_is_ascii_only() -> None: + source, model = _model(_meta(r"\d+")) + assert "re.ASCII" in source + assert model(code="12").code == "12" + with pytest.raises(ValidationError): + model(code=ARABIC_INDIC) + + +def test_full_match_rejects_trailing_newline_and_partial() -> None: + _, model = _model(_meta(r"[a-z]+")) + for bad in ("ab\n", "ab1", "1ab"): + with pytest.raises(ValidationError): + model(code=bad) + + +def test_lookahead_pattern_is_supported() -> None: + _, model = _model(_meta(r"(?=.{1,5}$)[a-z]+")) + assert model(code="abc").code == "abc" + with pytest.raises(ValidationError): + model(code="abcdef") + + +def test_hostname_format_is_ascii_full_match() -> None: + _, model = _model(_meta(None, "hostname")) + assert model(code="a.example.com").code == "a.example.com" + with pytest.raises(ValidationError): + model(code="a.example.com\n") + + +def test_unicode_property_class_fails_at_generation() -> None: + with pytest.raises(ValueError, match="not valid in Python's re"): + render_entity_model(_entity(_meta(r"\p{L}+"))) + + +def test_published_json_schema_pattern_is_ecma_valid() -> None: + _, model = _model(_meta(r"\d+")) + pattern = model.model_json_schema()["properties"]["code"]["pattern"] + assert pattern == r"^(?:\d+)$" + assert "\\Z" not in pattern diff --git a/server/python/tests/conformance/test_extract_conformance.py b/server/python/tests/conformance/test_extract_conformance.py index 2eda6d8f98..977215e779 100644 --- a/server/python/tests/conformance/test_extract_conformance.py +++ b/server/python/tests/conformance/test_extract_conformance.py @@ -61,7 +61,7 @@ def _cases() -> list[str]: def test_discovers_all_extract_conformance_cases() -> None: """FR-011: lock the corpus size so a deleted fixture fails CI rather than silently reducing coverage. Mirrors the TS / Java / C# count guards.""" - assert len(_cases()) == 48 + assert len(_cases()) == 58 _NORMALIZE_MODES = {"none", "collapse", "strip"} diff --git a/server/python/tests/render/extract/test_coerce.py b/server/python/tests/render/extract/test_coerce.py index 7402481942..f781ec1294 100644 --- a/server/python/tests/render/extract/test_coerce.py +++ b/server/python/tests/render/extract/test_coerce.py @@ -239,3 +239,9 @@ def test_enum_coerce_default_ignored_when_not_a_member() -> None: "s", True, ["IN_PROGRESS", "DONE"], {}, coerce_default="BOGUS", normalize="none" ) assert _coerce.value("banana", f, _normal(), "s", rep) is MALFORMED + + +def test_long_above_two_pow_53_keeps_its_precision() -> None: + f = FieldSpec("id", FieldKind.LONG, True) + rep = ExtractionReport() + assert _coerce.value("9007199254740993", f, _normal(), "id", rep) == 9007199254740993 diff --git a/server/python/tests/render/extract/test_json_forgiving_reader.py b/server/python/tests/render/extract/test_json_forgiving_reader.py index 9d01b5273a..b0a216858a 100644 --- a/server/python/tests/render/extract/test_json_forgiving_reader.py +++ b/server/python/tests/render/extract/test_json_forgiving_reader.py @@ -93,3 +93,8 @@ def test_empty_value_then_more_keys_continues() -> None: m = _read('{"a":,"b":"2"}') assert m["a"] is TRUNCATED assert m["b"] == "2" + + +def test_full_json_escape_grammar() -> None: + got = _read(r'{"a":"café 😀 a\/b\bx\fy \"q\" \\ \u12"}') + assert got["a"] == 'café 😀 a/b\x08x\x0cy "q" \\ u12' diff --git a/server/python/tests/render/extract/test_strip.py b/server/python/tests/render/extract/test_strip.py index 48c1e2f353..a8ab8e21ed 100644 --- a/server/python/tests/render/extract/test_strip.py +++ b/server/python/tests/render/extract/test_strip.py @@ -26,3 +26,21 @@ def test_no_fence_returns_trimmed_input() -> None: def test_null_safe() -> None: assert _strip.strip(None) == "" + + +def test_backticks_inside_a_json_string_do_not_close_the_fence() -> None: + out = _strip.strip('```json\n{"summary": "run ```make``` first", "score": 7}\n```') + assert out == '{"summary": "run ```make``` first", "score": 7}' + + +def test_longer_opening_fence_closes_only_on_an_equal_or_longer_run() -> None: + out = _strip.strip("````md\nbefore\n```\ninner\n```\nafter\n````\ntail") + assert out == "before\n```\ninner\n```\nafter\ntail" + + +def test_closing_fence_on_the_payload_line_is_accepted_at_the_end() -> None: + assert _strip.strip('```json\n{"a":1}```') == '{"a":1}' + + +def test_unclosed_fence_is_not_a_fence() -> None: + assert _strip.strip('```json\n{"a":1}') == '```json\n{"a":1}' diff --git a/server/python/tests/render/test_verify_conformance.py b/server/python/tests/render/test_verify_conformance.py index 9bd350633e..b6da4bbfe0 100644 --- a/server/python/tests/render/test_verify_conformance.py +++ b/server/python/tests/render/test_verify_conformance.py @@ -19,6 +19,7 @@ ERR_OUTPUT_TAG_MISSING, ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, + ERR_TEMPLATE_SYNTAX, ERR_VAR_NOT_ON_PAYLOAD, InMemoryProvider, PayloadField, @@ -34,6 +35,7 @@ ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, } ) @@ -93,10 +95,11 @@ def test_verify_conformance(name: str) -> None: fixture_dir = _CORPUS / name fields = _parse_fields(json.loads((fixture_dir / "payload.json").read_text())) template = (fixture_dir / "template.mustache").read_text() - expected = [ - VerifyError(e["code"], e["path"]) - for e in json.loads((fixture_dir / "expected-drift.json").read_text()) - ] + # expected-drift.python.json (optional) replaces expected-drift.json for this port: the + # Python renderer refuses every delimiter change, so its verify reports each one. + python_expected = fixture_dir / "expected-drift.python.json" + expected_path = python_expected if python_expected.is_file() else fixture_dir / "expected-drift.json" + expected = [VerifyError(e["code"], e["path"]) for e in json.loads(expected_path.read_text())] # Lint: every expected code is a known verify code (typo guard). for e in expected: diff --git a/server/python/tests/runtime/test_validator_runner.py b/server/python/tests/runtime/test_validator_runner.py index 0b29e1ce96..2e728fb366 100644 --- a/server/python/tests/runtime/test_validator_runner.py +++ b/server/python/tests/runtime/test_validator_runner.py @@ -138,6 +138,12 @@ def test_regex_is_full_match() -> None: }] +def test_regex_digit_class_is_ascii_only() -> None: + e = _load(_with(_string("code"), {"validator.regex": {"@pattern": "\\d+"}})) + assert _rules(e, {"code": "123"}) == [] + assert _rules(e, {"code": "\u0661\u0662"}) == ["code:regex"] # Arabic-Indic digits + + def test_regex_rejects_a_trailing_newline() -> None: e = _load(_with(_string("code"), {"validator.regex": {"@pattern": "[A-Z]+"}})) assert _rules(e, {"code": "ABC\n"}) == ["code:regex"] diff --git a/server/python/tests/source/test_yaml_core_schema.py b/server/python/tests/source/test_yaml_core_schema.py new file mode 100644 index 0000000000..e49e066f81 --- /dev/null +++ b/server/python/tests/source/test_yaml_core_schema.py @@ -0,0 +1,56 @@ +"""YAML 1.2 core-schema scalar resolution (regression for the YAML 1.1 coercion). + +PyYAML's SafeLoader resolves scalars as YAML 1.1, so ``off``/``no``/``1:30``/ +``0777``/``2024-01-02`` changed type silently while the TS loader (the ``yaml`` +package, 1.2 core schema) kept them as the authored text. The Python loader +must agree with it, and a duplicate mapping key is a parse error, not last-wins. +""" +from __future__ import annotations + +import pytest +import yaml # type: ignore[import-untyped] + +from metaobjects.source.yaml_positions import parse_yaml_with_positions + + +def _scalar(text: str): + return parse_yaml_with_positions(f"k: {text}\n")["k"] + + +@pytest.mark.parametrize( + "text", + ["off", "on", "no", "yes", "y", "n", "Off", "NO", "1:30", "2024-01-02", "1_000", "0b11", "+.inf_"], +) +def test_yaml11_scalars_stay_strings(text: str) -> None: + assert _scalar(text) == text + + +def test_core_schema_scalars() -> None: + assert _scalar("true") is True + assert _scalar("False") is False + assert _scalar("TRUE") is True + assert _scalar("null") is None + assert _scalar("~") is None + assert _scalar("") is None + assert _scalar("42") == 42 + assert _scalar("0777") == 777 # 1.2 core: decimal, as the TS loader reads it + assert _scalar("-7") == -7 + assert _scalar("0x1F") == 31 + assert _scalar("0o17") == 15 + assert _scalar("1.5") == 1.5 + assert _scalar("1e3") == 1000.0 + assert _scalar(".inf") == float("inf") + assert _scalar('"off"') == "off" + + +def test_duplicate_mapping_key_is_an_error() -> None: + with pytest.raises(yaml.YAMLError): + parse_yaml_with_positions("a: 1\na: 2\n") + + +def test_positions_still_tracked() -> None: + value = parse_yaml_with_positions("a: off\nb: no\n") + assert value["a"] == "off" + from metaobjects.source.yaml_positions import get_yaml_position + + assert get_yaml_position(value, "b").line == 2 diff --git a/server/python/tests/unit/test_relative_set_newlines.py b/server/python/tests/unit/test_relative_set_newlines.py new file mode 100644 index 0000000000..d9b3585008 --- /dev/null +++ b/server/python/tests/unit/test_relative_set_newlines.py @@ -0,0 +1,11 @@ +"""The codegen drift gate must compare bytes: a CRLF file is not the LF file the generator wrote.""" +from __future__ import annotations + +from pathlib import Path + +from metaobjects.cli import _relative_set + + +def test_relative_set_keeps_carriage_returns(tmp_path: Path) -> None: + (tmp_path / "a.py").write_bytes(b"x = 1\r\ny = 2\r\n") + assert _relative_set(tmp_path) == {"a.py": "x = 1\r\ny = 2\r\n"} diff --git a/server/python/uv.lock b/server/python/uv.lock index 8215c4ad79..f87a38a7f7 100644 --- a/server/python/uv.lock +++ b/server/python/uv.lock @@ -443,7 +443,7 @@ requires-dist = [ { name = "jinja2", marker = "extra == 'jinja'", specifier = ">=3.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "pg8000", marker = "extra == 'integration'", specifier = ">=1.31" }, - { name = "pydantic", marker = "extra == 'dev'", specifier = ">=2" }, + { name = "pydantic", marker = "extra == 'dev'", specifier = ">=2.8" }, { name = "pymysql", marker = "extra == 'integration'", specifier = ">=1.1" }, { name = "pymysql", marker = "extra == 'mysql'", specifier = ">=1.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, @@ -457,7 +457,7 @@ provides-extras = ["dev", "integration", "mysql", "jinja"] dev = [ { name = "jinja2", specifier = ">=3.1" }, { name = "mypy", specifier = ">=1.10" }, - { name = "pydantic", specifier = ">=2" }, + { name = "pydantic", specifier = ">=2.8" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-xdist", specifier = ">=3.6" }, { name = "ruff", specifier = ">=0.6" }, diff --git a/server/typescript/packages/cli/src/commands/eject.ts b/server/typescript/packages/cli/src/commands/eject.ts index 41769c9114..98eab51682 100644 --- a/server/typescript/packages/cli/src/commands/eject.ts +++ b/server/typescript/packages/cli/src/commands/eject.ts @@ -91,6 +91,16 @@ function resolveSource(name: string): TemplateSource | undefined { } +/** Line counts ignore order, so a pure reorder reads 0/0; say so (Python and C# do too). */ +export function reorderedSuffix(behind: number, own: number): string { + return behind === 0 && own === 0 ? ", reordered" : ""; +} + +/** The `[DIFFERS: ...]` tag for a reference copy that is not the ejected one, shared by every listing. */ +function differsTag(path: string, referenceOnly: number, localOnly: number): string { + return ` ${path} [DIFFERS: ${referenceOnly} line(s) only in the package, ${localOnly} line(s) only in yours${reorderedSuffix(referenceOnly, localOnly)}]`; +} + /** Every ejectable name, in registry order (stable — matches `meta eject --list`). */ export function ejectableNames(): string[] { return SOURCES.flatMap((s) => s.names); @@ -349,7 +359,7 @@ export async function runtimeCopyLines(cwd: string): Promise { else { differing.push(r.path); lines.push( - ` ${r.path} [DIFFERS: ${r.referenceOnly} line(s) only in the package, ${r.localOnly} line(s) only in yours]`, + differsTag(r.path, r.referenceOnly, r.localOnly), ); } } @@ -387,7 +397,7 @@ export async function templateCopyLines(cwd: string): Promise { const diff = templateDiffCommand(r); if (diff !== undefined) differing.push(diff); lines.push( - ` ${r.path} [DIFFERS: ${r.referenceOnly} line(s) only in the package, ${r.localOnly} line(s) only in yours]`, + differsTag(r.path, r.referenceOnly, r.localOnly), ); } } @@ -535,11 +545,11 @@ async function listOutput(cwd: string): Promise { ]; lines.push( generatorFileDiffers(r) - ? ` ${r.name} [owned — DIFFERS: ${r.behind} line(s) behind, ${r.own} line(s) of your own]` + ? ` ${r.name} [owned — DIFFERS: ${r.behind} line(s) behind, ${r.own} line(s) of your own${reorderedSuffix(r.behind, r.own)}]` : ` ${r.name} [owned — DIFFERS: generator identical, ${companions.join(" and ")} differ]`, ); for (const t of differing) { - lines.push(` renderer ${t.path} [DIFFERS: ${t.behind} line(s) behind, ${t.own} line(s) of your own]`); + lines.push(` renderer ${t.path} [DIFFERS: ${t.behind} line(s) behind, ${t.own} line(s) of your own${reorderedSuffix(t.behind, t.own)}]`); } for (const path of r.runtime ?? []) lines.push(` runtime ${path} [DIFFERS]`); } diff --git a/server/typescript/packages/cli/src/lib/dependency-sync.ts b/server/typescript/packages/cli/src/lib/dependency-sync.ts index 4624de81dc..e55102508e 100644 --- a/server/typescript/packages/cli/src/lib/dependency-sync.ts +++ b/server/typescript/packages/cli/src/lib/dependency-sync.ts @@ -288,6 +288,17 @@ function pep440Normalized(version: string): string | undefined { return `${epoch}${release}${pre}${post}${dev}${local}`; } +/** PEP 440 equality: release segments compare with trailing zeros ignored, so `1.0` and + * `1.0.0` are the same version. A string that is not PEP 440 compares verbatim. */ +function pep440Equal(a: string, b: string): boolean { + const key = (v: string): string => { + const n = pep440Normalized(v); + if (n === undefined) return v; + return n.replace(/^((?:\d+!)?\d+(?:\.\d+)*)/, (rel) => rel.replace(/(?:\.0+)+$/, "")); + }; + return key(a) === key(b); +} + /** The installed package's own version should be the manifest's: `sharedModelFile()` * defaults the manifest version to the host package's. A difference means the * publisher bumped the package without regenerating the artifact (or set @@ -301,7 +312,7 @@ export function packageVersionWarning( if (packageVersion === undefined) return undefined; const same = "python" in spec - ? (pep440Normalized(packageVersion) ?? packageVersion) === (pep440Normalized(manifestVersion) ?? manifestVersion) + ? pep440Equal(packageVersion, manifestVersion) : packageVersion === manifestVersion; if (same) return undefined; const pkg = "npm" in spec ? `npm package "${spec.npm}"` : "python" in spec ? `Python package "${spec.python}"` : "package"; diff --git a/server/typescript/packages/cli/src/lib/owned-copy.ts b/server/typescript/packages/cli/src/lib/owned-copy.ts index 6633750401..575349920b 100644 --- a/server/typescript/packages/cli/src/lib/owned-copy.ts +++ b/server/typescript/packages/cli/src/lib/owned-copy.ts @@ -23,11 +23,10 @@ // a list moves which member is last) // import statement AND specifier order → exploded one per line, then sorted // -// The cost is deliberate and stated: sorting lines also makes the comparison blind to -// a pure REORDERING of real code. That is the right trade for a report line whose job -// is "have you fallen behind upstream" — a reorder with no content change is not -// staleness — and every message that carries this verdict also prints the `diff` -// command, which is not blind to anything. +// The sorted lines feed only the "N behind, M of your own" COUNTS, which are information. +// The VERDICT is decided on the same canonical lines in their original ORDER: a pure +// reordering of real code (swapping two statements) changes behaviour, and `eject --undo` +// deletes a copy whose verdict is stock, so it must read `differs` and need `--force`. // // Fail-safe: if the formatter is unavailable or rejects the input (an adopter's copy is // theirs and may not even parse), canonicalization degrades to trimmed raw lines. The @@ -73,9 +72,35 @@ function getBiome(): Promise { return _biome; } -/** The canonical line multiset of one file, sorted. Exported for the tests that pin - * each formatter-insensitivity claim above against real reformatted input. */ -export async function canonicalLines(src: string): Promise { +/** + * Sort each run of import statements in place. The order of imports and of their specifiers + * is layout, not behaviour (formatters and organize-imports rewrite it), so it stays + * insensitive; the order of everything else is kept. + */ +function sortImportRuns(lines: string[]): string[] { + const out: string[] = []; + let run: string[] = []; + let inImport = false; + const flush = (): void => { + out.push(...run.sort()); + run = []; + }; + for (const l of lines) { + if (!inImport && !l.startsWith("import ")) { + flush(); + out.push(l); + continue; + } + run.push(l); + // An exploded import is `import {`, one specifier per line, then `} from "x";`. + inImport = !l.endsWith(";"); + } + flush(); + return out; +} + +/** The canonical lines of one file, in file order apart from each run of imports. */ +async function orderedLines(src: string): Promise { let text = src; const biome = await getBiome(); if (biome !== undefined) { @@ -86,14 +111,19 @@ export async function canonicalLines(src: string): Promise { /* keep the raw text — see the fail-safe note above */ } } - return text + return sortImportRuns(text .split("\n") // The trailing-comma strip is per LINE and only meaningful because every construct // is already on its own line: reordering an import list changes which specifier is // last, and only the last one lacks a comma. .map((l) => l.trim().replace(/,$/, "")) - .filter((l) => l !== "") - .sort(); + .filter((l) => l !== "")); +} + +/** The canonical line multiset of one file, sorted. Exported for the tests that pin + * each formatter-insensitivity claim above against real reformatted input. */ +export async function canonicalLines(src: string): Promise { + return (await orderedLines(src)).sort(); } /** How many members of `a` are not in `b`, counting duplicates. */ @@ -111,11 +141,14 @@ function countMissing(a: string[], b: string[]): number { export async function compareOwnedCopy(owned: string, reference: string): Promise { if (owned === reference) return { verdict: "identical", localOnly: 0, referenceOnly: 0 }; - const [a, b] = await Promise.all([canonicalLines(owned), canonicalLines(reference)]); + const [ordA, ordB] = await Promise.all([orderedLines(owned), orderedLines(reference)]); + const a = [...ordA].sort(); + const b = [...ordB].sort(); const localOnly = countMissing(a, b); const referenceOnly = countMissing(b, a); + const sameOrder = ordA.join("\n") === ordB.join("\n"); return { - verdict: localOnly === 0 && referenceOnly === 0 ? "reformatted" : "differs", + verdict: sameOrder ? "reformatted" : "differs", localOnly, referenceOnly, }; diff --git a/server/typescript/packages/cli/test/deps-sync-packages.test.ts b/server/typescript/packages/cli/test/deps-sync-packages.test.ts index a379f2785b..4d53a8a425 100644 --- a/server/typescript/packages/cli/test/deps-sync-packages.test.ts +++ b/server/typescript/packages/cli/test/deps-sync-packages.test.ts @@ -349,6 +349,12 @@ describe("the installed-vs-manifest version warning", () => { ["1.0.dev0", "1.0-dev"], ["1.0.dev3", "1.0_dev3"], ["1.0.0", "v1.0.0"], + // PEP 440 ignores trailing zero release segments: 1.0 is 1.0.0. + ["1.0", "1.0.0"], + ["1.0.0", "1.0"], + ["2", "2.0.0"], + ["1.0.0rc1", "1.0rc1"], + ["1!1.0", "1!1.0.0"], ])("python: installed %s matches manifest %s", (installed, manifest) => { expect(packageVersionWarning(DEP, py, installed, manifest)).toBeUndefined(); }); @@ -357,7 +363,8 @@ describe("the installed-vs-manifest version warning", () => { expect(packageVersionWarning(DEP, py, "1.0.0rc2", "1.0.0-rc.1")).toContain("records 1.0.0-rc.1"); expect(packageVersionWarning(DEP, py, "1.0.0rc1", "1.0.0rc2")).toContain("records 1.0.0rc2"); expect(packageVersionWarning(DEP, py, "1.0.0rc1.post2", "1.0.0rc1")).toContain("records 1.0.0rc1"); - expect(packageVersionWarning(DEP, py, "1.0", "1.0.0")).toContain("records 1.0.0"); + expect(packageVersionWarning(DEP, py, "1.0.1", "1.0")).toContain("records 1.0"); + expect(packageVersionWarning(DEP, py, "1.10", "1.1")).toContain("records 1.1"); expect(packageVersionWarning(DEP, py, "1.1.0", "1.0.0")).toContain("is 1.1.0"); }); diff --git a/server/typescript/packages/cli/test/eject-reordered.test.ts b/server/typescript/packages/cli/test/eject-reordered.test.ts new file mode 100644 index 0000000000..5c0273cd81 --- /dev/null +++ b/server/typescript/packages/cli/test/eject-reordered.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import { reorderedSuffix } from "../src/commands/eject.js"; +import { compareOwnedCopy } from "../src/lib/owned-copy.js"; + +describe("a pure reorder of an owned copy", () => { + test("is a `differs` verdict with 0/0 counts, and the listing says reordered", async () => { + const reference = "export const a = 1;\nexport const b = 2;\n"; + const owned = "export const b = 2;\nexport const a = 1;\n"; + const r = await compareOwnedCopy(owned, reference); + expect(r.verdict).toBe("differs"); + expect(r.localOnly).toBe(0); + expect(r.referenceOnly).toBe(0); + expect(reorderedSuffix(r.referenceOnly, r.localOnly)).toBe(", reordered"); + }); + + test("a real difference carries no reordered suffix", () => { + expect(reorderedSuffix(0, 1)).toBe(""); + expect(reorderedSuffix(2, 0)).toBe(""); + }); +}); diff --git a/server/typescript/packages/cli/test/eject-undo.test.ts b/server/typescript/packages/cli/test/eject-undo.test.ts index d0d71edae8..9446ca3128 100644 --- a/server/typescript/packages/cli/test/eject-undo.test.ts +++ b/server/typescript/packages/cli/test/eject-undo.test.ts @@ -104,6 +104,29 @@ describe("meta eject --undo", () => { } }); + test("a copy whose only edit is a reordering is customised, not stock", async () => { + const dir = tmp(); + try { + await ejectCommand(["barrel"], dir, "text"); + // Swap two statements of the generator body: the same lines in a different order. + const lines = readFileSync(gen(dir, "barrel"), "utf8").split("\n"); + const i = lines.findIndex((l) => l.includes("const sorted = [")); + expect(i).toBeGreaterThan(0); + [lines[i], lines[i + 1]] = [lines[i + 1] as string, lines[i] as string]; + writeFileSync(gen(dir, "barrel"), lines.join("\n"), "utf8"); + logged = []; erred = []; + expect(await ejectCommand(["--undo"], dir, "text")).toBe(0); + expect(existsSync(gen(dir, "barrel"))).toBe(true); + expect(text()).toContain("customised"); + expect(await ejectCommand(["--undo", "barrel"], dir, "text")).toBe(2); + expect(existsSync(gen(dir, "barrel"))).toBe(true); + expect(await ejectCommand(["--undo", "--force", "barrel"], dir, "text")).toBe(0); + expect(existsSync(gen(dir, "barrel"))).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("--force removes a named customised copy", async () => { const dir = tmp(); try { diff --git a/server/typescript/packages/codegen-ts/src/naming/collection-name-collision.ts b/server/typescript/packages/codegen-ts/src/naming/collection-name-collision.ts index a9ed4a3752..84395be27f 100644 --- a/server/typescript/packages/codegen-ts/src/naming/collection-name-collision.ts +++ b/server/typescript/packages/codegen-ts/src/naming/collection-name-collision.ts @@ -16,7 +16,9 @@ // WHOLE set, with a message naming both entities and the colliding name. import { type ErrorCode, OBJECT_SUBTYPE_VALUE, type MetaObject } from "@metaobjectsdev/metadata"; -import { pluralize } from "@metaobjectsdev/metadata"; +import { pluralize, toSnakeCase } from "@metaobjectsdev/metadata"; +import { emitsInstanceArtifacts } from "../instance-artifacts.js"; +import { isTphSubtype } from "../tph.js"; // Already promoted to the shared cross-language error-code ledger // (fixtures/conformance/ERROR-CODES.json) and to TS's own ERROR_CODES array @@ -59,4 +61,30 @@ export function assertNoCollectionNameCollisions(entities: readonly MetaObject[] } if (existing === undefined) ownerOfPlural.set(plural, obj); } + assertNoRouteSegmentCollisions(entities); +} + +/** + * The route half. The REST collection segment is the entity NAME snake_cased and then + * pluralized (`entity-ui-descriptor.ts`), so `ApiKey` and `APIKey` both serve `/api_keys` + * while their pluralized PascalCase names differ and pass the check above. Counts only + * objects that actually get a route: not a value object, not abstract, not sourceless, + * not a TPH subtype (folded into its base's table and served under the base's route). + */ +function assertNoRouteSegmentCollisions(entities: readonly MetaObject[]): void { + const ownerOfSegment = new Map(); + for (const obj of entities) { + if (obj.subType === OBJECT_SUBTYPE_VALUE) continue; + if (!emitsInstanceArtifacts(obj) || isTphSubtype(obj)) continue; + const segment = pluralize(toSnakeCase(obj.name)); + const existing = ownerOfSegment.get(segment); + if (existing !== undefined && existing.resolutionKey() !== obj.resolutionKey()) { + throw new Error( + `${ERR_COLLECTION_NAME_COLLISION}: "${existing.name}" and "${obj.name}" both resolve ` + + `to the REST collection segment "${segment}" — rename one entity so its snake_cased, ` + + `pluralized route segment is distinct.`, + ); + } + if (existing === undefined) ownerOfSegment.set(segment, obj); + } } diff --git a/server/typescript/packages/codegen-ts/test/naming/collection-name-collision.test.ts b/server/typescript/packages/codegen-ts/test/naming/collection-name-collision.test.ts index 6d1f8b99ab..16b2c5c629 100644 --- a/server/typescript/packages/codegen-ts/test/naming/collection-name-collision.test.ts +++ b/server/typescript/packages/codegen-ts/test/naming/collection-name-collision.test.ts @@ -105,4 +105,31 @@ describe("assertNoCollectionNameCollisions", () => { expect(() => assertNoCollectionNameCollisions(objects)).toThrow(ERR_COLLECTION_NAME_COLLISION); } }); + + test("ApiKey and APIKey collide on the snake_cased route segment", async () => { + const objects = await loadObjects([entity("ApiKey"), entity("APIKey")]); + expect(() => assertNoCollectionNameCollisions(objects)).toThrow( + new RegExp(`${ERR_COLLECTION_NAME_COLLISION}.*"ApiKey".*"APIKey".*"api_keys"`), + ); + }); + + test("a route-segment twin that is abstract or sourceless gets no route and does not collide", async () => { + const abstractTwin = { + "object.entity": { + name: "APIKey", + "@isAbstract": true, + children: [{ "field.long": { name: "id" } }, { "identity.primary": { "@fields": "id" } }], + }, + }; + const sourceless = { + "object.entity": { + name: "APIKey", + children: [{ "field.long": { name: "id" } }, { "identity.primary": { "@fields": "id" } }], + }, + }; + for (const twin of [abstractTwin, sourceless]) { + const objects = await loadObjects([entity("ApiKey"), twin]); + expect(() => assertNoCollectionNameCollisions(objects)).not.toThrow(); + } + }); }); diff --git a/server/typescript/packages/metadata/src/core/field/validate-enum-normalize-ambiguity.ts b/server/typescript/packages/metadata/src/core/field/validate-enum-normalize-ambiguity.ts index ee9a84a835..16ec25c610 100644 --- a/server/typescript/packages/metadata/src/core/field/validate-enum-normalize-ambiguity.ts +++ b/server/typescript/packages/metadata/src/core/field/validate-enum-normalize-ambiguity.ts @@ -27,6 +27,7 @@ // can merge tokens across a delimiter like "|" — both are skipped. import type { MetaData } from "../../shared/meta-data.js"; +import { ParseError } from "../../errors.js"; import type { LoaderWarning } from "../../source.js"; import { TYPE_FIELD } from "../../shared/base-types.js"; import { TYPE_OBJECT } from "../../shared/base-types.js"; @@ -163,3 +164,55 @@ export function validateEnumNormalizeAmbiguity(root: MetaData): EnumNormalizeAmb visit(root); return { warnings }; } + +/** + * ERR_ENUM_NORMALIZE_COLLISION — two declared members of one enum fold to the SAME key + * under `@normalize: strip` (e.g. `READ_ONLY` and `READONLY`). The extract engine can + * only resolve a reply to one of them, so the other becomes unreachable and a reply + * naming it is coerced to the wrong member (wrong-and-green). An ERROR, unlike the + * concatenation advisory above: the vocabulary itself is ambiguous, whatever the input. + * `collapse` / `none` are not gated (they keep the separators that distinguish members). + */ +export function validateEnumNormalizeCollision(root: MetaData): ParseError[] { + const errors: ParseError[] = []; + const visit = (node: MetaData): void => { + if (node.type === TYPE_FIELD && node.subType === FIELD_SUBTYPE_ENUM) { + // ADR-0039: own — the vocabulary DECLARED here; an inheriting enum shares the + // super's set, already checked there (one hazard, one error). + const rawValues = node.ownAttrs().get(FIELD_ATTR_VALUES); + if (Array.isArray(rawValues) && rawValues.length > 1) { + const normalize = effectiveNormalize(node); + if (normalize.mode === NORMALIZE_DEFAULT) { + const byKey = new Map(); + for (const m of rawValues) { + const member = String(m); + const key = stripNormalize(member); + if (key.length === 0) continue; + byKey.set(key, [...(byKey.get(key) ?? []), member]); + } + for (const [key, members] of byKey) { + // Distinct declared members only: an exact duplicate is ERR_BAD_ATTR_VALUE's job. + const distinct = [...new Set(members)]; + if (distinct.length < 2) continue; + errors.push( + new ParseError( + `field.enum "${node.name}" members ${distinct.map((d) => `'${d}'`).join(", ")} all ` + + `normalize to '${key}' under @${FIELD_ATTR_NORMALIZE}: '${NORMALIZE_DEFAULT}'` + + `${normalize.explicit ? "" : " (the default)"}, which upper-cases and deletes ` + + `everything outside A-Z0-9, so a reply naming one of them is indistinguishable ` + + `from the others. Rename the members so they differ after that fold, or set ` + + `@${FIELD_ATTR_NORMALIZE}: 'collapse' (keeps '_' distinct) or 'none' (exact ` + + `match) on this field.`, + { code: "ERR_ENUM_NORMALIZE_COLLISION", source: node.source }, + ), + ); + break; // one error per declaring node — the first collision is enough to act on + } + } + } + } + for (const child of node.ownChildren()) visit(child); + }; + visit(root); + return errors; +} diff --git a/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts b/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts index bdd25669c6..af2b7526ca 100644 --- a/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts +++ b/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts @@ -129,7 +129,12 @@ export function validateDiscriminator(root: MetaData): ParseError[] { for (const [, bindings] of bindingsByRoot) { const seen = new Map(); for (const b of bindings) { - const prev = seen.get(b.value); + // "1" and "01" are one value to a numeric column, so they are one claim. + const key = + NUMERIC_DISCRIMINATOR_SUBTYPES.has(b.discField.subType) && /^-?[0-9]+$/.test(b.value) + ? BigInt(b.value).toString() + : b.value; + const prev = seen.get(key); if (prev !== undefined) { errors.push( new ParseError( @@ -139,7 +144,7 @@ export function validateDiscriminator(root: MetaData): ParseError[] { ), ); } else { - seen.set(b.value, b.subtype); + seen.set(key, b.subtype); } } } diff --git a/server/typescript/packages/metadata/src/errors.ts b/server/typescript/packages/metadata/src/errors.ts index da75f58441..504ca1ec0d 100644 --- a/server/typescript/packages/metadata/src/errors.ts +++ b/server/typescript/packages/metadata/src/errors.ts @@ -120,6 +120,9 @@ export const ERROR_CODES = [ "ERR_DERIVED_FIELD_NO_READ_SOURCE", "ERR_ENTITY_PRIMARY_SOURCE_READONLY", "ERR_INVALID_TEMPLATE", + // Two fields of one stored object resolve to the same PHYSICAL column (snake_case + // folds `userId` and `userID` together, or two fields share an @column). + "ERR_COLUMN_NAME_COLLISION", // FR-017 — M:N relationship validation (slim vocabulary): @through must name a // junction declaring two identity.reference children; @sourceRefField must match // one of them; M:N attrs are invalid on a 1:N (@cardinality:one / no @through). @@ -275,6 +278,11 @@ export const ERROR_CODES = [ // an integer array. An array-of-enum stays string-backed: drop @intValueMap, // or make the field scalar. "ERR_ENUM_INT_VALUE_MAP_ARRAY", + // Two declared members of one field.enum normalize to the same key under the + // default `@normalize: strip` (upper-case, delete everything outside A-Z0-9), so + // one is unreachable and a reply naming it coerces to the other. Rename a member, + // or set `@normalize: collapse` / `none`. + "ERR_ENUM_NORMALIZE_COLLISION", // Phase-1 metadata-source-resolution — a path source declared in // .metaobjects/config.json does not exist on disk. "ERR_SOURCE_UNRESOLVED", diff --git a/server/typescript/packages/metadata/src/loader/meta-data-loader.ts b/server/typescript/packages/metadata/src/loader/meta-data-loader.ts index 9285fab035..f939431360 100644 --- a/server/typescript/packages/metadata/src/loader/meta-data-loader.ts +++ b/server/typescript/packages/metadata/src/loader/meta-data-loader.ts @@ -25,7 +25,7 @@ import { ParseError } from "../errors.js"; import type { LoaderWarning } from "../source.js"; import { codeSource, resolvedSource } from "../source.js"; import { parseJson } from "../parser-json.js"; -import { validateDataGridSortFields, validateFilterableHasIndex, validateFilterableHasSupportedOps, validateSortableHasSupportedSubtype, validateOriginPaths, validateDerivedFieldProvidability, validateDataGridFilterValues, validateFieldObjectStorage, validateFieldMap, validateTemplatePayloadRefs, validateFieldDefaults, validateRelationships, validateOneSideReferenceResolution, validateM2MJunctionPairing, validateIndexLookupFields, validateProjectionFilter, validateRetiredRequirementLinks } from "./validation-passes.js"; +import { validateDataGridSortFields, validateFilterableHasIndex, validateFilterableHasSupportedOps, validateSortableHasSupportedSubtype, validateOriginPaths, validateDerivedFieldProvidability, validateDataGridFilterValues, validateFieldObjectStorage, validateFieldMap, validateTemplatePayloadRefs, validateFieldDefaults, validateRelationships, validateOneSideReferenceResolution, validateM2MJunctionPairing, validateIndexLookupFields, validateProjectionFilter, validateRetiredRequirementLinks, validateColumnNameCollisions } from "./validation-passes.js"; import { validateReporting } from "./reporting-validation.js"; import { runRegisteredValidation } from "./validation-registry.js"; import { validateSourceRoles } from "../persistence/source/validate-source-roles.js"; @@ -33,7 +33,7 @@ import { validateSourceEscapes } from "../persistence/source/validate-source-esc import { validateSourcePhysicalNames } from "../persistence/source/validate-source-physical-names.js"; import { validateSourceParameterRef } from "../persistence/source/validate-source-parameter-ref.js"; import { validateFieldMutability } from "../core/field/validate-field-mutability.js"; -import { validateEnumNormalizeAmbiguity } from "../core/field/validate-enum-normalize-ambiguity.js"; +import { validateEnumNormalizeAmbiguity, validateEnumNormalizeCollision } from "../core/field/validate-enum-normalize-ambiguity.js"; import { validateDiscriminator } from "../core/object/validate-discriminator.js"; import { resolveDeferredSupers, EXTENDS_TARGET_MISMATCH_RULE } from "../super-resolve.js"; import { resolveReferences } from "./resolve-references.js"; @@ -691,6 +691,9 @@ export class MetaDataLoader { // (ADR-0039: resolving accessor, so inherited fields via extends are visible). errors.push(...validateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + errors.push(...validateColumnNameCollisions(root)); + // FR-039 — a retired requirement carries no @implementedBy (refused, not // exempted, so the dangling-ref class is unreachable) and @supersededBy is // legal only on `retired`. @@ -752,6 +755,8 @@ export class MetaDataLoader { // others), which would silently coerce delimited input to a wrong-but-valid // member. WARN_ENUM_NORMALIZE_AMBIGUOUS. envelopeWarnings.push(...validateEnumNormalizeAmbiguity(root).warnings); + // Two members folding to the same key under `strip` — ERR_ENUM_NORMALIZE_COLLISION. + errors.push(...validateEnumNormalizeCollision(root)); // FR-015 — source.rdb @parameterRef typed-input validation. errors.push(...validateSourceParameterRef(root)); diff --git a/server/typescript/packages/metadata/src/loader/sources/directory-source.ts b/server/typescript/packages/metadata/src/loader/sources/directory-source.ts index 3fc4aa889a..561568f1e8 100644 --- a/server/typescript/packages/metadata/src/loader/sources/directory-source.ts +++ b/server/typescript/packages/metadata/src/loader/sources/directory-source.ts @@ -6,12 +6,16 @@ // across environments and language ports. import { readdir, stat } from "node:fs/promises"; -import { basename, extname, join } from "node:path"; +import { basename, extname, join, relative, sep } from "node:path"; import { FileSource } from "./file-source.js"; import type { MetaDataSource } from "../meta-data-source.js"; export interface DirectoryOptions { - /** Filename patterns to exclude. Supports literal match and `*` / `**` globs. */ + /** + * Patterns to exclude. Supports literal match and `*` / `**` globs. A pattern without `/` + * matches the file's basename; a pattern containing `/` matches its `/`-separated path + * relative to the directory (so `vendor/**` excludes a subtree). + */ exclude?: string[]; /** Recurse into subdirectories. Default: true. */ recurse?: boolean; @@ -44,7 +48,8 @@ export class DirectorySource { for (const p of files) { if (!SUPPORTED_EXTS.has(extname(p).toLowerCase())) continue; const name = basename(p); - if (exclude.some((pat) => matchSimpleGlob(pat, name))) continue; + const rel = relative(this.directory, p).split(sep).join("/"); + if (exclude.some((pat) => matchSimpleGlob(pat, pat.includes("/") ? rel : name))) continue; filtered.push(p); } diff --git a/server/typescript/packages/metadata/src/loader/validation-passes.ts b/server/typescript/packages/metadata/src/loader/validation-passes.ts index 5f3650e8da..116eed45a8 100644 --- a/server/typescript/packages/metadata/src/loader/validation-passes.ts +++ b/server/typescript/packages/metadata/src/loader/validation-passes.ts @@ -55,6 +55,8 @@ import { OBJECT_PROJECTION_ATTR_FILTER, } from "../core/object/object-constants.js"; import { MetaSource } from "../persistence/source/meta-source.js"; +import { resolveColumnName } from "../naming.js"; +import { isMetaField } from "../shared/node-guards.js"; import { LAYOUT_SUBTYPE_DATA_GRID, LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_FIELD, @@ -2762,3 +2764,43 @@ export function validateRetiredRequirementLinks(root: MetaData): ParseError[] { walk(root); return errors; } + +// --------------------------------------------------------------------------- +// Physical column-name collisions +// +// Two fields of one stored object that resolve to the SAME physical column +// (`userId` + `userID` both snake_case to `user_id`; or two fields sharing an +// explicit @column) silently produce a duplicate column. Only the RESOLVED name +// is compared, through the single door `resolveColumnName`, so an explicit @column +// override, an inherited field and the project's column strategy are all honoured. +// Scope: a concrete object that declares at least one source (a source-less +// object generates nothing); derived fields (origin.*) and field.object (which may +// flatten to several columns) are not columns of this table and are skipped. +// ADR-0039: effective fields, so an inherited field collides like an own one. +// --------------------------------------------------------------------------- + +export function validateColumnNameCollisions(root: MetaData): ParseError[] { + const errors: ParseError[] = []; + for (const obj of root.children().filter((c) => c.type === TYPE_OBJECT)) { + if (obj.isAbstract) continue; + if (!obj.children().some((c) => c.type === TYPE_SOURCE)) continue; + const seen = new Map(); + for (const field of obj.children()) { + if (!isMetaField(field) || field.subType === FIELD_SUBTYPE_OBJECT || field.isDerived()) continue; + const column = resolveColumnName(field); + const first = seen.get(column); + if (first === undefined) { + seen.set(column, field); + continue; + } + errors.push( + new ParseError( + `fields "${first.name}" and "${field.name}" on "${obj.name}" both resolve to physical column ` + + `"${column}". Rename one field, or give one an explicit @column so the two columns differ.`, + { code: "ERR_COLUMN_NAME_COLLISION", source: field.source }, + ), + ); + } + } + return errors; +} diff --git a/server/typescript/packages/metadata/src/vocabulary-rewrite.ts b/server/typescript/packages/metadata/src/vocabulary-rewrite.ts index 926876b5aa..76ac210f82 100644 --- a/server/typescript/packages/metadata/src/vocabulary-rewrite.ts +++ b/server/typescript/packages/metadata/src/vocabulary-rewrite.ts @@ -153,6 +153,38 @@ function atOrBefore(a: string, b: string): boolean { return true; } +/** + * `text` with every JSONC comment overwritten by spaces (newlines kept), string literals + * untouched. Offsets are preserved exactly, so every scan below can run on the masked copy + * while edits are applied to the original: a `}` or `,` inside a comment can no longer end a + * node body or a scalar value early, and a commented-out key no longer matches. A comment + * the scan erases is also erased if its key is dropped, which is the right outcome for a + * trailing note about that key. + */ +function maskComments(text: string): string { + const out = text.split(""); + let inStr = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (inStr) { + if (c === "\\") i++; + else if (c === '"') inStr = false; + continue; + } + if (c === '"') { + inStr = true; + } else if (c === "/" && text[i + 1] === "/") { + for (; i < text.length && text[i] !== "\n"; i++) out[i] = " "; + } else if (c === "/" && text[i + 1] === "*") { + const end = text.indexOf("*/", i + 2); + const stop = end < 0 ? text.length : end + 2; + for (; i < stop; i++) if (text[i] !== "\n") out[i] = " "; + i--; + } + } + return out.join(""); +} + /** 1-indexed line of `index` in `text`. */ function lineAt(text: string, index: number): number { let line = 1; @@ -316,7 +348,10 @@ function valueSpan(text: string, from: number): { start: number; end: number; ra if (c === '"') inStr = true; else if (c === "," || c === "}" || c === "\n") break; } - return { start: i, end: j, raw: text.slice(i, j).trim() }; + // End at the last token character, not at the delimiter: the gap before it may be a + // (masked) comment, which a value replacement must not swallow. + const raw = text.slice(i, j).trim(); + return { start: i, end: i + raw.length, raw }; } /** Does the raw token equal this literal? Compares JSON-decoded where possible so @@ -371,7 +406,9 @@ function dropSpan(source: string, keyStart: number, valueEnd: number): { start: * Pure: no filesystem, no loader, no registry. Returns the new text plus every change and * every refusal, so a caller can print a diff and exit non-zero when work remains. */ -export function rewriteDocument(source: string, opts: RewriteOpts = {}): RewriteResult { +export function rewriteDocument(original: string, opts: RewriteOpts = {}): RewriteResult { + // Scan the comment-masked copy; apply edits to the original (offsets are identical). + const source = maskComments(original); const changes: RewriteChange[] = []; const refusals: RewriteRefusal[] = []; @@ -519,7 +556,7 @@ export function rewriteDocument(source: string, opts: RewriteOpts = {}): Rewrite } edits.sort((a, b) => b.start - a.start); - let text = source; + let text = original; for (const e of edits) text = text.slice(0, e.start) + e.text + text.slice(e.end); return { text, changes, refusals }; diff --git a/server/typescript/packages/metadata/test/enum-normalize-ambiguity.test.ts b/server/typescript/packages/metadata/test/enum-normalize-ambiguity.test.ts index 4ffc55625a..e3329a98e3 100644 --- a/server/typescript/packages/metadata/test/enum-normalize-ambiguity.test.ts +++ b/server/typescript/packages/metadata/test/enum-normalize-ambiguity.test.ts @@ -119,3 +119,27 @@ describe("WARN_ENUM_NORMALIZE_AMBIGUOUS scope", () => { expect(warnings.filter((x) => x.code === "WARN_ENUM_NORMALIZE_AMBIGUOUS").length).toBe(1); }); }); + +describe("ERR_ENUM_NORMALIZE_COLLISION", () => { + const codes = async (values: string[], normalize?: string) => + (await load(enumDoc(values, normalize))).errors.map((e) => (e as { code?: string }).code); + + test("default strip: members folding to one key are refused, naming both", async () => { + expect(await codes(["READ_ONLY", "READONLY", "WRITE"])).toContain("ERR_ENUM_NORMALIZE_COLLISION"); + const { errors } = await load(enumDoc(["READ_ONLY", "READONLY", "WRITE"])); + expect(errors[0]!.message).toContain("'READ_ONLY', 'READONLY'"); + }); + + test("explicit strip is refused too", async () => { + expect(await codes(["X_1", "X1"], "strip")).toContain("ERR_ENUM_NORMALIZE_COLLISION"); + }); + + test("collapse and none do not error", async () => { + expect(await codes(["READ_ONLY", "READONLY"], "collapse")).not.toContain("ERR_ENUM_NORMALIZE_COLLISION"); + expect(await codes(["READ_ONLY", "READONLY"], "none")).not.toContain("ERR_ENUM_NORMALIZE_COLLISION"); + }); + + test("collision-free control loads", async () => { + expect(await codes(["READ_ONLY", "WRITE", "ADMIN"])).toEqual([]); + }); +}); diff --git a/server/typescript/packages/metadata/test/loader/sources.test.ts b/server/typescript/packages/metadata/test/loader/sources.test.ts index a93c376d18..3e63d38c3d 100644 --- a/server/typescript/packages/metadata/test/loader/sources.test.ts +++ b/server/typescript/packages/metadata/test/loader/sources.test.ts @@ -79,6 +79,20 @@ describe("DirectorySource", () => { } }); + test("exclude with a slash matches the path relative to the directory", async () => { + const dir = await mkdtemp(join(tmpdir(), "ds-")); + try { + await mkdir(join(dir, "vendor", "deep"), { recursive: true }); + await mkdir(join(dir, "app"), { recursive: true }); + await writeFile(join(dir, "vendor", "deep", "meta.v.json"), "{}"); + await writeFile(join(dir, "app", "meta.app.json"), "{}"); + const expanded = await new DirectorySource(dir, { exclude: ["vendor/**"] }).expand(); + expect(expanded.map((s) => s.id)).toEqual(["meta.app.json"]); + } finally { + await rm(dir, { recursive: true }); + } + }); + test("recurses by default into subdirectories", async () => { const dir = await mkdtemp(join(tmpdir(), "ds-")); try { diff --git a/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts b/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts index 77ed2e2c9c..3f5de6e24f 100644 --- a/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts +++ b/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts @@ -241,3 +241,38 @@ describe("nothing exits clean while the metadata still will not load", () => { expect(() => JSON.parse(r.text)).not.toThrow(); }); }); + +describe("JSONC comments (audit finding 11)", () => { + test("a brace inside a // comment does not end the node scope early", () => { + const src = `{ "field.string": { "name": "ref", // closes } early? + "@readOnly": true } }`; + const r = rewriteDocument(src); + expect(r.text).toContain('"@mutability": "readOnly"'); + expect(r.text).toContain("// closes } early?"); + }); + + test("a trailing // comment after a value-scoped retired value still matches", () => { + const src = `{ "requirement.functional": { "name": "r", + "@status": "abandoned" // gave up +} }`; + const r = rewriteDocument(src); + expect(r.changes.length).toBeGreaterThan(0); + expect(r.text).not.toContain('"abandoned"'); + expect(r.text).toContain("// gave up"); + }); + + test("a trailing block comment does not hide a rewrite", () => { + const src = `{ "field.string": { "name": "ref", "@readOnly": true /* old */ } }`; + const r = rewriteDocument(src); + expect(r.text).toContain('"@mutability": "readOnly"'); + }); + + test("a commented-out retired key is left alone", () => { + const src = `{ "field.string": { "name": "ref", + // "@readOnly": true, + "@maxLength": 5 } }`; + const r = rewriteDocument(src); + expect(r.changes).toEqual([]); + expect(r.text).toBe(src); + }); +}); diff --git a/server/typescript/packages/migrate-ts/src/apply/apply.ts b/server/typescript/packages/migrate-ts/src/apply/apply.ts index 8076c122af..894d240dad 100644 --- a/server/typescript/packages/migrate-ts/src/apply/apply.ts +++ b/server/typescript/packages/migrate-ts/src/apply/apply.ts @@ -141,7 +141,7 @@ export async function applyPending( // directory of migrations, and `--replay` guessing at a remedy it could not // have chosen correctly without knowing the position. try { - await runSqlFileWithLedgerMutation(db, text, (trx) => + await runSqlFileWithLedgerMutation(db, text, dialect, (trx) => recordApplied(trx, m.name, checksum, dialect, ledger), ); } catch (err) { @@ -218,14 +218,14 @@ export async function rollbackTo( } // Comments alone reverse nothing: running them would only delete the ledger row // and report a rollback that never happened. - if (splitSqlStatements(downText).length === 0) { + if (splitSqlStatements(downText, { dialect }).length === 0) { throw new Error( `rollback '${name}': down.sql has no executable statement (only comments) — ` + `nothing would be reversed. Author the down by hand, or roll forward instead.`, ); } // Run the down SQL + the ledger delete in ONE transaction. - await runSqlFileWithLedgerMutation(db, downText, (trx) => + await runSqlFileWithLedgerMutation(db, downText, dialect, (trx) => deleteApplied(trx, name, dialect, ledger), ); rolledBack.push(name); @@ -338,6 +338,7 @@ function advisoryKey(name: string): string { async function runSqlFileWithLedgerMutation( db: Kysely>, sqlText: string, + dialect: LedgerDialect, mutateLedger: (trx: Transaction>) => Promise, ): Promise { // Adapt the file to the transaction we are about to open. A SQLite rebuild script @@ -346,7 +347,7 @@ async function runSqlFileWithLedgerMutation( // scaffold's default dialect) and its own `PRAGMA foreign_keys = OFF`, which is a // no-op inside a transaction. See runner-transaction-pass.ts for the full rationale. const { statements, requiresForeignKeysOff, postTransactionChecks } = - prepareForRunnerTransaction(sqlText); + prepareForRunnerTransaction(sqlText, dialect); // OUTSIDE the transaction, deliberately: `PRAGMA foreign_keys` is a no-op inside one, // and `defer_foreign_keys` cannot stand in for it here — `DROP TABLE`'s implicit delete diff --git a/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts b/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts index b19b9b3435..4119cd0f98 100644 --- a/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts +++ b/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts @@ -1,4 +1,4 @@ -import { splitSqlStatements, stripLeadingComments } from "../sql/split-statements.js"; +import { type SplitSqlDialect, splitSqlStatements, stripLeadingComments } from "../sql/split-statements.js"; /** * Adapt a migration file's statements for execution INSIDE the apply runner's own @@ -63,7 +63,8 @@ import { splitSqlStatements, stripLeadingComments } from "../sql/split-statement * caught anything; the caller now runs it after commit and fails on any row. * * Postgres migrations contain none of these constructs, so this is a no-op for them — - * which is why it keys on statement text rather than needing the dialect threaded in. + * which is why it keys on statement text; the dialect only tells the splitter whether `E'…'` + * is a backslash-escaped literal (Postgres) or not. */ export interface RunnerTransactionPassResult { /** Statements to execute, in order, inside the runner's transaction. */ @@ -80,18 +81,21 @@ export interface RunnerTransactionPassResult { postTransactionChecks: string[]; } -const TRANSACTION_CONTROL = /^\s*(BEGIN|COMMIT|END\s+TRANSACTION|ROLLBACK|SAVEPOINT|RELEASE)\b/i; +const TRANSACTION_CONTROL = /^\s*(BEGIN|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE)\b/i; const FK_OFF = /^\s*PRAGMA\s+foreign_keys\s*=\s*(OFF|0|false)\s*$/i; const FK_ON = /^\s*PRAGMA\s+foreign_keys\s*=\s*(ON|1|true)\s*$/i; const FK_CHECK = /^\s*PRAGMA\s+foreign_key_check\s*$/i; -export function prepareForRunnerTransaction(sqlText: string): RunnerTransactionPassResult { +export function prepareForRunnerTransaction( + sqlText: string, + dialect: SplitSqlDialect = "postgres", +): RunnerTransactionPassResult { const notes: string[] = []; const statements: string[] = []; const postTransactionChecks: string[] = []; let requiresForeignKeysOff = false; - for (const stmt of splitSqlStatements(sqlText)) { + for (const stmt of splitSqlStatements(sqlText, { dialect })) { // Classify on the statement's SQL, not its prose: a rebuild down leads with a comment // header, and `^\s*PRAGMA` never matched a statement that started with one. const code = stripLeadingComments(stmt); diff --git a/server/typescript/packages/migrate-ts/src/emit/d1-safety-pass.ts b/server/typescript/packages/migrate-ts/src/emit/d1-safety-pass.ts index a2ee1b03ac..90f1b67d1e 100644 --- a/server/typescript/packages/migrate-ts/src/emit/d1-safety-pass.ts +++ b/server/typescript/packages/migrate-ts/src/emit/d1-safety-pass.ts @@ -28,7 +28,7 @@ export function applyD1SafetyPass(sql: string, opts?: { collectWarnings?: boolea // fragments (a down's trailing `-- WARNING…` block) are kept so the prose survives // the rewrite, and are re-emitted WITHOUT a terminator: a `;` after them would make // an empty statement for whatever executes the file. - const statements = splitSqlStatements(sql, { keepCommentOnly: true }); + const statements = splitSqlStatements(sql, { keepCommentOnly: true, dialect: "d1" }); const kept: string[] = []; for (const stmt of statements) { @@ -67,7 +67,7 @@ export function applyD1SafetyPass(sql: string, opts?: { collectWarnings?: boolea /** True when a (kept) fragment carries no SQL — the splitter drops those by default. */ function isCommentOnly(fragment: string): boolean { - return splitSqlStatements(fragment).length === 0; + return splitSqlStatements(fragment, { dialect: "d1" }).length === 0; } function byteLength(s: string): number { diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index 828f54c07a..8104ab2dce 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -160,5 +160,5 @@ export { type D1Binding, type WranglerConfig, } from "./wrangler-config.js"; -export { splitSqlStatements } from "./sql/split-statements.js"; +export { splitSqlStatements, type SplitSqlDialect } from "./sql/split-statements.js"; export { viewFingerprint } from "./view-fingerprint.js"; diff --git a/server/typescript/packages/migrate-ts/src/introspect/d1.ts b/server/typescript/packages/migrate-ts/src/introspect/d1.ts index fa809e65e8..b14afc35e7 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/d1.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/d1.ts @@ -4,7 +4,7 @@ import type { } from "../types.js"; import { parseSqliteDefault, sqliteTypeToSqlType, sqliteRuleToAction, parseSqliteChecks, - buildSqliteIndexDescriptor, + buildSqliteIndexDescriptor, hasAutoincrementKeyword, } from "./sqlite-shared.js"; /** @@ -86,7 +86,7 @@ export async function introspectD1(opts: IntrospectD1Options): Promise c.name === pk[0]); if (pkCol) pkCol.identity = "increment"; diff --git a/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts b/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts index d682afb987..fc94ddffd4 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts @@ -337,3 +337,14 @@ export function sqliteRuleToAction(rule: string): FkAction { if (r === "RESTRICT") return "restrict"; return "no-action"; } + +/** + * True when a CREATE TABLE declares AUTOINCREMENT as a keyword: the scan runs on a masked + * copy, so a comment, a string literal (`DEFAULT 'AUTOINCREMENT'`) cannot trip it. A + * double-quoted identifier is preserved by the mask, so it is blanked here too. + */ +export function hasAutoincrementKeyword(createSql: string | null | undefined): boolean { + if (createSql === null || createSql === undefined) return false; + const masked = maskCommentsAndStrings(createSql).replace(/"(?:[^"]|"")*"/g, (m) => " ".repeat(m.length)); + return /\bAUTOINCREMENT\b/i.test(masked); +} diff --git a/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts b/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts index 910ad9d2f2..2435a66602 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts @@ -6,7 +6,7 @@ import type { } from "../types.js"; import { parseSqliteDefault, sqliteTypeToSqlType, sqliteRuleToAction, parseSqliteChecks, - buildSqliteIndexDescriptor, + buildSqliteIndexDescriptor, hasAutoincrementKeyword, } from "./sqlite-shared.js"; import { MIGRATIONS_TABLE } from "../apply/ledger.js"; @@ -36,7 +36,7 @@ export async function introspectSqlite(db: Kysely>): Pro const cols = await readSqliteColumns(k, t.name); const pk = await readSqlitePrimaryKey(k, t.name); // Detect AUTOINCREMENT via the CREATE TABLE statement (sqlite has no PRAGMA for it). - const hasAutoincrement = (t.sql ?? "").toUpperCase().includes("AUTOINCREMENT"); + const hasAutoincrement = hasAutoincrementKeyword(t.sql); if (hasAutoincrement && pk.length === 1) { const pkCol = cols.find((c) => c.name === pk[0]); if (pkCol) pkCol.identity = "increment"; diff --git a/server/typescript/packages/migrate-ts/src/sql/split-statements.ts b/server/typescript/packages/migrate-ts/src/sql/split-statements.ts index c445db1824..d2f7036dc5 100644 --- a/server/typescript/packages/migrate-ts/src/sql/split-statements.ts +++ b/server/typescript/packages/migrate-ts/src/sql/split-statements.ts @@ -5,6 +5,12 @@ * used both when applying migration files ({@link ../apply/apply.ts}) and when * post-processing emitted DDL for the D1 target ({@link ../emit/d1-safety-pass.ts}). * + * Two engine rules a plain `;` scan gets wrong are honoured: a SQLite + * `CREATE [TEMP] TRIGGER … BEGIN … END` is ONE statement (its body holds `;`; as in + * `sqlite3_complete`, only an `END` that follows a `;` closes it, so a `CASE … END` + * inside a body does not), and, on Postgres only, an `E'…'` literal treats `\` as an escape, so + * `E'it\'s; fine'` is one literal. + * * Returned statements are trimmed and DO NOT include the trailing `;` * separator; callers that need a terminator re-add one. * @@ -15,7 +21,15 @@ * statement stays attached to it. Re-emitters that must preserve the prose (the D1 * safety pass rewrites a file it does not execute) pass `keepCommentOnly`. */ +export type SplitSqlDialect = "postgres" | "sqlite" | "mysql" | "d1"; + export interface SplitSqlOptions { + /** + * The engine the script targets. Only Postgres reads `E'…'` as a backslash-escaped + * literal; on SQLite, MySQL and D1 `e'\'` is the identifier-ish word `e` followed by + * an ordinary literal. Omitted means Postgres (the original behaviour). + */ + dialect?: SplitSqlDialect; /** Keep comment-only fragments in the result (re-emitters only — never execute them). */ keepCommentOnly?: boolean; } @@ -25,8 +39,24 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st let start = 0; let i = 0; const n = text.length; + const backslashEscapes = (opts.dialect ?? "postgres") === "postgres"; // Whether the current fragment holds anything besides comments and whitespace. let hasCode = false; + // Trigger tracking (per statement): the leading words, whether we are inside the + // BEGIN … END body, whether the previous token was `;`, and whether END was just seen. + let leadWords: string[] = []; + let inTriggerBody = false; + let afterSemi = false; + let afterEnd = false; + const resetStatement = (): void => { + leadWords = []; + inTriggerBody = false; + afterSemi = false; + afterEnd = false; + }; + const isTriggerStart = (): boolean => + leadWords[0] === "CREATE" && + (leadWords[1] === "TRIGGER" || ((leadWords[1] === "TEMP" || leadWords[1] === "TEMPORARY") && leadWords[2] === "TRIGGER")); const pushFragment = (fragment: string): void => { const stmt = fragment.trim(); if (stmt.length === 0) return; @@ -54,6 +84,32 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st if (ch !== ";" && !/\s/.test(ch as string)) hasCode = true; + // A bare word: feeds trigger detection, and an `E` prefix opens a backslash-escaped literal. + if (/[A-Za-z_]/.test(ch as string)) { + let j = i + 1; + while (j < n && /[A-Za-z0-9_]/.test(text[j] as string)) j++; + const word = text.slice(i, j).toUpperCase(); + if (word === "E" && text[j] === "'" && backslashEscapes) { + i = skipBackslashLiteral(text, j); + afterSemi = false; + afterEnd = false; + continue; + } + if (leadWords.length < 3) leadWords.push(word); + if (inTriggerBody) { + afterEnd = afterSemi && word === "END"; + } else if (word === "BEGIN" && isTriggerStart()) { + inTriggerBody = true; + } + afterSemi = false; + i = j; + continue; + } + if (inTriggerBody && ch !== ";" && !/\s/.test(ch as string)) { + afterSemi = false; + afterEnd = false; + } + // Single-quoted literal: consume to the closing quote, treating '' as escape. if (ch === "'") { i++; @@ -107,7 +163,13 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st } // Statement separator (only reached when in no special state). + if (ch === ";" && inTriggerBody && !afterEnd) { + afterSemi = true; + i++; + continue; + } if (ch === ";") { + resetStatement(); pushFragment(text.slice(start, i)); hasCode = false; i++; @@ -122,6 +184,26 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st return statements; } +/** Skip a Postgres `E'…'` literal starting at the opening quote; `\x` and `''` escape. */ +function skipBackslashLiteral(text: string, quote: number): number { + let i = quote + 1; + while (i < text.length) { + if (text[i] === "\\") { + i += 2; + continue; + } + if (text[i] === "'") { + if (text[i + 1] === "'") { + i += 2; + continue; + } + return i + 1; + } + i++; + } + return i; +} + /** * The statement with any leading `--` / block comments and whitespace removed, for * callers that classify a statement by its first keyword. diff --git a/server/typescript/packages/migrate-ts/src/view-fingerprint.ts b/server/typescript/packages/migrate-ts/src/view-fingerprint.ts index 59d7e5e660..2552d1c215 100644 --- a/server/typescript/packages/migrate-ts/src/view-fingerprint.ts +++ b/server/typescript/packages/migrate-ts/src/view-fingerprint.ts @@ -29,6 +29,7 @@ // the body comparator (view-sql-compare.ts) is exact there. import { createHash } from "node:crypto"; +import { collapseWhitespaceOutsideQuotes } from "./view-sql-compare.js"; /** * Marker format version. Bump ONLY if the normalization rules or the marker grammar @@ -48,20 +49,17 @@ const CREATE_VIEW_PREFIX = /** * Canonicalize a view body for hashing. * - * Collapses whitespace (so reindenting the emitter does not re-stamp every deployed + * Collapses whitespace outside quoted literals/identifiers (so reindenting the emitter does not re-stamp every deployed * view) and strips an optional CREATE VIEW wrapper and trailing semicolon — and * NOTHING else. * - * Deliberately does NOT lowercase, unlike `normalizeViewSql`. That lowercasing exists - * only to chase Postgres's deparser, and it masks drift inside case-sensitive string - * literals — which `origin.aggregate @filter` predicates now carry (`status = 'Active'` - * and `status = 'active'` are different views). We never chase the deparser here, so - * there is nothing to buy and real drift to lose. + * Deliberately does NOT lowercase (nor does `normalizeViewSql` any more): lowercasing + * masks drift inside case-sensitive string literals — which `origin.aggregate @filter` + * predicates now carry (`status = 'Active'` and `status = 'active'` are different views). + * We never chase the deparser here, so there is nothing to buy and real drift to lose. */ export function normalizeForFingerprint(body: string): string { - return body - .replace(CREATE_VIEW_PREFIX, "") - .replace(/\s+/g, " ") + return collapseWhitespaceOutsideQuotes(body.replace(CREATE_VIEW_PREFIX, "")) .replace(/;\s*$/, "") .trim(); } diff --git a/server/typescript/packages/migrate-ts/src/view-sql-compare.ts b/server/typescript/packages/migrate-ts/src/view-sql-compare.ts index 82c7642dcc..cb220b295b 100644 --- a/server/typescript/packages/migrate-ts/src/view-sql-compare.ts +++ b/server/typescript/packages/migrate-ts/src/view-sql-compare.ts @@ -16,30 +16,57 @@ // deparser at all. Do not point this function at Postgres again. // // normalizeViewSql strips a leading `CREATE [OR REPLACE] VIEW AS`, collapses -// whitespace runs, drops a trailing `;`, and lower-cases. -// -// CAVEAT (accepted tradeoff): lower-casing can mask a difference living ONLY in a -// case-sensitive string literal (e.g. `WHERE status = 'Active'` vs `'active'`), which -// an `origin.aggregate @filter` predicate can now carry. The fingerprint path -// deliberately does NOT lowercase for exactly this reason; this SQLite path keeps it -// for backwards-compatible behavior. The name regex matches one whitespace/`(`-free -// token, so a quoted view name containing a space would not strip cleanly — a non-issue -// for generated identifiers. +// whitespace runs OUTSIDE quoted literals and identifiers, and drops a trailing `;`. +// It does NOT lower-case: SQLite stores the text verbatim, so an exact comparison is +// sound, and a case change inside a literal (`status = 'Active'` vs `'active'`) or a +// whitespace change inside one (`'a b'` vs `'a b'`) is a real change of the view. The +// name regex matches one whitespace/`(`-free token, so a quoted view name containing a +// space would not strip cleanly — a non-issue for generated identifiers. const CREATE_VIEW_PREFIX = /^\s*create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?view\s+(?:if\s+not\s+exists\s+)?[^\s(]+(?:\s*\([^)]*\))?\s+as\s+/i; +/** + * Collapse every whitespace run to one space, except inside `'…'` literals, `"…"` and + * `` `…` `` identifiers (doubled quote = escape), which are copied verbatim. + */ +export function collapseWhitespaceOutsideQuotes(sql: string): string { + let out = ""; + let i = 0; + const n = sql.length; + while (i < n) { + const ch = sql[i] as string; + if (ch === "'" || ch === '"' || ch === "`") { + let j = i + 1; + while (j < n) { + if (sql[j] === ch) { + if (sql[j + 1] === ch) { j += 2; continue; } + j++; + break; + } + j++; + } + out += sql.slice(i, j); + i = j; + } else if (/\s/.test(ch)) { + while (i < n && /\s/.test(sql[i] as string)) i++; + out += " "; + } else { + out += ch; + i++; + } + } + return out; +} + /** * Collapse a view definition (full `CREATE VIEW ... AS body` OR a bare body) - * to a canonical, comparable string. Whitespace-, case-, and wrapper-insensitive. + * to a canonical, comparable string. Whitespace- (outside quotes) and wrapper-insensitive; case-sensitive. */ export function normalizeViewSql(sql: string): string { - return sql - .replace(CREATE_VIEW_PREFIX, "") - .replace(/\s+/g, " ") + return collapseWhitespaceOutsideQuotes(sql.replace(CREATE_VIEW_PREFIX, "")) .replace(/;\s*$/, "") - .trim() - .toLowerCase(); + .trim(); } /** True when two view definitions are equivalent after normalization. */ diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts index 289becd8f4..c91b7b51d6 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts @@ -162,7 +162,7 @@ describe("D3 — a view whose definition differs says so", () => { expect(c !== undefined && "reason" in c ? c.reason : undefined).toEqual({ kind: "definition", compared: "text", - firstDifference: { expected: "select id, label as title from t", actual: "select id, label from t" }, + firstDifference: { expected: "SELECT id, label AS title FROM t", actual: "SELECT id, label FROM t" }, }); }); diff --git a/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts b/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts new file mode 100644 index 0000000000..633e098133 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { splitSqlStatements } from "../../src/sql/split-statements.js"; +import { prepareForRunnerTransaction } from "../../src/apply/runner-transaction-pass.js"; +import { viewSqlEquals } from "../../src/view-sql-compare.js"; +import { viewFingerprint } from "../../src/view-fingerprint.js"; +import { hasAutoincrementKeyword } from "../../src/introspect/sqlite-shared.js"; + +const TRIGGER = + "CREATE TRIGGER t_touch AFTER UPDATE ON t BEGIN UPDATE t SET updated_at = datetime('now') WHERE id = NEW.id; END"; + +describe("splitSqlStatements: engine rules", () => { + test("a SQLite trigger is one statement the engine accepts", () => { + const parts = splitSqlStatements(`CREATE TABLE t (id INTEGER, updated_at TEXT);\n${TRIGGER};\nSELECT 1;`); + expect(parts).toEqual(["CREATE TABLE t (id INTEGER, updated_at TEXT)", TRIGGER, "SELECT 1"]); + const db = new Database(":memory:"); + for (const p of parts) db.exec(p); + }); + + test("CASE ... END and several body statements stay inside the trigger", () => { + const trig = + "CREATE TEMP TRIGGER x AFTER INSERT ON t BEGIN SELECT CASE WHEN 1 THEN 2 END; UPDATE t SET a = 1; END"; + expect(splitSqlStatements(`${trig}; SELECT 2;`)).toEqual([trig, "SELECT 2"]); + }); + + test("an identifier containing begin does not open a body", () => { + expect(splitSqlStatements("CREATE TABLE begin_x (a); SELECT 1;")).toEqual(["CREATE TABLE begin_x (a)", "SELECT 1"]); + }); + + test("E'\\' is not a backslash escape on SQLite, MySQL or D1", () => { + const sql = "INSERT INTO e VALUES (e'\\'); SELECT 1;"; + for (const dialect of ["sqlite", "mysql", "d1"] as const) { + expect(splitSqlStatements(sql, { dialect })).toEqual(["INSERT INTO e VALUES (e'\\')", "SELECT 1"]); + } + // On Postgres the same text is one unterminated E-literal. + expect(splitSqlStatements(sql, { dialect: "postgres" })).toHaveLength(1); + }); + + test("Postgres E'' literal honours backslash escapes", () => { + const sql = "INSERT INTO t VALUES (E'it\\'s; fine');\nSELECT 1;"; + expect(splitSqlStatements(sql)).toEqual(["INSERT INTO t VALUES (E'it\\'s; fine')", "SELECT 1"]); + }); + + test("a plain literal ending in e is not an E string", () => { + expect(splitSqlStatements("SELECT 'e'; SELECT 'a\\'; SELECT 2;")).toEqual(["SELECT 'e'", "SELECT 'a\\'", "SELECT 2"]); + }); + + test("a bare END is transaction control", () => { + const r = prepareForRunnerTransaction(`END;\n${TRIGGER};`); + expect(r.statements).toEqual([TRIGGER]); + }); +}); + +describe("view comparators are literal-exact", () => { + test("SQLite comparator: literal case and literal whitespace differ", () => { + expect(viewSqlEquals("SELECT 1 WHERE s = 'Active'", "SELECT 1 WHERE s = 'active'")).toBe(false); + expect(viewSqlEquals("SELECT 1 WHERE s = 'a b'", "SELECT 1 WHERE s = 'a b'")).toBe(false); + expect(viewSqlEquals("CREATE VIEW v AS SELECT 1\n WHERE s = 'a b';", "SELECT 1 WHERE s = 'a b'")).toBe(true); + }); + test("fingerprint: literal whitespace differs, outside whitespace does not", () => { + expect(viewFingerprint("SELECT 'a b'")).not.toBe(viewFingerprint("SELECT 'a b'")); + expect(viewFingerprint("SELECT 1")).toBe(viewFingerprint("SELECT 1")); + }); +}); + +describe("AUTOINCREMENT detection", () => { + test("keyword only", () => { + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT)")).toBe(true); + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY) -- AUTOINCREMENT")).toBe(false); + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY, n TEXT DEFAULT 'AUTOINCREMENT')")).toBe(false); + expect(hasAutoincrementKeyword('CREATE TABLE t (id INTEGER PRIMARY KEY, "AUTOINCREMENT" TEXT)')).toBe(false); + }); +}); diff --git a/server/typescript/packages/render/src/escapers.ts b/server/typescript/packages/render/src/escapers.ts index ced15faf81..645ba6419a 100644 --- a/server/typescript/packages/render/src/escapers.ts +++ b/server/typescript/packages/render/src/escapers.ts @@ -24,7 +24,11 @@ const xml = (s: string): string => .replace(/'/g, "'"); // OWASP CSV/Excel formula-injection guard: neutralize a leading active char. -const injectionGuard = (s: string): string => (/^[=+\-@\t\r]/.test(s) ? "'" + s : s); +// A strict numeric literal (ASCII digits, no trailing newline: `$` without the m flag) is +// exempt, so a negative number stays a number instead of becoming text. +const STRICT_NUMERIC = /^-?[0-9]+(\.[0-9]+)?$/; +const injectionGuard = (s: string): string => + /^[=+\-@\t\r]/.test(s) && !STRICT_NUMERIC.test(s) ? "'" + s : s; const csv = (s: string): string => { const v = injectionGuard(s); diff --git a/server/typescript/packages/render/src/extract/json-forgiving-reader.ts b/server/typescript/packages/render/src/extract/json-forgiving-reader.ts index afd456dd2a..743467f0bd 100644 --- a/server/typescript/packages/render/src/extract/json-forgiving-reader.ts +++ b/server/typescript/packages/render/src/extract/json-forgiving-reader.ts @@ -127,19 +127,46 @@ class Reader { private readString(quoteChar: string): string { this.i++; // opening quote let sb = ""; - let esc = false; while (this.i < this.s.length) { const c = this.s.charAt(this.i++); - if (esc) { - sb += unescape(c); - esc = false; - } else if (c === "\\") esc = true; - else if (c === quoteChar) return sb; + if (c === "\\") { + sb += this.readEscape(); + } else if (c === quoteChar) return sb; else sb += c; } return sb; // unterminated string → return what we have } + /** The escape after a consumed backslash: the full JSON grammar (\\uXXXX, with surrogate pairs + * carried through as two code units). An unknown escape keeps its character; a truncated + * \\u keeps its text, so a cut-off reply still yields what it has. */ + private readEscape(): string { + if (this.i >= this.s.length) return ""; + const c = this.s.charAt(this.i++); + switch (c) { + case "n": + return "\n"; + case "t": + return "\t"; + case "r": + return "\r"; + case "b": + return "\b"; + case "f": + return "\f"; + case "u": { + const hex = this.s.substring(this.i, this.i + 4); + if (/^[0-9a-fA-F]{4}$/.test(hex)) { + this.i += 4; + return String.fromCharCode(Number.parseInt(hex, 16)); + } + return "u"; + } + default: + return c; // \" \\ \/ and any unknown escape + } + } + private readBareScalar(): string | null | typeof NULL_LITERAL { const start = this.i; while ( @@ -187,19 +214,6 @@ export function commentEnd(s: string, i: number): number { return close < 0 ? s.length : close + 2; } -function unescape(c: string): string { - switch (c) { - case "n": - return "\n"; - case "t": - return "\t"; - case "r": - return "\r"; - default: - return c; - } -} - function isPlainObject(o: unknown): boolean { return typeof o === "object" && o !== null && !Array.isArray(o); } diff --git a/server/typescript/packages/render/src/extract/locate.ts b/server/typescript/packages/render/src/extract/locate.ts index 85dfd8eda2..e2fe2f8a82 100644 --- a/server/typescript/packages/render/src/extract/locate.ts +++ b/server/typescript/packages/render/src/extract/locate.ts @@ -3,15 +3,11 @@ // declared field); locateJson (first-closed-else-first-open) is its fallback. Mirrors Java Locate. import { commentEnd } from "./json-forgiving-reader.js"; +import { matchFrom, maskXmlNonMarkup, quote } from "./xml-forgiving-reader.js"; /** The characters a JSON comment may follow: whitespace or a structural separator. */ const COMMENT_MAY_FOLLOW = /[\s,{[]/; -/** Escape regex metacharacters (equivalent of Java Pattern.quote). */ -function quote(s: string): string { - return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - /** First balanced {...}; if none closes, first '{' to end; null if no '{'. */ export function locateJson(text: string | null | undefined): string | null { if (text == null) return null; @@ -84,7 +80,12 @@ function scanBalanced(s: string, open: number): number { return -1; } -/** Span of ...; if close absent, opener to end; null if no opener. */ +/** + * Span of ...; if close absent, opener to end; null if no opener. Comments and + * CDATA are not searched, and when another opener of the same root sits between the first one + * and its close (the tag mentioned in prose, then the real element) the span starts at the + * LAST such opener: a root does not nest in itself. + */ export function locateXml( text: string | null | undefined, rootName: string | null | undefined, @@ -92,22 +93,21 @@ export function locateXml( ): string | null { if (text == null || rootName == null) return null; const flags = caseInsensitive ? "i" : ""; - const open = new RegExp(`<${quote(rootName)}(\\s[^>]*)?>`, flags).exec(text); + const view = maskXmlNonMarkup(text); + const openSrc = `<${quote(rootName)}(\\s[^>]*)?>`; + const open = new RegExp(openSrc, flags).exec(view); if (open == null) return null; - const start = open.index; const openEnd = open.index + open[0].length; - const closeRe = new RegExp(``, flags); - closeRe.lastIndex = openEnd; // Search for the close tag at-or-after the opener's end (Java close.find(open.end())). - const close = matchFrom(closeRe, text, openEnd); - if (close != null) return text.substring(start, close.index + close[0].length); - return text.substring(start); -} - -/** Find the first regex match at index >= `from` (emulates Java Matcher.find(int)). */ -function matchFrom(re: RegExp, text: string, from: number): RegExpExecArray | null { - const g = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g"); - g.lastIndex = from; - return g.exec(text); + const close = matchFrom(``, flags, view, openEnd); + if (close == null) return text.substring(open.index); + let start = open.index; + for (let at = openEnd; ; ) { + const next = matchFrom(openSrc, flags, view, at); + if (next == null || next.index >= close.index) break; + start = next.index; + at = next.index + next[0].length; + } + return text.substring(start, close.index + close[0].length); } diff --git a/server/typescript/packages/render/src/extract/strip.ts b/server/typescript/packages/render/src/extract/strip.ts index fbb2b9060d..bd6d361a25 100644 --- a/server/typescript/packages/render/src/extract/strip.ts +++ b/server/typescript/packages/render/src/extract/strip.ts @@ -1,18 +1,72 @@ // Stage 1: remove markdown code-fence markers. Prose around the payload is left for Locate. // Mirrors Java Strip. -// Captures the body inside a fenced block; optional language tag (json/xml/etc) is dropped. -const FENCE = /```[a-zA-Z0-9_-]*\s*\r?\n([\s\S]*?)\r?\n?```/; +// An opening fence: a run of 3+ backticks, an optional language tag (json/xml/etc, dropped), a line end. +const OPEN = /(`{3,})[a-zA-Z0-9_-]*\s*\r?\n/g; + +interface Fence { + /** Index of the opening backticks. */ + start: number; + /** Index just past the closing backticks. */ + end: number; + body: string; +} + +/** + * Every fenced block, in order. A fence closes at a line that starts with a backtick run at least + * as long as the opening one and carries nothing else, so backticks inside the payload (a JSON + * string holding `make` or a nested code sample) do not truncate it. A reply that closes on the + * payload's own line (`{...}` + three backticks) is accepted when that run ends the text. + */ +function fences(raw: string): Fence[] { + const out: Fence[] = []; + OPEN.lastIndex = 0; + for (let m = OPEN.exec(raw); m != null; m = OPEN.exec(raw)) { + const bodyStart = m.index + m[0].length; + const close = findClose(raw, bodyStart, m[1]?.length ?? 3); + if (close == null) continue; + out.push({ start: m.index, end: close.end, body: raw.substring(bodyStart, close.bodyEnd) }); + OPEN.lastIndex = close.end; + } + return out; +} + +function findClose(raw: string, from: number, width: number): { bodyEnd: number; end: number } | null { + for (let ls = from; ls <= raw.length; ) { + let i = ls; + while (i < raw.length && (raw.charAt(i) === " " || raw.charAt(i) === "\t") && i - ls < 4) i++; + let j = i; + while (j < raw.length && raw.charAt(j) === "`") j++; + if (j - i >= width) { + let k = j; + while (k < raw.length && (raw.charAt(k) === " " || raw.charAt(k) === "\t")) k++; + if (k >= raw.length || raw.charAt(k) === "\n" || raw.charAt(k) === "\r") { + return { bodyEnd: bodyEnd(raw, from, ls), end: j }; + } + } + const nl = raw.indexOf("\n", ls); + if (nl < 0) break; + ls = nl + 1; + } + // No closing line: accept a final backtick run that ends the text on the payload's own line. + const tail = raw.trimEnd(); + let k = tail.length; + while (k > from && tail.charAt(k - 1) === "`") k--; + if (tail.length - k >= width && k > from) return { bodyEnd: k, end: tail.length }; + return null; +} + +function bodyEnd(raw: string, from: number, lineStart: number): number { + let e = lineStart; + if (e > from && raw.charAt(e - 1) === "\n") e--; + if (e > from && raw.charAt(e - 1) === "\r") e--; + return e; +} export function strip(raw: string | null | undefined): string { if (raw == null) return ""; - const m = FENCE.exec(raw); - if (m && m.index >= 0) { - const before = raw.substring(0, m.index); - const body = m[1] ?? ""; - const after = raw.substring(m.index + m[0].length); - return (before + body + after).trim(); - } + const f = fences(raw)[0]; + if (f != null) return (raw.substring(0, f.start) + f.body + raw.substring(f.end)).trim(); return raw.trim(); } @@ -20,8 +74,5 @@ export function strip(raw: string | null | undefined): string { * because a model told to fence its answer puts the answer there. */ export function fencedBodies(raw: string | null | undefined): string[] { if (raw == null) return []; - const all = new RegExp(FENCE.source, "g"); - const out: string[] = []; - for (let m = all.exec(raw); m != null; m = all.exec(raw)) out.push(m[1] ?? ""); - return out; + return fences(raw).map((f) => f.body); } diff --git a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts index b7ff9ffacc..df9dc6ebae 100644 --- a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts +++ b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts @@ -17,12 +17,13 @@ * collides with a real attribute or child-element name. */ export const TEXT_KEY = "#text"; -function quote(s: string): string { +/** Escape regex metacharacters (equivalent of Java Pattern.quote). */ +export function quote(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } /** Find first regex match at index >= `from` (emulates Java Matcher.find(int)). */ -function matchFrom(source: string, flags: string, text: string, from: number): RegExpExecArray | null { +export function matchFrom(source: string, flags: string, text: string, from: number): RegExpExecArray | null { const g = new RegExp(source, flags.includes("g") ? flags : flags + "g"); g.lastIndex = from; return g.exec(text); @@ -37,14 +38,92 @@ const CLOSE_TAG_SRC = ""; // one attribute: name = "double" | 'single' | bareword. const ATTR_SRC = "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"; -export function readXml(span: string | null | undefined, caseInsensitive: boolean): Record { +/** A character an identifier can end with: `<` right after one is a type argument, not a tag. */ +const IDENT_CHAR = /^[A-Za-z0-9_]$/; + +const COMMENT_OPEN = ""; +const CDATA_OPEN = ""; +/** The five predefined XML entities plus numeric character references. */ +const ENTITY_RE = /&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);/g; +const NAMED_ENTITIES: Readonly> = { lt: "<", gt: ">", amp: "&", quot: '"', apos: "'" }; + +type XmlRegionKind = "comment" | "cdata"; + +/** + * Rewrite every XML comment and CDATA section in `text` through `replace`, scanning left to + * right so a comment marker inside CDATA (or the reverse) is never mistaken for markup. An + * unclosed comment is left as text; an unclosed CDATA section runs to the end. + */ +function rewriteRegions(text: string, replace: (kind: XmlRegionKind, region: string) => string): string { + let out = ""; + let pos = 0; + let from = 0; + for (;;) { + const c = text.indexOf(COMMENT_OPEN, from); + const d = text.indexOf(CDATA_OPEN, from); + const isComment = c >= 0 && (d < 0 || c < d); + const start = isComment ? c : d; + if (start < 0) break; + let end: number; + if (isComment) { + const e = text.indexOf(COMMENT_CLOSE, start + COMMENT_OPEN.length); + if (e < 0) { + from = start + COMMENT_OPEN.length; + continue; + } + end = e + COMMENT_CLOSE.length; + } else { + const e = text.indexOf(CDATA_CLOSE, start + CDATA_OPEN.length); + end = e < 0 ? text.length : e + CDATA_CLOSE.length; + } + out += text.substring(pos, start) + replace(isComment ? "comment" : "cdata", text.substring(start, end)); + pos = end; + from = end; + } + return out + text.substring(pos); +} + +/** `text` with comments and CDATA sections blanked to spaces (same length, so indices hold): + * the view the tag locators search, so markup inside them is never located. */ +export function maskXmlNonMarkup(text: string): string { + return rewriteRegions(text, (_kind, region) => " ".repeat(region.length)); +} + +/** Drop comments; turn each CDATA section into its text with `& < >` entity-escaped, so the + * tag scanner cannot split it and the entity decode restores it exactly. */ +function prepareXml(text: string): string { + return rewriteRegions(text, (kind, region) => { + if (kind === "comment") return ""; + const body = region.endsWith(CDATA_CLOSE) + ? region.substring(CDATA_OPEN.length, region.length - CDATA_CLOSE.length) + : region.substring(CDATA_OPEN.length); + return body.replace(/&/g, "&").replace(//g, ">"); + }); +} + +/** Decode predefined entities and numeric references in one pass (so `&lt;` is `<`). + * An unknown or out-of-range reference stays as written. */ +function decodeEntities(s: string): string { + if (!s.includes("&")) return s; + return s.replace(ENTITY_RE, (whole, body: string) => { + if (!body.startsWith("#")) return NAMED_ENTITIES[body] ?? whole; + const cp = body[1] === "x" ? parseInt(body.substring(2), 16) : parseInt(body.substring(1), 10); + const surrogate = cp >= 0xd800 && cp <= 0xdfff; + return cp >= 1 && cp <= 0x10ffff && !surrogate ? String.fromCodePoint(cp) : whole; + }); +} + +export function readXml(rawSpan: string | null | undefined, caseInsensitive: boolean): Record { const out: Record = {}; - if (span == null || span.trim().length === 0) return out; + if (rawSpan == null || rawSpan.trim().length === 0) return out; + const span = prepareXml(rawSpan); const gt = span.indexOf(">"); if (gt < 0) return out; const rootEnd = span.lastIndexOf(" { const out: Record = {}; if (text == null || text.trim().length === 0) return out; - parseChildren(text, caseInsensitive, out, null); + parseChildren(prepareXml(text), caseInsensitive, out, null, false); return out; } /** * Parse `inner`'s elements into `out`. When `loose` is non-null, the text BETWEEN those * elements (an element's own text in mixed content) is appended to it, one segment per gap. + * `literalUnclosed` is set for the body of an element that DID close: an open tag in there that + * directly follows an identifier character and has no close tag anywhere after it + * (`List and ...`) is prose, not an element. One after `>`, whitespace or the start of the + * body is a child whose close the model forgot. */ function parseChildren( inner: string, ci: boolean, out: Record, loose: string[] | null, + literalUnclosed: boolean, ): void { const flags = ci ? "i" : ""; let pos = 0; + let scan = 0; for (;;) { - const m = matchFrom(OPEN_TAG_SRC, flags, inner, pos); + const m = matchFrom(OPEN_TAG_SRC, flags, inner, scan); if (m == null) break; + const afterOpen = m.index + m[0].length; + if ( + literalUnclosed && + IDENT_CHAR.test(inner[m.index - 1] ?? "") && + !(m[2] ?? "").trimEnd().endsWith("/") && + matchFrom(CLOSE_TAG_SRC, flags, inner, afterOpen) == null + ) { + scan = afterOpen; + continue; + } loose?.push(inner.substring(pos, m.index)); const tag = m[1] ?? ""; const key = ci ? tag.toLowerCase() : tag; @@ -90,6 +185,7 @@ function parseChildren( if (selfClosing) { accumulate(out, key, Object.keys(attrs).length === 0 ? "" : attrs); pos = m.index + m[0].length; + scan = pos; continue; } @@ -137,8 +233,9 @@ function parseChildren( } const content = inner.substring(contentStart, contentEnd); - accumulate(out, key, combine(attrs, content, ci)); + accumulate(out, key, combine(attrs, content, ci, close != null)); pos = next; + scan = next; } loose?.push(inner.substring(pos)); } @@ -168,11 +265,11 @@ function findStrayClose(inner: string, from: number, to: number, ci: boolean): [ * keeps BOTH: the children, and the element's own text under TEXT_KEY, so a scalar or * `@xmlText` consumer still reads the prose around a child element. */ -function combine(attrs: Record, content: string, ci: boolean): unknown { +function combine(attrs: Record, content: string, ci: boolean, closed: boolean): unknown { if (content.includes("<")) { const nested: Record = {}; const loose: string[] = []; - parseChildren(content, ci, nested, loose); + parseChildren(content, ci, nested, loose, closed); if (Object.keys(nested).length > 0) { // attributes first; a child element wins a name collision const merged: Record = { ...attrs, ...nested }; @@ -191,14 +288,14 @@ function mixedText(segments: readonly string[], ci: boolean): string { const parts: string[] = []; for (const seg of segments) { closeRe.lastIndex = 0; - const t = seg.replace(closeRe, "").trim(); + const t = decodeEntities(seg.replace(closeRe, "").trim()); if (t.length > 0) parts.push(t); } return parts.join(" "); } function textValue(attrs: Record, content: string): unknown { - const text = content.trim(); + const text = decodeEntities(content.trim()); if (Object.keys(attrs).length === 0) return text; return { ...attrs, [TEXT_KEY]: text }; } @@ -212,7 +309,7 @@ function parseAttrs(rawAttrs: string, ci: boolean): Record { const rawName = a[1]; if (rawName === undefined) continue; // group 1 is mandatory in a match; guards strict TS const name = ci ? rawName.toLowerCase() : rawName; - const val = a[2] ?? a[3] ?? a[4] ?? ""; + const val = decodeEntities(a[2] ?? a[3] ?? a[4] ?? ""); if (!Object.prototype.hasOwnProperty.call(attrs, name)) attrs[name] = val; } return attrs; diff --git a/server/typescript/packages/render/src/render.ts b/server/typescript/packages/render/src/render.ts index f6ada2390b..86209fb919 100644 --- a/server/typescript/packages/render/src/render.ts +++ b/server/typescript/packages/render/src/render.ts @@ -1,7 +1,14 @@ import Mustache from "mustache"; import type { Provider } from "./provider.js"; import { ESCAPERS, type RenderFormat } from "./escapers.js"; -import { verify, ERR_REQUIRED_SLOT_UNUSED, type PayloadField } from "./verify.js"; +import { + verify, + parseTemplate, + partialTokens, + delimiterConflict, + ERR_REQUIRED_SLOT_UNUSED, + type PayloadField, +} from "./verify.js"; import { withDerivedAccessors } from "./payload-accessors.js"; import { TEMPLATE_ENGINE_JINJA, @@ -14,15 +21,23 @@ import { import { renderJinja } from "./jinja/render.js"; import { MAX_DEPTH } from "./verify-shared.js"; -const PARTIAL = /\{\{>\s*([^}\s]+)\s*\}\}/g; - // Resolve `{{> group/source }}` partials by inlining their text, recursively, // BEFORE Mustache parses — giving us deterministic whitespace and true, // path-based cycle/depth detection (no reliance on a per-lib partial loader). // Inlined text still renders in the surrounding context (e.g. once per loop // item), exactly like a native partial, because Mustache renders the result. -function expand(text: string, provider: Provider, path: readonly string[]): string { - return text.replace(PARTIAL, (_match, ref: string) => { +// Partial tags are located by Mustache's own parser, so a `{{> x}}` written inside +// a comment is not a partial, and a `{{=<% %>=}}` delimiter change is honoured. +function expand(text: string, provider: Provider, path: readonly string[], inlinedAs?: string): string { + const tokens = parseTemplate(text); + if (delimiterConflict(tokens, inlinedAs) !== undefined) { + throw new Error("a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + } + const partials = partialTokens(tokens); + if (partials.length === 0) return text; + let out = ""; + let pos = 0; + for (const [ref, start, end] of partials) { if (path.includes(ref)) throw new Error(`partial cycle: ${[...path, ref].join(" -> ")}`); if (path.length >= MAX_DEPTH) throw new Error(`partial depth exceeded ${MAX_DEPTH}: ${ref}`); const t = provider.resolve(ref); @@ -30,8 +45,10 @@ function expand(text: string, provider: Provider, path: readonly string[]): stri if (partialEngineMismatch(provider, ref, TEMPLATE_ENGINE_MUSTACHE)) { throw new Error(partialEngineMessage(ref, TEMPLATE_ENGINE_MUSTACHE)); } - return expand(t, provider, [...path, ref]); - }); + out += text.slice(pos, start) + expand(t, provider, [...path, ref], ref); + pos = end; + } + return out + text.slice(pos); } export interface RenderOptions { diff --git a/server/typescript/packages/render/src/verify.ts b/server/typescript/packages/render/src/verify.ts index 08e64d1896..a868a0edae 100644 --- a/server/typescript/packages/render/src/verify.ts +++ b/server/typescript/packages/render/src/verify.ts @@ -17,6 +17,7 @@ import { ERR_VAR_NOT_ON_PAYLOAD, ERR_PARTIAL_UNRESOLVED, ERR_TEMPLATE_ENGINE_UNSUPPORTED, + ERR_TEMPLATE_SYNTAX, MAX_DEPTH, slotAndTagFindings, type PayloadField, @@ -119,6 +120,77 @@ function parse(text: string): Token[] { return Mustache.parse(text) as unknown as Token[]; } +/** Parse `text`; a Mustache syntax error becomes an ERR_TEMPLATE_SYNTAX finding, anything else rethrows. */ +function parseOrSyntaxError(text: string): Token[] | VerifyError { + try { + return parse(text); + } catch (e) { + const section = syntaxSection(e); + if (section === undefined) throw e; + return { code: ERR_TEMPLATE_SYNTAX, path: section }; + } +} + +const TOKEN_PARTIAL = ">"; +const TOKEN_COMMENT = "!"; +const TOKEN_DELIMITERS = "="; + +/** Every token in a parse, depth-first, section bodies included. */ +function flatTokens(tokens: readonly Token[]): Token[] { + const out: Token[] = []; + for (const tok of tokens) { + out.push(tok); + if (Array.isArray(tok[4])) out.push(...flatTokens(tok[4] as Token[])); + } + return out; +} + +/** Every partial tag in a parse, in text order, as `[ref, start, end]` (nested sections included). */ +export function partialTokens(tokens: readonly Token[]): [string, number, number][] { + return flatTokens(tokens) + .filter((tok) => tok[0] === TOKEN_PARTIAL) + .map((tok) => [tok[1] as string, tok[2] as number, tok[3] as number]); +} + +/** Whether the parse contains a `{{=<% %>=}}` delimiter change. */ +function hasDelimiterChange(tokens: readonly Token[]): boolean { + return flatTokens(tokens).some((tok) => tok[0] === TOKEN_DELIMITERS); +} + +/** + * Where a text that changes delimiters cannot render, or undefined when it can: inlined text is + * parsed under the surrounding delimiters. A partial body (`inlinedAs` is its ref) that changes + * them is reported at that ref; a text that changes them and inlines a partial is reported at its + * first partial. + */ +export function delimiterConflict(tokens: readonly Token[], inlinedAs: string | undefined): string | undefined { + if (!hasDelimiterChange(tokens)) return undefined; + return inlinedAs ?? partialTokens(tokens)[0]?.[0]; +} + +function commentSpans(tokens: readonly Token[]): [number, number][] { + return flatTokens(tokens) + .filter((tok) => tok[0] === TOKEN_COMMENT) + .map((tok) => [tok[2] as number, tok[3] as number]); +} + +/** `text` with its comments cut out: what a tag-presence check may scan. */ +function withoutComments(text: string, tokens: readonly Token[]): string { + let out = ""; + let pos = 0; + for (const [start, end] of commentSpans(tokens)) { + out += text.slice(pos, start); + pos = end; + } + return out + text.slice(pos); +} + +/** The section a Mustache parse error is about (`Unopened section "x" at 4`), else undefined. */ +function syntaxSection(e: unknown): string | undefined { + const m = /^(?:Unopened|Unclosed) section "(.*)" at \d+$/.exec(e instanceof Error ? e.message : ""); + return m?.[1]; +} + /** * Parse a template into Mustache tokens (`[type, value, start, end, subTokens?]`), * the SAME parse verify walks. Exported so the docs annotator tokenizes through @@ -220,8 +292,15 @@ export function verify( errors.push({ code: ERR_TEMPLATE_ENGINE_UNSUPPORTED, path: value }); break; } - staticTexts.push(text); - walk(parse(text), stack, [...seen, value], positional); + const parsed = parseOrSyntaxError(text); + if (!Array.isArray(parsed)) { + errors.push(parsed); + break; + } + const conflict = delimiterConflict(parsed, value); + if (conflict !== undefined) errors.push({ code: ERR_TEMPLATE_SYNTAX, path: conflict }); + staticTexts.push(withoutComments(text, parsed)); + walk(parsed, stack, [...seen, value], positional); break; } default: @@ -230,7 +309,12 @@ export function verify( } } - walk(parse(templateText), [root], []); + const bodyTokens = parseOrSyntaxError(templateText); + if (!Array.isArray(bodyTokens)) return [bodyTokens]; + const conflict = delimiterConflict(bodyTokens, undefined); + if (conflict !== undefined) errors.push({ code: ERR_TEMPLATE_SYNTAX, path: conflict }); + staticTexts[0] = withoutComments(templateText, bodyTokens); + walk(bodyTokens, [root], []); errors.push(...slotAndTagFindings(referencedAtRoot, staticTexts, opts)); return errors; diff --git a/server/typescript/packages/render/test/escapers-numeric.test.ts b/server/typescript/packages/render/test/escapers-numeric.test.ts new file mode 100644 index 0000000000..8def33a060 --- /dev/null +++ b/server/typescript/packages/render/test/escapers-numeric.test.ts @@ -0,0 +1,14 @@ +import { describe, test, expect } from "bun:test"; +import { ESCAPERS } from "../src/escapers.js"; + +describe("csv injection guard numeric exemption", () => { + test("strict numeric literals are not prefixed", () => { + expect(ESCAPERS.csv("-5")).toBe("-5"); + expect(ESCAPERS.csv("-5.25")).toBe("-5.25"); + }); + test("everything else keeps the guard", () => { + for (const v of ["=1+1", "-cmd", "@x", "+5", "-5.", "-.5", "-5\n", "-5e3", "-٣", "-5 "]) { + expect(ESCAPERS.csv(v).replace(/^"/, "").startsWith("'")).toBe(true); + } + }); +}); diff --git a/server/typescript/packages/render/test/extract/extract-conformance.test.ts b/server/typescript/packages/render/test/extract/extract-conformance.test.ts index 35cbf70c29..5492ee3955 100644 --- a/server/typescript/packages/render/test/extract/extract-conformance.test.ts +++ b/server/typescript/packages/render/test/extract/extract-conformance.test.ts @@ -203,7 +203,7 @@ describe("extract-conformance corpus", () => { .filter((n) => existsSync(join(corpus, n, "schema.json"))) .sort(); - expect(cases.length).toBe(48); + expect(cases.length).toBe(58); for (const caseName of cases) { test(caseName, () => { diff --git a/server/typescript/packages/render/test/extract/strip.test.ts b/server/typescript/packages/render/test/extract/strip.test.ts index cc7b1a0eb1..da25563d58 100644 --- a/server/typescript/packages/render/test/extract/strip.test.ts +++ b/server/typescript/packages/render/test/extract/strip.test.ts @@ -28,3 +28,23 @@ describe("strip", () => { expect(strip(undefined)).toBe(""); }); }); + +describe("strip: fence closing", () => { + test("backticks inside a JSON string do not close the fence", () => { + const out = strip('```json\n{"summary": "run ```make``` first", "score": 7}\n```'); + expect(out).toBe('{"summary": "run ```make``` first", "score": 7}'); + }); + + test("a longer opening fence is closed only by an equal-or-longer run", () => { + const out = strip('````md\nbefore\n```\ninner\n```\nafter\n````\ntail'); + expect(out).toBe("before\n```\ninner\n```\nafter\ntail"); + }); + + test("a closing fence on the payload's own line is accepted at the end of the text", () => { + expect(strip('```json\n{"a":1}```')).toBe('{"a":1}'); + }); + + test("an unclosed fence is not a fence", () => { + expect(strip('```json\n{"a":1}')).toBe('```json\n{"a":1}'); + }); +}); diff --git a/server/typescript/packages/render/test/render.test.ts b/server/typescript/packages/render/test/render.test.ts index 75b5ca9e27..f29fb633ba 100644 --- a/server/typescript/packages/render/test/render.test.ts +++ b/server/typescript/packages/render/test/render.test.ts @@ -30,6 +30,20 @@ describe("render engine", () => { expect(render({ ref: "g/main", payload: { items: [{ n: 1 }, { n: 2 }] }, provider: p, format: "text" })).toBe("<1><2>"); }); + test("a delimiter change in a partial body is refused, not leaked into the parent", () => { + const p = P({ "g/main": "{{> g/leaf }} {{name}}", "g/leaf": "{{=<% %>=}}hi" }); + expect(() => render({ ref: "g/main", payload: { name: "Ada" }, provider: p })).toThrow(/change delimiters/); + }); + + test("a delimiter change in a root template resolved by ref, with no partials, renders", () => { + const p = P({ "g/main": "{{=<% %>=}}Hi <% name %>." }); + expect(render({ ref: "g/main", payload: { name: "Ada" }, provider: p })).toBe("Hi Ada."); + }); + + test("a delimiter change in a template without partials renders", () => { + expect(render({ template: "{{=<% %>=}}Hi <% name %>.", payload: { name: "Ada" }, provider: P({}) })).toBe("Hi Ada."); + }); + test("detects partial cycles", () => { const p = P({ "g/a": "{{> g/b }}", "g/b": "{{> g/a }}" }); expect(() => render({ ref: "g/a", payload: {}, provider: p })).toThrow(/cycle/i); diff --git a/server/typescript/packages/render/test/verify-conformance.test.ts b/server/typescript/packages/render/test/verify-conformance.test.ts index d4e351f181..dd1f5ac2dc 100644 --- a/server/typescript/packages/render/test/verify-conformance.test.ts +++ b/server/typescript/packages/render/test/verify-conformance.test.ts @@ -8,6 +8,7 @@ import { ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, type PayloadField, type VerifyError, type VerifyOptions, @@ -35,6 +36,7 @@ const VERIFY_CODES = new Set([ ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, ]); interface FixtureOptions { diff --git a/server/typescript/packages/sdk/package.json b/server/typescript/packages/sdk/package.json index 32fee616f4..b1ddf1a429 100644 --- a/server/typescript/packages/sdk/package.json +++ b/server/typescript/packages/sdk/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "@metaobjectsdev/metadata": "workspace:*", + "yaml": "^2.9.0", "zod": "^4.6.6" }, "devDependencies": { diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index dd263d17f2..c33e5397be 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -231,10 +231,14 @@ export const ConfigSchema = z.object({ .optional(), /** * C#-scoped settings. `outDir` is where `dotnet meta gen` writes and a bare `dotnet meta - * verify` diffs, relative to the directory holding `.metaobjects/`. Only the C# port reads - * it; it is declared here so the shared file validates and this reader does not warn on it. + * verify` diffs, relative to the directory holding `.metaobjects/`; `namespace` is the C# + * namespace generated types that no package binding places land in. Only the C# port reads + * them; they are declared here so the shared file validates and this reader does not warn on them. */ - csharp: z.strictObject({ outDir: z.string().min(1).optional() }).optional(), + csharp: z.strictObject({ + outDir: z.string().min(1).optional(), + namespace: z.string().min(1).optional(), + }).optional(), }).strict(); export type Config = z.infer; diff --git a/server/typescript/packages/sdk/src/workspace.ts b/server/typescript/packages/sdk/src/workspace.ts index b9c620ca11..53e0d12d4c 100644 --- a/server/typescript/packages/sdk/src/workspace.ts +++ b/server/typescript/packages/sdk/src/workspace.ts @@ -14,8 +14,9 @@ // - Lockfile-style resolution + version range matching // - Better error messages on circular extends, missing packages -import { readFile, readdir, stat } from "node:fs/promises"; -import { dirname, join, resolve, basename } from "node:path"; +import { readFile, readdir, realpath, stat } from "node:fs/promises"; +import { dirname, join, resolve, basename, relative, sep } from "node:path"; +import { parse as parseYaml } from "yaml"; import { type PackageManifest, PackageManifestSchema, @@ -72,35 +73,22 @@ async function readPnpmWorkspaceFile(path: string): Promise typeof p === "string"); } async function readPackageJsonWorkspaces(path: string): Promise { @@ -143,64 +131,136 @@ async function findWorkspaceConfig(start: string): Promise { - // Strip trailing `/` for normalization - const pattern = glob.replace(/\/$/, ""); +/** Directories pnpm's workspace walk never enters, whatever the pattern says. */ +const WORKSPACE_IGNORED_DIRS = new Set(["node_modules", "bower_components", "test", "tests"]); - // No wildcard: literal directory - if (!pattern.includes("*")) { - const literal = join(root, pattern); - try { - const s = await stat(literal); - return s.isDirectory() ? [literal] : []; - } catch { - return []; +/** Split a pattern into path segments, dropping `./` prefixes and trailing slashes. */ +function patternSegments(glob: string): string[] { + return glob.split("/").filter((seg) => seg !== "" && seg !== "."); +} + +/** One path segment against one pattern segment (`*` = any run, `?` = one char; no `/`). */ +function matchGlobSegment(pattern: string): (value: string) => boolean { + if (pattern === "*") return () => true; + const regex = new RegExp( + "^" + + pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, "[^/]*") + .replace(/\?/g, "[^/]") + + "$", + ); + return (value) => regex.test(value); +} + +/** Does the whole path match the whole pattern? `**` spans zero or more segments. */ +function matchSegments(pattern: readonly string[], path: readonly string[]): boolean { + const [head, ...rest] = pattern; + if (head === undefined) return path.length === 0; + if (head === "**") { + for (let skip = 0; skip <= path.length; skip++) { + if (matchSegments(rest, path.slice(skip))) return true; } + return false; + } + const [first, ...remaining] = path; + return first !== undefined && matchGlobSegment(head)(first) && matchSegments(rest, remaining); +} + +async function subdirectories(dir: string): Promise { + try { + const entries = await readdir(dir, { withFileTypes: true }); + return entries + .filter((e) => !WORKSPACE_IGNORED_DIRS.has(e.name)) + .map((e) => e.name); + } catch { + return []; } +} + +async function isDirectory(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} - // Split into prefix segments (literal) + last segment with wildcard. - // Supports the common cases: "packages/*", "apps/*", "packages/foo-*". - // Doesn't support deep `**` globs in this prototype. - const segments = pattern.split("/"); - const lastSegment = segments.pop()!; - const prefixDir = join(root, ...segments); +/** Whether a wildcard segment may match `name`: as in pnpm (micromatch without `dot`), a + * wildcard matches a dot-directory only when the segment itself starts with a dot. */ +function wildcardMay(name: string, segment: string): boolean { + return !name.startsWith(".") || segment.startsWith("."); +} - let entries: string[]; +/** + * `**` expansion: zero or more directory levels, the base included. Symlinks are followed + * but each real directory is visited once (cycle guard); dot-directories are not descended + * into. + */ +async function expandGlobstar( + dir: string, + rest: readonly string[], + visited: Set, +): Promise { + let real: string; try { - entries = await readdir(prefixDir); + real = await realpath(dir); } catch { return []; } + if (visited.has(real)) return []; + visited.add(real); + const found = await expandSegments(dir, rest); + for (const name of await subdirectories(dir)) { + if (!wildcardMay(name, "**")) continue; + const child = join(dir, name); + if (!(await isDirectory(child))) continue; + found.push(...(await expandGlobstar(child, rest, visited))); + } + return found; +} - const matcher = matchGlobSegment(lastSegment); - const result: string[] = []; - for (const entry of entries) { - if (!matcher(entry)) continue; - const full = join(prefixDir, entry); - try { - const s = await stat(full); - if (s.isDirectory()) result.push(full); - } catch { - // skip unreadable - } +/** Directories under `dir` matching the remaining pattern segments. */ +async function expandSegments(dir: string, segments: readonly string[]): Promise { + const [head, ...rest] = segments; + if (head === undefined) return [dir]; + if (head === "**") { + return expandGlobstar(dir, rest, new Set()); + } + if (!/[*?]/.test(head)) { + if (WORKSPACE_IGNORED_DIRS.has(head)) return []; + const child = join(dir, head); + return (await isDirectory(child)) ? expandSegments(child, rest) : []; } - return result; + const matcher = matchGlobSegment(head); + const found: string[] = []; + for (const name of await subdirectories(dir)) { + const child = join(dir, name); + if (wildcardMay(name, head) && matcher(name) && (await isDirectory(child))) found.push(...(await expandSegments(child, rest))); + } + return found; } -function matchGlobSegment(pattern: string): (value: string) => boolean { - if (pattern === "*") return () => true; - const regex = new RegExp( - "^" + - pattern - .replace(/[.+^${}()|[\]\\]/g, "\\$&") - .replace(/\*/g, "[^/]*") + - "$", - ); - return (value) => regex.test(value); +/** + * Expand workspace globs relative to `root`: every positive pattern contributes its + * matching directories, then every `!`-negated pattern removes the ones it matches. + */ +async function expandGlobs(root: string, globs: readonly string[]): Promise { + const negations = globs + .filter((g) => g.startsWith("!")) + .map((g) => patternSegments(g.slice(1))); + const found = new Set(); + for (const glob of globs) { + if (glob.startsWith("!")) continue; + for (const dir of await expandSegments(root, patternSegments(glob))) found.add(dir); + } + return [...found].filter((dir) => { + const rel = relative(root, dir).split(sep); + return !negations.some((neg) => matchSegments(neg, rel)); + }); } // --------------------------------------------------------------------------- @@ -223,10 +283,7 @@ export async function discoverWorkspace(cwd: string): Promise { test("dependencies: absent means []", () => { expect(ConfigSchema.parse({ schema_version: 1 }).dependencies).toEqual([]); }); }); +describe("csharp.namespace (the C# port's flat namespace)", () => { + test("is a known, validated key beside outDir", () => { + const ok = ConfigSchema.safeParse({ schema_version: 1, csharp: { outDir: "Generated", namespace: "Acme.Blog" } }); + expect(ok.success).toBe(true); + expect(ConfigSchema.safeParse({ schema_version: 1, csharp: { namespace: "" } }).success).toBe(false); + expect(ConfigSchema.safeParse({ schema_version: 1, csharp: { Namespace: "Acme" } }).success).toBe(false); + }); +}); + describe("csharp.outDir (the C# port's generated-code directory)", () => { test("is a known, validated key", () => { const ok = ConfigSchema.safeParse({ schema_version: 1, csharp: { outDir: "src/Generated" } }); diff --git a/server/typescript/packages/sdk/test/workspace.test.ts b/server/typescript/packages/sdk/test/workspace.test.ts index 0984652ca7..ee5bd1fad2 100644 --- a/server/typescript/packages/sdk/test/workspace.test.ts +++ b/server/typescript/packages/sdk/test/workspace.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -250,3 +250,133 @@ describe("resolveExtendsOrder", () => { } }); }); + +describe("extractPnpmPackages — real YAML (audit finding 17)", () => { + test("zero-indent block list", () => { + expect(extractPnpmPackages("packages:\n- 'packages/*'\n- apps/*\n")).toEqual(["packages/*", "apps/*"]); + }); + test("flow list", () => { + expect(extractPnpmPackages("packages: ['packages/*', \"apps/*\"]\n")).toEqual(["packages/*", "apps/*"]); + }); + test("a # inside a quoted pattern is not a comment", () => { + expect(extractPnpmPackages("packages:\n - 'a#b/*' # trailing\n")).toEqual(["a#b/*"]); + }); + test("negations are kept as negations for the matcher", () => { + expect(extractPnpmPackages("packages:\n - 'apps/*'\n - '!apps/legacy'\n")).toEqual(["apps/*", "!apps/legacy"]); + }); + test("invalid YAML yields no globs rather than throwing", () => { + expect(extractPnpmPackages("packages: [unclosed\n")).toEqual([]); + }); +}); + +describe("discoverWorkspace — glob semantics (audit finding 17)", () => { + async function names(root: string, from: string): Promise { + const ws = await discoverWorkspace(join(root, from)); + return (ws?.packages ?? []).map((p) => p.manifest.name).sort(); + } + + test("a negation excludes a directory", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-neg-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'apps/*'\n - '!apps/legacy'\n"); + scaffoldPackage(root, "apps/web", makeManifest("@acme/web", { metaobjectsPackage: "acme::web" })); + scaffoldPackage(root, "apps/legacy", makeManifest("@acme/legacy", { metaobjectsPackage: "acme::legacy" })); + expect(await names(root, "apps/web")).toEqual(["@acme/web"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("** matches nested directories at any depth", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-glob-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'libs/**'\n"); + scaffoldPackage(root, "libs/a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "libs/group/b", makeManifest("@acme/b", { metaobjectsPackage: "acme::b" })); + expect(await names(root, "libs/a")).toEqual(["@acme/a", "@acme/b"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a single * stays one level deep", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-star-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'libs/*'\n"); + scaffoldPackage(root, "libs/a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "libs/group/b", makeManifest("@acme/b", { metaobjectsPackage: "acme::b" })); + expect(await names(root, "libs/a")).toEqual(["@acme/a"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a ** in the middle and a # in a directory name", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-mid-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'a#b/*'\n - 'x/**/pkg'\n"); + scaffoldPackage(root, "a#b/one", makeManifest("@acme/one", { metaobjectsPackage: "acme::one" })); + scaffoldPackage(root, "x/y/z/pkg", makeManifest("@acme/deep", { metaobjectsPackage: "acme::deep" })); + scaffoldPackage(root, "x/pkg", makeManifest("@acme/zero", { metaobjectsPackage: "acme::zero" })); + expect(await names(root, "a#b/one")).toEqual(["@acme/deep", "@acme/one", "@acme/zero"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("dist and build are ordinary directories; bower_components is never searched", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-build-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'tools/**'\n - 'libs/*'\n"); + scaffoldPackage(root, "tools/build", makeManifest("@acme/build", { metaobjectsPackage: "acme::build" })); + scaffoldPackage(root, "tools/dist/x", makeManifest("@acme/x", { metaobjectsPackage: "acme::x" })); + scaffoldPackage(root, "tools/bower_components/b", makeManifest("@acme/b", { metaobjectsPackage: "acme::b" })); + scaffoldPackage(root, "libs/.hidden", makeManifest("@acme/h", { metaobjectsPackage: "acme::h" })); + expect(await names(root, "tools/build")).toEqual(["@acme/build", "@acme/x"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("test and tests directories are never entered, as in pnpm", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-tests-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n - 'tools/**'\n - 'libs/tests'\n"); + scaffoldPackage(root, "packages/real", makeManifest("@acme/real", { metaobjectsPackage: "acme::real" })); + scaffoldPackage(root, "packages/test", makeManifest("@acme/t", { metaobjectsPackage: "acme::t" })); + scaffoldPackage(root, "tools/x", makeManifest("@acme/x", { metaobjectsPackage: "acme::x" })); + scaffoldPackage(root, "tools/x/tests/fixture", makeManifest("@acme/f", { metaobjectsPackage: "acme::f" })); + scaffoldPackage(root, "tools/test/y", makeManifest("@acme/y", { metaobjectsPackage: "acme::y" })); + scaffoldPackage(root, "libs/tests", makeManifest("@acme/lt", { metaobjectsPackage: "acme::lt" })); + expect(await names(root, "packages/real")).toEqual(["@acme/real", "@acme/x"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("node_modules is never searched", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-nm-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - '**'\n"); + scaffoldPackage(root, "a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "node_modules/dep", makeManifest("@acme/dep", { metaobjectsPackage: "acme::dep" })); + expect(await names(root, "a")).toEqual(["@acme/a"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("trailing ** matches the base dir, skips dot-dirs, and survives a symlink cycle", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-cycle-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'libs/**'\n"); + scaffoldPackage(root, "libs", makeManifest("@acme/base", { metaobjectsPackage: "acme::base" })); + scaffoldPackage(root, "libs/a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "libs/.hidden/h", makeManifest("@acme/h", { metaobjectsPackage: "acme::h" })); + symlinkSync(join(root, "libs"), join(root, "libs/a/loop"), "dir"); + expect(await names(root, "libs/a")).toEqual(["@acme/a", "@acme/base"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +});