From 7499e6295d12c0d870c67038d7e37a79502b5483 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 28 Oct 2026 21:49:03 -0400 Subject: [PATCH 01/23] fix: replace regex and string-rewrite matching with authoritative parsers across ports (audit findings 5, 7-15, 17, 18) Currency parsing via Intl, SQL splitter, view comparators, Python YAML 1.2, Mustache tag scanning, forgiving XML reader, JSONC-aware upgrade rewriter, leak-scan renames, Java authored-name serialization, ERR_COLUMN_NAME_COLLISION, pnpm workspace parsing and assorted smaller instances. --- .githooks/leak-patterns.sh | 15 ++ .githooks/leak-scan.sh | 16 +- .githooks/pre-commit | 16 +- CHANGELOG.md | 45 +++++ bun.lock | 1 + .../web/packages/runtime-web/src/currency.ts | 133 ++++++++++--- .../runtime-web/test/currency.test.ts | 45 +++++ docs/CONFORMANCE.md | 2 +- fixtures/conformance/ERROR-CODES.json | 3 +- .../expected.json | 44 +++++ .../input/meta.shop.json | 44 +++++ .../providers.json | 1 + .../column-name-no-collision/expected.json | 60 ++++++ .../input/meta.demo.json | 24 +++ .../column-name-no-collision/providers.json | 1 + .../expected-errors.json | 15 ++ .../input/meta.demo.json | 17 ++ .../expected-errors.json | 15 ++ .../input/meta.demo.json | 52 ++++++ .../providers.json | 1 + .../expected-errors.json | 15 ++ .../input/meta.demo.json | 44 +++++ .../providers.json | 1 + .../xml-cdata-with-tags/expected.json | 1 + .../xml-cdata-with-tags/input.txt | 1 + .../xml-cdata-with-tags/schema.json | 3 + .../xml-comment-ignored/expected.json | 1 + .../xml-comment-ignored/input.txt | 1 + .../xml-comment-ignored/schema.json | 3 + .../xml-entities-decoded/expected.json | 1 + .../xml-entities-decoded/input.txt | 1 + .../xml-entities-decoded/schema.json | 3 + .../xml-literal-angle-in-text/expected.json | 1 + .../xml-literal-angle-in-text/input.txt | 1 + .../xml-literal-angle-in-text/schema.json | 3 + .../xml-root-in-comment/expected.json | 1 + .../xml-root-in-comment/input.txt | 2 + .../xml-root-in-comment/schema.json | 3 + .../xml-root-mentioned-in-prose/expected.json | 1 + .../xml-root-mentioned-in-prose/input.txt | 2 + .../xml-root-mentioned-in-prose/schema.json | 3 + .../render-comment-with-partial/expected.txt | 1 + .../render-comment-with-partial/meta.json | 1 + .../render-comment-with-partial/payload.json | 1 + .../template.mustache | 1 + .../expected-drift.json | 1 + .../options.json | 1 + .../payload.json | 1 + .../template.mustache | 1 + .../expected-drift.json | 1 + .../verify-mismatched-close/payload.json | 1 + .../verify-mismatched-close/template.mustache | 1 + .../expected-drift.json | 1 + .../options.json | 1 + .../payload.json | 1 + .../template.mustache | 1 + .../expected-drift.json | 1 + .../verify-set-delimiter-clean/payload.json | 1 + .../template.mustache | 1 + .../expected-drift.json | 1 + .../verify-set-delimiter-drift/payload.json | 1 + .../template.mustache | 1 + .../verify-unopened-close/expected-drift.json | 1 + .../verify-unopened-close/payload.json | 1 + .../verify-unopened-close/template.mustache | 1 + scripts/leak-scan.test.ts | 88 +++++++++ .../ConstraintErrorsTests.cs | 50 +++++ .../EntityGeneratorEscapeTests.cs | 38 ++++ .../NamesExtendsChainTests.cs | 2 +- .../NamesGeneratorTests.cs | 6 +- .../Generators/EntityGenerator.cs | 28 ++- .../Runtime/ConstraintErrors.cs | 77 +++++++- .../Extract/ExtractConformanceTests.cs | 2 +- .../VerifyConformanceTests.cs | 1 + .../MetaObjects.Render/Extract/Locate.cs | 14 +- .../Extract/XmlForgivingReader.cs | 135 ++++++++++++-- .../csharp/MetaObjects.Render/MustacheTags.cs | 115 ++++++++++++ server/csharp/MetaObjects.Render/Renderer.cs | 31 ++-- server/csharp/MetaObjects.Render/Verify.cs | 91 +++++---- server/csharp/MetaObjects/Errors.cs | 3 + .../MetaObjects/Loader/MetaDataLoader.cs | 3 + .../MetaObjects/Loader/ValidationPasses.cs | 62 ++++++- .../csharp/MetaObjects/Meta/PhysicalNaming.cs | 14 ++ .../javacode/ExtractorCodeGenerator.java | 2 +- .../javacode/JavaObjectCodeGenerator.java | 2 +- .../generator/util/GeneratorUtil.java | 32 ++-- .../generator/util/JavaIdentifiers.java | 25 +++ .../generator/util/GeneratorUtilTest.java | 9 + .../util/JavaIdentifiersEscapeTest.java | 20 ++ .../kotlin/KotlinRelationsGenerator.kt | 44 +++-- .../kotlin/KotlinNamesGeneratorTest.kt | 4 +- .../kotlin/KotlinRelationsGeneratorTest.kt | 31 ++++ .../spring/SpringGeneratorSupport.java | 2 +- .../spring/SpringOutputParserGenerator.java | 14 +- .../spring/SpringNamesGeneratorTest.java | 4 +- .../metaobjects/mojo/MetaDataEjectMojo.java | 53 +++++- .../metaobjects/mojo/EjectPomReadingTest.java | 41 ++++ .../main/java/com/metaobjects/ErrorCode.java | 3 + .../io/json/CanonicalJsonSerializer.java | 41 ++-- .../metaobjects/library/LibrarySources.java | 84 +++++---- .../metaobjects/loader/ValidationPhase.java | 50 ++++- .../com/metaobjects/source/MetaSource.java | 24 +-- .../util/ErrorMessageConstants.java | 3 + .../io/json/CanonicalJsonSerializerTest.java | 17 ++ .../library/LibrarySourcesManifestTest.java | 33 ++++ .../com/metaobjects/render/MustacheTags.java | 127 +++++++++++++ .../java/com/metaobjects/render/Renderer.java | 34 ++-- .../java/com/metaobjects/render/Verify.java | 100 +++++----- .../metaobjects/render/extract/Locate.java | 21 ++- .../render/extract/XmlForgivingReader.java | 141 ++++++++++++-- .../render/VerifyConformanceTest.java | 3 +- .../snapshots/render-comment-with-partial.txt | 1 + server/python/src/metaobjects/cli.py | 4 +- .../codegen/generators/entity_model.py | 10 + .../src/metaobjects/codegen/project_config.py | 3 +- server/python/src/metaobjects/errors.py | 3 + .../loader/validate_discriminator.py | 12 +- .../metaobjects/loader/validation_passes.py | 41 ++++ .../src/metaobjects/render/extract/locate.py | 25 ++- .../render/extract/xml_forgiving_reader.py | 129 +++++++++++-- .../src/metaobjects/render/mustache_tags.py | 121 ++++++++++++ .../python/src/metaobjects/render/renderer.py | 35 ++-- .../python/src/metaobjects/render/verify.py | 108 +++++------ .../metaobjects/runtime/validator_runner.py | 6 +- .../metaobjects/source/yaml_core_schema.py | 122 ++++++++++++ .../src/metaobjects/source/yaml_positions.py | 21 ++- .../test_fr019_shared_provided_conformance.py | 9 + .../tests/codegen/test_names_extends_chain.py | 2 +- .../conformance/test_extract_conformance.py | 2 +- .../tests/render/test_verify_conformance.py | 2 + .../tests/runtime/test_validator_runner.py | 6 + .../tests/source/test_yaml_core_schema.py | 56 ++++++ .../tests/unit/test_relative_set_newlines.py | 11 ++ .../packages/cli/src/lib/dependency-sync.ts | 13 +- .../cli/test/deps-sync-packages.test.ts | 9 +- .../src/core/object/validate-discriminator.ts | 9 +- .../packages/metadata/src/errors.ts | 3 + .../metadata/src/loader/meta-data-loader.ts | 5 +- .../src/loader/sources/directory-source.ts | 11 +- .../metadata/src/loader/validation-passes.ts | 42 +++++ .../metadata/src/vocabulary-rewrite.ts | 43 ++++- .../metadata/test/loader/sources.test.ts | 14 ++ .../metadata/test/vocabulary-rewrite.test.ts | 35 ++++ .../src/apply/runner-transaction-pass.ts | 2 +- .../packages/migrate-ts/src/introspect/d1.ts | 4 +- .../src/introspect/sqlite-shared.ts | 11 ++ .../migrate-ts/src/introspect/sqlite.ts | 4 +- .../migrate-ts/src/sql/split-statements.ts | 73 ++++++++ .../migrate-ts/src/view-fingerprint.ts | 7 +- .../migrate-ts/src/view-sql-compare.ts | 57 ++++-- .../test/unit/diff-view-change-reason.test.ts | 2 +- .../test/unit/regex-followups-sql.test.ts | 64 +++++++ .../packages/render/src/extract/locate.ts | 28 ++- .../src/extract/xml-forgiving-reader.ts | 112 +++++++++-- .../typescript/packages/render/src/render.ts | 30 ++- .../typescript/packages/render/src/verify.ts | 72 ++++++- .../test/extract/extract-conformance.test.ts | 2 +- .../render/test/verify-conformance.test.ts | 2 + server/typescript/packages/sdk/package.json | 1 + .../typescript/packages/sdk/src/workspace.ts | 175 ++++++++++-------- .../packages/sdk/test/workspace.test.ts | 86 +++++++++ 161 files changed, 3426 insertions(+), 609 deletions(-) create mode 100644 .githooks/leak-patterns.sh create mode 100644 fixtures/conformance/authored-name-looks-autogenerated/expected.json create mode 100644 fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json create mode 100644 fixtures/conformance/authored-name-looks-autogenerated/providers.json create mode 100644 fixtures/conformance/column-name-no-collision/expected.json create mode 100644 fixtures/conformance/column-name-no-collision/input/meta.demo.json create mode 100644 fixtures/conformance/column-name-no-collision/providers.json create mode 100644 fixtures/conformance/error-column-name-collision/expected-errors.json create mode 100644 fixtures/conformance/error-column-name-collision/input/meta.demo.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json create mode 100644 fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json create mode 100644 fixtures/extract-conformance/xml-cdata-with-tags/expected.json create mode 100644 fixtures/extract-conformance/xml-cdata-with-tags/input.txt create mode 100644 fixtures/extract-conformance/xml-cdata-with-tags/schema.json create mode 100644 fixtures/extract-conformance/xml-comment-ignored/expected.json create mode 100644 fixtures/extract-conformance/xml-comment-ignored/input.txt create mode 100644 fixtures/extract-conformance/xml-comment-ignored/schema.json create mode 100644 fixtures/extract-conformance/xml-entities-decoded/expected.json create mode 100644 fixtures/extract-conformance/xml-entities-decoded/input.txt create mode 100644 fixtures/extract-conformance/xml-entities-decoded/schema.json create mode 100644 fixtures/extract-conformance/xml-literal-angle-in-text/expected.json create mode 100644 fixtures/extract-conformance/xml-literal-angle-in-text/input.txt create mode 100644 fixtures/extract-conformance/xml-literal-angle-in-text/schema.json create mode 100644 fixtures/extract-conformance/xml-root-in-comment/expected.json create mode 100644 fixtures/extract-conformance/xml-root-in-comment/input.txt create mode 100644 fixtures/extract-conformance/xml-root-in-comment/schema.json create mode 100644 fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json create mode 100644 fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt create mode 100644 fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json create mode 100644 fixtures/render-conformance/render-comment-with-partial/expected.txt create mode 100644 fixtures/render-conformance/render-comment-with-partial/meta.json create mode 100644 fixtures/render-conformance/render-comment-with-partial/payload.json create mode 100644 fixtures/render-conformance/render-comment-with-partial/template.mustache create mode 100644 fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-comment-with-partial-clean/options.json create mode 100644 fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json create mode 100644 fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache create mode 100644 fixtures/verify-conformance/verify-mismatched-close/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-mismatched-close/payload.json create mode 100644 fixtures/verify-conformance/verify-mismatched-close/template.mustache create mode 100644 fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-required-tag-in-comment/options.json create mode 100644 fixtures/verify-conformance/verify-required-tag-in-comment/payload.json create mode 100644 fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache create mode 100644 fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-clean/payload.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache create mode 100644 fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-drift/payload.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache create mode 100644 fixtures/verify-conformance/verify-unopened-close/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-unopened-close/payload.json create mode 100644 fixtures/verify-conformance/verify-unopened-close/template.mustache create mode 100644 scripts/leak-scan.test.ts create mode 100644 server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs create mode 100644 server/csharp/MetaObjects.Render/MustacheTags.cs create mode 100644 server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java create mode 100644 server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java create mode 100644 server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java create mode 100644 server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java create mode 100644 server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt create mode 100644 server/python/src/metaobjects/render/mustache_tags.py create mode 100644 server/python/src/metaobjects/source/yaml_core_schema.py create mode 100644 server/python/tests/source/test_yaml_core_schema.py create mode 100644 server/python/tests/unit/test_relative_set_newlines.py create mode 100644 server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts diff --git a/.githooks/leak-patterns.sh b/.githooks/leak-patterns.sh new file mode 100644 index 0000000000..cb03f7c8fe --- /dev/null +++ b/.githooks/leak-patterns.sh @@ -0,0 +1,15 @@ +# Shared leak-scan patterns, sourced by .githooks/pre-commit and .githooks/leak-scan.sh so the +# local hook and the hosted `leak-scan` status can never disagree about what a leak is. +# Names NO private project; private names come only from the untracked denylist. +# +# /Users/ is the macOS home shape. It was missing from leak-scan.sh until a rename-with-edits +# slipped past both (audit finding 12). + +# Generic, non-sensitive structural patterns (safe to commit): +LEAK_PATTERNS='/home/[A-Za-z0-9._-]+/|/Users/[A-Za-z0-9._-]+/|~/Development' + +# Legitimate matches to ignore (the published npm author email). +LEAK_ALLOW='doug@dougmealing\.com' + +# Diff filter: Added, Copied, Modified and Renamed (a `git mv` plus an edit is a rename). +LEAK_DIFF_FILTER='ACMR' diff --git a/.githooks/leak-scan.sh b/.githooks/leak-scan.sh index 255a5b8d11..a858e9bc13 100755 --- a/.githooks/leak-scan.sh +++ b/.githooks/leak-scan.sh @@ -15,13 +15,13 @@ # set -uo pipefail -# Generic, non-sensitive structural patterns (safe to commit — name no project): -PATTERNS='/home/[A-Za-z0-9._-]+/|~/Development' - -# Legitimate matches to ignore (the published npm author email). -ALLOW='doug@dougmealing\.com' - ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" +# Patterns, allow-list and diff filter are shared with .githooks/pre-commit. +# shellcheck source=leak-patterns.sh +. "$ROOT/.githooks/leak-patterns.sh" +PATTERNS="$LEAK_PATTERNS" +ALLOW="$LEAK_ALLOW" + DENY_FILE="$(git config --get hooks.denyListPath 2>/dev/null || true)" [ -z "$DENY_FILE" ] && DENY_FILE="$ROOT/.githooks/deny-list.local" if [ -f "$DENY_FILE" ]; then @@ -30,9 +30,9 @@ if [ -f "$DENY_FILE" ]; then fi if [ -n "${1:-}" ]; then - DIFF=$(git diff "$1...HEAD" -U0 --no-color --diff-filter=ACM 2>/dev/null) + DIFF=$(git diff "$1...HEAD" -U0 --no-color --diff-filter="$LEAK_DIFF_FILTER" 2>/dev/null) else - DIFF=$(git diff --cached --diff-filter=ACM -U0 --no-color 2>/dev/null) + DIFF=$(git diff --cached --diff-filter="$LEAK_DIFF_FILTER" -U0 --no-color 2>/dev/null) fi # Scan only newly-ADDED lines, attributed to their file; skip the hooks dir itself. diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 3fe220ff29..6eaae8e1c0 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -17,19 +17,17 @@ # set -uo pipefail -# Generic, non-sensitive structural patterns — safe to commit (name no project): -# /Users/ is the macOS home shape. It was missing until a plan doc authored on a Mac -# committed an absolute developer home path that this hook scanned and passed. -PATTERNS='/home/[A-Za-z0-9._-]+/|/Users/[A-Za-z0-9._-]+/|~/Development' - -# Legitimate matches to ignore (e.g. the published npm author email). -ALLOW='doug@dougmealing\.com' +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" +# Patterns, allow-list and diff filter are shared with .githooks/leak-scan.sh. +# shellcheck source=leak-patterns.sh +. "$ROOT/.githooks/leak-patterns.sh" +PATTERNS="$LEAK_PATTERNS" +ALLOW="$LEAK_ALLOW" # Merge in private-name patterns from a denylist. Source priority: # 1) git config hooks.denyListPath -> point at your private denylist file (kept in a private repo) # 2) .githooks/deny-list.local -> untracked local file or symlink # If neither resolves, only the generic structural patterns above are enforced. -ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" DENY_FILE="$(git config --get hooks.denyListPath 2>/dev/null || true)" [ -z "$DENY_FILE" ] && DENY_FILE="$ROOT/.githooks/deny-list.local" if [ -f "$DENY_FILE" ]; then @@ -38,7 +36,7 @@ if [ -f "$DENY_FILE" ]; then fi # Scan only newly-ADDED staged lines, attributed to their file; skip the hooks dir. -hits=$(git diff --cached --diff-filter=ACM -U0 --no-color 2>/dev/null | awk ' +hits=$(git diff --cached --diff-filter="$LEAK_DIFF_FILTER" -U0 --no-color 2>/dev/null | awk ' /^\+\+\+ / { f=$2; sub(/^b\//,"",f); next } /^\+/ && !/^\+\+\+/ { if (f !~ /^\.githooks\//) print f "\t" substr($0,2) } ' | grep -Ei "$PATTERNS" | grep -vEi "$ALLOW") diff --git a/CHANGELOG.md b/CHANGELOG.md index 7494cde3f7..209e894fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,11 @@ here.** is unchanged and the default output is byte-identical. The `@metaobjectsdev/docs-site` README lists the five files and how to serve them. Copying them into the output is not done for you. +- **`ERR_COLUMN_NAME_COLLISION`.** Two fields of one stored object that resolve to the same + physical column (`userId` and `userID` under snake_case, or two fields sharing `@column`) are a + load error in every port, naming both fields. A model that loaded before with a duplicate column + now fails at load. Fixtures `error-column-name-collision` and `column-name-no-collision`. + - **Jinja templates in Python and TypeScript, held to the same payload check (ADR-0061, closes #483).** A `template.prompt` or `template.output` body may be a Jinja file — `/.jinja`, `.j2` or `.jinja2` — instead of a `.mustache` one. Nothing in the @@ -347,6 +352,46 @@ here.** Separately, the rename rewrite no longer touches a cast type, a function name or the inside of an `E'..'` literal when a column of that word is renamed. +- **Regex and hand-rewrite audit follow-ups.** Places that decided a match by regex or string + rewriting now ask the authoritative source: + - `parseCurrency` reads the locale's separators and sign forms from `Intl.NumberFormat`, returns + `null` on junk, and reads `$-5.00`, `(5.00)` and `5.00-` as negative. A long fraction rounds + exactly (`1.005` is 101, not 100). + - The migration SQL splitter keeps a SQLite `CREATE TRIGGER … BEGIN … END` as one statement and + honours Postgres `E'…'` backslash escapes; a bare `END` is dropped as transaction control. + - SQLite/D1 view comparison is case-sensitive and whitespace-exact inside quoted literals (as is + the Postgres fingerprint), so literal-only view changes are detected. A Postgres view whose + body has a whitespace-bearing literal re-stamps once. SQLite/D1 `AUTOINCREMENT` detection + ignores comments, literals and quoted identifiers. + - Python loads YAML with YAML 1.2 core-schema scalars, matching the TypeScript loader: `off`, + `no`, `1:30` and `2024-01-02` stay strings; a duplicate mapping key is a load error. The + Python codegen drift gate compares bytes, so CRLF edits are no longer hidden; `@pattern` uses + ASCII `\d`/`\w`/`\s`; a shared enum whose values differ only by case fails generation. + - Template verify and render agree with the Mustache engine in C#, Java, Python and TypeScript: + a partial tag in a comment is not expanded, `{{=<% %>=}}` raises no false + `ERR_VAR_NOT_ON_PAYLOAD`, `@requiredTags` is not satisfied inside a comment, and an unbalanced + section is `ERR_TEMPLATE_SYNTAX`. Render rejects partials combined with a delimiter change, + and the Python renderer rejects `{{=`. + - The forgiving XML extract reader (all five ports) decodes entities, reads CDATA as text, + ignores comments, keeps literal `List` prose inside closed elements, and locates the real + root. Six new `extract-conformance` cases. + - `meta upgrade` ignores JSONC comments when scanning JSON metadata, so a brace or trailing + comment no longer skips a retirement silently. + - The hosted `leak-scan` checks renamed files and `/Users//` paths and shares its patterns + with the local pre-commit hook. + - The Java canonical serializer and `source.rdb` name resolution no longer treat an authored + name like `required7` or `rdb1` as auto-generated (new fixture + `authored-name-looks-autogenerated`). + - Workspace discovery parses `pnpm-workspace.yaml` as YAML and honours negations and `**`. + - Smaller instances: Java `` glob no longer throws on metacharacters; Java and C# + generated string literals escape newlines and control characters; `library.json` is read with + Gson; the Kotlin relations helper imports its primary-key type; `eject` reads `pom.xml` with + Maven's model reader; C# constraint-violation classification uses SQLSTATE / SQLite extended + codes before message text; numeric `@discriminatorValue` rejects non-ASCII digits and a + trailing newline and treats `"1"` and `"01"` as duplicates (new fixtures); `meta deps sync` + no longer warns that Python `1.0` differs from `1.0.0`; `DirectorySource` excludes containing + `/` match the relative path. + - **`@default: "now"` is the current time on Postgres, not the table's creation time.** The expected schema stored it as the literal `'now'`, and Postgres converts that literal to a timestamp once, when it creates the column's `DEFAULT`, so every row got the same value. It diff --git a/bun.lock b/bun.lock index 1fa3f2c880..0ea7786deb 100644 --- a/bun.lock +++ b/bun.lock @@ -422,6 +422,7 @@ "version": "1.1.4", "dependencies": { "@metaobjectsdev/metadata": "workspace:*", + "yaml": "^2.9.0", "zod": "^4.6.6", }, "devDependencies": { diff --git a/client/web/packages/runtime-web/src/currency.ts b/client/web/packages/runtime-web/src/currency.ts index 4260da58d3..b6b722f422 100644 --- a/client/web/packages/runtime-web/src/currency.ts +++ b/client/web/packages/runtime-web/src/currency.ts @@ -25,34 +25,123 @@ export function formatCurrency( return intlFor(currency, locale).format(cents / factor); } +/** Whitespace-like group separators (fr-FR uses U+202F) accept any ordinary space typed. */ +const SPACE_LIKE = /^[\s  ]+$/; +const SPACE_CLASS = "[\\s\\u00a0\\u202f]"; + +function escapeRe(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** What the locale writes: separators, digit grouping sizes, and the currency's spellings. */ +interface LocaleShape { + readonly group: string; + readonly decimal: string; + /** Size of the last integer group and of the ones before it (3/3, or 3/2 for en-IN). */ + readonly primary: number; + readonly secondary: number; + readonly tokens: readonly string[]; +} + +function localeShape(currency: string, locale: string): LocaleShape { + let group = ","; + let decimal = "."; + let primary = 3; + let secondary = 3; + const tokens = new Set(); + try { + const parts = new Intl.NumberFormat(locale, { style: "currency", currency, useGrouping: true }) + .formatToParts(123456789.12); + group = parts.find((p) => p.type === "group")?.value ?? group; + decimal = parts.find((p) => p.type === "decimal")?.value ?? decimal; + const groups = parts.filter((p) => p.type === "integer").map((p) => p.value.length); + primary = groups[groups.length - 1] ?? primary; + secondary = groups[groups.length - 2] ?? primary; + for (const display of ["symbol", "narrowSymbol", "code"] as const) { + const token = new Intl.NumberFormat(locale, { style: "currency", currency, currencyDisplay: display }) + .formatToParts(1) + .find((p) => p.type === "currency")?.value; + if (token) tokens.add(token); + } + } catch { + // unknown currency or locale: fall back to the en-US shape, with the code as the token + tokens.add(currency); + } + // Longest first so "US$" is peeled before "$". + return { group, decimal, primary, secondary, tokens: [...tokens].sort((x, y) => y.length - x.length) }; +} + +function groupClass(shape: LocaleShape): string { + return SPACE_LIKE.test(shape.group) ? SPACE_CLASS : escapeRe(shape.group); +} + +/** The unsigned number grammar for a locale: grouped or plain integer, optional fraction. */ +function numberGrammar(shape: LocaleShape): RegExp { + const g = groupClass(shape); + const d = escapeRe(shape.decimal); + const grouped = `\\d{1,${shape.secondary}}(?:${g}\\d{${shape.secondary}})*${g}\\d{${shape.primary}}`; + return new RegExp(`^(${grouped}|\\d+)?(?:${d}(\\d*))?$`); +} + +/** + * Parse text a person typed or pasted into integer minor units, or `null` when it is not a + * money amount in this locale. The locale's own separators, grouping and currency spellings + * come from `Intl.NumberFormat#formatToParts`; the accepted forms are a strict grammar, not + * "strip everything that is not a digit". One negative marker is allowed, in any of the + * forms `-5.00`, `$-5.00`, `-$5.00`, `5.00-` or `(5.00)`; two, or anything unrecognised + * (`1e3`, `12.50 (2 items)`, a separator the locale does not use), is `null`. + */ export function parseCurrency( text: string, currency = "USD", locale = "en-US", ): number | null { - if (typeof text !== "string" || text.trim() === "") return null; - - // Detect locale's decimal separator via Intl. - const sample = (0.1).toLocaleString(locale); - const decimalSep = sample.includes(",") ? "," : "."; + if (typeof text !== "string") return null; + const shape = localeShape(currency, locale); + let rest = text.replace(/[−‒-—]/g, "-").trim(); + if (rest === "") return null; - // Determine sign from leading minus. - const sign = text.trim().startsWith("-") ? -1 : 1; - - // Remove all non-numeric chars except the decimal separator. - const escapedSep = decimalSep === "." ? "\\." : decimalSep; - const digitsAndDecimal = text.replace(new RegExp(`[^0-9${escapedSep}]`, "g"), ""); - if (digitsAndDecimal === "" || digitsAndDecimal === decimalSep) return null; - - // Normalize to "." for parseFloat. - const normalized = - decimalSep === "." ? digitsAndDecimal : digitsAndDecimal.replace(decimalSep, "."); - - if ((normalized.match(/\./g) ?? []).length > 1) return null; + let markers = 0; // negative markers seen: sign or parentheses + let positive = false; + let tokenSeen = false; + for (let peeled = true; peeled; ) { + peeled = false; + rest = rest.trim(); + if (rest.startsWith("(") && rest.endsWith(")")) { + rest = rest.slice(1, -1); + markers++; + peeled = true; + } else if (rest.startsWith("-") || rest.startsWith("+")) { + if (rest[0] === "-") markers++; + else positive = true; + rest = rest.slice(1); + peeled = true; + } else if (rest.endsWith("-")) { + markers++; + rest = rest.slice(0, -1); + peeled = true; + } else if (!tokenSeen) { + const token = shape.tokens.find((t) => rest.startsWith(t) || rest.endsWith(t)); + if (token !== undefined) { + rest = rest.startsWith(token) ? rest.slice(token.length) : rest.slice(0, -token.length); + tokenSeen = true; + peeled = true; + } + } + } + if (markers > 1 || (markers === 1 && positive)) return null; - const parsed = Number.parseFloat(normalized); - if (!Number.isFinite(parsed)) return null; + const match = numberGrammar(shape).exec(rest.trim()); + if (match === null) return null; + const whole = (match[1] ?? "").replace(new RegExp(groupClass(shape), "g"), ""); + const fraction = match[2] ?? ""; + if (whole === "" && fraction === "") return null; - const factor = Math.pow(10, minorUnitsFor(currency, locale)); - return Math.round(sign * parsed * factor); + // Exact decimal-string arithmetic: no float multiply, half rounds away from zero. + const minor = minorUnitsFor(currency, locale); + const kept = fraction.slice(0, minor).padEnd(minor, "0"); + let units = Number(`${whole === "" ? "0" : whole}${kept}`); + if ((fraction[minor] ?? "0") >= "5") units += 1; + if (!Number.isSafeInteger(units)) return null; + return markers === 1 && units !== 0 ? -units : units; } diff --git a/client/web/packages/runtime-web/test/currency.test.ts b/client/web/packages/runtime-web/test/currency.test.ts index a6448ffdfb..9a2b3aaedc 100644 --- a/client/web/packages/runtime-web/test/currency.test.ts +++ b/client/web/packages/runtime-web/test/currency.test.ts @@ -74,3 +74,48 @@ describe("parseCurrency", () => { expect(parseCurrency("15.9.9.9")).toBeNull(); }); }); + +describe("parseCurrency — strict grammar and sign forms (audit finding 5)", () => { + test.each([ + ["$-5.00", -500], + ["-$5.00", -500], + ["(5.00)", -500], + ["($5.00)", -500], + ["5.00-", -500], + ["−5.00", -500], + ["+5.00", 500], + ["USD 1,234.50", 123450], + ["1.005", 101], + [".5", 50], + ["5.", 500], + ["0.00", 0], + ["-0.00", 0], + ])("%s → %p", (input, cents) => { + expect(parseCurrency(input)).toBe(cents); + }); + + test.each(["1e3", "12.50 (2 items)", "--5", "(-5.00)", "-5.00-", "$$5", "1,23.45", "1,2345", "5 apples", "-", "$", "()", "1.5.2"])( + "%s → null", + (input) => { + expect(parseCurrency(input)).toBeNull(); + }, + ); + + test("de-DE uses , as decimal and . as grouping; '1.5' is not an amount", () => { + expect(parseCurrency("1.234,56 €", "EUR", "de-DE")).toBe(123456); + expect(parseCurrency("-5,00 €", "EUR", "de-DE")).toBe(-500); + expect(parseCurrency("1.5", "EUR", "de-DE")).toBeNull(); + }); + + test("fr-FR accepts an ordinary space for its narrow no-break group separator", () => { + expect(parseCurrency("1 234,50 €", "EUR", "fr-FR")).toBe(123450); + }); + + test("en-IN groups 2,2,3", () => { + expect(parseCurrency("₹12,34,567.00", "INR", "en-IN")).toBe(123456700); + }); + + test("JPY has no minor units; a fraction rounds", () => { + expect(parseCurrency("¥1,000", "JPY")).toBe(1000); + }); +}); diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index c6d41380bb..743dfdc855 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -33,7 +33,7 @@ regenerate with `ls -d fixtures//*/ | wc -l` for directory-shaped corpor | [`fixtures/render-conformance/`](../fixtures/render-conformance/) | 16 | ✓ | ✓ | inherits via Java | ✓ | ✓ | | [`fixtures/jinja-render-conformance/`](../fixtures/jinja-render-conformance/) (ADR-0061 — Jinja bodies render the same in jinja2 and nunjucks) | 15 | ✓ (nunjucks) | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | refuses, via Java | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | ✓ (jinja2) | | [`fixtures/jinja-verify-conformance/`](../fixtures/jinja-verify-conformance/) (ADR-0061 — the payload check over a Jinja body) | 31 | ✓ | refuses Jinja | refuses, via Java | refuses Jinja | ✓ | -| [`fixtures/extract-conformance/`](../fixtures/extract-conformance/) | 48 | ✓ | ✓ | inherits the shared JVM engine | ✓ | ✓ | +| [`fixtures/extract-conformance/`](../fixtures/extract-conformance/) | 54 | ✓ | ✓ | inherits the shared JVM engine | ✓ | ✓ | | [`fixtures/output-prompt-conformance/`](../fixtures/output-prompt-conformance/) | 17 | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/persistence-conformance/`](../fixtures/persistence-conformance/) | 59 (53 query + 6 migration) | all 59; the 24 `report-*` and the `-15` decimal scenarios also on SQLite, MySQL and D1, `decimal-exact` also on MySQL (a scenario's `engines:` key is exact) | 53 query (migrations TS-only, ADR-0015); the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query (via Exposed); the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query; the `report-*` and `engines:` scenarios also on SQLite and MySQL | 53 query (pg8000); the `report-*` and `engines:` scenarios also on SQLite (`sqlite3`) and MySQL (PyMySQL) | | [`fixtures/api-contract-conformance/`](../fixtures/api-contract-conformance/) | 119 (31 core + 10 tph + 9 m2m + 5 jsonb + 2 write-through + 2 secret-fields + 12 projection + 37 report + 11 decimal) | ✓ (Fastify reference + generated lane); `report/` and `decimal/` also on SQLite, MySQL and D1 | ✓ (embedded HTTP + JDBC); `report/` and `decimal/` generated controllers over OMDB on Postgres, SQLite and MySQL | ✓ (embedded HTTP + Exposed); `report/` and `decimal/` also on SQLite and MySQL | ✓ (Kestrel + Npgsql); `report/` and `decimal/` also on SQLite and MySQL | ✓ (FastAPI + pg8000); `report/` and `decimal/` also on SQLite and MySQL | diff --git a/fixtures/conformance/ERROR-CODES.json b/fixtures/conformance/ERROR-CODES.json index f736a92d2e..f4d24d2e54 100644 --- a/fixtures/conformance/ERROR-CODES.json +++ b/fixtures/conformance/ERROR-CODES.json @@ -38,6 +38,7 @@ "ERR_EXTENDS_ORIGIN_MISMATCH": "FR-024 (ADR-0029 decision 7): a field declares both an entity-nested extends (shape lineage) and an origin.passthrough @from (data lineage) and they disagree — the resolved @from target is not the field's resolved extends target (nor anywhere on its extends chain). Host-agnostic (projections, entities, values). origin.aggregate is never judged (it computes something new); a top-level abstract extends target is never judged (shape-only reuse makes no lineage claim).", "ERR_PASSTHROUGH_TYPE_MISMATCH": "#185: a field carrying origin.passthrough @from: \"Entity.field\" declares a field. or array-ness that differs from its resolved source field — a passthrough forwards the value unchanged, so its type must be identical. Host-agnostic (projections, entities, values, stored-proc parameter refs). Nullability is NOT judged (a view over an outer join legitimately widens NOT NULL → nullable). Opt out of a deliberate type change with @convert: true on the origin.passthrough (an acknowledgement — it does not generate a cast); real type-converting projections are origin.expression's job (#159).", "ERR_DERIVED_FIELD_NO_READ_SOURCE": "FR-024 (spec §7): an object.entity field carrying an origin.* child is derived (read-only) and must be providable — the entity must declare at least one source with a read-only @kind (view/materializedView/storedProc/tableFunction, e.g. a @role replica view beside the table primary). Table-only or source-less entities with origin-bearing fields error on the field. Projections and object.value hosts are exempt. Until the Phase-E B4b cutover removes view-primary entities, a read-only-kind PRIMARY source also counts as providable (legacy spelling).", + "ERR_COLUMN_NAME_COLLISION": "Two fields of one stored object (a concrete object declaring a source) resolve to the same physical column name: snake_case folds `userId` and `userID` into `user_id`, or two fields share an explicit @column. Only RESOLVED physical names are compared (explicit @column honoured, inherited fields included, derived and source-less objects exempt). The message names both fields and the column.", "ERR_INVALID_TEMPLATE": "A template (prompt/output) declares a @payloadRef that does not resolve, or @requiredSlots that are not fields on its payload.", "ERR_INVALID_RELATIONSHIP": "FR-017: a M:N relationship's slim vocabulary is invalid — @through does not resolve to a junction declaring two identity.reference children (ADR-0042: @through resolves package-locally when bare, exactly when FQN — a bare cross-package @through no longer binds elsewhere), @sourceRefField does not match one of them, or a M:N-only attr (@through/@sourceRefField/@symmetric) is set on a non-M:N (1:N / @cardinality:one) relationship; OR a relationship's @objectRef does not resolve to any object in the loaded tree (a dangling target).", "ERR_INVALID_REFERENCE": "An identity.reference @references names an FK target object that does not resolve to any object in the loaded tree (a dangling cross-reference between metadata).", @@ -48,7 +49,7 @@ "ERR_REQUIRED_SLOT_UNUSED": "Build-time verify (warning): a template's declared @requiredSlots slot is never referenced by the template text.", "ERR_OUTPUT_TAG_MISSING": "Build-time verify: a template declares @requiredTags but its text (including provider-resolved partials) omits a required output tag's opening () form. path = the tag name.", "ERR_TEMPLATE_NOT_STATIC": "Build-time verify (ADR-0061, Jinja bodies, Python and TypeScript): a template reference the check cannot settle statically — a computed attribute or include name, a dynamic subscript, a call, a tag outside the supported set. Reported rather than passed; path = the expression or tag.", - "ERR_TEMPLATE_SYNTAX": "Build-time verify (ADR-0061, Jinja bodies, Python and TypeScript): the template body, or a template it includes or imports, does not parse. path = the parser's message, which is engine-specific.", + "ERR_TEMPLATE_SYNTAX": "Build-time verify: the template body, or a template it includes or imports, does not parse. Jinja bodies (ADR-0061, Python and TypeScript): path = the parser's message, which is engine-specific. Mustache bodies (every port): an unbalanced section (a close with no open, a close naming another section, a section never closed); path = the section's name.", "ERR_TEMPLATE_ENGINE_UNSUPPORTED": "Build-time verify and render (ADR-0061): a template body is written for an engine this port does not ship — a Jinja body (.jinja / .j2 / .jinja2) in the Java, Kotlin or C# port, which render Mustache only. path = the template ref.", "ERR_BAD_ATTR_FILTER": "An attr.filter references a field that is not addressable in that context or uses a disallowed op: a dataGrid @filter over a non-filterable field or an op disallowed for the field's subtype, or a projection row-scope @filter over a non-declared (dangling) or aggregate-derived field (#207). FR-044: a segment / measure.aggregate / object.report @filter names a field its entity (or @from) lacks or an op illegal for the field (S1); a relative-date value { \"now\": \"\" } appears outside the @filter of a segment, measure.aggregate or object.report (F1); or a relative-date value sits on a field that is not field.date/field.timestamp, under an op other than gt/gte/lt/lte, carries a value that is not an ISO-8601 duration, or has a startOf other than day, week or month, or a key besides now and startOf (F2).", "ERR_STORAGE_FLATTENED_ARRAY": "@storage \"flattened\" cannot be combined with isArray=true.", diff --git a/fixtures/conformance/authored-name-looks-autogenerated/expected.json b/fixtures/conformance/authored-name-looks-autogenerated/expected.json new file mode 100644 index 0000000000..4a058e20e3 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/expected.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "acme::shop", + "children": [ + { + "object.entity": { + "name": "Item", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "sku", + "children": [ + { + "validator.required": { + "name": "required7" + } + }, + { + "validator.length": { + "@max": 10 + } + } + ] + } + }, + { + "identity.primary": { + "name": "pk", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json b/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json new file mode 100644 index 0000000000..4a058e20e3 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/input/meta.shop.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "acme::shop", + "children": [ + { + "object.entity": { + "name": "Item", + "children": [ + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "sku", + "children": [ + { + "validator.required": { + "name": "required7" + } + }, + { + "validator.length": { + "@max": 10 + } + } + ] + } + }, + { + "identity.primary": { + "name": "pk", + "@fields": [ + "id" + ] + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/authored-name-looks-autogenerated/providers.json b/fixtures/conformance/authored-name-looks-autogenerated/providers.json new file mode 100644 index 0000000000..8ec88fc091 --- /dev/null +++ b/fixtures/conformance/authored-name-looks-autogenerated/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types"] diff --git a/fixtures/conformance/column-name-no-collision/expected.json b/fixtures/conformance/column-name-no-collision/expected.json new file mode 100644 index 0000000000..66921351cb --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/expected.json @@ -0,0 +1,60 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Account", + "children": [ + { + "source.rdb": { + "@kind": "table" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "field.string": { + "name": "userId" + } + }, + { + "field.string": { + "name": "userID", + "@column": "user_id_legacy" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": [ + "id" + ] + } + } + ] + } + }, + { + "object.value": { + "name": "Shape", + "children": [ + { + "field.string": { + "name": "userId" + } + }, + { + "field.string": { + "name": "userID" + } + } + ] + } + } + ] + } +} diff --git a/fixtures/conformance/column-name-no-collision/input/meta.demo.json b/fixtures/conformance/column-name-no-collision/input/meta.demo.json new file mode 100644 index 0000000000..ea776f41a2 --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/input/meta.demo.json @@ -0,0 +1,24 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { "object.entity": { + "name": "Account", + "children": [ + { "source.rdb": { "@kind": "table" } }, + { "field.long": { "name": "id" } }, + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID", "@column": "user_id_legacy" } }, + { "identity.primary": { "name": "id", "@fields": ["id"] } } + ] + }}, + { "object.value": { + "name": "Shape", + "children": [ + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID" } } + ] + }} + ] + } +} diff --git a/fixtures/conformance/column-name-no-collision/providers.json b/fixtures/conformance/column-name-no-collision/providers.json new file mode 100644 index 0000000000..aefa027b6a --- /dev/null +++ b/fixtures/conformance/column-name-no-collision/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types", "metaobjects-db"] diff --git a/fixtures/conformance/error-column-name-collision/expected-errors.json b/fixtures/conformance/error-column-name-collision/expected-errors.json new file mode 100644 index 0000000000..1ce5056dc8 --- /dev/null +++ b/fixtures/conformance/error-column-name-collision/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_COLUMN_NAME_COLLISION", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[0]['object.entity'].children[3]['field.string']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-column-name-collision/input/meta.demo.json b/fixtures/conformance/error-column-name-collision/input/meta.demo.json new file mode 100644 index 0000000000..e2ed439a7b --- /dev/null +++ b/fixtures/conformance/error-column-name-collision/input/meta.demo.json @@ -0,0 +1,17 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { "object.entity": { + "name": "Account", + "children": [ + { "source.rdb": { "@kind": "table" } }, + { "field.long": { "name": "id" } }, + { "field.string": { "name": "userId" } }, + { "field.string": { "name": "userID" } }, + { "identity.primary": { "name": "id", "@fields": ["id"] } } + ] + }} + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json new file mode 100644 index 0000000000..ec9d869b77 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_DISCRIMINATOR_VALUE_DUPLICATE", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[2]['object.entity']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json new file mode 100644 index 0000000000..4f3c5bf927 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/input/meta.demo.json @@ -0,0 +1,52 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Auth", + "@discriminator": "kind", + "children": [ + { + "source.rdb": { + "@table": "auths" + } + }, + { + "field.int": { + "name": "kind" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + }, + { + "object.entity": { + "name": "One", + "extends": "demo::Auth", + "@discriminatorValue": "1", + "children": [] + } + }, + { + "object.entity": { + "name": "AlsoOne", + "extends": "demo::Auth", + "@discriminatorValue": "01", + "children": [] + } + } + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json new file mode 100644 index 0000000000..f9bb435a37 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-equivalent/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types","metaobjects-db"] diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json new file mode 100644 index 0000000000..4b916f01a2 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/expected-errors.json @@ -0,0 +1,15 @@ +{ + "errors": [ + { + "code": "ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH", + "source": { + "format": "json", + "files": [ + "meta.demo.json" + ], + "jsonPath": "$['metadata.root'].children[1]['object.entity']" + } + } + ], + "warnings": [] +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json new file mode 100644 index 0000000000..2932beb01b --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/input/meta.demo.json @@ -0,0 +1,44 @@ +{ + "metadata.root": { + "package": "demo", + "children": [ + { + "object.entity": { + "name": "Auth", + "@discriminator": "kind", + "children": [ + { + "source.rdb": { + "@table": "auths" + } + }, + { + "field.int": { + "name": "kind" + } + }, + { + "field.long": { + "name": "id" + } + }, + { + "identity.primary": { + "name": "id", + "@fields": "id" + } + } + ] + } + }, + { + "object.entity": { + "name": "Five", + "extends": "demo::Auth", + "@discriminatorValue": "5\n", + "children": [] + } + } + ] + } +} diff --git a/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json new file mode 100644 index 0000000000..f9bb435a37 --- /dev/null +++ b/fixtures/conformance/error-tph-discriminator-numeric-trailing-newline/providers.json @@ -0,0 +1 @@ +["metaobjects-core-types","metaobjects-db"] diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/expected.json b/fixtures/extract-conformance/xml-cdata-with-tags/expected.json new file mode 100644 index 0000000000..07fc491b40 --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "x bold & ", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/input.txt b/fixtures/extract-conformance/xml-cdata-with-tags/input.txt new file mode 100644 index 0000000000..a282ddfbdd --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/input.txt @@ -0,0 +1 @@ +<![CDATA[x <b>bold</b> & <!-- not a comment -->]]>n \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-cdata-with-tags/schema.json b/fixtures/extract-conformance/xml-cdata-with-tags/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-cdata-with-tags/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-comment-ignored/expected.json b/fixtures/extract-conformance/xml-comment-ignored/expected.json new file mode 100644 index 0000000000..4ea7b4495e --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "final" } } diff --git a/fixtures/extract-conformance/xml-comment-ignored/input.txt b/fixtures/extract-conformance/xml-comment-ignored/input.txt new file mode 100644 index 0000000000..74217348ec --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/input.txt @@ -0,0 +1 @@ +realfinal \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-comment-ignored/schema.json b/fixtures/extract-conformance/xml-comment-ignored/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-comment-ignored/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-entities-decoded/expected.json b/fixtures/extract-conformance/xml-entities-decoded/expected.json new file mode 100644 index 0000000000..dff6155d70 --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "A & B AB", "note": "< stays" } } diff --git a/fixtures/extract-conformance/xml-entities-decoded/input.txt b/fixtures/extract-conformance/xml-entities-decoded/input.txt new file mode 100644 index 0000000000..f8e502488f --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/input.txt @@ -0,0 +1 @@ +A & B <ok> AB&lt; stays \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-entities-decoded/schema.json b/fixtures/extract-conformance/xml-entities-decoded/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-entities-decoded/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json b/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json new file mode 100644 index 0000000000..2acfc669fa --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "use List and Map here", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt b/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt new file mode 100644 index 0000000000..08e26f30e7 --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/input.txt @@ -0,0 +1 @@ +use List<int> and Map<string, int> heren \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json b/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-literal-angle-in-text/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-in-comment/expected.json b/fixtures/extract-conformance/xml-root-in-comment/expected.json new file mode 100644 index 0000000000..c907092dc6 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-root-in-comment/input.txt b/fixtures/extract-conformance/xml-root-in-comment/input.txt new file mode 100644 index 0000000000..136fd46f24 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/input.txt @@ -0,0 +1,2 @@ + +realn \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-in-comment/schema.json b/fixtures/extract-conformance/xml-root-in-comment/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-in-comment/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json b/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json new file mode 100644 index 0000000000..c907092dc6 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/expected.json @@ -0,0 +1 @@ +{ "empty": false, "states": { "title": "EXTRACTED", "note": "EXTRACTED" }, "data": { "title": "real", "note": "n" } } diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt b/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt new file mode 100644 index 0000000000..1b5224f227 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/input.txt @@ -0,0 +1,2 @@ +Wrap the answer in a element like so: +realn \ No newline at end of file diff --git a/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json b/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json new file mode 100644 index 0000000000..e9087e5c16 --- /dev/null +++ b/fixtures/extract-conformance/xml-root-mentioned-in-prose/schema.json @@ -0,0 +1,3 @@ +{ "format": "XML", "rootName": "doc", "fields": [ + { "name": "title", "kind": "STRING", "required": true }, + { "name": "note", "kind": "STRING", "required": false } ] } \ No newline at end of file diff --git a/fixtures/render-conformance/render-comment-with-partial/expected.txt b/fixtures/render-conformance/render-comment-with-partial/expected.txt new file mode 100644 index 0000000000..abfd547468 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/expected.txt @@ -0,0 +1 @@ +Hi Ada }} \ No newline at end of file diff --git a/fixtures/render-conformance/render-comment-with-partial/meta.json b/fixtures/render-conformance/render-comment-with-partial/meta.json new file mode 100644 index 0000000000..740389bd32 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/meta.json @@ -0,0 +1 @@ +{ "format": "text", "note": "a comment ends at the first closing delimiter, so a partial tag written inside one is never a partial (no unresolved-partial error)" } diff --git a/fixtures/render-conformance/render-comment-with-partial/payload.json b/fixtures/render-conformance/render-comment-with-partial/payload.json new file mode 100644 index 0000000000..8f842fdc28 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/payload.json @@ -0,0 +1 @@ +{ "name": "Ada" } diff --git a/fixtures/render-conformance/render-comment-with-partial/template.mustache b/fixtures/render-conformance/render-comment-with-partial/template.mustache new file mode 100644 index 0000000000..e1a2d1a2d0 --- /dev/null +++ b/fixtures/render-conformance/render-comment-with-partial/template.mustache @@ -0,0 +1 @@ +Hi {{name}}{{! {{> missing}} }} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/expected-drift.json @@ -0,0 +1 @@ +[] diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json new file mode 100644 index 0000000000..d2d8a2fbe6 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/options.json @@ -0,0 +1 @@ +{ "provider": "with" } diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json b/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache b/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache new file mode 100644 index 0000000000..e1a2d1a2d0 --- /dev/null +++ b/fixtures/verify-conformance/verify-comment-with-partial-clean/template.mustache @@ -0,0 +1 @@ +Hi {{name}}{{! {{> missing}} }} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json b/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json new file mode 100644 index 0000000000..11cb5ec64f --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "name" } ] diff --git a/fixtures/verify-conformance/verify-mismatched-close/payload.json b/fixtures/verify-conformance/verify-mismatched-close/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-mismatched-close/template.mustache b/fixtures/verify-conformance/verify-mismatched-close/template.mustache new file mode 100644 index 0000000000..2fbef13473 --- /dev/null +++ b/fixtures/verify-conformance/verify-mismatched-close/template.mustache @@ -0,0 +1 @@ +{{#name}}x{{/zzz}} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json b/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json new file mode 100644 index 0000000000..10adfc23b8 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_OUTPUT_TAG_MISSING", "path": "answer" } ] diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/options.json b/fixtures/verify-conformance/verify-required-tag-in-comment/options.json new file mode 100644 index 0000000000..4f863d00b1 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/options.json @@ -0,0 +1 @@ +{ "requiredTags": ["answer"] } diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json b/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache b/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache new file mode 100644 index 0000000000..fc54efbef8 --- /dev/null +++ b/fixtures/verify-conformance/verify-required-tag-in-comment/template.mustache @@ -0,0 +1 @@ +{{name}}{{! }} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.json @@ -0,0 +1 @@ +[] diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json b/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache new file mode 100644 index 0000000000..205b12f067 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/template.mustache @@ -0,0 +1 @@ +{{=<% %>=}}<% name %> {{literal}} \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json new file mode 100644 index 0000000000..e9119e2212 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_VAR_NOT_ON_PAYLOAD", "path": "nope" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json b/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache new file mode 100644 index 0000000000..8373c1983c --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/template.mustache @@ -0,0 +1 @@ +{{=<% %>=}}<% name %> <% nope %> \ No newline at end of file diff --git a/fixtures/verify-conformance/verify-unopened-close/expected-drift.json b/fixtures/verify-conformance/verify-unopened-close/expected-drift.json new file mode 100644 index 0000000000..1e9ea3a114 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "zzz" } ] diff --git a/fixtures/verify-conformance/verify-unopened-close/payload.json b/fixtures/verify-conformance/verify-unopened-close/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-unopened-close/template.mustache b/fixtures/verify-conformance/verify-unopened-close/template.mustache new file mode 100644 index 0000000000..4e67e9feb5 --- /dev/null +++ b/fixtures/verify-conformance/verify-unopened-close/template.mustache @@ -0,0 +1 @@ +x{{/zzz}} \ No newline at end of file diff --git a/scripts/leak-scan.test.ts b/scripts/leak-scan.test.ts new file mode 100644 index 0000000000..d0d19225a5 --- /dev/null +++ b/scripts/leak-scan.test.ts @@ -0,0 +1,88 @@ +// The public-repo leak scan, exercised against a throwaway repository. +// +// `.githooks/leak-scan.sh` is the hosted `leak-scan` status and `.githooks/pre-commit` is the +// local hook; both read `.githooks/leak-patterns.sh`. Two defects (audit finding 12) let a +// leak through: `--diff-filter=ACM` skipped renamed-with-edits files, and the hosted scan had +// no `/Users//` pattern while the local hook did. +import { afterAll, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const REPO = resolve(import.meta.dirname, ".."); +const tmp = mkdtempSync(join(tmpdir(), "leak-scan-")); +afterAll(() => rmSync(tmp, { recursive: true, force: true })); + +function git(cwd: string, ...args: string[]): void { + const r = spawnSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd }); + if (r.status !== 0) throw new Error(`git ${args.join(" ")}: ${r.stderr}`); +} + +function fixture(): string { + const dir = mkdtempSync(join(tmp, "repo-")); + git(dir, "init", "-q", "-b", "main"); + mkdirSync(join(dir, ".githooks")); + for (const f of ["leak-scan.sh", "leak-patterns.sh", "pre-commit"]) { + cpSync(join(REPO, ".githooks", f), join(dir, ".githooks", f)); + } + // Long enough that an edit stays above git's rename-similarity threshold. + writeFileSync(join(dir, "a.md"), Array.from({ length: 20 }, (_, i) => `line ${i}`).join("\n") + "\n"); + git(dir, "add", "-A"); + git(dir, "commit", "-q", "-m", "base"); + return dir; +} + +function scan(dir: string, script = "leak-scan.sh") { + const r = spawnSync("bash", [join(dir, ".githooks", script)], { cwd: dir, encoding: "utf8" }); + return { status: r.status, out: `${r.stdout}${r.stderr}` }; +} + +// Built from parts so this test file never trips the scan itself. +const mac = "/" + "Users" + "/someone/project"; +const linux = "/" + "home" + "/someone/project"; + +describe("leak-scan", () => { + test("clean staged change passes", () => { + const dir = fixture(); + writeFileSync(join(dir, "a.md"), "line 0\nnothing here\n"); + git(dir, "add", "-A"); + expect(scan(dir).status).toBe(0); + }); + + test("an added /home path is blocked (existing behaviour)", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${linux}\n`); + git(dir, "add", "-A"); + expect(scan(dir).status).toBe(1); + }); + + test("an added macOS home path is blocked (previously only the local hook caught it)", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${mac}\n`); + git(dir, "add", "-A"); + const r = scan(dir); + expect(r.status).toBe(1); + expect(r.out).toContain("someone"); + }); + + test("a leak added while RENAMING a file is blocked", () => { + const dir = fixture(); + git(dir, "mv", "a.md", "b.md"); + writeFileSync( + join(dir, "b.md"), + Array.from({ length: 20 }, (_, i) => `line ${i}`).join("\n") + `\nsee ${linux}\n`, + ); + git(dir, "add", "-A"); + const names = spawnSync("git", ["diff", "--cached", "--name-status"], { cwd: dir, encoding: "utf8" }).stdout; + expect(names).toMatch(/^R/); + expect(scan(dir).status).toBe(1); + }); + + test("the local pre-commit hook agrees with the hosted scan", () => { + const dir = fixture(); + writeFileSync(join(dir, "b.md"), `see ${mac}\n`); + git(dir, "add", "-A"); + expect(scan(dir, "pre-commit").status).toBe(1); + }); +}); diff --git a/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs b/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs index d0c01080b9..2e414d42d9 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/ConstraintErrorsTests.cs @@ -98,4 +98,54 @@ public void The_chain_walk_is_depth_bounded() Assert.Null(ConstraintErrors.Classify(deep)); } + + private sealed class CodedDbException(string message, string? sqlState) : System.Data.Common.DbException(message) + { + public override string? SqlState { get; } = sqlState; + } + + private sealed class FakeSqliteException(string message, int extended) : Exception(message) + { + public int SqliteExtendedErrorCode { get; } = extended; + } + + [Theory] + [InlineData("23505", 409, "unique")] + [InlineData("23503", 409, "foreign_key")] + [InlineData("23514", 400, "check")] + [InlineData("23502", 400, "not_null")] + public void A_sqlstate_decides_even_when_the_message_names_another_constraint( + string sqlState, int status, string constraint) + { + // The message would classify as a foreign key / not null by text; the code wins. + var failure = ConstraintErrors.Classify(new InvalidOperationException(EfWrapper, + new CodedDbException("duplicate value 'FOREIGN KEY' NOT NULL", sqlState))); + + Assert.NotNull(failure); + Assert.Equal(status, failure!.Status); + Assert.Equal(constraint, failure.Constraint); + } + + [Fact] + public void A_non_constraint_sqlstate_is_not_classified_from_message_text() + { + // 40001 is a serialization failure; its message mentioning NOT NULL must not make it a 400. + Assert.Null(ConstraintErrors.Classify(new CodedDbException("column NOT NULL retry", "40001"))); + } + + [Theory] + [InlineData(787, 409, "foreign_key")] + [InlineData(2067, 409, "unique")] + [InlineData(1555, 409, "unique")] + [InlineData(275, 400, "check")] + [InlineData(1299, 400, "not_null")] + public void A_sqlite_extended_code_decides_over_the_message(int extended, int status, string constraint) + { + var failure = ConstraintErrors.Classify(new InvalidOperationException(EfWrapper, + new FakeSqliteException("misleading UNIQUE CONSTRAINT text", extended))); + + Assert.NotNull(failure); + Assert.Equal(status, failure!.Status); + Assert.Equal(constraint, failure.Constraint); + } } diff --git a/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs b/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs new file mode 100644 index 0000000000..837fe29b04 --- /dev/null +++ b/server/csharp/MetaObjects.Codegen.Tests/EntityGeneratorEscapeTests.cs @@ -0,0 +1,38 @@ +using MetaObjects.Codegen.Generators; +using Xunit; + +namespace MetaObjects.Codegen.Tests; + +public class EntityGeneratorEscapeTests +{ + private sealed class Probe : EntityGenerator + { + public static string Escape(string s) => EscapeAttrString(s); + } + + [Fact] + public void Escapes_what_would_end_the_literal() + { + Assert.Equal("a\\\\b\\\"c", Probe.Escape("a\\b\"c")); + Assert.Equal("l1\\nl2\\r\\t", Probe.Escape("l1\nl2\r\t")); + Assert.Equal("\\u0001\\u2028\\u0085", Probe.Escape("\u0001" + (char)0x2028 + (char)0x85)); + } + + [Fact] + public void Leaves_ordinary_text_alone() + { + Assert.Equal("^[a-z]+é$", Probe.Escape("^[a-z]+é$")); + } + + [Fact] + public void An_escaped_value_survives_a_round_trip_through_the_compiler_literal_rules() + { + var original = "line1\nline2\t\"q\"\\ " + (char)0x2029; + var src = "class C { public const string V = \"" + Probe.Escape(original) + "\"; }"; + var tree = Microsoft.CodeAnalysis.CSharp.CSharpSyntaxTree.ParseText(src); + Assert.Empty(tree.GetDiagnostics()); + var literal = tree.GetRoot().DescendantNodes() + .OfType().Single(); + Assert.Equal(original, literal.Token.ValueText); + } +} diff --git a/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs b/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs index ddb0be54cc..cdc8435b58 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/NamesExtendsChainTests.cs @@ -243,7 +243,7 @@ public void A_child_field_colliding_with_an_INHERITED_one_is_refused_naming_the_ { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "CreatedAt" } }, + { "field.timestamp": { "name": "CreatedAt", "@column": "created_at_alt" } }, { "identity.primary": { "name": "pk", "@fields": "id", "@generation": "increment" } } ]}} ]}} diff --git a/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs b/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs index 03b614dd97..5ae9bb5e7c 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/NamesGeneratorTests.cs @@ -343,7 +343,9 @@ public void The_type_prefix_keeps_an_unnamed_primary_key_off_the_primary_ROLE_me [Fact] public void Two_fields_colliding_on_their_Pascal_form_is_refused_naming_the_model() { - // userId and UserId both Pascalize to "UserId" — two duplicate const members. + // userId and UserId both Pascalize to "UserId" — two duplicate const members. (The + // explicit @column keeps their PHYSICAL columns apart: equal physical columns are + // refused earlier, by the loader, as ERR_COLUMN_NAME_COLLISION.) // C# would refuse to compile the generated file, but the error would name a // generated .g.cs and read as a codegen bug rather than a model one. Fail here // instead, naming the entity and both offending field names. @@ -353,7 +355,7 @@ public void Two_fields_colliding_on_their_Pascal_form_is_refused_naming_the_mode { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": "id" } } ]}} ]}} diff --git a/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs b/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs index cbf1848f5f..810c3c78e7 100644 --- a/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs +++ b/server/csharp/MetaObjects.Codegen/Generators/EntityGenerator.cs @@ -1842,9 +1842,31 @@ protected static string DefaultInitializer(MetaField field, string baseType) System.Globalization.CultureInfo.InvariantCulture, out _) ? s : null; } - // Escapes a regex pattern for embedding in a C# string-literal attribute argument. - protected static string EscapeAttrString(string s) => - s.Replace("\\", "\\\\").Replace("\"", "\\\""); + // Escapes a regex pattern or default for embedding in a C# string-literal attribute argument. + // A raw newline, any other control character, or a Unicode line separator would end the + // literal and emit source that does not compile. + protected static string EscapeAttrString(string s) + { + var sb = new System.Text.StringBuilder(s.Length + 8); + foreach (var c in s) + { + switch (c) + { + case '\\': sb.Append("\\\\"); break; + case '"': sb.Append("\\\""); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (char.IsControl(c) || c is '\u2028' or '\u2029') + sb.Append("\\u").Append(((int)c).ToString("x4", System.Globalization.CultureInfo.InvariantCulture)); + else + sb.Append(c); + break; + } + } + return sb.ToString(); + } // An owned-type navigation property for an object-typed field. Returns null (with // a warning) when the @objectRef can't be resolved. diff --git a/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs b/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs index f4d49fddcb..0a3da9dd25 100644 --- a/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs +++ b/server/csharp/MetaObjects.Codegen/Runtime/ConstraintErrors.cs @@ -20,7 +20,15 @@ // KIND — matching TypeScript byte for byte rather than inventing a taxonomy the other // ports would then have to follow. // -// WHY TEXT AND NOT A TYPED DRIVER CHECK: this assembly must not take a dependency on any +// STRUCTURED CODES FIRST: when the chain holds a DbException that reports a SQLSTATE +// (DbException.SqlState, which Npgsql fills), or an exception exposing a SQLite extended +// result code (Microsoft.Data.Sqlite's SqliteExtendedErrorCode, read by property name so no +// provider is referenced), that code is the verdict and message text is never consulted: +// a unique-violation whose offending value happens to read "NOT NULL" is still a unique +// violation. Message matching survives only as the fallback for exceptions that carry no +// code at all, such as an in-memory double. +// +// WHY NOT A TYPED DRIVER CHECK: this assembly must not take a dependency on any // specific ADO.NET provider — a consumer may be on Npgsql, SQL Server, SQLite or an // in-memory double, and binding to one of them here would make the generated routes // unusable on the others. Npgsql puts the SQLSTATE at the front of PostgresException's @@ -79,6 +87,9 @@ public static class ConstraintErrors /// public static ConstraintFailure? Classify(Exception? error) { + var coded = CodedVerdict(error, out var hasCode); + if (hasCode) return coded is null ? null : Failure(coded.Value); + var haystack = ChainText(error); if (haystack.Length == 0) return null; @@ -87,8 +98,65 @@ public static class ConstraintErrors // 409 for referential/uniqueness conflicts with EXISTING state; 400 for a value // the request itself got wrong. Both are client errors — neither is a 500. + return Failure(kind.Value); + } + + private static ConstraintFailure Failure(ConstraintKind kind) + { var status = kind is ConstraintKind.ForeignKey or ConstraintKind.Unique ? 409 : 400; - return new ConstraintFailure(status, kind.Value); + return new ConstraintFailure(status, kind); + } + + private const int MaxDepth = 8; + + // SQLite extended result codes (https://www.sqlite.org/rescode.html). + private const int SqliteConstraintCheck = 275; + private const int SqliteConstraintForeignKey = 787; + private const int SqliteConstraintNotNull = 1299; + private const int SqliteConstraintPrimaryKey = 1555; + private const int SqliteConstraintUnique = 2067; + + /// + /// The verdict of the first exception in the chain that carries a structured code: + /// says whether one was found, and the return is its kind (null + /// for a code that is not one of the four constraint classes). + /// + private static ConstraintKind? CodedVerdict(Exception? error, out bool hasCode) + { + var seen = 0; + for (var e = error; e is not null && seen < MaxDepth; e = e.InnerException, seen++) + { + foreach (var candidate in e is AggregateException agg ? agg.InnerExceptions.Prepend(e) : new[] { e }) + { + if (candidate is System.Data.Common.DbException { SqlState: { Length: > 0 } state }) + { + hasCode = true; + return state switch + { + "23503" => ConstraintKind.ForeignKey, + "23505" => ConstraintKind.Unique, + "23514" => ConstraintKind.Check, + "23502" => ConstraintKind.NotNull, + _ => null, + }; + } + var extended = candidate.GetType().GetProperty("SqliteExtendedErrorCode")?.GetValue(candidate); + if (extended is int code) + { + hasCode = true; + return code switch + { + SqliteConstraintForeignKey => ConstraintKind.ForeignKey, + SqliteConstraintUnique or SqliteConstraintPrimaryKey => ConstraintKind.Unique, + SqliteConstraintCheck => ConstraintKind.Check, + SqliteConstraintNotNull => ConstraintKind.NotNull, + _ => null, + }; + } + } + } + hasCode = false; + return null; } private static ConstraintKind? KindOf(string haystack) @@ -132,9 +200,8 @@ private static bool Contains(string haystack, string needle) => /// private static string ChainText(Exception? error) { - const int maxDepth = 8; - var parts = new List(maxDepth); - for (var e = error; e is not null && parts.Count < maxDepth; e = e.InnerException) + var parts = new List(MaxDepth); + for (var e = error; e is not null && parts.Count < MaxDepth; e = e.InnerException) { if (!string.IsNullOrEmpty(e.Message)) parts.Add(e.Message); // An AggregateException holds its failures beside InnerException, so the chain diff --git a/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs b/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs index 81bc7ffe1a..754485d5e2 100644 --- a/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/Extract/ExtractConformanceTests.cs @@ -46,7 +46,7 @@ public void Discovers_all_extract_conformance_cases() { // FR-011: lock the corpus size so a deleted fixture fails CI rather than // silently reducing coverage. Mirrors the TS / Java / Python count guards. - Assert.Equal(48, Cases().Count()); + Assert.Equal(54, Cases().Count()); } [Theory] diff --git a/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs b/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs index 22a0f04d23..9526cb4407 100644 --- a/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs +++ b/server/csharp/MetaObjects.Render.Tests/VerifyConformanceTests.cs @@ -25,6 +25,7 @@ public class VerifyConformanceTests Verify.ERR_PARTIAL_UNRESOLVED, Verify.ERR_REQUIRED_SLOT_UNUSED, Verify.ERR_OUTPUT_TAG_MISSING, + Verify.ERR_TEMPLATE_SYNTAX, }; private static string CorpusRoot() diff --git a/server/csharp/MetaObjects.Render/Extract/Locate.cs b/server/csharp/MetaObjects.Render/Extract/Locate.cs index 0609f777a5..92fd0b4913 100644 --- a/server/csharp/MetaObjects.Render/Extract/Locate.cs +++ b/server/csharp/MetaObjects.Render/Extract/Locate.cs @@ -104,18 +104,28 @@ private static int ScanBalanced(string s, int open) var flags = caseInsensitive ? RegexOptions.IgnoreCase : RegexOptions.None; + // Comments and CDATA are not searched; the view has the same length as the text. + string view = XmlForgivingReader.MaskNonMarkup(text); var openPattern = new Regex($"<{Regex.Escape(rootName)}(\\s[^>]*)?>", flags); - var openMatch = openPattern.Match(text); + var openMatch = openPattern.Match(view); if (!openMatch.Success) return null; int start = openMatch.Index; int gt = openMatch.Index + openMatch.Length - 1; // position of the '>' closing the open tag var closePattern = new Regex($"", flags); - var closeMatch = closePattern.Match(text, openMatch.Index + openMatch.Length); + var closeMatch = closePattern.Match(view, openMatch.Index + openMatch.Length); // Guard: if close found but its end position <= gt, treat as no-close (shouldn't // normally occur, but protects against malformed input such as a bare close tag). if (closeMatch.Success && closeMatch.Index + closeMatch.Length > gt) { + // A root does not nest in itself: another opener between the first one and its + // close is the real element (the tag was mentioned in prose first). + var next = openPattern.Match(view, openMatch.Index + openMatch.Length); + while (next.Success && next.Index < closeMatch.Index) + { + start = next.Index; + next = openPattern.Match(view, next.Index + next.Length); + } return text[start..(closeMatch.Index + closeMatch.Length)]; } return text[start..]; diff --git a/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs b/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs index 6f88b9dec3..d83bd474d4 100644 --- a/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs +++ b/server/csharp/MetaObjects.Render/Extract/XmlForgivingReader.cs @@ -34,10 +34,102 @@ public sealed class XmlForgivingReader "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))", RegexOptions.Compiled); - public Dictionary Read(string? span, bool caseInsensitive) + private const string CommentOpen = ""; + private const string CdataOpen = ""; + // the five predefined XML entities plus numeric character references. + private static readonly Regex EntityRegex = new( + "&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);", RegexOptions.Compiled); + + private enum RegionKind { Comment, Cdata } + + /// + /// Rewrite every XML comment and CDATA section in through + /// , scanning left to right so a comment marker inside CDATA (or + /// the reverse) is never mistaken for markup. An unclosed comment is left as text; an + /// unclosed CDATA section runs to the end. + /// + private static string RewriteRegions(string text, Func replace) + { + var sb = new System.Text.StringBuilder(); + int pos = 0, from = 0; + while (true) + { + int c = text.IndexOf(CommentOpen, from, StringComparison.Ordinal); + int d = text.IndexOf(CdataOpen, from, StringComparison.Ordinal); + bool isComment = c >= 0 && (d < 0 || c < d); + int start = isComment ? c : d; + if (start < 0) break; + int end; + if (isComment) + { + int e = text.IndexOf(CommentClose, start + CommentOpen.Length, StringComparison.Ordinal); + if (e < 0) { from = start + CommentOpen.Length; continue; } + end = e + CommentClose.Length; + } + else + { + int e = text.IndexOf(CdataClose, start + CdataOpen.Length, StringComparison.Ordinal); + end = e < 0 ? text.Length : e + CdataClose.Length; + } + sb.Append(text, pos, start - pos) + .Append(replace(isComment ? RegionKind.Comment : RegionKind.Cdata, text.Substring(start, end - start))); + pos = end; + from = end; + } + return sb.Append(text, pos, text.Length - pos).ToString(); + } + + /// with comments and CDATA sections blanked to spaces (same + /// length, so indices hold): the view the tag locators search, so markup inside them is + /// never located. + internal static string MaskNonMarkup(string text) => + RewriteRegions(text, (_, region) => new string(' ', region.Length)); + + /// Drop comments; turn each CDATA section into its text with & < > + /// entity-escaped, so the tag scanner cannot split it and the entity decode restores it + /// exactly. + private static string Prepare(string text) => RewriteRegions(text, (kind, region) => + { + if (kind == RegionKind.Comment) return ""; + string body = region.EndsWith(CdataClose, StringComparison.Ordinal) + ? region.Substring(CdataOpen.Length, region.Length - CdataOpen.Length - CdataClose.Length) + : region.Substring(CdataOpen.Length); + return body.Replace("&", "&").Replace("<", "<").Replace(">", ">"); + }); + + /// Decode predefined entities and numeric references in one pass (so + /// &amp;lt; is &lt;). An unknown or out-of-range reference stays as + /// written. + private static string DecodeEntities(string s) + { + if (!s.Contains('&')) return s; + return EntityRegex.Replace(s, m => + { + string body = m.Groups[1].Value; + switch (body) + { + case "lt": return "<"; + case "gt": return ">"; + case "amp": return "&"; + case "quot": return "\""; + case "apos": return "'"; + } + bool hex = body[1] == 'x'; + string digits = body.Substring(hex ? 2 : 1); + if (!int.TryParse(digits, hex ? System.Globalization.NumberStyles.HexNumber : System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, out int cp)) return m.Value; + bool surrogate = cp >= 0xD800 && cp <= 0xDFFF; + return cp >= 1 && cp <= 0x10FFFF && !surrogate ? char.ConvertFromUtf32(cp) : m.Value; + }); + } + + public Dictionary Read(string? rawSpan, bool caseInsensitive) { var out_ = new Dictionary(); - if (string.IsNullOrWhiteSpace(span)) return out_; + if (string.IsNullOrWhiteSpace(rawSpan)) return out_; + string span = Prepare(rawSpan); int gt = span.IndexOf('>'); if (gt < 0) return out_; @@ -46,7 +138,7 @@ public sealed class XmlForgivingReader int innerEnd = (rootEnd < 0 || rootEnd <= gt) ? span.Length : rootEnd; string inner = span.Substring(gt + 1, innerEnd - (gt + 1)); - ParseChildren(inner, caseInsensitive, out_, null); + ParseChildren(inner, caseInsensitive, out_, null, false); return out_; } @@ -60,7 +152,7 @@ public sealed class XmlForgivingReader { var out_ = new Dictionary(); if (string.IsNullOrWhiteSpace(text)) return out_; - ParseChildren(text!, caseInsensitive, out_, null); + ParseChildren(Prepare(text!), caseInsensitive, out_, null, false); return out_; } @@ -68,15 +160,30 @@ public sealed class XmlForgivingReader /// Parse 's elements into . When /// is non-null, the text BETWEEN those elements (an element's own /// text in mixed content) is appended to it, one segment per gap. + /// is set for the body of an element that DID close: an + /// open tag in there with no close tag anywhere after it (List<int> and ...) is + /// prose, not an element. /// - private static void ParseChildren(string inner, bool ci, Dictionary out_, List? loose) + private static void ParseChildren(string inner, bool ci, Dictionary out_, List? loose, + bool literalUnclosed) { - var openTag = new Regex(OpenTagPattern, ci ? RegexOptions.IgnoreCase : RegexOptions.None); + var flags = ci ? RegexOptions.IgnoreCase : RegexOptions.None; + var openTag = new Regex(OpenTagPattern, flags); + var anyClose = new Regex(CloseTagPattern, flags); int pos = 0; - Match m = openTag.Match(inner, pos); + int scan = 0; + Match m = openTag.Match(inner, scan); while (m.Success) { + if (literalUnclosed && !m.Groups[2].Value.TrimEnd().EndsWith('/') + && !anyClose.Match(inner, m.Index + m.Length).Success) + { + scan = m.Index + m.Length; + if (scan >= inner.Length) break; + m = openTag.Match(inner, scan); + continue; + } loose?.Add(inner.Substring(pos, m.Index - pos)); string tag = m.Groups[1].Value; @@ -91,6 +198,7 @@ private static void ParseChildren(string inner, bool ci, Dictionary= inner.Length) break; m = openTag.Match(inner, pos); continue; @@ -152,8 +260,9 @@ private static void ParseChildren(string inner, bool ci, Dictionary= inner.Length) break; m = openTag.Match(inner, pos); } @@ -188,13 +297,13 @@ private static (int Start, int End)? FindStrayClose(string inner, int from, int /// content keeps BOTH: the children, and the element's own text under , /// so a scalar or @xmlText consumer still reads the prose around a child element. /// - private static object? Combine(Dictionary attrs, string content, bool ci) + private static object? Combine(Dictionary attrs, string content, bool ci, bool closed) { if (content.Contains('<')) { var nested = new Dictionary(); var loose = new List(); - ParseChildren(content, ci, nested, loose); + ParseChildren(content, ci, nested, loose, closed); if (nested.Count > 0) { // attributes first; a child element wins a name collision @@ -216,7 +325,7 @@ private static string MixedText(List segments, bool ci) var parts = new List(); foreach (string seg in segments) { - string t = closeRegex.Replace(seg, "").Trim(); + string t = DecodeEntities(closeRegex.Replace(seg, "").Trim()); if (t.Length > 0) parts.Add(t); } return string.Join(" ", parts); @@ -224,7 +333,7 @@ private static string MixedText(List segments, bool ci) private static object? TextValue(Dictionary attrs, string content) { - string text = content.Trim(); + string text = DecodeEntities(content.Trim()); if (attrs.Count == 0) return text; var m = new Dictionary(attrs) { [TextKey] = text }; return m; @@ -240,7 +349,7 @@ private static string MixedText(List segments, bool ci) string val = a.Groups[2].Success ? a.Groups[2].Value : a.Groups[3].Success ? a.Groups[3].Value : a.Groups[4].Success ? a.Groups[4].Value : ""; - if (!attrs.ContainsKey(name)) attrs[name] = val; + if (!attrs.ContainsKey(name)) attrs[name] = DecodeEntities(val); } return attrs; } diff --git a/server/csharp/MetaObjects.Render/MustacheTags.cs b/server/csharp/MetaObjects.Render/MustacheTags.cs new file mode 100644 index 0000000000..2b376a0eaa --- /dev/null +++ b/server/csharp/MetaObjects.Render/MustacheTags.cs @@ -0,0 +1,115 @@ +// One Mustache tag scanner shared by Verify and the renderer's partial expansion. +// +// Both used to find tags on their own (Verify with a "{{" search, Renderer with a +// "{{> x}}" regex), and neither knew what the engine knows: a comment ends at the +// first closing delimiter, "{{=<% %>=}}" changes the delimiters, and a section close +// must name the section it closes. Scan follows the Mustache spec for all three, so the +// two callers cannot disagree about where a tag is. Mirrors Mustache.parse in the +// TypeScript port and the Java and Python scanners. + +namespace MetaObjects.Render; + +/// One tag: its kind, trimmed name, and [Start, End) span in the text. +internal sealed record MustacheTag(MustacheTagKind Kind, string Value, int Start, int End); + +internal enum MustacheTagKind { Name, Triple, Amp, Section, Inverted, Close, Partial, Comment, Delimiters } + +/// An unbalanced section; is the section the finding is about. +internal sealed class MustacheSyntaxException(string message, string section) : Exception(message) +{ + public string Section { get; } = section; +} + +internal static class MustacheTags +{ + private const string DefaultOpen = "{{"; + private const string DefaultClose = "}}"; + + /// + /// Every tag in , in order, honouring comments and delimiter changes. + /// Throws for a close with no open, a close naming a + /// different section than the one open, or a section never closed. An unterminated tag ends + /// the scan (the rest is text), as it always did here. + /// + public static List Scan(string text) + { + var tags = new List(); + string open = DefaultOpen, close = DefaultClose; + var openSections = new Stack(); + int pos = 0; + while (pos < text.Length) + { + int start = text.IndexOf(open, pos, StringComparison.Ordinal); + if (start < 0) break; + int contentStart = start + open.Length; + bool triple = contentStart < text.Length && text[contentStart] == '{'; + string endDelim = triple ? "}" + close : close; + if (triple) contentStart++; + int closeIdx = text.IndexOf(endDelim, contentStart, StringComparison.Ordinal); + if (closeIdx < 0) break; + string content = text[contentStart..closeIdx].Trim(); + int end = closeIdx + endDelim.Length; + pos = end; + + if (triple) { tags.Add(new(MustacheTagKind.Triple, content, start, end)); continue; } + if (content.Length == 0) continue; + char sigil = content[0]; + string name = content.Length > 1 ? content[1..].Trim() : ""; + switch (sigil) + { + case '!': + tags.Add(new(MustacheTagKind.Comment, name, start, end)); + break; + case '=': + if (content.EndsWith('=')) + { + var parts = content[1..^1].Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) { open = parts[0]; close = parts[1]; } + } + tags.Add(new(MustacheTagKind.Delimiters, name, start, end)); + break; + case '#': + case '^': + openSections.Push(name); + tags.Add(new(sigil == '#' ? MustacheTagKind.Section : MustacheTagKind.Inverted, name, start, end)); + break; + case '/': + if (openSections.Count == 0) + throw new MustacheSyntaxException($"Unopened section \"{name}\"", name); + string opened = openSections.Pop(); + if (opened != name) + throw new MustacheSyntaxException($"Unclosed section \"{opened}\"", opened); + tags.Add(new(MustacheTagKind.Close, name, start, end)); + break; + case '>': + tags.Add(new(MustacheTagKind.Partial, name, start, end)); + break; + case '&': + tags.Add(new(MustacheTagKind.Amp, name, start, end)); + break; + default: + tags.Add(new(MustacheTagKind.Name, content, start, end)); + break; + } + } + if (openSections.Count > 0) + { + string opened = openSections.Peek(); + throw new MustacheSyntaxException($"Unclosed section \"{opened}\"", opened); + } + return tags; + } + + /// with every comment cut out: what a tag-presence check may scan. + public static string WithoutComments(string text, IReadOnlyList tags) + { + var sb = new System.Text.StringBuilder(); + int pos = 0; + foreach (var tag in tags.Where(t => t.Kind == MustacheTagKind.Comment)) + { + sb.Append(text, pos, tag.Start - pos); + pos = tag.End; + } + return sb.Append(text, pos, text.Length - pos).ToString(); + } +} diff --git a/server/csharp/MetaObjects.Render/Renderer.cs b/server/csharp/MetaObjects.Render/Renderer.cs index 42d4bf0b73..09d2d8cc5f 100644 --- a/server/csharp/MetaObjects.Render/Renderer.cs +++ b/server/csharp/MetaObjects.Render/Renderer.cs @@ -1,14 +1,13 @@ // Deterministic, logic-less render: (template + payload + provider) -> string. // // Ported from typescript/packages/render/src/render.ts. Partials are inlined by -// regex BEFORE the Mustache pass (deterministic whitespace + path-based cycle/ +// tag scan BEFORE the Mustache pass (deterministic whitespace + path-based cycle/ // depth detection, independent of any lib partial loader); escaping is engine- // owned (the format escaper is installed as Stubble's encoding function, the // equivalent of overriding mustache.js's Mustache.escape). `{{{x}}}` bypasses it. // // The render-time `verify` guard (FR-004 Plan #3) is intentionally not wired yet. -using System.Text.RegularExpressions; using Stubble.Core.Builders; using Stubble.Core.Settings; @@ -45,21 +44,29 @@ public sealed record RenderRequest } /// The logic-less, deterministic MetaObjects render engine. -public static partial class Renderer +public static class Renderer { private const int MaxDepth = 32; - [GeneratedRegex(@"\{\{>\s*([^}\s]+)\s*\}\}")] - private static partial Regex PartialPattern(); - // Resolve `{{> group/source }}` partials by inlining their text, recursively, // BEFORE Mustache parses — giving deterministic whitespace and true, path-based - // cycle/depth detection. + // cycle/depth detection. Partial tags are located by the shared scanner + // (MustacheTags), so one written inside a comment is not a partial. private static string Expand(string text, IProvider provider, IReadOnlyList path) { - return PartialPattern().Replace(text, match => + List tags; + try { tags = MustacheTags.Scan(text); } + catch (MustacheSyntaxException e) { throw new RenderException($"template syntax error: {e.Message}"); } + var partials = tags.Where(t => t.Kind == MustacheTagKind.Partial).ToList(); + if (partials.Count == 0) return text; + // Inlined text is parsed under the surrounding delimiters, not the default ones. + if (tags.Any(t => t.Kind == MustacheTagKind.Delimiters)) + throw new RenderException("a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + var sb = new System.Text.StringBuilder(); + int pos = 0; + foreach (var tag in partials) { - string reference = match.Groups[1].Value; + string reference = tag.Value; if (path.Contains(reference)) throw new RenderException($"partial cycle: {string.Join(" -> ", path.Append(reference))}"); if (path.Count >= MaxDepth) @@ -68,8 +75,10 @@ private static string Expand(string text, IProvider provider, IReadOnlyList(path) { reference }; - return Expand(t, provider, next); - }); + sb.Append(text, pos, tag.Start - pos).Append(Expand(t, provider, next)); + pos = tag.End; + } + return sb.Append(text, pos, text.Length - pos).ToString(); } /// Deterministic, logic-less render of a template against a payload. diff --git a/server/csharp/MetaObjects.Render/Verify.cs b/server/csharp/MetaObjects.Render/Verify.cs index ffa1d399db..449ed90f38 100644 --- a/server/csharp/MetaObjects.Render/Verify.cs +++ b/server/csharp/MetaObjects.Render/Verify.cs @@ -54,6 +54,9 @@ public static class Verify /// A declared @requiredTags output tag is absent from the template text. public const string ERR_OUTPUT_TAG_MISSING = "ERR_OUTPUT_TAG_MISSING"; + /// The template text does not parse: an unbalanced Mustache section. path = the section's name. + public const string ERR_TEMPLATE_SYNTAX = "ERR_TEMPLATE_SYNTAX"; + private const int MaxDepth = 32; // --- Mustache tag token model (rendering-agnostic; only tag structure) --- @@ -62,61 +65,43 @@ private sealed record VarTok(string Value) : Tok; private sealed record SectionTok(string Value, bool Inverted, IReadOnlyList Children) : Tok; private sealed record PartialTok(string Value) : Tok; + // The tag tree of `text`, built from the shared scanner (MustacheTags). Throws + // MustacheSyntaxException on an unbalanced section. private static List Tokenize(string text) { - int pos = 0; - return ParseTokens(text, ref pos, null); - } - - // Parse tokens until EOF or a matching {{/closeName}} (which is consumed). - private static List ParseTokens(string text, ref int pos, string? closeName) - { - var tokens = new List(); - while (pos < text.Length) + var root = new List(); + var levels = new Stack>(); + levels.Push(root); + foreach (var tag in MustacheTags.Scan(text)) { - int open = text.IndexOf("{{", pos, StringComparison.Ordinal); - if (open < 0) break; // remainder is plain text - bool triple = open + 2 < text.Length && text[open + 2] == '{'; - string closeDelim = triple ? "}}}" : "}}"; - int contentStart = open + (triple ? 3 : 2); - int close = text.IndexOf(closeDelim, contentStart, StringComparison.Ordinal); - if (close < 0) break; // malformed; stop - string content = text[contentStart..close].Trim(); - pos = close + closeDelim.Length; - - if (triple) - { - tokens.Add(new VarTok(content)); - continue; - } - if (content.Length == 0) continue; - - char sigil = content[0]; - string name = content.Length > 1 ? content[1..].Trim() : ""; - switch (sigil) + switch (tag.Kind) { - case '!': break; // comment - case '#': - case '^': - var children = ParseTokens(text, ref pos, name); - tokens.Add(new SectionTok(name, sigil == '^', children)); + case MustacheTagKind.Section: + case MustacheTagKind.Inverted: + var children = new List(); + levels.Peek().Add(new SectionTok(tag.Value, tag.Kind == MustacheTagKind.Inverted, children)); + levels.Push(children); break; - case '/': - return tokens; // close (matched or mismatched) ends this level - case '>': - tokens.Add(new PartialTok(name)); + case MustacheTagKind.Close: + levels.Pop(); // the scanner already matched it to its open break; - case '&': - tokens.Add(new VarTok(name)); + case MustacheTagKind.Partial: + levels.Peek().Add(new PartialTok(tag.Value)); break; - default: - tokens.Add(new VarTok(content)); // {{x}}, {{x.y}}, {{.}} + case MustacheTagKind.Name: + case MustacheTagKind.Triple: + case MustacheTagKind.Amp: + levels.Peek().Add(new VarTok(tag.Value)); // {{x}}, {{x.y}}, {{.}}, {{{x}}}, {{&x}} break; + // comments and delimiter changes carry no reference } } - return tokens; + return root; } + // `text` minus its comments, for the output-tag scan. + private static string StaticText(string text) => MustacheTags.WithoutComments(text, MustacheTags.Scan(text)); + private static PayloadField? Find(IReadOnlyList fields, string name) => fields.FirstOrDefault(f => f.Name == name); @@ -234,14 +219,28 @@ void Walk(IReadOnlyList tokens, List> stack, Li errors.Add(new VerifyError(ERR_PARTIAL_UNRESOLVED, p.Value)); break; } - staticTexts.Add(text); - Walk(Tokenize(text), stack, new List(seen) { p.Value }, inElement); + List partialToks; + try { partialToks = Tokenize(text); } + catch (MustacheSyntaxException e) + { + errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)); + break; + } + staticTexts.Add(StaticText(text)); + Walk(partialToks, stack, new List(seen) { p.Value }, inElement); break; } } } - Walk(Tokenize(templateText), [root], []); + List bodyToks; + try { bodyToks = Tokenize(templateText); } + catch (MustacheSyntaxException e) + { + return [new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)]; + } + staticTexts[0] = StaticText(templateText); + Walk(bodyToks, [root], []); foreach (var slot in options?.RequiredSlots ?? []) if (!referencedAtRoot.Contains(slot)) diff --git a/server/csharp/MetaObjects/Errors.cs b/server/csharp/MetaObjects/Errors.cs index d2498740bc..f3fcd25782 100644 --- a/server/csharp/MetaObjects/Errors.cs +++ b/server/csharp/MetaObjects/Errors.cs @@ -247,6 +247,9 @@ public enum ErrorCode // @fields), whichever is declared supplies no key, or a named field does not exist // in the entity's effective field set. ERR_INVALID_INDEX, + // Two fields of one stored object resolve to the same PHYSICAL column (snake_case folds + // `userId` and `userID` together, or two fields share an @column). + ERR_COLUMN_NAME_COLLISION, // FR-044 — reporting vocabulary (ValidationPasses.ValidateReporting). Rule ids // D1-D4 / M1-M6 / R1-R7 are the cross-port rule table; S1/F1/F2 reuse // ERR_BAD_ATTR_FILTER. diff --git a/server/csharp/MetaObjects/Loader/MetaDataLoader.cs b/server/csharp/MetaObjects/Loader/MetaDataLoader.cs index 41ef2e75e1..a805e6f65e 100644 --- a/server/csharp/MetaObjects/Loader/MetaDataLoader.cs +++ b/server/csharp/MetaObjects/Loader/MetaDataLoader.cs @@ -642,6 +642,9 @@ public LoadResult Load(IReadOnlyList sources) // and each must exist in the entity's effective field set (ERR_INVALID_INDEX). errors.AddRange(ValidationPasses.ValidateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + errors.AddRange(ValidationPasses.ValidateColumnNameCollisions(root)); + // FR-039 — a retired requirement carries no @implementedBy (refused, not // exempted, so the dangling-ref class is unreachable) and @supersededBy is // legal only there. diff --git a/server/csharp/MetaObjects/Loader/ValidationPasses.cs b/server/csharp/MetaObjects/Loader/ValidationPasses.cs index a49fbe06e0..34fbf0b0df 100644 --- a/server/csharp/MetaObjects/Loader/ValidationPasses.cs +++ b/server/csharp/MetaObjects/Loader/ValidationPasses.cs @@ -4105,7 +4105,7 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) } else if (NumericDiscriminatorSubtypes.Contains(field.SubType)) { - if (!Regex.IsMatch(value, "^-?\\d+$")) + if (!IsAsciiInteger(value)) { errors.Add(new MetaError( $"object.entity \"{obj.Name}\" @discriminatorValue: \"{value}\" does not coerce to " + @@ -4129,9 +4129,14 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) foreach (var discRoot in order) { var seen = new Dictionary(); + var rootField = discRoot.OwnAttr(OBJECT_ATTR_DISCRIMINATOR) is string rootFieldName + ? FindFieldOnEntity(discRoot, rootFieldName) : null; + var numericRoot = rootField != null && NumericDiscriminatorSubtypes.Contains(rootField.SubType); foreach (var (subtype, value) in bindingsByRoot[discRoot]) { - if (seen.TryGetValue(value, out var prev)) + // "1" and "01" are one value to a numeric column, so they are one claim. + var key = numericRoot && IsAsciiInteger(value) ? CanonicalInteger(value) : value; + if (seen.TryGetValue(key, out var prev)) { errors.Add(new MetaError( $"object.entity \"{subtype.Name}\" @discriminatorValue: \"{value}\" duplicates the " + @@ -4141,7 +4146,7 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) } else { - seen[value] = subtype; + seen[key] = subtype; } } } @@ -4164,6 +4169,21 @@ public static IReadOnlyList ValidateDiscriminator(MetaData root) return errors.AsReadOnly(); } + /// An optional minus sign and ASCII digits only. Not \d (which matches every Unicode + /// decimal digit) and not $ (which matches before a trailing newline). + private static bool IsAsciiInteger(string value) + { + var start = value.Length > 0 && value[0] == '-' ? 1 : 0; + if (value.Length == start) return false; + for (var i = start; i < value.Length; i++) + if (value[i] < '0' || value[i] > '9') return false; + return true; + } + + /// The integer's canonical decimal spelling: no leading zeros, and no sign on zero. + private static string CanonicalInteger(string value) => + System.Numerics.BigInteger.Parse(value, System.Globalization.CultureInfo.InvariantCulture).ToString(System.Globalization.CultureInfo.InvariantCulture); + private static MetaField? FindFieldOnEntity(MetaData entity, string name) { var f = entity.OwnChildren().FirstOrDefault(c => c.Type == TYPE_FIELD && c.Name == name) as MetaField; @@ -4353,6 +4373,42 @@ void Walk(MetaData node) return errors; } + // ValidateColumnNameCollisions — ERR_COLUMN_NAME_COLLISION: two fields of one stored + // object resolving to the SAME physical column (userId + userID both snake_case to + // user_id, or two fields sharing an explicit @column). Only the RESOLVED name is + // compared (PhysicalNaming.ResolveColumnName), so an explicit @column, an inherited + // field and the project's column strategy are honoured. Scope: a concrete object + // declaring at least one source (a source-less object generates nothing); derived + // fields and field.object (may flatten to several columns) are skipped. + // ADR-0039: effective children, so an inherited field collides like an own one. + public static IReadOnlyList ValidateColumnNameCollisions(MetaData root) + { + var errors = new List(); + foreach (var obj in root.OwnChildren().Where(c => c.Type == TYPE_OBJECT)) + { + if (obj.IsAbstract) continue; + if (!obj.Children().Any(c => c.Type == TYPE_SOURCE)) continue; + var seen = new Dictionary(StringComparer.Ordinal); + foreach (var field in obj.Children().OfType()) + { + if (field.SubType == FIELD_SUBTYPE_OBJECT || field.IsDerived()) continue; + var column = PhysicalNaming.ResolveColumnName(field); + if (!seen.TryGetValue(column, out var first)) + { + seen[column] = field; + continue; + } + errors.Add(new MetaError( + $"fields \"{first.Name}\" and \"{field.Name}\" on \"{obj.Name}\" both resolve to " + + $"physical column \"{column}\". Rename one field, or give one an explicit @column " + + "so the two columns differ.", + ErrorCode.ERR_COLUMN_NAME_COLLISION, + Envelope: field.Source)); + } + } + return errors; + } + public static IReadOnlyList ValidateIndexLookupFields(MetaData root) { var errors = new List(); diff --git a/server/csharp/MetaObjects/Meta/PhysicalNaming.cs b/server/csharp/MetaObjects/Meta/PhysicalNaming.cs index e30f5f80d4..0844b9eb1a 100644 --- a/server/csharp/MetaObjects/Meta/PhysicalNaming.cs +++ b/server/csharp/MetaObjects/Meta/PhysicalNaming.cs @@ -61,6 +61,20 @@ public sealed record PhysicalNaming(ColumnNamingStrategy Columns, TableNamingRul public static PhysicalNaming Of(MetaData node) => node.Root() is MetaRoot root ? root.PhysicalNaming : Default; + /// A field's physical column name: its explicit @column (resolving, so an + /// inherited override counts), else the field name through the project's column strategy. + /// The loader's collision check compares THESE resolved names, not the raw field names. + public static string ResolveColumnName(MetaField field) + { + if (field.DbColumn is { Length: > 0 } explicitColumn) return explicitColumn; + return PhysicalNaming.Of(field).Columns switch + { + ColumnNamingStrategy.SnakeCase => SourceNaming.ToSnakeCase(field.Name), + ColumnNamingStrategy.KebabCase => SourceNaming.ToSnakeCase(field.Name).Replace('_', '-'), + _ => field.Name, + }; + } + /// cased by the table rule — no pluralization (the /// caller decides whether to pluralize; see ). public static string ApplyTableNamingRule(string name, TableNamingRule rule) => diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java index 9feb1f34f3..aaa48bb669 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/ExtractorCodeGenerator.java @@ -224,6 +224,6 @@ private static boolean isBlank(String s) { } private static String escape(String s) { - return s.replace("\\", "\\\\").replace("\"", "\\\""); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(s); } } diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java index cf7aa81d39..8c1f25cfce 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/javacode/JavaObjectCodeGenerator.java @@ -210,7 +210,7 @@ private void emitServicesRegistration() throws IOException { } private static String escape(String s) { - return s.replace("\\", "\\\\").replace("\"", "\\\""); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(s); } @Override diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java index 56e74ffb59..81419a280a 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/GeneratorUtil.java @@ -172,25 +172,31 @@ public static boolean filterByName( String metaDataName, String nameFilter) { return Pattern.compile(createRegexFromGlob(nameFilter)).matcher( metaDataName ).matches(); } + /** + * Glob to regex for the {@code } name surface: {@code *} any run (crossing {@code ::}), + * {@code @} one segment, {@code ?} one character. Every other character is literal, quoted so + * regex metacharacters in a name (e.g. {@code a::B(}) cannot throw or change the match. + * Deliberately not {@code Scope.compilePattern}: that grammar's {@code *} never crosses {@code ::}. + */ private static String createRegexFromGlob(String glob) { - String out = "^"; + StringBuilder out = new StringBuilder("^"); + StringBuilder literal = new StringBuilder(); for(int i = 0; i < glob.length(); ++i) { final char c = glob.charAt(i); - switch(c) - { - case '*': out += ".*"; break; - case '@': out += "[^:]+"; break; - case '?': out += "."; break; - case '.': out += "\\."; break; - case ':': out += "\\:"; break; // TODO: This doesn't seem to work on enforcing the ::'s as a separator for * - case '\\': out += "\\\\"; break; - default: out += c; - } + String wildcard = switch(c) { + case '*' -> ".*"; + case '@' -> "[^:]+"; + case '?' -> "."; + default -> null; + }; + if (wildcard == null) { literal.append(c); continue; } + if (literal.length() > 0) { out.append(Pattern.quote(literal.toString())); literal.setLength(0); } + out.append(wildcard); } - out += '$'; - return out; + if (literal.length() > 0) out.append(Pattern.quote(literal.toString())); + return out.append('$').toString(); } public static String toRelativePackage( String p1, String p2 ) { diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java index a461aa1897..0976f921b9 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/util/JavaIdentifiers.java @@ -94,4 +94,29 @@ public static String escapeMember(String name) { public static String escapeIdentifier(String name) { return isKeyword(name) ? name + "_" : name; } + + /** + * {@code s} escaped for the inside of a Java double-quoted string literal. Beyond the + * backslash and quote, a raw newline, carriage return or other control character would end + * the literal (or the file) and generate source that does not compile. + */ + public static String escapeStringLiteral(String s) { + StringBuilder out = new StringBuilder(s.length() + 8); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': out.append("\\\\"); break; + case '"': out.append("\\\""); break; + case '\n': out.append("\\n"); break; + case '\r': out.append("\\r"); break; + case '\t': out.append("\\t"); break; + case '\b': out.append("\\b"); break; + case '\f': out.append("\\f"); break; + default: + if (c < 0x20 || c == 0x7f) out.append(String.format("\\u%04x", (int) c)); + else out.append(c); + } + } + return out.toString(); + } } diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java index e3088a4768..012747754e 100644 --- a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/GeneratorUtilTest.java @@ -9,6 +9,15 @@ public class GeneratorUtilTest { + @Test + public void testFilterTreatsRegexMetacharactersAsLiterals() { + assertTrue( filterByName( "a::B(", "a::B(" )); + assertTrue( filterByName( "a::B[1]", "a::B[1]" )); + assertFalse( filterByName( "a::B1", "a::B[1]" )); + assertFalse( filterByName( "a::Bxc", "a::B.c" )); + assertTrue( filterByName( "a::B+c", "a::B+*" )); + } + @Test public void testFilterMatching() { assertTrue( filterByName( "acme::store::PrimaryStore", "acme::*" )); diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java new file mode 100644 index 0000000000..037dab4c5e --- /dev/null +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/util/JavaIdentifiersEscapeTest.java @@ -0,0 +1,20 @@ +package com.metaobjects.generator.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class JavaIdentifiersEscapeTest { + + @Test + public void escapesWhatWouldEndTheLiteral() { + assertEquals("a\\\\b\\\"c", JavaIdentifiers.escapeStringLiteral("a\\b\"c")); + assertEquals("line1\\nline2\\r\\t", JavaIdentifiers.escapeStringLiteral("line1\nline2\r\t")); + assertEquals("\\u0001x\\u007f", JavaIdentifiers.escapeStringLiteral("\u0001x\u007f")); + } + + @Test + public void leavesOrdinaryTextAlone() { + assertEquals("^[a-z]+é$", JavaIdentifiers.escapeStringLiteral("^[a-z]+é$")); + } +} diff --git a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt index 0fa26f99b0..07fbf9de42 100644 --- a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt +++ b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGenerator.kt @@ -23,6 +23,7 @@ import com.metaobjects.loader.MetaDataLoader import com.metaobjects.`object`.MetaObject import com.metaobjects.relationship.CompositionRelationship import com.metaobjects.relationship.MetaRelationship +import com.squareup.kotlinpoet.ClassName import com.squareup.kotlinpoet.LONG import com.squareup.kotlinpoet.TypeName import java.io.OutputStream @@ -143,9 +144,10 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() val api = exposedApi() val (pkg, ownerShort) = PackageMapping.splitFqn(entity.name) val ownerTable = ownerShort + "Table" - // TypeName.toString() yields the FQN (`kotlin.Long`, `java.util.UUID`, ...); - // grab the trailing simple name for the generated source. - val pkParamSimpleName = primaryKeyKotlinType(entity).toString().substringAfterLast('.') + // The generated source is text, so the PK type is rendered as a simple name plus + // the import it needs (a ClassName), not by slicing TypeName.toString(). + val pkRendered = renderType(primaryKeyKotlinType(entity)) + val pkParamSimpleName = pkRendered.name // FK column matches KotlinExposedTableGenerator.buildInverseFkSpec: Id. val fkColName = ownerShort.replaceFirstChar { it.lowercaseChar() } + "Id" @@ -205,9 +207,9 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() // reverse-FK value type) is uuid — the param types were already derived, but // without this import the generated file had an unresolved reference. Emitted // conditionally so uuid-free files stay byte-identical. - if (pkParamSimpleName == "UUID" || reverseFks.any { it.valueType == "UUID" }) { - append("import java.util.UUID\n") - } + val typeImports = (listOf(pkRendered.import) + reverseFks.map { it.valueImport }) + .filterNotNull().toSortedSet() + for (line in typeImports) append("import $line\n") val crossPackageTables = KotlinM2mSupport.importLines(m2mNavs.map { it.nav }, pkg, junction = true, dataClass = false) + helpers.filter { it.targetPackage != pkg } .map { if (it.targetPackage.isEmpty()) it.targetTable else "${it.targetPackage}.${it.targetTable}" } @@ -342,7 +344,26 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() * the FK field name, the target entity short name, and the Kotlin value type of the * FK (the type a `findBy` arg / element takes — the FK column's own Kotlin type). */ - protected data class ReverseFk(val fkField: String, val targetShortName: String, val valueType: String) + protected data class ReverseFk( + val fkField: String, + val targetShortName: String, + val valueType: String, + val valueImport: String? = null, + ) + + /** A Kotlin type as it is written into generated source: the name to print and the import it needs, if any. */ + protected data class RenderedType(val name: String, val import: String?) + + /** + * Render [type] for text emission. A [ClassName] prints its simple name and imports its + * canonical name (nothing for the auto-imported `kotlin` / `java.lang` packages). Any other + * shape (parameterized, nullable, ...) prints fully qualified, which is always valid. + */ + protected fun renderType(type: TypeName): RenderedType { + if (type !is ClassName || type.isNullable) return RenderedType(type.toString(), null) + val autoImported = type.packageName == "kotlin" || type.packageName == "java.lang" + return RenderedType(type.simpleNames.joinToString("."), if (autoImported) null else type.topLevelClassName().canonicalName) + } /** * Collect every reverse FK [entity] declares, in declaration order — one per @@ -360,11 +381,10 @@ open class KotlinRelationsGenerator : MultiFileDirectGeneratorBase() if (fields.size != 1) continue // single-column FKs only val fkField = fields.first() val targetShort = child.targetEntity?.let { PackageMapping.splitFqn(it).second } ?: "" - val valueType = entity.metaFields.firstOrNull { it.name == fkField }?.let { f -> - runCatching { KotlinTypeMapper.kotlinTypeName(f).toString().substringAfterLast('.') } - .getOrDefault("Long") - } ?: "Long" - out.add(ReverseFk(fkField, targetShort, valueType)) + val rendered = entity.metaFields.firstOrNull { it.name == fkField }?.let { f -> + runCatching { renderType(KotlinTypeMapper.kotlinTypeName(f)) }.getOrNull() + } ?: renderType(LONG) + out.add(ReverseFk(fkField, targetShort, rendered.name, rendered.import)) } return out } diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt index 6f6a860ad5..ebc1a0dc83 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt @@ -343,7 +343,7 @@ class KotlinNamesGeneratorTest { { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } @@ -514,7 +514,7 @@ class KotlinNamesGeneratorTest { { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "created_at" } }, + { "field.timestamp": { "name": "created_at", "@column": "created_at_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt index 4900d90f21..b8430107f9 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinRelationsGeneratorTest.kt @@ -139,4 +139,35 @@ class KotlinRelationsGeneratorTest { outDir.toFile().deleteRecursively() } } + + @Test fun `a non-uuid library-typed primary key imports its own type`() { + val fixture = """{ + "metadata.root": { "package": "acme::blog", "children": [ + { "object.entity": { "name": "Day", "children": [ + { "field.date": { "name": "id" } }, + { "relationship.composition": { + "name": "posts", "@objectRef": "Post", "@cardinality": "many" + } }, + { "source.rdb": { "@table": "days" } }, + { "identity.primary": { "name": "pk", "@fields": ["id"] } } + ] } }, + { "object.entity": { "name": "Post", "children": [ + { "field.long": { "name": "id" } }, + { "source.rdb": { "@table": "posts" } }, + { "identity.primary": { "name": "pk", "@fields": ["id"], "@generation": "increment" } } + ] } } + ] } + }""".trimIndent() + val outDir = Files.createTempDirectory("krel-") + try { + val gen = KotlinRelationsGenerator() + gen.setArgs(mapOf("outputDir" to outDir.toString())) + gen.execute(loadString("rel-date-pk", fixture)) + val src = Files.readString(outDir.resolve("acme/blog/DayRelations.kt")) + assertTrue("import java.time.LocalDate" in src, src) + assertTrue("dayId: LocalDate): Query" in src, src) + } finally { + outDir.toFile().deleteRecursively() + } + } } diff --git a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java index 8d5bdf62dd..cccb653d7a 100644 --- a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java +++ b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringGeneratorSupport.java @@ -635,7 +635,7 @@ public static boolean hasValidator(MetaField field, Class used, /** Escape a value for embedding inside a Java double-quoted string literal. */ private static String escapeJava(String value) { - StringBuilder sb = new StringBuilder(value.length() + 4); - for (int i = 0; i < value.length(); i++) { - char c = value.charAt(i); - switch (c) { - case '\\': sb.append("\\\\"); break; - case '"': sb.append("\\\""); break; - case '\t': sb.append("\\t"); break; - case '\n': sb.append("\\n"); break; - case '\r': sb.append("\\r"); break; - default: sb.append(c); - } - } - return sb.toString(); + return com.metaobjects.generator.util.JavaIdentifiers.escapeStringLiteral(value); } /** diff --git a/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java b/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java index f09f761385..efde171a77 100644 --- a/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java +++ b/server/java/codegen-spring/src/test/java/com/metaobjects/generator/spring/SpringNamesGeneratorTest.java @@ -322,7 +322,7 @@ public void aChildFieldCollidingWithAnInheritedOneIsRefusedNamingTheModel() thro { "object.entity": { "name": "Row", "extends": "BaseRow", "children": [ { "source.rdb": { "@table": "rows" } }, { "field.long": { "name": "id" } }, - { "field.timestamp": { "name": "created_at" } }, + { "field.timestamp": { "name": "created_at", "@column": "created_at_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } @@ -435,7 +435,7 @@ public void collidingScreamingSnakeMembersAreRefusedNamingTheModel() throws IOEx { "source.rdb": { "@table": "weirds" } }, { "field.long": { "name": "id" } }, { "field.string": { "name": "userId" } }, - { "field.string": { "name": "UserId" } }, + { "field.string": { "name": "UserId", "@column": "user_id_alt" } }, { "identity.primary": { "@fields": ["id"], "@generation": "increment" } } ] } } ] } diff --git a/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java b/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java index a3bf34d194..aab627e11d 100644 --- a/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java +++ b/server/java/maven-plugin/src/main/java/com/metaobjects/mojo/MetaDataEjectMojo.java @@ -1,7 +1,12 @@ package com.metaobjects.mojo; import com.metaobjects.generator.GeneratorRegistry; +import org.apache.maven.model.Build; import org.apache.maven.model.Dependency; +import org.apache.maven.model.Model; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -11,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -21,8 +27,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -456,7 +460,7 @@ private Path resolveRuntimeRoot() throws MojoFailureException { Path childPom = base.resolve(module).resolve("pom.xml"); try { if (Files.isRegularFile(childPom) - && Files.readString(childPom, StandardCharsets.UTF_8).contains("metaobjects-maven-plugin")) { + && declares(Files.readString(childPom, StandardCharsets.UTF_8), "metaobjects-maven-plugin")) { candidates.add(module); } } catch (IOException ex) { @@ -710,11 +714,40 @@ private String codegenPom(String artifactId, Set ports) { + "\n"; } - private static final Pattern SOURCE_DIRECTORY = - Pattern.compile("\\s*([^<]*?)\\s*"); + /** The pom parsed by Maven's own reader, or null when it is not well-formed. */ + static Model parsePom(String pom) { + try { + return new MavenXpp3Reader().read(new StringReader(pom), false); + } catch (IOException | XmlPullParserException ex) { + return null; + } + } + + /** + * Whether the pom declares {@code artifactId} as a dependency or a build plugin (managed or + * not). Read from the parsed model, so the name inside a comment or a property does not count. + */ + static boolean declares(String pom, String artifactId) { + Model model = parsePom(pom); + if (model == null) return false; + for (Dependency d : model.getDependencies()) { + if (artifactId.equals(d.getArtifactId())) return true; + } + Build build = model.getBuild(); + if (build == null) return false; + List plugins = new ArrayList<>(build.getPlugins()); + if (build.getPluginManagement() != null) plugins.addAll(build.getPluginManagement().getPlugins()); + for (Plugin plugin : plugins) { + if (artifactId.equals(plugin.getArtifactId())) return true; + } + return false; + } - private static boolean declares(String pom, String artifactId) { - return pom.contains("" + artifactId + ""); + /** {@code } as written (trimmed), or null when absent. */ + static String declaredSourceDirectory(String pom) { + Model model = parsePom(pom); + if (model == null || model.getBuild() == null || model.getBuild().getSourceDirectory() == null) return null; + return model.getBuild().getSourceDirectory().trim(); } /** @@ -741,9 +774,9 @@ private void reportWhatThePomLacks(Path codegenRoot) throws MojoExecutionExcepti changes.add(" Add to :\n" + kotlinPlugin(KOTLIN_SOURCE_DIRS)); } if (port == EjectSupport.Port.JAVA) { - Matcher sourceDirectory = SOURCE_DIRECTORY.matcher(content); - if (sourceDirectory.find() && !sourceDirectory.group(1).endsWith(sourceRoot)) { - changes.add(" Remove " + sourceDirectory.group() + " from , so that Maven compiles " + String sourceDirectory = declaredSourceDirectory(content); + if (sourceDirectory != null && !sourceDirectory.endsWith(sourceRoot)) { + changes.add(" Remove " + sourceDirectory + " from , so that Maven compiles " + sourceRoot + ", and add to the of " + KOTLIN_PLUGIN + ":\n" + KOTLIN_SOURCE_DIRS); } diff --git a/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java b/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java new file mode 100644 index 0000000000..a1127dbc04 --- /dev/null +++ b/server/java/maven-plugin/src/test/java/com/metaobjects/mojo/EjectPomReadingTest.java @@ -0,0 +1,41 @@ +package com.metaobjects.mojo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** Eject reads a pom with Maven's model reader: a name in a comment declares nothing. */ +public class EjectPomReadingTest { + + private static final String POM = "\n" + + " \n" + + " 4.0.0\n" + + " greal-dep\n" + + " \n" + + " \n" + + " \n src/main/java\n \n" + + " real-plugin\n" + + " \n" + + ""; + + @Test + public void commentedArtifactIsNotDeclared() { + assertFalse(MetaDataEjectMojo.declares(POM, "metaobjects-maven-plugin")); + assertTrue(MetaDataEjectMojo.declares(POM, "real-dep")); + assertTrue(MetaDataEjectMojo.declares(POM, "real-plugin")); + } + + @Test + public void sourceDirectoryIgnoresCommentsAndWhitespace() { + assertEquals("src/main/java", MetaDataEjectMojo.declaredSourceDirectory(POM)); + assertNull(MetaDataEjectMojo.declaredSourceDirectory("")); + } + + @Test + public void malformedPomDeclaresNothing() { + assertFalse(MetaDataEjectMojo.declares("", "real-dep")); + } +} diff --git a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java index 91a9221b49..674237ca50 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java +++ b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java @@ -492,6 +492,9 @@ public enum ErrorCode { * entity's effective (resolved, via {@code extends:}) field set. */ ERR_INVALID_INDEX, + /** Two fields of one stored object resolve to the same PHYSICAL column (snake_case folds + * {@code userId} and {@code userID} together, or two fields share an @column). */ + ERR_COLUMN_NAME_COLLISION, /** FR-039 — @status: retired declares @implementedBy. Refused rather than exempted: * a retired capability has no implementation BY DEFINITION, so forbidding the * attribute makes the dangling-reference class unreachable instead of tolerated. */ diff --git a/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java b/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java index 0a4dc9d013..ed4e91473a 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java +++ b/server/java/metadata/src/main/java/com/metaobjects/io/json/CanonicalJsonSerializer.java @@ -673,43 +673,24 @@ private static boolean getNativeIsArray(MetaData node, boolean effective) { // --------------------------------------------------------------------------- /** - * Returns {@code true} if the node's short name was auto-generated by the parser. + * Returns {@code true} if the node's short name was synthesised by the parser. * - *

Auto-naming applies only to {@code validator} and {@code view} type nodes. - * {@link com.metaobjects.loader.parser.BaseMetaDataParser} generates names of the - * exact form {@code } where {@code prefix} is the node's subType - * lowercased (or the type lowercased when there is no subType), and {@code N} is - * a positive integer. Detection therefore uses the exact expected prefix - * derived from this node's own type/subType — not a blanket pattern.

- * - *

This prevents silently dropping an explicitly authored name that happens to - * end in a digit (e.g. a validator named {@code length2} whose subType is - * {@code regex} would only be auto-detected as {@code regex\d+}, so {@code length2} - * is correctly treated as authored).

- * - *

All other type nodes require explicit names; if the short name is present - * it was authored and must be emitted.

+ *

Decided by the flag the parser sets at the moment it generates a name + * ({@link MetaData#isAutoNamed()}), never by the shape of the name: an authored + * name that merely looks generated (a validator authored as {@code required7}) + * is an authored name and must survive a canonical round trip. Only the + * auto-naming types ({@link BaseMetaDataParser#isAutoNamingType}) can carry a + * synthesised name; an empty name is the degenerate "no name" case.

*/ private static boolean isAutoGeneratedName(MetaData node) { - String typeName = node.getType(); - // Only the types in isAutoNamingType (validator, view, source, identity) use - // auto-naming — delegate to the single source of truth. - if (!BaseMetaDataParser.isAutoNamingType(typeName)) { + if (!BaseMetaDataParser.isAutoNamingType(node.getType())) { return false; } String shortName = node.getShortName(); if (shortName == null || shortName.isEmpty()) { - return true; // empty = auto (degenerate case) - } - // Derive the exact prefix the parser would have used: - // subType lowercased if non-null/non-empty, otherwise type lowercased. - String subType = node.getSubType(); - String expectedPrefix = (subType != null && !subType.isEmpty()) - ? subType.toLowerCase() - : typeName.toLowerCase(); - // Auto-generated name matches EXACTLY . - // Using Pattern.quote so any regex-special chars in the prefix are treated literally. - return shortName.matches("^" + Pattern.quote(expectedPrefix) + "\\d+$"); + return true; + } + return node.isAutoNamed(); } // --------------------------------------------------------------------------- diff --git a/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java b/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java index e898fd3346..80ce804d74 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java +++ b/server/java/metadata/src/main/java/com/metaobjects/library/LibrarySources.java @@ -1,5 +1,8 @@ package com.metaobjects.library; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.metaobjects.loader.FileSource; import com.metaobjects.loader.InMemoryStringSource; import com.metaobjects.loader.MetaDataSource; @@ -48,30 +51,32 @@ private LibrarySources() {} private static Map>> buildLayers() { Map>> map = new LinkedHashMap<>(); for (String name : new TreeSet<>(EmbeddedLibrary.MANIFESTS.keySet())) { - // Hand-parsed rather than pulled through Jackson: this module is the metadata - // core and does not depend on a JSON binder, and the shape read here is four - // keys deep in a file this repo generates. A binder would be a dependency - // added for a manifest we also write. map.put(name, parseLayers(EmbeddedLibrary.MANIFESTS.get(name))); } return map; } + private static JsonObject manifestRoot(String manifestJson) { + return JsonParser.parseString(manifestJson).getAsJsonObject(); + } + + /** A string array member as a list; absent or not an array gives empty. */ + private static List stringArray(JsonObject obj, String key) { + List out = new ArrayList<>(); + JsonElement el = obj.get(key); + if (el == null || !el.isJsonArray()) return out; + for (JsonElement item : el.getAsJsonArray()) out.add(item.getAsString()); + return out; + } + /** The {@code "layers"} object of a manifest: token to refs, in declaration order. */ - private static Map> parseLayers(String manifestJson) { + static Map> parseLayers(String manifestJson) { Map> layers = new LinkedHashMap<>(); - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"layers\"\\s*:\\s*\\{(.*?)\\n \\}", java.util.regex.Pattern.DOTALL) - .matcher(manifestJson); - if (!block.find()) return layers; - java.util.regex.Matcher entry = java.util.regex.Pattern - .compile("\"([^\"]*)\"\\s*:\\s*\\{[^}]*?\"refs\"\\s*:\\s*\\[([^\\]]*)\\]", java.util.regex.Pattern.DOTALL) - .matcher(block.group(1)); - while (entry.find()) { - List refs = new ArrayList<>(); - java.util.regex.Matcher ref = java.util.regex.Pattern.compile("\"([^\"]+)\"").matcher(entry.group(2)); - while (ref.find()) refs.add(ref.group(1)); - layers.put(entry.group(1), refs); + JsonElement block = manifestRoot(manifestJson).get("layers"); + if (block == null || !block.isJsonObject()) return layers; + for (Map.Entry entry : block.getAsJsonObject().entrySet()) { + if (!entry.getValue().isJsonObject()) continue; + layers.put(entry.getKey(), stringArray(entry.getValue().getAsJsonObject(), "refs")); } return layers; } @@ -96,19 +101,21 @@ private static Map> parseLayers(String manifestJson) { public static List generatorAnchors(String generatorName) { List out = new ArrayList<>(); for (String name : new TreeSet<>(EmbeddedLibrary.MANIFESTS.keySet())) { - String manifest = EmbeddedLibrary.MANIFESTS.get(name); - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"generators\"\\s*:\\s*\\[(.*?)\\]", java.util.regex.Pattern.DOTALL) - .matcher(manifest); - if (!block.find()) continue; - java.util.regex.Matcher obj = java.util.regex.Pattern - .compile("\\{([^}]*)\\}").matcher(block.group(1)); - while (obj.find()) { - String body = obj.group(1); - String declared = manifestField(body, "name"); - String anchor = manifestField(body, "anchor"); - if (generatorName.equals(declared) && anchor != null) out.add(anchor); - } + out.addAll(generatorAnchors(EmbeddedLibrary.MANIFESTS.get(name), generatorName)); + } + return out; + } + + static List generatorAnchors(String manifest, String generatorName) { + List out = new ArrayList<>(); + JsonElement block = manifestRoot(manifest).get("generators"); + if (block == null || !block.isJsonArray()) return out; + for (JsonElement el : block.getAsJsonArray()) { + if (!el.isJsonObject()) continue; + JsonObject gen = el.getAsJsonObject(); + String declared = manifestField(gen, "name"); + String anchor = manifestField(gen, "anchor"); + if (generatorName.equals(declared) && anchor != null) out.add(anchor); } return out; } @@ -119,27 +126,22 @@ public static List generatorAnchors(String generatorName) { * *

The provenance key for requirement-coverage activation (FR-043 §5.4): a * requirement whose package is NOT in this set was authored by the adopter. Read from the - * manifests, hand-parsed for the reason {@link #parseLayers} is, and never regenerated.

+ * manifests and never regenerated.

* * @return the declared package names, sorted */ public static java.util.Set libraryPackages() { java.util.Set out = new TreeSet<>(); for (String manifest : EmbeddedLibrary.MANIFESTS.values()) { - java.util.regex.Matcher block = java.util.regex.Pattern - .compile("\"packages\"\\s*:\\s*\\[([^\\]]*)\\]").matcher(manifest); - if (!block.find()) continue; - java.util.regex.Matcher pkg = java.util.regex.Pattern.compile("\"([^\"]+)\"").matcher(block.group(1)); - while (pkg.find()) out.add(pkg.group(1)); + out.addAll(stringArray(manifestRoot(manifest), "packages")); } return out; } - /** One {@code "key": "value"} string field out of a flat JSON object body. */ - private static String manifestField(String objectBody, String key) { - java.util.regex.Matcher m = java.util.regex.Pattern - .compile("\"" + key + "\"\\s*:\\s*\"([^\"]*)\"").matcher(objectBody); - return m.find() ? m.group(1) : null; + /** One string member of a JSON object, or null when absent or not a string. */ + private static String manifestField(JsonObject obj, String key) { + JsonElement el = obj.get(key); + return el != null && el.isJsonPrimitive() ? el.getAsString() : null; } /** The prefix every library source id carries. */ diff --git a/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java b/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java index d252e72e37..59c2199ea8 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java +++ b/server/java/metadata/src/main/java/com/metaobjects/loader/ValidationPhase.java @@ -287,6 +287,8 @@ public static void run(MetaRoot root, MetaDataLoader loader) { pass(collected, () -> validateFilterableHasSupportedOps(root)); pass(collected, () -> validateSortableHasSupportedSubtype(root)); pass(collected, () -> validateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + pass(collected, () -> validateColumnNameCollisions(root)); // FR-039 — a retired requirement carries no @implementedBy (refused, not exempted, // so the dangling-ref class is unreachable) and @supersededBy is legal only there. pass(collected, () -> validateRetiredRequirementLinks(root)); @@ -5043,6 +5045,9 @@ private static Set primaryAssignedFieldNames(MetaObject obj) { private static final Set NUMERIC_DISCRIMINATOR_SUBTYPES = Set.of( IntegerField.SUBTYPE_INT, LongField.SUBTYPE_LONG); + /** An optional minus sign and ASCII digits; matched whole, so no newline slips past. */ + private static final java.util.regex.Pattern INTEGER_LITERAL = java.util.regex.Pattern.compile("-?[0-9]+"); + static void validateDiscriminator(MetaRoot root) { List entities = new java.util.ArrayList<>(); for (MetaData rc : root.getChildren(MetaData.class, false)) { @@ -5089,7 +5094,7 @@ static void validateDiscriminator(MetaRoot root) { ErrorCode.ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH, obj.getSource()); } } else if (NUMERIC_DISCRIMINATOR_SUBTYPES.contains(st)) { - if (!value.matches("-?\\d+")) { + if (!INTEGER_LITERAL.matcher(value).matches()) { throw new MetaDataException( "ERR_DISCRIMINATOR_VALUE_TYPE_MISMATCH" + ": object.entity \"" + shortNameOf(obj) + "\" @discriminatorValue: \"" @@ -5107,10 +5112,16 @@ static void validateDiscriminator(MetaRoot root) { // Pass 3: ERR_DISCRIMINATOR_VALUE_DUPLICATE within each root's subtypes. for (Map.Entry> e : bindingsByRoot.entrySet()) { Map seen = new java.util.HashMap<>(); + String rootFieldName = ownAttrString(e.getKey(), MetaObject.ATTR_DISCRIMINATOR); + MetaField rootField = rootFieldName == null ? null : findFieldOnEntity(e.getKey(), rootFieldName); + boolean numericRoot = rootField != null && NUMERIC_DISCRIMINATOR_SUBTYPES.contains(rootField.getSubType()); for (Object[] b : e.getValue()) { MetaObject sub = (MetaObject) b[0]; String value = (String) b[1]; - MetaObject prev = seen.get(value); + // "1" and "01" are one value to a numeric column, so they are one claim. + String key = numericRoot && INTEGER_LITERAL.matcher(value).matches() + ? new java.math.BigInteger(value).toString() : value; + MetaObject prev = seen.get(key); if (prev != null) { throw new MetaDataException( "ERR_DISCRIMINATOR_VALUE_DUPLICATE" @@ -5119,7 +5130,7 @@ static void validateDiscriminator(MetaRoot root) { + shortNameOf(prev) + "\"", ErrorCode.ERR_DISCRIMINATOR_VALUE_DUPLICATE, sub.getSource()); } - seen.put(value, sub); + seen.put(key, sub); } } @@ -5327,6 +5338,39 @@ private static void walkRetiredRequirementLinks(MetaData node) { } } + /** + * ERR_COLUMN_NAME_COLLISION: two fields of one stored object resolving to the SAME + * physical column ({@code userId} + {@code userID} both snake_case to {@code user_id}, + * or two fields sharing an explicit {@code @column}). + * + *

Only the RESOLVED name is compared, through {@link com.metaobjects.database.ColumnNaming#resolve}, + * so an explicit {@code @column}, an inherited field and the project's column strategy are + * honoured. Scope: a concrete object declaring at least one source (a source-less object + * generates nothing); derived fields and {@code field.object} (may flatten to several + * columns) are skipped. ADR-0039: resolving accessors, so an inherited field collides like + * an own one.

+ */ + static void validateColumnNameCollisions(MetaRoot root) { + for (MetaObject obj : root.getChildren(MetaObject.class, false)) { + if (isAbstract(obj)) continue; + if (obj.getChildren(MetaSource.class, true).isEmpty()) continue; + java.util.Map> seen = new java.util.HashMap<>(); + for (MetaField field : obj.getChildren(MetaField.class, true)) { + if (ObjectField.SUBTYPE_OBJECT.equals(field.getSubType()) || field.isDerived()) continue; + String column = com.metaobjects.database.ColumnNaming.resolve(field); + MetaField first = seen.putIfAbsent(column, field); + if (first == null) continue; + throw new MetaDataException( + ErrorMessageConstants.ERR_COLUMN_NAME_COLLISION + + ": fields \"" + first.getName() + "\" and \"" + field.getName() + + "\" on \"" + obj.getName() + "\" both resolve to physical column \"" + + column + "\". Rename one field, or give one an explicit @column so the" + + " two columns differ.", + ErrorCode.ERR_COLUMN_NAME_COLLISION, field.getSource()); + } + } + } + static void validateIndexLookupFields(MetaRoot root) { for (MetaData rootChild : root.getChildren(MetaData.class, false)) { walkIndexLookupFields(rootChild); diff --git a/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java b/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java index 1613893814..92d0edc772 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java +++ b/server/java/metadata/src/main/java/com/metaobjects/source/MetaSource.java @@ -303,7 +303,7 @@ && hasMetaAttr(ATTR_TABLE)) { // auto-generated N names so a nameless authored source falls // through to Step 4 (matches TS, where unnamed sources have name === ""). String shortName = getShortName(); - if (shortName != null && !shortName.isEmpty() && !isAutoGeneratedSourceName(shortName)) { + if (shortName != null && !shortName.isEmpty() && !isAutoNamed()) { return com.metaobjects.database.PhysicalNaming.applyTableRule(shortName, tableRule); } @@ -322,28 +322,6 @@ && hasMetaAttr(ATTR_TABLE)) { return ""; } - /** - * Returns {@code true} when the given short name matches the parser's - * auto-naming pattern for source nodes ({@code N} — e.g. {@code rdb1}). - * Mirrors {@link com.metaobjects.io.json.CanonicalJsonSerializer}'s - * suppression of auto-names on emit so the FR-016 four-step rule treats - * authored vs auto-generated source names consistently across ports. - */ - private boolean isAutoGeneratedSourceName(String shortName) { - String subType = getSubType(); - String expectedPrefix = (subType != null && !subType.isEmpty()) - ? subType.toLowerCase() - : TYPE_SOURCE.toLowerCase(); - // Auto-generated name matches EXACTLY . - if (!shortName.startsWith(expectedPrefix)) return false; - String tail = shortName.substring(expectedPrefix.length()); - if (tail.isEmpty()) return false; - for (int i = 0; i < tail.length(); i++) { - if (!Character.isDigit(tail.charAt(i))) return false; - } - return true; - } - /** * FROZEN — the DEFAULT PHYSICAL name derivation only (step 4 above, reached only * when metadata declares no explicit physical name). Tiny inflector mirroring the diff --git a/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java b/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java index 6af6112fdd..bb6c03f774 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java +++ b/server/java/metadata/src/main/java/com/metaobjects/util/ErrorMessageConstants.java @@ -256,6 +256,9 @@ private ErrorMessageConstants() { */ public static final String ERR_INVALID_INDEX = "ERR_INVALID_INDEX"; + /** Cross-language contract: {@code ERR_COLUMN_NAME_COLLISION}. */ + public static final String ERR_COLUMN_NAME_COLLISION = "ERR_COLUMN_NAME_COLLISION"; + /** FR-039 — a retired requirement may not declare @implementedBy. */ public static final String ERR_REQUIREMENT_RETIRED_HAS_IMPLEMENTORS = "ERR_REQUIREMENT_RETIRED_HAS_IMPLEMENTORS"; diff --git a/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java b/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java index ff616ce08f..2388c84395 100644 --- a/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java +++ b/server/java/metadata/src/test/java/com/metaobjects/io/json/CanonicalJsonSerializerTest.java @@ -429,4 +429,21 @@ public void testAuthoredValidatorNameEndingInDigitIsEmitted() { assertTrue("authored validator name 'length2' must be emitted (not suppressed as auto-generated), got: " + json, json.contains("\"name\": \"length2\"")); } + + // An authored name that merely LOOKS auto-generated ("required7" on a required + // validator) is an authored name: suppression is decided by the parser's + // auto-named flag, never by the shape of the name. + @Test + public void testAuthoredNameMatchingAutoPatternIsEmitted() { + MetaRoot root = new MetaRoot("test::pkg"); + ValueMetaObject obj = ValueMetaObject.create("test::pkg::Widget"); + RegexValidator validator = new RegexValidator("regex7"); + obj.addChild(validator); + root.addChild(obj); + + String json = CanonicalJsonSerializer.canonicalSerialize(root); + + assertTrue("authored 'regex7' must survive canonical serialization, got: " + json, + json.contains("\"name\": \"regex7\"")); + } } diff --git a/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java b/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java new file mode 100644 index 0000000000..6e772fe8ec --- /dev/null +++ b/server/java/metadata/src/test/java/com/metaobjects/library/LibrarySourcesManifestTest.java @@ -0,0 +1,33 @@ +package com.metaobjects.library; + +import static org.junit.Assert.assertEquals; + +import java.util.List; +import java.util.Map; +import org.junit.Test; + +/** The manifest reader is a JSON parser, not a pattern: braces and quotes inside strings are data. */ +public class LibrarySourcesManifestTest { + + private static final String MANIFEST = "{\n" + + " \"description\": \"has } and ] and \\\"layers\\\" inside\",\n" + + " \"generators\": [ { \"anchor\": \"a::B\", \"name\": \"g\", \"note\": \"x}y\" } ],\n" + + " \"layers\": {\n" + + " \"\": { \"description\": \"core } here\", \"refs\": [\"x/model\", \"x/req\"] },\n" + + " \"db\": { \"refs\": [\"x/db\"] }\n" + + " }\n" + + "}"; + + @Test + public void layersSurviveBracesInsideStrings() { + Map> layers = LibrarySources.parseLayers(MANIFEST); + assertEquals(List.of("x/model", "x/req"), layers.get("")); + assertEquals(List.of("x/db"), layers.get("db")); + } + + @Test + public void generatorAnchorReadsKeysInAnyOrder() { + assertEquals(List.of("a::B"), LibrarySources.generatorAnchors(MANIFEST, "g")); + assertEquals(List.of(), LibrarySources.generatorAnchors(MANIFEST, "other")); + } +} diff --git a/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java new file mode 100644 index 0000000000..42cf13579a --- /dev/null +++ b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java @@ -0,0 +1,127 @@ +package com.metaobjects.render; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; + +/** + * One Mustache tag scanner shared by {@link Verify} and the renderer's partial + * expansion. + * + *

Both used to find tags on their own ({@code Verify} with a {@code {{} search, + * {@code Renderer} with a {@code {{> x}}} regex), and neither knew what the engine + * knows: a comment ends at the first closing delimiter, {@code {{=<% %>=}}} changes + * the delimiters, and a section close must name the section it closes. {@link #scan} + * follows the Mustache spec for all three, so the two callers cannot disagree about + * where a tag is. Mirrors {@code Mustache.parse} in the TypeScript port and the C# and + * Python scanners. + */ +final class MustacheTags { + + enum Kind { NAME, TRIPLE, AMP, SECTION, INVERTED, CLOSE, PARTIAL, COMMENT, DELIMITERS } + + /** One tag: its kind, trimmed name, and {@code [start, end)} span in the text. */ + record Tag(Kind kind, String value, int start, int end) {} + + /** An unbalanced section; {@link #section()} is the section the finding is about. */ + static final class SyntaxException extends RuntimeException { + private final String section; + + SyntaxException(String message, String section) { + super(message); + this.section = section; + } + + String section() { + return section; + } + } + + private static final String DEFAULT_OPEN = "{{"; + private static final String DEFAULT_CLOSE = "}}"; + + private MustacheTags() {} + + /** + * Every tag in {@code text}, in order, honouring comments and delimiter changes. + * Throws {@link SyntaxException} for a close with no open, a close naming a different + * section than the one open, or a section never closed. An unterminated tag ends the + * scan (the rest is text), as it always did here. + */ + static List scan(String text) { + List tags = new ArrayList<>(); + String open = DEFAULT_OPEN; + String close = DEFAULT_CLOSE; + Deque openSections = new ArrayDeque<>(); + int pos = 0; + while (pos < text.length()) { + int start = text.indexOf(open, pos); + if (start < 0) break; + int contentStart = start + open.length(); + boolean triple = contentStart < text.length() && text.charAt(contentStart) == '{'; + String endDelim = triple ? "}" + close : close; + if (triple) contentStart++; + int closeIdx = text.indexOf(endDelim, contentStart); + if (closeIdx < 0) break; + String content = text.substring(contentStart, closeIdx).trim(); + int end = closeIdx + endDelim.length(); + pos = end; + + if (triple) { + tags.add(new Tag(Kind.TRIPLE, content, start, end)); + continue; + } + if (content.isEmpty()) continue; + char sigil = content.charAt(0); + String name = content.length() > 1 ? content.substring(1).trim() : ""; + switch (sigil) { + case '!' -> tags.add(new Tag(Kind.COMMENT, name, start, end)); + case '=' -> { + if (content.endsWith("=")) { + String[] parts = content.substring(1, content.length() - 1).trim().split("\\s+"); + if (parts.length == 2) { + open = parts[0]; + close = parts[1]; + } + } + tags.add(new Tag(Kind.DELIMITERS, name, start, end)); + } + case '#', '^' -> { + openSections.push(name); + tags.add(new Tag(sigil == '#' ? Kind.SECTION : Kind.INVERTED, name, start, end)); + } + case '/' -> { + if (openSections.isEmpty()) { + throw new SyntaxException("Unopened section \"" + name + "\"", name); + } + String opened = openSections.pop(); + if (!opened.equals(name)) { + throw new SyntaxException("Unclosed section \"" + opened + "\"", opened); + } + tags.add(new Tag(Kind.CLOSE, name, start, end)); + } + case '>' -> tags.add(new Tag(Kind.PARTIAL, name, start, end)); + case '&' -> tags.add(new Tag(Kind.AMP, name, start, end)); + default -> tags.add(new Tag(Kind.NAME, content, start, end)); + } + } + if (!openSections.isEmpty()) { + String opened = openSections.peek(); + throw new SyntaxException("Unclosed section \"" + opened + "\"", opened); + } + return tags; + } + + /** {@code text} with every comment cut out: what a tag-presence check may scan. */ + static String withoutComments(String text, List tags) { + StringBuilder sb = new StringBuilder(text.length()); + int pos = 0; + for (Tag tag : tags) { + if (tag.kind() != Kind.COMMENT) continue; + sb.append(text, pos, tag.start()); + pos = tag.end(); + } + return sb.append(text, pos, text.length()).toString(); + } +} diff --git a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java index 53d8825390..8c7e615438 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java +++ b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java @@ -10,9 +10,8 @@ import java.io.Writer; import java.util.ArrayDeque; import java.util.Deque; +import java.util.List; import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Mustache-driven render pipeline for the {@code template.*} metatype (FR-004). @@ -35,10 +34,6 @@ public final class Renderer { static final int MAX_DEPTH = 32; - // {{> name }} — captures the reference between the marker and the closing }}. - private static final Pattern PARTIAL_PATTERN = - Pattern.compile("\\{\\{>\\s*([^\\s}]+)\\s*\\}\\}"); - public String render(RenderRequest req) { validate(req); @@ -122,14 +117,27 @@ private static String preExpandPartials(String text, Provider provider, Deque ", stack) + ")"); } - Matcher m = PARTIAL_PATTERN.matcher(text); - if (!m.find()) return text; - m.reset(); + // Partial tags are located by the shared scanner, so one written inside a comment + // is not a partial and a {{=<% %>=}} delimiter change is honoured. + List tags; + try { + tags = MustacheTags.scan(text); + } catch (MustacheTags.SyntaxException e) { + throw new RenderException("template syntax error: " + e.getMessage()); + } + List partials = tags.stream() + .filter(t -> t.kind() == MustacheTags.Kind.PARTIAL).toList(); + if (partials.isEmpty()) return text; + // Inlined text is parsed under the surrounding delimiters, not the default ones. + if (tags.stream().anyMatch(t -> t.kind() == MustacheTags.Kind.DELIMITERS)) { + throw new RenderException( + "a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + } StringBuilder out = new StringBuilder(text.length()); int last = 0; - while (m.find()) { - String partialRef = m.group(1); + for (MustacheTags.Tag tag : partials) { + String partialRef = tag.value(); if (stack.contains(partialRef)) { throw new RenderException("partial cycle: " + String.join(" -> ", stack) + " -> " + partialRef); @@ -144,9 +152,9 @@ private static String preExpandPartials(String text, Provider provider, Deque check( // The static text the output-tag check scans: the body plus every // provider-resolved partial body, collected during the single walk below. List staticTexts = new ArrayList<>(); - staticTexts.add(templateText); + + List bodyToks; + try { + bodyToks = tokenize(templateText); + } catch (MustacheTags.SyntaxException e) { + errors.add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); + return errors; + } + staticTexts.add(staticText(templateText)); WalkContext ctx = new WalkContext(provider, root, errors, referencedAtRoot, staticTexts); - walk(tokenize(templateText), List.of(root), List.of(), ctx, false); + walk(bodyToks, List.of(root), List.of(), ctx, false); if (opts.requiredSlots() != null) { for (String slot : opts.requiredSlots()) { @@ -217,10 +229,17 @@ private static void walk( ctx.errors().add(new VerifyError(ERR_PARTIAL_UNRESOLVED, p.value())); break; } - ctx.staticTexts().add(text); + List partialToks; + try { + partialToks = tokenize(text); + } catch (MustacheTags.SyntaxException e) { + ctx.errors().add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); + break; + } + ctx.staticTexts().add(staticText(text)); List nextSeen = new ArrayList<>(seen); nextSeen.add(p.value()); - walk(tokenize(text), stack, Collections.unmodifiableList(nextSeen), ctx, inElement); + walk(partialToks, stack, Collections.unmodifiableList(nextSeen), ctx, inElement); } } } @@ -288,61 +307,32 @@ private static boolean isBooleanAccessor(List> stack, String * cares about: variables (incl. {@code {{&x}}} and {@code {{{x}}}}), * sections + inverted sections (with nested children), and partials. * Comments and set-delimiter changes are dropped; plain text is skipped. + * Tags come from the shared scanner ({@link MustacheTags}); an unbalanced + * section throws {@link MustacheTags.SyntaxException}. */ static List tokenize(String text) { - int[] pos = {0}; - return parseTokens(text, pos); - } - - /** - * Parse tokens until EOF or a matching {@code {{/...}}} (which is consumed). - * Mirrors C# {@code Verify.cs ParseTokens()} — note that any close tag - * (matched or mismatched) ends the current level, which is intentionally - * lenient (templates that drift to imbalanced sections still tokenize). - */ - private static List parseTokens(String text, int[] pos) { - List tokens = new ArrayList<>(); - while (pos[0] < text.length()) { - int open = text.indexOf("{{", pos[0]); - if (open < 0) break; // remainder is plain text - boolean triple = open + 2 < text.length() && text.charAt(open + 2) == '{'; - String closeDelim = triple ? "}}}" : "}}"; - int contentStart = open + (triple ? 3 : 2); - int close = text.indexOf(closeDelim, contentStart); - if (close < 0) break; // malformed; stop - String content = text.substring(contentStart, close).trim(); - pos[0] = close + closeDelim.length(); - - if (triple) { - tokens.add(new VarTok(content)); - continue; - } - if (content.isEmpty()) continue; - - char sigil = content.charAt(0); - String name = content.length() > 1 ? content.substring(1).trim() : ""; - switch (sigil) { - case '!': - break; // comment - case '#': - case '^': - List children = parseTokens(text, pos); - tokens.add(new SectionTok(name, sigil == '^', children)); - break; - case '/': - return tokens; // close (matched or mismatched) ends this level - case '>': - tokens.add(new PartialTok(name)); - break; - case '&': - tokens.add(new VarTok(name)); - break; - default: - tokens.add(new VarTok(content)); // {{x}}, {{x.y}}, {{.}} - break; + List root = new ArrayList<>(); + Deque> levels = new ArrayDeque<>(); + levels.push(root); + for (MustacheTags.Tag tag : MustacheTags.scan(text)) { + switch (tag.kind()) { + case SECTION, INVERTED -> { + List children = new ArrayList<>(); + levels.peek().add(new SectionTok(tag.value(), tag.kind() == MustacheTags.Kind.INVERTED, children)); + levels.push(children); + } + case CLOSE -> levels.pop(); // the scanner already matched it to its open + case PARTIAL -> levels.peek().add(new PartialTok(tag.value())); + case NAME, TRIPLE, AMP -> levels.peek().add(new VarTok(tag.value())); // {{x}}, {{x.y}}, {{.}} + default -> { } // comments and delimiter changes carry no reference } } - return tokens; + return root; + } + + /** {@code text} minus its comments, for the output-tag scan. */ + private static String staticText(String text) { + return MustacheTags.withoutComments(text, MustacheTags.scan(text)); } // ---------------- Required-tag matching ---------------- diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java b/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java index 1d98e9dda8..6f7692d165 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/Locate.java @@ -77,15 +77,26 @@ private static int scanBalanced(String s, int open) { return -1; } - /** Span of ...; if close absent, opener to end; null if no opener. */ + /** + * Span of ...; if close absent, opener to end; null if no opener. Comments and + * CDATA are not searched, and when another opener of the same root sits between the first + * one and its close (the tag mentioned in prose, then the real element) the span starts at + * the LAST such opener: a root does not nest in itself. + */ public static String xml(String text, String rootName, boolean caseInsensitive) { if (text == null || rootName == null) return null; int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0; - Matcher open = Pattern.compile("<" + Pattern.quote(rootName) + "(\\s[^>]*)?>", flags).matcher(text); + String view = XmlForgivingReader.maskNonMarkup(text); + Matcher open = Pattern.compile("<" + Pattern.quote(rootName) + "(\\s[^>]*)?>", flags).matcher(view); if (!open.find()) return null; int start = open.start(); - Matcher close = Pattern.compile("", flags).matcher(text); - if (close.find(open.end())) return text.substring(start, close.end()); - return text.substring(start); + Matcher close = Pattern.compile("", flags).matcher(view); + if (!close.find(open.end())) return text.substring(start); + int at = open.end(); + while (open.find(at) && open.start() < close.start()) { + start = open.start(); + at = open.end(); + } + return text.substring(start, close.end()); } } diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java index 3d7f285123..d5495b546b 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java @@ -4,6 +4,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,14 +51,113 @@ public final class XmlForgivingReader { private static final Pattern ATTR = Pattern.compile( "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"); - public Map read(String span, boolean caseInsensitive) { + private static final String COMMENT_OPEN = ""; + private static final String CDATA_OPEN = ""; + // the five predefined XML entities plus numeric character references. + private static final Pattern ENTITY = Pattern.compile("&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);"); + + private static final String KIND_COMMENT = "comment"; + private static final String KIND_CDATA = "cdata"; + + /** + * Rewrite every XML comment and CDATA section in {@code text} through {@code replace} + * (kind, region), scanning left to right so a comment marker inside CDATA (or the reverse) + * is never mistaken for markup. An unclosed comment is left as text; an unclosed CDATA + * section runs to the end. + */ + private static String rewriteRegions(String text, BiFunction replace) { + StringBuilder out = new StringBuilder(); + int pos = 0; + int from = 0; + while (true) { + int c = text.indexOf(COMMENT_OPEN, from); + int d = text.indexOf(CDATA_OPEN, from); + boolean isComment = c >= 0 && (d < 0 || c < d); + int start = isComment ? c : d; + if (start < 0) break; + int end; + if (isComment) { + int e = text.indexOf(COMMENT_CLOSE, start + COMMENT_OPEN.length()); + if (e < 0) { + from = start + COMMENT_OPEN.length(); + continue; + } + end = e + COMMENT_CLOSE.length(); + } else { + int e = text.indexOf(CDATA_CLOSE, start + CDATA_OPEN.length()); + end = e < 0 ? text.length() : e + CDATA_CLOSE.length(); + } + out.append(text, pos, start) + .append(replace.apply(isComment ? KIND_COMMENT : KIND_CDATA, text.substring(start, end))); + pos = end; + from = end; + } + return out.append(text.substring(pos)).toString(); + } + + /** {@code text} with comments and CDATA sections blanked to spaces (same length, so + * indices hold): the view the tag locators search, so markup inside them is never located. */ + static String maskNonMarkup(String text) { + return rewriteRegions(text, (kind, region) -> " ".repeat(region.length())); + } + + /** Drop comments; turn each CDATA section into its text with {@code & < >} entity-escaped, + * so the tag scanner cannot split it and the entity decode restores it exactly. */ + private static String prepare(String text) { + return rewriteRegions(text, (kind, region) -> { + if (kind.equals(KIND_COMMENT)) return ""; + String body = region.endsWith(CDATA_CLOSE) + ? region.substring(CDATA_OPEN.length(), region.length() - CDATA_CLOSE.length()) + : region.substring(CDATA_OPEN.length()); + return body.replace("&", "&").replace("<", "<").replace(">", ">"); + }); + } + + /** Decode predefined entities and numeric references in one pass (so {@code &lt;} is + * {@code <}). An unknown or out-of-range reference stays as written. */ + private static String decodeEntities(String s) { + if (s.indexOf('&') < 0) return s; + Matcher m = ENTITY.matcher(s); + StringBuilder sb = new StringBuilder(); + int last = 0; + while (m.find()) { + String body = m.group(1); + String rep = m.group(); + switch (body) { + case "lt" -> rep = "<"; + case "gt" -> rep = ">"; + case "amp" -> rep = "&"; + case "quot" -> rep = "\""; + case "apos" -> rep = "'"; + default -> { + try { + int cp = body.charAt(1) == 'x' + ? Integer.parseInt(body.substring(2), 16) + : Integer.parseInt(body.substring(1), 10); + boolean surrogate = cp >= 0xD800 && cp <= 0xDFFF; + if (cp >= 1 && cp <= 0x10FFFF && !surrogate) rep = new String(Character.toChars(cp)); + } catch (NumberFormatException ignored) { + // out-of-range reference: stays as written + } + } + } + sb.append(s, last, m.start()).append(rep); + last = m.end(); + } + return sb.append(s.substring(last)).toString(); + } + + public Map read(String rawSpan, boolean caseInsensitive) { Map out = new LinkedHashMap<>(); - if (span == null || span.isBlank()) return out; + if (rawSpan == null || rawSpan.isBlank()) return out; + String span = prepare(rawSpan); int gt = span.indexOf('>'); if (gt < 0) return out; int rootEnd = span.lastIndexOf(" read(String span, boolean caseInsensitive) { public Map readRootless(String text, boolean caseInsensitive) { Map out = new LinkedHashMap<>(); if (text == null || text.isBlank()) return out; - parseChildren(text, caseInsensitive, out, null); + parseChildren(prepare(text), caseInsensitive, out, null, false); return out; } /** * Parse {@code inner}'s elements into {@code out}. When {@code loose} is non-null, the text * BETWEEN those elements (an element's own text in mixed content) is appended to it, one - * segment per gap. + * segment per gap. {@code literalUnclosed} is set for the body of an element that DID + * close: an open tag in there with no close tag anywhere after it ({@code List and + * ...}) is prose, not an element. */ - private void parseChildren(String inner, boolean ci, Map out, List loose) { + private void parseChildren(String inner, boolean ci, Map out, List loose, + boolean literalUnclosed) { Pattern openTag = Pattern.compile(OPEN_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); + Pattern closeTag = Pattern.compile(CLOSE_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); Matcher m = openTag.matcher(inner); int pos = 0; - while (m.find(pos)) { + int scan = 0; + while (m.find(scan)) { + if (literalUnclosed && !m.group(2).stripTrailing().endsWith("/") + && !closeTag.matcher(inner).region(m.end(), inner.length()).find()) { + scan = m.end(); + continue; + } if (loose != null) loose.add(inner.substring(pos, m.start())); String tag = m.group(1); String key = ci ? tag.toLowerCase() : tag; @@ -96,6 +206,7 @@ private void parseChildren(String inner, boolean ci, Map out, Li if (selfClosing) { accumulate(out, key, attrs.isEmpty() ? "" : attrs); pos = m.end(); + scan = pos; continue; } @@ -103,7 +214,8 @@ private void parseChildren(String inner, boolean ci, Map out, Li String closeRe = ""; Matcher close = Pattern.compile(closeRe, ci ? Pattern.CASE_INSENSITIVE : 0).matcher(inner); int contentEnd, next; - if (close.find(contentStart)) { + boolean closed = close.find(contentStart); + if (closed) { contentEnd = close.start(); next = close.end(); } else { @@ -141,8 +253,9 @@ private void parseChildren(String inner, boolean ci, Map out, Li } } String content = inner.substring(contentStart, contentEnd); - accumulate(out, key, combine(attrs, content, ci)); + accumulate(out, key, combine(attrs, content, ci, closed)); pos = next; + scan = next; } if (loose != null) loose.add(inner.substring(pos)); } @@ -171,11 +284,11 @@ private static int[] findStrayClose(String inner, int from, int to, boolean ci) * content keeps BOTH: the children, and the element's own text under {@link #TEXT_KEY}, so * a scalar or {@code @xmlText} consumer still reads the prose around a child element. */ - private Object combine(Map attrs, String content, boolean ci) { + private Object combine(Map attrs, String content, boolean ci, boolean closed) { if (content.contains("<")) { Map nested = new LinkedHashMap<>(); List loose = new ArrayList<>(); - parseChildren(content, ci, nested, loose); + parseChildren(content, ci, nested, loose, closed); if (!nested.isEmpty()) { // attributes first; a child element takes precedence on a name collision Map merged = new LinkedHashMap<>(attrs); @@ -194,7 +307,7 @@ private static String mixedText(List segments, boolean ci) { Pattern close = Pattern.compile(CLOSE_TAG_RE, ci ? Pattern.CASE_INSENSITIVE : 0); List parts = new ArrayList<>(); for (String seg : segments) { - String t = close.matcher(seg).replaceAll("").trim(); + String t = decodeEntities(close.matcher(seg).replaceAll("").trim()); if (!t.isEmpty()) parts.add(t); } return String.join(" ", parts); @@ -203,7 +316,7 @@ private static String mixedText(List segments, boolean ci) { /** A text body: bare {@code String} when there are no attributes, else a map carrying the * attributes plus the text under {@link #TEXT_KEY}. */ private Object textValue(Map attrs, String content) { - String text = content.trim(); + String text = decodeEntities(content.trim()); if (attrs.isEmpty()) return text; Map m = new LinkedHashMap<>(attrs); m.put(TEXT_KEY, text); @@ -219,7 +332,7 @@ private Map parseAttrs(String rawAttrs, boolean ci) { String val = a.group(2) != null ? a.group(2) : a.group(3) != null ? a.group(3) : a.group(4) != null ? a.group(4) : ""; - attrs.putIfAbsent(name, val); + attrs.putIfAbsent(name, decodeEntities(val)); } return attrs; } diff --git a/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java b/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java index 09e59143ab..730876e30c 100644 --- a/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java +++ b/server/java/render/src/test/java/com/metaobjects/render/VerifyConformanceTest.java @@ -58,7 +58,8 @@ public class VerifyConformanceTest { Verify.ERR_VAR_NOT_ON_PAYLOAD, Verify.ERR_PARTIAL_UNRESOLVED, Verify.ERR_REQUIRED_SLOT_UNUSED, - Verify.ERR_OUTPUT_TAG_MISSING + Verify.ERR_OUTPUT_TAG_MISSING, + Verify.ERR_TEMPLATE_SYNTAX ); static { diff --git a/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt b/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt new file mode 100644 index 0000000000..abfd547468 --- /dev/null +++ b/server/java/render/src/test/resources/snapshots/render-comment-with-partial.txt @@ -0,0 +1 @@ +Hi Ada }} \ No newline at end of file diff --git a/server/python/src/metaobjects/cli.py b/server/python/src/metaobjects/cli.py index 79204d894d..06817f3b69 100644 --- a/server/python/src/metaobjects/cli.py +++ b/server/python/src/metaobjects/cli.py @@ -1877,7 +1877,9 @@ def _relative_set(root: Path, nested: list[Path] | None = None) -> dict[str, str if p.suffix in _UNCOMPARED_SUFFIXES: continue try: - files[str(rel)] = p.read_text(encoding="utf-8") + # Bytes, not read_text(): text mode translates \r\n and lone \r to \n, so a + # committed CRLF file would compare equal to the LF the generator wrote. + files[str(rel)] = p.read_bytes().decode("utf-8") except (UnicodeDecodeError, OSError): continue return files diff --git a/server/python/src/metaobjects/codegen/generators/entity_model.py b/server/python/src/metaobjects/codegen/generators/entity_model.py index 42587b2eec..f2b221e907 100644 --- a/server/python/src/metaobjects/codegen/generators/entity_model.py +++ b/server/python/src/metaobjects/codegen/generators/entity_model.py @@ -830,6 +830,16 @@ def _render_shared_enums_module(self, entities: list[MetaObject]) -> EmittedFile # Member names follow the Python convention (UPPER_CASE constants); the # value preserves the wire form, so ``Enum("statistical")`` and # ``member == "statistical"`` still work (str-enum). + names: dict[str, str] = {} + for m in e.values: + # Two values folding to one member name ('Open' / 'open') would make the + # second an alias of the first at import time, silently dropping a value. + if m.upper() in names: + raise ValueError( + f"shared enum '{e.name}': values '{names[m.upper()]}' and '{m}' both map " + f"to the Python member {m.upper()}; rename one so the members are distinct" + ) + names[m.upper()] = m parts += [f' {m.upper()} = "{m}"' for m in e.values] parts.append("") return EmittedFile(path="enums.py", content=ruff_format("\n".join(parts))) diff --git a/server/python/src/metaobjects/codegen/project_config.py b/server/python/src/metaobjects/codegen/project_config.py index 3f98021eed..8f57b78150 100644 --- a/server/python/src/metaobjects/codegen/project_config.py +++ b/server/python/src/metaobjects/codegen/project_config.py @@ -30,6 +30,7 @@ from metaobjects.codegen.config import DEFAULT_API_PREFIX from metaobjects.naming import removed_naming_source_message +from metaobjects.source.yaml_core_schema import safe_load_core #: Config filename looked up in the cwd when ``--config`` is not given. CONFIG_FILENAME = "metaobjects.config.yaml" @@ -374,7 +375,7 @@ def load_project_config(path: Path) -> ProjectConfig: except (OSError, UnicodeDecodeError) as exc: raise ConfigError(f"{path}: cannot read config file: {exc}") from exc try: - raw = yaml.safe_load(text) + raw = safe_load_core(text) except yaml.YAMLError as exc: raise ConfigError(f"{path}: invalid YAML: {exc}") from exc diff --git a/server/python/src/metaobjects/errors.py b/server/python/src/metaobjects/errors.py index 0e27deaa5c..d3934bb78c 100644 --- a/server/python/src/metaobjects/errors.py +++ b/server/python/src/metaobjects/errors.py @@ -201,6 +201,9 @@ class ErrorCode(str, Enum): # Cross-language vocabulary for features other ports added (FR-003 storage, FR-004 template); # the Python loader does not emit these yet, but the enum tracks the shared corpus codes. ERR_INVALID_TEMPLATE = "ERR_INVALID_TEMPLATE" + # Two fields of one stored object resolve to the same PHYSICAL column (snake_case + # folds `userId` and `userID` together, or two fields share an @column). + ERR_COLUMN_NAME_COLLISION = "ERR_COLUMN_NAME_COLLISION" ERR_STORAGE_FLATTENED_ARRAY = "ERR_STORAGE_FLATTENED_ARRAY" ERR_STORAGE_WITHOUT_OBJECT_REF = "ERR_STORAGE_WITHOUT_OBJECT_REF" # ADR-0013: a field.object REQUIRES @objectRef (open/untyped JSON uses the diff --git a/server/python/src/metaobjects/loader/validate_discriminator.py b/server/python/src/metaobjects/loader/validate_discriminator.py index 055ffbd79c..08adbd58f3 100644 --- a/server/python/src/metaobjects/loader/validate_discriminator.py +++ b/server/python/src/metaobjects/loader/validate_discriminator.py @@ -38,7 +38,8 @@ _NUMERIC_DISCRIMINATOR_SUBTYPES = frozenset( {FIELD_SUBTYPE_INT, FIELD_SUBTYPE_LONG} ) -_INT_RE = re.compile(r"^-?\d+$") +# ASCII digits only and `\Z`: `\d` matches every Unicode digit and `$` matches before a trailing newline. +_INT_RE = re.compile(r"-?[0-9]+\Z") def _find_field_on_entity(entity: MetaData, name: str) -> MetaData | None: @@ -154,8 +155,13 @@ def validate_discriminator(root: MetaData, errors: list[MetaError]) -> None: # Pass 3: ERR_DISCRIMINATOR_VALUE_DUPLICATE within each root's subtypes. for _disc_root, bindings in bindings_by_root: seen: dict[str, MetaData] = {} + root_field_name = _disc_root.attr(OBJECT_ATTR_DISCRIMINATOR) # Python attr() is own-only: the root declares it + root_field = _find_field_on_entity(_disc_root, root_field_name) if isinstance(root_field_name, str) else None + numeric_root = root_field is not None and root_field.sub_type in _NUMERIC_DISCRIMINATOR_SUBTYPES for subtype, value in bindings: - prev = seen.get(value) + # "1" and "01" are one value to a numeric column, so they are one claim. + key = str(int(value)) if numeric_root and _INT_RE.match(value) else value + prev = seen.get(key) if prev is not None: errors.append( MetaError( @@ -167,7 +173,7 @@ def validate_discriminator(root: MetaData, errors: list[MetaError]) -> None: ) ) else: - seen[value] = subtype + seen[key] = subtype # Pass 4: ERR_DISCRIMINATOR_VALUE_MISSING — every concrete entity that extends # a @discriminator-bearing root must declare a value. diff --git a/server/python/src/metaobjects/loader/validation_passes.py b/server/python/src/metaobjects/loader/validation_passes.py index 47c02628b5..7c8cf6c2fb 100644 --- a/server/python/src/metaobjects/loader/validation_passes.py +++ b/server/python/src/metaobjects/loader/validation_passes.py @@ -261,6 +261,8 @@ def run_validations( # #335 Half B — @sortable on an array field, or a subtype with no operator band → error. _validate_sortable_has_supported_subtype(root, errors) _validate_index_lookup_fields(root, errors) + # Two fields of one stored object resolving to the same physical column. + _validate_column_name_collisions(root, errors) # FR-039 -- a retired requirement carries no @implementedBy (refused, not exempted, # so the dangling-ref class is unreachable) and @supersededBy is legal only there. _validate_retired_requirement_links(root, errors) @@ -4191,6 +4193,45 @@ def _validate_index_lookup_fields(root: MetaData, errors: list[MetaError]) -> No # --------------------------------------------------------------------------- +def _validate_column_name_collisions(root: MetaData, errors: list[MetaError]) -> None: + """ERR_COLUMN_NAME_COLLISION: two fields of one stored object resolving to the SAME + physical column (``userId`` + ``userID`` both snake_case to ``user_id``, or two + fields sharing an explicit ``@column``). + + Only the RESOLVED name is compared, through :func:`resolve_column_name`, so an + explicit ``@column``, an inherited field and the project's column strategy are + honoured. Scope: a concrete object declaring at least one source (a source-less + object generates nothing); derived fields and ``field.object`` (may flatten to + several columns) are skipped. ADR-0039: effective children (inherited fields count). + """ + from ..naming import resolve_column_name + from ..meta.core.field.meta_field import MetaField + for obj in (c for c in root.children() if c.type == TYPE_OBJECT): + if obj.is_abstract: + continue + if not any(c.type == TYPE_SOURCE for c in obj.children()): + continue + seen: dict[str, MetaData] = {} + for field in obj.children(): + if not isinstance(field, MetaField) or field.sub_type == FIELD_SUBTYPE_OBJECT \ + or field.is_derived(): + continue + column = resolve_column_name(field) + first = seen.get(column) + if first is None: + seen[column] = field + continue + errors.append(MetaError( + code=ErrorCode.ERR_COLUMN_NAME_COLLISION, + message=( + f'fields "{first.name}" and "{field.name}" on "{obj.name}" both resolve ' + f'to physical column "{column}". Rename one field, or give one an ' + f"explicit @column so the two columns differ." + ), + envelope=field.source, + )) + + def _validate_retired_requirement_links(root: MetaData, errors: list[MetaError]) -> None: from ..meta.core.requirement.requirement_constants import ( REQUIREMENT_ATTR_IMPLEMENTED_BY, diff --git a/server/python/src/metaobjects/render/extract/locate.py b/server/python/src/metaobjects/render/extract/locate.py index b5d6c024e2..f791f61ea6 100644 --- a/server/python/src/metaobjects/render/extract/locate.py +++ b/server/python/src/metaobjects/render/extract/locate.py @@ -8,6 +8,7 @@ import re from metaobjects.render.extract.json_forgiving_reader import comment_end +from metaobjects.render.extract.xml_forgiving_reader import mask_non_markup def json(text: str | None) -> str | None: @@ -84,17 +85,29 @@ def _scan_balanced(s: str, open_idx: int) -> int: def xml(text: str | None, root_name: str | None, case_insensitive: bool) -> str | None: - """Span of ``...``; if close absent, opener to end; ``None`` if no opener.""" + """Span of ``...``; if close absent, opener to end; ``None`` if no opener. + + Comments and CDATA are not searched, and when another opener of the same root sits between + the first one and its close (the tag mentioned in prose, then the real element) the span + starts at the LAST such opener: a root does not nest in itself.""" if text is None or root_name is None: return None flags = re.IGNORECASE if case_insensitive else 0 + view = mask_non_markup(text) open_re = re.compile("<" + re.escape(root_name) + r"(\s[^>]*)?>", flags) - open_m = open_re.search(text) + open_m = open_re.search(view) if open_m is None: return None start = open_m.start() close_re = re.compile("", flags) - close_m = close_re.search(text, open_m.end()) - if close_m is not None: - return text[start : close_m.end()] - return text[start:] + close_m = close_re.search(view, open_m.end()) + if close_m is None: + return text[start:] + at = open_m.end() + while True: + nxt = open_re.search(view, at) + if nxt is None or nxt.start() >= close_m.start(): + break + start = nxt.start() + at = nxt.end() + return text[start : close_m.end()] diff --git a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py index 468be5a8e0..71e5cea595 100644 --- a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py +++ b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py @@ -21,6 +21,7 @@ from __future__ import annotations import re +from collections.abc import Callable #: Reserved key holding an element's own text content when the element is represented as a #: dict (because it also carries attributes). ``#`` is not a legal XML name char, so it never @@ -38,19 +39,101 @@ # one attribute: name = "double" | 'single' | bareword. _ATTR = re.compile(r"""([A-Za-z_:][A-Za-z0-9_:.\-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s/>]+))""") +_COMMENT_OPEN = "" +_CDATA_OPEN = "" +# the five predefined XML entities plus numeric character references. +_ENTITY = re.compile(r"&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);") +_NAMED_ENTITIES = {"lt": "<", "gt": ">", "amp": "&", "quot": '"', "apos": "'"} +_KIND_COMMENT = "comment" +_KIND_CDATA = "cdata" + + +def _rewrite_regions(text: str, replace: Callable[[str, str], str]) -> str: + """Rewrite every XML comment and CDATA section in ``text`` through ``replace(kind, region)``, + scanning left to right so a comment marker inside CDATA (or the reverse) is never mistaken + for markup. An unclosed comment is left as text; an unclosed CDATA section runs to the end.""" + out: list[str] = [] + pos = 0 + frm = 0 + while True: + c = text.find(_COMMENT_OPEN, frm) + d = text.find(_CDATA_OPEN, frm) + is_comment = c >= 0 and (d < 0 or c < d) + start = c if is_comment else d + if start < 0: + break + if is_comment: + e = text.find(_COMMENT_CLOSE, start + len(_COMMENT_OPEN)) + if e < 0: + frm = start + len(_COMMENT_OPEN) + continue + end = e + len(_COMMENT_CLOSE) + else: + e = text.find(_CDATA_CLOSE, start + len(_CDATA_OPEN)) + end = len(text) if e < 0 else e + len(_CDATA_CLOSE) + out.append(text[pos:start]) + out.append(replace(_KIND_COMMENT if is_comment else _KIND_CDATA, text[start:end])) + pos = end + frm = end + out.append(text[pos:]) + return "".join(out) + + +def mask_non_markup(text: str) -> str: + """``text`` with comments and CDATA sections blanked to spaces (same length, so indices + hold): the view the tag locators search, so markup inside them is never located.""" + return _rewrite_regions(text, lambda _kind, region: " " * len(region)) + + +def _prepare(text: str) -> str: + """Drop comments; turn each CDATA section into its text with ``& < >`` entity-escaped, so + the tag scanner cannot split it and the entity decode restores it exactly.""" + + def replace(kind: str, region: str) -> str: + if kind == _KIND_COMMENT: + return "" + body = ( + region[len(_CDATA_OPEN) : -len(_CDATA_CLOSE)] + if region.endswith(_CDATA_CLOSE) + else region[len(_CDATA_OPEN) :] + ) + return body.replace("&", "&").replace("<", "<").replace(">", ">") + + return _rewrite_regions(text, replace) + + +def _decode_entities(s: str) -> str: + """Decode predefined entities and numeric references in one pass (so ``&lt;`` is + ``<``). An unknown or out-of-range reference stays as written.""" + if "&" not in s: + return s + + def sub(m: re.Match[str]) -> str: + body = m.group(1) + if not body.startswith("#"): + return _NAMED_ENTITIES[body] + cp = int(body[2:], 16) if body[1] == "x" else int(body[1:], 10) + surrogate = 0xD800 <= cp <= 0xDFFF + return chr(cp) if 1 <= cp <= 0x10FFFF and not surrogate else m.group(0) + + return _ENTITY.sub(sub, s) + class XmlForgivingReader: - def read(self, span: str | None, case_insensitive: bool) -> dict[str, object]: + def read(self, raw_span: str | None, case_insensitive: bool) -> dict[str, object]: out: dict[str, object] = {} - if span is None or span.strip() == "": + if raw_span is None or raw_span.strip() == "": return out + span = _prepare(raw_span) gt = span.find(">") if gt < 0: return out root_end = span.rfind(" dict[str, object]: @@ -61,21 +144,37 @@ def read_rootless(self, text: str | None, case_insensitive: bool) -> dict[str, o out: dict[str, object] = {} if text is None or text.strip() == "": return out - self._parse_children(text, case_insensitive, out, None) + self._parse_children(_prepare(text), case_insensitive, out, None, False) return out def _parse_children( - self, inner: str, ci: bool, out: dict[str, object], loose: list[str] | None + self, + inner: str, + ci: bool, + out: dict[str, object], + loose: list[str] | None, + literal_unclosed: bool, ) -> None: """Parse ``inner``'s elements into ``out``. When ``loose`` is not ``None``, the text BETWEEN those elements (an element's own text in mixed content) is appended to it, - one segment per gap.""" + one segment per gap. ``literal_unclosed`` is set for the body of an element that DID + close: an open tag in there with no close tag anywhere after it (``List and ...``) + is prose, not an element.""" open_tag = _OPEN_TAG_CI if ci else _OPEN_TAG + any_close = _CLOSE_TAG_CI if ci else _CLOSE_TAG pos = 0 + scan = 0 while True: - m = open_tag.search(inner, pos) + m = open_tag.search(inner, scan) if m is None: break + if ( + literal_unclosed + and not (m.group(2) or "").rstrip().endswith("/") + and any_close.search(inner, m.end()) is None + ): + scan = m.end() + continue if loose is not None: loose.append(inner[pos : m.start()]) tag = m.group(1) @@ -90,6 +189,7 @@ def _parse_children( if self_closing: self._accumulate(out, key, "" if not attrs else attrs) pos = m.end() + scan = pos continue content_start = m.end() @@ -130,8 +230,9 @@ def _parse_children( if stray is not None: content_end, nxt = stray content = inner[content_start:content_end] - self._accumulate(out, key, self._combine(attrs, content, ci)) + self._accumulate(out, key, self._combine(attrs, content, ci, close_m is not None)) pos = nxt + scan = nxt if loose is not None: loose.append(inner[pos:]) @@ -154,7 +255,9 @@ def _find_stray_close( return (m.start(), m.end()) at = m.end() - def _combine(self, attrs: dict[str, object], content: str, ci: bool) -> object: + def _combine( + self, attrs: dict[str, object], content: str, ci: bool, closed: bool + ) -> object: """Combine an element's attributes with its body (nested children or plain text). Mixed content keeps BOTH: the children, and the element's own text under :data:`TEXT_KEY`, so a scalar or ``@xmlText`` consumer still reads the prose @@ -162,7 +265,7 @@ def _combine(self, attrs: dict[str, object], content: str, ci: bool) -> object: if "<" in content: nested: dict[str, object] = {} loose: list[str] = [] - self._parse_children(content, ci, nested, loose) + self._parse_children(content, ci, nested, loose, closed) if nested: # attributes first; a child element wins a name collision merged: dict[str, object] = dict(attrs) @@ -179,13 +282,13 @@ def _mixed_text(self, segments: list[str], ci: bool) -> str: close_re = _CLOSE_TAG_CI if ci else _CLOSE_TAG parts: list[str] = [] for seg in segments: - t = close_re.sub("", seg).strip() + t = _decode_entities(close_re.sub("", seg).strip()) if t: parts.append(t) return " ".join(parts) def _text_value(self, attrs: dict[str, object], content: str) -> object: - text = content.strip() + text = _decode_entities(content.strip()) if not attrs: return text m: dict[str, object] = dict(attrs) @@ -204,7 +307,7 @@ def _parse_attrs(self, raw_attrs: str, ci: bool) -> dict[str, object]: ) ) if name not in attrs: - attrs[name] = val + attrs[name] = _decode_entities(val) return attrs def _accumulate(self, out: dict[str, object], key: str, value: object) -> None: diff --git a/server/python/src/metaobjects/render/mustache_tags.py b/server/python/src/metaobjects/render/mustache_tags.py new file mode 100644 index 0000000000..95f4c68847 --- /dev/null +++ b/server/python/src/metaobjects/render/mustache_tags.py @@ -0,0 +1,121 @@ +"""One Mustache tag scanner shared by ``verify`` and the renderer's partial expansion. + +Both used to find tags with their own ``{{`` search (verify) or a ``{{> x}}`` regex +(render), and neither knew what the engine knows: a comment ends at the first +closing delimiter, ``{{=<% %>=}}`` changes the delimiters, and a section close must +name the section it closes. :func:`scan` follows the Mustache spec for all three, so +the two callers cannot disagree about where a tag is. + +Mirrors ``Mustache.parse`` in the TypeScript port (``mustache.js``), ``MustacheTags`` in +the C# and Java ports. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +DEFAULT_OPEN = "{{" +DEFAULT_CLOSE = "}}" + +#: Tag kinds. ``name`` is a plain ``{{x}}``; ``{`` is ``{{{x}}}``; the rest are the sigil. +KIND_NAME = "name" +KIND_TRIPLE = "{" +KIND_AMP = "&" +KIND_SECTION = "#" +KIND_INVERTED = "^" +KIND_CLOSE = "/" +KIND_PARTIAL = ">" +KIND_COMMENT = "!" +KIND_DELIMITERS = "=" + + +@dataclass(frozen=True) +class Tag: + """One tag: its kind, its trimmed name, and its ``[start, end)`` span in the text.""" + + kind: str + value: str + start: int + end: int + + +class MustacheSyntaxError(ValueError): + """An unbalanced section. ``section`` is the section name the finding is about: + the unopened close's name, or the section left open.""" + + def __init__(self, message: str, section: str) -> None: + super().__init__(message) + self.section = section + + +def scan(text: str) -> list[Tag]: + """Every tag in ``text``, in order, honouring comments and delimiter changes. + + Raises :class:`MustacheSyntaxError` for a close with no open, a close that names a + different section than the one open, or a section never closed. An unterminated tag + ends the scan (the rest is text), as it always did here. + """ + tags: list[Tag] = [] + open_delim, close_delim = DEFAULT_OPEN, DEFAULT_CLOSE + open_sections: list[str] = [] + pos = 0 + while pos < len(text): + start = text.find(open_delim, pos) + if start < 0: + break + content_start = start + len(open_delim) + triple = text.startswith("{", content_start) + end_delim = "}" + close_delim if triple else close_delim + if triple: + content_start += 1 + close = text.find(end_delim, content_start) + if close < 0: + break + content = text[content_start:close].strip() + end = close + len(end_delim) + pos = end + + if triple: + tags.append(Tag(KIND_TRIPLE, content, start, end)) + continue + if not content: + continue + sigil = content[0] + name = content[1:].strip() if len(content) > 1 else "" + if sigil == KIND_COMMENT: + tags.append(Tag(KIND_COMMENT, name, start, end)) + elif sigil == KIND_DELIMITERS: + parts = content[1:].removesuffix("=").split() if content.endswith("=") else [] + if len(parts) == 2: + open_delim, close_delim = parts + tags.append(Tag(KIND_DELIMITERS, name, start, end)) + elif sigil in (KIND_SECTION, KIND_INVERTED): + open_sections.append(name) + tags.append(Tag(sigil, name, start, end)) + elif sigil == KIND_CLOSE: + if not open_sections: + raise MustacheSyntaxError(f'Unopened section "{name}"', name) + opened = open_sections.pop() + if opened != name: + raise MustacheSyntaxError(f'Unclosed section "{opened}"', opened) + tags.append(Tag(KIND_CLOSE, name, start, end)) + elif sigil in (KIND_PARTIAL, KIND_AMP): + tags.append(Tag(sigil, name, start, end)) + else: + tags.append(Tag(KIND_NAME, content, start, end)) + if open_sections: + opened = open_sections[-1] + raise MustacheSyntaxError(f'Unclosed section "{opened}"', opened) + return tags + + +def without_comments(text: str, tags: list[Tag]) -> str: + """``text`` with every comment tag cut out — what a tag-presence check may scan.""" + out: list[str] = [] + pos = 0 + for tag in tags: + if tag.kind == KIND_COMMENT: + out.append(text[pos : tag.start]) + pos = tag.end + out.append(text[pos:]) + return "".join(out) diff --git a/server/python/src/metaobjects/render/renderer.py b/server/python/src/metaobjects/render/renderer.py index feb23860d5..3db9c144f9 100644 --- a/server/python/src/metaobjects/render/renderer.py +++ b/server/python/src/metaobjects/render/renderer.py @@ -10,15 +10,13 @@ cross-port whitespace and lets the engine own cycle detection. The Mustache subset implemented (interpolation, raw triple-stache, sections, inverted sections, partials, comments) is the corpus's vocabulary — set-delimiter -directives are intentionally NOT supported (matches the verify side and the -TS/C#/Java engines). +directives are rejected loudly (verify understands them; this interpreter does not). Mirrors ``server/java/render/.../Renderer.java`` and ``server/csharp/MetaObjects.Render/Renderer.cs``. """ from __future__ import annotations -import re from dataclasses import dataclass, field from typing import Any @@ -30,6 +28,7 @@ partial_engine_mismatch, template_engine_of, ) +from .mustache_tags import KIND_DELIMITERS, KIND_PARTIAL, MustacheSyntaxError, scan from .payload_accessors import with_derived_accessors from .verify import ( ERR_REQUIRED_SLOT_UNUSED, @@ -40,10 +39,6 @@ ) from .verify import verify as verify_template -# {{> name }} — captures the reference between the marker and the closing }}. -_PARTIAL_PATTERN = re.compile(r"\{\{>\s*([^\s}]+)\s*\}\}") - - class RenderError(Exception): """Anything that goes wrong during render (unresolved partial, cycle, etc.).""" @@ -145,9 +140,20 @@ def _pre_expand_partials(text: str, provider: Provider, stack: list[str]) -> str raise RenderError( f"partial depth exceeded {MAX_DEPTH} (chain: {' -> '.join(stack)})" ) - - def replace(match: re.Match[str]) -> str: - ref = match.group(1) + try: + tags = scan(text) + except MustacheSyntaxError as e: + raise RenderError(f"template syntax error: {e}") from e + if any(t.kind == KIND_DELIMITERS for t in tags): + raise RenderError( + "set-delimiter tags ({{=<% %>=}}) are not supported by the Python renderer" + ) + out: list[str] = [] + pos = 0 + for tag in tags: + if tag.kind != KIND_PARTIAL: + continue + ref = tag.value if ref in stack: raise RenderError(f"partial cycle: {' -> '.join(stack)} -> {ref}") body = provider.resolve(ref) @@ -157,11 +163,14 @@ def replace(match: re.Match[str]) -> str: raise RenderError(partial_engine_message(ref, TEMPLATE_ENGINE_MUSTACHE)) stack.append(ref) try: - return _pre_expand_partials(body, provider, stack) + expanded = _pre_expand_partials(body, provider, stack) finally: stack.pop() - - return _PARTIAL_PATTERN.sub(replace, text) + out.append(text[pos : tag.start]) + out.append(expanded) + pos = tag.end + out.append(text[pos:]) + return "".join(out) # --------------------------------------------------------------------------- diff --git a/server/python/src/metaobjects/render/verify.py b/server/python/src/metaobjects/render/verify.py index ed6fb9cbda..9ae8bf5c2a 100644 --- a/server/python/src/metaobjects/render/verify.py +++ b/server/python/src/metaobjects/render/verify.py @@ -11,11 +11,10 @@ ``csharp/MetaObjects.Render/Verify.cs``; the three share the ``fixtures/verify-conformance/`` corpus that proves they agree. -The tokenizer models only plain interpolation, sections, inverted sections, and -partials (matching the C# port). It deliberately does NOT model Mustache -set-delimiter directives (``{{=<% %>=}}``); the TS engine parses with the real -``mustache`` library, so a fixture using set-delimiters would diverge across -ports. The corpus uses only plain tags — keep it that way. +Tags come from :mod:`metaobjects.render.mustache_tags`, which follows the Mustache +spec (a comment ends at the first closing delimiter, ``{{=<% %>=}}`` changes the +delimiters, a close must name its open) and is shared with the renderer's partial +expansion, so verify and render cannot disagree about where a tag is. """ from __future__ import annotations @@ -24,6 +23,18 @@ from typing import Protocol from .engine import TEMPLATE_ENGINE_JINJA, TEMPLATE_ENGINE_MUSTACHE, partial_engine_mismatch +from .mustache_tags import ( + KIND_AMP, + KIND_CLOSE, + KIND_INVERTED, + KIND_NAME, + KIND_PARTIAL, + KIND_SECTION, + KIND_TRIPLE, + MustacheSyntaxError, + scan, + without_comments, +) from .payload_accessors import is_boolean_accessor, is_positional_accessor #: A ``{{var}}`` references a field the (contextual) payload does not declare. @@ -39,7 +50,8 @@ #: dynamic key on an object (``user[key]``), a computed ``attribute=`` argument, or a #: template name that is an expression (``{% include name %}``). Reported, never passed. ERR_TEMPLATE_NOT_STATIC = "ERR_TEMPLATE_NOT_STATIC" -#: Jinja: the template text does not parse. path = the parser's message. +#: The template text does not parse. Jinja: path = the parser's message. Mustache: an +#: unbalanced section, path = the section's name. ERR_TEMPLATE_SYNTAX = "ERR_TEMPLATE_SYNTAX" #: A body is written for an engine this port does not run (Java, Kotlin and C# run #: Mustache only, so a ``.jinja`` template there is refused with this code). @@ -134,48 +146,30 @@ class _Partial: def _tokenize(text: str) -> list[_Token]: - tokens, _ = _parse_tokens(text, 0) - return tokens - - -def _parse_tokens(text: str, pos: int) -> tuple[list[_Token], int]: - """Parse tokens until EOF or a matching ``{{/close}}`` (which is consumed).""" - tokens: list[_Token] = [] - while pos < len(text): - open_idx = text.find("{{", pos) - if open_idx < 0: - break # remainder is plain text - triple = open_idx + 2 < len(text) and text[open_idx + 2] == "{" - close_delim = "}}}" if triple else "}}" - content_start = open_idx + (3 if triple else 2) - close_idx = text.find(close_delim, content_start) - if close_idx < 0: - break # malformed; stop - content = text[content_start:close_idx].strip() - pos = close_idx + len(close_delim) - - if triple: - tokens.append(_Var(content)) - continue - if not content: - continue - - sigil = content[0] - name = content[1:].strip() if len(content) > 1 else "" - if sigil == "!": - continue # comment - if sigil in ("#", "^"): - children, pos = _parse_tokens(text, pos) - tokens.append(_Section(name, sigil == "^", children)) - elif sigil == "/": - return tokens, pos # close (matched or mismatched) ends this level - elif sigil == ">": - tokens.append(_Partial(name)) - elif sigil == "&": - tokens.append(_Var(name)) - else: - tokens.append(_Var(content)) # {{x}}, {{x.y}}, {{.}} - return tokens, pos + """The tag tree of ``text``, built from the shared scanner (:mod:`mustache_tags`). + + Raises :class:`MustacheSyntaxError` on an unbalanced section.""" + root: list[_Token] = [] + open_levels: list[list[_Token]] = [root] + for tag in scan(text): + level = open_levels[-1] + if tag.kind in (KIND_SECTION, KIND_INVERTED): + section = _Section(tag.value, tag.kind == KIND_INVERTED, []) + level.append(section) + open_levels.append(section.children) + elif tag.kind == KIND_CLOSE: + open_levels.pop() # the scanner already matched it to its open + elif tag.kind == KIND_PARTIAL: + level.append(_Partial(tag.value)) + elif tag.kind in (KIND_NAME, KIND_TRIPLE, KIND_AMP): + level.append(_Var(tag.value)) # {{x}}, {{x.y}}, {{.}}, {{{x}}}, {{&x}} + # comments and delimiter changes carry no reference + return root + + +def _static_text(text: str) -> str: + """``text`` minus its comments, for the output-tag scan.""" + return without_comments(text, scan(text)) def _find(fields: list[PayloadField], name: str) -> PayloadField | None: @@ -321,10 +315,20 @@ def walk( if partial_engine_mismatch(provider, tok.value, TEMPLATE_ENGINE_MUSTACHE): errors.append(VerifyError(ERR_TEMPLATE_ENGINE_UNSUPPORTED, tok.value)) continue - static_texts.append(body) - walk(_tokenize(body), stack, [*seen, tok.value], in_element) - - walk(_tokenize(template_text), [root], []) + try: + partial_tokens = _tokenize(body) + except MustacheSyntaxError as e: + errors.append(VerifyError(ERR_TEMPLATE_SYNTAX, e.section)) + continue + static_texts.append(_static_text(body)) + walk(partial_tokens, stack, [*seen, tok.value], in_element) + + try: + body_tokens = _tokenize(template_text) + except MustacheSyntaxError as e: + return [VerifyError(ERR_TEMPLATE_SYNTAX, e.section)] + static_texts[0] = _static_text(template_text) + walk(body_tokens, [root], []) errors.extend(slot_and_tag_findings(referenced_at_root, static_texts, required_slots, required_tags)) return errors diff --git a/server/python/src/metaobjects/runtime/validator_runner.py b/server/python/src/metaobjects/runtime/validator_runner.py index 306c64deca..1f0577243c 100644 --- a/server/python/src/metaobjects/runtime/validator_runner.py +++ b/server/python/src/metaobjects/runtime/validator_runner.py @@ -343,8 +343,10 @@ def _scalar_errors(field: MetaData, value: object, required: bool, label: str) - if not isinstance(pattern, str): continue try: - # @pattern is FULL-MATCH: the whole value must match. - matched = re.fullmatch(f"(?:{pattern})", value) is not None + # @pattern is FULL-MATCH: the whole value must match. re.ASCII keeps \d, \w + # and \s to ASCII like the TS and Java ports; str patterns default to + # Unicode, so \d would otherwise accept Arabic-Indic digits. + matched = re.fullmatch(f"(?:{pattern})", value, re.ASCII) is not None except re.error: errors.append(ValidationFailure( label, "regex", f"'{label}' has an invalid validator pattern: {pattern}", pattern, diff --git a/server/python/src/metaobjects/source/yaml_core_schema.py b/server/python/src/metaobjects/source/yaml_core_schema.py new file mode 100644 index 0000000000..ac6f7149da --- /dev/null +++ b/server/python/src/metaobjects/source/yaml_core_schema.py @@ -0,0 +1,122 @@ +"""YAML 1.2 core-schema loader. + +PyYAML resolves plain scalars as YAML 1.1, where ``off``/``no``/``y`` are +booleans, ``1:30`` is a sexagesimal int, ``0777`` is octal, ``1_000`` is an int +and ``2024-01-02`` is a ``date``. The TypeScript loader uses the ``yaml`` +package, which resolves the 1.2 core schema: only ``true``/``false`` are +booleans, only ``[-+]?[0-9]+`` / ``0o`` / ``0x`` are ints, and there is no +timestamp type. This loader gives Python the same scalar resolution, so one +YAML file loads to the same model in every port. +""" +from __future__ import annotations + +import math +import re +from typing import Any + +import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs + +_TAG_NULL = "tag:yaml.org,2002:null" +_TAG_BOOL = "tag:yaml.org,2002:bool" +_TAG_INT = "tag:yaml.org,2002:int" +_TAG_FLOAT = "tag:yaml.org,2002:float" + +# YAML 1.2 section 10.3.2 (core schema) tag resolution. +_NULL_RE = re.compile(r"^(?:~|null|Null|NULL|)$") +_BOOL_RE = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +_INT_RE = re.compile(r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$") +_FLOAT_RE = re.compile( + r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$" +) + + +class CoreSchemaLoader(yaml.SafeLoader): # type: ignore[misc] + """``SafeLoader`` whose implicit scalar resolvers are the YAML 1.2 core schema.""" + + # Own table: ``add_implicit_resolver`` copies the inherited one on first use, + # so reset it here to drop every 1.1 resolver (timestamp, merge, value, ...). + yaml_implicit_resolvers: dict[str, list[tuple[str, re.Pattern[str]]]] = {} + + +CoreSchemaLoader.add_implicit_resolver(_TAG_NULL, _NULL_RE, ["~", "n", "N", ""]) +CoreSchemaLoader.add_implicit_resolver(_TAG_BOOL, _BOOL_RE, list("tTfF")) +CoreSchemaLoader.add_implicit_resolver(_TAG_INT, _INT_RE, list("-+0123456789")) +CoreSchemaLoader.add_implicit_resolver(_TAG_FLOAT, _FLOAT_RE, list("-+0123456789.")) + + +def _construct_null(loader: yaml.SafeLoader, node: yaml.Node) -> None: + loader.construct_scalar(node) + return None + + +def _construct_bool(loader: yaml.SafeLoader, node: yaml.Node) -> bool: + return str(loader.construct_scalar(node)).lower() == "true" + + +def _construct_int(loader: yaml.SafeLoader, node: yaml.Node) -> int: + text = str(loader.construct_scalar(node)) + if text.startswith("0o"): + return int(text[2:], 8) + if text.startswith("0x"): + return int(text[2:], 16) + return int(text, 10) + + +def _construct_float(loader: yaml.SafeLoader, node: yaml.Node) -> float: + text = str(loader.construct_scalar(node)).lower() + if text.endswith(".nan"): + return math.nan + if text.endswith(".inf"): + return -math.inf if text.startswith("-") else math.inf + return float(text) + + +CoreSchemaLoader.add_constructor(_TAG_NULL, _construct_null) +CoreSchemaLoader.add_constructor(_TAG_BOOL, _construct_bool) +CoreSchemaLoader.add_constructor(_TAG_INT, _construct_int) +CoreSchemaLoader.add_constructor(_TAG_FLOAT, _construct_float) + + +def safe_load_core(text: str) -> Any: + """``yaml.safe_load`` with YAML 1.2 core-schema scalars and unique mapping keys.""" + loader = CoreSchemaLoader(text) + try: + node = loader.get_single_node() + if node is None: + return None + _reject_duplicate_keys(node) + return loader.construct_document(node) + finally: + loader.dispose() + + +def _reject_duplicate_keys(node: yaml.Node) -> None: + """Raise on a repeated scalar key in any mapping (the ``yaml`` package does too).""" + seen_nodes: set[int] = set() + stack = [node] + while stack: + current = stack.pop() + if id(current) in seen_nodes: + continue + seen_nodes.add(id(current)) + if isinstance(current, yaml.SequenceNode): + stack.extend(current.value) + elif isinstance(current, yaml.MappingNode): + check_unique_keys(current) + for key_node, value_node in current.value: + stack.append(key_node) + stack.append(value_node) + + +def check_unique_keys(node: yaml.MappingNode) -> None: + seen: set[str] = set() + for key_node, _ in node.value: + if not isinstance(key_node, yaml.ScalarNode): + continue + key = str(key_node.value) + if key in seen: + raise yaml.constructor.ConstructorError( + None, None, f"duplicate mapping key {key!r}", key_node.start_mark + ) + seen.add(key) diff --git a/server/python/src/metaobjects/source/yaml_positions.py b/server/python/src/metaobjects/source/yaml_positions.py index 5a65e70f55..0eb60bb8ec 100644 --- a/server/python/src/metaobjects/source/yaml_positions.py +++ b/server/python/src/metaobjects/source/yaml_positions.py @@ -36,6 +36,7 @@ import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs from .error_source import YamlPosition +from .yaml_core_schema import CoreSchemaLoader, check_unique_keys class YamlPositionMap(dict): @@ -100,8 +101,9 @@ def set_position_map( def parse_yaml_with_positions(text: str) -> Any: """Parse YAML text and return a Python structure with positions attached. - Mirrors the contract of :func:`yaml.safe_load` for the shapes the - metaobjects authoring grammar uses (mappings, sequences, scalars). Aliases + Mirrors the contract of the TS ``yaml.parse`` (YAML 1.2 core schema, unique + mapping keys) for the shapes the metaobjects authoring grammar uses + (mappings, sequences, scalars). Aliases are resolved via the underlying SafeLoader pipeline — i.e. they resolve as the library normally would. @@ -110,10 +112,10 @@ def parse_yaml_with_positions(text: str) -> Any: the position of the KEY token in the YAML source. Raises: - :class:`yaml.YAMLError`: on YAML syntax errors (same as - ``yaml.safe_load``). + :class:`yaml.YAMLError`: on YAML syntax errors and on a duplicate + mapping key. """ - loader = yaml.SafeLoader(text) + loader = CoreSchemaLoader(text) try: node = loader.get_single_node() if node is None: @@ -130,16 +132,17 @@ def _yaml_node_to_py(node: Any, loader: Any) -> Any: :class:`YamlPositionMap`) onto the resulting dict — the position of each key is the (line, col) of the KEY token in the YAML source. - Scalar nodes are constructed via the SafeLoader's constructor pipeline, - so YAML 1.1's typed scalars (null, bool, int, float) coerce exactly the - way :func:`yaml.safe_load` produces them. The desugar's D2 coercion - guard fires on those coerced values, matching TS / Java / C# behavior. + Scalar nodes are constructed via :class:`CoreSchemaLoader`, so typed scalars + (null, bool, int, float) resolve under the YAML 1.2 core schema, matching the + TS loader: ``off``/``no``/``1:30``/``0777`` stay strings. The desugar's D2 + coercion guard fires on the values that do coerce (``true``, ``42``). """ if isinstance(node, yaml.ScalarNode): return loader.construct_object(node, deep=True) if isinstance(node, yaml.SequenceNode): return [_yaml_node_to_py(item, loader) for item in node.value] if isinstance(node, yaml.MappingNode): + check_unique_keys(node) out = YamlPositionMap() positions: dict[str, YamlPosition] = {} for key_node, value_node in node.value: diff --git a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py index 05ba7d5a8e..0914caf1d6 100644 --- a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py +++ b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py @@ -292,3 +292,12 @@ def test_narrowed_run_on_only_the_projection_still_materializes_the_shared_enum( ticket_summary_src = (out_dir / "TicketSummary.py").read_text() assert "level: Priority" in ticket_summary_src assert "from .enums import Priority" in ticket_summary_src + + +def test_shared_enum_values_colliding_after_upper_are_a_codegen_error() -> None: + with pytest.raises(ValueError, match="both map to the Python member OPEN"): + _emit_doc({"metadata.root": {"package": "a::b", "children": [ + {"field.enum": {"name": "State", "abstract": True, "@values": ["Open", "open"]}}, + {"object.value": {"name": "M", "children": [ + {"field.enum": {"name": "st", "extends": "a::b::State"}}]}}, + ]}}) diff --git a/server/python/tests/codegen/test_names_extends_chain.py b/server/python/tests/codegen/test_names_extends_chain.py index c59864590a..866ab7a400 100644 --- a/server/python/tests/codegen/test_names_extends_chain.py +++ b/server/python/tests/codegen/test_names_extends_chain.py @@ -296,7 +296,7 @@ def test_a_child_field_colliding_with_an_inherited_one_is_refused_naming_the_mod "children": [ {"source.rdb": {"@table": "rows"}}, {"field.long": {"name": "id"}}, - {"field.timestamp": {"name": "created_at"}}, + {"field.timestamp": {"name": "created_at", "@column": "created_at_alt"}}, {"identity.primary": {"name": "pk", "@fields": "id", "@generation": "increment"}}, ], } diff --git a/server/python/tests/conformance/test_extract_conformance.py b/server/python/tests/conformance/test_extract_conformance.py index 2eda6d8f98..b8aa180c36 100644 --- a/server/python/tests/conformance/test_extract_conformance.py +++ b/server/python/tests/conformance/test_extract_conformance.py @@ -61,7 +61,7 @@ def _cases() -> list[str]: def test_discovers_all_extract_conformance_cases() -> None: """FR-011: lock the corpus size so a deleted fixture fails CI rather than silently reducing coverage. Mirrors the TS / Java / C# count guards.""" - assert len(_cases()) == 48 + assert len(_cases()) == 54 _NORMALIZE_MODES = {"none", "collapse", "strip"} diff --git a/server/python/tests/render/test_verify_conformance.py b/server/python/tests/render/test_verify_conformance.py index 9bd350633e..54e8d010ff 100644 --- a/server/python/tests/render/test_verify_conformance.py +++ b/server/python/tests/render/test_verify_conformance.py @@ -19,6 +19,7 @@ ERR_OUTPUT_TAG_MISSING, ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, + ERR_TEMPLATE_SYNTAX, ERR_VAR_NOT_ON_PAYLOAD, InMemoryProvider, PayloadField, @@ -34,6 +35,7 @@ ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, } ) diff --git a/server/python/tests/runtime/test_validator_runner.py b/server/python/tests/runtime/test_validator_runner.py index 0b29e1ce96..2e728fb366 100644 --- a/server/python/tests/runtime/test_validator_runner.py +++ b/server/python/tests/runtime/test_validator_runner.py @@ -138,6 +138,12 @@ def test_regex_is_full_match() -> None: }] +def test_regex_digit_class_is_ascii_only() -> None: + e = _load(_with(_string("code"), {"validator.regex": {"@pattern": "\\d+"}})) + assert _rules(e, {"code": "123"}) == [] + assert _rules(e, {"code": "\u0661\u0662"}) == ["code:regex"] # Arabic-Indic digits + + def test_regex_rejects_a_trailing_newline() -> None: e = _load(_with(_string("code"), {"validator.regex": {"@pattern": "[A-Z]+"}})) assert _rules(e, {"code": "ABC\n"}) == ["code:regex"] diff --git a/server/python/tests/source/test_yaml_core_schema.py b/server/python/tests/source/test_yaml_core_schema.py new file mode 100644 index 0000000000..e49e066f81 --- /dev/null +++ b/server/python/tests/source/test_yaml_core_schema.py @@ -0,0 +1,56 @@ +"""YAML 1.2 core-schema scalar resolution (regression for the YAML 1.1 coercion). + +PyYAML's SafeLoader resolves scalars as YAML 1.1, so ``off``/``no``/``1:30``/ +``0777``/``2024-01-02`` changed type silently while the TS loader (the ``yaml`` +package, 1.2 core schema) kept them as the authored text. The Python loader +must agree with it, and a duplicate mapping key is a parse error, not last-wins. +""" +from __future__ import annotations + +import pytest +import yaml # type: ignore[import-untyped] + +from metaobjects.source.yaml_positions import parse_yaml_with_positions + + +def _scalar(text: str): + return parse_yaml_with_positions(f"k: {text}\n")["k"] + + +@pytest.mark.parametrize( + "text", + ["off", "on", "no", "yes", "y", "n", "Off", "NO", "1:30", "2024-01-02", "1_000", "0b11", "+.inf_"], +) +def test_yaml11_scalars_stay_strings(text: str) -> None: + assert _scalar(text) == text + + +def test_core_schema_scalars() -> None: + assert _scalar("true") is True + assert _scalar("False") is False + assert _scalar("TRUE") is True + assert _scalar("null") is None + assert _scalar("~") is None + assert _scalar("") is None + assert _scalar("42") == 42 + assert _scalar("0777") == 777 # 1.2 core: decimal, as the TS loader reads it + assert _scalar("-7") == -7 + assert _scalar("0x1F") == 31 + assert _scalar("0o17") == 15 + assert _scalar("1.5") == 1.5 + assert _scalar("1e3") == 1000.0 + assert _scalar(".inf") == float("inf") + assert _scalar('"off"') == "off" + + +def test_duplicate_mapping_key_is_an_error() -> None: + with pytest.raises(yaml.YAMLError): + parse_yaml_with_positions("a: 1\na: 2\n") + + +def test_positions_still_tracked() -> None: + value = parse_yaml_with_positions("a: off\nb: no\n") + assert value["a"] == "off" + from metaobjects.source.yaml_positions import get_yaml_position + + assert get_yaml_position(value, "b").line == 2 diff --git a/server/python/tests/unit/test_relative_set_newlines.py b/server/python/tests/unit/test_relative_set_newlines.py new file mode 100644 index 0000000000..d9b3585008 --- /dev/null +++ b/server/python/tests/unit/test_relative_set_newlines.py @@ -0,0 +1,11 @@ +"""The codegen drift gate must compare bytes: a CRLF file is not the LF file the generator wrote.""" +from __future__ import annotations + +from pathlib import Path + +from metaobjects.cli import _relative_set + + +def test_relative_set_keeps_carriage_returns(tmp_path: Path) -> None: + (tmp_path / "a.py").write_bytes(b"x = 1\r\ny = 2\r\n") + assert _relative_set(tmp_path) == {"a.py": "x = 1\r\ny = 2\r\n"} diff --git a/server/typescript/packages/cli/src/lib/dependency-sync.ts b/server/typescript/packages/cli/src/lib/dependency-sync.ts index 4624de81dc..e55102508e 100644 --- a/server/typescript/packages/cli/src/lib/dependency-sync.ts +++ b/server/typescript/packages/cli/src/lib/dependency-sync.ts @@ -288,6 +288,17 @@ function pep440Normalized(version: string): string | undefined { return `${epoch}${release}${pre}${post}${dev}${local}`; } +/** PEP 440 equality: release segments compare with trailing zeros ignored, so `1.0` and + * `1.0.0` are the same version. A string that is not PEP 440 compares verbatim. */ +function pep440Equal(a: string, b: string): boolean { + const key = (v: string): string => { + const n = pep440Normalized(v); + if (n === undefined) return v; + return n.replace(/^((?:\d+!)?\d+(?:\.\d+)*)/, (rel) => rel.replace(/(?:\.0+)+$/, "")); + }; + return key(a) === key(b); +} + /** The installed package's own version should be the manifest's: `sharedModelFile()` * defaults the manifest version to the host package's. A difference means the * publisher bumped the package without regenerating the artifact (or set @@ -301,7 +312,7 @@ export function packageVersionWarning( if (packageVersion === undefined) return undefined; const same = "python" in spec - ? (pep440Normalized(packageVersion) ?? packageVersion) === (pep440Normalized(manifestVersion) ?? manifestVersion) + ? pep440Equal(packageVersion, manifestVersion) : packageVersion === manifestVersion; if (same) return undefined; const pkg = "npm" in spec ? `npm package "${spec.npm}"` : "python" in spec ? `Python package "${spec.python}"` : "package"; diff --git a/server/typescript/packages/cli/test/deps-sync-packages.test.ts b/server/typescript/packages/cli/test/deps-sync-packages.test.ts index a379f2785b..4d53a8a425 100644 --- a/server/typescript/packages/cli/test/deps-sync-packages.test.ts +++ b/server/typescript/packages/cli/test/deps-sync-packages.test.ts @@ -349,6 +349,12 @@ describe("the installed-vs-manifest version warning", () => { ["1.0.dev0", "1.0-dev"], ["1.0.dev3", "1.0_dev3"], ["1.0.0", "v1.0.0"], + // PEP 440 ignores trailing zero release segments: 1.0 is 1.0.0. + ["1.0", "1.0.0"], + ["1.0.0", "1.0"], + ["2", "2.0.0"], + ["1.0.0rc1", "1.0rc1"], + ["1!1.0", "1!1.0.0"], ])("python: installed %s matches manifest %s", (installed, manifest) => { expect(packageVersionWarning(DEP, py, installed, manifest)).toBeUndefined(); }); @@ -357,7 +363,8 @@ describe("the installed-vs-manifest version warning", () => { expect(packageVersionWarning(DEP, py, "1.0.0rc2", "1.0.0-rc.1")).toContain("records 1.0.0-rc.1"); expect(packageVersionWarning(DEP, py, "1.0.0rc1", "1.0.0rc2")).toContain("records 1.0.0rc2"); expect(packageVersionWarning(DEP, py, "1.0.0rc1.post2", "1.0.0rc1")).toContain("records 1.0.0rc1"); - expect(packageVersionWarning(DEP, py, "1.0", "1.0.0")).toContain("records 1.0.0"); + expect(packageVersionWarning(DEP, py, "1.0.1", "1.0")).toContain("records 1.0"); + expect(packageVersionWarning(DEP, py, "1.10", "1.1")).toContain("records 1.1"); expect(packageVersionWarning(DEP, py, "1.1.0", "1.0.0")).toContain("is 1.1.0"); }); diff --git a/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts b/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts index bdd25669c6..af2b7526ca 100644 --- a/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts +++ b/server/typescript/packages/metadata/src/core/object/validate-discriminator.ts @@ -129,7 +129,12 @@ export function validateDiscriminator(root: MetaData): ParseError[] { for (const [, bindings] of bindingsByRoot) { const seen = new Map(); for (const b of bindings) { - const prev = seen.get(b.value); + // "1" and "01" are one value to a numeric column, so they are one claim. + const key = + NUMERIC_DISCRIMINATOR_SUBTYPES.has(b.discField.subType) && /^-?[0-9]+$/.test(b.value) + ? BigInt(b.value).toString() + : b.value; + const prev = seen.get(key); if (prev !== undefined) { errors.push( new ParseError( @@ -139,7 +144,7 @@ export function validateDiscriminator(root: MetaData): ParseError[] { ), ); } else { - seen.set(b.value, b.subtype); + seen.set(key, b.subtype); } } } diff --git a/server/typescript/packages/metadata/src/errors.ts b/server/typescript/packages/metadata/src/errors.ts index da75f58441..4014d2d508 100644 --- a/server/typescript/packages/metadata/src/errors.ts +++ b/server/typescript/packages/metadata/src/errors.ts @@ -120,6 +120,9 @@ export const ERROR_CODES = [ "ERR_DERIVED_FIELD_NO_READ_SOURCE", "ERR_ENTITY_PRIMARY_SOURCE_READONLY", "ERR_INVALID_TEMPLATE", + // Two fields of one stored object resolve to the same PHYSICAL column (snake_case + // folds `userId` and `userID` together, or two fields share an @column). + "ERR_COLUMN_NAME_COLLISION", // FR-017 — M:N relationship validation (slim vocabulary): @through must name a // junction declaring two identity.reference children; @sourceRefField must match // one of them; M:N attrs are invalid on a 1:N (@cardinality:one / no @through). diff --git a/server/typescript/packages/metadata/src/loader/meta-data-loader.ts b/server/typescript/packages/metadata/src/loader/meta-data-loader.ts index 9285fab035..480e3978da 100644 --- a/server/typescript/packages/metadata/src/loader/meta-data-loader.ts +++ b/server/typescript/packages/metadata/src/loader/meta-data-loader.ts @@ -25,7 +25,7 @@ import { ParseError } from "../errors.js"; import type { LoaderWarning } from "../source.js"; import { codeSource, resolvedSource } from "../source.js"; import { parseJson } from "../parser-json.js"; -import { validateDataGridSortFields, validateFilterableHasIndex, validateFilterableHasSupportedOps, validateSortableHasSupportedSubtype, validateOriginPaths, validateDerivedFieldProvidability, validateDataGridFilterValues, validateFieldObjectStorage, validateFieldMap, validateTemplatePayloadRefs, validateFieldDefaults, validateRelationships, validateOneSideReferenceResolution, validateM2MJunctionPairing, validateIndexLookupFields, validateProjectionFilter, validateRetiredRequirementLinks } from "./validation-passes.js"; +import { validateDataGridSortFields, validateFilterableHasIndex, validateFilterableHasSupportedOps, validateSortableHasSupportedSubtype, validateOriginPaths, validateDerivedFieldProvidability, validateDataGridFilterValues, validateFieldObjectStorage, validateFieldMap, validateTemplatePayloadRefs, validateFieldDefaults, validateRelationships, validateOneSideReferenceResolution, validateM2MJunctionPairing, validateIndexLookupFields, validateProjectionFilter, validateRetiredRequirementLinks, validateColumnNameCollisions } from "./validation-passes.js"; import { validateReporting } from "./reporting-validation.js"; import { runRegisteredValidation } from "./validation-registry.js"; import { validateSourceRoles } from "../persistence/source/validate-source-roles.js"; @@ -691,6 +691,9 @@ export class MetaDataLoader { // (ADR-0039: resolving accessor, so inherited fields via extends are visible). errors.push(...validateIndexLookupFields(root)); + // Two fields of one stored object resolving to the same physical column. + errors.push(...validateColumnNameCollisions(root)); + // FR-039 — a retired requirement carries no @implementedBy (refused, not // exempted, so the dangling-ref class is unreachable) and @supersededBy is // legal only on `retired`. diff --git a/server/typescript/packages/metadata/src/loader/sources/directory-source.ts b/server/typescript/packages/metadata/src/loader/sources/directory-source.ts index 3fc4aa889a..561568f1e8 100644 --- a/server/typescript/packages/metadata/src/loader/sources/directory-source.ts +++ b/server/typescript/packages/metadata/src/loader/sources/directory-source.ts @@ -6,12 +6,16 @@ // across environments and language ports. import { readdir, stat } from "node:fs/promises"; -import { basename, extname, join } from "node:path"; +import { basename, extname, join, relative, sep } from "node:path"; import { FileSource } from "./file-source.js"; import type { MetaDataSource } from "../meta-data-source.js"; export interface DirectoryOptions { - /** Filename patterns to exclude. Supports literal match and `*` / `**` globs. */ + /** + * Patterns to exclude. Supports literal match and `*` / `**` globs. A pattern without `/` + * matches the file's basename; a pattern containing `/` matches its `/`-separated path + * relative to the directory (so `vendor/**` excludes a subtree). + */ exclude?: string[]; /** Recurse into subdirectories. Default: true. */ recurse?: boolean; @@ -44,7 +48,8 @@ export class DirectorySource { for (const p of files) { if (!SUPPORTED_EXTS.has(extname(p).toLowerCase())) continue; const name = basename(p); - if (exclude.some((pat) => matchSimpleGlob(pat, name))) continue; + const rel = relative(this.directory, p).split(sep).join("/"); + if (exclude.some((pat) => matchSimpleGlob(pat, pat.includes("/") ? rel : name))) continue; filtered.push(p); } diff --git a/server/typescript/packages/metadata/src/loader/validation-passes.ts b/server/typescript/packages/metadata/src/loader/validation-passes.ts index 5f3650e8da..116eed45a8 100644 --- a/server/typescript/packages/metadata/src/loader/validation-passes.ts +++ b/server/typescript/packages/metadata/src/loader/validation-passes.ts @@ -55,6 +55,8 @@ import { OBJECT_PROJECTION_ATTR_FILTER, } from "../core/object/object-constants.js"; import { MetaSource } from "../persistence/source/meta-source.js"; +import { resolveColumnName } from "../naming.js"; +import { isMetaField } from "../shared/node-guards.js"; import { LAYOUT_SUBTYPE_DATA_GRID, LAYOUT_DATA_GRID_ATTR_DEFAULT_SORT_FIELD, @@ -2762,3 +2764,43 @@ export function validateRetiredRequirementLinks(root: MetaData): ParseError[] { walk(root); return errors; } + +// --------------------------------------------------------------------------- +// Physical column-name collisions +// +// Two fields of one stored object that resolve to the SAME physical column +// (`userId` + `userID` both snake_case to `user_id`; or two fields sharing an +// explicit @column) silently produce a duplicate column. Only the RESOLVED name +// is compared, through the single door `resolveColumnName`, so an explicit @column +// override, an inherited field and the project's column strategy are all honoured. +// Scope: a concrete object that declares at least one source (a source-less +// object generates nothing); derived fields (origin.*) and field.object (which may +// flatten to several columns) are not columns of this table and are skipped. +// ADR-0039: effective fields, so an inherited field collides like an own one. +// --------------------------------------------------------------------------- + +export function validateColumnNameCollisions(root: MetaData): ParseError[] { + const errors: ParseError[] = []; + for (const obj of root.children().filter((c) => c.type === TYPE_OBJECT)) { + if (obj.isAbstract) continue; + if (!obj.children().some((c) => c.type === TYPE_SOURCE)) continue; + const seen = new Map(); + for (const field of obj.children()) { + if (!isMetaField(field) || field.subType === FIELD_SUBTYPE_OBJECT || field.isDerived()) continue; + const column = resolveColumnName(field); + const first = seen.get(column); + if (first === undefined) { + seen.set(column, field); + continue; + } + errors.push( + new ParseError( + `fields "${first.name}" and "${field.name}" on "${obj.name}" both resolve to physical column ` + + `"${column}". Rename one field, or give one an explicit @column so the two columns differ.`, + { code: "ERR_COLUMN_NAME_COLLISION", source: field.source }, + ), + ); + } + } + return errors; +} diff --git a/server/typescript/packages/metadata/src/vocabulary-rewrite.ts b/server/typescript/packages/metadata/src/vocabulary-rewrite.ts index 926876b5aa..76ac210f82 100644 --- a/server/typescript/packages/metadata/src/vocabulary-rewrite.ts +++ b/server/typescript/packages/metadata/src/vocabulary-rewrite.ts @@ -153,6 +153,38 @@ function atOrBefore(a: string, b: string): boolean { return true; } +/** + * `text` with every JSONC comment overwritten by spaces (newlines kept), string literals + * untouched. Offsets are preserved exactly, so every scan below can run on the masked copy + * while edits are applied to the original: a `}` or `,` inside a comment can no longer end a + * node body or a scalar value early, and a commented-out key no longer matches. A comment + * the scan erases is also erased if its key is dropped, which is the right outcome for a + * trailing note about that key. + */ +function maskComments(text: string): string { + const out = text.split(""); + let inStr = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (inStr) { + if (c === "\\") i++; + else if (c === '"') inStr = false; + continue; + } + if (c === '"') { + inStr = true; + } else if (c === "/" && text[i + 1] === "/") { + for (; i < text.length && text[i] !== "\n"; i++) out[i] = " "; + } else if (c === "/" && text[i + 1] === "*") { + const end = text.indexOf("*/", i + 2); + const stop = end < 0 ? text.length : end + 2; + for (; i < stop; i++) if (text[i] !== "\n") out[i] = " "; + i--; + } + } + return out.join(""); +} + /** 1-indexed line of `index` in `text`. */ function lineAt(text: string, index: number): number { let line = 1; @@ -316,7 +348,10 @@ function valueSpan(text: string, from: number): { start: number; end: number; ra if (c === '"') inStr = true; else if (c === "," || c === "}" || c === "\n") break; } - return { start: i, end: j, raw: text.slice(i, j).trim() }; + // End at the last token character, not at the delimiter: the gap before it may be a + // (masked) comment, which a value replacement must not swallow. + const raw = text.slice(i, j).trim(); + return { start: i, end: i + raw.length, raw }; } /** Does the raw token equal this literal? Compares JSON-decoded where possible so @@ -371,7 +406,9 @@ function dropSpan(source: string, keyStart: number, valueEnd: number): { start: * Pure: no filesystem, no loader, no registry. Returns the new text plus every change and * every refusal, so a caller can print a diff and exit non-zero when work remains. */ -export function rewriteDocument(source: string, opts: RewriteOpts = {}): RewriteResult { +export function rewriteDocument(original: string, opts: RewriteOpts = {}): RewriteResult { + // Scan the comment-masked copy; apply edits to the original (offsets are identical). + const source = maskComments(original); const changes: RewriteChange[] = []; const refusals: RewriteRefusal[] = []; @@ -519,7 +556,7 @@ export function rewriteDocument(source: string, opts: RewriteOpts = {}): Rewrite } edits.sort((a, b) => b.start - a.start); - let text = source; + let text = original; for (const e of edits) text = text.slice(0, e.start) + e.text + text.slice(e.end); return { text, changes, refusals }; diff --git a/server/typescript/packages/metadata/test/loader/sources.test.ts b/server/typescript/packages/metadata/test/loader/sources.test.ts index a93c376d18..3e63d38c3d 100644 --- a/server/typescript/packages/metadata/test/loader/sources.test.ts +++ b/server/typescript/packages/metadata/test/loader/sources.test.ts @@ -79,6 +79,20 @@ describe("DirectorySource", () => { } }); + test("exclude with a slash matches the path relative to the directory", async () => { + const dir = await mkdtemp(join(tmpdir(), "ds-")); + try { + await mkdir(join(dir, "vendor", "deep"), { recursive: true }); + await mkdir(join(dir, "app"), { recursive: true }); + await writeFile(join(dir, "vendor", "deep", "meta.v.json"), "{}"); + await writeFile(join(dir, "app", "meta.app.json"), "{}"); + const expanded = await new DirectorySource(dir, { exclude: ["vendor/**"] }).expand(); + expect(expanded.map((s) => s.id)).toEqual(["meta.app.json"]); + } finally { + await rm(dir, { recursive: true }); + } + }); + test("recurses by default into subdirectories", async () => { const dir = await mkdtemp(join(tmpdir(), "ds-")); try { diff --git a/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts b/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts index 77ed2e2c9c..3f5de6e24f 100644 --- a/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts +++ b/server/typescript/packages/metadata/test/vocabulary-rewrite.test.ts @@ -241,3 +241,38 @@ describe("nothing exits clean while the metadata still will not load", () => { expect(() => JSON.parse(r.text)).not.toThrow(); }); }); + +describe("JSONC comments (audit finding 11)", () => { + test("a brace inside a // comment does not end the node scope early", () => { + const src = `{ "field.string": { "name": "ref", // closes } early? + "@readOnly": true } }`; + const r = rewriteDocument(src); + expect(r.text).toContain('"@mutability": "readOnly"'); + expect(r.text).toContain("// closes } early?"); + }); + + test("a trailing // comment after a value-scoped retired value still matches", () => { + const src = `{ "requirement.functional": { "name": "r", + "@status": "abandoned" // gave up +} }`; + const r = rewriteDocument(src); + expect(r.changes.length).toBeGreaterThan(0); + expect(r.text).not.toContain('"abandoned"'); + expect(r.text).toContain("// gave up"); + }); + + test("a trailing block comment does not hide a rewrite", () => { + const src = `{ "field.string": { "name": "ref", "@readOnly": true /* old */ } }`; + const r = rewriteDocument(src); + expect(r.text).toContain('"@mutability": "readOnly"'); + }); + + test("a commented-out retired key is left alone", () => { + const src = `{ "field.string": { "name": "ref", + // "@readOnly": true, + "@maxLength": 5 } }`; + const r = rewriteDocument(src); + expect(r.changes).toEqual([]); + expect(r.text).toBe(src); + }); +}); diff --git a/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts b/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts index b19b9b3435..145b3b1abc 100644 --- a/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts +++ b/server/typescript/packages/migrate-ts/src/apply/runner-transaction-pass.ts @@ -80,7 +80,7 @@ export interface RunnerTransactionPassResult { postTransactionChecks: string[]; } -const TRANSACTION_CONTROL = /^\s*(BEGIN|COMMIT|END\s+TRANSACTION|ROLLBACK|SAVEPOINT|RELEASE)\b/i; +const TRANSACTION_CONTROL = /^\s*(BEGIN|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE)\b/i; const FK_OFF = /^\s*PRAGMA\s+foreign_keys\s*=\s*(OFF|0|false)\s*$/i; const FK_ON = /^\s*PRAGMA\s+foreign_keys\s*=\s*(ON|1|true)\s*$/i; const FK_CHECK = /^\s*PRAGMA\s+foreign_key_check\s*$/i; diff --git a/server/typescript/packages/migrate-ts/src/introspect/d1.ts b/server/typescript/packages/migrate-ts/src/introspect/d1.ts index fa809e65e8..b14afc35e7 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/d1.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/d1.ts @@ -4,7 +4,7 @@ import type { } from "../types.js"; import { parseSqliteDefault, sqliteTypeToSqlType, sqliteRuleToAction, parseSqliteChecks, - buildSqliteIndexDescriptor, + buildSqliteIndexDescriptor, hasAutoincrementKeyword, } from "./sqlite-shared.js"; /** @@ -86,7 +86,7 @@ export async function introspectD1(opts: IntrospectD1Options): Promise c.name === pk[0]); if (pkCol) pkCol.identity = "increment"; diff --git a/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts b/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts index d682afb987..fc94ddffd4 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/sqlite-shared.ts @@ -337,3 +337,14 @@ export function sqliteRuleToAction(rule: string): FkAction { if (r === "RESTRICT") return "restrict"; return "no-action"; } + +/** + * True when a CREATE TABLE declares AUTOINCREMENT as a keyword: the scan runs on a masked + * copy, so a comment, a string literal (`DEFAULT 'AUTOINCREMENT'`) cannot trip it. A + * double-quoted identifier is preserved by the mask, so it is blanked here too. + */ +export function hasAutoincrementKeyword(createSql: string | null | undefined): boolean { + if (createSql === null || createSql === undefined) return false; + const masked = maskCommentsAndStrings(createSql).replace(/"(?:[^"]|"")*"/g, (m) => " ".repeat(m.length)); + return /\bAUTOINCREMENT\b/i.test(masked); +} diff --git a/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts b/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts index 910ad9d2f2..2435a66602 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/sqlite.ts @@ -6,7 +6,7 @@ import type { } from "../types.js"; import { parseSqliteDefault, sqliteTypeToSqlType, sqliteRuleToAction, parseSqliteChecks, - buildSqliteIndexDescriptor, + buildSqliteIndexDescriptor, hasAutoincrementKeyword, } from "./sqlite-shared.js"; import { MIGRATIONS_TABLE } from "../apply/ledger.js"; @@ -36,7 +36,7 @@ export async function introspectSqlite(db: Kysely>): Pro const cols = await readSqliteColumns(k, t.name); const pk = await readSqlitePrimaryKey(k, t.name); // Detect AUTOINCREMENT via the CREATE TABLE statement (sqlite has no PRAGMA for it). - const hasAutoincrement = (t.sql ?? "").toUpperCase().includes("AUTOINCREMENT"); + const hasAutoincrement = hasAutoincrementKeyword(t.sql); if (hasAutoincrement && pk.length === 1) { const pkCol = cols.find((c) => c.name === pk[0]); if (pkCol) pkCol.identity = "increment"; diff --git a/server/typescript/packages/migrate-ts/src/sql/split-statements.ts b/server/typescript/packages/migrate-ts/src/sql/split-statements.ts index c445db1824..4950c7b16c 100644 --- a/server/typescript/packages/migrate-ts/src/sql/split-statements.ts +++ b/server/typescript/packages/migrate-ts/src/sql/split-statements.ts @@ -5,6 +5,12 @@ * used both when applying migration files ({@link ../apply/apply.ts}) and when * post-processing emitted DDL for the D1 target ({@link ../emit/d1-safety-pass.ts}). * + * Two engine rules a plain `;` scan gets wrong are honoured: a SQLite + * `CREATE [TEMP] TRIGGER … BEGIN … END` is ONE statement (its body holds `;`; as in + * `sqlite3_complete`, only an `END` that follows a `;` closes it, so a `CASE … END` + * inside a body does not), and a Postgres `E'…'` literal treats `\` as an escape, so + * `E'it\'s; fine'` is one literal. + * * Returned statements are trimmed and DO NOT include the trailing `;` * separator; callers that need a terminator re-add one. * @@ -27,6 +33,21 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st const n = text.length; // Whether the current fragment holds anything besides comments and whitespace. let hasCode = false; + // Trigger tracking (per statement): the leading words, whether we are inside the + // BEGIN … END body, whether the previous token was `;`, and whether END was just seen. + let leadWords: string[] = []; + let inTriggerBody = false; + let afterSemi = false; + let afterEnd = false; + const resetStatement = (): void => { + leadWords = []; + inTriggerBody = false; + afterSemi = false; + afterEnd = false; + }; + const isTriggerStart = (): boolean => + leadWords[0] === "CREATE" && + (leadWords[1] === "TRIGGER" || ((leadWords[1] === "TEMP" || leadWords[1] === "TEMPORARY") && leadWords[2] === "TRIGGER")); const pushFragment = (fragment: string): void => { const stmt = fragment.trim(); if (stmt.length === 0) return; @@ -54,6 +75,32 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st if (ch !== ";" && !/\s/.test(ch as string)) hasCode = true; + // A bare word: feeds trigger detection, and an `E` prefix opens a backslash-escaped literal. + if (/[A-Za-z_]/.test(ch as string)) { + let j = i + 1; + while (j < n && /[A-Za-z0-9_]/.test(text[j] as string)) j++; + const word = text.slice(i, j).toUpperCase(); + if (word === "E" && text[j] === "'") { + i = skipBackslashLiteral(text, j); + afterSemi = false; + afterEnd = false; + continue; + } + if (leadWords.length < 3) leadWords.push(word); + if (inTriggerBody) { + afterEnd = afterSemi && word === "END"; + } else if (word === "BEGIN" && isTriggerStart()) { + inTriggerBody = true; + } + afterSemi = false; + i = j; + continue; + } + if (inTriggerBody && ch !== ";" && !/\s/.test(ch as string)) { + afterSemi = false; + afterEnd = false; + } + // Single-quoted literal: consume to the closing quote, treating '' as escape. if (ch === "'") { i++; @@ -107,7 +154,13 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st } // Statement separator (only reached when in no special state). + if (ch === ";" && inTriggerBody && !afterEnd) { + afterSemi = true; + i++; + continue; + } if (ch === ";") { + resetStatement(); pushFragment(text.slice(start, i)); hasCode = false; i++; @@ -122,6 +175,26 @@ export function splitSqlStatements(text: string, opts: SplitSqlOptions = {}): st return statements; } +/** Skip a Postgres `E'…'` literal starting at the opening quote; `\x` and `''` escape. */ +function skipBackslashLiteral(text: string, quote: number): number { + let i = quote + 1; + while (i < text.length) { + if (text[i] === "\\") { + i += 2; + continue; + } + if (text[i] === "'") { + if (text[i + 1] === "'") { + i += 2; + continue; + } + return i + 1; + } + i++; + } + return i; +} + /** * The statement with any leading `--` / block comments and whitespace removed, for * callers that classify a statement by its first keyword. diff --git a/server/typescript/packages/migrate-ts/src/view-fingerprint.ts b/server/typescript/packages/migrate-ts/src/view-fingerprint.ts index 59d7e5e660..d4b86b3aed 100644 --- a/server/typescript/packages/migrate-ts/src/view-fingerprint.ts +++ b/server/typescript/packages/migrate-ts/src/view-fingerprint.ts @@ -29,6 +29,7 @@ // the body comparator (view-sql-compare.ts) is exact there. import { createHash } from "node:crypto"; +import { collapseWhitespaceOutsideQuotes } from "./view-sql-compare.js"; /** * Marker format version. Bump ONLY if the normalization rules or the marker grammar @@ -48,7 +49,7 @@ const CREATE_VIEW_PREFIX = /** * Canonicalize a view body for hashing. * - * Collapses whitespace (so reindenting the emitter does not re-stamp every deployed + * Collapses whitespace outside quoted literals/identifiers (so reindenting the emitter does not re-stamp every deployed * view) and strips an optional CREATE VIEW wrapper and trailing semicolon — and * NOTHING else. * @@ -59,9 +60,7 @@ const CREATE_VIEW_PREFIX = * there is nothing to buy and real drift to lose. */ export function normalizeForFingerprint(body: string): string { - return body - .replace(CREATE_VIEW_PREFIX, "") - .replace(/\s+/g, " ") + return collapseWhitespaceOutsideQuotes(body.replace(CREATE_VIEW_PREFIX, "")) .replace(/;\s*$/, "") .trim(); } diff --git a/server/typescript/packages/migrate-ts/src/view-sql-compare.ts b/server/typescript/packages/migrate-ts/src/view-sql-compare.ts index 82c7642dcc..cb220b295b 100644 --- a/server/typescript/packages/migrate-ts/src/view-sql-compare.ts +++ b/server/typescript/packages/migrate-ts/src/view-sql-compare.ts @@ -16,30 +16,57 @@ // deparser at all. Do not point this function at Postgres again. // // normalizeViewSql strips a leading `CREATE [OR REPLACE] VIEW AS`, collapses -// whitespace runs, drops a trailing `;`, and lower-cases. -// -// CAVEAT (accepted tradeoff): lower-casing can mask a difference living ONLY in a -// case-sensitive string literal (e.g. `WHERE status = 'Active'` vs `'active'`), which -// an `origin.aggregate @filter` predicate can now carry. The fingerprint path -// deliberately does NOT lowercase for exactly this reason; this SQLite path keeps it -// for backwards-compatible behavior. The name regex matches one whitespace/`(`-free -// token, so a quoted view name containing a space would not strip cleanly — a non-issue -// for generated identifiers. +// whitespace runs OUTSIDE quoted literals and identifiers, and drops a trailing `;`. +// It does NOT lower-case: SQLite stores the text verbatim, so an exact comparison is +// sound, and a case change inside a literal (`status = 'Active'` vs `'active'`) or a +// whitespace change inside one (`'a b'` vs `'a b'`) is a real change of the view. The +// name regex matches one whitespace/`(`-free token, so a quoted view name containing a +// space would not strip cleanly — a non-issue for generated identifiers. const CREATE_VIEW_PREFIX = /^\s*create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?view\s+(?:if\s+not\s+exists\s+)?[^\s(]+(?:\s*\([^)]*\))?\s+as\s+/i; +/** + * Collapse every whitespace run to one space, except inside `'…'` literals, `"…"` and + * `` `…` `` identifiers (doubled quote = escape), which are copied verbatim. + */ +export function collapseWhitespaceOutsideQuotes(sql: string): string { + let out = ""; + let i = 0; + const n = sql.length; + while (i < n) { + const ch = sql[i] as string; + if (ch === "'" || ch === '"' || ch === "`") { + let j = i + 1; + while (j < n) { + if (sql[j] === ch) { + if (sql[j + 1] === ch) { j += 2; continue; } + j++; + break; + } + j++; + } + out += sql.slice(i, j); + i = j; + } else if (/\s/.test(ch)) { + while (i < n && /\s/.test(sql[i] as string)) i++; + out += " "; + } else { + out += ch; + i++; + } + } + return out; +} + /** * Collapse a view definition (full `CREATE VIEW ... AS body` OR a bare body) - * to a canonical, comparable string. Whitespace-, case-, and wrapper-insensitive. + * to a canonical, comparable string. Whitespace- (outside quotes) and wrapper-insensitive; case-sensitive. */ export function normalizeViewSql(sql: string): string { - return sql - .replace(CREATE_VIEW_PREFIX, "") - .replace(/\s+/g, " ") + return collapseWhitespaceOutsideQuotes(sql.replace(CREATE_VIEW_PREFIX, "")) .replace(/;\s*$/, "") - .trim() - .toLowerCase(); + .trim(); } /** True when two view definitions are equivalent after normalization. */ diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts index 289becd8f4..c91b7b51d6 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts @@ -162,7 +162,7 @@ describe("D3 — a view whose definition differs says so", () => { expect(c !== undefined && "reason" in c ? c.reason : undefined).toEqual({ kind: "definition", compared: "text", - firstDifference: { expected: "select id, label as title from t", actual: "select id, label from t" }, + firstDifference: { expected: "SELECT id, label AS title FROM t", actual: "SELECT id, label FROM t" }, }); }); diff --git a/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts b/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts new file mode 100644 index 0000000000..09a81b2c72 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/unit/regex-followups-sql.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { splitSqlStatements } from "../../src/sql/split-statements.js"; +import { prepareForRunnerTransaction } from "../../src/apply/runner-transaction-pass.js"; +import { viewSqlEquals } from "../../src/view-sql-compare.js"; +import { viewFingerprint } from "../../src/view-fingerprint.js"; +import { hasAutoincrementKeyword } from "../../src/introspect/sqlite-shared.js"; + +const TRIGGER = + "CREATE TRIGGER t_touch AFTER UPDATE ON t BEGIN UPDATE t SET updated_at = datetime('now') WHERE id = NEW.id; END"; + +describe("splitSqlStatements: engine rules", () => { + test("a SQLite trigger is one statement the engine accepts", () => { + const parts = splitSqlStatements(`CREATE TABLE t (id INTEGER, updated_at TEXT);\n${TRIGGER};\nSELECT 1;`); + expect(parts).toEqual(["CREATE TABLE t (id INTEGER, updated_at TEXT)", TRIGGER, "SELECT 1"]); + const db = new Database(":memory:"); + for (const p of parts) db.exec(p); + }); + + test("CASE ... END and several body statements stay inside the trigger", () => { + const trig = + "CREATE TEMP TRIGGER x AFTER INSERT ON t BEGIN SELECT CASE WHEN 1 THEN 2 END; UPDATE t SET a = 1; END"; + expect(splitSqlStatements(`${trig}; SELECT 2;`)).toEqual([trig, "SELECT 2"]); + }); + + test("an identifier containing begin does not open a body", () => { + expect(splitSqlStatements("CREATE TABLE begin_x (a); SELECT 1;")).toEqual(["CREATE TABLE begin_x (a)", "SELECT 1"]); + }); + + test("Postgres E'' literal honours backslash escapes", () => { + const sql = "INSERT INTO t VALUES (E'it\\'s; fine');\nSELECT 1;"; + expect(splitSqlStatements(sql)).toEqual(["INSERT INTO t VALUES (E'it\\'s; fine')", "SELECT 1"]); + }); + + test("a plain literal ending in e is not an E string", () => { + expect(splitSqlStatements("SELECT 'e'; SELECT 'a\\'; SELECT 2;")).toEqual(["SELECT 'e'", "SELECT 'a\\'", "SELECT 2"]); + }); + + test("a bare END is transaction control", () => { + const r = prepareForRunnerTransaction(`END;\n${TRIGGER};`); + expect(r.statements).toEqual([TRIGGER]); + }); +}); + +describe("view comparators are literal-exact", () => { + test("SQLite comparator: literal case and literal whitespace differ", () => { + expect(viewSqlEquals("SELECT 1 WHERE s = 'Active'", "SELECT 1 WHERE s = 'active'")).toBe(false); + expect(viewSqlEquals("SELECT 1 WHERE s = 'a b'", "SELECT 1 WHERE s = 'a b'")).toBe(false); + expect(viewSqlEquals("CREATE VIEW v AS SELECT 1\n WHERE s = 'a b';", "SELECT 1 WHERE s = 'a b'")).toBe(true); + }); + test("fingerprint: literal whitespace differs, outside whitespace does not", () => { + expect(viewFingerprint("SELECT 'a b'")).not.toBe(viewFingerprint("SELECT 'a b'")); + expect(viewFingerprint("SELECT 1")).toBe(viewFingerprint("SELECT 1")); + }); +}); + +describe("AUTOINCREMENT detection", () => { + test("keyword only", () => { + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT)")).toBe(true); + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY) -- AUTOINCREMENT")).toBe(false); + expect(hasAutoincrementKeyword("CREATE TABLE t (id INTEGER PRIMARY KEY, n TEXT DEFAULT 'AUTOINCREMENT')")).toBe(false); + expect(hasAutoincrementKeyword('CREATE TABLE t (id INTEGER PRIMARY KEY, "AUTOINCREMENT" TEXT)')).toBe(false); + }); +}); diff --git a/server/typescript/packages/render/src/extract/locate.ts b/server/typescript/packages/render/src/extract/locate.ts index 85dfd8eda2..2a891155ed 100644 --- a/server/typescript/packages/render/src/extract/locate.ts +++ b/server/typescript/packages/render/src/extract/locate.ts @@ -3,6 +3,7 @@ // declared field); locateJson (first-closed-else-first-open) is its fallback. Mirrors Java Locate. import { commentEnd } from "./json-forgiving-reader.js"; +import { maskXmlNonMarkup } from "./xml-forgiving-reader.js"; /** The characters a JSON comment may follow: whitespace or a structural separator. */ const COMMENT_MAY_FOLLOW = /[\s,{[]/; @@ -84,7 +85,12 @@ function scanBalanced(s: string, open: number): number { return -1; } -/** Span of ...; if close absent, opener to end; null if no opener. */ +/** + * Span of ...; if close absent, opener to end; null if no opener. Comments and + * CDATA are not searched, and when another opener of the same root sits between the first one + * and its close (the tag mentioned in prose, then the real element) the span starts at the + * LAST such opener: a root does not nest in itself. + */ export function locateXml( text: string | null | undefined, rootName: string | null | undefined, @@ -92,17 +98,23 @@ export function locateXml( ): string | null { if (text == null || rootName == null) return null; const flags = caseInsensitive ? "i" : ""; - const open = new RegExp(`<${quote(rootName)}(\\s[^>]*)?>`, flags).exec(text); + const view = maskXmlNonMarkup(text); + const openSrc = `<${quote(rootName)}(\\s[^>]*)?>`; + const open = new RegExp(openSrc, flags).exec(view); if (open == null) return null; - const start = open.index; const openEnd = open.index + open[0].length; - const closeRe = new RegExp(``, flags); - closeRe.lastIndex = openEnd; // Search for the close tag at-or-after the opener's end (Java close.find(open.end())). - const close = matchFrom(closeRe, text, openEnd); - if (close != null) return text.substring(start, close.index + close[0].length); - return text.substring(start); + const close = matchFrom(new RegExp(``, flags), view, openEnd); + if (close == null) return text.substring(open.index); + let start = open.index; + for (let at = openEnd; ; ) { + const next = matchFrom(new RegExp(openSrc, flags), view, at); + if (next == null || next.index >= close.index) break; + start = next.index; + at = next.index + next[0].length; + } + return text.substring(start, close.index + close[0].length); } /** Find the first regex match at index >= `from` (emulates Java Matcher.find(int)). */ diff --git a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts index b7ff9ffacc..5edee8bc51 100644 --- a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts +++ b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts @@ -37,14 +37,89 @@ const CLOSE_TAG_SRC = ""; // one attribute: name = "double" | 'single' | bareword. const ATTR_SRC = "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"; -export function readXml(span: string | null | undefined, caseInsensitive: boolean): Record { +const COMMENT_OPEN = ""; +const CDATA_OPEN = ""; +/** The five predefined XML entities plus numeric character references. */ +const ENTITY_RE = /&(#x[0-9a-fA-F]+|#[0-9]+|lt|gt|amp|quot|apos);/g; +const NAMED_ENTITIES: Readonly> = { lt: "<", gt: ">", amp: "&", quot: '"', apos: "'" }; + +type XmlRegionKind = "comment" | "cdata"; + +/** + * Rewrite every XML comment and CDATA section in `text` through `replace`, scanning left to + * right so a comment marker inside CDATA (or the reverse) is never mistaken for markup. An + * unclosed comment is left as text; an unclosed CDATA section runs to the end. + */ +function rewriteRegions(text: string, replace: (kind: XmlRegionKind, region: string) => string): string { + let out = ""; + let pos = 0; + let from = 0; + for (;;) { + const c = text.indexOf(COMMENT_OPEN, from); + const d = text.indexOf(CDATA_OPEN, from); + const isComment = c >= 0 && (d < 0 || c < d); + const start = isComment ? c : d; + if (start < 0) break; + let end: number; + if (isComment) { + const e = text.indexOf(COMMENT_CLOSE, start + COMMENT_OPEN.length); + if (e < 0) { + from = start + COMMENT_OPEN.length; + continue; + } + end = e + COMMENT_CLOSE.length; + } else { + const e = text.indexOf(CDATA_CLOSE, start + CDATA_OPEN.length); + end = e < 0 ? text.length : e + CDATA_CLOSE.length; + } + out += text.substring(pos, start) + replace(isComment ? "comment" : "cdata", text.substring(start, end)); + pos = end; + from = end; + } + return out + text.substring(pos); +} + +/** `text` with comments and CDATA sections blanked to spaces (same length, so indices hold): + * the view the tag locators search, so markup inside them is never located. */ +export function maskXmlNonMarkup(text: string): string { + return rewriteRegions(text, (_kind, region) => " ".repeat(region.length)); +} + +/** Drop comments; turn each CDATA section into its text with `& < >` entity-escaped, so the + * tag scanner cannot split it and the entity decode restores it exactly. */ +function prepareXml(text: string): string { + return rewriteRegions(text, (kind, region) => { + if (kind === "comment") return ""; + const body = region.endsWith(CDATA_CLOSE) + ? region.substring(CDATA_OPEN.length, region.length - CDATA_CLOSE.length) + : region.substring(CDATA_OPEN.length); + return body.replace(/&/g, "&").replace(//g, ">"); + }); +} + +/** Decode predefined entities and numeric references in one pass (so `&lt;` is `<`). + * An unknown or out-of-range reference stays as written. */ +function decodeEntities(s: string): string { + if (!s.includes("&")) return s; + return s.replace(ENTITY_RE, (whole, body: string) => { + if (!body.startsWith("#")) return NAMED_ENTITIES[body] ?? whole; + const cp = body[1] === "x" ? parseInt(body.substring(2), 16) : parseInt(body.substring(1), 10); + const surrogate = cp >= 0xd800 && cp <= 0xdfff; + return cp >= 1 && cp <= 0x10ffff && !surrogate ? String.fromCodePoint(cp) : whole; + }); +} + +export function readXml(rawSpan: string | null | undefined, caseInsensitive: boolean): Record { const out: Record = {}; - if (span == null || span.trim().length === 0) return out; + if (rawSpan == null || rawSpan.trim().length === 0) return out; + const span = prepareXml(rawSpan); const gt = span.indexOf(">"); if (gt < 0) return out; const rootEnd = span.lastIndexOf(" { const out: Record = {}; if (text == null || text.trim().length === 0) return out; - parseChildren(text, caseInsensitive, out, null); + parseChildren(prepareXml(text), caseInsensitive, out, null, false); return out; } /** * Parse `inner`'s elements into `out`. When `loose` is non-null, the text BETWEEN those * elements (an element's own text in mixed content) is appended to it, one segment per gap. + * `literalUnclosed` is set for the body of an element that DID close: an open tag in there with + * no close tag anywhere after it (`List and ...`) is prose, not an element. */ function parseChildren( inner: string, ci: boolean, out: Record, loose: string[] | null, + literalUnclosed: boolean, ): void { const flags = ci ? "i" : ""; let pos = 0; + let scan = 0; for (;;) { - const m = matchFrom(OPEN_TAG_SRC, flags, inner, pos); + const m = matchFrom(OPEN_TAG_SRC, flags, inner, scan); if (m == null) break; + const afterOpen = m.index + m[0].length; + if ( + literalUnclosed && + !(m[2] ?? "").trimEnd().endsWith("/") && + matchFrom(CLOSE_TAG_SRC, flags, inner, afterOpen) == null + ) { + scan = afterOpen; + continue; + } loose?.push(inner.substring(pos, m.index)); const tag = m[1] ?? ""; const key = ci ? tag.toLowerCase() : tag; @@ -90,6 +178,7 @@ function parseChildren( if (selfClosing) { accumulate(out, key, Object.keys(attrs).length === 0 ? "" : attrs); pos = m.index + m[0].length; + scan = pos; continue; } @@ -137,8 +226,9 @@ function parseChildren( } const content = inner.substring(contentStart, contentEnd); - accumulate(out, key, combine(attrs, content, ci)); + accumulate(out, key, combine(attrs, content, ci, close != null)); pos = next; + scan = next; } loose?.push(inner.substring(pos)); } @@ -168,11 +258,11 @@ function findStrayClose(inner: string, from: number, to: number, ci: boolean): [ * keeps BOTH: the children, and the element's own text under TEXT_KEY, so a scalar or * `@xmlText` consumer still reads the prose around a child element. */ -function combine(attrs: Record, content: string, ci: boolean): unknown { +function combine(attrs: Record, content: string, ci: boolean, closed: boolean): unknown { if (content.includes("<")) { const nested: Record = {}; const loose: string[] = []; - parseChildren(content, ci, nested, loose); + parseChildren(content, ci, nested, loose, closed); if (Object.keys(nested).length > 0) { // attributes first; a child element wins a name collision const merged: Record = { ...attrs, ...nested }; @@ -191,14 +281,14 @@ function mixedText(segments: readonly string[], ci: boolean): string { const parts: string[] = []; for (const seg of segments) { closeRe.lastIndex = 0; - const t = seg.replace(closeRe, "").trim(); + const t = decodeEntities(seg.replace(closeRe, "").trim()); if (t.length > 0) parts.push(t); } return parts.join(" "); } function textValue(attrs: Record, content: string): unknown { - const text = content.trim(); + const text = decodeEntities(content.trim()); if (Object.keys(attrs).length === 0) return text; return { ...attrs, [TEXT_KEY]: text }; } @@ -212,7 +302,7 @@ function parseAttrs(rawAttrs: string, ci: boolean): Record { const rawName = a[1]; if (rawName === undefined) continue; // group 1 is mandatory in a match; guards strict TS const name = ci ? rawName.toLowerCase() : rawName; - const val = a[2] ?? a[3] ?? a[4] ?? ""; + const val = decodeEntities(a[2] ?? a[3] ?? a[4] ?? ""); if (!Object.prototype.hasOwnProperty.call(attrs, name)) attrs[name] = val; } return attrs; diff --git a/server/typescript/packages/render/src/render.ts b/server/typescript/packages/render/src/render.ts index f6ada2390b..aeadbcb221 100644 --- a/server/typescript/packages/render/src/render.ts +++ b/server/typescript/packages/render/src/render.ts @@ -1,7 +1,14 @@ import Mustache from "mustache"; import type { Provider } from "./provider.js"; import { ESCAPERS, type RenderFormat } from "./escapers.js"; -import { verify, ERR_REQUIRED_SLOT_UNUSED, type PayloadField } from "./verify.js"; +import { + verify, + parseTemplate, + partialTokens, + hasDelimiterChange, + ERR_REQUIRED_SLOT_UNUSED, + type PayloadField, +} from "./verify.js"; import { withDerivedAccessors } from "./payload-accessors.js"; import { TEMPLATE_ENGINE_JINJA, @@ -14,15 +21,24 @@ import { import { renderJinja } from "./jinja/render.js"; import { MAX_DEPTH } from "./verify-shared.js"; -const PARTIAL = /\{\{>\s*([^}\s]+)\s*\}\}/g; - // Resolve `{{> group/source }}` partials by inlining their text, recursively, // BEFORE Mustache parses — giving us deterministic whitespace and true, // path-based cycle/depth detection (no reliance on a per-lib partial loader). // Inlined text still renders in the surrounding context (e.g. once per loop // item), exactly like a native partial, because Mustache renders the result. +// Partial tags are located by Mustache's own parser, so a `{{> x}}` written inside +// a comment is not a partial, and a `{{=<% %>=}}` delimiter change is honoured. function expand(text: string, provider: Provider, path: readonly string[]): string { - return text.replace(PARTIAL, (_match, ref: string) => { + const tokens = parseTemplate(text); + const partials = partialTokens(tokens); + if (partials.length === 0) return text; + // Inlined text is parsed under the surrounding delimiters, not the default ones. + if (hasDelimiterChange(tokens)) { + throw new Error("a template that uses partials cannot also change delimiters ({{=<% %>=}})"); + } + let out = ""; + let pos = 0; + for (const [ref, start, end] of partials) { if (path.includes(ref)) throw new Error(`partial cycle: ${[...path, ref].join(" -> ")}`); if (path.length >= MAX_DEPTH) throw new Error(`partial depth exceeded ${MAX_DEPTH}: ${ref}`); const t = provider.resolve(ref); @@ -30,8 +46,10 @@ function expand(text: string, provider: Provider, path: readonly string[]): stri if (partialEngineMismatch(provider, ref, TEMPLATE_ENGINE_MUSTACHE)) { throw new Error(partialEngineMessage(ref, TEMPLATE_ENGINE_MUSTACHE)); } - return expand(t, provider, [...path, ref]); - }); + out += text.slice(pos, start) + expand(t, provider, [...path, ref]); + pos = end; + } + return out + text.slice(pos); } export interface RenderOptions { diff --git a/server/typescript/packages/render/src/verify.ts b/server/typescript/packages/render/src/verify.ts index 08e64d1896..818595dcd3 100644 --- a/server/typescript/packages/render/src/verify.ts +++ b/server/typescript/packages/render/src/verify.ts @@ -17,6 +17,7 @@ import { ERR_VAR_NOT_ON_PAYLOAD, ERR_PARTIAL_UNRESOLVED, ERR_TEMPLATE_ENGINE_UNSUPPORTED, + ERR_TEMPLATE_SYNTAX, MAX_DEPTH, slotAndTagFindings, type PayloadField, @@ -119,6 +120,53 @@ function parse(text: string): Token[] { return Mustache.parse(text) as unknown as Token[]; } +const TOKEN_PARTIAL = ">"; +const TOKEN_COMMENT = "!"; +const TOKEN_DELIMITERS = "="; + +/** Every partial tag in a parse, in text order, as `[ref, start, end]` (nested sections included). */ +export function partialTokens(tokens: readonly Token[]): [string, number, number][] { + const out: [string, number, number][] = []; + for (const tok of tokens) { + if (tok[0] === TOKEN_PARTIAL) out.push([tok[1] as string, tok[2] as number, tok[3] as number]); + else if (Array.isArray(tok[4])) out.push(...partialTokens(tok[4] as Token[])); + } + return out; +} + +/** Whether the parse contains a `{{=<% %>=}}` delimiter change. */ +export function hasDelimiterChange(tokens: readonly Token[]): boolean { + return tokens.some( + (tok) => tok[0] === TOKEN_DELIMITERS || (Array.isArray(tok[4]) && hasDelimiterChange(tok[4] as Token[])), + ); +} + +function commentSpans(tokens: readonly Token[]): [number, number][] { + const out: [number, number][] = []; + for (const tok of tokens) { + if (tok[0] === TOKEN_COMMENT) out.push([tok[2] as number, tok[3] as number]); + else if (Array.isArray(tok[4])) out.push(...commentSpans(tok[4] as Token[])); + } + return out; +} + +/** `text` with its comments cut out: what a tag-presence check may scan. */ +function withoutComments(text: string, tokens: readonly Token[]): string { + let out = ""; + let pos = 0; + for (const [start, end] of commentSpans(tokens)) { + out += text.slice(pos, start); + pos = end; + } + return out + text.slice(pos); +} + +/** The section a Mustache parse error is about (`Unopened section "x" at 4`), else undefined. */ +function syntaxSection(e: unknown): string | undefined { + const m = /^(?:Unopened|Unclosed) section "(.*)" at \d+$/.exec(e instanceof Error ? e.message : ""); + return m?.[1]; +} + /** * Parse a template into Mustache tokens (`[type, value, start, end, subTokens?]`), * the SAME parse verify walks. Exported so the docs annotator tokenizes through @@ -220,8 +268,17 @@ export function verify( errors.push({ code: ERR_TEMPLATE_ENGINE_UNSUPPORTED, path: value }); break; } - staticTexts.push(text); - walk(parse(text), stack, [...seen, value], positional); + let parsed: Token[]; + try { + parsed = parse(text); + } catch (e) { + const section = syntaxSection(e); + if (section === undefined) throw e; + errors.push({ code: ERR_TEMPLATE_SYNTAX, path: section }); + break; + } + staticTexts.push(withoutComments(text, parsed)); + walk(parsed, stack, [...seen, value], positional); break; } default: @@ -230,7 +287,16 @@ export function verify( } } - walk(parse(templateText), [root], []); + let bodyTokens: Token[]; + try { + bodyTokens = parse(templateText); + } catch (e) { + const section = syntaxSection(e); + if (section === undefined) throw e; + return [{ code: ERR_TEMPLATE_SYNTAX, path: section }]; + } + staticTexts[0] = withoutComments(templateText, bodyTokens); + walk(bodyTokens, [root], []); errors.push(...slotAndTagFindings(referencedAtRoot, staticTexts, opts)); return errors; diff --git a/server/typescript/packages/render/test/extract/extract-conformance.test.ts b/server/typescript/packages/render/test/extract/extract-conformance.test.ts index 35cbf70c29..369336e637 100644 --- a/server/typescript/packages/render/test/extract/extract-conformance.test.ts +++ b/server/typescript/packages/render/test/extract/extract-conformance.test.ts @@ -203,7 +203,7 @@ describe("extract-conformance corpus", () => { .filter((n) => existsSync(join(corpus, n, "schema.json"))) .sort(); - expect(cases.length).toBe(48); + expect(cases.length).toBe(54); for (const caseName of cases) { test(caseName, () => { diff --git a/server/typescript/packages/render/test/verify-conformance.test.ts b/server/typescript/packages/render/test/verify-conformance.test.ts index d4e351f181..dd1f5ac2dc 100644 --- a/server/typescript/packages/render/test/verify-conformance.test.ts +++ b/server/typescript/packages/render/test/verify-conformance.test.ts @@ -8,6 +8,7 @@ import { ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, type PayloadField, type VerifyError, type VerifyOptions, @@ -35,6 +36,7 @@ const VERIFY_CODES = new Set([ ERR_PARTIAL_UNRESOLVED, ERR_REQUIRED_SLOT_UNUSED, ERR_OUTPUT_TAG_MISSING, + ERR_TEMPLATE_SYNTAX, ]); interface FixtureOptions { diff --git a/server/typescript/packages/sdk/package.json b/server/typescript/packages/sdk/package.json index 32fee616f4..b1ddf1a429 100644 --- a/server/typescript/packages/sdk/package.json +++ b/server/typescript/packages/sdk/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "@metaobjectsdev/metadata": "workspace:*", + "yaml": "^2.9.0", "zod": "^4.6.6" }, "devDependencies": { diff --git a/server/typescript/packages/sdk/src/workspace.ts b/server/typescript/packages/sdk/src/workspace.ts index b9c620ca11..2061367abd 100644 --- a/server/typescript/packages/sdk/src/workspace.ts +++ b/server/typescript/packages/sdk/src/workspace.ts @@ -15,7 +15,8 @@ // - Better error messages on circular extends, missing packages import { readFile, readdir, stat } from "node:fs/promises"; -import { dirname, join, resolve, basename } from "node:path"; +import { dirname, join, resolve, basename, relative, sep } from "node:path"; +import { parse as parseYaml } from "yaml"; import { type PackageManifest, PackageManifestSchema, @@ -72,35 +73,22 @@ async function readPnpmWorkspaceFile(path: string): Promise typeof p === "string"); } async function readPackageJsonWorkspaces(path: string): Promise { @@ -143,64 +131,106 @@ async function findWorkspaceConfig(start: string): Promise { - // Strip trailing `/` for normalization - const pattern = glob.replace(/\/$/, ""); +/** Directories no workspace pattern descends into (pnpm ignores these by default). */ +const SKIPPED_DIRS = new Set(["node_modules", ".git"]); - // No wildcard: literal directory - if (!pattern.includes("*")) { - const literal = join(root, pattern); - try { - const s = await stat(literal); - return s.isDirectory() ? [literal] : []; - } catch { - return []; +/** Split a pattern into path segments, dropping `./` prefixes and trailing slashes. */ +function patternSegments(glob: string): string[] { + return glob.split("/").filter((seg) => seg !== "" && seg !== "."); +} + +/** One path segment against one pattern segment (`*` = any run, `?` = one char; no `/`). */ +function matchGlobSegment(pattern: string): (value: string) => boolean { + if (pattern === "*") return () => true; + const regex = new RegExp( + "^" + + pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, "[^/]*") + .replace(/\?/g, "[^/]") + + "$", + ); + return (value) => regex.test(value); +} + +/** Does the whole path match the whole pattern? `**` spans zero or more segments. */ +function matchSegments(pattern: readonly string[], path: readonly string[]): boolean { + const [head, ...rest] = pattern; + if (head === undefined) return path.length === 0; + if (head === "**") { + for (let skip = 0; skip <= path.length; skip++) { + if (matchSegments(rest, path.slice(skip))) return true; } + return false; } + const [first, ...remaining] = path; + return first !== undefined && matchGlobSegment(head)(first) && matchSegments(rest, remaining); +} - // Split into prefix segments (literal) + last segment with wildcard. - // Supports the common cases: "packages/*", "apps/*", "packages/foo-*". - // Doesn't support deep `**` globs in this prototype. - const segments = pattern.split("/"); - const lastSegment = segments.pop()!; - const prefixDir = join(root, ...segments); - - let entries: string[]; +async function subdirectories(dir: string): Promise { try { - entries = await readdir(prefixDir); + const entries = await readdir(dir, { withFileTypes: true }); + return entries + .filter((e) => !SKIPPED_DIRS.has(e.name)) + .map((e) => e.name); } catch { return []; } +} - const matcher = matchGlobSegment(lastSegment); - const result: string[] = []; - for (const entry of entries) { - if (!matcher(entry)) continue; - const full = join(prefixDir, entry); - try { - const s = await stat(full); - if (s.isDirectory()) result.push(full); - } catch { - // skip unreadable +async function isDirectory(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} + +/** Directories under `dir` matching the remaining pattern segments. */ +async function expandSegments(dir: string, segments: readonly string[]): Promise { + const [head, ...rest] = segments; + if (head === undefined) return [dir]; + if (head === "**") { + const found = await expandSegments(dir, rest); + for (const name of await subdirectories(dir)) { + const child = join(dir, name); + if (await isDirectory(child)) found.push(...(await expandSegments(child, segments))); } + return found; } - return result; + if (!/[*?]/.test(head)) { + const child = join(dir, head); + return (await isDirectory(child)) ? expandSegments(child, rest) : []; + } + const matcher = matchGlobSegment(head); + const found: string[] = []; + for (const name of await subdirectories(dir)) { + const child = join(dir, name); + if (matcher(name) && (await isDirectory(child))) found.push(...(await expandSegments(child, rest))); + } + return found; } -function matchGlobSegment(pattern: string): (value: string) => boolean { - if (pattern === "*") return () => true; - const regex = new RegExp( - "^" + - pattern - .replace(/[.+^${}()|[\]\\]/g, "\\$&") - .replace(/\*/g, "[^/]*") + - "$", - ); - return (value) => regex.test(value); +/** + * Expand workspace globs relative to `root`: every positive pattern contributes its + * matching directories, then every `!`-negated pattern removes the ones it matches. + */ +async function expandGlobs(root: string, globs: readonly string[]): Promise { + const negations = globs + .filter((g) => g.startsWith("!")) + .map((g) => patternSegments(g.slice(1))); + const found = new Set(); + for (const glob of globs) { + if (glob.startsWith("!")) continue; + for (const dir of await expandSegments(root, patternSegments(glob))) found.add(dir); + } + return [...found].filter((dir) => { + const rel = relative(root, dir).split(sep); + return !negations.some((neg) => matchSegments(neg, rel)); + }); } // --------------------------------------------------------------------------- @@ -223,10 +253,7 @@ export async function discoverWorkspace(cwd: string): Promise { } }); }); + +describe("extractPnpmPackages — real YAML (audit finding 17)", () => { + test("zero-indent block list", () => { + expect(extractPnpmPackages("packages:\n- 'packages/*'\n- apps/*\n")).toEqual(["packages/*", "apps/*"]); + }); + test("flow list", () => { + expect(extractPnpmPackages("packages: ['packages/*', \"apps/*\"]\n")).toEqual(["packages/*", "apps/*"]); + }); + test("a # inside a quoted pattern is not a comment", () => { + expect(extractPnpmPackages("packages:\n - 'a#b/*' # trailing\n")).toEqual(["a#b/*"]); + }); + test("negations are kept as negations for the matcher", () => { + expect(extractPnpmPackages("packages:\n - 'apps/*'\n - '!apps/legacy'\n")).toEqual(["apps/*", "!apps/legacy"]); + }); + test("invalid YAML yields no globs rather than throwing", () => { + expect(extractPnpmPackages("packages: [unclosed\n")).toEqual([]); + }); +}); + +describe("discoverWorkspace — glob semantics (audit finding 17)", () => { + async function names(root: string, from: string): Promise { + const ws = await discoverWorkspace(join(root, from)); + return (ws?.packages ?? []).map((p) => p.manifest.name).sort(); + } + + test("a negation excludes a directory", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-neg-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'apps/*'\n - '!apps/legacy'\n"); + scaffoldPackage(root, "apps/web", makeManifest("@acme/web", { metaobjectsPackage: "acme::web" })); + scaffoldPackage(root, "apps/legacy", makeManifest("@acme/legacy", { metaobjectsPackage: "acme::legacy" })); + expect(await names(root, "apps/web")).toEqual(["@acme/web"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("** matches nested directories at any depth", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-glob-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'libs/**'\n"); + scaffoldPackage(root, "libs/a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "libs/group/b", makeManifest("@acme/b", { metaobjectsPackage: "acme::b" })); + expect(await names(root, "libs/a")).toEqual(["@acme/a", "@acme/b"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a single * stays one level deep", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-star-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'libs/*'\n"); + scaffoldPackage(root, "libs/a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "libs/group/b", makeManifest("@acme/b", { metaobjectsPackage: "acme::b" })); + expect(await names(root, "libs/a")).toEqual(["@acme/a"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a ** in the middle and a # in a directory name", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-mid-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - 'a#b/*'\n - 'x/**/pkg'\n"); + scaffoldPackage(root, "a#b/one", makeManifest("@acme/one", { metaobjectsPackage: "acme::one" })); + scaffoldPackage(root, "x/y/z/pkg", makeManifest("@acme/deep", { metaobjectsPackage: "acme::deep" })); + scaffoldPackage(root, "x/pkg", makeManifest("@acme/zero", { metaobjectsPackage: "acme::zero" })); + expect(await names(root, "a#b/one")).toEqual(["@acme/deep", "@acme/one", "@acme/zero"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("node_modules is never searched", async () => { + const root = mkdtempSync(join(tmpdir(), "ws-nm-")); + try { + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages:\n - '**'\n"); + scaffoldPackage(root, "a", makeManifest("@acme/a", { metaobjectsPackage: "acme::a" })); + scaffoldPackage(root, "node_modules/dep", makeManifest("@acme/dep", { metaobjectsPackage: "acme::dep" })); + expect(await names(root, "a")).toEqual(["@acme/a"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 1b01f3653a6099c2ae0195aaa868ef1f5eccf571 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 28 Oct 2026 21:55:15 -0400 Subject: [PATCH 02/23] fix: import the shared support class in the Spring and Kotlin filter-allowlist generators An ejected copy is package-renamed, so same-package bare-name resolution stopped compiling once #503 started calling the support class there. --- .../generator/kotlin/KotlinFilterAllowlistGenerator.kt | 1 + .../generator/spring/SpringFilterAllowlistGenerator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt index baddea1d3e..f3a12ea074 100644 --- a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt +++ b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinFilterAllowlistGenerator.kt @@ -4,6 +4,7 @@ package com.metaobjects.generator.kotlin // com.metaobjects.generator.kotlin when this generator is copied out via // `mvn metaobjects:eject` and its own package is renamed — an explicit import, not // same-package bare-name resolution, is what keeps the ejected copy compiling. +import com.metaobjects.generator.kotlin.KotlinGenUtil import com.metaobjects.generator.kotlin.KotlinNaming import com.metaobjects.generator.kotlin.KotlinTphPlan import com.metaobjects.generator.kotlin.PackageMapping diff --git a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java index ee88b7fbf9..03af95a822 100644 --- a/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java +++ b/server/java/codegen-spring/src/main/java/com/metaobjects/generator/spring/SpringFilterAllowlistGenerator.java @@ -4,6 +4,7 @@ // com.metaobjects.generator.spring when this generator is copied out via // `mvn metaobjects:eject` and its own package is renamed — an explicit import, not // same-package bare-name resolution, is what keeps the ejected copy compiling. +import com.metaobjects.generator.spring.SpringGeneratorSupport; import com.metaobjects.generator.spring.SpringNaming; import com.metaobjects.generator.spring.TphPlan; From b172556a62448f548e0d2d938b7b536dd5c11e6d Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 28 Oct 2026 22:07:26 -0400 Subject: [PATCH 03/23] fix: Roslyn namespace/handoff detection, ASCII Pydantic patterns, route-segment collision (audit finding 18 follow-ups) --- CHANGELOG.md | 13 +++ .../already-plural-pluralize.json | 6 ++ .../CodegenHandoffTests.cs | 23 +++++ .../csharp/MetaObjects.Cli/CodegenHandoff.cs | 2 +- .../CSharpNamingTests.cs | 12 +++ .../CodegenDriftTests.cs | 19 ++++ .../MetaObjects.Codegen/CSharpNaming.cs | 33 +++++++ .../MetaObjects.Codegen/CSharpSource.cs | 52 +++++++++++ .../MetaObjects.Codegen/CodegenDrift.cs | 11 +-- .../MetaObjects.Codegen.csproj | 3 + .../generator/util/RouteNamingTest.java | 10 ++ .../codegen/collection_name_collision.py | 34 ++++++- .../codegen/generators/entity_model.py | 45 ++++++++- .../codegen/test_collection_name_collision.py | 9 ++ .../tests/codegen/test_pattern_ascii.py | 92 +++++++++++++++++++ .../src/naming/collection-name-collision.ts | 30 +++++- .../naming/collection-name-collision.test.ts | 27 ++++++ 17 files changed, 406 insertions(+), 15 deletions(-) create mode 100644 server/csharp/MetaObjects.Codegen/CSharpSource.cs create mode 100644 server/python/tests/codegen/test_pattern_ascii.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 209e894fd1..deec816b32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -383,6 +383,19 @@ here.** name like `required7` or `rdb1` as auto-generated (new fixture `authored-name-looks-autogenerated`). - Workspace discovery parses `pnpm-workspace.yaml` as YAML and honours negations and `**`. + - C# `dotnet meta verify --codegen` infers the generated namespace, and `dotnet meta` detects + whether `codegen/Program.cs` runs `Generators.All`, by parsing the source with Roslyn instead of a + regex and a substring match. `MetaObjects.Codegen` and the `dotnet meta` tool now depend on + `Microsoft.CodeAnalysis.CSharp`. + - Python generated Pydantic models match `validator.regex` `@pattern` (and the hostname format) + as an ASCII full match: `\d`/`\w`/`\s` no longer accept non-ASCII, lookahead works, a trailing + newline is rejected. A pattern Python's `re` cannot compile, such as `\p{L}`, now fails at `gen` + with a clear error instead of at validation time. + - Codegen refuses, with `ERR_COLLECTION_NAME_COLLISION`, two served entities whose snake-cased, + pluralized REST segments are equal (`ApiKey` and `APIKey`, both `/api_keys`); TS, C# and Python + compared only the pluralized PascalCase name. Java and Kotlin already refused it. + - The Spring and Kotlin filter-allowlist generators import their support class explicitly, so an + ejected, package-renamed copy compiles again (broken since #503). - Smaller instances: Java `` glob no longer throws on metacharacters; Java and C# generated string literals escape newlines and control characters; `library.json` is read with Gson; the Kotlin relations helper imports its primary-key type; `eject` reads `pom.xml` with diff --git a/fixtures/naming-conformance/already-plural-pluralize.json b/fixtures/naming-conformance/already-plural-pluralize.json index 396d37439a..e52a9a434d 100644 --- a/fixtures/naming-conformance/already-plural-pluralize.json +++ b/fixtures/naming-conformance/already-plural-pluralize.json @@ -63,6 +63,12 @@ "entityB": "Orders", "collidesOn": "Orders", "note": "same shape as Address/Addresses — Order legacy-pluralizes to Orders; Orders is already-plural and stays Orders" + }, + { + "entityA": "ApiKey", + "entityB": "APIKey", + "collidesOn": "api_keys", + "note": "route-segment twins — the pluralized PascalCase names differ (ApiKeys / APIKeys) but both snake_case to api_key and serve /api_keys" } ] } diff --git a/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs b/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs index 48ba8f60a4..4906eb8776 100644 --- a/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs @@ -39,4 +39,27 @@ public void A_generators_dir_alone_with_no_csproj_is_not_a_handoff_target() Assert.False(CodegenHandoff.HasOwnedCodegen(_root)); } + + private bool Ignores(string program) + { + Directory.CreateDirectory(Path.Combine(_root, "codegen")); + File.WriteAllText(Path.Combine(_root, "codegen", "Program.cs"), program); + return CodegenHandoff.ProgramIgnoresList(_root); + } + + [Fact] + public void Program_running_Generators_All_does_not_ignore_the_list() + { + Assert.False(Ignores("return CodegenCli.Run(args, Generators.All);")); + Assert.False(Ignores("return Run(args, My.Codegen.Generators.All);")); + Assert.False(Ignores("using static Acme.Generators;\nreturn Run(args, All);")); + } + + [Fact] + public void Generators_All_in_a_comment_or_string_does_not_count() + { + Assert.True(Ignores("// uses Generators.All\nreturn Run(args, new IGenerator[0]);")); + Assert.True(Ignores("/* Generators.All */ Console.WriteLine(\"Generators.All\");")); + Assert.True(Ignores("return Run(args, MyGenerators.AllOfThem);")); + } } diff --git a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs index 2488b79c48..e9029094e6 100644 --- a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs +++ b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs @@ -27,7 +27,7 @@ public static bool HasGeneratorList(string projectRoot) => public static bool ProgramIgnoresList(string projectRoot) { var program = Path.Combine(projectRoot, "codegen", "Program.cs"); - return File.Exists(program) && !File.ReadAllText(program).Contains("Generators.All", StringComparison.Ordinal); + return File.Exists(program) && !MetaObjects.Codegen.CSharpSource.ReferencesMember(File.ReadAllText(program), "Generators", "All"); } ///

diff --git a/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs b/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs index 39e06fc052..0d083ab4b7 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/CSharpNamingTests.cs @@ -266,4 +266,16 @@ public void NamingConformance_collisionCasesAreRefused(CollisionCase c) Assert.Throws( () => CSharpNaming.AssertNoCollectionNameCollisions(entities)); } + + [Fact] + public void AssertNoCollectionNameCollisions_throws_forRouteSegmentTwins() + { + // ApiKey / APIKey pluralize to different names but both snake_case to /api_keys. + var entities = MultiEntityRoot(("ApiKey", false), ("APIKey", false)); + var ex = Assert.Throws( + () => CSharpNaming.AssertNoCollectionNameCollisions(entities)); + Assert.Contains("\"ApiKey\"", ex.Message); + Assert.Contains("\"APIKey\"", ex.Message); + Assert.Contains("api_keys", ex.Message); + } } diff --git a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs index 1ae5bacbf0..a3852a2992 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs @@ -193,6 +193,25 @@ public void InferNamespace_returns_null_when_no_cs_has_a_namespace() Assert.Null(CodegenDrift.InferNamespace(OutDir)); } + [Fact] + public void InferNamespace_ignores_a_namespace_in_a_comment_or_string() + { + Directory.CreateDirectory(OutDir); + File.WriteAllText(Path.Combine(OutDir, "A.g.cs"), + "// namespace Fake.Comment;\n/*\nnamespace Fake.Block {}\n*/\n" + + "public static class K { public const string S = @\"\nnamespace Fake.Str;\n\"; }\n"); + File.WriteAllText(Path.Combine(OutDir, "B.g.cs"), "namespace Real.Ns;\n"); + Assert.Equal("Real.Ns", CodegenDrift.InferNamespace(OutDir)); + } + + [Fact] + public void InferNamespace_returns_null_when_only_comments_and_strings_mention_one() + { + Directory.CreateDirectory(OutDir); + File.WriteAllText(Path.Combine(OutDir, "A.g.cs"), "// namespace Fake.Comment;\nvar s = \"namespace Fake.Str;\";\n"); + Assert.Null(CodegenDrift.InferNamespace(OutDir)); + } + private static Dictionary SnapshotDir(string dir) { var snap = new Dictionary(StringComparer.Ordinal); diff --git a/server/csharp/MetaObjects.Codegen/CSharpNaming.cs b/server/csharp/MetaObjects.Codegen/CSharpNaming.cs index cbbebe0be4..2d39b5ef6e 100644 --- a/server/csharp/MetaObjects.Codegen/CSharpNaming.cs +++ b/server/csharp/MetaObjects.Codegen/CSharpNaming.cs @@ -12,6 +12,7 @@ using static MetaObjects.Core.Field.FieldConstants; using static MetaObjects.Core.Identity.IdentityConstants; using static MetaObjects.Core.Index.IndexConstants; +using static MetaObjects.Core.Object.ObjectConstants; using static MetaObjects.Persistence.Source.SourceConstants; using static MetaObjects.Shared.BaseTypes; @@ -1177,6 +1178,38 @@ public static void AssertNoCollectionNameCollisions(IReadOnlyList en ownerOfPlural[plural] = obj; } } + AssertNoRouteSegmentCollisions(entities); + } + + /// + /// The route half. is the entity NAME snake_cased then + /// pluralized, so "ApiKey" and "APIKey" both serve "/api_keys" while their pluralized + /// PascalCase names differ and pass the DbSet-name check above. Counts only objects that + /// actually get a route: not a value object, not abstract, not sourceless, and not a + /// TPH subtype (folded into its base's table and served under the base's route). + /// + private static void AssertNoRouteSegmentCollisions(IReadOnlyList entities) + { + var ownerOfSegment = new Dictionary(StringComparer.Ordinal); + foreach (var obj in entities) + { + if (obj.IsValue() || !InstanceArtifacts.EmitsInstanceArtifacts(obj)) continue; + // ADR-0039: OwnAttr is REQUIRED — a TPH subtype declares @discriminatorValue itself. + if (obj.OwnAttr(OBJECT_ATTR_DISCRIMINATOR_VALUE) is string { Length: > 0 }) continue; + var segment = RoutePath(obj); + if (ownerOfSegment.TryGetValue(segment, out var existing)) + { + if (existing.ResolutionKey() != obj.ResolutionKey()) + throw new InvalidOperationException( + $"\"{existing.Name}\" and \"{obj.Name}\" both resolve " + + $"to the REST collection segment \"{segment}\" — rename one entity so its " + + $"snake_cased, pluralized route segment is distinct."); + } + else + { + ownerOfSegment[segment] = obj; + } + } } private static readonly char[] CSharpEscapeChars = { '\\', '"', '\t', '\n', '\r' }; diff --git a/server/csharp/MetaObjects.Codegen/CSharpSource.cs b/server/csharp/MetaObjects.Codegen/CSharpSource.cs new file mode 100644 index 0000000000..7aedf2fb17 --- /dev/null +++ b/server/csharp/MetaObjects.Codegen/CSharpSource.cs @@ -0,0 +1,52 @@ +// Reads of C# source go through Roslyn's syntax tree, never a regex or substring over the text: +// a comment or string literal that merely mentions `namespace X` or `Generators.All` is trivia or +// a literal token there, not a declaration or a member access. + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace MetaObjects.Codegen; + +public static class CSharpSource +{ + /// The name of the first top-level namespace (file-scoped or block) declared in + /// , or null when it declares none. + public static string? TopLevelNamespace(string source) + { + var root = CSharpSyntaxTree.ParseText(source).GetCompilationUnitRoot(); + return root.Members.OfType() + .Select(n => n.Name.ToString().Replace(" ", "")) + .FirstOrDefault(); + } + + /// True iff evaluates the member of a type + /// named (Generators.All, My.Ns.Generators.All), or names + /// member bare under a using static of that type. Comments and string literals never count. + public static bool ReferencesMember(string source, string typeName, string member) + { + var root = CSharpSyntaxTree.ParseText(source).GetCompilationUnitRoot(); + var nodes = root.DescendantNodes().ToList(); + + if (nodes.OfType().Any(m => + m.Name.Identifier.ValueText == member && LastName(m.Expression) == typeName)) + return true; + + var staticallyImported = nodes.OfType().Any(u => + u.StaticKeyword.IsKind(SyntaxKind.StaticKeyword) + && u.NamespaceOrType is { } t && LastName(t) == typeName); + return staticallyImported && nodes.OfType().Any(i => + i.Identifier.ValueText == member + && !(i.Parent is MemberAccessExpressionSyntax m && m.Name == i)); + } + + /// The rightmost simple name of A, A.B or global::A.B; null for anything else. + private static string? LastName(SyntaxNode node) => node switch + { + IdentifierNameSyntax id => id.Identifier.ValueText, + QualifiedNameSyntax q => LastName(q.Right), + AliasQualifiedNameSyntax a => LastName(a.Name), + MemberAccessExpressionSyntax m => LastName(m.Name), + _ => null, + }; +} diff --git a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs index 1d60595dd7..28ff23057d 100644 --- a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs +++ b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs @@ -11,7 +11,6 @@ // SAFETY: the regen is written ONLY under a fresh temp dir; the real OutDir is // read but never written. On any outcome the temp tree is removed. -using System.Text.RegularExpressions; using MetaObjects.Meta; namespace MetaObjects.Codegen; @@ -19,12 +18,6 @@ namespace MetaObjects.Codegen; /// Regenerate-to-temp + diff vs committed output (ADR-0021 D2). public static class CodegenDrift { - // Matches the FIRST top-level namespace declaration in a C# source file: - // file-scoped `namespace X.Y;` or block `namespace X.Y {` / `namespace X.Y`. - // Multiline so `^` anchors each line; we scan line-leading declarations only. - private static readonly Regex NamespaceDecl = - new(@"^\s*namespace\s+([A-Za-z_][\w.]*)\s*;?", RegexOptions.Multiline | RegexOptions.Compiled); - /// /// Infer the C# namespace embedded in the committed generated output under /// . Scans the FIRST .cs file (ordinal-sorted) @@ -48,8 +41,8 @@ public static class CodegenDrift try { text = File.ReadAllText(file); } catch { continue; } - var m = NamespaceDecl.Match(text); - if (m.Success) return m.Groups[1].Value; + var ns = CSharpSource.TopLevelNamespace(text); + if (ns is not null) return ns; } return null; } diff --git a/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj b/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj index 0a9a096a62..2056d640e7 100644 --- a/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj +++ b/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj @@ -21,6 +21,9 @@ + + "; private const string CdataOpen = " is non-null, the text BETWEEN those elements (an element's own /// text in mixed content) is appended to it, one segment per gap. /// is set for the body of an element that DID close: an - /// open tag in there with no close tag anywhere after it (List<int> and ...) is - /// prose, not an element. + /// open tag in there that directly follows an identifier character and has no close tag + /// anywhere after it (List<int> and ...) is prose, not an element. One after + /// >, whitespace or the start of the body is a child whose close the model forgot. /// private static void ParseChildren(string inner, bool ci, Dictionary out_, List? loose, bool literalUnclosed) @@ -176,7 +180,8 @@ private static void ParseChildren(string inner, bool ci, Dictionary 0 && IsIdentChar(inner[m.Index - 1]) + && !m.Groups[2].Value.TrimEnd().EndsWith('/') && !anyClose.Match(inner, m.Index + m.Length).Success) { scan = m.Index + m.Length; diff --git a/server/csharp/MetaObjects.Render/MustacheTags.cs b/server/csharp/MetaObjects.Render/MustacheTags.cs index 2b376a0eaa..234e05ba7f 100644 --- a/server/csharp/MetaObjects.Render/MustacheTags.cs +++ b/server/csharp/MetaObjects.Render/MustacheTags.cs @@ -100,6 +100,13 @@ public static List Scan(string text) return tags; } + /// The first partial of a text that also changes delimiters, or null. Such a text + /// does not render: the inlined partial would be parsed under the changed delimiters. + public static string? DelimitedPartial(IReadOnlyList tags) => + tags.Any(t => t.Kind == MustacheTagKind.Delimiters) + ? tags.FirstOrDefault(t => t.Kind == MustacheTagKind.Partial)?.Value + : null; + /// with every comment cut out: what a tag-presence check may scan. public static string WithoutComments(string text, IReadOnlyList tags) { diff --git a/server/csharp/MetaObjects.Render/Renderer.cs b/server/csharp/MetaObjects.Render/Renderer.cs index 09d2d8cc5f..08b737497b 100644 --- a/server/csharp/MetaObjects.Render/Renderer.cs +++ b/server/csharp/MetaObjects.Render/Renderer.cs @@ -59,8 +59,7 @@ private static string Expand(string text, IProvider provider, IReadOnlyList t.Kind == MustacheTagKind.Partial).ToList(); if (partials.Count == 0) return text; - // Inlined text is parsed under the surrounding delimiters, not the default ones. - if (tags.Any(t => t.Kind == MustacheTagKind.Delimiters)) + if (MustacheTags.DelimitedPartial(tags) is not null) throw new RenderException("a template that uses partials cannot also change delimiters ({{=<% %>=}})"); var sb = new System.Text.StringBuilder(); int pos = 0; diff --git a/server/csharp/MetaObjects.Render/Verify.cs b/server/csharp/MetaObjects.Render/Verify.cs index 449ed90f38..f7220fc218 100644 --- a/server/csharp/MetaObjects.Render/Verify.cs +++ b/server/csharp/MetaObjects.Render/Verify.cs @@ -100,6 +100,14 @@ private static List Tokenize(string text) } // `text` minus its comments, for the output-tag scan. + /// A text that changes delimiters and also uses a partial does not render: report it + /// against its first partial. + private static void ReportDelimitedPartial(string text, List errors) + { + if (MustacheTags.DelimitedPartial(MustacheTags.Scan(text)) is { } partial) + errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, partial)); + } + private static string StaticText(string text) => MustacheTags.WithoutComments(text, MustacheTags.Scan(text)); private static PayloadField? Find(IReadOnlyList fields, string name) => @@ -226,6 +234,7 @@ void Walk(IReadOnlyList tokens, List> stack, Li errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)); break; } + ReportDelimitedPartial(text, errors); staticTexts.Add(StaticText(text)); Walk(partialToks, stack, new List(seen) { p.Value }, inElement); break; @@ -239,6 +248,7 @@ void Walk(IReadOnlyList tokens, List> stack, Li { return [new VerifyError(ERR_TEMPLATE_SYNTAX, e.Section)]; } + ReportDelimitedPartial(templateText, errors); staticTexts[0] = StaticText(templateText); Walk(bodyToks, [root], []); diff --git a/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java index 42cf13579a..4de481a952 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java +++ b/server/java/render/src/main/java/com/metaobjects/render/MustacheTags.java @@ -113,6 +113,15 @@ static List scan(String text) { return tags; } + /** + * The first partial of a text that also changes delimiters, or {@code null}. Such a text does + * not render: the inlined partial would be parsed under the changed delimiters. + */ + static String delimitedPartial(List tags) { + if (tags.stream().noneMatch(t -> t.kind() == Kind.DELIMITERS)) return null; + return tags.stream().filter(t -> t.kind() == Kind.PARTIAL).map(Tag::value).findFirst().orElse(null); + } + /** {@code text} with every comment cut out: what a tag-presence check may scan. */ static String withoutComments(String text, List tags) { StringBuilder sb = new StringBuilder(text.length()); diff --git a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java index 8c7e615438..bf30230d27 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/Renderer.java +++ b/server/java/render/src/main/java/com/metaobjects/render/Renderer.java @@ -128,8 +128,7 @@ private static String preExpandPartials(String text, Provider provider, Deque partials = tags.stream() .filter(t -> t.kind() == MustacheTags.Kind.PARTIAL).toList(); if (partials.isEmpty()) return text; - // Inlined text is parsed under the surrounding delimiters, not the default ones. - if (tags.stream().anyMatch(t -> t.kind() == MustacheTags.Kind.DELIMITERS)) { + if (MustacheTags.delimitedPartial(tags) != null) { throw new RenderException( "a template that uses partials cannot also change delimiters ({{=<% %>=}})"); } diff --git a/server/java/render/src/main/java/com/metaobjects/render/Verify.java b/server/java/render/src/main/java/com/metaobjects/render/Verify.java index 5f8819ec5a..496beba8b6 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/Verify.java +++ b/server/java/render/src/main/java/com/metaobjects/render/Verify.java @@ -107,6 +107,7 @@ public static List check( errors.add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); return errors; } + reportDelimitedPartial(templateText, errors); staticTexts.add(staticText(templateText)); WalkContext ctx = new WalkContext(provider, root, errors, referencedAtRoot, staticTexts); @@ -236,6 +237,7 @@ private static void walk( ctx.errors().add(new VerifyError(ERR_TEMPLATE_SYNTAX, e.section())); break; } + reportDelimitedPartial(text, ctx.errors()); ctx.staticTexts().add(staticText(text)); List nextSeen = new ArrayList<>(seen); nextSeen.add(p.value()); @@ -330,6 +332,13 @@ static List tokenize(String text) { return root; } + /** A text that changes delimiters and also uses a partial does not render: report it against + * its first partial. */ + private static void reportDelimitedPartial(String text, List errors) { + String partial = MustacheTags.delimitedPartial(MustacheTags.scan(text)); + if (partial != null) errors.add(new VerifyError(ERR_TEMPLATE_SYNTAX, partial)); + } + /** {@code text} minus its comments, for the output-tag scan. */ private static String staticText(String text) { return MustacheTags.withoutComments(text, MustacheTags.scan(text)); diff --git a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java index d5495b546b..4b1110957e 100644 --- a/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java +++ b/server/java/render/src/main/java/com/metaobjects/render/extract/XmlForgivingReader.java @@ -51,6 +51,11 @@ public final class XmlForgivingReader { private static final Pattern ATTR = Pattern.compile( "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"); + // a character an identifier can end with: '<' right after one is a type argument, not a tag. + private static boolean isIdentChar(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_'; + } + private static final String COMMENT_OPEN = ""; private static final String CDATA_OPEN = " readRootless(String text, boolean caseInsensitive) { * Parse {@code inner}'s elements into {@code out}. When {@code loose} is non-null, the text * BETWEEN those elements (an element's own text in mixed content) is appended to it, one * segment per gap. {@code literalUnclosed} is set for the body of an element that DID - * close: an open tag in there with no close tag anywhere after it ({@code List and - * ...}) is prose, not an element. + * close: an open tag in there that directly follows an identifier character and has no close + * tag anywhere after it ({@code List and ...}) is prose, not an element. One after + * {@code >}, whitespace or the start of the body is a child whose close the model forgot. */ private void parseChildren(String inner, boolean ci, Map out, List loose, boolean literalUnclosed) { @@ -189,7 +195,8 @@ private void parseChildren(String inner, boolean ci, Map out, Li int pos = 0; int scan = 0; while (m.find(scan)) { - if (literalUnclosed && !m.group(2).stripTrailing().endsWith("/") + if (literalUnclosed && m.start() > 0 && isIdentChar(inner.charAt(m.start() - 1)) + && !m.group(2).stripTrailing().endsWith("/") && !closeTag.matcher(inner).region(m.end(), inner.length()).find()) { scan = m.end(); continue; diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index 6be28f0479..39f613418a 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -260,14 +260,19 @@ def packaged_module_path(entry: GeneratorEntry) -> Path: return Path(module.__file__) +def _ordered(text: str) -> list[str]: + return [line.rstrip() for line in text.splitlines() if line.strip()] + + def _normalised(text: str) -> Counter[str]: - return Counter(line.rstrip() for line in text.splitlines() if line.strip()) + return Counter(_ordered(text)) def owned_status(root: Path, entry: GeneratorEntry) -> str | None: """``identical`` / ``DIFFERS: N behind, M of your own``, or ``None`` if not owned. - ``identical`` means the exact text (line endings aside): an edit that only reorders + ``identical`` means the reference lines in the reference order (trailing whitespace and + blank lines aside): an edit that only reorders lines is a customisation, because ``--undo`` deletes an ``identical`` copy. *behind* counts reference lines the copy lacks, *of your own* counts copy lines the reference lacks, as in TS ``owned-copy.ts``. @@ -319,9 +324,10 @@ def _status(behind: int, own: int) -> str: def _compare(mine_text: str, ref_text: str) -> str: - """``identical`` only for the exact reference text, line endings aside. The line counts - ignore order, so a reorder reads ``DIFFERS: 0 behind, 0 of your own, reordered``.""" - if mine_text.replace("\r\n", "\n") == ref_text.replace("\r\n", "\n"): + """``identical`` only for the reference lines in the reference order, trailing whitespace and + blank lines aside. The line counts ignore order, so a reorder reads ``DIFFERS: 0 behind, 0 of + your own, reordered``.""" + if _ordered(mine_text) == _ordered(ref_text): return "identical" behind, own = _line_diff(mine_text, ref_text) status = _status(behind, own) diff --git a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py index 71e5cea595..4ee86c899c 100644 --- a/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py +++ b/server/python/src/metaobjects/render/extract/xml_forgiving_reader.py @@ -36,6 +36,8 @@ # the body) — LLMs commonly end a block with the wrong tag. _CLOSE_TAG = re.compile(r"") _CLOSE_TAG_CI = re.compile(r"", re.IGNORECASE) +# a character an identifier can end with: '<' right after one is a type argument, not a tag. +_IDENT_CHAR = re.compile(r"[A-Za-z0-9_]") # one attribute: name = "double" | 'single' | bareword. _ATTR = re.compile(r"""([A-Za-z_:][A-Za-z0-9_:.\-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s/>]+))""") @@ -158,8 +160,9 @@ def _parse_children( """Parse ``inner``'s elements into ``out``. When ``loose`` is not ``None``, the text BETWEEN those elements (an element's own text in mixed content) is appended to it, one segment per gap. ``literal_unclosed`` is set for the body of an element that DID - close: an open tag in there with no close tag anywhere after it (``List and ...``) - is prose, not an element.""" + close: an open tag in there that directly follows an identifier character and has no + close tag anywhere after it (``List and ...``) is prose, not an element. One after + ``>``, whitespace or the start of the body is a child whose close the model forgot.""" open_tag = _OPEN_TAG_CI if ci else _OPEN_TAG any_close = _CLOSE_TAG_CI if ci else _CLOSE_TAG pos = 0 @@ -170,6 +173,8 @@ def _parse_children( break if ( literal_unclosed + and m.start() > 0 + and _IDENT_CHAR.match(inner[m.start() - 1]) is not None and not (m.group(2) or "").rstrip().endswith("/") and any_close.search(inner, m.end()) is None ): diff --git a/server/python/src/metaobjects/render/mustache_tags.py b/server/python/src/metaobjects/render/mustache_tags.py index 95f4c68847..9b1f54f891 100644 --- a/server/python/src/metaobjects/render/mustache_tags.py +++ b/server/python/src/metaobjects/render/mustache_tags.py @@ -109,6 +109,14 @@ def scan(text: str) -> list[Tag]: return tags +def delimited_partial(tags: list[Tag]) -> str | None: + """The first partial of a text that also changes delimiters, or ``None``. Such a text does + not render: the inlined partial would be parsed under the changed delimiters.""" + if not any(t.kind == KIND_DELIMITERS for t in tags): + return None + return next((t.value for t in tags if t.kind == KIND_PARTIAL), None) + + def without_comments(text: str, tags: list[Tag]) -> str: """``text`` with every comment tag cut out — what a tag-presence check may scan.""" out: list[str] = [] diff --git a/server/python/src/metaobjects/render/verify.py b/server/python/src/metaobjects/render/verify.py index 9ae8bf5c2a..8b8e2847e0 100644 --- a/server/python/src/metaobjects/render/verify.py +++ b/server/python/src/metaobjects/render/verify.py @@ -32,6 +32,7 @@ KIND_SECTION, KIND_TRIPLE, MustacheSyntaxError, + delimited_partial, scan, without_comments, ) @@ -167,6 +168,13 @@ def _tokenize(text: str) -> list[_Token]: return root +def _delimited_partial_errors(text: str) -> list[VerifyError]: + """A text that changes delimiters and also uses a partial does not render: report it + against its first partial.""" + partial = delimited_partial(scan(text)) + return [] if partial is None else [VerifyError(ERR_TEMPLATE_SYNTAX, partial)] + + def _static_text(text: str) -> str: """``text`` minus its comments, for the output-tag scan.""" return without_comments(text, scan(text)) @@ -320,6 +328,7 @@ def walk( except MustacheSyntaxError as e: errors.append(VerifyError(ERR_TEMPLATE_SYNTAX, e.section)) continue + errors.extend(_delimited_partial_errors(body)) static_texts.append(_static_text(body)) walk(partial_tokens, stack, [*seen, tok.value], in_element) @@ -327,6 +336,7 @@ def walk( body_tokens = _tokenize(template_text) except MustacheSyntaxError as e: return [VerifyError(ERR_TEMPLATE_SYNTAX, e.section)] + errors.extend(_delimited_partial_errors(template_text)) static_texts[0] = _static_text(template_text) walk(body_tokens, [root], []) diff --git a/server/python/tests/codegen/test_eject_undo.py b/server/python/tests/codegen/test_eject_undo.py index 88cb4085c8..2488b783c9 100644 --- a/server/python/tests/codegen/test_eject_undo.py +++ b/server/python/tests/codegen/test_eject_undo.py @@ -163,3 +163,14 @@ def test_undo_keeps_a_copy_whose_only_edit_is_a_reordering(tmp_path: Path, monke assert path.exists() assert main(["eject", "--undo", "--force", "names"]) == 0 assert not path.exists() + + +def test_undo_removes_a_copy_whose_only_edit_is_whitespace(tmp_path: Path, monkeypatch, capsys) -> None: + monkeypatch.chdir(tmp_path) + assert main(["eject", "names"]) == 0 + path = tmp_path / GEN / "names.py" + # An editor's save-time cleanup: trailing whitespace and an extra blank line, nothing reordered. + path.write_text("\n".join(line + " " for line in path.read_text().splitlines()) + "\n\n\n") + capsys.readouterr() + assert main(["eject", "--undo", "names"]) == 0 + assert not path.exists() diff --git a/server/python/tests/conformance/test_extract_conformance.py b/server/python/tests/conformance/test_extract_conformance.py index 915dc960ae..977215e779 100644 --- a/server/python/tests/conformance/test_extract_conformance.py +++ b/server/python/tests/conformance/test_extract_conformance.py @@ -61,7 +61,7 @@ def _cases() -> list[str]: def test_discovers_all_extract_conformance_cases() -> None: """FR-011: lock the corpus size so a deleted fixture fails CI rather than silently reducing coverage. Mirrors the TS / Java / C# count guards.""" - assert len(_cases()) == 57 + assert len(_cases()) == 58 _NORMALIZE_MODES = {"none", "collapse", "strip"} diff --git a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts index 84175fe417..df9dc6ebae 100644 --- a/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts +++ b/server/typescript/packages/render/src/extract/xml-forgiving-reader.ts @@ -38,6 +38,9 @@ const CLOSE_TAG_SRC = ""; // one attribute: name = "double" | 'single' | bareword. const ATTR_SRC = "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))"; +/** A character an identifier can end with: `<` right after one is a type argument, not a tag. */ +const IDENT_CHAR = /^[A-Za-z0-9_]$/; + const COMMENT_OPEN = ""; const CDATA_OPEN = " and ...`) is prose, not an element. + * `literalUnclosed` is set for the body of an element that DID close: an open tag in there that + * directly follows an identifier character and has no close tag anywhere after it + * (`List and ...`) is prose, not an element. One after `>`, whitespace or the start of the + * body is a child whose close the model forgot. */ function parseChildren( inner: string, @@ -161,6 +166,7 @@ function parseChildren( const afterOpen = m.index + m[0].length; if ( literalUnclosed && + IDENT_CHAR.test(inner[m.index - 1] ?? "") && !(m[2] ?? "").trimEnd().endsWith("/") && matchFrom(CLOSE_TAG_SRC, flags, inner, afterOpen) == null ) { diff --git a/server/typescript/packages/render/src/verify.ts b/server/typescript/packages/render/src/verify.ts index 7ced1db61b..1632005643 100644 --- a/server/typescript/packages/render/src/verify.ts +++ b/server/typescript/packages/render/src/verify.ts @@ -157,6 +157,13 @@ export function hasDelimiterChange(tokens: readonly Token[]): boolean { return flatTokens(tokens).some((tok) => tok[0] === TOKEN_DELIMITERS); } +/** A text that changes delimiters and also uses a partial does not render: the inlined partial + * would be parsed under the changed delimiters. Reported against the text's first partial. */ +function delimiterConflict(tokens: readonly Token[]): VerifyError | undefined { + const first = partialTokens(tokens)[0]; + return first !== undefined && hasDelimiterChange(tokens) ? { code: ERR_TEMPLATE_SYNTAX, path: first[0] } : undefined; +} + function commentSpans(tokens: readonly Token[]): [number, number][] { return flatTokens(tokens) .filter((tok) => tok[0] === TOKEN_COMMENT) @@ -286,6 +293,8 @@ export function verify( errors.push(parsed); break; } + const conflict = delimiterConflict(parsed); + if (conflict) errors.push(conflict); staticTexts.push(withoutComments(text, parsed)); walk(parsed, stack, [...seen, value], positional); break; @@ -298,6 +307,8 @@ export function verify( const bodyTokens = parseOrSyntaxError(templateText); if (!Array.isArray(bodyTokens)) return [bodyTokens]; + const conflict = delimiterConflict(bodyTokens); + if (conflict) errors.push(conflict); staticTexts[0] = withoutComments(templateText, bodyTokens); walk(bodyTokens, [root], []); diff --git a/server/typescript/packages/render/test/extract/extract-conformance.test.ts b/server/typescript/packages/render/test/extract/extract-conformance.test.ts index c0675b8a14..5492ee3955 100644 --- a/server/typescript/packages/render/test/extract/extract-conformance.test.ts +++ b/server/typescript/packages/render/test/extract/extract-conformance.test.ts @@ -203,7 +203,7 @@ describe("extract-conformance corpus", () => { .filter((n) => existsSync(join(corpus, n, "schema.json"))) .sort(); - expect(cases.length).toBe(57); + expect(cases.length).toBe(58); for (const caseName of cases) { test(caseName, () => { From de09ddf14846f5e6c3ec6cebbdd1d510eb15dd4e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 28 Oct 2026 23:47:25 -0400 Subject: [PATCH 10/23] no-mistakes(review): apply reviewer inline fixes --- server/csharp/MetaObjects.Render/Verify.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/csharp/MetaObjects.Render/Verify.cs b/server/csharp/MetaObjects.Render/Verify.cs index f7220fc218..d4ef45e6b3 100644 --- a/server/csharp/MetaObjects.Render/Verify.cs +++ b/server/csharp/MetaObjects.Render/Verify.cs @@ -99,7 +99,6 @@ private static List Tokenize(string text) return root; } - // `text` minus its comments, for the output-tag scan. /// A text that changes delimiters and also uses a partial does not render: report it /// against its first partial. private static void ReportDelimitedPartial(string text, List errors) @@ -108,6 +107,7 @@ private static void ReportDelimitedPartial(string text, List errors errors.Add(new VerifyError(ERR_TEMPLATE_SYNTAX, partial)); } + // `text` minus its comments, for the output-tag scan. private static string StaticText(string text) => MustacheTags.WithoutComments(text, MustacheTags.Scan(text)); private static PayloadField? Find(IReadOnlyList fields, string name) => From 27ad36b8dc77d323607736dec559b6ed8ff169b2 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Thu, 29 Oct 2026 20:14:57 -0400 Subject: [PATCH 11/23] no-mistakes(review): Drop C# source inference and Roslyn; tighten delimiter checks --- CHANGELOG.md | 22 ++++--- docs/CONFORMANCE.md | 2 +- .../migrations/docs-are-generators.md | 11 ++-- docs/ports/csharp.md | 4 +- .../expected-drift.python.json | 1 + .../expected-drift.python.json | 1 + .../expected-drift.json | 1 + .../partials/leaf.mustache | 1 + .../payload.json | 1 + .../template.mustache | 1 + .../CodegenHandoffTests.cs | 23 ------- .../OneGeneratorListTests.cs | 63 ------------------- .../VerifySubverbTests.cs | 60 ++++-------------- .../csharp/MetaObjects.Cli/CodegenHandoff.cs | 8 --- server/csharp/MetaObjects.Cli/EjectCommand.cs | 10 +-- server/csharp/MetaObjects.Cli/InitCommand.cs | 18 +----- server/csharp/MetaObjects.Cli/Program.cs | 10 +-- .../csharp/MetaObjects.Cli/VerifyCommand.cs | 13 +--- .../CodegenDriftTests.cs | 61 ------------------ .../MetaObjects.Codegen/CSharpSource.cs | 52 --------------- .../csharp/MetaObjects.Codegen/CodegenCli.cs | 18 +----- .../MetaObjects.Codegen/CodegenDrift.cs | 29 --------- .../MetaObjects.Codegen.csproj | 3 - .../csharp/MetaObjects.Render/MustacheTags.cs | 14 +++-- server/csharp/MetaObjects.Render/Renderer.cs | 4 +- server/csharp/MetaObjects.Render/Verify.cs | 13 ++-- .../com/metaobjects/render/MustacheTags.java | 9 ++- .../java/com/metaobjects/render/Renderer.java | 8 +-- .../java/com/metaobjects/render/Verify.java | 13 ++-- .../src/metaobjects/render/mustache_tags.py | 11 ++-- .../python/src/metaobjects/render/renderer.py | 4 +- .../python/src/metaobjects/render/verify.py | 21 ++++--- .../tests/render/test_verify_conformance.py | 9 +-- .../typescript/packages/render/src/render.ts | 9 ++- .../typescript/packages/render/src/verify.ts | 24 ++++--- .../packages/render/test/render.test.ts | 9 +++ 36 files changed, 135 insertions(+), 426 deletions(-) create mode 100644 fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache create mode 100644 fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json create mode 100644 fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache delete mode 100644 server/csharp/MetaObjects.Codegen/CSharpSource.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 78afeb37ce..ab8666c287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,16 @@ here.** ### Changed +- **C# no longer reads source code to work out its configuration.** `dotnet meta verify + --codegen` regenerates under the namespace `gen` resolves (`--namespace`, else `Generated`) + instead of reading it from the committed output, so output generated under another namespace + is drift. Pass the same `--namespace` to both. `dotnet meta gen`, `verify --codegen`, `init` + and `eject` no longer check whether `codegen/Program.cs` runs `Generators.All`. + **Upgrade step:** a 1.0 project whose `codegen/Program.cs` builds its own generator list + still runs that list and ignores `codegen/Generators.cs`, and nothing warns about it. Move + each entry into `Generators.All` and replace `Program.cs` with the two lines + `using MetaObjects.Codegen;` and `return CodegenCli.Run(args, Generators.All);`. + - **On Postgres, `meta verify --db` and `meta migrate` ask Postgres whether a CHECK or an index expression is the one stored, instead of comparing rewritten text.** For each table the plan creates a temporary table with the live columns inside a transaction it rolls back (a savepoint @@ -391,9 +401,11 @@ here.** - Template verify and render agree with the Mustache engine in C#, Java, Python and TypeScript: a partial tag in a comment is not expanded, `{{=<% %>=}}` raises no false `ERR_VAR_NOT_ON_PAYLOAD`, `@requiredTags` is not satisfied inside a comment, and an unbalanced - section is `ERR_TEMPLATE_SYNTAX`. Render rejects partials combined with a delimiter change, - and verify reports that combination as `ERR_TEMPLATE_SYNTAX` on the first partial (fixture - `verify-set-delimiter-with-partial`). The Python renderer rejects every `{{=`. + section is `ERR_TEMPLATE_SYNTAX`. Render rejects a delimiter change in a template that uses + partials, or in a partial body, and verify reports the same as `ERR_TEMPLATE_SYNTAX` at the + first partial or the partial's ref (fixtures `verify-set-delimiter-with-partial`, + `verify-set-delimiter-in-partial`). The Python renderer rejects every `{{=`, so Python verify + reports every one (`expected-drift.python.json` overrides). - The forgiving XML extract reader (all five ports) decodes entities, reads CDATA as text, ignores comments, keeps literal `List` prose inside closed elements, and locates the real root. An unclosed open tag after `>`, whitespace or the start of the body is still a child @@ -406,10 +418,6 @@ here.** name like `required7` or `rdb1` as auto-generated (new fixture `authored-name-looks-autogenerated`). - Workspace discovery parses `pnpm-workspace.yaml` as YAML and honours negations and `**`. - - C# `dotnet meta verify --codegen` infers the generated namespace, and `dotnet meta` detects - whether `codegen/Program.cs` runs `Generators.All`, by parsing the source with Roslyn instead of a - regex and a substring match. `MetaObjects.Codegen` and the `dotnet meta` tool now depend on - `Microsoft.CodeAnalysis.CSharp`. - Python generated Pydantic models match `validator.regex` `@pattern` (and the hostname format) as an ASCII full match: `\d`/`\w`/`\s` no longer accept non-ASCII, lookahead works, a trailing newline is rejected. A pattern Python's `re` cannot compile, such as `\p{L}`, now fails at `gen` diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index 9adf6f0a3b..5d861ca936 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -28,7 +28,7 @@ regenerate with `ls -d fixtures//*/ | wc -l` for directory-shaped corpor |---|---|---|---|---|---|---| | [`fixtures/conformance/`](../fixtures/conformance/) (metamodel) | 413 | ✓ | ✓ | inherits via `metadata-ktx` | ✓ | ✓ | | [`fixtures/yaml-conformance/`](../fixtures/yaml-conformance/) | 16 | 16 / 16 | 15 / 16 (1 ledgered: `yaml-quoted-leading-zero` — Java pipeline strips quotes off `"007"`) | inherits via Java | 15 / 16 (1 ledgered: `error-yaml-coerced-hex-in-string` — YamlDotNet doesn't coerce `0xFF`) | 16 / 16 | -| [`fixtures/verify-conformance/`](../fixtures/verify-conformance/) | 44 | ✓ | ✓ | inherits via Java | ✓ | ✓ | +| [`fixtures/verify-conformance/`](../fixtures/verify-conformance/) | 45 | ✓ | ✓ | inherits via Java | ✓ | ✓ | | [`fixtures/verify-strict-conformance/`](../fixtures/verify-strict-conformance/) | 1 | ✓ | — | — | — | ✓ | | [`fixtures/render-conformance/`](../fixtures/render-conformance/) | 16 | ✓ | ✓ | inherits via Java | ✓ | ✓ | | [`fixtures/jinja-render-conformance/`](../fixtures/jinja-render-conformance/) (ADR-0061 — Jinja bodies render the same in jinja2 and nunjucks) | 15 | ✓ (nunjucks) | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | refuses, via Java | refuses Jinja (`ERR_TEMPLATE_ENGINE_UNSUPPORTED`) | ✓ (jinja2) | diff --git a/docs/features/migrations/docs-are-generators.md b/docs/features/migrations/docs-are-generators.md index ef3ff7f10b..de92e9b139 100644 --- a/docs/features/migrations/docs-are-generators.md +++ b/docs/features/migrations/docs-are-generators.md @@ -324,14 +324,11 @@ A project that already has `codegen/Program.cs` with an inline list: move each e `new Codegen.Generators.()`, a packaged one `new MetaObjects.Codegen.Generators.()`), and replace `Program.cs` with the two lines above. `dotnet meta init` writes `Generators.cs` beside an existing `Program.cs` and leaves `Program.cs` alone. Until -`Program.cs` runs `Generators.All`, `dotnet meta gen` prints this and exits 2, writing nothing -(`dotnet meta verify --codegen` in place of `dotnet meta gen` for the gate): +`Program.cs` runs `Generators.All`, `dotnet meta gen` and `dotnet meta verify --codegen` run +the list `Program.cs` builds and ignore `Generators.cs`; nothing warns about it, so replace +`Program.cs` as part of the upgrade. -``` -dotnet meta gen: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and `return CodegenCli.Run(args, Generators.All);`. See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section "C#". -``` - -`init` and `eject` say the same, and warn when `codegen/Codegen.csproj` still pins an older +`init` and `eject` warn when `codegen/Codegen.csproj` still pins an older `MetaObjects.Codegen`: set that version to the tool's, or the 1.0 codegen runs. **A template spec.** Through 1.0, `dotnet meta gen` and `verify --codegen` also read a diff --git a/docs/ports/csharp.md b/docs/ports/csharp.md index 16c7ee5844..58e85ff317 100644 --- a/docs/ports/csharp.md +++ b/docs/ports/csharp.md @@ -184,8 +184,8 @@ in `.metaobjects/config.json`, below). dotnet meta gen ./metaobjects --out ./Generated --namespace Acme.Blog # The drift gate: regenerate into a temp tree and fail (exit 1, naming the files) if the -# committed ./Generated differs. Run it in CI; the namespace is inferred from ./Generated. -dotnet meta verify ./metaobjects --codegen --out ./Generated +# committed ./Generated differs. Run it in CI with the --namespace gen used. +dotnet meta verify ./metaobjects --codegen --out ./Generated --namespace Acme.Blog ``` `verify --codegen` is the gate for generated code. `dotnet meta verify ./metaobjects diff --git a/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json new file mode 100644 index 0000000000..751ee879de --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-clean/expected-drift.python.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "<% %>" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json new file mode 100644 index 0000000000..8782129305 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-drift/expected-drift.python.json @@ -0,0 +1 @@ +[ { "code": "ERR_VAR_NOT_ON_PAYLOAD", "path": "nope" }, { "code": "ERR_TEMPLATE_SYNTAX", "path": "<% %>" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json b/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json new file mode 100644 index 0000000000..9787342aa4 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/expected-drift.json @@ -0,0 +1 @@ +[ { "code": "ERR_TEMPLATE_SYNTAX", "path": "partials/leaf" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache b/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache new file mode 100644 index 0000000000..27c658ae77 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/partials/leaf.mustache @@ -0,0 +1 @@ +{{=<% %>=}}hi diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json b/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json new file mode 100644 index 0000000000..c71e5ee8a7 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/payload.json @@ -0,0 +1 @@ +[ { "name": "name" } ] diff --git a/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache b/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache new file mode 100644 index 0000000000..cd616c89f3 --- /dev/null +++ b/fixtures/verify-conformance/verify-set-delimiter-in-partial/template.mustache @@ -0,0 +1 @@ +{{> partials/leaf}} {{name}} diff --git a/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs b/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs index 4906eb8776..48ba8f60a4 100644 --- a/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/CodegenHandoffTests.cs @@ -39,27 +39,4 @@ public void A_generators_dir_alone_with_no_csproj_is_not_a_handoff_target() Assert.False(CodegenHandoff.HasOwnedCodegen(_root)); } - - private bool Ignores(string program) - { - Directory.CreateDirectory(Path.Combine(_root, "codegen")); - File.WriteAllText(Path.Combine(_root, "codegen", "Program.cs"), program); - return CodegenHandoff.ProgramIgnoresList(_root); - } - - [Fact] - public void Program_running_Generators_All_does_not_ignore_the_list() - { - Assert.False(Ignores("return CodegenCli.Run(args, Generators.All);")); - Assert.False(Ignores("return Run(args, My.Codegen.Generators.All);")); - Assert.False(Ignores("using static Acme.Generators;\nreturn Run(args, All);")); - } - - [Fact] - public void Generators_All_in_a_comment_or_string_does_not_count() - { - Assert.True(Ignores("// uses Generators.All\nreturn Run(args, new IGenerator[0]);")); - Assert.True(Ignores("/* Generators.All */ Console.WriteLine(\"Generators.All\");")); - Assert.True(Ignores("return Run(args, MyGenerators.AllOfThem);")); - } } diff --git a/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs b/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs index 9d55499853..66303243e7 100644 --- a/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/OneGeneratorListTests.cs @@ -300,8 +300,6 @@ public void Eject_into_an_empty_project_lists_the_ejected_generator() Assert.Equal(CodegenScaffold.ProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); } - // ---- a 1.0 Program.cs, which never runs Generators.All, is refused -------------------------- - private const string LegacyProgramCs = "using MetaObjects.Codegen;\n" + "IReadOnlyList generators = [new MetaObjects.Codegen.Generators.EntityGenerator()];\n" + @@ -317,67 +315,6 @@ private void WriteLegacyCodegenProject(bool withGeneratorList) File.WriteAllText(Path.Combine(_tmp, "codegen", "Generators.cs"), CodegenScaffold.GeneratorsCs([])); } - private static string LegacyProgramLine(string command) => - $"{command}: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in " + - "codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and " + - "`return CodegenCli.Run(args, Generators.All);`. See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\"."; - - [Fact] - public void Gen_with_a_1_0_program_refuses_with_the_migration_line_and_writes_nothing() - { - WriteLegacyCodegenProject(withGeneratorList: true); - - var (exit, _, stderr) = CliProcess.Run(_tmp, "gen", MetaDir, "--out", OutDir); - - Assert.Equal(2, exit); - Assert.Equal(LegacyProgramLine("dotnet meta gen"), stderr.Trim()); - Assert.Equal(CodegenCli.LegacyProgram("dotnet meta gen"), stderr.Trim()); - Assert.Empty(FilesUnder(OutDir)); - } - - [Fact] - public void Verify_codegen_with_a_1_0_program_refuses_with_the_migration_line() - { - WriteLegacyCodegenProject(withGeneratorList: true); - - var (exit, _, stderr) = CliProcess.Run(_tmp, "verify", MetaDir, "--codegen", "--out", OutDir); - - Assert.Equal(2, exit); - Assert.Contains(LegacyProgramLine("dotnet meta verify --codegen"), stderr); - } - - [Fact] - public void Init_beside_a_1_0_program_says_to_replace_it_instead_of_claiming_gen_runs_the_list() - { - WriteLegacyCodegenProject(withGeneratorList: false); - - var lines = InitCommand.Run(_tmp); - - Assert.Contains(" wrote codegen/Generators.cs", lines); - Assert.Contains(" kept codegen/Program.cs (already yours)", lines); - Assert.DoesNotContain(lines, l => l.Contains("is the generator list `dotnet meta gen` runs", StringComparison.Ordinal)); - Assert.Contains(lines, l => l.StartsWith("codegen/Program.cs does not run Generators.All", StringComparison.Ordinal)); - Assert.Contains(" return CodegenCli.Run(args, Generators.All);", lines); - Assert.Contains("See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\".", lines); - Assert.Equal(LegacyProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); - } - - [Fact] - public void Eject_beside_a_1_0_program_says_to_replace_it_instead_of_claiming_gen_runs_the_list() - { - WriteLegacyCodegenProject(withGeneratorList: false); - - var outcome = EjectCommand.Run(["entity"], _tmp, force: false); - - Assert.Equal(0, outcome.ExitCode); - Assert.Contains(outcome.Out, l => l.StartsWith("codegen/Generators.cs lists the generator(s) you just ejected", StringComparison.Ordinal)); - Assert.DoesNotContain(outcome.Out, l => l.Contains("run that list", StringComparison.Ordinal)); - Assert.Contains(outcome.Out, l => l.StartsWith("codegen/Program.cs does not run Generators.All", StringComparison.Ordinal)); - Assert.Contains(" return CodegenCli.Run(args, Generators.All);", outcome.Out); - Assert.Contains("See https://github.com/metaobjectsdev/metaobjects/blob/v1.1.0/docs/features/migrations/docs-are-generators.md, section \"C#\".", outcome.Out); - Assert.Equal(LegacyProgramCs, File.ReadAllText(Path.Combine(_tmp, "codegen", "Program.cs"))); - } - [Fact] public void Init_in_a_project_pinned_to_an_older_package_warns_and_names_the_line_to_change() { diff --git a/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs b/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs index 4da708f582..c20e1eef7e 100644 --- a/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs +++ b/server/csharp/MetaObjects.Cli.Tests/VerifySubverbTests.cs @@ -253,70 +253,33 @@ public void Codegen_with_metadata_that_does_not_load_is_exit1_not_the_usage_exit Assert.Contains("did not load", r.Codegen!.Error); } - // -------------------- the namespace-inference footgun -------------------- - // gen used a CUSTOM namespace; verify --codegen WITHOUT --namespace must infer - // it from the committed output (else every file would spuriously drift on the - // embedded `namespace {ns};`). Build the Options with NO explicit namespace. + // -------------------- the namespace is gen's, never read from the output -------------------- + // verify --codegen regenerates under the namespace gen resolves (--namespace, else the + // default). Output generated under another namespace is drift, not something to adopt. - /// Codegen opts with NO explicit namespace (the footgun scenario). - private VerifyCommand.Options CodegenOptsNoNamespace() => - new() - { - MetadataDir = MetaDir, - // verify --codegen re-runs the SELECTION; there is no default suite to fall - // back on, so the mechanics tests name the one their fixtures were written for. - Generators = GenSuite.Names, - OutDir = OutDir, - Codegen = true, - // Namespace intentionally NOT set + NamespaceExplicit defaults to false. - }; + private VerifyCommand.Options CodegenOpts(string? ns) => + ns is null + ? new() { MetadataDir = MetaDir, Generators = GenSuite.Names, OutDir = OutDir, Codegen = true } + : new() { MetadataDir = MetaDir, Generators = GenSuite.Names, OutDir = OutDir, Codegen = true, Namespace = ns }; [Fact] - public void Codegen_infers_custom_namespace_from_committed_output_no_flag() + public void Codegen_output_under_another_namespace_is_drift_without_the_flag() { File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); - // Committed output generated with a CUSTOM namespace. GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // verify --codegen WITHOUT --namespace → must infer "Acme.Generated" from the - // committed files and produce a byte-identical regen → exit 0 (no spurious drift). - var r = VerifyCommand.RunSubverbs(CodegenOptsNoNamespace()); - Assert.Equal(0, r.ExitCode); - Assert.True(r.Codegen!.Clean, string.Join("; ", r.Codegen!.Lines)); - } - - [Fact] - public void Codegen_inference_still_detects_real_drift_with_custom_namespace() - { - File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); - GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // A real hand-edit on top of the custom namespace must still be drift. - File.AppendAllText(Path.Combine(OutDir, "Subscriber.g.cs"), "\n// real drift\n"); - - var r = VerifyCommand.RunSubverbs(CodegenOptsNoNamespace()); + var r = VerifyCommand.RunSubverbs(CodegenOpts(null)); Assert.NotEqual(0, r.ExitCode); Assert.Contains(r.Codegen!.DriftedFiles, f => f.EndsWith("Subscriber.g.cs")); } [Fact] - public void Codegen_explicit_namespace_still_wins_over_inference() + public void Codegen_with_the_namespace_gen_used_is_clean() { File.WriteAllText(Path.Combine(MetaDir, "meta.ai.json"), EntityMetadata); GenSuite.Run(MetaDir, OutDir, "Acme.Generated"); - // Explicit namespace set (matching) → wins, byte-identical regen → exit 0. - var opts = new VerifyCommand.Options - { - MetadataDir = MetaDir, - // verify --codegen re-runs the SELECTION; there is no default suite to fall - // back on, so the mechanics tests name the one their fixtures were written for. - Generators = GenSuite.Names, - OutDir = OutDir, - Namespace = "Acme.Generated", - NamespaceExplicit = true, - Codegen = true, - }; - var r = VerifyCommand.RunSubverbs(opts); + var r = VerifyCommand.RunSubverbs(CodegenOpts("Acme.Generated")); Assert.Equal(0, r.ExitCode); Assert.True(r.Codegen!.Clean, string.Join("; ", r.Codegen!.Lines)); } @@ -371,7 +334,6 @@ private void WriteNamingConfig(string columns) Generators = GenSuite.Names, OutDir = OutDir, Namespace = "Acme.Generated", - NamespaceExplicit = true, Codegen = true, Naming = new PhysicalNaming(verifyStrategy, TableNamingRule.SnakeCase), }; diff --git a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs index e9029094e6..2fb7aeecad 100644 --- a/server/csharp/MetaObjects.Cli/CodegenHandoff.cs +++ b/server/csharp/MetaObjects.Cli/CodegenHandoff.cs @@ -22,14 +22,6 @@ public static bool HasOwnedCodegen(string projectRoot) => public static bool HasGeneratorList(string projectRoot) => File.Exists(Path.Combine(projectRoot, "codegen", "Generators.cs")); - /// True iff has a codegen/Program.cs that does not run - /// Generators.All: a 1.0 entry point, which declares its own list and ignores codegen/Generators.cs. - public static bool ProgramIgnoresList(string projectRoot) - { - var program = Path.Combine(projectRoot, "codegen", "Program.cs"); - return File.Exists(program) && !MetaObjects.Codegen.CSharpSource.ReferencesMember(File.ReadAllText(program), "Generators", "All"); - } - /// /// When has an ejected codegen project, run `dotnet /// run --project codegen -- <args>` (inheriting this process's console) and set diff --git a/server/csharp/MetaObjects.Cli/EjectCommand.cs b/server/csharp/MetaObjects.Cli/EjectCommand.cs index c6be8be129..8cd3d7e2af 100644 --- a/server/csharp/MetaObjects.Cli/EjectCommand.cs +++ b/server/csharp/MetaObjects.Cli/EjectCommand.cs @@ -221,12 +221,11 @@ public static Outcome Run(IReadOnlyList names, string root, bool force) outLines.Add(""); outLines.Add($"Also wrote {string.Join(", ", scaffolded)}."); } - var programIgnoresList = CodegenHandoff.ProgramIgnoresList(root); if (scaffolded.Contains("codegen/Generators.cs")) { outLines.Add( - $"codegen/Generators.cs lists the generator(s) you just ejected ({string.Join(", ", entries)})." + - (programIgnoresList ? "" : " `dotnet meta gen` and `dotnet meta verify --codegen` run that list.")); + $"codegen/Generators.cs lists the generator(s) you just ejected ({string.Join(", ", entries)}). " + + "`dotnet meta gen` and `dotnet meta verify --codegen` run that list."); } else { @@ -237,11 +236,6 @@ public static Outcome Run(IReadOnlyList names, string root, bool force) outLines.Add("to the list there, in place of the packaged entry if it lists one " + "(MetaObjects.Codegen.Generators.)."); } - if (programIgnoresList) - { - outLines.Add(""); - outLines.AddRange(CodegenScaffold.ReplaceProgram()); - } if (csprojExisted) outLines.AddRange(OlderPinWarning(csprojPath, "eject")); return new Outcome(0, outLines, []); diff --git a/server/csharp/MetaObjects.Cli/InitCommand.cs b/server/csharp/MetaObjects.Cli/InitCommand.cs index 4b1483eb29..3caf2eebc9 100644 --- a/server/csharp/MetaObjects.Cli/InitCommand.cs +++ b/server/csharp/MetaObjects.Cli/InitCommand.cs @@ -49,15 +49,6 @@ public static class CodegenScaffold "using MetaObjects.Codegen;\n" + "return CodegenCli.Run(args, Generators.All);\n"; - /// What init and eject say in place of "gen runs the list" when the project's own - /// codegen/Program.cs does not run it (). - public static IReadOnlyList ReplaceProgram() => - [ - "codegen/Program.cs does not run Generators.All, so `dotnet meta gen` and `dotnet meta verify --codegen` " + - "refuse until it does. It is yours, so init and eject never edit it: replace it with the two-line entry", - .. ProgramCs.TrimEnd('\n').Split('\n').Select(l => " " + l), - $"See {CodegenCli.DocsMigrationGuide}.", - ]; /// /// codegen/Generators.cs, the project's generator list, holding : @@ -140,12 +131,9 @@ public static IReadOnlyList Run(string root, bool withDocs = false) foreach (var rel in CodegenScaffold.Files) lines.Add(written.Contains(rel) ? $" wrote {rel}" : $" kept {rel} (already yours)"); lines.Add(""); - if (CodegenHandoff.ProgramIgnoresList(root)) - lines.AddRange(CodegenScaffold.ReplaceProgram()); - else - lines.Add( - "codegen/Generators.cs is the generator list `dotnet meta gen` runs. It declares nothing yet, " + - "so gen writes nothing until you add entries: `dotnet meta gen --list` is the catalog."); + lines.Add( + "codegen/Generators.cs is the generator list `dotnet meta gen` runs. It declares nothing yet, " + + "so gen writes nothing until you add entries: `dotnet meta gen --list` is the catalog."); if (withDocs) { if (listExisted) diff --git a/server/csharp/MetaObjects.Cli/Program.cs b/server/csharp/MetaObjects.Cli/Program.cs index 541f5cf41f..286bcead1f 100644 --- a/server/csharp/MetaObjects.Cli/Program.cs +++ b/server/csharp/MetaObjects.Cli/Program.cs @@ -46,8 +46,8 @@ " --codegen regen-to-temp vs committed --out\n" + " (--out defaults to csharp.outDir in\n" + " .metaobjects/config.json, as for gen)\n" + - " --namespace codegen regen namespace; inferred\n" + - " from the committed --out when omitted\n" + + " --namespace codegen regen namespace; pass the one\n" + + " gen used (default Generated)\n" + " --db NOT supported in C# (migrate engine)\n" + " --lax load lax (legacy); strict-by-default\n" + " rejects an unregistered @attr (ADR-0023)\n" + @@ -173,11 +173,6 @@ static int RunGen(string[] rest) "`dotnet meta init` writes the missing files and keeps the ones you have."); return 2; } - if (CodegenHandoff.ProgramIgnoresList(projectRoot)) - { - Console.Error.WriteLine(CodegenCli.LegacyProgram(command)); - return 2; - } return null; } @@ -540,7 +535,6 @@ static int RunVerify(string[] rest) TemplatesRoot = templatesRoot, OutDir = outDir, Namespace = ns, - NamespaceExplicit = nsExplicit, Generators = generators, TemplateRoot = templateRoot, Templates = templates, diff --git a/server/csharp/MetaObjects.Cli/VerifyCommand.cs b/server/csharp/MetaObjects.Cli/VerifyCommand.cs index 7a8a578ce0..5f10a5fe31 100644 --- a/server/csharp/MetaObjects.Cli/VerifyCommand.cs +++ b/server/csharp/MetaObjects.Cli/VerifyCommand.cs @@ -125,10 +125,6 @@ public sealed record Options public string? OutDir { get; init; } /// Code-gen namespace (only affects the temp regen; default mirrors `gen`). public string Namespace { get; init; } = "Generated"; - /// True when the user explicitly passed --namespace. When false and - /// --codegen runs, the namespace is inferred from the committed output so a - /// regen matches output generated with any namespace (avoids spurious drift). - public bool NamespaceExplicit { get; init; } /// Generator selection for the --codegen regen. There is no default suite: name the ones gen ran. public IReadOnlyList? Generators { get; init; } /// Template root for render-helper/template generators in the codegen regen. @@ -432,14 +428,9 @@ private static Codegen.CodegenDrift.Result RunCodegenDrift(Options opts, out boo ], }; - // The C# namespace is embedded in every generated file (`namespace {ns};`). - // If the user did NOT pass --namespace, infer it from the committed output so - // the regen matches output produced with any namespace (otherwise every file - // would spuriously drift). An explicit --namespace always wins (back-compat / - // override); inference falling back to null leaves the default in place. + // The namespace resolves exactly as `gen` resolves it (--namespace, else the default), + // so output generated under another namespace is drift. var ns = opts.Namespace; - if (!opts.NamespaceExplicit) - ns = Codegen.CodegenDrift.InferNamespace(opts.OutDir) ?? opts.Namespace; // C1 — same presence gate `gen` computes (GeneratorRegistry.IncludesNames over // the SAME resolved `names` list above), or a regen under a `--generators` diff --git a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs index a3852a2992..6fa7a1e2ae 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/CodegenDriftTests.cs @@ -151,67 +151,6 @@ public void Drift_run_never_writes_to_the_real_out_dir() Assert.NotEqual(before, afterMutation); } - // -------------------- namespace inference from committed output ---------- - - [Fact] - public void InferNamespace_reads_file_scoped_namespace() - { - Gen(); // writes files with `namespace Acme.Generated;` - Assert.Equal("Acme.Generated", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_reads_block_scoped_namespace() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "Block.g.cs"), - "// \nnamespace Acme.Block\n{\n public class Foo { }\n}\n"); - Assert.Equal("Acme.Block", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_picks_the_first_sorted_file() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "Zeta.g.cs"), "namespace Zeta.Ns;\n"); - File.WriteAllText(Path.Combine(OutDir, "Alpha.g.cs"), "namespace Alpha.Ns;\n"); - Assert.Equal("Alpha.Ns", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_returns_null_for_missing_dir() - { - Assert.Null(CodegenDrift.InferNamespace(Path.Combine(_tmp, "does-not-exist"))); - } - - [Fact] - public void InferNamespace_returns_null_when_no_cs_has_a_namespace() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "NoNs.g.cs"), "// top-level statements only\nvar x = 1;\n"); - File.WriteAllText(Path.Combine(OutDir, "notes.txt"), "namespace NotCSharp;\n"); - Assert.Null(CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_ignores_a_namespace_in_a_comment_or_string() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "A.g.cs"), - "// namespace Fake.Comment;\n/*\nnamespace Fake.Block {}\n*/\n" + - "public static class K { public const string S = @\"\nnamespace Fake.Str;\n\"; }\n"); - File.WriteAllText(Path.Combine(OutDir, "B.g.cs"), "namespace Real.Ns;\n"); - Assert.Equal("Real.Ns", CodegenDrift.InferNamespace(OutDir)); - } - - [Fact] - public void InferNamespace_returns_null_when_only_comments_and_strings_mention_one() - { - Directory.CreateDirectory(OutDir); - File.WriteAllText(Path.Combine(OutDir, "A.g.cs"), "// namespace Fake.Comment;\nvar s = \"namespace Fake.Str;\";\n"); - Assert.Null(CodegenDrift.InferNamespace(OutDir)); - } - private static Dictionary SnapshotDir(string dir) { var snap = new Dictionary(StringComparer.Ordinal); diff --git a/server/csharp/MetaObjects.Codegen/CSharpSource.cs b/server/csharp/MetaObjects.Codegen/CSharpSource.cs deleted file mode 100644 index 7aedf2fb17..0000000000 --- a/server/csharp/MetaObjects.Codegen/CSharpSource.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Reads of C# source go through Roslyn's syntax tree, never a regex or substring over the text: -// a comment or string literal that merely mentions `namespace X` or `Generators.All` is trivia or -// a literal token there, not a declaration or a member access. - -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.CSharp.Syntax; - -namespace MetaObjects.Codegen; - -public static class CSharpSource -{ - /// The name of the first top-level namespace (file-scoped or block) declared in - /// , or null when it declares none. - public static string? TopLevelNamespace(string source) - { - var root = CSharpSyntaxTree.ParseText(source).GetCompilationUnitRoot(); - return root.Members.OfType() - .Select(n => n.Name.ToString().Replace(" ", "")) - .FirstOrDefault(); - } - - /// True iff evaluates the member of a type - /// named (Generators.All, My.Ns.Generators.All), or names - /// member bare under a using static of that type. Comments and string literals never count. - public static bool ReferencesMember(string source, string typeName, string member) - { - var root = CSharpSyntaxTree.ParseText(source).GetCompilationUnitRoot(); - var nodes = root.DescendantNodes().ToList(); - - if (nodes.OfType().Any(m => - m.Name.Identifier.ValueText == member && LastName(m.Expression) == typeName)) - return true; - - var staticallyImported = nodes.OfType().Any(u => - u.StaticKeyword.IsKind(SyntaxKind.StaticKeyword) - && u.NamespaceOrType is { } t && LastName(t) == typeName); - return staticallyImported && nodes.OfType().Any(i => - i.Identifier.ValueText == member - && !(i.Parent is MemberAccessExpressionSyntax m && m.Name == i)); - } - - /// The rightmost simple name of A, A.B or global::A.B; null for anything else. - private static string? LastName(SyntaxNode node) => node switch - { - IdentifierNameSyntax id => id.Identifier.ValueText, - QualifiedNameSyntax q => LastName(q.Right), - AliasQualifiedNameSyntax a => LastName(a.Name), - MemberAccessExpressionSyntax m => LastName(m.Name), - _ => null, - }; -} diff --git a/server/csharp/MetaObjects.Codegen/CodegenCli.cs b/server/csharp/MetaObjects.Codegen/CodegenCli.cs index d4eb86f360..e3cdd1c10a 100644 --- a/server/csharp/MetaObjects.Codegen/CodegenCli.cs +++ b/server/csharp/MetaObjects.Codegen/CodegenCli.cs @@ -316,14 +316,6 @@ public static string NoGeneratorList(string command) => $"{command}: no codegen/Generators.cs. The generator list lives there in 1.1.0, documentation included; " + "`dotnet meta init` writes it. See " + DocsMigrationGuide + "."; - /// What a gen or verify --codegen run prints, exit 2, when codegen/Program.cs does not - /// run Generators.All: a 1.0 entry point declares its generators inline and would ignore the list. - /// The command's own name: dotnet meta gen or dotnet meta verify --codegen. - public static string LegacyProgram(string command) => - $"{command}: codegen/Program.cs does not run Generators.All, so it would ignore the generator list in " + - "codegen/Generators.cs. Replace it with the two-line entry `using MetaObjects.Codegen;` and " + - "`return CodegenCli.Run(args, Generators.All);`. See " + DocsMigrationGuide + "."; - /// What a gen run prints, exit 2, when it finds 1.0 template-spec configuration: a /// project-root template-spec.json no entry declares, or --template-spec. Through /// 1.0 the gen command read either; a spec now runs only as a @@ -548,7 +540,7 @@ private sealed class OwnedArgs public string? MetadataDir, OutDir, TemplateRoot, TemplateSpecPath, ColumnNamingRaw, GeneratorsCsv; public string Namespace = "Generated"; public string Baseline = "default"; - public bool NamespaceExplicit, EmitAbstractShapes, Codegen, Lax, Strict; + public bool EmitAbstractShapes, Codegen, Lax, Strict; /// 1.0 docs flags given (), in order. public List LegacyDocs = []; /// The usage error the arguments earn, or null. @@ -584,7 +576,7 @@ public static OwnedArgs Parse(string[] rest, string command, string[] valueFlags switch (flag) { case "--out": a.OutDir = value; break; - case "--namespace": a.Namespace = value; a.NamespaceExplicit = true; break; + case "--namespace": a.Namespace = value; break; case "--column-naming": a.ColumnNamingRaw = value; break; case "--baseline": a.Baseline = value; break; case "--template-root": a.TemplateRoot = value; break; @@ -954,10 +946,6 @@ private static int RunVerifyArgs(string[] rest, IReadOnlyList declar return 0; } - // Same inference rule `dotnet meta verify --codegen` has always used: an explicit - // --namespace always wins; otherwise infer from the committed output so a regen - // matches output produced with any namespace (avoids spurious drift). - var effectiveNs = a.NamespaceExplicit ? a.Namespace : (CodegenDrift.InferNamespace(a.OutDir) ?? a.Namespace); // ADR-0058 — the loaded root carries the project's column rule; `gen` read it the // same way producing the committed output, so this regen agrees with it by // construction instead of needing a flag told to match. @@ -965,7 +953,7 @@ private static int RunVerifyArgs(string[] rest, IReadOnlyList declar var config = new GenConfig { OutDir = a.OutDir, - Namespace = effectiveNs, + Namespace = a.Namespace, ColumnNamingStrategy = columnNaming, IncludeNames = declared.Any(g => g.Name == "names"), // The manifest gen keeps: an extra file is classified by what gen recorded writing. diff --git a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs index 28ff23057d..6430b4336c 100644 --- a/server/csharp/MetaObjects.Codegen/CodegenDrift.cs +++ b/server/csharp/MetaObjects.Codegen/CodegenDrift.cs @@ -18,35 +18,6 @@ namespace MetaObjects.Codegen; /// Regenerate-to-temp + diff vs committed output (ADR-0021 D2). public static class CodegenDrift { - /// - /// Infer the C# namespace embedded in the committed generated output under - /// . Scans the FIRST .cs file (ordinal-sorted) - /// that carries a top-level namespace declaration and returns it. - /// Returns null when the dir is missing or no .cs file declares a - /// namespace — the caller then falls back to its default (there is nothing - /// meaningful to diff against in that case anyway). Used by - /// verify --codegen so a regen matches output produced with ANY namespace - /// when --namespace is omitted. - /// - public static string? InferNamespace(string outDir) - { - if (!Directory.Exists(outDir)) return null; - - var csFiles = Directory.EnumerateFiles(outDir, "*.cs", SearchOption.AllDirectories) - .OrderBy(p => p.Replace(Path.DirectorySeparatorChar, '/'), StringComparer.Ordinal); - - foreach (var file in csFiles) - { - string text; - try { text = File.ReadAllText(file); } - catch { continue; } - - var ns = CSharpSource.TopLevelNamespace(text); - if (ns is not null) return ns; - } - return null; - } - /// The outcome of a codegen-drift computation (pure; no console I/O). public sealed record Result { diff --git a/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj b/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj index 2056d640e7..0a9a096a62 100644 --- a/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj +++ b/server/csharp/MetaObjects.Codegen/MetaObjects.Codegen.csproj @@ -21,9 +21,6 @@ - -