From 38046a8697ed07627ef40de60647c0912dae13d1 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 1 Oct 2026 19:31:41 +0800 Subject: [PATCH] fix: plain parallel blocks take the replace-tree's duplicate semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against Ruby: an all-plain parallel block compiles to a replace tree where a later duplicate from overwrites an earlier one — masm-mon lists sub "i" "й" ... sub "i" "и" precisely so the last wins. A block containing guards, or any rule whose from or result holds a boundary-like token (Ruby: "Can't use boundary in a par context"), falls back to the megaregexp, where the earliest equal-key rule wins — bgnpcgn-bal's و -> o, not the later w. The engine now drops every duplicate plain pattern except the last when the block is par-safe, and keeps index order otherwise. Direct corpus sweep: 46 failures / 14 maps -> 35 / 13 — masm-mon (11) fully healed, bal restored. --- src/interscript/engine.py | 25 +++++++++++++++++++++-- src/interscript/expr.py | 11 ++++++++++ tests/test_engine.py | 43 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/interscript/engine.py b/src/interscript/engine.py index e209f65..14a1899 100644 --- a/src/interscript/engine.py +++ b/src/interscript/engine.py @@ -17,7 +17,7 @@ import re import unicodedata -from .expr import expr_lookbehind, expr_max_length, expr_neg_lookbehind, expr_to_literal, expr_to_regex +from .expr import expr_is_par_safe, expr_lookbehind, expr_max_length, expr_neg_lookbehind, expr_to_literal, expr_to_regex, is_plain_string class ExecutionError(ValueError): @@ -27,9 +27,30 @@ class ExecutionError(ValueError): def _compile_parallel(subs: list[dict]) -> tuple[re.Pattern[str], dict[str, str]]: """Compile one parallel group: longest-pattern-first alternation with a named group per sub; lookaround guards for before:/after:. - Plain-string patterns additionally feed the casing maps.""" + + An all-plain group compiles to a replace tree in Ruby, where a + later duplicate from overwrites an earlier one; a guarded group + takes the megaregexp, where the earliest equal-key rule wins.""" + guards = ("before", "after", "not_before", "not_after") + all_plain = not any(sub.get(g) for sub in subs for g in guards) and all( + sub.get("result") is not None + and expr_is_par_safe(sub["pattern"]) + and expr_is_par_safe(sub["result"]) + for sub in subs + ) + drop: set[int] = set() + if all_plain: + last: dict[str, int] = {} + for i, sub in enumerate(subs): + if is_plain_string(sub["pattern"]): + last[expr_to_literal(sub["pattern"])] = i + for i, sub in enumerate(subs): + if is_plain_string(sub["pattern"]) and last[expr_to_literal(sub["pattern"])] != i: + drop.add(i) indexed = [] for i, sub in enumerate(subs): + if i in drop: + continue pat = expr_to_regex(sub["pattern"]) full = pat if sub.get("before"): diff --git a/src/interscript/expr.py b/src/interscript/expr.py index 3765f2a..5885a77 100644 --- a/src/interscript/expr.py +++ b/src/interscript/expr.py @@ -300,6 +300,17 @@ def is_plain_string(expr: str) -> bool: return bool(re.fullmatch(r'"(?:[^"\\]|\\.)*"', expr.strip())) +def expr_is_par_safe(expr: str) -> bool: + """Ruby's parallel tree path builds literal from/to strings; a + boundary-like token there raises ("Can't use boundary in a par + context") and the whole block falls back to the megaregexp.""" + try: + toks = _scan(expr, "par") + except ValueError: + return False + return all(k not in ("boundary", "nwb", "anchor") for k, _ in toks) + + def expr_max_length(expr: str) -> int: """The Ruby runtime's parallel-selection key: Rule::Sub#max_length = from + before + after + not_before + not_after (+ priority), where a diff --git a/tests/test_engine.py b/tests/test_engine.py index 7624285..76041e5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -303,3 +303,46 @@ def test_subst_boundary_treats_combining_marks_as_word_chars(): and the schwa-killing rule fired — kaṁganā came out kṁganā.""" e = _load("un-mar-Deva-Latn-2016") assert e.transliterate("कंगना") == "kaṁganā" + + +def test_plain_parallel_duplicate_last_wins(): + """Measured against Ruby: an all-plain parallel block compiles to a + replace tree where a later duplicate from overwrites an earlier one + (masm-mon lists sub "i" "й" ... sub "i" "и" to let the last win). + A guarded block takes the megaregexp path instead, where the + earliest equal-key rule that matches wins.""" + tree = parse_imp( + 'stage {\n parallel {\n' + ' sub "i", "й"\n' + ' sub "ab", "X"\n' + ' sub "i", "и"\n' + ' sub "ab", "Y"\n' + ' }\n}\n' + ) + e = Engine(tree) + assert e.transliterate("i ab") == "и Y" + + tree2 = parse_imp( + 'stage {\n parallel {\n' + ' sub "i", "X", before: "a"\n' + ' sub "i", "Y"\n' + ' }\n}\n' + ) + e2 = Engine(tree2) + assert e2.transliterate("ai") == "aX" + assert e2.transliterate("bi") == "bY" + + +def test_par_unsafe_rule_forces_megaregexp_first_wins(): + """Measured via Ruby on bgnpcgn-bal: one rule with a boundary in a + par-unsafe position makes the whole block fall back from the + replace tree (last duplicate wins) to the megaregexp (earliest + equal-key rule wins) — و maps to o there, not the later w.""" + tree = parse_imp( + 'stage {\n parallel {\n' + ' sub "i", "X"\n' + ' sub "i", "Y"\n' + ' sub "z" + boundary, "B"\n' + ' }\n}\n' + ) + assert Engine(tree).transliterate("i") == "X"