Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions src/interscript/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import re
import unicodedata

from .expr import expr_lookbehind, expr_max_length, expr_neg_lookbehind, expr_to_literal, expr_to_regex
from .expr import expr_is_par_safe, expr_lookbehind, expr_max_length, expr_neg_lookbehind, expr_to_literal, expr_to_regex, is_plain_string


class ExecutionError(ValueError):
Expand All @@ -27,9 +27,30 @@ class ExecutionError(ValueError):
def _compile_parallel(subs: list[dict]) -> tuple[re.Pattern[str], dict[str, str]]:
"""Compile one parallel group: longest-pattern-first alternation
with a named group per sub; lookaround guards for before:/after:.
Plain-string patterns additionally feed the casing maps."""

An all-plain group compiles to a replace tree in Ruby, where a
later duplicate from overwrites an earlier one; a guarded group
takes the megaregexp, where the earliest equal-key rule wins."""
guards = ("before", "after", "not_before", "not_after")
all_plain = not any(sub.get(g) for sub in subs for g in guards) and all(
sub.get("result") is not None
and expr_is_par_safe(sub["pattern"])
and expr_is_par_safe(sub["result"])
for sub in subs
)
drop: set[int] = set()
if all_plain:
last: dict[str, int] = {}
for i, sub in enumerate(subs):
if is_plain_string(sub["pattern"]):
last[expr_to_literal(sub["pattern"])] = i
for i, sub in enumerate(subs):
if is_plain_string(sub["pattern"]) and last[expr_to_literal(sub["pattern"])] != i:
drop.add(i)
indexed = []
for i, sub in enumerate(subs):
if i in drop:
continue
pat = expr_to_regex(sub["pattern"])
full = pat
if sub.get("before"):
Expand Down
11 changes: 11 additions & 0 deletions src/interscript/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,17 @@ def is_plain_string(expr: str) -> bool:
return bool(re.fullmatch(r'"(?:[^"\\]|\\.)*"', expr.strip()))


def expr_is_par_safe(expr: str) -> bool:
"""Ruby's parallel tree path builds literal from/to strings; a
boundary-like token there raises ("Can't use boundary in a par
context") and the whole block falls back to the megaregexp."""
try:
toks = _scan(expr, "par")
except ValueError:
return False
return all(k not in ("boundary", "nwb", "anchor") for k, _ in toks)


def expr_max_length(expr: str) -> int:
"""The Ruby runtime's parallel-selection key: Rule::Sub#max_length =
from + before + after + not_before + not_after (+ priority), where a
Expand Down
43 changes: 43 additions & 0 deletions tests/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,3 +303,46 @@ def test_subst_boundary_treats_combining_marks_as_word_chars():
and the schwa-killing rule fired — kaṁganā came out kṁganā."""
e = _load("un-mar-Deva-Latn-2016")
assert e.transliterate("कंगना") == "kaṁganā"


def test_plain_parallel_duplicate_last_wins():
"""Measured against Ruby: an all-plain parallel block compiles to a
replace tree where a later duplicate from overwrites an earlier one
(masm-mon lists sub "i" "й" ... sub "i" "и" to let the last win).
A guarded block takes the megaregexp path instead, where the
earliest equal-key rule that matches wins."""
tree = parse_imp(
'stage {\n parallel {\n'
' sub "i", "й"\n'
' sub "ab", "X"\n'
' sub "i", "и"\n'
' sub "ab", "Y"\n'
' }\n}\n'
)
e = Engine(tree)
assert e.transliterate("i ab") == "и Y"

tree2 = parse_imp(
'stage {\n parallel {\n'
' sub "i", "X", before: "a"\n'
' sub "i", "Y"\n'
' }\n}\n'
)
e2 = Engine(tree2)
assert e2.transliterate("ai") == "aX"
assert e2.transliterate("bi") == "bY"


def test_par_unsafe_rule_forces_megaregexp_first_wins():
"""Measured via Ruby on bgnpcgn-bal: one rule with a boundary in a
par-unsafe position makes the whole block fall back from the
replace tree (last duplicate wins) to the megaregexp (earliest
equal-key rule wins) — و maps to o there, not the later w."""
tree = parse_imp(
'stage {\n parallel {\n'
' sub "i", "X"\n'
' sub "i", "Y"\n'
' sub "z" + boundary, "B"\n'
' }\n}\n'
)
assert Engine(tree).transliterate("i") == "X"
Loading