fix: seven ISC pipeline conformance bugs — the corpus sweep runs end-to-end

.github/workflows/rake.yml

@@ -18,6 +18,12 @@ jobs:
     env:
       BUNDLE_WITHOUT: "secryst"
       SKIP_JS: "1"
+      # The ISC corpus sweep (ISC_SWEEP=1) stays opt-in until the
+      # conformance gap it measures is closed: it currently exposes real
+      # per-map mismatches across the corpus. Run it locally with
+      #   ISC_SWEEP=1 INTERSCRIPT_MAPS_PATH=../maps/maps \
+      #   rspec spec/interscript_spec.rb
+      # to track that gap; flip it on here once it goes green.
 
     steps:
       - name: Checkout monorepo

lib/interscript/compiler.rb

@@ -9,18 +9,10 @@ class Interscript::Compiler
   attr_accessor :code
 
   def self.call(map, **kwargs)
-    if String === map
-      path = begin
-        Interscript.locate(map)
-      rescue
-        nil
-      end
-      map = if path&.end_with?(".isc")
-        parse_isc(path)
-      else
-        Interscript::DSL.parse(map)
-      end
-    end
+    # DSL.parse locates, dispatches .isc and caches the parsed document —
+    # calling parse_isc here re-ran a ~30 s Parslet parse on every
+    # compiler instantiation.
+    map = Interscript::DSL.parse(map) if String === map
     compiler = new
     compiler.compile(map, **kwargs)
     compiler

lib/interscript/dsl.rb

@@ -40,7 +40,9 @@ def self.parse(map_name, reverse: true)
       end
     end
     # ISC documents route through the ISC parser, not the .imp DSL.
-    return Interscript::Compiler.parse_isc(path) if path.end_with?(".isc")
+    # The parse is expensive (Parslet over the whole map) — cache it like
+    # the .imp path does, or every transliterate call re-parses the map.
+    return @cache[map_name] = Interscript::Compiler.parse_isc(path) if path.end_with?(".isc")
 
     library = path.end_with?(".iml")
 

lib/interscript/isc/node_adapter.rb

@@ -26,10 +26,22 @@ def build
         Interscript::Node::Document.new.tap do |doc|
           doc.metadata = build_metadata
           doc.tests = build_tests
-          doc.aliases = build_aliases
-          build_stages.each { |name, stage| doc.stages[name] = stage }
+          # Maps are addressed by their file-derived name (locate() and
+          # Maps.transliterate use it); registering under the system code
+          # made every map whose code differs from its file name resolve
+          # to an auto-created empty entry — stages[:main].call crashed.
+          doc.name = if @isc_doc[:filename]
+            File.basename(@isc_doc[:filename].to_s, ".isc")
+          else
+            @isc_doc[:systemCode]
+          end
+          # The legacy DSL stamps doc_name on every alias and stage; the
+          # Ruby/JS compilers emit run directives and alias lookups as
+          # Maps.transliterate(stage.doc_name, ...) — without it every
+          # aliased-dependency run compiled to transliterate(nil).
+          doc.aliases = build_aliases(doc.name)
+          build_stages(doc.name).each { |name, stage| doc.stages[name] = stage }
           build_dependencies(doc)
-          doc.name = @isc_doc[:systemCode]
         end
       end
 
@@ -53,12 +65,14 @@ def build_tests
         tests
       end
 
-      def build_aliases
+      def build_aliases(doc_name)
         @isc_doc[:aliases].each_with_object({}) do |a, h|
           # The runtime resolves aliases through AliasDef#data (see
           # Interpreter::Stage#build_item); a bare item here hands it a
           # raw Array once Any#data unrolls.
-          h[a[:name].to_sym] = Interscript::Node::AliasDef.new(a[:name].to_sym, convert_item(a[:value]))
+          alias_def = Interscript::Node::AliasDef.new(a[:name].to_sym, convert_item(a[:value]))
+          alias_def.doc_name = doc_name
+          h[a[:name].to_sym] = alias_def
         end
       end
 
@@ -90,9 +104,11 @@ def load_dependency_document(full_name)
         end
       end
 
-      def build_stages
+      def build_stages(doc_name)
         @isc_doc[:stages].each_with_object({}) do |stage, h|
-          h[stage[:name].to_sym] = build_stage(stage)
+          built = build_stage(stage)
+          built.doc_name = doc_name
+          h[stage[:name].to_sym] = built
         end
       end
 
@@ -174,9 +190,13 @@ def convert_item(item)
         when Items::Some
           Interscript::Node::Item::Some.new(convert_item(item.inner))
         when Items::Range
-          Interscript::Node::Item::Any.new(
-            (item.lo..item.hi).map { |c| Interscript::Node::Item::String.new(c) }
-          )
+          # Keep the range native: both runtimes compile Any(Range) to a
+          # codepoint character class [lo-hi]. Expanding it to an Array
+          # iterates Ruby String#succ (a, b, … z, aa, ab …), which never
+          # reaches non-ASCII codepoints and blows up the node size.
+          lo = item.lo.is_a?(Items::StringValue) ? item.lo.value : item.lo
+          hi = item.hi.is_a?(Items::StringValue) ? item.hi.value : item.hi
+          Interscript::Node::Item::Any.new(lo..hi)
         when Items::Set
           convert_set(item)
         else
@@ -209,10 +229,19 @@ def convert_set(set)
             case c
             when ::String
               Interscript::Node::Item::String.new(c)
+            when Items::StringValue
+              Interscript::Node::Item::String.new(c.value)
             when Items::Primitive
               convert_primitive(c)
             when Items::AliasRef
-              convert_alias_ref(c)
+              # Legacy parity: a bare alias inside any(...) compiles to the
+              # Stdlib value (Any(nil) there for imported aliases). Keeping
+              # the Alias item resolves to the imported charset *string* at
+              # build time, baking a 1000-char literal into the constraint
+              # regexp — a lookbehind that never matches.
+              name = convert_alias_ref(c).name
+              val = Interscript::Stdlib::ALIASES[name]
+              val ? Interscript::Node::Item::Any.new(val) : Interscript::Node::Item::String.new("")
             else
               convert_item(c)
             end

lib/interscript/isc/normalizer.rb

@@ -36,10 +36,12 @@ def normalize
       def scalar(node)
         return node if node.is_a?(::String) || node.nil?
         if node.key?(:string)
-          # Parts are {char:} hashes in the general grammar; some
-          # constructions (e.g. the rababa directive) capture bare
-          # slices instead.
-          Array(node[:string]).map { |part| part.is_a?(Hash) ? part[:char].to_s : part.to_s }.join
+          # Parts are {char:} hashes and escape fragments ({dquote:},
+          # {unicode:}, …) in the general grammar; some constructions
+          # (e.g. the rababa directive) capture bare slices instead.
+          # Escapes decode via the shared fold — folding only :char here
+          # silently dropped every escaped character.
+          Interscript::Isc::Transform.decode_string_parts(node[:string])
         elsif node.key?(:identifier)
           node[:identifier].to_s
         elsif node.key?(:raw)

lib/interscript/isc/transform.rb

@@ -16,33 +16,7 @@ class Transform < Parslet::Transform
       # StringValue.
       rule(string: simple(:s)) { Items::StringValue.new(s.to_s) }
       rule(string: sequence(:parts)) do
-        combined = parts.map do |p|
-          case p
-          when Hash
-            # Escape sequence fragment: e.g. {newline: "n"}, {unicode: "1234"},
-            # {dquote: '"'}, {char: "a"}
-            if p.key?(:char)
-              p[:char].to_s
-            elsif p.key?(:newline)
-              "\n"
-            elsif p.key?(:carriage_return)
-              "\r"
-            elsif p.key?(:tab)
-              "\t"
-            elsif p.key?(:dquote)
-              '"'
-            elsif p.key?(:backslash)
-              "\\"
-            elsif p.key?(:unicode)
-              [p[:unicode].to_s.to_i(16)].pack("U")
-            else
-              p.to_s
-            end
-          else
-            p.to_s
-          end
-        end.join
-        Items::StringValue.new(combined)
+        Items::StringValue.new(Transform.decode_string_parts(parts))
       end
       rule(char: simple(:c)) { c.to_s }
 
@@ -55,6 +29,11 @@ class Transform < Parslet::Transform
         Items::AliasRef.new(q.to_s, map: n.to_s)
       }
       rule(alias: simple(:n)) { Items::AliasRef.new(n.to_s) }
+      # Bare identifiers inside any(...) lists parse as :alias_ref (the
+      # :alias key only covers direct item position). Unmatched, the hash
+      # degraded downstream into the debug string "[:alias_ref, \"name\"]".
+      rule(alias_ref: {identifier: simple(:n)}) { Items::AliasRef.new(n.to_s) }
+      rule(alias_ref: simple(:n)) { Items::AliasRef.new(n.to_s) }
       rule(ref: subtree(:h)) { Items::Capture.new(h[:digit].to_s.to_i) }
       rule(capture_inner: subtree(:inner)) { Items::CaptureGroup.new(Interscript::Isc::Transform.materialize_item(inner)) }
       rule(maybe_inner: subtree(:inner)) { Items::Maybe.new(Interscript::Isc::Transform.materialize_item(inner)) }
@@ -71,14 +50,52 @@ class Transform < Parslet::Transform
         hex.to_s
       end
 
+      # Shared decoder for the pieces of a :string capture — used by the
+      # transform rules above and by the Normalizer's scalar folding. A
+      # fragment without an escape key is a raw character slice.
+      def self.decode_string_parts(parts)
+        Array(parts).map do |p|
+          case p
+          when Hash
+            if p.key?(:char)
+              p[:char].to_s
+            elsif p.key?(:newline)
+              "\n"
+            elsif p.key?(:carriage_return)
+              "\r"
+            elsif p.key?(:tab)
+              "\t"
+            elsif p.key?(:dquote)
+              '"'
+            elsif p.key?(:backslash)
+              "\\"
+            elsif p.key?(:unicode)
+              [p[:unicode].to_s.to_i(16)].pack("U")
+            else
+              p.to_s
+            end
+          else
+            p.to_s
+          end
+        end.join
+      end
+
       rule(lo: simple(:lo), hi: simple(:hi)) do
         Items::Range.new(lo.to_s, hi.to_s)
       end
       rule(single: simple(:s)) { Items::Set.from_string(s.to_s) }
+      # List entries are already Items (strings, primitives, alias refs).
+      # Stringifying here baked an object inspect into the char set.
       rule(list: sequence(:arr)) do
-        Items::Set.from_strings(arr.map(&:to_s))
+        Items::Set.new(arr)
+      end
+      rule(any: subtree(:h)) do
+        # A single alias argument keeps set semantics: the legacy runtime
+        # builds Any(Alias) which resolves through Stdlib (Any(nil) for
+        # imported aliases). Unwrapped, the bare Alias resolves to the
+        # imported charset string and compiles as a literal.
+        h.is_a?(Items::AliasRef) ? Items::Set.new([h]) : h
       end
-      rule(any: subtree(:h)) { h }
 
       rule(concatenation: subtree(:parts)) do
         Items::Concat.from_parts(Array(parts))

spec/interscript/isc/node_adapter_spec.rb

@@ -263,3 +263,113 @@ def parse_and_adapt(src)
     expect(node.name).to eq("X:a-b:C-D:1")
   end
 end
+
+RSpec.describe "NodeAdapter range handling" do
+  it "keeps ISC ranges as native Any ranges — codepoint semantics, not string-succ expansion" do
+    src = <<~ISC
+      system "TEST:aze-Arab:Latn:2026" {
+        metadata { name "T" }
+        stage main {
+          parallel { sub "q" "k" }
+          sub { from any("a".."￿") to upcase before boundary }
+        }
+      }
+    ISC
+    node = Interscript::Isc::NodeAdapter.to_interscript_node(
+      Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
+    )
+    upcase_rule = node.stages[:main].children
+      .select { |c| c.is_a?(Interscript::Node::Rule::Sub) }
+      .find { |r| r.to == :upcase }
+    # A Ruby String range expands via String#succ (a, b, ..., z, aa, ab …)
+    # which never reaches non-ASCII codepoints — the range must stay a range.
+    expect(upcase_rule.from.value).to be_a(Range)
+    interp = Interscript::Interpreter.new
+    interp.compile(node)
+    expect(interp.call("īş")).to eq("Īş")
+  end
+end
+
+RSpec.describe "NodeAdapter any-list constraints" do
+  it "keeps primitives as aliases, never their inspect" do
+    tree = Interscript::Isc::Parser.parse(
+      'system "x" { stage main { sub { from "ം" to "m" after any([boundary, "‌", "‍"]) } } }'
+    )
+    doc = Interscript::Isc::DocumentBuilder.build(tree)
+    node = Interscript::Isc::NodeAdapter.to_interscript_node(doc)
+    stage = Interscript::Interpreter::Stage.new(node, "")
+    re = stage.send(:build_regexp, node.stages[:main].children.first)
+    # The constraint must match boundary positions — an inspect leak
+    # turns the lookahead into a garbage char class.
+    expect(re).not_to include("Primitive")
+    expect("പ്പം ഹ").to match(Regexp.new(re))
+  end
+end
+
+RSpec.describe "NodeAdapter string escapes" do
+  it "decodes escape sequences in test strings" do
+    src = <<~'ISC'
+      system "T:a-b:C-D:1" {
+        metadata { name "T" }
+        tests {
+          "pod\"ezd" -> "p\"ezd"
+          "a\tb\nc" -> "déjà"
+        }
+        stage main { sub "a" "b" }
+      }
+    ISC
+    node = Interscript::Isc::NodeAdapter.to_interscript_node(
+      Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
+    )
+    tests = node.tests.data
+    expect(tests[0]).to eq(['pod"ezd', 'p"ezd'])
+    expect(tests[1]).to eq(["a\tb\nc", "déjà"])
+  end
+end
+
+RSpec.describe "NodeAdapter alias refs in constraints" do
+  it "compiles bare alias refs in sets like the legacy DSL — no debug dump, no literal-charset lookbehind" do
+    src = <<~ISC
+      system "t:g-c:C-D:1" {
+        metadata { name "T" }
+        stage main {
+          sub {
+            from "γ"
+            to "n"
+            before any(greek)
+            after any("κΚ") + any(greek)
+            not_before boundary
+          }
+        }
+      }
+    ISC
+    node = Interscript::Isc::NodeAdapter.to_interscript_node(
+      Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
+    )
+    rule = node.stages[:main].children.first
+    expect(rule.before).to be_a(Interscript::Node::Item::Any)
+    expect(rule.before.inspect).not_to include("alias_ref")
+    expect(rule.after.inspect).not_to include("alias_ref")
+    # The legacy runtime compiles an imported alias in a set to a vacuous
+    # fragment (Any(nil)); resolving it to the charset *string* would bake
+    # a 1000-char literal into the lookbehind and never match.
+    interp = Interscript::Interpreter.new
+    interp.compile(node)
+    expect(interp.call("αγκα")).to eq("αnκα")
+  end
+end
+
+RSpec.describe "NodeAdapter document name derivation" do
+  it "names the document after the file, not the system code — transliteration addresses maps by file name" do
+    tree = Interscript::Isc::Parser.parse(
+      'system "TOTALLY:diff-Code:frm:File" { stage main { sub { from "a" to "b" } } }'
+    )
+    doc = Interscript::Isc::DocumentBuilder.build(tree, filename: "test-map.isc")
+    node = Interscript::Isc::NodeAdapter.to_interscript_node(doc)
+    # The Ruby compiler registers compiled maps under Document#name and
+    # Maps.transliterate looks them up by the map name (file-derived).
+    # Registering under the system code auto-creates an empty entry via
+    # the Hash default block and crashes on stages[:main].call.
+    expect(node.name).to eq("test-map")
+  end
+end

spec/interscript_spec.rb

@@ -7,8 +7,16 @@
 # per-map failures are expected until the ISC-era conformance work
 # completes — the sweep exists to MEASURE that gap, gated so CI stays
 # green while it closes.
+# The sweep honors INTERSCRIPT_MAPS_PATH (like the corpus specs) and
+# enumerates through the load path — matching what locate() resolves.
+# Bare maps() would enumerate the installed gem's frozen 2.4.x .imp
+# corpus wherever the maps checkout isn't the active gem.
 maps = if ENV["ISC_SWEEP"]
-  Interscript.maps(basename: false, select: mask)
+  sweep_root = ENV.fetch("INTERSCRIPT_MAPS_PATH", "../maps/maps")
+  unless Interscript.load_path.first == File.expand_path(sweep_root)
+    Interscript.load_path.unshift(File.expand_path(sweep_root))
+  end
+  Interscript.maps(basename: false, select: mask, load_path: true)
 else
   legacy_maps(select: mask)
 end