diff --git a/.github/workflows/rake.yml b/.github/workflows/rake.yml index cc352e78..c15a542d 100644 --- a/.github/workflows/rake.yml +++ b/.github/workflows/rake.yml @@ -38,9 +38,13 @@ jobs: - name: Install Python helper and gems working-directory: ruby run: | - pip install regex bundle config set --local with jsexec bundle install + # PyCall binds whichever libpython it was built against, which + # is not necessarily the python that bare `pip` targets — + # install the regex helper into that interpreter. + PYBIN=$(bundle exec ruby -e 'require "pycall"; v = PyCall.sys.version[/\d+\.\d+/]; puts "python#{v}"') + "$PYBIN" -m pip install regex - name: RSpec working-directory: ruby diff --git a/lib/interscript/compiler/python.rb b/lib/interscript/compiler/python.rb index 37705ad2..bc9046b8 100644 --- a/lib/interscript/compiler/python.rb +++ b/lib/interscript/compiler/python.rb @@ -290,10 +290,14 @@ def compile_item i, doc = @map, target = nil class << self attr_accessor :maps_loaded attr_accessor :ctx + attr_accessor :maps_dir end def load - if !self.class.maps_loaded[@map.name] + # Keyed by document identity: synthetic spec documents reuse names + # (example-1, example-2, ...), and a stale engine under a reused + # name would silently serve the previous document. + if self.class.maps_loaded[@map.name] != @map.object_id @map.dependencies.each do |dep| dep = dep.full_name if !self.class.maps_loaded[dep] @@ -302,27 +306,42 @@ def load end ctx = self.class.ctx - python_src_path = File.join(__dir__, "..", "..", "..", "..", "python", "src") + python_src_path = ENV["INTERSCRIPT_PYTHON_SRC"] || + File.join(__dir__, "..", "..", "..", "..", "python", "src") unless ctx - PyCall.sys.path.append(python_src_path) + # Insert at 0: an editable install of interscript-python may + # shadow the monorepo checkout on sys.path. + PyCall.sys.path.insert(0, python_src_path) self.class.ctx = PyCall.import_module("interscript") + ctx = self.class.ctx end - # puts @code - begin - Dir.mkdir("#{python_src_path}/interscript/maps") - rescue - nil + # The rewritten python runtime executes ISC sources from its load + # paths, so in-memory documents cross the boundary as generated + # ISC text in a per-process temp dir — never inside the python + # package tree. + unless self.class.maps_dir + require "tmpdir" unless defined?(Dir.mktmpdir) + self.class.maps_dir = Dir.mktmpdir("interscript-python") + ctx.add_load_path(self.class.maps_dir) end - File.write("#{python_src_path}/interscript/maps/#{@map.name}.py", @code) + File.write(File.join(self.class.maps_dir, "#{@map.name}.isc"), + Interscript::Isc::Generator.generate(@map)) self.class.ctx.load_map(@map.name) - self.class.maps_loaded[@map.name] = true + if self.class.maps_loaded.key?(@map.name) + self.class.ctx.unload_map(@map.name) + end + self.class.maps_loaded[@map.name] = @map.object_id end end def call(str, stage = :main) load - self.class.ctx.transliterate(@map.name, str, stage.to_s) + unless stage.to_s == "main" + raise Interscript::MapLogicError, + "the python runtime transliterates the whole map; stage #{stage} can't be selected" + end + self.class.ctx.transliterate(@map.name, str) end def self.read_debug_data diff --git a/lib/interscript/isc.rb b/lib/interscript/isc.rb index d5aa2f93..a1aeb766 100644 --- a/lib/interscript/isc.rb +++ b/lib/interscript/isc.rb @@ -15,6 +15,7 @@ module Isc autoload :YamlBridge, "interscript/isc/yaml_bridge" autoload :Serializer, "interscript/isc/serializer" autoload :Normalizer, "interscript/isc/normalizer" + autoload :Generator, "interscript/isc/generator" SCHEMA_VERSION = 1 diff --git a/lib/interscript/isc/generator.rb b/lib/interscript/isc/generator.rb new file mode 100644 index 00000000..1505afce --- /dev/null +++ b/lib/interscript/isc/generator.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +module Interscript + module Isc + # Serializes a Node::Document back to ISC source — the inverse of + # Parser + DocumentBuilder + NodeAdapter. The Python compiler + # bridge generates ISC from in-memory documents because the + # rewritten python runtime executes ISC sources, not compiled + # modules. Every construct the generator emits must round-trip: + # spec/interscript/isc/generator_spec.rb pins behavior equality. + class Generator + def self.generate(doc) + new(doc).generate + end + + def initialize(doc) + @doc = doc + end + + def generate + out = "system #{quote(@doc.name.to_s)} {\n" + out << generate_aliases + out << generate_dependencies + @doc.stages.each do |name, group| + out << " stage #{name} {\n" << generate_group(group, 2) << " }\n" + end + out << "}\n" + end + + private + + def generate_aliases + return "" if @doc.aliases.empty? + + out = +" aliases {\n" + @doc.aliases.each_value do |alias_def| + out << " #{alias_def.name} = #{item(alias_def.data)}\n" + end + out << " }\n" + end + + def generate_dependencies + return "" if @doc.dependencies.empty? + + out = +"" + @doc.dependencies.each do |dep| + out << " dependency #{quote(dep.full_name.to_s)}" + out << " as #{dep.name}" if dep.name + out << "\n" + end + out + end + + def generate_group(group, indent) + Array(group.children).map { |child| emit(child, indent) }.join + end + + def emit(child, indent) + pad = " " * indent + case child + when Interscript::Node::Group::Parallel + "#{pad}parallel {\n#{generate_group(child, indent + 2)}#{pad}}\n" + when Interscript::Node::Group + generate_group(child, indent) + when Interscript::Node::Rule::Sub + rule(child, pad) + when Interscript::Node::Rule::Run + "#{pad}run #{run_target(child)}\n" + when Interscript::Node::Rule::Funcall + funcall(child, pad) + else + raise UnsupportedConstruct, "cannot generate #{child.class} as a stage item" + end + end + + def rule(sub, pad) + out = "#{pad}sub {\n" + out << "#{pad} from #{item(sub.from)}\n" + out << "#{pad} to #{target(sub.to)}\n" + {before: sub.before, after: sub.after, + not_before: sub.not_before, not_after: sub.not_after}.each do |kw, value| + out << "#{pad} #{kw} #{item(value)}\n" if value + end + out << "#{pad}}\n" + end + + def target(to) + return to.to_s if %i[upcase downcase].include?(to) + + item(to) + end + + def run_target(run) + stage = run.stage + stage.map ? "map.#{stage.map}.stage.#{stage.name}" : "stage.#{stage.name}" + end + + def funcall(rule, pad) + unless rule.kwargs.empty? + raise UnsupportedConstruct, + "cannot generate #{rule.name}(...) with kwargs as ISC" + end + + "#{pad}#{rule.name}\n" + end + + # -- items -- + + def item(i) + case i + when Interscript::Node::Item::String + quote(i.data) + when Interscript::Node::Item::Any + any(i) + when Interscript::Node::Item::Alias + identifier(i.name) + when Interscript::Node::Item::CaptureGroup + "capture(#{item(i.data)})" + when Interscript::Node::Item::CaptureRef + "ref(#{i.id})" + when Interscript::Node::Item::Group + i.children.map { |c| item(c) }.join(" + ") + when Interscript::Node::Item::Maybe + "maybe(#{item(i.data)})" + when Interscript::Node::Item::Some + "some(#{item(i.data)})" + when ::String + quote(i) + when ::Symbol + identifier(i) + else + raise UnsupportedConstruct, "cannot generate #{i.class} as an item" + end + end + + def any(node) + case node.value + when ::String + "any(#{quote(node.value)})" + when ::Range + "any(#{quote(node.value.first)}..#{quote(node.value.last)})" + when ::Array + inner = node.value.map { |v| item(v) } + return "any(#{inner.first})" if inner.size == 1 + + "any([#{inner.join(", ")}])" + else + raise UnsupportedConstruct, "cannot generate Any of #{node.value.class}" + end + end + + def identifier(sym) + name = sym.to_s + raise UnsupportedConstruct, "#{name.inspect} is not a bare identifier" unless /\A[A-Za-z_][A-Za-z0-9_]*\z/.match?(name) + + name + end + + ESCAPE_MAP = { + "\\" => "\\\\", + "\"" => "\\\"" + }.freeze + + def quote(str) + escaped = str.to_s.gsub(/[\\"]/) { |c| ESCAPE_MAP.fetch(c) } + # Only control and DEL characters need escaping; both parsers + # read raw UTF-8, including astral code points (surrogate-pair + # \u escapes are NOT re-joined). + escaped = escaped.each_char.map { |c| + (c.ord < 0x20 || c.ord == 0x7f) ? format("\\u%04x", c.ord) : c + }.join + "\"" + escaped + "\"" + end + + class UnsupportedConstruct < StandardError; end + end + end +end diff --git a/spec/interscript/isc/generator_spec.rb b/spec/interscript/isc/generator_spec.rb new file mode 100644 index 00000000..be8e0532 --- /dev/null +++ b/spec/interscript/isc/generator_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# Generator round-trip: every document the DSL can build must survive a +# generate → parse → adapt cycle with identical runtime behavior. This +# is the contract the Python bridge relies on — the rewritten python +# runtime executes ISC sources, so in-memory documents cross the +# boundary as generated ISC text. +require "spec_helper" + +RSpec.describe Interscript::Isc::Generator do + def round_trip(doc) + isc = described_class.generate(doc) + tree = Interscript::Isc::Parser.parse(isc) + built = Interscript::Isc::DocumentBuilder.build(tree) + Interscript::Isc::NodeAdapter.to_interscript_node(built) + end + + def round_trip_stage(&block) + original = stage(&block) + reparsed = round_trip(original) + [original, reparsed] + end + + it "generates parseable ISC for a stage document" do + doc = stage { sub "b", "e" } + isc = described_class.generate(doc) + expect(isc).to start_with(%(system "example-)) + expect { Interscript::Isc::Parser.parse(isc) }.not_to raise_error + end + + it "round-trips basic substitution" do + original, reparsed = round_trip_stage { sub "b", "e" } + expect(reparsed.call("abcd")).to eq(original.call("abcd")) + end + + it "round-trips substitution with all four constraints" do + original, reparsed = round_trip_stage { + sub "a", "A", before: space + sub "a", "B", not_before: space + sub "a", "C", after: space + sub "a", "D", not_after: space + } + expect(reparsed.call("abcda abcda abada")).to eq(original.call("abcda abcda abada")) + end + + it "round-trips a casing target" do + original, reparsed = round_trip_stage { sub "b", :upcase } + expect(reparsed.call("aba")).to eq(original.call("aba")) + end + + it "round-trips parallel blocks" do + original, reparsed = round_trip_stage { + parallel { + sub "ab", "X" + sub "a", "Y" + } + } + expect(reparsed.call("aba")).to eq(original.call("aba")) + end + + it "round-trips any() sets and ranges" do + original, reparsed = round_trip_stage { + sub any("abc"), "X" + sub "z", "W", before: any("aeiou") + } + expect(reparsed.call("zaza zab")).to eq(original.call("zaza zab")) + end + + it "round-trips capture groups and references" do + original, reparsed = round_trip_stage { + sub capture("b") + capture("c"), ref(2) + ref(1) + } + expect(reparsed.call("xbc")).to eq(original.call("xbc")) + end + + it "round-trips document-level aliases" do + original = document { + aliases do + def_alias :vowel, any("aeiou") + end + stage { + sub "v", "V", before: vowel + } + } + reparsed = round_trip(original) + expect(reparsed.call("ava eve")).to eq(original.call("ava eve")) + end + + it "round-trips multiple stages with a stage run" do + original = document { + stage :first do + sub "b", "e" + end + stage { + run Interscript::Node::Item::Stage.new(:first) + } + } + reparsed = round_trip(original) + expect(reparsed.call("abcd")).to eq(original.call("abcd")) + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7963afdf..d33efa25 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -19,6 +19,15 @@ # Disable RSpec exposing methods globally on `Module` and `main` config.disable_monkey_patching! + # dsl_stage_spec sets the $compiler global per example; without a + # reset it leaks into every file that runs afterwards, silently + # rerouting suites that never opted into a specific compiler. + # standard:disable Style/GlobalVars (deliberate $DEBUG / -d-flag debug idiom) + config.before(:example) do + $compiler = nil + end + # standard:enable Style/GlobalVars + config.expect_with :rspec do |c| c.syntax = :expect end