From 5ed4972e38fd1f74d5315ceae66d47d57eb04541 Mon Sep 17 00:00:00 2001 From: Adrian Madrid <65470515+aemadrid-twx@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:16:32 -0700 Subject: [PATCH 1/2] loopguard: count minitest, rake and rails test runs as verifications A Ruby project's checks run as `ruby` given a test file, `rake test` or `bin/rails test`, and minitest prints its own summary line, so the guard saw no verification in such a turn: one whose last run failed still counted as ended green. Co-Authored-By: Claude Opus 5.5 --- tests/test_live_core.py | 45 +++++++++++++++++++++++++++++++ tokeneconomizer/loopguard/turn.py | 10 +++++++ 2 files changed, 55 insertions(+) diff --git a/tests/test_live_core.py b/tests/test_live_core.py index 24c9648..07e5aa7 100644 --- a/tests/test_live_core.py +++ b/tests/test_live_core.py @@ -1398,6 +1398,47 @@ def test_a_runner_started_by_a_launcher_counts_as_a_verification(cmd): assert g.verifications == 1 and not g.resolved_proxy() +@pytest.mark.parametrize("cmd", ["ruby test/ledger_test.rb", "ruby -w -Ilib test/slugify_test.rb", + "ruby -I lib test/x_test.rb", "ruby /w/repo/test/x_test.rb", + "ruby -Itest test/models/user_test.rb", "ruby spec/foo_spec.rb", + "ruby -W0 --disable-gems test/test_parser.rb", + "bundle exec ruby -Itest test/a_test.rb -n test_x", + "rake test", "bundle exec rake test", "rake test:models", + "bin/rake test TEST=test/a_test.rb", "rake spec", "bin/rails test", + "rails test test/models/user_test.rb"]) +def test_a_ruby_test_run_counts_as_a_verification(cmd): + """minitest and test-unit are run as `ruby` given a test file, or through rake or rails, not + by a runner of their own as `rspec` is.""" + assert turn.status_of_tests(cmd) == "own" + g = guard.TurnGuard(turn_key="t") + g.fold("Edit", {"file_path": "/r/lib/a.rb", "old_string": "x", "new_string": "y"}, "ok") + g.fold("Bash", {"command": cmd}, "8 runs, 8 assertions, 1 failures, 0 errors, 0 skips", 1) + assert g.verifications == 1 and not g.resolved_proxy() + + +@pytest.mark.parametrize("cmd", ["ruby -Ilib -e 'require \"a\"; p A.new'", "ruby -wc lib/retry.rb", + "ruby -c test/a_test.rb", "ruby repro.rb", "ruby -v", + "ruby -e 'load \"test/a_test.rb\"'", "ruby lib/contest.rb", + "ruby -Icore lib/x.rb", "rake", "rake db:migrate", + "rails server", "cat test/a_test.rb"]) +def test_ruby_that_runs_no_test_file_is_not_a_verification(cmd): + """A one-off `ruby -e`, a syntax check and a script are not the project's checks.""" + assert turn.status_of_tests(cmd) is None + + +def test_a_piped_minitest_run_is_judged_by_its_summary(): + """`ruby test/x_test.rb | tail -5` exits with `tail`'s status, so minitest's summary line + decides whether the turn ended green.""" + g = guard.TurnGuard(turn_key="t") + g.fold("Edit", {"file_path": "/r/lib/a.rb", "old_string": "x", "new_string": "y"}, "ok") + g.fold("Bash", {"command": "ruby test/a_test.rb 2>&1 | tail -5"}, + "8 runs, 8 assertions, 1 failures, 0 errors, 0 skips", 0) + assert g.verifications == 1 and not g.resolved_proxy() + g.fold("Bash", {"command": "ruby test/a_test.rb 2>&1 | tail -5"}, + "8 runs, 8 assertions, 0 failures, 0 errors, 0 skips", 0) + assert g.verifications == 2 and g.resolved_proxy() + + @pytest.mark.parametrize("cmd,status", [("uv run ruff check", None), ("npx prettier --check .", None), ("pnpm exec jest --help", None), @@ -1517,6 +1558,10 @@ def test_a_test_run_followed_by_other_commands_is_judged_by_its_runners_summary( ("ok \tacme/x\t0.012s", False), ("test result: FAILED. 1 passed; 1 failed", True), ("test result: ok. 3 passed; 0 failed", False), ("5 examples, 1 failure", True), ("Tests run: 4, Failures: 0, Errors: 1", True), ("[INFO] BUILD SUCCESS", False), + ("8 runs, 8 assertions, 1 failures, 0 errors, 0 skips", True), + ("8 runs, 8 assertions, 0 failures, 2 errors, 0 skips", True), + ("Finished in 0.01s\n5 runs, 9 assertions, 0 failures, 0 errors, 1 skips", False), + ("1 tests, 1 assertions, 0 failures, 0 errors, 0 pendings, 0 omissions", False), ("+ raise ValueError('no')", None), ("ok 1 - add", None)]) def test_a_runners_summary_gives_its_verdict(out, failed): assert turn.runner_verdict(out) is failed diff --git a/tokeneconomizer/loopguard/turn.py b/tokeneconomizer/loopguard/turn.py index cf2f2e9..b0c124c 100644 --- a/tokeneconomizer/loopguard/turn.py +++ b/tokeneconomizer/loopguard/turn.py @@ -41,6 +41,12 @@ r"(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?test|go\s+test|cargo\s+(?:test|nextest)|" r"(?:mvn|mvnw|\./mvnw)\s+(?:\S+\s+)*test|(?:gradle|gradlew|\./gradlew)\s+(?:\S+\s+)*test|" r"make\s+(?:test|check)|ctest|rspec|phpunit|dotnet\s+test|swift\s+test|mix\s+test|" + # Ruby: `rake test`, `bin/rails test`, and `ruby` given a test file (`ruby -Itest + # test/models/user_test.rb`), but not `ruby -e '...'`, a syntax check (`ruby -wc x.rb`) or a + # script that is not a test (`ruby repro.rb`): the options it takes are spelled out. + r"rake(?:\s+\w+=\S*)*\s+(?:test|spec)(?::[\w:]+)?|rails\s+test(?::[\w:]+)?|" + r"ruby(?:\s+(?:-[IrC](?:\S+|\s+[^\s\-]\S*)|-W\S*|--[\w\-]+(?:=\S+)?|-[abdf-z]+))*\s+" + r"(?:[\w.\-~]*/)*(?:test_[\w.\-]*|[\w.\-]*_(?:test|spec))\.rb|" r"deno\s+test|node\s+--test|bazel\s+test|(?:\./)?bin/test)(?![\w.\-/])") # A command line's exit status is its LAST command's, so it may not be the test run's: `npm test @@ -178,6 +184,9 @@ def status_of_tests(cmd): r"^(?:--- FAIL: |FAIL\t|FAIL$)", # go r"^test result: FAILED", # cargo r"\b\d+ examples?, [1-9]\d* failures?\b", # rspec + # minitest (`8 runs, 9 assertions, 1 failures, 0 errors, 0 skips`), test-unit (`tests`) + r"^\d+ (?:runs|tests), \d+ assertions, (?:[1-9]\d* failures, \d+ errors|\d+ failures, " + r"[1-9]\d* errors)\b", r"Tests run: \d+, Failures: (?:[1-9]|0, Errors: [1-9])", # JUnit (Maven, Gradle) r"BUILD FAIL(?:URE|ED)")), re.MULTILINE) _RUN_PASSED_RE = re.compile("|".join(( @@ -192,6 +201,7 @@ def status_of_tests(cmd): r"^ok\s+\S+\s+(?:\d+(?:\.\d+)?s|\(cached\))", r"^test result: ok\.", r"\b\d+ examples?, 0 failures\b", + r"^\d+ (?:runs|tests), \d+ assertions, 0 failures, 0 errors\b", # minitest, test-unit r"BUILD SUCCESS")), re.MULTILINE) From 64608e476efb1ab4217a7fe3cd5e6d4e92c58f42 Mon Sep 17 00:00:00 2001 From: Adrian Madrid <65470515+aemadrid-twx@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:44:29 -0700 Subject: [PATCH 2/2] loopguard: read a ruby command's options case-sensitively Under VERIFY_RE's (?i), `-W\S*` also took `-wc` and `-[IrC]` took `-cw`, so a syntax check of a test file (`ruby -wc test/a_test.rb`) counted as a verification. Found in review. Co-Authored-By: Claude Opus 5.5 --- tests/test_live_core.py | 6 +++++- tokeneconomizer/loopguard/turn.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_live_core.py b/tests/test_live_core.py index 07e5aa7..da2c9e1 100644 --- a/tests/test_live_core.py +++ b/tests/test_live_core.py @@ -1402,6 +1402,8 @@ def test_a_runner_started_by_a_launcher_counts_as_a_verification(cmd): "ruby -I lib test/x_test.rb", "ruby /w/repo/test/x_test.rb", "ruby -Itest test/models/user_test.rb", "ruby spec/foo_spec.rb", "ruby -W0 --disable-gems test/test_parser.rb", + "ruby -W:no-deprecated test/a_test.rb", "ruby -wv test/a_test.rb", + "ruby -C /w -Itest test/a_test.rb", "bundle exec ruby -Itest test/a_test.rb -n test_x", "rake test", "bundle exec rake test", "rake test:models", "bin/rake test TEST=test/a_test.rb", "rake spec", "bin/rails test", @@ -1417,7 +1419,9 @@ def test_a_ruby_test_run_counts_as_a_verification(cmd): @pytest.mark.parametrize("cmd", ["ruby -Ilib -e 'require \"a\"; p A.new'", "ruby -wc lib/retry.rb", - "ruby -c test/a_test.rb", "ruby repro.rb", "ruby -v", + "ruby -c test/a_test.rb", "ruby -wc test/a_test.rb", + "ruby -Wc test/a_test.rb", "ruby -cw test/a_test.rb", + "ruby -we 'load \"test/a_test.rb\"'", "ruby repro.rb", "ruby -v", "ruby -e 'load \"test/a_test.rb\"'", "ruby lib/contest.rb", "ruby -Icore lib/x.rb", "rake", "rake db:migrate", "rails server", "cat test/a_test.rb"]) diff --git a/tokeneconomizer/loopguard/turn.py b/tokeneconomizer/loopguard/turn.py index b0c124c..c23255a 100644 --- a/tokeneconomizer/loopguard/turn.py +++ b/tokeneconomizer/loopguard/turn.py @@ -43,9 +43,11 @@ r"make\s+(?:test|check)|ctest|rspec|phpunit|dotnet\s+test|swift\s+test|mix\s+test|" # Ruby: `rake test`, `bin/rails test`, and `ruby` given a test file (`ruby -Itest # test/models/user_test.rb`), but not `ruby -e '...'`, a syntax check (`ruby -wc x.rb`) or a - # script that is not a test (`ruby repro.rb`): the options it takes are spelled out. + # script that is not a test (`ruby repro.rb`): the options it takes are spelled out, and + # case-sensitive, so `-W` does not also take `-wc` under the pattern's `(?i)`. r"rake(?:\s+\w+=\S*)*\s+(?:test|spec)(?::[\w:]+)?|rails\s+test(?::[\w:]+)?|" - r"ruby(?:\s+(?:-[IrC](?:\S+|\s+[^\s\-]\S*)|-W\S*|--[\w\-]+(?:=\S+)?|-[abdf-z]+))*\s+" + r"ruby(?-i:\s+(?:-[Ir](?:\S+|\s+[^\s\-]\S*)|-C\s*\S+|-W[0-2]?(?::\S+)?|" + r"--[\w\-]+(?:=\S+)?|-[abdf-zABDF-Z]+))*\s+" r"(?:[\w.\-~]*/)*(?:test_[\w.\-]*|[\w.\-]*_(?:test|spec))\.rb|" r"deno\s+test|node\s+--test|bazel\s+test|(?:\./)?bin/test)(?![\w.\-/])")