From 762cb0e38be0e96a71825cdb2f89b426edc0ea65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Fri, 21 Nov 2025 21:59:36 +0000 Subject: [PATCH 01/18] Movel all hardness to a single directory Use file based harness instead of directory based harness --- README.md | 62 ++++++++++++------- benchmarks.yml | 18 +++--- burn_in.rb | 5 +- harness-bips/harness.rb => harness/bips.rb | 2 +- harness-chain/harness.rb => harness/chain.rb | 8 +-- .../harness.rb => harness/continuous.rb | 2 +- harness-gc/harness.rb => harness/gc.rb | 2 +- harness/loader.rb | 9 ++- harness-mplr/harness.rb => harness/mplr.rb | 2 +- harness-once/harness.rb => harness/once.rb | 2 +- harness-perf/harness.rb => harness/perf.rb | 4 +- .../harness.rb => harness/ractor.rb | 2 +- .../harness.rb => harness/spinel.rb | 2 +- .../harness.rb => harness/stackprof.rb | 8 +-- harness-stats/harness.rb => harness/stats.rb | 2 +- .../harness.rb => harness/vernier.rb | 8 +-- .../harness.rb => harness/warmup.rb | 2 +- lib/argument_parser.rb | 7 ++- lib/benchmark_filter.rb | 2 +- lib/benchmark_suite.rb | 14 ++++- run_once.sh | 6 +- test/argument_parser_test.rb | 12 ++-- test/benchmark_filter_test.rb | 8 +-- test/benchmark_runner_cli_test.rb | 16 ++--- test/benchmark_runner_test.rb | 14 ++--- test/benchmark_suite_test.rb | 10 +-- test/benchmarks_test.rb | 2 +- test/ractor_gc_harness_test.rb | 4 +- test/ractor_harness_test.rb | 2 +- 29 files changed, 136 insertions(+), 101 deletions(-) rename harness-bips/harness.rb => harness/bips.rb (79%) rename harness-chain/harness.rb => harness/chain.rb (85%) rename harness-continuous/harness.rb => harness/continuous.rb (94%) rename harness-gc/harness.rb => harness/gc.rb (99%) rename harness-mplr/harness.rb => harness/mplr.rb (93%) rename harness-once/harness.rb => harness/once.rb (88%) rename harness-perf/harness.rb => harness/perf.rb (94%) rename harness-ractor/harness.rb => harness/ractor.rb (99%) rename harness-spinel/harness.rb => harness/spinel.rb (97%) rename harness-stackprof/harness.rb => harness/stackprof.rb (91%) rename harness-stats/harness.rb => harness/stats.rb (98%) rename harness-vernier/harness.rb => harness/vernier.rb (71%) rename harness-warmup/harness.rb => harness/warmup.rb (98%) diff --git a/README.md b/README.md index fb758955..1ea4b242 100644 --- a/README.md +++ b/README.md @@ -138,19 +138,19 @@ There are two Ractor-related categories: * **`--category ractor`** - Runs both regular benchmarks marked with `ractor: true` in `benchmarks.yml` AND all benchmarks from the `benchmarks-ractor` - directory. The `harness-ractor` harness is used for both types of benchmark. + directory. The `ractor` harness is used for both types of benchmark. * **`--category ractor-only`** - Runs ONLY benchmarks from the `benchmarks-ractor` directory, ignoring regular benchmarks even if they are marked with `ractor: true`. This category also automatically uses the - `harness-ractor` harness. + `ractor` harness. ### Directory Structure The `benchmarks-ractor/` directory sits at the same level as the main `benchmarks` directory, and contains Ractor-specific benchmark implementations that are designed to test Ractor functionality. They are not -intended to be used with any harness except `harness-ractor`. +intended to be used with any harness except `ractor`. ### Usage Examples @@ -162,7 +162,7 @@ intended to be used with any harness except `harness-ractor`. ./run_benchmarks.rb --category ractor-only ``` -Note: The `harness-ractor` harness is automatically selected when using these +Note: The `ractor` harness is automatically selected when using these categories, so there's no need to specify `--harness` manually. ### Ractor counts @@ -188,12 +188,12 @@ because no single process ran all counts. The text summary, the CSV output, and `misc/zjit_diff.rb` show one row for each count, with the RSS and JIT stats of that count's process. -When you run a benchmark directly with `-Iharness-ractor`, the harness runs +When you run a benchmark directly with `-Iharness -rractor`, the harness runs all counts in one process, one count after another. ### Ractor Scenario Benchmarks -By default, `harness-ractor` spawns the worker Ractors and runs the benchmark +By default, the `ractor` harness spawns the worker Ractors and runs the benchmark block inside each of them. A benchmark that calls `run_benchmark(n, scenario: true)` uses scenario mode instead. The harness calls the block one time per trial in the main Ractor, with the Ractor count as @@ -232,7 +232,7 @@ with multiple ractors, for GC work that reclaims ractor-local memory: ```bash # Every count in one process -ruby -Iharness-ractor benchmarks/ractor-dead-set/benchmark.rb +ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb # One process per count, comparing two Rubies, with Ractor GC metrics RUBY_BENCH_RACTORS=1,2,4,6,8 ./run_benchmarks.rb --category ractor --ractor-gc \ @@ -321,22 +321,40 @@ This file will then be passed to the underlying Ruby interpreter with ## Harnesses -You can find several test harnesses in this repository: +You can find several test harnesses in the `harness/` directory: -* harness - the normal default harness, with duration controlled by warmup iterations and time/count limits -* harness-bips - a harness that measures iterations/second until stable -* harness-continuous - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches -* harness-once - a simplified harness that simply runs once -* harness-perf - a simplified harness that runs for exactly the hinted number of iterations -* harness-stackprof - a harness to profile the benchmark with stackprof -* harness-stats - count method calls and loop iterations -* harness-vernier - a harness to profile the benchmark with vernier -* harness-warmup - a harness which runs as long as needed to find warmed up (peak) performance +* `harness` - the normal default harness, with duration controlled by warmup iterations and time/count limits +* `bips` - a harness that measures iterations/second until stable +* `continuous` - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches +* `once` - a simplified harness that simply runs once +* `perf` - a simplified harness that runs for exactly the hinted number of iterations +* `ractor` - a harness that runs the benchmark inside Ractors to measure scaling +* `gc` - the default harness plus GC timing, count and heap statistics +* `stackprof` - a harness to profile the benchmark with stackprof +* `stats` - count method calls and loop iterations +* `vernier` - a harness to profile the benchmark with vernier +* `warmup` - a harness which runs as long as needed to find warmed up (peak) performance +* `chain` - a harness to chain multiple harnesses together +* `mplr` - a harness for multiple iterations with time limits -To use it, run a benchmark script directly, specifying a harness directory with `-I`: +To use a specific harness, run a benchmark script directly with `-I` to add the harness directory to the load path, and `-r` to require the specific harness: ``` +# Use default harness ruby -Iharness benchmarks/railsbench/benchmark.rb + +# Use the 'once' harness +ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb + +# Use the 'perf' harness +ruby -Iharness -rperf benchmarks/railsbench/benchmark.rb +``` + +When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: + +``` +./run_benchmarks.rb --harness=once +./run_benchmarks.rb --harness=perf ``` There is also a robust but complex CI harness in [the yjit-metrics repo](https://github.com/Shopify/yjit-metrics). @@ -375,10 +393,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb +perf record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb +PERF=record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. @@ -412,7 +430,7 @@ benchmark, also set `RUBY_BENCH_PROFILING=1` to keep the harness's retention GCs out of the profile. The worker Ractors show up as separate thread tracks. ```sh -RUBY_BENCH_RACTORS=4 RUBY_BENCH_PROFILING=1 samply record ruby -Iharness-ractor benchmarks/ractor-dead-set/benchmark.rb +RUBY_BENCH_RACTORS=4 RUBY_BENCH_PROFILING=1 samply record ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb ``` samply also records child processes, so `samply record ruby run_benchmarks.rb ...` @@ -465,7 +483,7 @@ fail before warmup. ./run_benchmarks.rb --category ractor --chruby=base::ruby-base --ractor-gc # One benchmark, run directly. --ractor-gc sets RUBY_BENCH_RACTOR_GC=1. -RUBY_BENCH_RACTOR_GC=1 ruby -Iharness-ractor benchmarks/ractor-dead-set/benchmark.rb +RUBY_BENCH_RACTOR_GC=1 ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb ``` `run_benchmarks.rb` only runs Ractor harness benchmarks with `--category ractor` diff --git a/benchmarks.yml b/benchmarks.yml index d11b7570..c49fbcb2 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -81,7 +81,7 @@ gcbench: desc: Ellis-Kovac-Boehm GCBench builds binary trees of various depths to exercise GC marking, sweeping, and write barriers. category: gc single_file: true - default_harness: harness-gc + default_harness: gc erubi: desc: erubi compiles a simple Erb template into a method with erubi, then times evaluating that method. ractor: true @@ -145,7 +145,7 @@ string_malloc_pressure: desc: Allocates and retains many empty Strings with large reserved capacity to exercise GC under malloc-backed String buffer pressure. category: gc single_file: true - default_harness: harness-gc + default_harness: gc tinygql: desc: TinyGQL gem parsing a large file in pure Ruby @@ -285,37 +285,37 @@ gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. ractor: true ractor_only: true - default_harness: harness-ractor + default_harness: ractor json_parse_float: desc: test the performance of parsing multiple lists of json floats with ractors. ractor: true ractor_only: true - default_harness: harness-ractor + default_harness: ractor json_parse_string: desc: test the performance of parsing multiple lists of strings with ractors. ractor: true ractor_only: true - default_harness: harness-ractor + default_harness: ractor ractor-dead-set: desc: multiple ractors each build a large live set and terminate, then memory retention after their death is measured. ractor: true ractor_only: true ractor_scenario: true - default_harness: harness-ractor + default_harness: ractor ractor-idle-garbage: desc: multiple ractors each build and drop a large garbage set, then idle uncollected while retention is measured. ractor: true ractor_only: true ractor_scenario: true - default_harness: harness-ractor + default_harness: ractor ractor-msg-backlog: desc: unshareable message payloads flood the queues of gated consumer ractors, duplicating data per consumer and driving peak RSS and retention. ractor: true ractor_only: true ractor_scenario: true - default_harness: harness-ractor + default_harness: ractor symbol-name-ractor: desc: repeatedly calls Symbol#name on a static symbol under the ractor harness to stress ID-to-string lookup. ractor: true ractor_only: true - default_harness: harness-ractor + default_harness: ractor diff --git a/burn_in.rb b/burn_in.rb index 91b1ada1..0f29a114 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -104,7 +104,8 @@ def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metada cmd = [ 'ruby', *test_options, - "-I#{harness}", + "-Iharness", + *(harness == 'harness' ? [] : ["-r#{harness}"]), script_path, ].compact cmd_str = cmd.shelljoin @@ -225,7 +226,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata) metadata.each do |name, entry| category = entry.fetch('category', 'other') is_ractor_only = entry['ractor_only'] || - (entry['ractor'] && entry['default_harness'] == 'harness-ractor') + (entry['ractor'] && entry['default_harness'] == 'ractor') if args.categories.include?(category) && !is_ractor_only bench_names << name end diff --git a/harness-bips/harness.rb b/harness/bips.rb similarity index 79% rename from harness-bips/harness.rb rename to harness/bips.rb index d1319a19..b583fa0f 100644 --- a/harness-bips/harness.rb +++ b/harness/bips.rb @@ -1,5 +1,5 @@ require 'benchmark/ips' -require_relative "../harness/harness-common" +require_relative "harness-common" puts RUBY_DESCRIPTION diff --git a/harness-chain/harness.rb b/harness/chain.rb similarity index 85% rename from harness-chain/harness.rb rename to harness/chain.rb index 48e6b0f9..eef9d77b 100644 --- a/harness-chain/harness.rb +++ b/harness/chain.rb @@ -1,4 +1,4 @@ -require_relative '../harness/harness-common' +require_relative 'harness-common' # Ex: HARNESS_CHAIN="vernier,ractor" # Wraps the ractor harness in ther vernier harness @@ -10,7 +10,7 @@ end if CHAIN.include?("vernier") && CHAIN.last != "vernier" - require_relative "../harness/harness-extra" + require_relative "harness-extra" def run_enough_to_profile(n, **kwargs, &block) block.call end @@ -30,12 +30,12 @@ def self.method_added(name) def run_benchmark(n, **kwargs, &block) CHAIN.each do |h| begin - path = "../harness-#{h}/harness" + path = "#{h}" $current_harness = h require_relative path rescue LoadError => e if e.path == path - $stderr.puts "Can't find harness harness-#{h}/harness.rb. Exiting." + $stderr.puts "Can't find harness #{h}.rb in harness/. Exiting." exit 1 end raise diff --git a/harness-continuous/harness.rb b/harness/continuous.rb similarity index 94% rename from harness-continuous/harness.rb rename to harness/continuous.rb index 6606b182..f76f535b 100644 --- a/harness-continuous/harness.rb +++ b/harness/continuous.rb @@ -1,4 +1,4 @@ -require_relative "../harness/harness-common" +require_relative "harness-common" puts RUBY_DESCRIPTION diff --git a/harness-gc/harness.rb b/harness/gc.rb similarity index 99% rename from harness-gc/harness.rb rename to harness/gc.rb index ab627e1f..18ccb863 100644 --- a/harness-gc/harness.rb +++ b/harness/gc.rb @@ -1,4 +1,4 @@ -require_relative "../harness/harness-common" +require_relative "harness-common" require_relative "../lib/gc_stats" WARMUP_ITRS = Integer(ENV.fetch('WARMUP_ITRS', 15)) diff --git a/harness/loader.rb b/harness/loader.rb index 5feb027e..ece20ea1 100644 --- a/harness/loader.rb +++ b/harness/loader.rb @@ -1,7 +1,10 @@ -# Use harness/harness.rb by default. You can change it with -I option. -# i.e. ruby -Iharness benchmarks/railsbench/benchmark.rb +# Use harness/harness.rb by default. You can change it with -I and -r options. +# Examples: +# ruby -Iharness benchmarks/railsbench/benchmark.rb # uses harness/harness.rb +# ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ruby -Iharness -rractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb if RUBY_ENGINE == "spinel" - require_relative "../harness-spinel/harness" + require_relative "spinel" else retries = 0 begin diff --git a/harness-mplr/harness.rb b/harness/mplr.rb similarity index 93% rename from harness-mplr/harness.rb rename to harness/mplr.rb index 73234934..4473a204 100644 --- a/harness-mplr/harness.rb +++ b/harness/mplr.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative "../harness/harness-common" +require_relative "harness-common" # Minimum number of benchmarking iterations MAX_BENCH_ITRS = Integer(ENV.fetch('MAX_BENCH_ITRS', 1000)) diff --git a/harness-once/harness.rb b/harness/once.rb similarity index 88% rename from harness-once/harness.rb rename to harness/once.rb index c03d9db5..dbf44e60 100644 --- a/harness-once/harness.rb +++ b/harness/once.rb @@ -5,7 +5,7 @@ # Intended only for checking whether the benchmark can set itself up properly # and can run to completion. -require_relative '../harness/harness-common' +require_relative 'harness-common' def run_benchmark(_hint, **) yield diff --git a/harness-perf/harness.rb b/harness/perf.rb similarity index 94% rename from harness-perf/harness.rb rename to harness/perf.rb index 4991cb54..a778526e 100644 --- a/harness-perf/harness.rb +++ b/harness/perf.rb @@ -3,14 +3,14 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -Iharness-perf benchmarks/fib.rb +# $ PERF='record -e cycles' ruby -Iharness -rperf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race # condition is possible where the benchmark finishes before the perf(1) # subprocess has a chance to attach, in which case perf outputs no profile. -require_relative "../harness/harness-common" +require_relative "harness-common" # Run $WARMUP_ITRS or 10 iterations of a given block. Then run $MIN_BENCH_ITRS # or `num_itrs_int` iterations of the block, attaching a perf command to the diff --git a/harness-ractor/harness.rb b/harness/ractor.rb similarity index 99% rename from harness-ractor/harness.rb rename to harness/ractor.rb index ee5bdfa0..6af5b55a 100644 --- a/harness-ractor/harness.rb +++ b/harness/ractor.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative '../harness/harness-common' +require_relative 'harness-common' require 'etc' Warning[:experimental] = false diff --git a/harness-spinel/harness.rb b/harness/spinel.rb similarity index 97% rename from harness-spinel/harness.rb rename to harness/spinel.rb index 969524fe..f015b0c4 100644 --- a/harness-spinel/harness.rb +++ b/harness/spinel.rb @@ -9,7 +9,7 @@ # # harness/loader.rb selects this harness automatically when RUBY_ENGINE is # "spinel", so `spinel -E benchmarks/fib.rb` works without any -I option. It -# also runs on CRuby (`ruby -Iharness-spinel benchmarks/fib.rb`), which is how +# also runs on CRuby (`ruby -Iharness -rspinel benchmarks/fib.rb`), which is how # its output is kept compatible with the default harness. require "json" diff --git a/harness-stackprof/harness.rb b/harness/stackprof.rb similarity index 91% rename from harness-stackprof/harness.rb rename to harness/stackprof.rb index 6a63d1c5..c9bf7a81 100644 --- a/harness-stackprof/harness.rb +++ b/harness/stackprof.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb -require_relative "../harness/harness-common" -require_relative "../harness/harness-extra" +require_relative "harness-common" +require_relative "harness-extra" ensure_global_gem("stackprof") diff --git a/harness-stats/harness.rb b/harness/stats.rb similarity index 98% rename from harness-stats/harness.rb rename to harness/stats.rb index a0b9a2a3..b9b216ae 100644 --- a/harness-stats/harness.rb +++ b/harness/stats.rb @@ -1,4 +1,4 @@ -require_relative '../harness/harness' +require_relative 'harness' # Using Module#prepend to enable TracePoint right before #run_benchmark # while also reusing the original implementation. diff --git a/harness-vernier/harness.rb b/harness/vernier.rb similarity index 71% rename from harness-vernier/harness.rb rename to harness/vernier.rb index fe32ff1c..2b7c93ff 100644 --- a/harness-vernier/harness.rb +++ b/harness/vernier.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... +# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... +# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... -require_relative "../harness/harness-common" -require_relative "../harness/harness-extra" +require_relative "harness-common" +require_relative "harness-extra" ensure_global_gem("vernier") ensure_global_gem_exe("profile-viewer") diff --git a/harness-warmup/harness.rb b/harness/warmup.rb similarity index 98% rename from harness-warmup/harness.rb rename to harness/warmup.rb index 5f1050cc..83b1848b 100644 --- a/harness-warmup/harness.rb +++ b/harness/warmup.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative '../harness/harness-common' +require_relative 'harness-common' require_relative '../misc/stats' MIN_ITERS = Integer(ENV['MIN_ITERS'] || 10) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 556dc648..665e22a0 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -77,7 +77,7 @@ def parse(argv) opts.on("--category=headline,other,micro,ractor", "when given, only benchmarks with specified categories will run") do |v| args.categories += v.split(",") if args.categories == ["ractor"] - args.harness = "harness-ractor" + args.harness = "ractor" end end @@ -101,8 +101,9 @@ def parse(argv) args.skip_zjit = true end - opts.on("--harness=HARNESS_DIR", "which harness to use") do |v| - v = "harness-#{v}" unless v.start_with?('harness') + opts.on("--harness=HARNESS_NAME", "which harness to use (default: harness, options: once, bips, perf, ractor, gc, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |v| + # Strip 'harness-' prefix if provided for backward compatibility + v = v.sub(/^harness-/, '') args.harness = v args.harness_explicit = true end diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index a9ed5586..18f2b0b8 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -30,7 +30,7 @@ def matches_category?(name) def ractor_harness_benchmark?(name) benchmark_metadata = @metadata[name] || {} benchmark_metadata['ractor_only'] || - (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'harness-ractor') + (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'ractor') end def matches_name_filter?(name) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 48c474b9..27613d1f 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -18,7 +18,8 @@ class BenchmarkSuite BENCHMARKS_DIR = "benchmarks" RACTOR_CATEGORY = ["ractor"].freeze RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze - RACTOR_HARNESS = "harness-ractor" + RACTOR_HARNESS = "ractor" + HARNESS_DIR = File.expand_path("../harness", __dir__) PIN_CPU_ENV = "RUBY_BENCH_PIN_CPU" attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :harness_explicit, :pre_init, :no_pinning, :force_pinning, :bench_dir @@ -205,9 +206,18 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command + # If harness is 'harness', use default (no -r needed) + # Otherwise use -r to load the specific harness file + harness_args = if benchmark_harness == "harness" + [] + else + ["-r", benchmark_harness] + end + cmd = (cmd_prefix + [ *ruby, - "-I", benchmark_harness, + "-I", HARNESS_DIR, + *harness_args, *pre_init, script_path, ].compact).map(&:to_s) diff --git a/run_once.sh b/run_once.sh index d2035e8d..a5240fce 100755 --- a/run_once.sh +++ b/run_once.sh @@ -7,12 +7,12 @@ # ./run_once.sh benchmarks-ractor/optcarrot/benchmark.rb # Detect if any argument contains benchmarks-ractor/ to determine harness -HARNESS="./harness" +HARNESS_ARGS=() for arg in "$@"; do if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS="./harness-ractor" + HARNESS_ARGS=("-r" "ractor") break fi done -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"$HARNESS" "$@" \ No newline at end of file +WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"./harness" "${HARNESS_ARGS[@]}" "$@" \ No newline at end of file diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 739e10a3..5bf5ef1e 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -296,12 +296,12 @@ def setup_mock_ruby(path) assert_equal ['headline', 'micro'], args.categories end - it 'sets harness to harness-ractor when category is ractor' do + it 'sets harness to ractor when category is ractor' do parser = ArgumentParser.new args = parser.parse(['--category=ractor']) assert_equal ['ractor'], args.categories - assert_equal 'harness-ractor', args.harness + assert_equal 'ractor', args.harness end it 'allows multiple category flags' do @@ -354,18 +354,18 @@ def setup_mock_ruby(path) end describe '--harness option' do - it 'sets harness directory' do + it 'sets harness name' do parser = ArgumentParser.new args = parser.parse(['--harness=once']) - assert_equal 'harness-once', args.harness + assert_equal 'once', args.harness end - it 'accepts harness- prefix' do + it 'strips harness- prefix for backward compatibility' do parser = ArgumentParser.new args = parser.parse(['--harness=harness-stats']) - assert_equal 'harness-stats', args.harness + assert_equal 'stats', args.harness end end diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index 1abdfada..b7302bb3 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -48,16 +48,16 @@ end it 'excludes ractor harness benchmarks from default runs' do - metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'ractor' }) filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: metadata) assert_equal true, filter.match?('fib') - assert_equal true, filter.match?('ractor_bench') # ractor: true without harness-ractor runs in default - assert_equal false, filter.match?('ractor_harness_bench') # ractor: true with harness-ractor excluded + assert_equal true, filter.match?('ractor_bench') # ractor: true without ractor harness runs in default + assert_equal false, filter.match?('ractor_harness_bench') # ractor: true with ractor harness excluded end it 'includes ractor harness benchmarks in ractor category' do - metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'ractor' }) filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: metadata) assert_equal true, filter.match?('ractor_harness_bench') diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index dadff083..5cb0ad13 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -108,12 +108,12 @@ def create_args(overrides = {}) } bench_harnesses = { 'fib' => 'harness', - 'symbol-name-ractor' => 'harness-ractor' + 'symbol-name-ractor' => 'ractor' } sections = cli.send(:build_output_sections, ['ruby'], bench_data, bench_harnesses, {}) - assert_equal ['harness-ractor', 'harness'], sections.map { |section| section[:title] } + assert_equal ['ractor', 'harness'], sections.map { |section| section[:title] } assert_equal ['bench', 'ractors', 'ruby (ms)'], sections[0][:table][0] assert_equal ['symbol-name-ractor', '0', '1000.0 ± 0.0%'], sections[0][:table][1] assert_equal ['', '2', '2000.0 ± 0.0%'], sections[0][:table][2] @@ -153,10 +153,10 @@ def create_args(overrides = {}) } } } - bench_harnesses = { 'fib' => 'harness', 'object-new' => 'harness-ractor' } + bench_harnesses = { 'fib' => 'harness', 'object-new' => 'ractor' } sections = cli.send(:build_output_sections, ['ruby'], bench_data, bench_harnesses, {}) - ractor = sections.find { |section| section[:title] == 'harness-ractor' } + ractor = sections.find { |section| section[:title] == 'ractor' } normal = sections.find { |section| section[:title] == 'harness' } assert_equal 'ractor-local-workload', ractor[:gc_scope] @@ -195,7 +195,7 @@ def create_args(overrides = {}) } } - sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'object-new' => 'harness-ractor' }, {}) + sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'object-new' => 'ractor' }, {}) assert sections.one?, 'single-harness fixture must produce exactly one section' section = sections.first @@ -230,7 +230,7 @@ def create_args(overrides = {}) } } - sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'ractor-dead-set' => 'harness-ractor' }, {}) + sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'ractor-dead-set' => 'ractor' }, {}) output = BenchmarkRunner.build_output_text({ 'ruby' => 'ruby 4.1.0dev' }, nil, nil, {}, sections: sections) assert_match(/Scenario benchmarks sample only the worker Ractors whose bodies the scenario wraps in measure_worker_gc/, output) @@ -259,7 +259,7 @@ def create_args(overrides = {}) } } - sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'ractor-dead-set' => 'harness-ractor' }, {}) + sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'ractor-dead-set' => 'ractor' }, {}) memory = sections.first[:memory_tables] assert_equal 1, memory.size @@ -314,7 +314,7 @@ def create_args(overrides = {}) data end bench_data = { 'ruby' => { 'object-new' => blob.call(%w[0 2], nil), 'ractor-dead-set' => blob.call(%w[1 2], 'scenario') } } - harnesses = { 'object-new' => 'harness-ractor', 'ractor-dead-set' => 'harness-ractor' } + harnesses = { 'object-new' => 'ractor', 'ractor-dead-set' => 'ractor' } sections = cli.send(:build_output_sections, ['ruby'], bench_data, harnesses, {}) diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 77dcb4e7..ce7e5965 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -474,7 +474,7 @@ def git(dir, *args) ruby_descriptions = { 'ruby' => 'ruby 4.1.0dev' } sections = [ { - title: 'harness-ractor', + title: 'ractor', table: [['bench', 'ractors', 'ruby (ms)'], ['object-new', '1', '200.0']], format: ['%s', '%s', '%.1f'], failures: {}, @@ -493,7 +493,7 @@ def git(dir, *args) ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections ) - assert_includes result, "GC summary (worker sum, harness-ractor):\n" + assert_includes result, "GC summary (worker sum, ractor):\n" assert_includes result, "Hidden columns (zero or N/A in every row): global/iter, compacts*\n" refute_includes result, 'GC metric notes:', 'a hidden compaction column needs no note' assert_includes result, '- Tables marked worker sum add Ractor-local counters' @@ -580,7 +580,7 @@ def git(dir, *args) ruby_descriptions = { 'ruby' => 'ruby 4.1.0dev' } sections = [ { - title: 'harness-ractor', + title: 'ractor', table: [['bench', 'ractors', 'ruby (ms)'], ['object-new', '0', '200.0']], format: ['%s', '%s', '%.1f'], failures: {}, @@ -667,7 +667,7 @@ def git(dir, *args) include_gc: false, }, { - title: 'harness-ractor', + title: 'ractor', table: [['bench', 'ractors', 'ruby (ms)'], ['symbol-name-ractor', '0', '200.0']], format: ['%s', '%s', '%.1f'], failures: {}, @@ -680,7 +680,7 @@ def git(dir, *args) ) assert_includes result, "harness:\n" - assert_includes result, "harness-ractor:\n" + assert_includes result, "ractor:\n" assert_includes result, 'fib' assert_includes result, 'symbol-name-ractor' end @@ -689,7 +689,7 @@ def git(dir, *args) ruby_descriptions = { 'ruby' => 'ruby 3.3.0', 'exp' => 'ruby 3.3.0 exp' } sections = [ { - title: 'harness-gc', + title: 'gc', table: [['bench', 'ruby (ms)', 'exp (ms)'], ['gcbench', '100.0', '90.0']], format: ['%s', '%.1f', '%.1f'], failures: {}, @@ -702,7 +702,7 @@ def git(dir, *args) ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections ) - assert_includes result, "GC time ratios (harness-gc):\n" + assert_includes result, "GC time ratios (gc):\n" assert_includes result, '- GC time ratios are ruby/comparison;' end diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index 27ae3c46..9cebcfae 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -87,6 +87,7 @@ assert_equal ['micro'], suite.categories end + it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( categories: ['ractor'], @@ -114,7 +115,7 @@ categories: [], name_filters: [], out_path: @out_path, - harness: 'harness-ractor' + harness: 'ractor' ) assert_equal false, suite_auto.harness_explicit end @@ -134,12 +135,12 @@ categories: [], name_filters: [], out_path: @out_path, - harness: 'harness-ractor', + harness: 'ractor', harness_explicit: false ) assert_equal 'harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') - assert_equal 'harness-ractor', suite.send(:benchmark_harness_for, 'simple') + assert_equal 'ractor', suite.send(:benchmark_harness_for, 'simple') end it 'ignores default_harness when harness is explicit' do @@ -162,7 +163,7 @@ categories: ['ractor'], name_filters: [], out_path: @out_path, - harness: 'harness-ractor' + harness: 'ractor' ) assert_equal true, suite.send(:ractor_category_run?) @@ -357,6 +358,7 @@ def base_cmd_for(ruby_description) assert_includes bench_data, 'ractor_test' refute_includes bench_data, 'simple' assert_empty bench_failures + end it 'runs each Ractor count in its own process and merges the results' do diff --git a/test/benchmarks_test.rb b/test/benchmarks_test.rb index a5ca16d3..c249f737 100644 --- a/test/benchmarks_test.rb +++ b/test/benchmarks_test.rb @@ -31,7 +31,7 @@ benchmarks_yml.each do |name, metadata| category = metadata.fetch('category') do # Ractor scaling benchmarks are intentionally kept in their own section. - metadata['default_harness'] == 'harness-ractor' ? 'ractor-scaling' : 'other' + metadata['default_harness'] == 'ractor' ? 'ractor-scaling' : 'other' end benchmark_names_by_category[category] << name end diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index 8c76c51d..522b19c9 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -200,7 +200,7 @@ def run_workload(body, env: {}) 'MIN_BENCH_TIME' => '0', 'RESULT_JSON_PATH' => result_path ).merge(env) - stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness-ractor')}", script, chdir: ROOT) + stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness')}", "-rractor", script, chdir: ROOT) yield stdout, stderr, status, result_path end end @@ -465,7 +465,7 @@ def run_workload(body, env: {}) 'RESULT_JSON_PATH' => result_path ) script = File.join(ROOT, 'benchmarks', bench, 'benchmark.rb') - stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness-ractor')}", script, chdir: ROOT) + stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness')}", "-rractor", script, chdir: ROOT) assert status.success?, "#{bench} failed:\n#{stdout}\n#{stderr}" data = JSON.parse(File.read(result_path)) diff --git a/test/ractor_harness_test.rb b/test/ractor_harness_test.rb index cab55a44..d6c937c1 100644 --- a/test/ractor_harness_test.rb +++ b/test/ractor_harness_test.rb @@ -28,7 +28,7 @@ 'RESULT_JSON_PATH' => result_path } - stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness-ractor')}", script) + stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness')}", "-rractor", script) assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" data = JSON.parse(File.read(result_path)) From c0b0bc773c8bdf9dfe5182412d8f5cd12f3e4558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Fri, 21 Nov 2025 22:20:13 +0000 Subject: [PATCH 02/18] Keep the harness implementation separated from the harness framework --- README.md | 40 ++++++---- benchmarks.yml | 2 +- benchmarks/30k_ifelse.rb | 2 +- benchmarks/30k_methods.rb | 2 +- benchmarks/30k_variables.rb | 2 +- benchmarks/activerecord/benchmark.rb | 2 +- benchmarks/addressable/equality.rb | 2 +- benchmarks/addressable/getters.rb | 2 +- benchmarks/addressable/join.rb | 2 +- benchmarks/addressable/merge.rb | 2 +- benchmarks/addressable/new.rb | 2 +- benchmarks/addressable/normalize.rb | 2 +- benchmarks/addressable/parse.rb | 2 +- benchmarks/addressable/setters.rb | 2 +- benchmarks/addressable/to-s.rb | 2 +- benchmarks/attr_accessor.rb | 2 +- benchmarks/binarytrees/benchmark.rb | 2 +- benchmarks/block_methods.rb | 2 +- benchmarks/blurhash/benchmark.rb | 2 +- benchmarks/cfunc_itself.rb | 2 +- benchmarks/chunky-png/benchmark.rb | 2 +- benchmarks/erubi-rails/benchmark.rb | 2 +- benchmarks/erubi/benchmark.rb | 2 +- benchmarks/etanni/benchmark.rb | 2 +- benchmarks/fannkuchredux/benchmark.rb | 2 +- benchmarks/fib.rb | 2 +- benchmarks/fluentd/benchmark.rb | 2 +- benchmarks/gcbench.rb | 2 +- benchmarks/getivar-module.rb | 2 +- benchmarks/getivar.rb | 2 +- benchmarks/grape/benchmark.rb | 2 +- benchmarks/graphql-native/benchmark.rb | 2 +- benchmarks/graphql/benchmark.rb | 2 +- benchmarks/gvl_release_acquire/benchmark.rb | 2 +- benchmarks/hexapdf/benchmark.rb | 2 +- benchmarks/json_parse_float/benchmark.rb | 2 +- benchmarks/json_parse_string/benchmark.rb | 2 +- benchmarks/keyword_args.rb | 2 +- benchmarks/knucleotide-ractor/benchmark.rb | 2 +- benchmarks/knucleotide/benchmark.rb | 2 +- benchmarks/lee/benchmark.rb | 2 +- benchmarks/liquid-c/benchmark.rb | 2 +- benchmarks/liquid-compile/benchmark.rb | 2 +- benchmarks/liquid-il/benchmark.rb | 2 +- benchmarks/liquid-render/benchmark.rb | 2 +- benchmarks/lobsters/benchmark.rb | 2 +- benchmarks/loops-times.rb | 2 +- benchmarks/mail/benchmark.rb | 2 +- benchmarks/matmul.rb | 2 +- benchmarks/nbody/benchmark.rb | 2 +- benchmarks/nqueens.rb | 2 +- benchmarks/object-new-initialize.rb | 2 +- benchmarks/object-new-no-escape.rb | 2 +- benchmarks/object-new.rb | 2 +- benchmarks/optcarrot/benchmark.rb | 2 +- benchmarks/protoboeuf-encode/benchmark.rb | 2 +- benchmarks/protoboeuf/benchmark.rb | 2 +- benchmarks/psych-load/benchmark.rb | 2 +- benchmarks/rack/benchmark.rb | 2 +- benchmarks/ractor-dead-set/benchmark.rb | 2 +- benchmarks/ractor-idle-garbage/benchmark.rb | 2 +- benchmarks/ractor-msg-backlog/benchmark.rb | 2 +- benchmarks/railsbench/benchmark.rb | 2 +- benchmarks/respond_to.rb | 2 +- benchmarks/rubocop/benchmark.rb | 2 +- benchmarks/ruby-json/benchmark.rb | 2 +- benchmarks/ruby-lsp/benchmark.rb | 2 +- benchmarks/ruby-xor.rb | 2 +- benchmarks/rubyboy/benchmark.rb | 2 +- benchmarks/rubykon/benchmark.rb | 2 +- benchmarks/send_bmethod.rb | 2 +- benchmarks/send_cfunc_block.rb | 2 +- benchmarks/send_rubyfunc_block.rb | 2 +- benchmarks/send_rubyfunc_inline.rb | 2 +- benchmarks/sequel/benchmark.rb | 2 +- benchmarks/setivar.rb | 2 +- benchmarks/setivar_object.rb | 2 +- benchmarks/setivar_young.rb | 2 +- benchmarks/shipit/benchmark.rb | 2 +- benchmarks/splay.rb | 2 +- benchmarks/str_concat.rb | 2 +- benchmarks/string_malloc_pressure.rb | 2 +- benchmarks/struct-new-no-escape.rb | 2 +- benchmarks/structaref.rb | 2 +- benchmarks/structaset.rb | 2 +- benchmarks/sudoku.rb | 2 +- benchmarks/symbol-name-ractor/benchmark.rb | 2 +- benchmarks/throw.rb | 2 +- benchmarks/tinygql/benchmark.rb | 2 +- burn_in.rb | 4 +- harness/bips.rb | 2 +- harness/chain.rb | 4 +- harness/continuous.rb | 2 +- harness/{harness.rb => default.rb} | 2 +- harness/gc.rb | 2 +- harness/loader.rb | 20 ----- harness/mplr.rb | 2 +- harness/once.rb | 2 +- harness/perf.rb | 4 +- harness/ractor.rb | 2 +- harness/spinel.rb | 6 +- harness/stackprof.rb | 8 +- harness/stats.rb | 2 +- harness/vernier.rb | 8 +- harness/warmup.rb | 2 +- lib/argument_parser.rb | 2 +- lib/benchmark_suite.rb | 10 +-- harness/harness-common.rb => lib/harness.rb | 0 .../harness-extra.rb => lib/harness/extra.rb | 5 +- lib/harness/loader.rb | 26 +++++++ misc/gen_30k_variables.rb | 2 +- run_once.sh | 4 +- test/argument_parser_test.rb | 2 +- test/benchmark_runner_cli_test.rb | 12 +-- test/benchmark_runner_test.rb | 4 +- test/benchmark_suite_test.rb | 78 +++++++++---------- test/ractor_gc_harness_test.rb | 6 +- test/ractor_harness_test.rb | 4 +- 118 files changed, 228 insertions(+), 215 deletions(-) rename harness/{harness.rb => default.rb} (98%) delete mode 100644 harness/loader.rb rename harness/harness-common.rb => lib/harness.rb (100%) rename harness/harness-extra.rb => lib/harness/extra.rb (95%) create mode 100644 lib/harness/loader.rb diff --git a/README.md b/README.md index 1ea4b242..788c62ae 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Small set of benchmarks and scripts for the Ruby programming language. The benchmarks are found in the `benchmarks` directory. Individual Ruby files in `benchmarks` are microbenchmarks. Subdirectories under `benchmarks` are larger macrobenchmarks. Each benchmark relies on a harness found in -[./harness/harness.rb](harness/harness.rb). The harness controls the number of times a benchmark is +[./harness/default.rb](harness/default.rb). The harness controls the number of times a benchmark is run, and writes timing values into an output file. The `run_benchmarks.rb` script (optional) traverses the `benchmarks` directory and @@ -188,7 +188,7 @@ because no single process ran all counts. The text summary, the CSV output, and `misc/zjit_diff.rb` show one row for each count, with the RSS and JIT stats of that count's process. -When you run a benchmark directly with `-Iharness -rractor`, the harness runs +When you run a benchmark directly with `-r./harness/ractor`, the harness runs all counts in one process, one count after another. ### Ractor Scenario Benchmarks @@ -232,7 +232,7 @@ with multiple ractors, for GC work that reclaims ractor-local memory: ```bash # Every count in one process -ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb +ruby -r./harness/ractor benchmarks/ractor-dead-set/benchmark.rb # One process per count, comparing two Rubies, with Ractor GC metrics RUBY_BENCH_RACTORS=1,2,4,6,8 ./run_benchmarks.rb --category ractor --ractor-gc \ @@ -323,7 +323,7 @@ This file will then be passed to the underlying Ruby interpreter with You can find several test harnesses in the `harness/` directory: -* `harness` - the normal default harness, with duration controlled by warmup iterations and time/count limits +* `default` - the normal default harness, with duration controlled by warmup iterations and time/count limits * `bips` - a harness that measures iterations/second until stable * `continuous` - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches * `once` - a simplified harness that simply runs once @@ -337,17 +337,23 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, run a benchmark script directly with `-I` to add the harness directory to the load path, and `-r` to require the specific harness: +To use a specific harness, run a benchmark script directly with `-r` to require the harness: ``` -# Use default harness -ruby -Iharness benchmarks/railsbench/benchmark.rb +# Use default harness (loads harness/default.rb automatically) +ruby benchmarks/railsbench/benchmark.rb # Use the 'once' harness -ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb +ruby -r./harness/once benchmarks/railsbench/benchmark.rb # Use the 'perf' harness -ruby -Iharness -rperf benchmarks/railsbench/benchmark.rb +ruby -r./harness/perf benchmarks/railsbench/benchmark.rb + +# Use the 'stackprof' harness +ruby -r./harness/stackprof benchmarks/railsbench/benchmark.rb + +# Use the 'vernier' harness +ruby -r./harness/vernier benchmarks/railsbench/benchmark.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: @@ -393,10 +399,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb +perf record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb +PERF=record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. @@ -408,14 +414,14 @@ Linux that shows its profiles in the Firefox Profiler. Run the benchmark script directly under `samply record`: ```sh -samply record ruby -Iharness benchmarks/railsbench/benchmark.rb +samply record ruby benchmarks/railsbench/benchmark.rb # Name JIT-compiled methods. Both JITs write /tmp/perf-.map, which samply reads. -samply record ruby --yjit --yjit-perf -Iharness benchmarks/railsbench/benchmark.rb -samply record ruby --zjit --zjit-perf -Iharness benchmarks/railsbench/benchmark.rb +samply record ruby --yjit --yjit-perf benchmarks/railsbench/benchmark.rb +samply record ruby --zjit --zjit-perf benchmarks/railsbench/benchmark.rb # Save the profile without opening it, and open it later -samply record --save-only -o railsbench.json.gz ruby -Iharness benchmarks/railsbench/benchmark.rb +samply record --save-only -o railsbench.json.gz ruby benchmarks/railsbench/benchmark.rb samply load railsbench.json.gz ``` @@ -430,7 +436,7 @@ benchmark, also set `RUBY_BENCH_PROFILING=1` to keep the harness's retention GCs out of the profile. The worker Ractors show up as separate thread tracks. ```sh -RUBY_BENCH_RACTORS=4 RUBY_BENCH_PROFILING=1 samply record ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb +RUBY_BENCH_RACTORS=4 RUBY_BENCH_PROFILING=1 samply record ruby -r./harness/ractor benchmarks/ractor-dead-set/benchmark.rb ``` samply also records child processes, so `samply record ruby run_benchmarks.rb ...` @@ -483,7 +489,7 @@ fail before warmup. ./run_benchmarks.rb --category ractor --chruby=base::ruby-base --ractor-gc # One benchmark, run directly. --ractor-gc sets RUBY_BENCH_RACTOR_GC=1. -RUBY_BENCH_RACTOR_GC=1 ruby -Iharness -rractor benchmarks/ractor-dead-set/benchmark.rb +RUBY_BENCH_RACTOR_GC=1 ruby -r./harness/ractor benchmarks/ractor-dead-set/benchmark.rb ``` `run_benchmarks.rb` only runs Ractor harness benchmarks with `--category ractor` diff --git a/benchmarks.yml b/benchmarks.yml index c49fbcb2..aa848b9e 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -105,7 +105,7 @@ knucleotide-ractor: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors ractor: true ractor_only: true - default_harness: harness + default_harness: default lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: diff --git a/benchmarks/30k_ifelse.rb b/benchmarks/30k_ifelse.rb index ff9f32c3..a525fda4 100644 --- a/benchmarks/30k_ifelse.rb +++ b/benchmarks/30k_ifelse.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' if ENV["RUBY_BENCH_RACTOR_HARNESS"] eval_recv = Object.new eval_meth = :instance_eval diff --git a/benchmarks/30k_methods.rb b/benchmarks/30k_methods.rb index f3d75f61..4d7bdccd 100644 --- a/benchmarks/30k_methods.rb +++ b/benchmarks/30k_methods.rb @@ -121011,7 +121011,7 @@ def fun_l29_n999() end end -require_relative '../harness/loader' +require_relative '../lib/harness/loader' INTERNAL_ITRS = Integer(ENV.fetch("INTERNAL_ITRS", 200)) diff --git a/benchmarks/30k_variables.rb b/benchmarks/30k_variables.rb index 44e77de0..342171d2 100644 --- a/benchmarks/30k_variables.rb +++ b/benchmarks/30k_variables.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true # Generated by misc/gen_30k_variables.rb: 300 methods x 100 local variables = 30000 local variables -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def fun_l0_n0(x) v0 = x diff --git a/benchmarks/activerecord/benchmark.rb b/benchmarks/activerecord/benchmark.rb index e820d5ae..4f336daa 100644 --- a/benchmarks/activerecord/benchmark.rb +++ b/benchmarks/activerecord/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/equality.rb b/benchmarks/addressable/equality.rb index ca48dffe..95141423 100644 --- a/benchmarks/addressable/equality.rb +++ b/benchmarks/addressable/equality.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/getters.rb b/benchmarks/addressable/getters.rb index 301439f2..43d0d03a 100644 --- a/benchmarks/addressable/getters.rb +++ b/benchmarks/addressable/getters.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/join.rb b/benchmarks/addressable/join.rb index a4df7b39..4d600795 100644 --- a/benchmarks/addressable/join.rb +++ b/benchmarks/addressable/join.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/merge.rb b/benchmarks/addressable/merge.rb index 39c07f2d..20f33283 100644 --- a/benchmarks/addressable/merge.rb +++ b/benchmarks/addressable/merge.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/new.rb b/benchmarks/addressable/new.rb index 5fd9731e..ae5d44ae 100644 --- a/benchmarks/addressable/new.rb +++ b/benchmarks/addressable/new.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/normalize.rb b/benchmarks/addressable/normalize.rb index 121a8ad0..014c81fa 100644 --- a/benchmarks/addressable/normalize.rb +++ b/benchmarks/addressable/normalize.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/parse.rb b/benchmarks/addressable/parse.rb index 02d29a83..d95f1094 100644 --- a/benchmarks/addressable/parse.rb +++ b/benchmarks/addressable/parse.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/setters.rb b/benchmarks/addressable/setters.rb index 41ee88b3..3431bba3 100644 --- a/benchmarks/addressable/setters.rb +++ b/benchmarks/addressable/setters.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/to-s.rb b/benchmarks/addressable/to-s.rb index 355d931f..9f337ff7 100644 --- a/benchmarks/addressable/to-s.rb +++ b/benchmarks/addressable/to-s.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/attr_accessor.rb b/benchmarks/attr_accessor.rb index d5e397dc..1ab4ae7f 100644 --- a/benchmarks/attr_accessor.rb +++ b/benchmarks/attr_accessor.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass attr_accessor :levar diff --git a/benchmarks/binarytrees/benchmark.rb b/benchmarks/binarytrees/benchmark.rb index 4e54528d..ca460c07 100644 --- a/benchmarks/binarytrees/benchmark.rb +++ b/benchmarks/binarytrees/benchmark.rb @@ -22,7 +22,7 @@ def bottom_up_tree(depth) MAX_DEPTH = MIN_DEPTH + 2 if MIN_DEPTH + 2 > MAX_DEPTH STRETCH_DEPTH = MAX_DEPTH + 1 -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' run_benchmark(60) do max_depth = MAX_DEPTH diff --git a/benchmarks/block_methods.rb b/benchmarks/block_methods.rb index 3db42080..8fc910c7 100644 --- a/benchmarks/block_methods.rb +++ b/benchmarks/block_methods.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def array_each(array) array.each do |x| diff --git a/benchmarks/blurhash/benchmark.rb b/benchmarks/blurhash/benchmark.rb index 1f081f4b..327fc225 100644 --- a/benchmarks/blurhash/benchmark.rb +++ b/benchmarks/blurhash/benchmark.rb @@ -172,7 +172,7 @@ def blurHashForPixels(xComponents, yComponents, width, height, rgb, bytesPerRow) # :startdoc: end -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" FILE = File.join(__dir__, "test.bin") diff --git a/benchmarks/cfunc_itself.rb b/benchmarks/cfunc_itself.rb index 2c39137e..8560da66 100644 --- a/benchmarks/cfunc_itself.rb +++ b/benchmarks/cfunc_itself.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' run_benchmark(500) do # 500K calls diff --git a/benchmarks/chunky-png/benchmark.rb b/benchmarks/chunky-png/benchmark.rb index ef511893..29054d29 100644 --- a/benchmarks/chunky-png/benchmark.rb +++ b/benchmarks/chunky-png/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/erubi-rails/benchmark.rb b/benchmarks/erubi-rails/benchmark.rb index 38c22634..1817a0f8 100644 --- a/benchmarks/erubi-rails/benchmark.rb +++ b/benchmarks/erubi-rails/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/erubi/benchmark.rb b/benchmarks/erubi/benchmark.rb index 0a75f1bb..f62a2f3d 100644 --- a/benchmarks/erubi/benchmark.rb +++ b/benchmarks/erubi/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/etanni/benchmark.rb b/benchmarks/etanni/benchmark.rb index 77e86386..48c41f1a 100644 --- a/benchmarks/etanni/benchmark.rb +++ b/benchmarks/etanni/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ # This is an Etanni translation of the Erb template in the Erubi diff --git a/benchmarks/fannkuchredux/benchmark.rb b/benchmarks/fannkuchredux/benchmark.rb index e73dc0ee..2e44c7d1 100644 --- a/benchmarks/fannkuchredux/benchmark.rb +++ b/benchmarks/fannkuchredux/benchmark.rb @@ -56,7 +56,7 @@ def fannkuch(n) #n = (ARGV[0] || 1).to_i N = 9 # Benchmarks Game uses n = 12, but it's too slow -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' run_benchmark(10) do sum, flips = fannkuch(N) diff --git a/benchmarks/fib.rb b/benchmarks/fib.rb index 4e5b3428..02be31b8 100644 --- a/benchmarks/fib.rb +++ b/benchmarks/fib.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def fib(n) if n < 2 diff --git a/benchmarks/fluentd/benchmark.rb b/benchmarks/fluentd/benchmark.rb index c503a653..4e3f9a6d 100644 --- a/benchmarks/fluentd/benchmark.rb +++ b/benchmarks/fluentd/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/gcbench.rb b/benchmarks/gcbench.rb index be6399fc..582551a3 100644 --- a/benchmarks/gcbench.rb +++ b/benchmarks/gcbench.rb @@ -11,7 +11,7 @@ # Tree construction uses both top-down (populate — creates old-to-young pointers, # exercises write barriers) and bottom-up (make_tree — young-to-young only). -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class GCBench class Node diff --git a/benchmarks/getivar-module.rb b/benchmarks/getivar-module.rb index e99930a2..ee6e1d06 100644 --- a/benchmarks/getivar-module.rb +++ b/benchmarks/getivar-module.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass @v0 = 1 diff --git a/benchmarks/getivar.rb b/benchmarks/getivar.rb index 80acfae1..d77cc33d 100644 --- a/benchmarks/getivar.rb +++ b/benchmarks/getivar.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/grape/benchmark.rb b/benchmarks/grape/benchmark.rb index 17b513ae..c139b41b 100644 --- a/benchmarks/grape/benchmark.rb +++ b/benchmarks/grape/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/graphql-native/benchmark.rb b/benchmarks/graphql-native/benchmark.rb index d4fd7077..ff5858c9 100644 --- a/benchmarks/graphql-native/benchmark.rb +++ b/benchmarks/graphql-native/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/graphql/benchmark.rb b/benchmarks/graphql/benchmark.rb index 85b5da1f..26c1feb6 100644 --- a/benchmarks/graphql/benchmark.rb +++ b/benchmarks/graphql/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/gvl_release_acquire/benchmark.rb b/benchmarks/gvl_release_acquire/benchmark.rb index e735bfe2..b1b22863 100644 --- a/benchmarks/gvl_release_acquire/benchmark.rb +++ b/benchmarks/gvl_release_acquire/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" # Windows has no /dev/zero, so read the same zeros from a regular file kept in data/. ZERO_PATH = if File.exist?("/dev/zero") diff --git a/benchmarks/hexapdf/benchmark.rb b/benchmarks/hexapdf/benchmark.rb index bcd2cf90..147eb50f 100644 --- a/benchmarks/hexapdf/benchmark.rb +++ b/benchmarks/hexapdf/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/json_parse_float/benchmark.rb b/benchmarks/json_parse_float/benchmark.rb index ef81bc49..2aef033f 100644 --- a/benchmarks/json_parse_float/benchmark.rb +++ b/benchmarks/json_parse_float/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/json_parse_string/benchmark.rb b/benchmarks/json_parse_string/benchmark.rb index a579f805..71417cd3 100644 --- a/benchmarks/json_parse_string/benchmark.rb +++ b/benchmarks/json_parse_string/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/keyword_args.rb b/benchmarks/keyword_args.rb index a9b035f3..386ef894 100644 --- a/benchmarks/keyword_args.rb +++ b/benchmarks/keyword_args.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def add(left:, right:) left + right diff --git a/benchmarks/knucleotide-ractor/benchmark.rb b/benchmarks/knucleotide-ractor/benchmark.rb index 41d265d6..779ae3c6 100644 --- a/benchmarks/knucleotide-ractor/benchmark.rb +++ b/benchmarks/knucleotide-ractor/benchmark.rb @@ -6,7 +6,7 @@ Warning[:experimental] = false -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' def frequency(seq, length) frequencies = Hash.new(0) diff --git a/benchmarks/knucleotide/benchmark.rb b/benchmarks/knucleotide/benchmark.rb index 94bbb028..4ea79b37 100644 --- a/benchmarks/knucleotide/benchmark.rb +++ b/benchmarks/knucleotide/benchmark.rb @@ -4,7 +4,7 @@ # k-nucleotide benchmark - Fastest implementation # Based on Ruby #1 with byteslice optimization -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' def frequency(seq, length) frequencies = Hash.new(0) diff --git a/benchmarks/lee/benchmark.rb b/benchmarks/lee/benchmark.rb index 5d0475fc..5e6d3c84 100644 --- a/benchmarks/lee/benchmark.rb +++ b/benchmarks/lee/benchmark.rb @@ -84,7 +84,7 @@ def lay(depth, solution) end end -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-c/benchmark.rb b/benchmarks/liquid-c/benchmark.rb index e074e33c..4a649dff 100644 --- a/benchmarks/liquid-c/benchmark.rb +++ b/benchmarks/liquid-c/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-compile/benchmark.rb b/benchmarks/liquid-compile/benchmark.rb index 0e0d0f1c..956e0fb8 100644 --- a/benchmarks/liquid-compile/benchmark.rb +++ b/benchmarks/liquid-compile/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-il/benchmark.rb b/benchmarks/liquid-il/benchmark.rb index 144894ca..99dac298 100644 --- a/benchmarks/liquid-il/benchmark.rb +++ b/benchmarks/liquid-il/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-render/benchmark.rb b/benchmarks/liquid-render/benchmark.rb index 661015a8..b851474c 100644 --- a/benchmarks/liquid-render/benchmark.rb +++ b/benchmarks/liquid-render/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/lobsters/benchmark.rb b/benchmarks/lobsters/benchmark.rb index 3a28fe81..63c36100 100644 --- a/benchmarks/lobsters/benchmark.rb +++ b/benchmarks/lobsters/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' ENV['DISABLE_DATABASE_ENVIRONMENT_CHECK'] = '1' # Benchmarks don't really have 'production', so trash it at will. diff --git a/benchmarks/loops-times.rb b/benchmarks/loops-times.rb index 653fe989..22c196bd 100644 --- a/benchmarks/loops-times.rb +++ b/benchmarks/loops-times.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # Fix these values for determinism U = 5 diff --git a/benchmarks/mail/benchmark.rb b/benchmarks/mail/benchmark.rb index 0d625650..9bef261d 100644 --- a/benchmarks/mail/benchmark.rb +++ b/benchmarks/mail/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/matmul.rb b/benchmarks/matmul.rb index c4087dbb..80187ade 100644 --- a/benchmarks/matmul.rb +++ b/benchmarks/matmul.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def matgen(n) tmp = 1.0 / n / n diff --git a/benchmarks/nbody/benchmark.rb b/benchmarks/nbody/benchmark.rb index 737e298b..86a11a3c 100644 --- a/benchmarks/nbody/benchmark.rb +++ b/benchmarks/nbody/benchmark.rb @@ -5,7 +5,7 @@ # modified by Jesse Millikan # modified by Yusuke Endoh -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' SOLAR_MASS = 4 * Math::PI**2 DAYS_PER_YEAR = 365.24 diff --git a/benchmarks/nqueens.rb b/benchmarks/nqueens.rb index 9a1b7a8c..0912a64a 100644 --- a/benchmarks/nqueens.rb +++ b/benchmarks/nqueens.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def nq_solve(n) a = Array.new(n, -1) diff --git a/benchmarks/object-new-initialize.rb b/benchmarks/object-new-initialize.rb index bb90dbf1..b51881a7 100644 --- a/benchmarks/object-new-initialize.rb +++ b/benchmarks/object-new-initialize.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class C def initialize(a, b, c, d) diff --git a/benchmarks/object-new-no-escape.rb b/benchmarks/object-new-no-escape.rb index 5c38e03d..069784df 100644 --- a/benchmarks/object-new-no-escape.rb +++ b/benchmarks/object-new-no-escape.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class Point attr_reader :x, :y diff --git a/benchmarks/object-new.rb b/benchmarks/object-new.rb index b079c393..ed3b37b8 100644 --- a/benchmarks/object-new.rb +++ b/benchmarks/object-new.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' run_benchmark(100) do i = 0 diff --git a/benchmarks/optcarrot/benchmark.rb b/benchmarks/optcarrot/benchmark.rb index 3c48515f..29e6e58f 100755 --- a/benchmarks/optcarrot/benchmark.rb +++ b/benchmarks/optcarrot/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' require_relative "lib/optcarrot" if ENV["RUBY_BENCH_RACTOR_HARNESS"] diff --git a/benchmarks/protoboeuf-encode/benchmark.rb b/benchmarks/protoboeuf-encode/benchmark.rb index a7afcdb7..16551acc 100644 --- a/benchmarks/protoboeuf-encode/benchmark.rb +++ b/benchmarks/protoboeuf-encode/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' # Protoboeuf decoder require_relative 'benchmark_pb' diff --git a/benchmarks/protoboeuf/benchmark.rb b/benchmarks/protoboeuf/benchmark.rb index 6e3078f4..9f477b79 100644 --- a/benchmarks/protoboeuf/benchmark.rb +++ b/benchmarks/protoboeuf/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' # Protoboeuf decoder require_relative 'benchmark_pb' diff --git a/benchmarks/psych-load/benchmark.rb b/benchmarks/psych-load/benchmark.rb index 795135f3..08b99c64 100644 --- a/benchmarks/psych-load/benchmark.rb +++ b/benchmarks/psych-load/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/rack/benchmark.rb b/benchmarks/rack/benchmark.rb index a3fde656..57c25052 100644 --- a/benchmarks/rack/benchmark.rb +++ b/benchmarks/rack/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/ractor-dead-set/benchmark.rb b/benchmarks/ractor-dead-set/benchmark.rb index 4f48e50e..9a693b97 100644 --- a/benchmarks/ractor-dead-set/benchmark.rb +++ b/benchmarks/ractor-dead-set/benchmark.rb @@ -4,7 +4,7 @@ Warning[:experimental] = false -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" DEAD_SET_ITEMS = Integer(ENV.fetch("RACTOR_DEAD_SET_ITEMS", 100_000)) diff --git a/benchmarks/ractor-idle-garbage/benchmark.rb b/benchmarks/ractor-idle-garbage/benchmark.rb index 7c69366f..87b66d01 100644 --- a/benchmarks/ractor-idle-garbage/benchmark.rb +++ b/benchmarks/ractor-idle-garbage/benchmark.rb @@ -6,7 +6,7 @@ Warning[:experimental] = false -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" IDLE_GARBAGE_ITEMS = Integer(ENV.fetch("RACTOR_IDLE_GARBAGE_ITEMS", 100_000)) diff --git a/benchmarks/ractor-msg-backlog/benchmark.rb b/benchmarks/ractor-msg-backlog/benchmark.rb index 1648bbdc..fdca9adb 100644 --- a/benchmarks/ractor-msg-backlog/benchmark.rb +++ b/benchmarks/ractor-msg-backlog/benchmark.rb @@ -6,7 +6,7 @@ Warning[:experimental] = false -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" BACKLOG_MESSAGES = Integer(ENV.fetch("RACTOR_BACKLOG_MESSAGES", 20_000)) BACKLOG_GATE_SLEEP = Float(ENV.fetch("RACTOR_BACKLOG_GATE_SLEEP", 1.0)) diff --git a/benchmarks/railsbench/benchmark.rb b/benchmarks/railsbench/benchmark.rb index f89f91a3..965c70e3 100644 --- a/benchmarks/railsbench/benchmark.rb +++ b/benchmarks/railsbench/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' diff --git a/benchmarks/respond_to.rb b/benchmarks/respond_to.rb index 9a419060..55f49f70 100644 --- a/benchmarks/respond_to.rb +++ b/benchmarks/respond_to.rb @@ -1,7 +1,7 @@ # Microbenchmark to test the performance of respond_to? # This is one of the top most called methods in rack/railsbench -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class A def foo diff --git a/benchmarks/rubocop/benchmark.rb b/benchmarks/rubocop/benchmark.rb index 60b49d05..12f415a0 100644 --- a/benchmarks/rubocop/benchmark.rb +++ b/benchmarks/rubocop/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/ruby-json/benchmark.rb b/benchmarks/ruby-json/benchmark.rb index 3c2d3e4e..f8c085b8 100644 --- a/benchmarks/ruby-json/benchmark.rb +++ b/benchmarks/ruby-json/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" require "json" require "strscan" diff --git a/benchmarks/ruby-lsp/benchmark.rb b/benchmarks/ruby-lsp/benchmark.rb index 232a0e8d..0bbc304b 100644 --- a/benchmarks/ruby-lsp/benchmark.rb +++ b/benchmarks/ruby-lsp/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/ruby-xor.rb b/benchmarks/ruby-xor.rb index 5352c76a..2a9f9d1b 100644 --- a/benchmarks/ruby-xor.rb +++ b/benchmarks/ruby-xor.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # XOR strings in place # Expected to behave like Xorcist.xor! from the xorcist gem diff --git a/benchmarks/rubyboy/benchmark.rb b/benchmarks/rubyboy/benchmark.rb index 5a086fc2..18afcbd9 100644 --- a/benchmarks/rubyboy/benchmark.rb +++ b/benchmarks/rubyboy/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/rubykon/benchmark.rb b/benchmarks/rubykon/benchmark.rb index e8f0e527..d312daaf 100644 --- a/benchmarks/rubykon/benchmark.rb +++ b/benchmarks/rubykon/benchmark.rb @@ -2,7 +2,7 @@ # https://github.com/benchmark-driver/sky2-bench/blob/master/benchmark/rubykon-benchmark.rb, # part of benchmark-driver's default benchmarking suite, by Takashi Kokubun. -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' require_relative 'lib/rubykon' diff --git a/benchmarks/send_bmethod.rb b/benchmarks/send_bmethod.rb index 649df891..07774ea0 100644 --- a/benchmarks/send_bmethod.rb +++ b/benchmarks/send_bmethod.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # Call with and without args define_method(:zero) { :b } diff --git a/benchmarks/send_cfunc_block.rb b/benchmarks/send_cfunc_block.rb index 2c7dc34f..1238b041 100644 --- a/benchmarks/send_cfunc_block.rb +++ b/benchmarks/send_cfunc_block.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' ARR = [] diff --git a/benchmarks/send_rubyfunc_block.rb b/benchmarks/send_rubyfunc_block.rb index ae3f3734..95db2e01 100644 --- a/benchmarks/send_rubyfunc_block.rb +++ b/benchmarks/send_rubyfunc_block.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class C CALL_BLOCK = false diff --git a/benchmarks/send_rubyfunc_inline.rb b/benchmarks/send_rubyfunc_inline.rb index 50f0efd8..230ba169 100644 --- a/benchmarks/send_rubyfunc_inline.rb +++ b/benchmarks/send_rubyfunc_inline.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' INNER_ITERATIONS = 10_000_000 EXPECTED_RESULT = 2 * INNER_ITERATIONS * INNER_ITERATIONS + 4 * INNER_ITERATIONS diff --git a/benchmarks/sequel/benchmark.rb b/benchmarks/sequel/benchmark.rb index 81b07c67..7c2491b4 100644 --- a/benchmarks/sequel/benchmark.rb +++ b/benchmarks/sequel/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" require "securerandom" # Provides `Random::Formatter` in Ruby 2.6+ Dir.chdir __dir__ diff --git a/benchmarks/setivar.rb b/benchmarks/setivar.rb index e64dca4d..9e9ebf3a 100644 --- a/benchmarks/setivar.rb +++ b/benchmarks/setivar.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/setivar_object.rb b/benchmarks/setivar_object.rb index 6f34cba6..0c316833 100644 --- a/benchmarks/setivar_object.rb +++ b/benchmarks/setivar_object.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/setivar_young.rb b/benchmarks/setivar_young.rb index 2f673756..d463d811 100644 --- a/benchmarks/setivar_young.rb +++ b/benchmarks/setivar_young.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/shipit/benchmark.rb b/benchmarks/shipit/benchmark.rb index d7b2ce7d..afe83a71 100644 --- a/benchmarks/shipit/benchmark.rb +++ b/benchmarks/shipit/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' ENV['DISABLE_DATABASE_ENVIRONMENT_CHECK'] = '1' # Benchmarks don't really have 'production', so trash it at will. diff --git a/benchmarks/splay.rb b/benchmarks/splay.rb index e38e0f84..f9903551 100644 --- a/benchmarks/splay.rb +++ b/benchmarks/splay.rb @@ -29,7 +29,7 @@ # Ported to Ruby from the V8/WebKit JavaScript benchmark suite. # https://browserbench.org/JetStream2.1/Octane/splay.js -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class SplayTree class Node diff --git a/benchmarks/str_concat.rb b/benchmarks/str_concat.rb index ee514cc4..9bbeaf2b 100644 --- a/benchmarks/str_concat.rb +++ b/benchmarks/str_concat.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative '../harness/loader' +require_relative '../lib/harness/loader' NUM_ITERS = 10 * 1024 TEST_STR = 'sssssséé'.freeze diff --git a/benchmarks/string_malloc_pressure.rb b/benchmarks/string_malloc_pressure.rb index ebda6860..39026d22 100644 --- a/benchmarks/string_malloc_pressure.rb +++ b/benchmarks/string_malloc_pressure.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def build_string_malloc_pressure live = [] diff --git a/benchmarks/struct-new-no-escape.rb b/benchmarks/struct-new-no-escape.rb index 1a45269f..18751bde 100644 --- a/benchmarks/struct-new-no-escape.rb +++ b/benchmarks/struct-new-no-escape.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' Point = Struct.new(:x, :y) diff --git a/benchmarks/structaref.rb b/benchmarks/structaref.rb index 95bb3575..b319092e 100644 --- a/benchmarks/structaref.rb +++ b/benchmarks/structaref.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' TheClass = Struct.new(:v0, :v1, :v2, :levar) diff --git a/benchmarks/structaset.rb b/benchmarks/structaset.rb index 5d73035e..bcff9b0e 100644 --- a/benchmarks/structaset.rb +++ b/benchmarks/structaset.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' TheClass = Struct.new(:v0, :v1, :v2, :levar) diff --git a/benchmarks/sudoku.rb b/benchmarks/sudoku.rb index 62308b36..e8a298e0 100644 --- a/benchmarks/sudoku.rb +++ b/benchmarks/sudoku.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def sd_genmat mr = Array.new(324) { [] } diff --git a/benchmarks/symbol-name-ractor/benchmark.rb b/benchmarks/symbol-name-ractor/benchmark.rb index 3d60356f..367e71e2 100644 --- a/benchmarks/symbol-name-ractor/benchmark.rb +++ b/benchmarks/symbol-name-ractor/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" SYMBOL = :ruby_bench_id2str_pressure_static_symbol EXPECTED_NAME = SYMBOL.name diff --git a/benchmarks/throw.rb b/benchmarks/throw.rb index 3d3a7b2b..e3aa8b9c 100644 --- a/benchmarks/throw.rb +++ b/benchmarks/throw.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def foo yield diff --git a/benchmarks/tinygql/benchmark.rb b/benchmarks/tinygql/benchmark.rb index 7195a9ed..8457772d 100644 --- a/benchmarks/tinygql/benchmark.rb +++ b/benchmarks/tinygql/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/burn_in.rb b/burn_in.rb index 0f29a114..f205c203 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -62,7 +62,7 @@ def free_file_path(parent_dir, name_prefix) def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) bench_dir = 'benchmarks' entry = metadata[bench_name] || {} - harness = entry.fetch('default_harness', 'harness') + harness = entry.fetch('default_harness', 'default') script_path = File.join(bench_dir, bench_name, 'benchmark.rb') if not File.exist?(script_path) @@ -105,7 +105,7 @@ def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metada 'ruby', *test_options, "-Iharness", - *(harness == 'harness' ? [] : ["-r#{harness}"]), + "-r#{harness}", script_path, ].compact cmd_str = cmd.shelljoin diff --git a/harness/bips.rb b/harness/bips.rb index b583fa0f..dddb546e 100644 --- a/harness/bips.rb +++ b/harness/bips.rb @@ -1,5 +1,5 @@ require 'benchmark/ips' -require_relative "harness-common" +require_relative "../lib/harness" puts RUBY_DESCRIPTION diff --git a/harness/chain.rb b/harness/chain.rb index eef9d77b..62d50712 100644 --- a/harness/chain.rb +++ b/harness/chain.rb @@ -1,4 +1,4 @@ -require_relative 'harness-common' +require_relative '../lib/harness' # Ex: HARNESS_CHAIN="vernier,ractor" # Wraps the ractor harness in ther vernier harness @@ -10,7 +10,7 @@ end if CHAIN.include?("vernier") && CHAIN.last != "vernier" - require_relative "harness-extra" + require_relative "../lib/harness/extra" def run_enough_to_profile(n, **kwargs, &block) block.call end diff --git a/harness/continuous.rb b/harness/continuous.rb index f76f535b..05a9162f 100644 --- a/harness/continuous.rb +++ b/harness/continuous.rb @@ -1,4 +1,4 @@ -require_relative "harness-common" +require_relative "../lib/harness" puts RUBY_DESCRIPTION diff --git a/harness/harness.rb b/harness/default.rb similarity index 98% rename from harness/harness.rb rename to harness/default.rb index f6bd6099..1d4aae93 100644 --- a/harness/harness.rb +++ b/harness/default.rb @@ -1,4 +1,4 @@ -require_relative "./harness-common" +require_relative "../lib/harness" # Warmup iterations WARMUP_ITRS = Integer(ENV.fetch('WARMUP_ITRS', 15)) diff --git a/harness/gc.rb b/harness/gc.rb index 18ccb863..4c93fc69 100644 --- a/harness/gc.rb +++ b/harness/gc.rb @@ -1,4 +1,4 @@ -require_relative "harness-common" +require_relative "../lib/harness" require_relative "../lib/gc_stats" WARMUP_ITRS = Integer(ENV.fetch('WARMUP_ITRS', 15)) diff --git a/harness/loader.rb b/harness/loader.rb deleted file mode 100644 index ece20ea1..00000000 --- a/harness/loader.rb +++ /dev/null @@ -1,20 +0,0 @@ -# Use harness/harness.rb by default. You can change it with -I and -r options. -# Examples: -# ruby -Iharness benchmarks/railsbench/benchmark.rb # uses harness/harness.rb -# ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb # uses harness/once.rb -# ruby -Iharness -rractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb -if RUBY_ENGINE == "spinel" - require_relative "spinel" -else - retries = 0 - begin - require "harness" - rescue LoadError => e - if retries == 0 && e.path == "harness" - retries += 1 - $LOAD_PATH << File.expand_path(__dir__) - retry - end - raise - end -end diff --git a/harness/mplr.rb b/harness/mplr.rb index 4473a204..f91bfd94 100644 --- a/harness/mplr.rb +++ b/harness/mplr.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative "harness-common" +require_relative "../lib/harness" # Minimum number of benchmarking iterations MAX_BENCH_ITRS = Integer(ENV.fetch('MAX_BENCH_ITRS', 1000)) diff --git a/harness/once.rb b/harness/once.rb index dbf44e60..29b3340a 100644 --- a/harness/once.rb +++ b/harness/once.rb @@ -5,7 +5,7 @@ # Intended only for checking whether the benchmark can set itself up properly # and can run to completion. -require_relative 'harness-common' +require_relative '../lib/harness' def run_benchmark(_hint, **) yield diff --git a/harness/perf.rb b/harness/perf.rb index a778526e..afc339ef 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,14 +3,14 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -Iharness -rperf benchmarks/fib.rb +# $ PERF='record -e cycles' ruby -r./harness/perf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race # condition is possible where the benchmark finishes before the perf(1) # subprocess has a chance to attach, in which case perf outputs no profile. -require_relative "harness-common" +require_relative "../lib/harness" # Run $WARMUP_ITRS or 10 iterations of a given block. Then run $MIN_BENCH_ITRS # or `num_itrs_int` iterations of the block, attaching a perf command to the diff --git a/harness/ractor.rb b/harness/ractor.rb index 6af5b55a..a74bc6a4 100644 --- a/harness/ractor.rb +++ b/harness/ractor.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative 'harness-common' +require_relative '../lib/harness' require 'etc' Warning[:experimental] = false diff --git a/harness/spinel.rb b/harness/spinel.rb index f015b0c4..3abb72c5 100644 --- a/harness/spinel.rb +++ b/harness/spinel.rb @@ -1,5 +1,5 @@ # A harness for Ruby implementations that compile the whole program ahead of -# time, such as Spinel (https://github.com/matz/spinel). harness/harness.rb +# time, such as Spinel (https://github.com/matz/spinel). harness/default.rb # relies on CRuby-only libraries and reflection (RubyVM, RbConfig, Fiddle, # Bundler, $LOAD_PATH) that a whole-program compiler refuses to compile, so # this file reimplements the same protocol in a portable subset of Ruby: @@ -7,9 +7,9 @@ # * WARMUP_ITRS, MIN_BENCH_ITRS and MIN_BENCH_TIME select the iteration count. # * RESULT_JSON_PATH receives the JSON that run_benchmarks.rb reads. # -# harness/loader.rb selects this harness automatically when RUBY_ENGINE is +# lib/harness/loader.rb selects this harness automatically when RUBY_ENGINE is # "spinel", so `spinel -E benchmarks/fib.rb` works without any -I option. It -# also runs on CRuby (`ruby -Iharness -rspinel benchmarks/fib.rb`), which is how +# also runs on CRuby (`ruby -r./harness/spinel benchmarks/fib.rb`), which is how # its output is kept compatible with the default harness. require "json" diff --git a/harness/stackprof.rb b/harness/stackprof.rb index c9bf7a81..d11860a6 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb -require_relative "harness-common" -require_relative "harness-extra" +require_relative "../lib/harness" +require_relative "../lib/harness/extra" ensure_global_gem("stackprof") diff --git a/harness/stats.rb b/harness/stats.rb index b9b216ae..2971c667 100644 --- a/harness/stats.rb +++ b/harness/stats.rb @@ -1,4 +1,4 @@ -require_relative 'harness' +require_relative 'default' # Using Module#prepend to enable TracePoint right before #run_benchmark # while also reusing the original implementation. diff --git a/harness/vernier.rb b/harness/vernier.rb index 2b7c93ff..747ce223 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... +# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... +# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... -require_relative "harness-common" -require_relative "harness-extra" +require_relative "../lib/harness" +require_relative "../lib/harness/extra" ensure_global_gem("vernier") ensure_global_gem_exe("profile-viewer") diff --git a/harness/warmup.rb b/harness/warmup.rb index 83b1848b..84321234 100644 --- a/harness/warmup.rb +++ b/harness/warmup.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative 'harness-common' +require_relative '../lib/harness' require_relative '../misc/stats' MIN_ITERS = Integer(ENV['MIN_ITERS'] || 10) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 665e22a0..2d852d60 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -247,7 +247,7 @@ def default_args executables: {}, out_path: File.expand_path("../data", __dir__), out_override: nil, - harness: "harness", + harness: "default", harness_explicit: false, yjit_opts: "", zjit_opts: "", diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 27613d1f..7af54347 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -206,17 +206,17 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command - # If harness is 'harness', use default (no -r needed) - # Otherwise use -r to load the specific harness file - harness_args = if benchmark_harness == "harness" + # If harness is 'default', use default (no -r needed) + # Otherwise use -r to load the specific harness file with full path + harness_args = if benchmark_harness == "default" [] else - ["-r", benchmark_harness] + harness_path = File.join(HARNESS_DIR, benchmark_harness) + ["-r", harness_path] end cmd = (cmd_prefix + [ *ruby, - "-I", HARNESS_DIR, *harness_args, *pre_init, script_path, diff --git a/harness/harness-common.rb b/lib/harness.rb similarity index 100% rename from harness/harness-common.rb rename to lib/harness.rb diff --git a/harness/harness-extra.rb b/lib/harness/extra.rb similarity index 95% rename from harness/harness-extra.rb rename to lib/harness/extra.rb index 18a3654f..99f3cbf4 100644 --- a/harness/harness-extra.rb +++ b/lib/harness/extra.rb @@ -33,11 +33,12 @@ def benchmark_name # Get name of harness (stackprof, vernier, etc) from the file path of the loaded harness. def harness_name $LOADED_FEATURES.reverse_each do |feat| - if m = feat.match(%r{/harness-([^/]+)/harness\.rb$}) + if m = feat.match(%r{/harness/([^/]+)\.rb$}) return m[1] end end - raise "Unable to determine harness name" + # Default to 'harness' if we can't determine the name + 'harness' end # Share a single timestamp for everything from this execution. diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb new file mode 100644 index 00000000..329c6dca --- /dev/null +++ b/lib/harness/loader.rb @@ -0,0 +1,26 @@ +# Use harness/default.rb by default. You can change it with -r option. +# The -r option should be specified BEFORE requiring loader. +# Examples: +# ruby benchmarks/railsbench/benchmark.rb # uses harness/default.rb +# ruby -r./harness/once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ruby -r./harness/ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb + +if RUBY_ENGINE == "spinel" + require_relative "../../harness/spinel" +else + # Only load the default harness if no other harness has defined run_benchmark + unless defined?(run_benchmark) + retries = 0 + begin + require "default" + rescue LoadError => e + if retries == 0 && e.path == "default" + retries += 1 + # Add the harness directory to the load path + $LOAD_PATH << File.expand_path("../../harness", __dir__) + retry + end + raise + end + end +end diff --git a/misc/gen_30k_variables.rb b/misc/gen_30k_variables.rb index a55db4ad..df24c92e 100644 --- a/misc/gen_30k_variables.rb +++ b/misc/gen_30k_variables.rb @@ -22,7 +22,7 @@ out = +"" out << "# frozen_string_literal: true\n" out << "# Generated by misc/gen_30k_variables.rb: #{num_methods} methods x #{vars_per_method} local variables = #{num_methods * vars_per_method} local variables\n" -out << "require_relative '../harness/loader'\n\n" +out << "require_relative '../lib/harness/loader'\n\n" num_methods.times do |m| out << "def fun_l0_n#{m}(x)\n" diff --git a/run_once.sh b/run_once.sh index a5240fce..bef1d8bd 100755 --- a/run_once.sh +++ b/run_once.sh @@ -10,9 +10,9 @@ HARNESS_ARGS=() for arg in "$@"; do if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS_ARGS=("-r" "ractor") + HARNESS_ARGS=("-r" "./harness/ractor") break fi done -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"./harness" "${HARNESS_ARGS[@]}" "$@" \ No newline at end of file +WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby "${HARNESS_ARGS[@]}" "$@" diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 5bf5ef1e..57670a87 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -44,7 +44,7 @@ def setup_mock_ruby(path) assert_equal({ 'ruby' => [mock_ruby] }, args.executables) assert_equal File.expand_path("./data"), args.out_path assert_nil args.out_override - assert_equal "harness", args.harness + assert_equal "default", args.harness assert_equal "", args.yjit_opts assert_equal [], args.categories assert_equal [], args.name_filters diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 5cb0ad13..ef039e3c 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -44,7 +44,7 @@ def create_args(overrides = {}) executables: executables, out_path: nil, out_override: nil, - harness: 'harness', + harness: 'default', yjit_opts: '', categories: [], name_filters: [], @@ -107,13 +107,13 @@ def create_args(overrides = {}) } } bench_harnesses = { - 'fib' => 'harness', + 'fib' => 'default', 'symbol-name-ractor' => 'ractor' } sections = cli.send(:build_output_sections, ['ruby'], bench_data, bench_harnesses, {}) - assert_equal ['ractor', 'harness'], sections.map { |section| section[:title] } + assert_equal ['ractor', 'default'], sections.map { |section| section[:title] } assert_equal ['bench', 'ractors', 'ruby (ms)'], sections[0][:table][0] assert_equal ['symbol-name-ractor', '0', '1000.0 ± 0.0%'], sections[0][:table][1] assert_equal ['', '2', '2000.0 ± 0.0%'], sections[0][:table][2] @@ -153,11 +153,11 @@ def create_args(overrides = {}) } } } - bench_harnesses = { 'fib' => 'harness', 'object-new' => 'ractor' } + bench_harnesses = { 'fib' => 'default', 'object-new' => 'ractor' } sections = cli.send(:build_output_sections, ['ruby'], bench_data, bench_harnesses, {}) ractor = sections.find { |section| section[:title] == 'ractor' } - normal = sections.find { |section| section[:title] == 'harness' } + normal = sections.find { |section| section[:title] == 'default' } assert_equal 'ractor-local-workload', ractor[:gc_scope] refute normal.key?(:gc_scope) @@ -283,7 +283,7 @@ def create_args(overrides = {}) 'ruby' => { 'fib' => { 'warmup' => [0.1], 'bench' => [0.1], 'rss' => 10 * 1024 * 1024 } } } - sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'fib' => 'harness' }, {}) + sections = cli.send(:build_output_sections, ['ruby'], bench_data, { 'fib' => 'default' }, {}) assert_nil sections.first[:memory_tables] output = BenchmarkRunner.build_output_text({ 'ruby' => 'ruby 4.1.0dev' }, nil, nil, {}, sections: sections) diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index ce7e5965..e7c66a94 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -660,7 +660,7 @@ def git(dir, *args) ruby_descriptions = { 'ruby' => 'ruby 3.3.0' } sections = [ { - title: 'harness', + title: 'default', table: [['bench', 'ruby (ms)'], ['fib', '100.0']], format: ['%s', '%.1f'], failures: {}, @@ -679,7 +679,7 @@ def git(dir, *args) ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections ) - assert_includes result, "harness:\n" + assert_includes result, "default:\n" assert_includes result, "ractor:\n" assert_includes result, 'fib' assert_includes result, 'symbol-name-ractor' diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index 9cebcfae..5d257908 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -51,13 +51,13 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal ['micro'], suite.categories assert_equal [], suite.name_filters assert_equal @out_path, suite.out_path - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_nil suite.pre_init assert_equal false, suite.no_pinning end @@ -67,7 +67,7 @@ categories: [], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -79,25 +79,24 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal 'benchmarks', suite.bench_dir - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_equal ['micro'], suite.categories end - it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( categories: ['ractor'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal 'benchmarks', suite.bench_dir - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_equal ['ractor'], suite.categories end @@ -125,7 +124,7 @@ before do @metadata_with_harness = { 'simple' => { 'category' => 'micro' }, - 'custom_harness_bench' => { 'category' => 'other', 'default_harness' => 'harness' } + 'custom_harness_bench' => { 'category' => 'other', 'default_harness' => 'default' } } File.write('benchmarks.yml', YAML.dump(@metadata_with_harness)) end @@ -139,7 +138,7 @@ harness_explicit: false ) - assert_equal 'harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'default', suite.send(:benchmark_harness_for, 'custom_harness_bench') assert_equal 'ractor', suite.send(:benchmark_harness_for, 'simple') end @@ -174,7 +173,7 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal false, suite.send(:ractor_category_run?) @@ -187,7 +186,7 @@ def base_cmd_for(ruby_description) categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) suite.stub :linux?, true do suite.stub :setarch_prefix, [] do @@ -213,7 +212,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -235,7 +234,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -258,7 +257,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -279,7 +278,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['failing'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -310,7 +309,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['subdir'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -345,7 +344,7 @@ def base_cmd_for(ruby_description) categories: ['ractor'], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -359,6 +358,7 @@ def base_cmd_for(ruby_description) refute_includes bench_data, 'simple' assert_empty bench_failures + end it 'runs each Ractor count in its own process and merges the results' do @@ -379,7 +379,7 @@ def base_cmd_for(ruby_description) custom_json_path = File.join(@out_path, 'nested', 'custom_result.json') ENV['RESULT_JSON_PATH'] = custom_json_path ENV['RUBY_BENCH_RACTORS'] = '0,2,1' - suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'default', no_pinning: true) bench_data, bench_failures = nil capture_io do @@ -410,7 +410,7 @@ def base_cmd_for(ruby_description) File.write('benchmarks.yml', YAML.dump('fails_at_two' => { 'category' => 'other', 'ractor' => true })) ENV['RUBY_BENCH_RACTORS'] = '0,2,4' - suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'default', no_pinning: true) bench_data, bench_failures = nil capture_io do @@ -432,7 +432,7 @@ def base_cmd_for(ruby_description) File.write(ENV['RESULT_JSON_PATH'], JSON.generate('warmup' => [], 'bench' => [count.to_f], 'rss' => 1)) RUBY File.write('benchmarks.yml', YAML.dump('scenario' => { 'category' => 'other', 'ractor' => true, 'ractor_scenario' => true })) - suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', no_pinning: true) + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'default', no_pinning: true) ENV['RUBY_BENCH_RACTORS'] = '0,1,2' bench_data, bench_failures = nil @@ -463,7 +463,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -485,7 +485,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -505,7 +505,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -522,7 +522,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: '/nonexistent/file.rb', no_pinning: true ) @@ -538,7 +538,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: @temp_dir, no_pinning: true ) @@ -552,7 +552,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -571,7 +571,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -592,7 +592,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -626,7 +626,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['bench_a'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -646,7 +646,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -661,7 +661,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -675,7 +675,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -690,7 +690,7 @@ def base_cmd_for(ruby_description) assert_includes result[:data], 'warmup' assert_includes result[:data], 'bench' assert_includes result[:data], 'rss' - assert_equal 'harness', result[:harness] + assert_equal 'default', result[:harness] assert_nil result[:failure] end @@ -701,7 +701,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['failing'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -714,7 +714,7 @@ def base_cmd_for(ruby_description) assert_equal 'failing', result[:name] assert_nil result[:data] assert_equal 1, result[:failure] - assert_equal 'harness', result[:harness] + assert_equal 'default', result[:harness] end it 'produces same data as run for the same benchmark' do @@ -722,7 +722,7 @@ def base_cmd_for(ruby_description) categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -749,7 +749,7 @@ def base_cmd_for(ruby_description) end def pin_cpu_seen_by_benchmark(ruby_description: 'ruby 3.2.0', **options) - suite = BenchmarkSuite.new(categories: [], name_filters: ['pin_cpu'], out_path: @out_path, harness: 'harness', **options) + suite = BenchmarkSuite.new(categories: [], name_filters: ['pin_cpu'], out_path: @out_path, harness: 'default', **options) result = nil suite.stub(:windows?, true) do suite.stub(:linux?, false) do @@ -802,7 +802,7 @@ def pin_cpu_seen_by_benchmark(ruby_description: 'ruby 3.2.0', **options) categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) diff --git a/test/ractor_gc_harness_test.rb b/test/ractor_gc_harness_test.rb index 522b19c9..f4df718c 100644 --- a/test/ractor_gc_harness_test.rb +++ b/test/ractor_gc_harness_test.rb @@ -190,7 +190,7 @@ def run_workload(body, env: {}) Dir.mktmpdir do |dir| result_path = File.join(dir, 'results.json') script = File.join(dir, 'workload.rb') - File.write(script, "Warning[:experimental] = false\nrequire #{File.join(ROOT, 'harness', 'loader').inspect}\n" + body) + File.write(script, "Warning[:experimental] = false\nrequire #{File.join(ROOT, 'lib', 'harness', 'loader').inspect}\n" + body) env = CLEAN_ENV.merge( 'RUBY_BENCH_RACTOR_GC' => '1', 'RUBY_BENCH_RACTORS' => '0,1,2', @@ -200,7 +200,7 @@ def run_workload(body, env: {}) 'MIN_BENCH_TIME' => '0', 'RESULT_JSON_PATH' => result_path ).merge(env) - stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness')}", "-rractor", script, chdir: ROOT) + stdout, stderr, status = Open3.capture3(env, @ruby, "-r#{File.join(ROOT, 'harness', 'ractor')}", script, chdir: ROOT) yield stdout, stderr, status, result_path end end @@ -465,7 +465,7 @@ def run_workload(body, env: {}) 'RESULT_JSON_PATH' => result_path ) script = File.join(ROOT, 'benchmarks', bench, 'benchmark.rb') - stdout, stderr, status = Open3.capture3(env, @ruby, "-I#{File.join(ROOT, 'harness')}", "-rractor", script, chdir: ROOT) + stdout, stderr, status = Open3.capture3(env, @ruby, "-r#{File.join(ROOT, 'harness', 'ractor')}", script, chdir: ROOT) assert status.success?, "#{bench} failed:\n#{stdout}\n#{stderr}" data = JSON.parse(File.read(result_path)) diff --git a/test/ractor_harness_test.rb b/test/ractor_harness_test.rb index d6c937c1..2c4568c0 100644 --- a/test/ractor_harness_test.rb +++ b/test/ractor_harness_test.rb @@ -13,7 +13,7 @@ Dir.mkdir(calls_dir) script = File.join(dir, 'workload.rb') File.write(script, <<~RUBY) - require #{File.join(root, 'harness', 'loader').inspect} + require #{File.join(root, 'lib', 'harness', 'loader').inspect} run_benchmark(1) do |count| File.open(File.join(#{calls_dir.inspect}, Ractor.current.object_id.to_s), 'a') { |f| f.puts(count) } end @@ -28,7 +28,7 @@ 'RESULT_JSON_PATH' => result_path } - stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-I#{File.join(root, 'harness')}", "-rractor", script) + stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, "-r#{File.join(root, 'harness', 'ractor')}", script) assert status.success?, "workload failed:\n#{stdout}\n#{stderr}" data = JSON.parse(File.read(result_path)) From e0139ba230b229a40769cc9ab7033cf6aaf54cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Mon, 24 Nov 2025 20:43:25 +0000 Subject: [PATCH 03/18] Improve the run_once script to better handle Ruby options and harness selection So we don't need to deal with the `-r` ruby argument directly for selecting harnesses. --- .github/workflows/test.yml | 4 +- README.md | 37 +++--- harness/perf.rb | 2 +- harness/stackprof.rb | 4 +- harness/vernier.rb | 4 +- lib/harness/loader.rb | 10 +- run_once.rb | 90 ++++++++++++++ run_once.sh | 18 --- test/run_once_test.rb | 249 +++++++++++++++++++++++++++++++++++++ 9 files changed, 372 insertions(+), 46 deletions(-) create mode 100755 run_once.rb delete mode 100755 run_once.sh create mode 100644 test/run_once_test.rb diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0960f70c..3f1f1cec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -152,8 +152,8 @@ jobs: with: ruby-version: head - - name: Test run_once.sh - run: ./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb + - name: Test run_once.rb + run: ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb all-tests: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 788c62ae..e2fa7c31 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,14 @@ To run one or more specific benchmarks and record the data: ### Running a single benchmark -This is the easiest way to run a single benchmark. -It requires no setup at all and assumes nothing about the Ruby you are benchmarking. -It's also convenient for profiling, debugging, etc, especially since all benchmarked code runs in that process. +The easiest way to run a single benchmark once is using the `run_once.rb` script: ``` -ruby benchmarks/some_benchmark.rb +./run_once.rb benchmarks/some_benchmark.rb ``` +This automatically sets the environment to run the benchmark once (no warmup iterations). + ### Benchmark organization Benchmarks can be organized in three ways: @@ -337,23 +337,29 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, run a benchmark script directly with `-r` to require the harness: +To use a specific harness, use the `run_once.rb` script: ``` -# Use default harness (loads harness/default.rb automatically) -ruby benchmarks/railsbench/benchmark.rb +# Use default harness +./run_once.rb benchmarks/railsbench/benchmark.rb # Use the 'once' harness -ruby -r./harness/once benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # Use the 'perf' harness -ruby -r./harness/perf benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb # Use the 'stackprof' harness -ruby -r./harness/stackprof benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=stackprof benchmarks/railsbench/benchmark.rb # Use the 'vernier' harness -ruby -r./harness/vernier benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=vernier benchmarks/railsbench/benchmark.rb + +# Pass Ruby options like --yjit (use -- separator) +./run_once.rb -- --yjit benchmarks/railsbench/benchmark.rb + +# Combine harness option with Ruby options +./run_once.rb --harness=default -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: @@ -384,12 +390,11 @@ You can also use `--warmup`, `--bench`, or `--once` to set these environment var ./run_benchmarks.rb railsbench --once ``` -There is also a handy script for running benchmarks just once using -`WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0`, for example +You can also use the `run_once.rb` script to run benchmarks just once, for example with the `--yjit-stats` command-line option: ``` -./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb +./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` ### Using perf @@ -399,10 +404,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb +perf record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb +PERF=record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/harness/perf.rb b/harness/perf.rb index afc339ef..086b70b6 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,7 +3,7 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -r./harness/perf benchmarks/fib.rb +# $ PERF='record -e cycles' ./run_once.rb --harness=perf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race diff --git a/harness/stackprof.rb b/harness/stackprof.rb index d11860a6..931d5122 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/harness/vernier.rb b/harness/vernier.rb index 747ce223..3accb74d 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... +# ./run_once.rb --harness=vernier benchmarks/... +# NO_VIEWER=1 ./run_once.rb --harness=vernier benchmarks/... require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index 329c6dca..ac3c7bd2 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -1,9 +1,9 @@ -# Use harness/default.rb by default. You can change it with -r option. -# The -r option should be specified BEFORE requiring loader. +# Use harness/default.rb by default. You can change it with the --harness option in run_once.rb +# or with -r option when calling Ruby directly. # Examples: -# ruby benchmarks/railsbench/benchmark.rb # uses harness/default.rb -# ruby -r./harness/once benchmarks/railsbench/benchmark.rb # uses harness/once.rb -# ruby -r./harness/ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb +# ./run_once.rb benchmarks/railsbench/benchmark.rb # uses harness/default.rb +# ./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ./run_once.rb --harness=ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb if RUBY_ENGINE == "spinel" require_relative "../../harness/spinel" diff --git a/run_once.rb b/run_once.rb new file mode 100755 index 00000000..9576fdd1 --- /dev/null +++ b/run_once.rb @@ -0,0 +1,90 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Script to run a single benchmark once +# Provides a clean interface for running benchmarks with different harnesses. +# Examples: +# ./run_once.rb benchmarks/railsbench/benchmark.rb +# ./run_once.rb --harness=once benchmarks/fib.rb +# ./run_once.rb --harness=stackprof benchmarks/fib.rb +# ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb +# ./run_once.rb --harness=default -- --yjit benchmarks/fib.rb + +require 'optparse' +require 'shellwords' + +# Parse options +harness = nil +ruby_args = [] +benchmark_file = nil + +parser = OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options] [--] [ruby-options] BENCHMARK_FILE" + + opts.on("--harness=HARNESS", "Harness to use (default: default, options: once, bips, perf, ractor, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |h| + harness = h + end + + opts.on("-h", "--help", "Show this help message") do + puts opts + exit + end +end + +# Parse our options - this will stop at '--' or first non-option argument +begin + parser.parse! +rescue OptionParser::InvalidOption => e + puts "Error: #{e.message}" + puts parser + exit 1 +end + +# After parsing, ARGV contains remaining args (Ruby options + benchmark file) +ARGV.each do |arg| + if arg.end_with?('.rb') && !benchmark_file + benchmark_file = arg + else + ruby_args << arg + end +end + +if !benchmark_file + puts "Error: No benchmark file specified" + puts parser + exit 1 +end + +unless File.exist?(benchmark_file) + puts "Error: Benchmark file not found: #{benchmark_file}" + exit 1 +end + +# Automatically detect ractor benchmarks +if !harness && benchmark_file.include?('benchmarks-ractor/') + harness = 'ractor' +end + +# Build the command +harness_dir = File.expand_path('harness', __dir__) +harness_args = if harness && harness != 'default' + harness_path = File.join(harness_dir, harness) + unless File.exist?("#{harness_path}.rb") + puts "Error: Harness not found: #{harness}" + puts "Available harnesses: #{Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.join(', ')}" + exit 1 + end + ['-r', harness_path] +else + [] +end + +# Set environment for running once +ENV['WARMUP_ITRS'] = '0' +ENV['MIN_BENCH_ITRS'] = '1' +ENV['MIN_BENCH_TIME'] = '0' + +# Build and execute the command +cmd = ['ruby', *ruby_args, *harness_args, benchmark_file] + +exec(*cmd) diff --git a/run_once.sh b/run_once.sh deleted file mode 100755 index bef1d8bd..00000000 --- a/run_once.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Script to run a single benchmark once -# You can pass --yjit-stats and other ruby arguments to this script. -# Automatically detects Ractor benchmarks and uses the appropriate harness. -# Examples: -# ./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb -# ./run_once.sh benchmarks-ractor/optcarrot/benchmark.rb - -# Detect if any argument contains benchmarks-ractor/ to determine harness -HARNESS_ARGS=() -for arg in "$@"; do - if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS_ARGS=("-r" "./harness/ractor") - break - fi -done - -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby "${HARNESS_ARGS[@]}" "$@" diff --git a/test/run_once_test.rb b/test/run_once_test.rb new file mode 100644 index 00000000..6e4acc20 --- /dev/null +++ b/test/run_once_test.rb @@ -0,0 +1,249 @@ +require_relative 'test_helper' +require 'open3' +require 'tmpdir' +require 'fileutils' + +describe 'run_once.rb' do + before do + @script_path = File.expand_path('../run_once.rb', __dir__) + @original_env = ENV.to_h + + # Create a temp directory with test benchmarks + @tmpdir = Dir.mktmpdir + @test_benchmark = File.join(@tmpdir, 'test_benchmark.rb') + File.write(@test_benchmark, <<~RUBY) + puts "Benchmark executed" + puts "WARMUP_ITRS=\#{ENV['WARMUP_ITRS']}" + puts "MIN_BENCH_ITRS=\#{ENV['MIN_BENCH_ITRS']}" + puts "MIN_BENCH_TIME=\#{ENV['MIN_BENCH_TIME']}" + RUBY + end + + after do + FileUtils.rm_rf(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir) + # Restore original environment + ENV.replace(@original_env) + end + + def run_script(*args) + # Run the script and capture output + Open3.capture3('ruby', @script_path, *args) + end + + describe 'basic execution' do + it 'executes a benchmark file' do + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success?, "Script should execute successfully. stderr: #{_stderr}" + assert_match(/Benchmark executed/, stdout) + end + + it 'sets environment variables for single iteration' do + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success? + assert_match(/WARMUP_ITRS=0/, stdout) + assert_match(/MIN_BENCH_ITRS=1/, stdout) + assert_match(/MIN_BENCH_TIME=0/, stdout) + end + + it 'shows error when no benchmark file specified' do + stdout, _stderr, status = run_script + + refute status.success? + assert_match(/No benchmark file specified/, stdout) + end + + it 'shows error when benchmark file does not exist' do + stdout, _stderr, status = run_script('/nonexistent/benchmark.rb') + + refute status.success? + assert_match(/Benchmark file not found/, stdout) + end + end + + describe '--harness option' do + it 'accepts default harness' do + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'loads custom harness when specified' do + # Create a test harness + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + test_harness = File.join(harness_dir, 'test_harness.rb') + + begin + File.write(test_harness, <<~RUBY) + puts "TEST HARNESS LOADED" + RUBY + + stdout, _stderr, status = run_script('--harness=test_harness', @test_benchmark) + + assert status.success? + assert_match(/TEST HARNESS LOADED/, stdout) + ensure + File.delete(test_harness) if File.exist?(test_harness) + end + end + + it 'shows error for non-existent harness' do + stdout, _stderr, status = run_script('--harness=nonexistent', @test_benchmark) + + refute status.success? + assert_match(/Harness not found/, stdout) + assert_match(/Available harnesses/, stdout) + end + end + + describe 'ractor benchmark detection' do + it 'automatically uses ractor harness for ractor benchmarks' do + # Create a ractor benchmark + ractor_dir = File.join(@tmpdir, 'benchmarks-ractor', 'test') + FileUtils.mkdir_p(ractor_dir) + ractor_benchmark = File.join(ractor_dir, 'benchmark.rb') + File.write(ractor_benchmark, 'puts "Ractor benchmark"') + + _stdout, _stderr, _status = run_script(ractor_benchmark) + + # The script will try to load ractor harness + # We just verify it detected the ractor path + assert_match(/benchmarks-ractor/, ractor_benchmark) + end + end + + describe 'Ruby options pass-through' do + it 'passes Ruby options after -- separator' do + # Create a benchmark that checks for a warning flag + warning_benchmark = File.join(@tmpdir, 'warning_test.rb') + File.write(warning_benchmark, <<~RUBY) + x = 1 + x = 2 + puts "Benchmark with warnings" + RUBY + + stdout, _stderr, status = run_script('--', '-W2', warning_benchmark) + + assert status.success? + assert_match(/Benchmark with warnings/, stdout) + end + + it 'passes YJIT options after -- separator' do + yjit_benchmark = File.join(@tmpdir, 'yjit_test.rb') + File.write(yjit_benchmark, <<~RUBY) + puts "YJIT enabled" if defined?(RubyVM::YJIT) + puts "Benchmark complete" + RUBY + + stdout, _stderr, status = run_script('--', '--yjit', yjit_benchmark) + + assert status.success? + assert_match(/Benchmark complete/, stdout) + end + end + + describe '--help option' do + it 'shows help message' do + stdout, _stderr, status = run_script('--help') + + assert status.success? + assert_match(/Usage:/, stdout) + assert_match(/--harness/, stdout) + end + + it 'shows help with -h' do + stdout, _stderr, status = run_script('-h') + + assert status.success? + assert_match(/Usage:/, stdout) + end + end + + describe 'argument parsing order' do + it 'handles harness option before benchmark' do + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'handles Ruby options after -- separator' do + stdout, _stderr, status = run_script('--', '-W0', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'handles mixed options with -- separator' do + stdout, _stderr, status = run_script('--harness=default', '--', '-W0', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + end + + describe 'script examples' do + it 'works with simple benchmark path' do + # Simulates: ./run_once.rb benchmarks/fib.rb + fib_benchmark = File.join(@tmpdir, 'fib.rb') + File.write(fib_benchmark, 'puts "Fibonacci benchmark"') + + stdout, _stderr, status = run_script(fib_benchmark) + + assert status.success? + assert_match(/Fibonacci benchmark/, stdout) + end + + it 'works with harness option' do + # Simulates: ./run_once.rb --harness=default benchmarks/fib.rb + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'works with Ruby options after -- separator' do + # Simulates: ./run_once.rb -- --yjit benchmarks/fib.rb + stdout, _stderr, status = run_script('--', '--yjit', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + end + + describe 'edge cases' do + it 'handles benchmark files with spaces in path' do + space_dir = File.join(@tmpdir, 'dir with spaces') + FileUtils.mkdir_p(space_dir) + space_benchmark = File.join(space_dir, 'benchmark.rb') + File.write(space_benchmark, 'puts "Space test"') + + stdout, _stderr, status = run_script(space_benchmark) + + assert status.success? + assert_match(/Space test/, stdout) + end + + it 'identifies first .rb file as benchmark' do + # Even if multiple .rb files mentioned (shouldn't happen), first one wins + other_file = File.join(@tmpdir, 'other.rb') + File.write(other_file, 'puts "Wrong file"') + + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + refute_match(/Wrong file/, stdout) + end + + it 'rejects invalid options gracefully' do + stdout, _stderr, status = run_script('--invalid-option', @test_benchmark) + + refute status.success? + assert_match(/invalid option|Error/, stdout) + end + end +end From f072f0700c045c53cb67783be42e7fdc3c01ed17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 27 Nov 2025 19:30:32 +0000 Subject: [PATCH 04/18] Allow using HARNESS environment variable to select harness This allow `ruby` script direct calls to still select the harness without using any wrapper script. --- README.md | 61 ++++++++++++++++++++------------------- harness/perf.rb | 2 +- harness/stackprof.rb | 4 +-- harness/vernier.rb | 4 +-- lib/harness/loader.rb | 23 +++++++++++---- test/run_once_test.rb | 66 +++++++++++++++++++++++++++++++++++-------- 6 files changed, 108 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index e2fa7c31..10454db9 100644 --- a/README.md +++ b/README.md @@ -93,13 +93,13 @@ To run one or more specific benchmarks and record the data: ### Running a single benchmark -The easiest way to run a single benchmark once is using the `run_once.rb` script: +This is the easiest way to run a single benchmark. +It requires no setup at all and assumes nothing about the Ruby you are benchmarking. +It's also convenient for profiling, debugging, etc, especially since all benchmarked code runs in that process. +```bash +ruby benchmarks/fib.rb ``` -./run_once.rb benchmarks/some_benchmark.rb -``` - -This automatically sets the environment to run the benchmark once (no warmup iterations). ### Benchmark organization @@ -337,34 +337,35 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, use the `run_once.rb` script: +### Selecting a harness -``` -# Use default harness -./run_once.rb benchmarks/railsbench/benchmark.rb +**Using the `HARNESS` environment variable** -# Use the 'once' harness -./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb +```bash +# Run with specific harness +HARNESS=perf ruby benchmarks/fib.rb +HARNESS=stackprof ruby benchmarks/railsbench/benchmark.rb +HARNESS=vernier ruby benchmarks/optcarrot/benchmark.rb -# Use the 'perf' harness -./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb +# Combine with Ruby options +HARNESS=perf ruby --yjit benchmarks/fib.rb +HARNESS=once ruby --yjit-stats benchmarks/railsbench/benchmark.rb +``` -# Use the 'stackprof' harness -./run_once.rb --harness=stackprof benchmarks/railsbench/benchmark.rb +**Alternative: Use `run_once.rb` with `--harness` option**: -# Use the 'vernier' harness -./run_once.rb --harness=vernier benchmarks/railsbench/benchmark.rb +```bash +./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=stackprof benchmarks/fib.rb -# Pass Ruby options like --yjit (use -- separator) +# With Ruby options (use -- separator) ./run_once.rb -- --yjit benchmarks/railsbench/benchmark.rb - -# Combine harness option with Ruby options -./run_once.rb --harness=default -- --yjit-stats benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=perf -- --yjit-stats benchmarks/fib.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: -``` +```bash ./run_benchmarks.rb --harness=once ./run_benchmarks.rb --harness=perf ``` @@ -390,10 +391,13 @@ You can also use `--warmup`, `--bench`, or `--once` to set these environment var ./run_benchmarks.rb railsbench --once ``` -You can also use the `run_once.rb` script to run benchmarks just once, for example -with the `--yjit-stats` command-line option: +You can also run a single benchmark directly with Ruby: -``` +```bash +# Run once with YJIT stats +ruby --yjit-stats benchmarks/railsbench/benchmark.rb + +# Or use run_once.rb script ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` @@ -402,13 +406,12 @@ with the `--yjit-stats` command-line option: There is also a harness to use Linux perf. By default, it only runs a fixed number of iterations. If `PERF` environment variable is present, it starts the perf subcommand after warmup. -```sh +```bash # Use `perf record` for both warmup and benchmark -perf record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb +HARNESS=perf perf record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb -``` +HARNESS=perf PERF=record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/harness/perf.rb b/harness/perf.rb index 086b70b6..e13978ea 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,7 +3,7 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ./run_once.rb --harness=perf benchmarks/fib.rb +# $ PERF='record -e cycles' HARNESS=perf ruby benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race diff --git a/harness/stackprof.rb b/harness/stackprof.rb index 931d5122..8a33524e 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' HARNESS=stackprof MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 HARNESS=stackprof MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/harness/vernier.rb b/harness/vernier.rb index 3accb74d..186da9de 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# ./run_once.rb --harness=vernier benchmarks/... -# NO_VIEWER=1 ./run_once.rb --harness=vernier benchmarks/... +# HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... +# NO_VIEWER=1 HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index ac3c7bd2..5d2d5be7 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -1,8 +1,11 @@ -# Use harness/default.rb by default. You can change it with the --harness option in run_once.rb -# or with -r option when calling Ruby directly. +# Use harness/default.rb by default. You can change it with: +# 1. HARNESS environment variable (preferred, keeps harness close to code) +# 2. --harness option in run_once.rb +# 3. -r option when calling Ruby directly +# # Examples: -# ./run_once.rb benchmarks/railsbench/benchmark.rb # uses harness/default.rb -# ./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# HARNESS=perf ruby benchmarks/railsbench/benchmark.rb # uses harness/perf.rb +# HARNESS=once ruby benchmarks/railsbench/benchmark.rb # uses harness/once.rb # ./run_once.rb --harness=ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb if RUBY_ENGINE == "spinel" @@ -10,16 +13,24 @@ else # Only load the default harness if no other harness has defined run_benchmark unless defined?(run_benchmark) + harness_name = ENV['HARNESS'] || 'default' + retries = 0 begin - require "default" + require harness_name rescue LoadError => e - if retries == 0 && e.path == "default" + if retries == 0 && e.path == harness_name retries += 1 # Add the harness directory to the load path $LOAD_PATH << File.expand_path("../../harness", __dir__) retry end + # Provide helpful error message for invalid harness + if e.path == harness_name + harness_dir = File.expand_path("../../harness", __dir__) + available = Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.sort + raise LoadError, "Harness '#{harness_name}' not found. Available harnesses: #{available.join(', ')}" + end raise end end diff --git a/test/run_once_test.rb b/test/run_once_test.rb index 6e4acc20..1d96bc31 100644 --- a/test/run_once_test.rb +++ b/test/run_once_test.rb @@ -8,10 +8,11 @@ @script_path = File.expand_path('../run_once.rb', __dir__) @original_env = ENV.to_h - # Create a temp directory with test benchmarks @tmpdir = Dir.mktmpdir @test_benchmark = File.join(@tmpdir, 'test_benchmark.rb') + loader_path = File.expand_path('../lib/harness/loader', __dir__) File.write(@test_benchmark, <<~RUBY) + require "#{loader_path}" puts "Benchmark executed" puts "WARMUP_ITRS=\#{ENV['WARMUP_ITRS']}" puts "MIN_BENCH_ITRS=\#{ENV['MIN_BENCH_ITRS']}" @@ -21,12 +22,10 @@ after do FileUtils.rm_rf(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir) - # Restore original environment ENV.replace(@original_env) end def run_script(*args) - # Run the script and capture output Open3.capture3('ruby', @script_path, *args) end @@ -71,7 +70,6 @@ def run_script(*args) end it 'loads custom harness when specified' do - # Create a test harness harness_dir = File.join(File.dirname(@script_path), 'harness') FileUtils.mkdir_p(harness_dir) test_harness = File.join(harness_dir, 'test_harness.rb') @@ -90,6 +88,26 @@ def run_script(*args) end end + it 'respects HARNESS environment variable when no --harness option provided' do + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + test_harness = File.join(harness_dir, 'test_harness.rb') + + begin + File.write(test_harness, <<~RUBY) + puts "TEST HARNESS LOADED" + RUBY + + env = { 'HARNESS' => 'test_harness' } + stdout, _stderr, status = Open3.capture3(env, 'ruby', @script_path, @test_benchmark) + + assert status.success? + assert_match(/TEST HARNESS LOADED/, stdout) + ensure + File.delete(test_harness) if File.exist?(test_harness) + end + end + it 'shows error for non-existent harness' do stdout, _stderr, status = run_script('--harness=nonexistent', @test_benchmark) @@ -101,7 +119,6 @@ def run_script(*args) describe 'ractor benchmark detection' do it 'automatically uses ractor harness for ractor benchmarks' do - # Create a ractor benchmark ractor_dir = File.join(@tmpdir, 'benchmarks-ractor', 'test') FileUtils.mkdir_p(ractor_dir) ractor_benchmark = File.join(ractor_dir, 'benchmark.rb') @@ -109,15 +126,12 @@ def run_script(*args) _stdout, _stderr, _status = run_script(ractor_benchmark) - # The script will try to load ractor harness - # We just verify it detected the ractor path assert_match(/benchmarks-ractor/, ractor_benchmark) end end describe 'Ruby options pass-through' do it 'passes Ruby options after -- separator' do - # Create a benchmark that checks for a warning flag warning_benchmark = File.join(@tmpdir, 'warning_test.rb') File.write(warning_benchmark, <<~RUBY) x = 1 @@ -187,7 +201,6 @@ def run_script(*args) describe 'script examples' do it 'works with simple benchmark path' do - # Simulates: ./run_once.rb benchmarks/fib.rb fib_benchmark = File.join(@tmpdir, 'fib.rb') File.write(fib_benchmark, 'puts "Fibonacci benchmark"') @@ -198,7 +211,6 @@ def run_script(*args) end it 'works with harness option' do - # Simulates: ./run_once.rb --harness=default benchmarks/fib.rb stdout, _stderr, status = run_script('--harness=default', @test_benchmark) assert status.success? @@ -206,7 +218,6 @@ def run_script(*args) end it 'works with Ruby options after -- separator' do - # Simulates: ./run_once.rb -- --yjit benchmarks/fib.rb stdout, _stderr, status = run_script('--', '--yjit', @test_benchmark) assert status.success? @@ -228,7 +239,6 @@ def run_script(*args) end it 'identifies first .rb file as benchmark' do - # Even if multiple .rb files mentioned (shouldn't happen), first one wins other_file = File.join(@tmpdir, 'other.rb') File.write(other_file, 'puts "Wrong file"') @@ -246,4 +256,36 @@ def run_script(*args) assert_match(/invalid option|Error/, stdout) end end + + describe 'direct ruby execution with HARNESS env var' do + it 'uses HARNESS environment variable when running benchmark directly' do + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + direct_harness = File.join(harness_dir, 'direct_test.rb') + + begin + File.write(direct_harness, <<~RUBY) + puts "DIRECT HARNESS RUNNING" + RUBY + + env = { 'HARNESS' => 'direct_test' } + stdout, _stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) + + assert status.success?, "Direct Ruby execution should work with HARNESS env var" + assert_match(/DIRECT HARNESS RUNNING/, stdout) + assert_match(/Benchmark executed/, stdout) + ensure + File.delete(direct_harness) if File.exist?(direct_harness) + end + end + + it 'shows helpful error for invalid HARNESS value' do + env = { 'HARNESS' => 'nonexistent_harness' } + _stdout, stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) + + refute status.success? + assert_match(/Harness 'nonexistent_harness' not found/, stderr) + assert_match(/Available harnesses/, stderr) + end + end end From ecd6717eb827889dbccff75a108a5a8c92a38dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 27 Nov 2025 19:45:13 +0000 Subject: [PATCH 05/18] Improve chain harness documentation and error handling --- README.md | 19 +++++++++++++++++++ harness/chain.rb | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 10454db9..b240fed5 100644 --- a/README.md +++ b/README.md @@ -372,6 +372,25 @@ When using `run_benchmarks.rb`, you can specify a harness with the `--harness` o There is also a robust but complex CI harness in [the yjit-metrics repo](https://github.com/Shopify/yjit-metrics). +### Chain harness + +The `chain` harness allows you to combine multiple harnesses together. This is useful when you want to run a benchmark through multiple analysis tools or measurement approaches in sequence. + +Use the `HARNESS_CHAIN` environment variable to specify which harnesses to chain (comma-separated, at least 2 required): + +```bash +# Chain the 'once' and 'default' harnesses (runs benchmark once, then with default iterations) +HARNESS=chain HARNESS_CHAIN="once,default" ruby benchmarks/fib.rb + +# Profile with vernier while using ractor harness +HARNESS=chain HARNESS_CHAIN="vernier,ractor" ruby benchmarks-ractor/some_benchmark.rb + +# Using run_once.rb +HARNESS_CHAIN="once,default" ./run_once.rb --harness=chain benchmarks/fib.rb +``` + +The harnesses are executed in the order specified, with each harness wrapping the previous one. + ### Iterations and duration With the default harness, the number of iterations and duration diff --git a/harness/chain.rb b/harness/chain.rb index 62d50712..20e1fa66 100644 --- a/harness/chain.rb +++ b/harness/chain.rb @@ -34,7 +34,8 @@ def run_benchmark(n, **kwargs, &block) $current_harness = h require_relative path rescue LoadError => e - if e.path == path + expected_path = File.expand_path(h, __dir__) + if e.path == expected_path $stderr.puts "Can't find harness #{h}.rb in harness/. Exiting." exit 1 end From 45e110e44b2cbfa9c0912ad6f0d37e6e5ae258ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Mon, 1 Dec 2025 17:38:24 +0000 Subject: [PATCH 06/18] Build harness arguments once during initialization --- lib/benchmark_suite.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 7af54347..f1834b66 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -206,18 +206,9 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command - # If harness is 'default', use default (no -r needed) - # Otherwise use -r to load the specific harness file with full path - harness_args = if benchmark_harness == "default" - [] - else - harness_path = File.join(HARNESS_DIR, benchmark_harness) - ["-r", harness_path] - end - cmd = (cmd_prefix + [ *ruby, - *harness_args, + *harness_args_for(benchmark_harness), *pre_init, script_path, ].compact).map(&:to_s) @@ -339,6 +330,17 @@ def setarch_prefix prefix end + # If harness is 'default', use default (no -r needed) + # Otherwise use -r to load the specific harness file with full path + def harness_args_for(benchmark_harness) + @harness_args ||= {} + @harness_args[benchmark_harness] ||= if benchmark_harness == "default" + [] + else + ["-r", File.join(HARNESS_DIR, benchmark_harness)] + end + end + # Resolve the pre_init file path into a form that can be required def expand_pre_init(path) path = Pathname.new(path) From 0920f782db6925ca57e8cb607b220a3502aa411c Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 14:57:07 +0100 Subject: [PATCH 07/18] Write profiler output to the top level data directory lib/harness/extra.rb moved from harness/ to lib/harness/ but still built its output path from ../data, so the stackprof and vernier harnesses wrote to lib/data, which does not exist. harness_name also matched any */harness/*.rb feature, so lib/harness/loader.rb won and vernier named its profile loader-*.json. In this commit we build the output path from ../../data and take the harness name only from files directly inside the top level harness directory. --- lib/harness/extra.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/harness/extra.rb b/lib/harness/extra.rb index 99f3cbf4..c3e526bb 100644 --- a/lib/harness/extra.rb +++ b/lib/harness/extra.rb @@ -32,10 +32,9 @@ def benchmark_name # Get name of harness (stackprof, vernier, etc) from the file path of the loaded harness. def harness_name + harness_dir = File.expand_path("../../harness", __dir__) $LOADED_FEATURES.reverse_each do |feat| - if m = feat.match(%r{/harness/([^/]+)\.rb$}) - return m[1] - end + return File.basename(feat, ".rb") if File.dirname(feat) == harness_dir end # Default to 'harness' if we can't determine the name 'harness' @@ -47,7 +46,7 @@ def harness_name # Create a consistent file path in the data directory # so that the data can be further analyzed. def output_file_path(prefix: harness_name, suffix: benchmark_name, ruby_info: ruby_version_info, timestamp: TIMESTAMP, ext: "bin") - File.expand_path("../data/#{prefix}-#{timestamp}-#{ruby_info}-#{suffix}.#{ext}", __dir__) + File.expand_path("../../data/#{prefix}-#{timestamp}-#{ruby_info}-#{suffix}.#{ext}", __dir__) end # Can we get the benchmark config name from somewhere? From 2cca989cc886d3bcae67215e91d0838da7881f81 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 14:57:07 +0100 Subject: [PATCH 08/18] Load the HARNESS harness by path, not by feature name The loader passed HARNESS to require as a bare feature name. When a gem has the same name, such as stackprof or vernier, require loaded the gem and run_benchmark was never defined. In this commit we require harness/.rb by its absolute path and keep the error that lists the available harnesses. --- lib/harness/loader.rb | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index 5d2d5be7..1d0b0a8d 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -14,24 +14,14 @@ # Only load the default harness if no other harness has defined run_benchmark unless defined?(run_benchmark) harness_name = ENV['HARNESS'] || 'default' + harness_dir = File.expand_path("../../harness", __dir__) + harness_path = File.join(harness_dir, "#{harness_name}.rb") - retries = 0 - begin - require harness_name - rescue LoadError => e - if retries == 0 && e.path == harness_name - retries += 1 - # Add the harness directory to the load path - $LOAD_PATH << File.expand_path("../../harness", __dir__) - retry - end - # Provide helpful error message for invalid harness - if e.path == harness_name - harness_dir = File.expand_path("../../harness", __dir__) - available = Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.sort - raise LoadError, "Harness '#{harness_name}' not found. Available harnesses: #{available.join(', ')}" - end - raise + unless File.file?(harness_path) + available = Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.sort + raise LoadError, "Harness '#{harness_name}' not found. Available harnesses: #{available.join(', ')}" end + + require harness_path end end From 4aa87be588e2843ade59a51015c32b20652de03f Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 15:40:38 +0100 Subject: [PATCH 09/18] Pin the selected harness in the benchmark environment The default harness was selected by passing no -r at all, which left the choice to lib/harness/loader.rb. The loader reads HARNESS first, so an exported HARNESS replaced --harness=default and default_harness: default in benchmarks.yml, while the result was still labelled default. In this commit we set HARNESS to the resolved harness in the environment of the benchmark process. The harness that BenchmarkSuite picks is now the harness that the benchmark loads, whatever the parent environment holds. --- lib/benchmark_suite.rb | 2 +- test/benchmark_suite_test.rb | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index f1834b66..f8ca472a 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -214,7 +214,7 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b ].compact).map(&:to_s) # Do the benchmarking - result = BenchmarkRunner.check_call(cmd, env: env, raise_error: false, quiet: quiet) + result = BenchmarkRunner.check_call(cmd, env: env.merge("HARNESS" => benchmark_harness), raise_error: false, quiet: quiet) result[:command] = cmd.shelljoin result ensure diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index 5d257908..a3ad79ef 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -740,6 +740,30 @@ def base_cmd_for(ruby_description) assert_equal run_data['simple'].keys.sort, single_result[:data].keys.sort end + it 'loads the selected harness when HARNESS is set in the environment' do + loader_path = File.expand_path('../lib/harness/loader', __dir__) + harness_dir = File.expand_path('../harness', __dir__) + File.write('benchmarks/harness_probe.rb', <<~RUBY) + require #{loader_path.inspect} + require 'json' + harnesses = $LOADED_FEATURES.filter_map { |f| File.basename(f, '.rb') if File.dirname(f) == #{harness_dir.inspect} } + File.write(ENV['RESULT_JSON_PATH'], JSON.generate('warmup' => [], 'bench' => [0.001], 'harnesses' => harnesses)) + RUBY + suite = BenchmarkSuite.new(categories: [], name_filters: ['harness_probe'], out_path: @out_path, harness: 'default', harness_explicit: true, no_pinning: true) + + result = nil + ENV['HARNESS'] = 'once' + begin + capture_io do + result = suite.run_benchmark(suite.benchmarks.first, ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + ensure + ENV.delete('HARNESS') + end + + assert_equal ['default'], result[:data]['harnesses'] + end + describe 'on Windows' do before do File.write('benchmarks/pin_cpu.rb', <<~RUBY) From 281457d2c5d63a5ea94c8f603c58c26d64441871 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 15:43:36 +0100 Subject: [PATCH 10/18] Load the pre-init file before the harness The harness was passed as -r ahead of the --with-pre-init library, so Ruby loaded a non-default harness first. The Ractor harness calls Ractor.make_shareable(self) at load time, which froze main before the pre-init file ran, and a pre-init that set a top-level instance variable raised FrozenError. The default harness still loaded after pre-init, from the benchmark, so the order also depended on the harness. In this commit we pass the pre-init library before the harness. The pre-init file now runs before every harness, as it did when the benchmark itself loaded the harness. --- lib/benchmark_suite.rb | 2 +- test/benchmark_suite_test.rb | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index f8ca472a..1a247748 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -208,8 +208,8 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b # Set up the benchmarking command cmd = (cmd_prefix + [ *ruby, - *harness_args_for(benchmark_harness), *pre_init, + *harness_args_for(benchmark_harness), script_path, ].compact).map(&:to_s) diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index a3ad79ef..3a203c2e 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -515,6 +515,28 @@ def base_cmd_for(ruby_description) assert_equal 'setup', suite.pre_init[3].to_s end + it 'runs pre_init before the Ractor harness freezes main' do + pre_init_file = File.join(@temp_dir, 'pre_init.rb') + File.write(pre_init_file, "@pre_init_ran = true\n") + File.write('benchmarks/ractor_pre_init.rb', <<~RUBY) + require 'json' + File.write(ENV['RESULT_JSON_PATH'], JSON.generate('warmup' => [], 'bench' => [1.0], 'pre_init_ran' => @pre_init_ran)) + RUBY + File.write('benchmarks.yml', YAML.dump('ractor_pre_init' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'ractor' })) + suite = BenchmarkSuite.new(categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'default', pre_init: pre_init_file, no_pinning: true) + + ENV['RUBY_BENCH_RACTORS'] = '1' + bench_data, bench_failures = nil + capture_io do + bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') + end + + assert_empty bench_failures + assert_equal true, bench_data['ractor_pre_init']['pre_init_ran'] + ensure + ENV.delete('RUBY_BENCH_RACTORS') + end + it 'exits when pre_init file does not exist' do output = capture_io do assert_raises(SystemExit) do From 1060a641e1c28378d80ffcba9035b5f300b8aa81 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 15:48:48 +0100 Subject: [PATCH 11/18] Map the old --harness=harness name to the default harness Before the harnesses moved to harness/.rb, --harness=harness selected the default harness. The prefix stripping turned harness- into but left harness unchanged, so every benchmark tried to load harness/harness.rb and failed with LoadError. The help text still named harness as the default. In this commit we map harness to default and change the help text to say default. --- lib/argument_parser.rb | 6 +++--- test/argument_parser_test.rb | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 2d852d60..18b45dc9 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -101,9 +101,9 @@ def parse(argv) args.skip_zjit = true end - opts.on("--harness=HARNESS_NAME", "which harness to use (default: harness, options: once, bips, perf, ractor, gc, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |v| - # Strip 'harness-' prefix if provided for backward compatibility - v = v.sub(/^harness-/, '') + opts.on("--harness=HARNESS_NAME", "which harness to use (default: default, options: once, bips, perf, ractor, gc, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |v| + # Accept the old directory names (harness, harness-) for backward compatibility + v = v == 'harness' ? 'default' : v.sub(/^harness-/, '') args.harness = v args.harness_explicit = true end diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 57670a87..5e732078 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -367,6 +367,13 @@ def setup_mock_ruby(path) assert_equal 'stats', args.harness end + + it 'maps the old default harness name to default' do + parser = ArgumentParser.new + args = parser.parse(['--harness=harness']) + + assert_equal 'default', args.harness + end end describe '--warmup option' do From 68c769c11523c83c6d9f270240257882c0ed318a Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 12 Oct 2026 15:49:49 +0100 Subject: [PATCH 12/18] Close the perf code block in the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b240fed5..f805d547 100644 --- a/README.md +++ b/README.md @@ -431,6 +431,7 @@ HARNESS=perf perf record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark HARNESS=perf PERF=record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb +``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. From 997e288f51499f6dcb0f581eb46398114be06d65 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 7 Oct 2026 17:05:39 +0100 Subject: [PATCH 13/18] Remove run_once.rb There are already two ways to run a benchmark: run_benchmarks.rb, and ruby on the benchmark file, where HARNESS or -r selects the harness and WARMUP_ITRS, MIN_BENCH_ITRS and MIN_BENCH_TIME set the iterations. run_once.rb added a third way that wrapped the second one in an exec. In this commit we remove run_once.rb and its tests. The CI job that ran it now runs railsbench once directly with ruby, and the README and lib/harness/loader.rb no longer refer to the script. The README example that runs a benchmark directly now sets WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0, the same values that run_benchmarks.rb --once sets. --- .github/workflows/test.yml | 8 +- README.md | 21 +-- lib/harness/loader.rb | 9 +- run_once.rb | 90 ------------ test/run_once_test.rb | 291 ------------------------------------- 5 files changed, 12 insertions(+), 407 deletions(-) delete mode 100755 run_once.rb delete mode 100644 test/run_once_test.rb diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f1f1cec..0c6e32d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -152,8 +152,12 @@ jobs: with: ruby-version: head - - name: Test run_once.rb - run: ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb + - name: Run a benchmark once directly with Ruby + run: ruby --yjit-stats benchmarks/railsbench/benchmark.rb + env: + WARMUP_ITRS: '0' + MIN_BENCH_ITRS: '1' + MIN_BENCH_TIME: '0' all-tests: runs-on: ubuntu-latest diff --git a/README.md b/README.md index f805d547..d2e1c30d 100644 --- a/README.md +++ b/README.md @@ -352,17 +352,6 @@ HARNESS=perf ruby --yjit benchmarks/fib.rb HARNESS=once ruby --yjit-stats benchmarks/railsbench/benchmark.rb ``` -**Alternative: Use `run_once.rb` with `--harness` option**: - -```bash -./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb -./run_once.rb --harness=stackprof benchmarks/fib.rb - -# With Ruby options (use -- separator) -./run_once.rb -- --yjit benchmarks/railsbench/benchmark.rb -./run_once.rb --harness=perf -- --yjit-stats benchmarks/fib.rb -``` - When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: ```bash @@ -384,9 +373,6 @@ HARNESS=chain HARNESS_CHAIN="once,default" ruby benchmarks/fib.rb # Profile with vernier while using ractor harness HARNESS=chain HARNESS_CHAIN="vernier,ractor" ruby benchmarks-ractor/some_benchmark.rb - -# Using run_once.rb -HARNESS_CHAIN="once,default" ./run_once.rb --harness=chain benchmarks/fib.rb ``` The harnesses are executed in the order specified, with each harness wrapping the previous one. @@ -410,14 +396,11 @@ You can also use `--warmup`, `--bench`, or `--once` to set these environment var ./run_benchmarks.rb railsbench --once ``` -You can also run a single benchmark directly with Ruby: +To run a single benchmark once directly with Ruby, set the same variables as `--once`: ```bash # Run once with YJIT stats -ruby --yjit-stats benchmarks/railsbench/benchmark.rb - -# Or use run_once.rb script -./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb +WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby --yjit-stats benchmarks/railsbench/benchmark.rb ``` ### Using perf diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index 1d0b0a8d..e4a5800e 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -1,12 +1,11 @@ # Use harness/default.rb by default. You can change it with: # 1. HARNESS environment variable (preferred, keeps harness close to code) -# 2. --harness option in run_once.rb -# 3. -r option when calling Ruby directly +# 2. -r option when calling Ruby directly # # Examples: -# HARNESS=perf ruby benchmarks/railsbench/benchmark.rb # uses harness/perf.rb -# HARNESS=once ruby benchmarks/railsbench/benchmark.rb # uses harness/once.rb -# ./run_once.rb --harness=ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb +# HARNESS=perf ruby benchmarks/railsbench/benchmark.rb # uses harness/perf.rb +# HARNESS=once ruby benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ruby -r./harness/ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb if RUBY_ENGINE == "spinel" require_relative "../../harness/spinel" diff --git a/run_once.rb b/run_once.rb deleted file mode 100755 index 9576fdd1..00000000 --- a/run_once.rb +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Script to run a single benchmark once -# Provides a clean interface for running benchmarks with different harnesses. -# Examples: -# ./run_once.rb benchmarks/railsbench/benchmark.rb -# ./run_once.rb --harness=once benchmarks/fib.rb -# ./run_once.rb --harness=stackprof benchmarks/fib.rb -# ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb -# ./run_once.rb --harness=default -- --yjit benchmarks/fib.rb - -require 'optparse' -require 'shellwords' - -# Parse options -harness = nil -ruby_args = [] -benchmark_file = nil - -parser = OptionParser.new do |opts| - opts.banner = "Usage: #{$0} [options] [--] [ruby-options] BENCHMARK_FILE" - - opts.on("--harness=HARNESS", "Harness to use (default: default, options: once, bips, perf, ractor, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |h| - harness = h - end - - opts.on("-h", "--help", "Show this help message") do - puts opts - exit - end -end - -# Parse our options - this will stop at '--' or first non-option argument -begin - parser.parse! -rescue OptionParser::InvalidOption => e - puts "Error: #{e.message}" - puts parser - exit 1 -end - -# After parsing, ARGV contains remaining args (Ruby options + benchmark file) -ARGV.each do |arg| - if arg.end_with?('.rb') && !benchmark_file - benchmark_file = arg - else - ruby_args << arg - end -end - -if !benchmark_file - puts "Error: No benchmark file specified" - puts parser - exit 1 -end - -unless File.exist?(benchmark_file) - puts "Error: Benchmark file not found: #{benchmark_file}" - exit 1 -end - -# Automatically detect ractor benchmarks -if !harness && benchmark_file.include?('benchmarks-ractor/') - harness = 'ractor' -end - -# Build the command -harness_dir = File.expand_path('harness', __dir__) -harness_args = if harness && harness != 'default' - harness_path = File.join(harness_dir, harness) - unless File.exist?("#{harness_path}.rb") - puts "Error: Harness not found: #{harness}" - puts "Available harnesses: #{Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.join(', ')}" - exit 1 - end - ['-r', harness_path] -else - [] -end - -# Set environment for running once -ENV['WARMUP_ITRS'] = '0' -ENV['MIN_BENCH_ITRS'] = '1' -ENV['MIN_BENCH_TIME'] = '0' - -# Build and execute the command -cmd = ['ruby', *ruby_args, *harness_args, benchmark_file] - -exec(*cmd) diff --git a/test/run_once_test.rb b/test/run_once_test.rb deleted file mode 100644 index 1d96bc31..00000000 --- a/test/run_once_test.rb +++ /dev/null @@ -1,291 +0,0 @@ -require_relative 'test_helper' -require 'open3' -require 'tmpdir' -require 'fileutils' - -describe 'run_once.rb' do - before do - @script_path = File.expand_path('../run_once.rb', __dir__) - @original_env = ENV.to_h - - @tmpdir = Dir.mktmpdir - @test_benchmark = File.join(@tmpdir, 'test_benchmark.rb') - loader_path = File.expand_path('../lib/harness/loader', __dir__) - File.write(@test_benchmark, <<~RUBY) - require "#{loader_path}" - puts "Benchmark executed" - puts "WARMUP_ITRS=\#{ENV['WARMUP_ITRS']}" - puts "MIN_BENCH_ITRS=\#{ENV['MIN_BENCH_ITRS']}" - puts "MIN_BENCH_TIME=\#{ENV['MIN_BENCH_TIME']}" - RUBY - end - - after do - FileUtils.rm_rf(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir) - ENV.replace(@original_env) - end - - def run_script(*args) - Open3.capture3('ruby', @script_path, *args) - end - - describe 'basic execution' do - it 'executes a benchmark file' do - stdout, _stderr, status = run_script(@test_benchmark) - - assert status.success?, "Script should execute successfully. stderr: #{_stderr}" - assert_match(/Benchmark executed/, stdout) - end - - it 'sets environment variables for single iteration' do - stdout, _stderr, status = run_script(@test_benchmark) - - assert status.success? - assert_match(/WARMUP_ITRS=0/, stdout) - assert_match(/MIN_BENCH_ITRS=1/, stdout) - assert_match(/MIN_BENCH_TIME=0/, stdout) - end - - it 'shows error when no benchmark file specified' do - stdout, _stderr, status = run_script - - refute status.success? - assert_match(/No benchmark file specified/, stdout) - end - - it 'shows error when benchmark file does not exist' do - stdout, _stderr, status = run_script('/nonexistent/benchmark.rb') - - refute status.success? - assert_match(/Benchmark file not found/, stdout) - end - end - - describe '--harness option' do - it 'accepts default harness' do - stdout, _stderr, status = run_script('--harness=default', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - - it 'loads custom harness when specified' do - harness_dir = File.join(File.dirname(@script_path), 'harness') - FileUtils.mkdir_p(harness_dir) - test_harness = File.join(harness_dir, 'test_harness.rb') - - begin - File.write(test_harness, <<~RUBY) - puts "TEST HARNESS LOADED" - RUBY - - stdout, _stderr, status = run_script('--harness=test_harness', @test_benchmark) - - assert status.success? - assert_match(/TEST HARNESS LOADED/, stdout) - ensure - File.delete(test_harness) if File.exist?(test_harness) - end - end - - it 'respects HARNESS environment variable when no --harness option provided' do - harness_dir = File.join(File.dirname(@script_path), 'harness') - FileUtils.mkdir_p(harness_dir) - test_harness = File.join(harness_dir, 'test_harness.rb') - - begin - File.write(test_harness, <<~RUBY) - puts "TEST HARNESS LOADED" - RUBY - - env = { 'HARNESS' => 'test_harness' } - stdout, _stderr, status = Open3.capture3(env, 'ruby', @script_path, @test_benchmark) - - assert status.success? - assert_match(/TEST HARNESS LOADED/, stdout) - ensure - File.delete(test_harness) if File.exist?(test_harness) - end - end - - it 'shows error for non-existent harness' do - stdout, _stderr, status = run_script('--harness=nonexistent', @test_benchmark) - - refute status.success? - assert_match(/Harness not found/, stdout) - assert_match(/Available harnesses/, stdout) - end - end - - describe 'ractor benchmark detection' do - it 'automatically uses ractor harness for ractor benchmarks' do - ractor_dir = File.join(@tmpdir, 'benchmarks-ractor', 'test') - FileUtils.mkdir_p(ractor_dir) - ractor_benchmark = File.join(ractor_dir, 'benchmark.rb') - File.write(ractor_benchmark, 'puts "Ractor benchmark"') - - _stdout, _stderr, _status = run_script(ractor_benchmark) - - assert_match(/benchmarks-ractor/, ractor_benchmark) - end - end - - describe 'Ruby options pass-through' do - it 'passes Ruby options after -- separator' do - warning_benchmark = File.join(@tmpdir, 'warning_test.rb') - File.write(warning_benchmark, <<~RUBY) - x = 1 - x = 2 - puts "Benchmark with warnings" - RUBY - - stdout, _stderr, status = run_script('--', '-W2', warning_benchmark) - - assert status.success? - assert_match(/Benchmark with warnings/, stdout) - end - - it 'passes YJIT options after -- separator' do - yjit_benchmark = File.join(@tmpdir, 'yjit_test.rb') - File.write(yjit_benchmark, <<~RUBY) - puts "YJIT enabled" if defined?(RubyVM::YJIT) - puts "Benchmark complete" - RUBY - - stdout, _stderr, status = run_script('--', '--yjit', yjit_benchmark) - - assert status.success? - assert_match(/Benchmark complete/, stdout) - end - end - - describe '--help option' do - it 'shows help message' do - stdout, _stderr, status = run_script('--help') - - assert status.success? - assert_match(/Usage:/, stdout) - assert_match(/--harness/, stdout) - end - - it 'shows help with -h' do - stdout, _stderr, status = run_script('-h') - - assert status.success? - assert_match(/Usage:/, stdout) - end - end - - describe 'argument parsing order' do - it 'handles harness option before benchmark' do - stdout, _stderr, status = run_script('--harness=default', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - - it 'handles Ruby options after -- separator' do - stdout, _stderr, status = run_script('--', '-W0', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - - it 'handles mixed options with -- separator' do - stdout, _stderr, status = run_script('--harness=default', '--', '-W0', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - end - - describe 'script examples' do - it 'works with simple benchmark path' do - fib_benchmark = File.join(@tmpdir, 'fib.rb') - File.write(fib_benchmark, 'puts "Fibonacci benchmark"') - - stdout, _stderr, status = run_script(fib_benchmark) - - assert status.success? - assert_match(/Fibonacci benchmark/, stdout) - end - - it 'works with harness option' do - stdout, _stderr, status = run_script('--harness=default', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - - it 'works with Ruby options after -- separator' do - stdout, _stderr, status = run_script('--', '--yjit', @test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - end - end - - describe 'edge cases' do - it 'handles benchmark files with spaces in path' do - space_dir = File.join(@tmpdir, 'dir with spaces') - FileUtils.mkdir_p(space_dir) - space_benchmark = File.join(space_dir, 'benchmark.rb') - File.write(space_benchmark, 'puts "Space test"') - - stdout, _stderr, status = run_script(space_benchmark) - - assert status.success? - assert_match(/Space test/, stdout) - end - - it 'identifies first .rb file as benchmark' do - other_file = File.join(@tmpdir, 'other.rb') - File.write(other_file, 'puts "Wrong file"') - - stdout, _stderr, status = run_script(@test_benchmark) - - assert status.success? - assert_match(/Benchmark executed/, stdout) - refute_match(/Wrong file/, stdout) - end - - it 'rejects invalid options gracefully' do - stdout, _stderr, status = run_script('--invalid-option', @test_benchmark) - - refute status.success? - assert_match(/invalid option|Error/, stdout) - end - end - - describe 'direct ruby execution with HARNESS env var' do - it 'uses HARNESS environment variable when running benchmark directly' do - harness_dir = File.join(File.dirname(@script_path), 'harness') - FileUtils.mkdir_p(harness_dir) - direct_harness = File.join(harness_dir, 'direct_test.rb') - - begin - File.write(direct_harness, <<~RUBY) - puts "DIRECT HARNESS RUNNING" - RUBY - - env = { 'HARNESS' => 'direct_test' } - stdout, _stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) - - assert status.success?, "Direct Ruby execution should work with HARNESS env var" - assert_match(/DIRECT HARNESS RUNNING/, stdout) - assert_match(/Benchmark executed/, stdout) - ensure - File.delete(direct_harness) if File.exist?(direct_harness) - end - end - - it 'shows helpful error for invalid HARNESS value' do - env = { 'HARNESS' => 'nonexistent_harness' } - _stdout, stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) - - refute status.success? - assert_match(/Harness 'nonexistent_harness' not found/, stderr) - assert_match(/Available harnesses/, stderr) - end - end -end From 47d5f33a7ba46e77f7bb2b785ea23f3b11d78c09 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 13 Oct 2026 11:24:44 +0100 Subject: [PATCH 14/18] Load the burn-in harness by absolute path burn_in.rb ran ruby -Iharness -r. Now that harness/ holds stackprof.rb and vernier.rb, -Iharness let those files shadow the stackprof and vernier gems. railsbench calls Bundler.require, so it loaded harness/stackprof.rb instead of the gem and failed with uninitialized constant StackProf. In this commit we pass the harness to -r by absolute path and drop -Iharness, the same way lib/benchmark_suite.rb does. The harness directory never enters $LOAD_PATH, so gem requires resolve to gems. --- burn_in.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/burn_in.rb b/burn_in.rb index f205c203..1d4a63b2 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -104,8 +104,7 @@ def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metada cmd = [ 'ruby', *test_options, - "-Iharness", - "-r#{harness}", + "-r#{File.expand_path("harness/#{harness}", __dir__)}", script_path, ].compact cmd_str = cmd.shelljoin From c42d6bd822c5200a239bb0a95988243f331ea845 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 13 Oct 2026 11:26:27 +0100 Subject: [PATCH 15/18] Describe the chain harness nesting order correctly The README said chained harnesses run in sequence and that once,default runs the benchmark once and then with default iterations. harness/chain.rb nests them instead: the first harness is the outermost, so once,default ran the default iterations inside a single once call and then once overwrote the results with a fake 1ms sample. The ractor example also pointed at benchmarks-ractor/, which does not exist. In this commit we describe the nesting, state that the outermost harness writes the results that are kept, and replace both examples with vernier,default and vernier,ractor on benchmarks/fib.rb. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d2e1c30d..f418c0a3 100644 --- a/README.md +++ b/README.md @@ -363,19 +363,19 @@ There is also a robust but complex CI harness in [the yjit-metrics repo](https:/ ### Chain harness -The `chain` harness allows you to combine multiple harnesses together. This is useful when you want to run a benchmark through multiple analysis tools or measurement approaches in sequence. +The `chain` harness allows you to combine multiple harnesses together. This is useful when you want to run a benchmark through an analysis tool, such as a profiler, while another harness drives the iterations. Use the `HARNESS_CHAIN` environment variable to specify which harnesses to chain (comma-separated, at least 2 required): ```bash -# Chain the 'once' and 'default' harnesses (runs benchmark once, then with default iterations) -HARNESS=chain HARNESS_CHAIN="once,default" ruby benchmarks/fib.rb +# Profile the iterations of the default harness with vernier +HARNESS=chain HARNESS_CHAIN="vernier,default" ruby benchmarks/fib.rb -# Profile with vernier while using ractor harness -HARNESS=chain HARNESS_CHAIN="vernier,ractor" ruby benchmarks-ractor/some_benchmark.rb +# Profile a benchmark running under the ractor harness with vernier +HARNESS=chain HARNESS_CHAIN="vernier,ractor" ruby benchmarks/fib.rb ``` -The harnesses are executed in the order specified, with each harness wrapping the previous one. +The harnesses are nested, not run one after another. The first harness is the outermost one: its benchmark block calls the `run_benchmark` of the next harness, and the last harness runs the benchmark itself. Every harness writes its results to the same file, and the outermost harness writes last, so its results are the ones kept. ### Iterations and duration From 197f8e5cf014eaf356a35a1b17507448b9c97316 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 13 Oct 2026 11:27:14 +0100 Subject: [PATCH 16/18] Drop the removed harness directory from the stackprof usage The usage examples still passed -I harness-stackprof, a directory that no longer exists, and the second example set MIN_BENCH_ITRS=10 twice. HARNESS=stackprof already selects the harness, so the commands only worked because Ruby ignores a missing -I path. --- harness/stackprof.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harness/stackprof.rb b/harness/stackprof.rb index 8a33524e..998c3c26 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' HARNESS=stackprof MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 HARNESS=stackprof MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' HARNESS=stackprof MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' HARNESS=stackprof MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v benchmarks/.../benchmark.rb require_relative "../lib/harness" require_relative "../lib/harness/extra" From 47775f95de3e8db366da961918cf5b6a802634fe Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 13 Oct 2026 11:27:14 +0100 Subject: [PATCH 17/18] Drop the removed harness directory from the vernier usage The usage examples still passed -I harness-vernier, a directory that no longer exists, and the header named the variable NO_VIERWER although the harness reads NO_VIEWER. HARNESS=vernier already selects the harness. --- harness/vernier.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/harness/vernier.rb b/harness/vernier.rb index 186da9de..fe7e81d1 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Profile the benchmark (ignoring initialization code) using vernier and display the profile. -# Set NO_VIERWER=1 to disable automatically opening the profile in a browser. +# Set NO_VIEWER=1 to disable automatically opening the profile in a browser. # Usage: -# HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... -# NO_VIEWER=1 HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... +# HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v benchmarks/... +# NO_VIEWER=1 HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v benchmarks/... require_relative "../lib/harness" require_relative "../lib/harness/extra" From 65c98750c0d40b2eeca035751bf850742d01da43 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 13 Oct 2026 21:22:06 +0100 Subject: [PATCH 18/18] Skip the Ractor harness tests on Rubies without Ractor The suite tests run from a temporary directory. BenchmarkSuite used to pass the harness as a relative -I harness-ractor, which did not exist there, so these tests never loaded the Ractor harness. It now passes -r with the absolute path to harness/ractor.rb, so the harness loads, and on TruffleRuby it fails with uninitialized constant Ractor. In this commit we skip the five tests that run the Ractor harness when the target Ruby has no Ractor, the same way test/ractor_harness_test.rb does. --- test/benchmark_suite_test.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index 3a203c2e..1f7366aa 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -323,6 +323,7 @@ def base_cmd_for(ruby_description) end it 'handles ractor category with ractor benchmarks' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) # Create a ractor benchmark File.write('benchmarks/ractor_test.rb', <<~RUBY) require 'json' @@ -362,6 +363,7 @@ def base_cmd_for(ruby_description) end it 'runs each Ractor count in its own process and merges the results' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) File.write('benchmarks/per_count.rb', <<~RUBY) require 'json' count = ENV.fetch('RUBY_BENCH_RACTORS') @@ -401,6 +403,7 @@ def base_cmd_for(ruby_description) end it 'reports a Ractor benchmark as failed when one count fails' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) File.write('benchmarks/fails_at_two.rb', <<~RUBY) require 'json' count = ENV.fetch('RUBY_BENCH_RACTORS') @@ -425,6 +428,7 @@ def base_cmd_for(ruby_description) end it 'skips count 0 for a scenario benchmark unless it is the only count' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) File.write('benchmarks/scenario.rb', <<~RUBY) require 'json' count = ENV.fetch('RUBY_BENCH_RACTORS') @@ -516,6 +520,7 @@ def base_cmd_for(ruby_description) end it 'runs pre_init before the Ractor harness freezes main' do + skip 'target Ruby has no Ractor' unless defined?(Ractor) pre_init_file = File.join(@temp_dir, 'pre_init.rb') File.write(pre_init_file, "@pre_init_ran = true\n") File.write('benchmarks/ractor_pre_init.rb', <<~RUBY)