diff --git a/README.md b/README.md index 2d1480ec..d67eb5b4 100644 --- a/README.md +++ b/README.md @@ -521,8 +521,10 @@ Ractor's own object space. The JSON output records the scope as The text summary shows GC data in separate tables after the timing table. A single-executable report has one `GC summary` table. A comparison report -has a `GC time ratios` table (base/comparison) and a `GC counts` table -(base → comparison). A table hides a column that has no data in any row and +has a `GC per ruby` table, a `GC time ratios` table (base/comparison), and a +`GC counts` table (base → comparison). The `GC per ruby` table has one row +for each ruby, including the base, with the absolute `GC summary` columns. +A table hides a column that has no data in any row and lists the hidden columns below the table. A ratio column has no data when it is `N/A` in every row; a `0.000` ratio stays visible. Any other column has no data when it is zero or `N/A` in every row. @@ -530,9 +532,10 @@ data when it is zero or `N/A` in every row. * Tables marked `worker sum` add the Ractor-local counters and GC times of the sampled workers of each iteration. `GCs/iter` is the sum of `minor/iter`, `major/iter`, and `global/iter`; a global cycle counts under `global` on - the Ractor that initiated it, not under `major`. Single-executable reports - also show `GC ms/worker`, which divides each iteration's worker-sum GC - time by its sampled worker count, then averages. + the Ractor that initiated it, not under `major`. The absolute GC tables + (the single-executable `GC summary` and the `GC per ruby` table) also show + `GC ms/worker`, which divides each iteration's worker-sum GC time by its + sampled worker count, then averages. * `compacts*` shows the main Ractor's `GC.stat(:compact_count)` delta. Every global compacting cycle increments it in every object space, so it is not summed across workers. diff --git a/harness/gc.rb b/harness/gc.rb index 4c93fc69..8ac5af53 100644 --- a/harness/gc.rb +++ b/harness/gc.rb @@ -148,8 +148,9 @@ def run_benchmark(_num_itrs_hint, **, &block) heap_snapshot.each do |idx, stats| slot_size = stats[:slot_size] || 0 eden_slots = stats[:heap_eden_slots] || 0 - live_slots = stats[:heap_live_slots] || 0 - free_slots = stats[:heap_free_slots] || 0 + # Ruby < 4.0 omits heap_live_slots/heap_free_slots; derive them from the per-heap counters. + live_slots = stats[:heap_live_slots] || stats[:total_allocated_objects].to_i - stats[:total_freed_objects].to_i + free_slots = stats[:heap_free_slots] || eden_slots - live_slots eden_pages = stats[:heap_eden_pages] || 0 live_pct = eden_slots > 0 ? (live_slots * 100.0 / eden_slots) : 0.0 mem_kib = page_size ? (eden_pages * page_size / 1024.0) : 0.0 diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 43c427ff..f2a6dfeb 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -74,6 +74,8 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ sections ||= [{ table: table, format: format, failures: bench_failures, include_gc: include_gc, gc_tables: gc_tables, memory_tables: memory_tables }] has_gc_summary = sections.any? { |section| section[:include_gc] && section[:gc_tables] } has_memory_summary = sections.any? { |section| section[:memory_tables] } + gc_table_list = sections.flat_map { |section| Array(section[:gc_tables]) } + per_ruby_headers = gc_table_list.select { |gc_table| gc_table[:name] == 'GC per ruby' }.flat_map { |gc_table| gc_table[:rows].first } sections.each do |section| title = section[:title] output_str << "#{title}:\n" if title @@ -93,10 +95,11 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ end end if has_gc_summary + output_str << "- GC per ruby shows the mean GC values per benchmark iteration for each ruby. These values are not ratios. Benchmarks with no GC activity on any ruby are omitted.\n" unless per_ruby_headers.empty? output_str << "- GC time ratios are #{base_name}/comparison; above 1 means the comparison spent less GC time.\n" output_str << "- gc/iter, mark/iter, and sweep/iter compare total GC (or phase) time per benchmark iteration, so they include both per-GC cost and GC frequency changes.\n" output_str << "- gc/GC, mark/GC, and sweep/GC divide average GC (or phase) time by the same run's GCs/iter count; they are not complete per-cycle attribution of process-wide GC.\n" - output_str << "- GC counts show #{base_name} → comparison values, not ratios. Rows with no GC activity are omitted.\n" + output_str << "- GC counts show #{base_name} → comparison values, not ratios. In GC time ratios and GC counts, a row is omitted when neither #{base_name} nor the comparison has GC activity.\n" end if has_memory_summary output_str << "- Scenario memory shows #{base_name} → comparison values, not ratios. Lower is better.\n" @@ -105,10 +108,13 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ output_str << "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)\n" end end - gc_headers = sections.flat_map { |section| Array(section[:gc_tables]).flat_map { |gc_table| gc_table[:rows].first } } + gc_headers = gc_table_list.flat_map { |gc_table| gc_table[:rows].first } if gc_headers.include?('compacts*') output_str << "GC metric notes:\n" - output_str << "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration. Every global compacting cycle increments compact_count in every object space. Do not sum it across workers.#{other_names.empty? ? '' : " Comparison tables show #{base_name} → comparison values."}\n" + compaction_note = +"" + compaction_note << " GC counts show #{base_name} → comparison values." unless other_names.empty? + compaction_note << " GC per ruby shows the value for each ruby." if per_ruby_headers.include?('compacts*') + output_str << "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration. Every global compacting cycle increments compact_count in every object space. Do not sum it across workers.#{compaction_note}\n" end ractor_gc_sections = sections.select { |section| section[:gc_scope] == 'ractor-local-workload' && section[:gc_tables] } @@ -118,7 +124,7 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_ output_str << "Ractor GC scope note:\n" scope_columns = +"- Tables marked worker sum add Ractor-local counters and GC times across the sampled workers of each iteration" scope_columns << (worker_mode ? "; the main Ractor performs the count-0 workload of per-worker benchmarks." : ".") - scope_columns << " GC ms/worker divides each iteration's worker-sum GC time by its sampled worker count, then averages." if other_names.empty? + scope_columns << " GC ms/worker divides each iteration's worker-sum GC time by its sampled worker count, then averages." if gc_headers.include?('GC ms/worker') output_str << "#{scope_columns} Controller snapshots and per-worker heap detail are in the JSON output, not these tables.\n" output_str << "- Ruby's Ractor-retirement GC (after a worker's stack is torn down) is not sampled." output_str << " Ractors created by the workload of a per-worker benchmark are not sampled." if worker_mode diff --git a/lib/results_table_builder.rb b/lib/results_table_builder.rb index 6f48a09a..405158b8 100644 --- a/lib/results_table_builder.rb +++ b/lib/results_table_builder.rb @@ -124,7 +124,7 @@ def build_gc_tables return nil unless @include_gc label_header = ["bench", *@row_layout.extra_header_columns] - tables = @other_names.empty? ? [build_gc_absolute_table(label_header)] : build_gc_comparison_tables(label_header) + tables = @other_names.empty? ? [build_gc_absolute_table(label_header)] : [build_gc_per_ruby_table(label_header), *build_gc_comparison_tables(label_header)] tables.compact! tables.empty? ? nil : tables end @@ -192,6 +192,19 @@ def gc_count_columns columns end + def build_gc_per_ruby_table(label_header) + rows = [] + gc_entries.each do |entry| + data_by_name = @executable_names.to_h { |name| [name, bench_data_for(name, entry.data_key)] } + next unless gc_activity?(*data_by_name.values.flat_map { |data| data.values_at(*GC_SERIES_KEYS) }) + + data_by_name.each do |name, data| + rows << [gc_label_cells(entry) + [name], [data]] if GC_SERIES_KEYS.any? { |key| data.key?(key) } + end + end + assemble_gc_table("GC per ruby", label_header + ["ruby"], rows, gc_absolute_columns) + end + def build_gc_absolute_table(label_header) rows = gc_entries.filter_map do |entry| data = bench_data_for(@base_name, entry.data_key) diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 580a8b92..f1055c95 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -524,7 +524,8 @@ def git(dir, *args) assert_includes result, "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration." assert_includes result, 'Every global compacting cycle increments compact_count in every object space.' assert_includes result, 'Do not sum it across workers.' - assert_includes result, 'Comparison tables show ruby-base → comparison values.' + assert_includes result, 'GC counts show ruby-base → comparison values.' + refute_includes result, 'GC per ruby shows the value for each ruby.' refute_includes result, 'global GCs/iter' end @@ -546,7 +547,7 @@ def git(dir, *args) refute_includes result, 'Legend:' assert_includes result, "GC metric notes:\n" assert_includes result, "- compacts*: the main Ractor's GC.stat(:compact_count) delta per iteration." - refute_includes result, 'Comparison tables show' + refute_includes result, 'GC counts show' refute_includes result, 'global GCs/iter' end @@ -573,7 +574,7 @@ def git(dir, *args) refute_includes result, 'GC metric notes:' assert_includes result, "the same run's GCs/iter count" - assert_includes result, 'GC counts show ruby-base → comparison values, not ratios' + assert_includes result, 'GC counts show ruby-base → comparison values, not ratios. In GC time ratios and GC counts, a row is omitted when neither ruby-base nor the comparison has GC activity.' end it 'omits the Ractor scope note when the section rendered no GC table' do @@ -597,6 +598,80 @@ def git(dir, *args) refute_includes result, 'Ractor GC scope note:' end + it 'prints the GC per ruby table before the GC ratio table, with its legend line' do + ruby_descriptions = { 'master' => 'ruby 4.1.0dev', 'four' => 'ruby 4.0.6', 'three' => 'ruby 3.4.10' } + sections = [ + { + title: 'harness-gc', + table: [['bench', 'master (ms)', 'four (ms)', 'three (ms)'], ['gcbench', '1.0', '2.0', '3.0']], + format: ['%s', '%s', '%s', '%s'], + failures: {}, + include_gc: true, + gc_tables: [ + { name: 'GC per ruby', scope: nil, hidden: [], rows: [ + ['bench', 'ruby', 'GC ms/iter', 'compacts*'], + ['gcbench', 'master', '1.000', '0.0'], + ['gcbench', 'four', '2.000', '0.0'], + ['gcbench', 'three', '3.000', '0.0'] + ] }, + { name: 'GC time ratios', scope: nil, hidden: [], rows: [ + ['bench', 'comparison', 'gc/iter'], ['gcbench', 'four', '0.500'], ['gcbench', 'three', '0.333'] + ] }, + ], + }, + { + title: 'harness', + table: [['bench', 'master (ms)', 'four (ms)', 'three (ms)'], ['fib', '1.0', '2.0', '3.0']], + format: ['%s', '%s', '%s', '%s'], + failures: {}, + include_gc: false, + } + ] + + result = BenchmarkRunner.build_output_text( + ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections + ) + + per_ruby_at = result.index("GC per ruby (harness-gc):\n") + summary_at = result.index("GC time ratios (harness-gc):\n") + refute_nil per_ruby_at + refute_nil summary_at + assert_operator per_ruby_at, :<, summary_at + assert_match(/^gcbench\s+master\s+1\.000\s+0\.0$/, result) + assert_includes result, '- GC per ruby shows the mean GC values per benchmark iteration for each ruby. These values are not ratios. Benchmarks with no GC activity on any ruby are omitted.' + assert_includes result, 'Do not sum it across workers. GC counts show master → comparison values. GC per ruby shows the value for each ruby.' + refute_includes result, 'GC per ruby (harness):' + end + + it 'explains GC ms/worker in the Ractor scope note when only the GC per ruby table shows it' do + ruby_descriptions = { 'base' => 'ruby 4.1.0dev', 'exp' => 'ruby 4.1.0dev experiment' } + sections = [ + { + title: 'harness-ractor', + table: [['bench', 'ractors', 'base (ms)', 'exp (ms)'], ['object-new', '2', '1.0', '2.0']], + format: ['%s', '%s', '%s', '%s'], + failures: {}, + include_gc: true, + gc_tables: [ + { name: 'GC per ruby', scope: 'worker sum', hidden: [], rows: [ + ['bench', 'ractors', 'ruby', 'GC ms/iter', 'GC ms/worker'], ['object-new', '2', 'base', '4.000', '2.000'] + ] }, + { name: 'GC time ratios', scope: 'worker sum', hidden: [], rows: [ + ['bench', 'ractors', 'gc/iter'], ['object-new', '2', '2.000'] + ] }, + ], + gc_scope: 'ractor-local-workload', + } + ] + + result = BenchmarkRunner.build_output_text( + ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections + ) + + assert_includes result, "Ractor GC scope note:\n" + assert_includes result, "GC ms/worker divides each iteration's worker-sum GC time by its sampled worker count, then averages." + end + it 'includes RSS ratio legend when include_rss is true' do ruby_descriptions = { 'ruby-base' => 'ruby 3.3.0', diff --git a/test/results_table_builder_test.rb b/test/results_table_builder_test.rb index c0ad30f3..280dc176 100644 --- a/test/results_table_builder_test.rb +++ b/test/results_table_builder_test.rb @@ -750,9 +750,9 @@ def scenario_layout(name, counts) assert_equal ['bench', 'ruby-base (ms)', 'ruby-exp (ms)', 'ruby-exp 1st itr', 'ruby-base/ruby-exp'], table[0] assert_equal ['%s', '%s', '%s', '%.3f', '%s'], format - ratios, counts = gc_tables - assert_equal ['GC time ratios', 'GC counts'], gc_tables.map { |t| t[:name] } - assert_equal [nil, nil], gc_tables.map { |t| t[:scope] }, 'non-Ractor tables are not worker sums' + _per_ruby, ratios, counts = gc_tables + assert_equal ['GC per ruby', 'GC time ratios', 'GC counts'], gc_tables.map { |t| t[:name] } + assert_equal [nil, nil, nil], gc_tables.map { |t| t[:scope] }, 'non-Ractor tables are not worker sums' assert_equal [ ['bench', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ['fib', '1.333', '1.000', '0.667', '0.500'] @@ -761,7 +761,7 @@ def scenario_layout(name, counts) ['bench', 'GCs/iter', 'major/iter', 'minor/iter', 'minor GC %'], ['fib', '10.0 → 5.0', ' 2.0 → 1.0', ' 8.0 → 4.0', ' 80% → 80%'] ], counts[:rows] - assert_equal [[], []], gc_tables.map { |t| t[:hidden] } + assert_equal [[], []], [ratios, counts].map { |t| t[:hidden] } end it 'omits benchmarks with no GC activity from the GC summary' do @@ -848,7 +848,7 @@ def scenario_layout(name, counts) _table, _format, gc_tables = builder.build - ratios, counts = gc_tables + _per_ruby, ratios, counts = gc_tables assert_equal [['bench', 'GCs/iter', 'major/iter'], ['fib', ' 3.0 → 2.0', ' 3.0 → 2.0']], counts[:rows] assert_equal ['minor/iter', 'minor GC %'], counts[:hidden], 'a 0% → 0% column has no data' assert_equal 'GC time ratios', ratios[:name], 'a table with rows stays even when every column is hidden' @@ -888,6 +888,69 @@ def scenario_layout(name, counts) refute_includes counts[:rows][0], 'global/iter', 'a column of N/A → N/A cells has no data' assert_includes counts[:hidden], 'global/iter' end + + it 'gives every executable, including the baseline, its own row in the GC per ruby table' do + gc_data = ->(mark, major, minor) do + { + 'warmup' => [0.1], + 'bench' => [0.1, 0.1], + 'rss' => 10, + 'gc_marking_time_bench' => [mark, mark], + 'gc_sweeping_time_bench' => [1.0, 1.0], + 'gc_major_count_bench' => [major, major], + 'gc_minor_count_bench' => [minor, minor] + } + end + bench_data = { + 'master' => { 'gcbench' => gc_data.(4.0, 1, 9) }, + 'four' => { 'gcbench' => gc_data.(2.0, 0, 6) }, + 'three' => { 'gcbench' => gc_data.(3.0, 2, 4) } + } + + builder = ResultsTableBuilder.new( + executable_names: ['master', 'four', 'three'], + bench_data: bench_data + ) + + _table, _format, gc_tables = builder.build + + assert_equal ['GC per ruby', 'GC time ratios', 'GC counts'], gc_tables.map { |t| t[:name] } + per_ruby = gc_tables.first + assert_equal ['bench', 'ruby', 'mark ms/iter', 'sweep ms/iter', 'GCs/iter', 'major/iter', 'minor/iter'], per_ruby[:rows][0] + assert_equal ['GC ms/iter'], per_ruby[:hidden] + assert_equal [ + ['gcbench', 'master', '4.000', '1.000', '10.0', '1.0', '9.0'], + ['gcbench', 'four', '2.000', '1.000', '6.0', '0.0', '6.0'], + ['gcbench', 'three', '3.000', '1.000', '6.0', '2.0', '4.0'] + ], per_ruby[:rows][1..] + end + + it 'omits the GC per ruby table for one executable or idle benchmarks, and keeps idle rubies when another is active' do + idle = { + 'warmup' => [0.1], + 'bench' => [0.1], + 'rss' => 10, + 'gc_marking_time_bench' => [0.0], + 'gc_sweeping_time_bench' => [0.0], + 'gc_major_count_bench' => [0], + 'gc_minor_count_bench' => [0] + } + + per_ruby = ->(builder) { Array(builder.build[2]).find { |t| t[:name] == 'GC per ruby' } } + + single = ResultsTableBuilder.new(executable_names: ['ruby'], bench_data: { 'ruby' => { 'fib' => idle.merge('gc_minor_count_bench' => [3]) } }) + assert_nil per_ruby.(single) + + multi = ResultsTableBuilder.new(executable_names: ['a', 'b'], bench_data: { 'a' => { 'fib' => idle }, 'b' => { 'fib' => idle } }) + assert_nil per_ruby.(multi) + + mixed = ResultsTableBuilder.new(executable_names: ['a', 'b'], bench_data: { 'a' => { 'fib' => idle }, 'b' => { 'fib' => idle.merge('gc_minor_count_bench' => [3]) } }) + assert_equal [ + ['bench', 'ruby', 'GCs/iter', 'minor/iter'], + ['fib', 'a', '0.0', '0.0'], + ['fib', 'b', '3.0', '3.0'] + ], per_ruby.(mixed)[:rows], 'an idle ruby keeps its row when another ruby has GC activity' + end end describe 'RSS sampling (rss_samples)' do @@ -1066,8 +1129,8 @@ def build_ractor_gc(bench_data) table, _format, gc_tables = build_ractor_gc(bench_data) assert_equal ['bench', 'ractors', 'base (ms)', 'candidate (ms)', 'candidate 1st itr', 'base/candidate'], table[0] - assert_equal ['worker sum', 'worker sum'], gc_tables.map { |t| t[:scope] } - ratios, counts = gc_tables.map { |t| t[:rows] } + assert_equal ['worker sum', 'worker sum', 'worker sum'], gc_tables.map { |t| t[:scope] } + _per_ruby, ratios, counts = gc_tables.map { |t| t[:rows] } assert_equal ['bench', 'ractors', 'gc/iter', 'gc/GC', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ratios[0] assert_equal ['bench', 'ractors', 'GCs/iter', 'major/iter', 'minor/iter', 'minor GC %'], counts[0] @@ -1101,7 +1164,7 @@ def build_ractor_gc(bench_data) _table, _format, gc_tables = build_ractor_gc(bench_data) - rows = gc_tables.first[:rows][1..].to_h { |row| [row[1], row] } + rows = gc_tables.find { |t| t[:name] == 'GC time ratios' }[:rows][1..].to_h { |row| [row[1], row] } assert_equal '2.000', rows['0'][2], 'count 0 keeps its own gc/iter ratio' assert_equal '1.000', rows['0'][4] assert_equal 'N/A', rows['2'][2], 'missing total-time series must not reuse count 0 data or zero' @@ -1199,7 +1262,7 @@ def build_ractor_gc(bench_data) _table, _format, gc_tables = build_ractor_gc(bench_data) - ratios, counts = gc_tables.map { |t| t[:rows] } + _per_ruby, ratios, counts = gc_tables.map { |t| t[:rows] } assert_equal '2.500', ratios[1][ratios[0].index('gc/iter')] assert_equal '1.500', ratios[1][ratios[0].index('gc/GC')] header = counts[0] @@ -1246,7 +1309,7 @@ def build_ractor_gc(bench_data) _table, _format, gc_tables = build_ractor_gc(bench_data) - ratios, counts = gc_tables.map { |t| t[:rows] } + _per_ruby, ratios, counts = gc_tables.map { |t| t[:rows] } assert_equal ['bench', 'ractors', 'gc/iter', 'gc/GC', 'mark/iter', 'sweep/iter', 'mark/GC', 'sweep/GC'], ratios[0], 'only time ratios' assert_equal ['bench', 'ractors', 'GCs/iter', 'major/iter', 'minor/iter', 'global/iter', 'compacts*', 'minor GC %'], counts[0] assert_equal ' 4.0 → 2.0', counts[1][counts[0].index('global/iter')], 'global GCs are counts, not a time ratio' @@ -1266,7 +1329,7 @@ def build_ractor_gc(bench_data) _table, _format, gc_tables = build_ractor_gc(bench_data) refute_nil gc_tables, 'the nil entry must not hide a row whose other metrics have activity' - ratios, counts = gc_tables + _per_ruby, ratios, counts = gc_tables assert_equal ['mark/iter', 'mark/GC'], ratios[:hidden], 'a null entry makes the whole series unavailable in the only row' header = ratios[:rows][0] assert_equal '1.000', ratios[:rows][1][header.index('sweep/iter')], 'neighboring series keep their values' @@ -1288,5 +1351,28 @@ def build_ractor_gc(bench_data) assert_equal ['object-new', '0', '4.000', '4.000', '7.0', '1.0', '3.0', '3.0', '1.0'], gc_table[:rows][1] assert_equal ['mark ms/iter', 'sweep ms/iter'], gc_table[:hidden] end + + it 'puts the ruby column after ractors in the GC per ruby table' do + bench_data = { + 'base' => { 'object-new' => ractor_gc_blob('2' => { bench: [1.0, 1.0], gc: gc_group(total: [12.0, 12.0], major: [2, 2], minor: [6, 6]) }) }, + 'candidate' => { 'object-new' => ractor_gc_blob('2' => { bench: [1.0, 1.0], gc: gc_group(total: [3.0, 3.0], major: [1, 1], minor: [3, 3]) }) } + } + expanded = RactorBreakdown.expand(bench_data) + builder = ResultsTableBuilder.new( + executable_names: bench_data.keys, + bench_data: expanded.bench_data, + row_layout: RactorRowLayout.new(groups: expanded.groups) + ) + + _table, _format, gc_tables = builder.build + + per_ruby = gc_tables.find { |t| t[:name] == 'GC per ruby' } + assert_equal 'worker sum', per_ruby[:scope] + assert_equal [ + 'bench', 'ractors', 'ruby', 'GC ms/iter', 'GC ms/worker', 'GCs/iter', 'major/iter', 'minor/iter' + ], per_ruby[:rows][0] + assert_equal ['object-new', '2', 'base', '12.000', '12.000', '8.0', '2.0', '6.0'], per_ruby[:rows][1] + assert_equal ['object-new', '2', 'candidate', '3.000', '3.000', '4.0', '1.0', '3.0'], per_ruby[:rows][2] + end end end