Split summary table by harness

Author: eightbitraptor

lib/benchmark_runner.rb

@@ -48,7 +48,7 @@ def write_csv(output_path, ruby_descriptions, table)
     end
 
     # Build output text string with metadata, table, and legend
-    def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false, include_gc: false, include_pvalue: false, gc_table: nil, gc_format: nil)
+    def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false, include_gc: false, include_pvalue: false, gc_table: nil, gc_format: nil, sections: nil)
       base_name, *other_names = ruby_descriptions.keys
 
       output_str = +""
@@ -58,11 +58,17 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_
       end
 
       output_str << "\n"
-      output_str << TableFormatter.new(table, format, bench_failures).to_s + "\n"
-
-      if include_gc && gc_table && gc_format
-        output_str << "GC summary:\n"
-        output_str << TableFormatter.new(gc_table, gc_format, {}).to_s + "\n"
+      sections ||= [{ table: table, format: format, failures: bench_failures, include_gc: include_gc, gc_table: gc_table, gc_format: gc_format }]
+      has_gc_summary = sections.any? { |section| section[:include_gc] && section[:gc_table] }
+      sections.each do |section|
+        title = section[:title]
+        output_str << "#{title}:\n" if title
+        output_str << TableFormatter.new(section[:table], section[:format], section.fetch(:failures, {})).to_s + "\n"
+
+        if section[:include_gc] && section[:gc_table] && section[:gc_format]
+          output_str << (title ? "GC summary (#{title}):\n" : "GC summary:\n")
+          output_str << TableFormatter.new(section[:gc_table], section[:gc_format], {}).to_s + "\n"
+        end
       end
 
       unless other_names.empty?
@@ -74,7 +80,7 @@ def build_output_text(ruby_descriptions, table, format, bench_failures, include_
             output_str << "- RSS #{base_name}/#{name}: ratio of #{base_name}/#{name} RSS. Higher is better for #{name}. Above 1 means lower memory usage.\n"
           end
         end
-        if include_gc && gc_table
+        if has_gc_summary
           output_str << "- GC summary compares #{base_name} → comparison. Ratio columns are #{base_name}/comparison; above 1 means the comparison spent less GC time.\n"
           output_str << "- mark/iter ratio and sweep/iter ratio compare total GC phase time per benchmark iteration, so they include both per-GC cost and GC frequency changes.\n"
           output_str << "- mark/GC ratio and sweep/GC ratio compare average phase time per GC, isolating whether each GC became cheaper or more expensive.\n"

lib/benchmark_runner/cli.rb

@@ -6,6 +6,8 @@
 require_relative '../benchmark_runner'
 require_relative '../benchmark_suite'
 require_relative '../results_table_builder'
+require_relative '../ractor_breakdown'
+require_relative '../row_layout'
 
 module BenchmarkRunner
   class CLI
@@ -63,6 +65,9 @@ def run
       bench_start_time = Time.now.to_f
       bench_data = {}
       bench_failures = {}
+      bench_harnesses = suite.benchmarks.each_with_object({}) do |entry, h|
+        h[entry.name] = suite.harness_for(entry.name)
+      end
 
       if args.interleave
         args.executables.each_key { |name| bench_data[name] = {} }
@@ -104,7 +109,7 @@ def run
 
       puts
 
-      # Build results table
+      # Build the results table
       builder = ResultsTableBuilder.new(
         executable_names: ruby_descriptions.keys,
         bench_data: bench_data,
@@ -125,7 +130,8 @@ def run
       BenchmarkRunner.write_csv(output_path, ruby_descriptions, table)
 
       # Save the output in a text file that we can easily refer to
-      output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss, include_gc: builder.include_gc?, include_pvalue: args.pvalue, gc_table: gc_table, gc_format: gc_format)
+      output_sections = build_output_sections(ruby_descriptions.keys, bench_data, bench_harnesses, bench_failures)
+      output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss, include_gc: builder.include_gc?, include_pvalue: args.pvalue, gc_table: gc_table, gc_format: gc_format, sections: output_sections)
       out_txt_path = output_path + ".txt"
       File.open(out_txt_path, "w") { |f| f.write output_str }
 
@@ -149,5 +155,81 @@ def run
         exit(1)
       end
     end
+
+    private
+
+    def build_output_sections(executable_names, bench_data, bench_harnesses, bench_failures)
+      ordered_names = sorted_benchmark_names(executable_names, bench_data)
+      failed_names = bench_failures.values.flat_map(&:keys).uniq
+      ordered_names.concat(failed_names.reject { |name| ordered_names.include?(name) })
+
+      names_by_harness = {}
+      ordered_names.each do |bench_name|
+        harness = bench_harnesses.fetch(bench_name, args.harness)
+        names_by_harness[harness] ||= []
+        names_by_harness[harness] << bench_name
+      end
+
+      show_titles = names_by_harness.size > 1
+      names_by_harness.map do |harness, names|
+        section = build_output_section(executable_names, bench_data, bench_failures, harness, names)
+        section[:title] = nil unless show_titles
+        section
+      end
+    end
+
+    def build_output_section(executable_names, bench_data, bench_failures, harness, bench_names)
+      section_data = slice_bench_data(bench_data, bench_names)
+      breakdown = RactorBreakdown.expand(section_data)
+      use_ractor_layout = harness == BenchmarkSuite::RACTOR_HARNESS && !breakdown.groups.empty?
+      layout = use_ractor_layout ? RactorRowLayout.new(groups: breakdown.groups) : FlatRowLayout.new
+      display_data = use_ractor_layout ? breakdown.bench_data : section_data
+
+      builder = ResultsTableBuilder.new(
+        executable_names: executable_names,
+        bench_data: display_data,
+        include_rss: args.rss,
+        include_pvalue: args.pvalue,
+        zjit_stats: args.zjit_stats,
+        row_layout: layout
+      )
+      table, format, gc_table, gc_format = builder.build
+
+      {
+        title: harness,
+        table: table,
+        format: format,
+        failures: slice_failures(bench_failures, bench_names),
+        include_gc: builder.include_gc?,
+        gc_table: gc_table,
+        gc_format: gc_format,
+      }
+    end
+
+    def sorted_benchmark_names(executable_names, bench_data)
+      builder = ResultsTableBuilder.new(
+        executable_names: executable_names,
+        bench_data: bench_data,
+        include_rss: args.rss,
+        include_pvalue: args.pvalue,
+        zjit_stats: args.zjit_stats
+      )
+      builder.bench_names
+    end
+
+    def slice_bench_data(bench_data, bench_names)
+      wanted = bench_names.each_with_object({}) { |name, h| h[name] = true }
+      bench_data.each_with_object({}) do |(executable, benchmarks), sliced|
+        sliced[executable] = benchmarks.select { |name, _data| wanted[name] }
+      end
+    end
+
+    def slice_failures(bench_failures, bench_names)
+      wanted = bench_names.each_with_object({}) { |name, h| h[name] = true }
+      bench_failures.each_with_object({}) do |(executable, failures), sliced|
+        selected = failures.select { |name, _failure| wanted[name] }
+        sliced[executable] = selected unless selected.empty?
+      end
+    end
   end
 end

lib/benchmark_suite.rb

@@ -38,6 +38,10 @@ def benchmarks
     @benchmarks ||= discover_benchmarks
   end
 
+  def harness_for(benchmark_name)
+    benchmark_harness_for(benchmark_name)
+  end
+
   # Run a single benchmark entry on a single executable.
   # Returns { name:, data: } on success, { name:, failure: } on error.
   def run_benchmark(entry, ruby:, ruby_description:)
@@ -50,19 +54,21 @@ def run_benchmark(entry, ruby:, ruby_description:)
 
     # Clear project-level Bundler environment so benchmarks run in a clean context.
     # Benchmarks that need Bundler (e.g., railsbench) set up their own via use_gemfile.
+    benchmark_harness = benchmark_harness_for(entry.name)
+
     result = if defined?(Bundler)
       Bundler.with_unbundled_env do
-        run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, entry.name, quiet: quiet)
+        run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet)
       end
     else
-      run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, entry.name, quiet: quiet)
+      run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: quiet)
     end
 
     if result[:success]
-      { name: entry.name, data: process_benchmark_result(result_json_path, result[:command], delete_file: !caller_json_path) }
+      { name: entry.name, data: process_benchmark_result(result_json_path, result[:command], delete_file: !caller_json_path), harness: benchmark_harness }
     else
       FileUtils.rm_f(result_json_path) unless caller_json_path
-      { name: entry.name, failure: result[:status].exitstatus }
+      { name: entry.name, failure: result[:status].exitstatus, harness: benchmark_harness }
     end
   end
 
@@ -145,7 +151,7 @@ def filter_entries(entries, categories:, name_filters:, excludes:, directory_map
     entries.select { |entry| filter.match?(entry.name) }
   end
 
-  def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_name, quiet: false)
+  def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_harness, quiet: false)
     # Fix for jruby/jruby#7394 in JRuby 9.4.2.0
     script_path = File.expand_path(script_path)
 
@@ -154,9 +160,6 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, b
     original_result_json_path = ENV["RESULT_JSON_PATH"]
     ENV["RESULT_JSON_PATH"] = result_json_path
 
-    # Use per-benchmark default_harness if set, otherwise use global harness
-    benchmark_harness = benchmark_harness_for(benchmark_name)
-
     # Set up the benchmarking command
     cmd = cmd_prefix + [
       *ruby,

test/benchmark_runner_cli_test.rb

@@ -84,6 +84,41 @@ def create_args(overrides = {})
     end
   end
 
+  describe '#build_output_sections' do
+    it 'splits text summary tables by harness in sorted benchmark order' do
+      args = create_args
+      cli = BenchmarkRunner::CLI.new(args)
+      bench_data = {
+        'ruby' => {
+          'fib' => {
+            'warmup' => [],
+            'bench' => [0.1],
+            'rss' => 10 * 1024 * 1024
+          },
+          'symbol-name-ractor' => {
+            'warmup' => [],
+            'bench' => [1.0, 2.0],
+            'bench_by_ractors' => { '0' => [1.0], '2' => [2.0] },
+            'rss' => 10 * 1024 * 1024
+          }
+        }
+      }
+      bench_harnesses = {
+        'fib' => 'harness',
+        'symbol-name-ractor' => 'harness-ractor'
+      }
+
+      sections = cli.send(:build_output_sections, ['ruby'], bench_data, bench_harnesses, {})
+
+      assert_equal ['harness-ractor', 'harness'], sections.map { |section| section[:title] }
+      assert_equal ['bench', 'ractors', 'ruby (ms)'], sections[0][:table][0]
+      assert_equal ['symbol-name-ractor', '0', '1000.0 ± 0.0%'], sections[0][:table][1]
+      assert_equal ['', '2', '2000.0 ± 0.0%'], sections[0][:table][2]
+      assert_equal ['bench', 'ruby (ms)'], sections[1][:table][0]
+      assert_equal ['fib', '100.0 ± 0.0%'], sections[1][:table][1]
+    end
+  end
+
   describe '#run integration test' do
     it 'runs a simple benchmark end-to-end and produces all output files' do
       Dir.mktmpdir do |tmpdir|

test/benchmark_runner_test.rb

@@ -496,6 +496,57 @@
       assert_includes result, '100.0'
     end
 
+    it 'prints multiple table sections when provided' do
+      ruby_descriptions = { 'ruby' => 'ruby 3.3.0' }
+      sections = [
+        {
+          title: 'harness',
+          table: [['bench', 'ruby (ms)'], ['fib', '100.0']],
+          format: ['%s', '%.1f'],
+          failures: {},
+          include_gc: false,
+        },
+        {
+          title: 'harness-ractor',
+          table: [['bench', 'ractors', 'ruby (ms)'], ['symbol-name-ractor', '0', '200.0']],
+          format: ['%s', '%s', '%.1f'],
+          failures: {},
+          include_gc: false,
+        }
+      ]
+
+      result = BenchmarkRunner.build_output_text(
+        ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections
+      )
+
+      assert_includes result, "harness:\n"
+      assert_includes result, "harness-ractor:\n"
+      assert_includes result, 'fib'
+      assert_includes result, 'symbol-name-ractor'
+    end
+
+    it 'labels GC summaries by section title' do
+      ruby_descriptions = { 'ruby' => 'ruby 3.3.0', 'exp' => 'ruby 3.3.0 exp' }
+      sections = [
+        {
+          title: 'harness-gc',
+          table: [['bench', 'ruby (ms)', 'exp (ms)'], ['gcbench', '100.0', '90.0']],
+          format: ['%s', '%.1f', '%.1f'],
+          failures: {},
+          include_gc: true,
+          gc_table: [['bench', 'mark/iter ratio'], ['gcbench', '1.100']],
+          gc_format: ['%s', '%s'],
+        }
+      ]
+
+      result = BenchmarkRunner.build_output_text(
+        ruby_descriptions, sections.first[:table], sections.first[:format], {}, sections: sections
+      )
+
+      assert_includes result, "GC summary (harness-gc):\n"
+      assert_includes result, '- GC summary compares ruby → comparison.'
+    end
+
     it 'omits legend when no other executables' do
       ruby_descriptions = { 'ruby' => 'ruby 3.3.0' }
       table = [['bench', 'ruby (ms)'], ['fib', '100.0']]

test/benchmark_suite_test.rb

@@ -569,6 +569,7 @@
       assert_includes result[:data], 'warmup'
       assert_includes result[:data], 'bench'
       assert_includes result[:data], 'rss'
+      assert_equal 'harness', result[:harness]
       assert_nil result[:failure]
     end
 
@@ -592,6 +593,7 @@
       assert_equal 'failing', result[:name]
       assert_nil result[:data]
       assert_equal 1, result[:failure]
+      assert_equal 'harness', result[:harness]
     end
 
     it 'produces same data as run for the same benchmark' do