Sort benchmark definitions alphabetically (#524)

Author: k0kubun

benchmarks.yml

@@ -130,12 +130,12 @@ rubyboy:
   ractor: true
 rubykon:
   desc: Ruby solver for Go (the boardgame.) Runs many iterations forward from an initial starting board.
-sudoku:
-  desc: sudoku solver
-  ractor: true
 splay:
   desc: Splay tree operations (insert, find, remove) to stress GC.
   single_file: true
+sudoku:
+  desc: sudoku solver
+  ractor: true
 string_malloc_pressure:
   desc: Allocates and retains many empty Strings with large reserved capacity to exercise GC under malloc-backed String buffer pressure.
   category: gc
@@ -166,18 +166,6 @@ cfunc_itself:
   desc: cfunc_itself just calls the 'itself' method many, many times.
   category: micro
   single_file: true
-send_cfunc_block:
-  desc: send_cfunc_block just calls a known C function with a block many, many times.
-  category: micro
-  single_file: true
-send_rubyfunc_block:
-  desc: send_rubyfunc_block just calls a known Ruby function with a block many, many times.
-  category: micro
-  single_file: true
-send_bmethod:
-  desc: send_bmethod just calls known Ruby bmethods many, many times.
-  category: micro
-  single_file: true
 fib:
   desc: Fib is a simple exponential-time recursive Fibonacci number generator.
   category: micro
@@ -193,14 +181,6 @@ getivar-module:
   category: micro
   single_file: true
   ractor: true
-structaref:
-  desc: structaref tests the performance of getting Struct members
-  category: micro
-  single_file: true
-structaset:
-  desc: structaset tests the performance of setting Struct members
-  category: micro
-  single_file: true
 keyword_args:
   desc: keyword_args tests the performance of method calls with keyword arguments.
   category: micro
@@ -233,6 +213,22 @@ ruby-xor:
   category: micro
   single_file: true
   ractor: true
+send_bmethod:
+  desc: send_bmethod just calls known Ruby bmethods many, many times.
+  category: micro
+  single_file: true
+send_cfunc_block:
+  desc: send_cfunc_block just calls a known C function with a block many, many times.
+  category: micro
+  single_file: true
+send_rubyfunc_block:
+  desc: send_rubyfunc_block just calls a known Ruby function with a block many, many times.
+  category: micro
+  single_file: true
+send_rubyfunc_inline:
+  desc: send_rubyfunc_inline repeatedly calls a small inlinable Ruby method with multiple arguments.
+  category: micro
+  single_file: true
 setivar:
   desc: setivar tests the performance of setting instance variable values.
   category: micro
@@ -251,24 +247,23 @@ str_concat:
   category: micro
   single_file: true
   ractor: true
+structaref:
+  desc: structaref tests the performance of getting Struct members
+  category: micro
+  single_file: true
+structaset:
+  desc: structaset tests the performance of setting Struct members
+  category: micro
+  single_file: true
 throw:
   desc: microbenchmark for the throw instruction and stack unwinding.
   category: micro
   single_file: true
   ractor: true
-send_rubyfunc_inline:
-  desc: send_rubyfunc_inline repeatedly calls a small inlinable Ruby method with multiple arguments.
-  category: micro
-  single_file: true
 
 #
 # Ractor scaling benchmarks
 #
-symbol-name-ractor:
-  desc: repeatedly calls Symbol#name on a static symbol under the ractor harness to stress ID-to-string lookup.
-  ractor: true
-  ractor_only: true
-  default_harness: harness-ractor
 gvl_release_acquire:
   desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors.
   ractor: true
@@ -284,3 +279,8 @@ json_parse_string:
   ractor: true
   ractor_only: true
   default_harness: harness-ractor
+symbol-name-ractor:
+  desc: repeatedly calls Symbol#name on a static symbol under the ractor harness to stress ID-to-string lookup.
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor

test/benchmarks_test.rb

@@ -22,4 +22,27 @@
 
     assert_equal yml_keys, discovered_keys
   end
+
+  it 'sorts benchmarks alphabetically within each category' do
+    yjit_bench = File.expand_path('..', __dir__)
+    benchmarks_yml = YAML.load_file("#{yjit_bench}/benchmarks.yml")
+
+    benchmark_names_by_category = Hash.new { |hash, key| hash[key] = [] }
+    benchmarks_yml.each do |name, metadata|
+      category = metadata.fetch('category') do
+        # Ractor scaling benchmarks are intentionally kept in their own section.
+        metadata['default_harness'] == 'harness-ractor' ? 'ractor-scaling' : 'other'
+      end
+      benchmark_names_by_category[category] << name
+    end
+
+    benchmark_names_by_category.each do |category, benchmark_names|
+      assert_equal format_benchmark_names(benchmark_names.sort), format_benchmark_names(benchmark_names),
+        "#{category} benchmarks should be sorted alphabetically"
+    end
+  end
+
+  def format_benchmark_names(benchmark_names)
+    "[\n#{benchmark_names.map { |name| "  #{name.inspect}," }.join("\n")}\n]\n"
+  end
 end