From 1154a882659f21f18f5f8055b77512c2921b5772 Mon Sep 17 00:00:00 2001 From: Orjan Ameye Date: Wed, 24 Jun 2026 11:49:39 +0200 Subject: [PATCH] bench: report minimum and gcsample the heavy ODE benchmarks Reduce the run-to-run and cross-runner variance that has been making the benchmark CI flag phantom regressions. Save minimum(results) instead of median(results). The minimum is BenchmarkTools' recommended tracking estimator: measurement noise (GC pauses, scheduler preemption, frequency scaling) only ever adds time, so the minimum is the most reproducible estimate of the true cost and the least sensitive to hardware differences between runners. Set gcsample=true on the two allocation-heavy ODE-solve benchmarks (Correlations two-time, Interaction Picture numerical mode evolution). They rebuild operators every ODE step and spend ~50% of their runtime in GC, which gives a ~4x run-to-run spread; running a GC before each sample brings that down to ~1.4x. The fast ns/us benchmarks are left untouched: their minimum is already stable and gcsample would only cost samples. --- benchmarks/correlations.jl | 6 +++++- benchmarks/interaction_picture.jl | 4 +++- benchmarks/runbenchmarks.jl | 12 ++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/benchmarks/correlations.jl b/benchmarks/correlations.jl index c0914862..dedbdb31 100644 --- a/benchmarks/correlations.jl +++ b/benchmarks/correlations.jl @@ -59,8 +59,12 @@ function benchmark_correlations!(SUITE) SUITE["Correlations"]["two-time"] = BenchmarkGroup() + # Allocation-heavy: a full master-equation solve that rebuilds operators every + # ODE step (~0.5 GB / call). `gcsample=true` runs a GC before each sample so the + # timing isn't dominated by non-deterministic GC pauses carried over between + # samples, which otherwise gives this benchmark a ~4x run-to-run spread. SUITE["Correlations"]["two-time"]["single photon cavity"] = - @benchmarkable correlation_matrix($T, $ρt, $input_output, $Ls) + @benchmarkable correlation_matrix($T, $ρt, $input_output, $Ls) gcsample = true return nothing end diff --git a/benchmarks/interaction_picture.jl b/benchmarks/interaction_picture.jl index 627b0091..dec20435 100644 --- a/benchmarks/interaction_picture.jl +++ b/benchmarks/interaction_picture.jl @@ -40,8 +40,10 @@ function benchmark_interaction_picture!(SUITE) SUITE["Interaction Picture"]["coefficient matrix M"] = BenchmarkGroup() + # Allocation-heavy ODE solve: GC before each sample to stabilize the timing + # (see the note in runbenchmarks.jl). SUITE["Interaction Picture"]["coefficient matrix M"]["numerical (ODE)"] = - @benchmarkable solve_mode_evolution($A_uv, $T) + @benchmarkable solve_mode_evolution($A_uv, $T) gcsample = true SUITE["Interaction Picture"]["coefficient matrix M"]["analytical (2 equal modes)"] = @benchmarkable solve_mode_evolution_symmetric($u, $T) diff --git a/benchmarks/runbenchmarks.jl b/benchmarks/runbenchmarks.jl index 58bb934b..e6912547 100644 --- a/benchmarks/runbenchmarks.jl +++ b/benchmarks/runbenchmarks.jl @@ -24,6 +24,14 @@ benchmark_correlations!(SUITE) BenchmarkTools.tune!(SUITE) results = BenchmarkTools.run(SUITE; verbose = true) -display(median(results)) -BenchmarkTools.save("benchmarks_output.json", median(results)) +# Report the minimum rather than the median. The minimum is BenchmarkTools' +# recommended estimator for tracking: measurement noise (GC pauses, scheduler +# preemption, frequency scaling) is strictly additive, so the minimum is the +# most reproducible estimate of the underlying cost and the least sensitive to +# cross-runner variance. Allocation-heavy benchmarks additionally set +# `gcsample=true` so each sample starts from a clean heap (see the individual +# `@benchmarkable`s). +display(minimum(results)) + +BenchmarkTools.save("benchmarks_output.json", minimum(results))