diff --git a/Cargo.lock b/Cargo.lock index 28f21f5..c16fa79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,19 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] +name = "bench-suite" +version = "0.1.0" +dependencies = [ + "component-test-sdk", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -227,6 +240,7 @@ dependencies = [ "tokio", "wasmtime", "wasmtime-wasi", + "wasmtime-wizer", ] [[package]] @@ -1032,6 +1046,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128" version = "0.2.7" @@ -1083,6 +1103,39 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "mach2" version = "0.6.0" @@ -1326,6 +1379,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "runner-cli" version = "0.1.0" @@ -1906,6 +1965,18 @@ dependencies = [ "wasmparser 0.254.0", ] +[[package]] +name = "wasm-wave" +version = "0.252.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054d19e2b53e3bf158deac6e8cc003d9eaf3ec7ecaef579988ef7b5c666f9546" +dependencies = [ + "anyhow", + "logos", + "thiserror 2.0.19", + "wit-parser 0.252.0", +] + [[package]] name = "wasmparser" version = "0.243.0" @@ -2002,6 +2073,7 @@ dependencies = [ "tempfile", "wasm-compose", "wasm-encoder 0.252.0", + "wasm-wave", "wasmparser 0.252.0", "wasmtime-environ", "wasmtime-internal-cache", @@ -2245,6 +2317,19 @@ dependencies = [ "wasmtime", ] +[[package]] +name = "wasmtime-wizer" +version = "47.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc5f1c0bf98aa22e4551cb81ff035d8103ec473ce663d441e7c485b45818a7f" +dependencies = [ + "log", + "rayon", + "wasm-encoder 0.252.0", + "wasmparser 0.252.0", + "wasmtime", +] + [[package]] name = "wast" version = "35.0.2" diff --git a/Cargo.toml b/Cargo.toml index d0f027a..15eb1a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/component-test-sdk-macro", "crates/component-test-cli", "crates/component-test-runner", + "components/bench-suite", "components/drift-fixture", "components/fixture-suite", "components/hang-fixture", diff --git a/components/bench-suite/Cargo.toml b/components/bench-suite/Cargo.toml new file mode 100644 index 0000000..24ffda2 --- /dev/null +++ b/components/bench-suite/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "bench-suite" +version.workspace = true +edition.workspace = true + +[lib] +crate-type = ["cdylib"] + +[dependencies] +component-test-sdk = { workspace = true } diff --git a/components/bench-suite/src/lib.rs b/components/bench-suite/src/lib.rs new file mode 100644 index 0000000..09fe613 --- /dev/null +++ b/components/bench-suite/src/lib.rs @@ -0,0 +1,33 @@ +//! Synthetic benchmark suite: N trivially-passing cases, no corpus, no +//! per-case data — the pure cost of registering, minting, and lifting +//! `test-case` handles. Built for measuring `all()` overhead at scale +//! (issues #22/#25 territory: instance-per-case pays this per instance). +//! +//! `BENCH_CASES` (wasi:cli env) sets the case count, default 10_000. +//! Bench drivers vary it per instance; under a plain runner (which +//! passes no env) the suite is deterministic at the default, but this +//! is a measurement fixture, not a lockfile citizen — don't lock it. + +#[component_test_sdk::suite] +mod bench { + use component_test_sdk::{ArcStr, Registry, Tags}; + + /// Registers `bench/mint/c00000`..`c`: leaf-only allocation, + /// zero-capture bodies, no tags. The registry build cost this loop + /// represents is measured separately from the mint+lift (the + /// drivers time a second `all()` call, which reuses the registry). + #[case_row(prefix = "mint")] + fn mint(reg: &mut Registry, prefix: &ArcStr, tags: &Tags) { + let n: usize = std::env::var("BENCH_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10_000); + for i in 0..n { + reg.generated( + prefix, + tags, + Case::new(format!("c{i:05}"), |_ctx| Box::pin(async { Ok(()) })), + ); + } + } +} diff --git a/crates/component-test-runner/Cargo.toml b/crates/component-test-runner/Cargo.toml index f2231da..edf087d 100644 --- a/crates/component-test-runner/Cargo.toml +++ b/crates/component-test-runner/Cargo.toml @@ -17,10 +17,25 @@ futures = { workspace = true } tokio = { workspace = true, features = ["time"] } wasmtime = { version = "47", features = ["component-model-async"] } wasmtime-wasi = "47" +# For the feature-gated `wizer-preinit` bin only (component-level +# pre-initialization experiments, #25 / findings.md #22). +wasmtime-wizer = { version = "47", features = ["wasmtime", "component-model"], optional = true } + +[features] +wizer = ["dep:wasmtime-wizer"] [[bin]] name = "ct-runner" path = "src/bin/ct-runner.rs" +[[bin]] +name = "bench-mint" +path = "src/bin/bench-mint.rs" + +[[bin]] +name = "wizer-preinit" +path = "src/bin/wizer-preinit.rs" +required-features = ["wizer"] + [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/component-test-runner/src/bin/bench-mint.rs b/crates/component-test-runner/src/bin/bench-mint.rs new file mode 100644 index 0000000..6f2a435 --- /dev/null +++ b/crates/component-test-runner/src/bin/bench-mint.rs @@ -0,0 +1,281 @@ +//! Synthetic handle-mint benchmark (`bench-mint`): measures the +//! per-fresh-instance cost of the `tests.all()` protocol at suite +//! scale, phase by phase, against a suite whose case count is set via +//! the `BENCH_CASES` env import (see `components/bench-suite`). +//! +//! Phases, timed per fresh store+instance: +//! instantiate store creation + linker instantiation +//! all#1 first `all()`: guest registry build + mint + lift +//! all#2 second `all()`: mint + lift only (registry cached) +//! name[0] one `test-case.name` boundary call (string lift) +//! run[0] one trivial case execution (borrowed context) +//! drop store teardown (frees 2N lifted handles + guest heap) +//! +//! Engine configuration mirrors the production `Runner` (pooling +//! allocator, CoW images, epoch instrumentation compiled in, untyped +//! `Val` calls), so numbers transfer to the real instance-per-case +//! path. Usage: +//! +//! cargo run --release -p component-test-runner --bin bench-mint -- \ +//! target/wasm32-wasip2/release/bench_suite.wasm \ +//! [--cases 100,1000,10000] [--instances 20] + +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use anyhow::{anyhow, bail, Context as _, Result}; +use wasmtime::component::{Component, Func, Instance, Linker, Resource, ResourceType, Val}; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; + +const TESTS_INSTANCE: &str = "polymorph:test/tests@0.1.0"; +const CONTEXT_INSTANCE: &str = "polymorph:test/test-context@0.1.0"; + +struct BenchCtx { + wasi: WasiCtx, + table: ResourceTable, +} + +impl WasiView for BenchCtx { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +/// Unit host rep for `test-context.context`; diagnostics are discarded +/// (the bench's cases emit none). +struct HostContext; + +#[derive(Default, Clone)] +struct Phases { + instantiate: Duration, + all1: Duration, + all2: Duration, + name0: Duration, + run0: Duration, + drop: Duration, +} + +fn main() -> Result<()> { + let mut suite: Option = None; + let mut cases: Vec = vec![100, 1000, 10000]; + let mut instances: usize = 20; + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--cases" => { + let list = args.next().ok_or_else(|| anyhow!("--cases needs a list"))?; + cases = list + .split(',') + .map(|s| s.parse().context("--cases: invalid count")) + .collect::>()?; + } + "--instances" => { + instances = args + .next() + .ok_or_else(|| anyhow!("--instances needs a number"))? + .parse() + .context("--instances: invalid number")?; + } + s if s.starts_with('-') => bail!("unknown flag `{s}`"), + _ if suite.is_none() => suite = Some(PathBuf::from(arg)), + _ => bail!("unexpected argument `{arg}`"), + } + } + let suite = suite.ok_or_else(|| { + anyhow!("usage: bench-mint [--cases N,N,...] [--instances M]") + })?; + + // Engine config: byte-for-byte the production Runner's choices. + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + config.epoch_interruption(true); + let mut pool = wasmtime::PoolingAllocationConfig::new(); + pool.total_memories(64) + .total_tables(64) + .total_core_instances(64) + .total_component_instances(32) + .max_memory_size(1 << 30) + .max_component_instance_size(1 << 20); + config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pool)); + let engine = Engine::new(&config)?; + + let wasm = std::fs::read(&suite)?; + let t = Instant::now(); + let component = Component::new(&engine, &wasm)?; + eprintln!("compile: {:?} ({} bytes)", t.elapsed(), wasm.len()); + + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; + let mut ctx_instance = linker.instance(CONTEXT_INSTANCE)?; + ctx_instance.resource( + "context", + ResourceType::host::(), + |_, _| Ok(()), + )?; + ctx_instance.func_wrap_concurrent( + "[method]context.diagnostic", + |_accessor, (_this, _msg): (Resource, String)| Box::pin(async move { Ok(()) }), + )?; + + println!( + "{:>6} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>10}", + "cases", "instantiate", "all#1", "all#2", "name[0]", "run[0]", "drop", "ns/handle" + ); + for n in cases { + let samples = + wasmtime_wasi::runtime::in_tokio(bench_n(&engine, &component, &linker, n, instances))?; + report(n, &samples); + } + Ok(()) +} + +async fn bench_n( + engine: &Engine, + component: &Component, + linker: &Linker, + n: usize, + instances: usize, +) -> Result> { + let warmup = 3.min(instances); + let mut samples = Vec::with_capacity(instances); + for i in 0..instances + warmup { + let mut p = Phases::default(); + + let t = Instant::now(); + let mut store = Store::new( + engine, + BenchCtx { + wasi: WasiCtxBuilder::new() + .inherit_stderr() + .env("BENCH_CASES", n.to_string()) + .build(), + table: ResourceTable::new(), + }, + ); + store.set_epoch_deadline(1); // parity: checks compiled in, never trips + let instance = linker.instantiate_async(&mut store, component).await?; + p.instantiate = t.elapsed(); + + let funcs = TestsFuncs::new(&mut store, &instance)?; + + let t = Instant::now(); + let cases1 = funcs.all(&mut store).await?; + p.all1 = t.elapsed(); + if cases1.len() != n { + bail!("suite minted {} cases, expected {n}", cases1.len()); + } + + let t = Instant::now(); + let cases2 = funcs.all(&mut store).await?; + p.all2 = t.elapsed(); + + let t = Instant::now(); + let name = funcs.name(&mut store, &cases1[0]).await?; + p.name0 = t.elapsed(); + if !name.starts_with("bench/mint/") { + bail!("unexpected case name `{name}`"); + } + + let ctx = Resource::::new_own(1); + let ctx_any = ctx.try_into_resource_any(&mut store)?; + let t = Instant::now(); + let mut results = [Val::Bool(false)]; + funcs + .run + .call_async( + &mut store, + &[cases1[0].clone(), Val::Resource(ctx_any)], + &mut results, + ) + .await?; + p.run0 = t.elapsed(); + if !matches!(&results[0], Val::Result(Ok(_))) { + bail!("bench case did not pass: {:?}", results[0]); + } + + drop(cases1); + drop(cases2); + let t = Instant::now(); + drop(store); + p.drop = t.elapsed(); + + if i >= warmup { + samples.push(p); + } + } + Ok(samples) +} + +fn report(n: usize, samples: &[Phases]) { + let med = |f: fn(&Phases) -> Duration| -> Duration { + let mut v: Vec<_> = samples.iter().map(f).collect(); + v.sort(); + v[v.len() / 2] + }; + let all2 = med(|p| p.all2); + println!( + "{:>6} {:>12?} {:>12?} {:>12?} {:>12?} {:>12?} {:>12?} {:>10.0}", + n, + med(|p| p.instantiate), + med(|p| p.all1), + all2, + med(|p| p.name0), + med(|p| p.run0), + med(|p| p.drop), + all2.as_nanos() as f64 / n as f64, + ); +} + +/// The suite's `tests` export surface via the untyped `Val` API — the +/// same calls the production runner makes. +struct TestsFuncs { + all: Func, + name: Func, + run: Func, +} + +impl TestsFuncs { + fn new(store: &mut Store, instance: &Instance) -> Result { + let (_, tests) = instance + .get_export(&mut *store, None, TESTS_INSTANCE) + .ok_or_else(|| anyhow!("suite does not export `{TESTS_INSTANCE}`"))?; + let lookup = |store: &mut Store, name: &str| -> Result { + let (_, idx) = instance + .get_export(&mut *store, Some(&tests), name) + .ok_or_else(|| anyhow!("`{TESTS_INSTANCE}` does not export `{name}`"))?; + instance + .get_func(&mut *store, idx) + .ok_or_else(|| anyhow!("`{name}` is not a function")) + }; + Ok(Self { + all: lookup(store, "all")?, + name: lookup(store, "[method]test-case.name")?, + run: lookup(store, "[method]test-case.run")?, + }) + } + + async fn all(&self, store: &mut Store) -> Result> { + let mut results = [Val::Bool(false)]; + self.all.call_async(&mut *store, &[], &mut results).await?; + match results.into_iter().next().unwrap() { + Val::List(cases) => Ok(cases), + other => bail!("unexpected tests.all result: {other:?}"), + } + } + + async fn name(&self, store: &mut Store, case: &Val) -> Result { + let mut results = [Val::Bool(false)]; + self.name + .call_async(&mut *store, std::slice::from_ref(case), &mut results) + .await?; + match results.into_iter().next().unwrap() { + Val::String(s) => Ok(s), + other => bail!("unexpected test-case.name result: {other:?}"), + } + } +} diff --git a/crates/component-test-runner/src/bin/wizer-preinit.rs b/crates/component-test-runner/src/bin/wizer-preinit.rs new file mode 100644 index 0000000..415dfe6 --- /dev/null +++ b/crates/component-test-runner/src/bin/wizer-preinit.rs @@ -0,0 +1,118 @@ +//! Component-level wizer pre-initialization driver (`wizer-preinit`, +//! feature `wizer`): snapshots a suite component after running its own +//! `tests.all()`, so every fresh instance is born with the case table +//! built and `all()` costs only mint+lift (#25). +//! +//! Works on any unmodified suite artifact: the init function is the +//! contract's own `all()` — named in the version-last invoke syntax +//! `polymorph:test/tests.all@0.1.0()` (the parenthesized wave-call +//! form; the bare item-name form requires a `[] -> []` signature). +//! The handles `all()` returns are per-call state; only the guest heap +//! (the built registry) lands in the snapshot, at no measurable size +//! cost over a dedicated no-op init export (findings.md #22). +//! +//! Exists because the `wasmtime wizer` CLI cannot wizen a *suite*: +//! the suite world imports `test-context`, whose `context` resource +//! cannot be synthesized by unknown-import stubbing ("resource +//! implementation is missing"), and composed bundles hit "nested +//! components with modules not currently supported". Driving +//! wasmtime-wizer as a library with our own linker — WASI plus a host +//! `context` resource whose methods init never calls — sidesteps both. +//! +//! cargo run --release -p component-test-runner --features wizer \ +//! --bin wizer-preinit -- [ENV=VAL ...] + +use wasmtime::component::{Linker, Resource, ResourceType}; +use wasmtime::error::format_err; +use wasmtime::{Config, Engine, Result, Store}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; + +struct Ctx { + wasi: WasiCtx, + table: ResourceTable, +} + +impl WasiView for Ctx { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +struct HostContext; + +fn main() -> Result<()> { + let mut args = std::env::args().skip(1); + let usage = "usage: wizer-preinit [ENV=VAL ...]"; + let input = args.next().ok_or_else(|| format_err!("{usage}"))?; + let output = args.next().ok_or_else(|| format_err!("{usage}"))?; + let env: Vec<(String, String)> = args + .map(|kv| { + kv.split_once('=') + .map(|(k, v)| (k.to_string(), v.to_string())) + .ok_or_else(|| format_err!("bad env pair `{kv}` (want NAME=VAL)\n{usage}")) + }) + .collect::>()?; + + let wasm = std::fs::read(&input)?; + + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + let engine = Engine::new(&config)?; + + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; + let mut ctx_instance = linker.instance("polymorph:test/test-context@0.1.0")?; + ctx_instance.resource( + "context", + ResourceType::host::(), + |_, _| Ok(()), + )?; + // Present so the component type-checks; never called during init + // (registry builds are pure — anything else is a suite bug this + // driver would surface as a snapshot-time diagnostic call). + ctx_instance.func_wrap_concurrent( + "[method]context.diagnostic", + |_accessor, (_this, _msg): (Resource, String)| Box::pin(async move { Ok(()) }), + )?; + + let mut wasi = WasiCtxBuilder::new(); + wasi.inherit_stderr(); + for (k, v) in &env { + wasi.env(k, v); + } + let mut store = Store::new( + &engine, + Ctx { + wasi: wasi.build(), + table: ResourceTable::new(), + }, + ); + + let mut wizer = wasmtime_wizer::Wizer::new(); + // The contract's own enumeration is the init function (version-last + // invoke syntax; the parens select the wave-call path, which + // permits results). + wizer.init_func("polymorph:test/tests.all@0.1.0()"); + // Mandatory: "stripping" the init function would remove the + // `tests.all` export itself. (Stripping a dedicated init export is + // also currently broken upstream — dangling core-instance export + // reference, bytecodealliance/wasmtime#13168.) + wizer.keep_init_func(true); + let (wizened, _rets) = wasmtime_wasi::runtime::in_tokio(wizer.run_component( + &mut store, + &wasm, + async |store: &mut Store, component| linker.instantiate_async(store, component).await, + ))?; + + std::fs::write(&output, &wizened)?; + eprintln!( + "wizened: {} -> {} bytes ({output})", + wasm.len(), + wizened.len(), + ); + Ok(()) +} diff --git a/docs/findings.md b/docs/findings.md index cf2d7a1..18b7f67 100644 --- a/docs/findings.md +++ b/docs/findings.md @@ -129,3 +129,103 @@ Rust `wasm32-wasip2` target. WASI sleep: `std::thread::sleep` on wasip2 suspends in the host's async `poll`, no wasm executing). Hence `hang/wedge` sleeps via WASI rather than awaiting `pending()`. + +## Handle mint/lift at scale (bench-suite; #22/#25 context) + +Synthetic measurement of the `all()` protocol per fresh instance — +N trivially-passing cases, no corpus, no per-case data +(`components/bench-suite`, count via `BENCH_CASES` env). Drivers: +`bench-mint` bin (wasmtime, production `Runner` config: pooling, CoW, +epoch instrumentation, untyped `Val` calls) and +`js/runner-deltic/bench-mint.mjs` (pinned deltic embedder, plain +Node). Medians over 20/10 fresh instances, one dev box (17-core +x86_64 Linux), wasmtime 47.0.3 / deltic pre-83fff30 / Node 24. + +19. **wasmtime: `all()` splits ~3:1 registry-build : mint+lift, both + linear.** At 10k cases: all#1 (build + mint + lift) 3.2ms, all#2 + (mint + lift only, registry cached) 0.79ms ≈ **75–80ns/handle**; + registry build (the OnceCell `IndexMap` + boxed-closure loop) ≈ + 240ns/case. Instantiate 21µs; store drop 83µs at 10k (scales with + lifted-handle count); `name`/`run` boundary calls 2–3µs each. + Instance-per-case on a 10k suite therefore pays ~3.3ms/case of + pure protocol overhead (matches #22's campaign arithmetic), and + the guest-side registry build — not the handle lift — is the + larger share, so SDK-side table work (static case table, #25 + wizer) buys more than lift avoidance alone; a direct-access + interface caps out at ~25% unless stacked on a lazy/static + registry. +20. **deltic (runtime linker, callback ABI): same shape, bigger + constants.** Instantiate ~650µs (~30× wasmtime); mint+lift + ~340–370ns/handle (~4.3×; mildly superlinear by 30k — V8 GC on + the wrapper objects); per-call boundary overhead ~25µs (~10×); + all#1 at 10k ≈ 5.2ms. Translator init + component translate are + one-off ~16ms + ~25ms. Per-fresh-instance topologies are + tolerable here only while per-instance work stays O(cases + served), not O(suite). +21. **`harness.mjs` fresh-instance relocation was the real JS-leg + quadratic** (fixed — positional relocation, PR #83): `freshCases` + re-found each case by a linear `name()` scan. Hot-loop `name()` + costs ~3.4µs under deltic (a cold single call measures ~26µs — + promise/JIT overhead that amortizes), so the scan averaged ~N/2 × + 3.4µs ≈ **17ms per case** at 10k (33ms worst, measured), a + multiple of the `all()` re-enumeration it followed and O(N²) + across a run. The contract guarantees `all()` order is + deterministic across instances, so positional relocation (index + + one `name()` verify) is sound and erases it: 33.4ms → 0.0ms for + the last case at 10k. + +## Component-level wizer pre-initialization (#25) + +Follow-up to the bench above: pre-build the registry at build time so +fresh instances are born initialized. `wasmtime-wizer` 47 as a +library; drivers: `wizer-preinit` bin (feature `wizer`) over the +bench-suite artifact built with its `wizer-init` feature. + +22. **Component-level pre-init works today (wasmtime-wizer 47) — #25's + "core-module level only" constraint is stale — and the suite needs + no init export: the contract's own `all()` is the init function.** + Named in the *version-last* invoke syntax, + `polymorph:test/tests.all@0.1.0()` — the wave/`ItemName` grammar + places `@version` after the item name to resolve the dot + ambiguity (`pkg:ns/iface.func@1.2.3`), and rejects the + export-name-order form `pkg:ns/iface@1.2.3.func` by design (the + resulting "invalid token" error points at the run-command docs + and hints at neither). The parenthesized wave-call form is + required: the bare item-name path demands a `[] -> []` signature, + and `all` has a result. The returned handles are per-call state; + only the built registry lands in the snapshot (measured: same + size as via a dedicated no-op init export, ±3KB). Wizening a + *suite* still needs wasmtime-wizer as a library — + `Wizer::run_component` takes a caller-supplied instantiate + closure, so a custom linker satisfies `test-context` (a host + resource whose methods init never calls) plus full WASI (env + reads during init work) — because the CLI cannot: unknown-import + stubbing cannot synthesize **resource** types ("resource + implementation is missing"), composed bundles fail ("nested + components with modules not currently supported"), and the CLI + additionally defaults WASI *off* for wizening (`-S cli` + required for env-reading inits). Two upstream edges: + `keep_init_func(false)` — moot here (stripping would remove + `tests.all`) — emits an invalid component for dedicated init + exports (dangling core-instance export reference; known, + bytecodealliance/wasmtime#13168); and the wave func-name lexer's + semver subpattern is bare `X.Y.Z`, so prerelease-versioned + interfaces (`@0.3.0-rc-…`) cannot be named in call form. Custom + sections **survive** the rewrite (tags inventory intact — the + runner's scheduling and drift checks work on the wizened + artifact, unlike wac-composed bundles, finding 14). +23. **wasmtime, wizened 10k suite: the registry-build half vanishes + exactly.** all#1 3.15ms → 663µs ≈ all#2; instantiate unchanged + (~19µs — CoW absorbs the 1.29MB snapshot, 122KB → 1.29MB); store + drop 80µs → 12µs (fewer runtime-dirtied pages). End-to-end K=1 + full-isolation run: 30.8s → 7.1s sequential (4.3×), **1.14s at + jobs=8** — per-case isolation on a wizened suite undercuts the + shared-instance numbers that motivated relaxing isolation in #22. +24. **deltic, wizened suite: net ~1.5× only.** all#1 6.9ms → 2.9ms, + but instantiate 0.78ms → 2.17ms: V8 has no CoW memory images, so + the 1.29MB active data segment is copied at every instantiation, + eating most of the build win (translate also 24ms → 44ms, + one-off). Wizening pays on JS legs only when enumeration is + genuinely expensive; instance-granularity K>1 remains the lever + there. + diff --git a/js/runner-deltic/bench-mint.mjs b/js/runner-deltic/bench-mint.mjs new file mode 100644 index 0000000..66c1c7a --- /dev/null +++ b/js/runner-deltic/bench-mint.mjs @@ -0,0 +1,142 @@ +// Synthetic handle-mint benchmark for the deltic leg — the runtime-linked +// sibling of crates/component-test-runner/src/bin/bench-mint.rs, measuring +// the same phases per fresh instance under the pinned deltic embedder on +// plain Node (callback ABI, no engine flags): +// +// instantiate deltic.instantiate (runtime link + wasm instantiation) +// all#1 first all(): guest registry build + mint + lift (N wrappers) +// all#2 second all(): mint + lift only (guest registry cached) +// name[0] one test-case.name boundary call +// run[0] one trivial case execution (borrowed context) +// +// No teardown phase: deltic instances are GC-reclaimed, there is no +// dispose surface. Case count rides the BENCH_CASES wasi env import +// (see components/bench-suite). +// +// node js/runner-deltic/bench-mint.mjs \ +// \ +// [--cases 100,1000,10000] [--instances 10] + +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const args = process.argv.slice(2); +const positional = []; +let caseCounts = [100, 1000, 10000]; +let instances = 10; +for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case "--cases": + caseCounts = args[++i].split(",").map((s) => Number(s)); + break; + case "--instances": + instances = Number(args[++i]); + break; + default: + if (args[i].startsWith("--")) throw new Error(`unknown flag '${args[i]}'`); + positional.push(args[i]); + } +} +const [bundlePath, translatorPath, suitePath] = positional; +if (!suitePath) { + console.error( + "usage: node bench-mint.mjs " + + " [--cases N,N,...] [--instances M]", + ); + process.exit(2); +} + +const deltic = await import(pathToFileURL(bundlePath).href); +const suiteBytes = new Uint8Array(readFileSync(suitePath)); + +let t = performance.now(); +const translator = await deltic.Translator.create( + new Uint8Array(readFileSync(translatorPath)), +); +const translatorMs = performance.now() - t; + +t = performance.now(); +const { plan, adapters } = translator.translate(suiteBytes); +const translateMs = performance.now() - t; +console.error( + `translator init: ${translatorMs.toFixed(1)}ms ` + + `translate: ${translateMs.toFixed(1)}ms (${suiteBytes.length} bytes)`, +); + +const artifacts = { plan, componentBytes: suiteBytes, adapters }; +const TESTS = "polymorph:test/tests@0.1.0"; + +function median(xs) { + const v = [...xs].sort((a, b) => a - b); + return v[Math.floor(v.length / 2)]; +} + +const fmt = (ms) => (ms >= 1 ? `${ms.toFixed(2)}ms` : `${(ms * 1000).toFixed(1)}µs`); + +console.log( + "cases".padStart(6) + + ["instantiate", "all#1", "all#2", "name[0]", "run[0]", "ns/handle"] + .map((h) => h.padStart(12)) + .join(""), +); + +for (const n of caseCounts) { + const imports = { + ...deltic.wasiShims({ cli: { env: { BENCH_CASES: String(n) } } }), + ...deltic.testContextImportRecord(), + }; + const warmup = Math.min(3, instances); + const samples = { instantiate: [], all1: [], all2: [], name0: [], run0: [] }; + for (let i = 0; i < instances + warmup; i++) { + let t = performance.now(); + const inst = await deltic.instantiate(artifacts, imports); + const instantiate = performance.now() - t; + const tests = inst.exports[TESTS] ?? inst.exports["tests"]; + + t = performance.now(); + const cases1 = await tests.all(); + const all1 = performance.now() - t; + if (cases1.length !== n) { + throw new Error(`suite minted ${cases1.length} cases, expected ${n}`); + } + + t = performance.now(); + const cases2 = await tests.all(); + const all2 = performance.now() - t; + if (cases2.length !== n) throw new Error("second all() disagreed"); + + t = performance.now(); + const name = String(await cases1[0].name()); + const name0 = performance.now() - t; + if (!name.startsWith("bench/mint/")) { + throw new Error(`unexpected case name '${name}'`); + } + + const ctx = new deltic.Context(() => {}); + t = performance.now(); + await cases1[0].run(ctx); // resolves = pass; throws = fail/trap + const run0 = performance.now() - t; + + if (i >= warmup) { + samples.instantiate.push(instantiate); + samples.all1.push(all1); + samples.all2.push(all2); + samples.name0.push(name0); + samples.run0.push(run0); + } + } + const all2 = median(samples.all2); + console.log( + String(n).padStart(6) + + [ + fmt(median(samples.instantiate)), + fmt(median(samples.all1)), + fmt(all2), + fmt(median(samples.name0)), + fmt(median(samples.run0)), + ((all2 * 1e6) / n).toFixed(0), + ] + .map((s) => s.padStart(12)) + .join(""), + ); +}