Skip to content

Unify memcpy, memmove, memset and commit on one accelerator #5295

Unify memcpy, memmove, memset and commit on one accelerator

Unify memcpy, memmove, memset and commit on one accelerator #5295

Workflow file for this run

name: Integration Tests
on:
push:
branches: ["main"]
merge_group:
pull_request:
branches: ["**"]
paths-ignore: ["spec/**"]
permissions:
contents: read
actions: write
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
lint:
# "Lint" is a required check, don't change the name
name: Lint
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-lint"
cache-all-crates: "true"
- name: Run lint checks
run: make lint
- name: Check ethrex fixture checksums
run: make check-ethrex-fixture-checksums
# Five manifests carry the ethrex rev and they must agree: a skew is not a build
# failure, it is a fixture the guest decodes as the all-zero default. `--show`
# exits non-zero on MIXED. It reads manifests only; the matching locks are held by
# `--locked` on each workspace's own test/build step.
- name: Check the ethrex rev is consistent across manifests
run: scripts/set_ethrex_rev.sh --show
# The benchmark comment harness: verifies every steps.X.outputs.Y consumed in
# benchmark-pr.yml has a producer (the orphaned-wiring class that once broke
# /bench-growth silently), then renders the full comment scenario suite.
# Pure node, no dependencies, ~10 s. Lives in Lint so it runs on every PR —
# a path filter would have to enumerate every file that can break the wiring.
- name: Check benchmark comment wiring
run: node scripts/render_bench_comment.js
test-executor:
name: Executor tests
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-test"
cache-all-crates: "true"
# ethrex-tests is a detached workspace (own Cargo.lock, own target dir), so
# the default `. -> target` misses it and its release tree would rebuild
# from scratch every run. `cache-all-crates` only covers the registry, not
# compiled artifacts.
#
# ethrex-block-converter is deliberately NOT here: this job no longer builds it.
# That workspace is the expensive one (~335 packages, including the blst,
# c-kzg and secp256k1-sys C builds, malachite and ark-ff/asm) and it now
# builds only in ethrex-block-converter.yml, which carries its own cache entry.
workspaces: |
. -> target
tooling/ethrex-tests -> target
- name: Cache compiled ASM ELF artifacts
id: cache-asm-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/asm
key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }}
- name: Compile ASM programs to ELF
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-asm
- name: Cache compiled Rust ELF artifacts and build cache
id: cache-rust-elfs
uses: actions/cache@v4
with:
path: |
executor/program_artifacts/rust
executor/shared_target
key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }}
restore-keys: |
rust-elf-artifacts-
- name: Compile Rust programs to ELF
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-rust
- name: Run asm tests (no compile)
run: |
cargo test --release -p executor --test asm
- name: Run rust tests (no compile)
run: |
cargo test --release -p executor --test rust
- name: Run flamegraph tests
run: |
cargo test --release -p executor --test flamegraph
# The unit tests under `executor/src/tests/` are a *lib* target (`pub mod tests;`
# in lib.rs), which none of the `--test <name>` steps above select — and the
# `test_ckzg` step below filters by name, so it doesn't run them either. Without
# this step they never run in CI. It shares the lib test binary with that step,
# so it costs a test run, not an extra compile.
- name: Run executor lib unit tests
run: |
cargo test --release -p executor --lib
- name: Run ignored executor tests
run: |
cargo test --release -p executor test_ckzg -- --ignored
# ethrex host-reference tests live in the detached `tooling/ethrex-tests`
# workspace (ethrex pins rkyv's `unaligned` feature, which must not
# feature-unify with the main workspace's aligned proof format), so run
# them from that directory to use its isolated Cargo.lock. The guest ELF
# and committed fixtures are already present from the steps above.
# --include-ignored also runs the heavier synthetic-block test.
#
# `--skip test_ethrex_real_block` is a substring match, so it drops BOTH
# real-block tests (the `_vm` and `_native` ones) and with them this job's need
# for the real-block fixture. That fixture is a benchmark input: it has to be
# right once, not on every PR, and fetching plus screening it here put a network
# download and a ~335-package build in the required gate for an artifact with no
# bearing on correctness. Both moved to ethrex-block-converter.yml, which runs when
# the converter or the block pin actually changes. `no_kzg_backend_linked` still
# runs here — it is a pure unit test, and it is the property the real-block
# screen over there depends on.
#
# This is `make test-ethrex-offline` spelled out, matching the other test steps
# in this job: they all bypass make so its dependency check cannot decide the
# cache-restored ELFs are stale and trigger a rebuild.
- name: Run ethrex host-reference tests (detached workspace)
run: |
cd tooling/ethrex-tests && \
cargo test --locked --release -- --include-ignored --skip test_ethrex_real_block
test-cli:
name: CLI tests
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-cli-test"
cache-all-crates: "true"
- name: Run CLI tests
run: cargo test -p cli
- name: Run syscalls host tests (allocator + keccak differential vs sha3)
run: make test-syscalls
# The dlmalloc fallback is feature-selected, so nothing else in CI compiles it and it
# can rot silently. Its tests run here too.
- name: Test the dlmalloc guest allocator fallback
run: |
cd syscalls
cargo test --features dlmalloc-alloc
cargo test --release --features dlmalloc-alloc
- name: Run ethrex-crypto host tests (hint verify-then-fallback + ecrecover)
run: make test-ethrex-crypto
# "Test" is a required check — keep this name to avoid branch protection changes.
# This gate job passes only when CLI, executor, disk-spill, and prover tests succeed.
test:
name: Test
if: always()
needs: [test-executor, test-cli, test-prover, test-disk-spill, test-stark-cuda-lib]
runs-on: ubuntu-latest
steps:
- name: Check results
run: |
executor="${{ needs.test-executor.result }}"
cli="${{ needs.test-cli.result }}"
prover="${{ needs.test-prover.result }}"
disk_spill="${{ needs.test-disk-spill.result }}"
stark_cuda_lib="${{ needs.test-stark-cuda-lib.result }}"
echo "test-executor: $executor"
echo "test-cli: $cli"
echo "test-prover: $prover"
echo "test-disk-spill: $disk_spill"
echo "test-stark-cuda-lib: $stark_cuda_lib"
# Allow "success" or "skipped" (skipped on merge queue pushes)
if [[ "$executor" != "success" && "$executor" != "skipped" ]]; then
exit 1
fi
if [[ "$cli" != "success" && "$cli" != "skipped" ]]; then
exit 1
fi
if [[ "$prover" != "success" && "$prover" != "skipped" ]]; then
exit 1
fi
if [[ "$disk_spill" != "success" && "$disk_spill" != "skipped" ]]; then
exit 1
fi
if [[ "$stark_cuda_lib" != "success" && "$stark_cuda_lib" != "skipped" ]]; then
exit 1
fi
test-disk-spill:
name: Disk-spill tests
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-disk-spill"
cache-all-crates: "true"
- name: Cache compiled ASM ELF artifacts
id: cache-asm-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/asm
key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }}
- name: Install clang and lld
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: sudo apt-get update && sudo apt-get install -y clang lld
- name: Compile ASM programs to ELF
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-asm
- name: Cache compiled Rust ELF artifacts and build cache
id: cache-rust-elfs
uses: actions/cache@v4
with:
path: |
executor/program_artifacts/rust
executor/shared_target
key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }}
restore-keys: |
rust-elf-artifacts-
- name: Compile Rust programs to ELF
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-rust
- name: Run stark disk-spill tests
run: |
cargo test --release -p stark --features disk-spill disk_spill
- name: Run prover disk-spill tests
env:
FORCE_DISK_SPILL: "1"
run: |
cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths
test-stark-cuda-lib:
name: Stark cuda-feature lib tests
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-stark-cuda-lib"
cache-all-crates: "true"
# The cuda feature gates the logup_gpu module, whose descriptor /
# CPU-mirror parity tests are pure CPU (they never touch the driver).
# Without nvcc the kernels build as empty PTX stubs, so these run on a
# plain runner; GPU-dependent tests are #[ignore] and stay skipped.
# Scoped to logup_gpu: the rest of the cuda-feature lib suite dispatches
# to the GPU and needs the CUDA driver.
- name: Run stark logup_gpu parity tests (no GPU required)
run: |
cargo test --release -p stark --features cuda --lib logup_gpu
build-prover-tests:
name: Build prover tests
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.actor != 'github-merge-queue[bot]'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Rust Environment
uses: ./.github/actions/setup-rust
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@v2
with:
shared-key: "lambda-vm-prover-test"
cache-all-crates: "true"
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: Build and archive prover + crypto tests
run: |
cargo nextest archive --release \
-p lambda-vm-prover -p stark -p crypto -p ecsm \
--archive-file prover-tests.tar.zst
- name: Upload test archive
uses: actions/upload-artifact@v4
with:
name: prover-tests
path: prover-tests.tar.zst
retention-days: 1
test-prover:
name: Prover tests (${{ matrix.partition }}/4)
needs: build-prover-tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
partition: [1, 2, 3, 4]
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Cache compiled ASM ELF artifacts
id: cache-asm-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/asm
key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }}
- name: Install clang and lld
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: sudo apt-get update && sudo apt-get install -y clang lld
- name: Compile ASM programs to ELF
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-asm
- name: Cache compiled Rust ELF artifacts and build cache
id: cache-rust-elfs
uses: actions/cache@v4
with:
path: |
executor/program_artifacts/rust
executor/shared_target
key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }}
restore-keys: |
rust-elf-artifacts-
- name: Setup Rust Environment
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-rust
- name: Compile Rust programs to ELF
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-rust
- name: Cache compiled recursion guest ELF artifacts
id: cache-recursion-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/recursion
key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }}
restore-keys: |
recursion-elf-artifacts-
- name: Setup Rust Environment (recursion ELFs)
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true'
uses: ./.github/actions/setup-rust
- name: Compile recursion guest ELFs
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true'
run: |
make compile-recursion-elfs
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: Download test archive
uses: actions/download-artifact@v4
with:
name: prover-tests
- name: Run prover tests (shard ${{ matrix.partition }}/4)
# Shard 1 only: force k > 1 so the per-table admission scheduler really
# runs several table closures concurrently. ubuntu-latest has 2-4 vCPU
# and table_parallelism() defaults to (cores / 3).max(1), so every
# other shard proves with a single driver thread and never exercises
# the concurrent path or VramGate's blocking path on a PR. The other
# three shards keep the default-k coverage. An empty value on those
# fails to parse and falls back to the default, so this is inert there.
env:
TABLE_PARALLELISM: ${{ matrix.partition == 1 && '6' || '' }}
run: |
cargo nextest run \
--archive-file prover-tests.tar.zst \
--partition hash:${{ matrix.partition }}/4 \
--test-threads=1
test-prover-comprehensive:
name: Prover comprehensive tests
needs: build-prover-tests
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Cache compiled ASM ELF artifacts
id: cache-asm-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/asm
key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }}
- name: Install clang and lld
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: sudo apt-get update && sudo apt-get install -y clang lld
- name: Compile ASM programs to ELF
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-asm
- name: Cache compiled Rust ELF artifacts and build cache
id: cache-rust-elfs
uses: actions/cache@v4
with:
path: |
executor/program_artifacts/rust
executor/shared_target
key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }}
restore-keys: |
rust-elf-artifacts-
- name: Setup Rust Environment
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-rust
- name: Compile Rust programs to ELF
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
run: |
make compile-programs-rust
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: Download test archive
uses: actions/download-artifact@v4
with:
name: prover-tests
- name: Cache compiled recursion guest ELF artifacts
id: cache-recursion-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/recursion
key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }}
restore-keys: |
recursion-elf-artifacts-
- name: Setup Rust Environment (recursion ELFs)
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true'
uses: ./.github/actions/setup-rust
- name: Compile recursion guest ELFs
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true'
run: |
make compile-recursion-elfs
- name: Run comprehensive prover tests
run: |
cargo nextest run \
--archive-file prover-tests.tar.zst \
--test-threads=1 \
-E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_execute)' \
--run-ignored ignored-only
# Seed ELF caches on refs/heads/main so merge-queue runs can restore them.
# GitHub Actions caches are branch-scoped: merge_group branches can only read
# caches from their base branch (main). The test jobs above are skipped on
# push-to-main (already validated by the merge queue), so without this job
# no ELF cache is ever saved on main and every merge-queue entry recompiles.
seed-elf-cache:
name: Seed ELF cache
runs-on: ubuntu-latest
if: github.event_name == 'push'
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Cache compiled ASM ELF artifacts
id: cache-asm-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/asm
key: asm-elf-artifacts-${{ hashFiles('executor/programs/asm/**') }}
- name: Install clang and lld
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: sudo apt-get update && sudo apt-get install -y clang lld
- name: Compile ASM programs to ELF
if: steps.cache-asm-elfs.outputs.cache-hit != 'true'
run: make compile-programs-asm
- name: Cache compiled Rust ELF artifacts and build cache
id: cache-rust-elfs
uses: actions/cache@v4
with:
path: |
executor/program_artifacts/rust
executor/shared_target
key: rust-elf-artifacts-${{ hashFiles('executor/programs/rust/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }}
restore-keys: |
rust-elf-artifacts-
- name: Setup Rust Environment
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-rust
- name: Compile Rust programs to ELF
if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
run: make compile-programs-rust
- name: Cache compiled recursion guest ELF artifacts
id: cache-recursion-elfs
uses: actions/cache@v4
with:
path: executor/program_artifacts/recursion
key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }}
restore-keys: |
recursion-elf-artifacts-
- name: Setup Rust Environment (recursion ELFs)
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true'
uses: ./.github/actions/setup-rust
- name: Compile recursion guest ELFs
if: steps.cache-recursion-elfs.outputs.cache-hit != 'true'
run: make compile-recursion-elfs