From 0ad3242fbddb0778f8dd3a489f5c5c59b5283f2c Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Mon, 10 Aug 2026 14:14:20 -0300 Subject: [PATCH 1/6] fix(make): build the guest ELFs the prover targets read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-prover, test-prover-all, test-prover-debug, test-prover-cuda and test-disk-spill depended on compile-recursion-elfs only, so on a clean checkout every test reading executor/program_artifacts/rust/*.elf or */asm/*.elf panicked with "elf not found". test-fast additionally runs the executor suite, which reads most of the Rust guests. compile-prover-test-elfs builds the asm guests, the recursion guests, and the eight Rust guests the prover suite reads — not all 35, since the .elf rules are FORCE and would re-enter cargo per guest on every run. check-prover-test-elfs greps the prover sources and fails if one is missing from the list, so the next test to read a new guest cannot silently reintroduce the failure; it runs from lint, which CI already invokes. --- Makefile | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index a4b05b507..36d5f649e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: deps deps-linux deps-macos compile-programs-asm compile-programs-rust compile-bench \ -compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-shared \ +compile-programs compile-recursion-elfs compile-prover-test-elfs check-prover-test-elfs \ +clean-asm clean-rust clean-bench clean-shared \ clean-recursion-elfs clean test test-asm \ test-rust test-ethrex test-ethrex-offline test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ test-profile-recursion-block recursion-profile-block-input \ @@ -48,6 +49,13 @@ RUST_PROGRAM_DIRS := $(dir $(wildcard $(RUST_PROGRAMS_DIR)/*/Cargo.toml)) RUST_PROGRAMS := $(notdir $(basename $(RUST_PROGRAM_DIRS:%/=%))) RUST_ARTIFACTS := $(addprefix $(RUST_ARTIFACTS_DIR)/, $(addsuffix .elf, $(RUST_PROGRAMS))) +# The Rust guests the prover test suite reads as prebuilt artifacts. The prover +# targets build these instead of all $(RUST_PROGRAMS), because the .elf rules are +# FORCE (see below) and would re-enter cargo once per guest on every test run. +# `check-prover-test-elfs` fails if a prover test reads one that is missing here. +PROVER_TEST_GUESTS := allocator commit_sum ecsm ef_io_demo ethrex hint_min hint_multi pure_commit +PROVER_TEST_ARTIFACTS := $(addprefix $(RUST_ARTIFACTS_DIR)/, $(addsuffix .elf, $(PROVER_TEST_GUESTS))) + BENCH_PROGRAM_DIRS := $(dir $(wildcard $(BENCH_PROGRAMS_DIR)/*/Cargo.toml)) BENCH_PROGRAMS := $(notdir $(basename $(BENCH_PROGRAM_DIRS:%/=%))) BENCH_ARTIFACTS := $(addprefix $(BENCH_ARTIFACTS_DIR)/, $(addsuffix .elf, $(BENCH_PROGRAMS))) @@ -171,6 +179,18 @@ compile-programs: compile-programs-asm compile-programs-rust compile-bench compi compile-recursion-elfs: prepare-sysroot $(RECURSION_ARTIFACTS) $(RECURSION_VERIFIER_ARTIFACTS) +# Everything the prover suite reads off disk: the asm guests (via run_asm_elf), +# the Rust guests listed above, and the recursion guests. +compile-prover-test-elfs: compile-programs-asm compile-recursion-elfs $(PROVER_TEST_ARTIFACTS) + +# Guards the list against drift: a new prover test that reads a Rust guest not in +# $(PROVER_TEST_GUESTS) would pass in CI (which builds every guest) and panic with +# "elf not found" on a clean checkout running `make test-prover`. Run from `lint`. +check-prover-test-elfs: + @grep -rhoE 'program_artifacts/rust/[A-Za-z0-9_]+\.elf' prover/src prover/tests \ + | sed -E 's#.*/##; s#\.elf$$##' | sort -u \ + | awk -v known="$(PROVER_TEST_GUESTS)" 'BEGIN { n = split(known, a, " "); for (i = 1; i <= n; i++) k[a[i]] = 1 } !($$0 in k) { if (!missing++) print "check-prover-test-elfs: prover tests read Rust guests missing from PROVER_TEST_GUESTS:"; print " " $$0 } END { if (missing) { print "Add them to PROVER_TEST_GUESTS in the Makefile."; exit 1 } }' + $(RECURSION_ARTIFACTS_DIR): mkdir -p $@ @@ -535,29 +555,30 @@ test: compile-programs test-syscalls test-ethrex-crypto # === Quick test shortcuts === -# Fast prover tests (skips ignored slow tests). Recursion smoke/PoC tests read -# prebuilt guest ELFs, so build them first. -test-fast: compile-recursion-elfs +# Fast prover tests (skips ignored slow tests). The executor leg reads most of +# the Rust guests, so this one needs the full set rather than the prover subset. +test-fast: compile-programs-asm compile-programs-rust compile-recursion-elfs cargo test -p lambda-vm-prover -p stark -p executor -F stark/parallel # Prover tests only -test-prover: compile-recursion-elfs +test-prover: compile-prover-test-elfs cargo test -p lambda-vm-prover # Prover tests including slow ones. The recursion smoke tests read prebuilt # guest ELFs from executor/program_artifacts/recursion/ — the fast ones on every # run, the slow ones (still #[ignore]d) only under --include-ignored — so build # them first. -test-prover-all: compile-recursion-elfs +test-prover-all: compile-prover-test-elfs cargo test -p lambda-vm-prover -- --include-ignored # Prover tests with debug-checks (shows bus balance report). Also unfiltered, so # it runs the non-ignored recursion tests that read prebuilt guest ELFs. -test-prover-debug: compile-recursion-elfs +test-prover-debug: compile-prover-test-elfs cargo test -p lambda-vm-prover --features debug-checks -- --nocapture # Disk-spill tests (stark + prover). FORCE_DISK_SPILL is required by the prover tests. -test-disk-spill: +# The count_table_lengths filter picks up the drift tests, which read hint_min.elf. +test-disk-spill: compile-prover-test-elfs cargo test --release -p stark --features disk-spill disk_spill FORCE_DISK_SPILL=1 cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths @@ -583,9 +604,9 @@ test-cuda-fallback: # GPU + nvcc). The GPU CI counterpart of CPU CI's sharded prover tests. Single-threaded: the # GPU serializes proves and the dispatch counters are process-global. cuda on prover cascades # to stark; crypto/ecsm build without it (they have no GPU path). -# compile-recursion-elfs: this unfiltered run executes the non-ignored recursion -# smoke tests, which read prebuilt guest ELFs; scripts/gpu_test.sh otherwise never builds them. -test-prover-cuda: compile-recursion-elfs +# compile-prover-test-elfs: this unfiltered run executes the non-ignored tests that +# read prebuilt guest ELFs; scripts/gpu_test.sh otherwise never builds them. +test-prover-cuda: compile-prover-test-elfs cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \ --features lambda-vm-prover/cuda -- --test-threads=1 @@ -630,7 +651,7 @@ fmt: cargo fmt --all # Run clippy + fmt check (used by CI) -lint: +lint: check-prover-test-elfs cargo fmt --check --all cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref From 5aa1d67edb16be0762ea6d394dfbbff704eb5042 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Mon, 10 Aug 2026 17:06:08 -0300 Subject: [PATCH 2/6] fix(make): let the ELF guard match hyphenated guest names The pattern matched [A-Za-z0-9_] only, so a guest directory with a hyphen would have been skipped silently. Also states what the guard does not cover. --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 36d5f649e..76444d1f7 100644 --- a/Makefile +++ b/Makefile @@ -186,8 +186,11 @@ compile-prover-test-elfs: compile-programs-asm compile-recursion-elfs $(PROVER_T # Guards the list against drift: a new prover test that reads a Rust guest not in # $(PROVER_TEST_GUESTS) would pass in CI (which builds every guest) and panic with # "elf not found" on a clean checkout running `make test-prover`. Run from `lint`. +# Rust guests only — the asm and recursion artifacts are built wholesale, so no +# list can go stale for them. A prover test reading a bench guest (none do today) +# would need its own entry here. check-prover-test-elfs: - @grep -rhoE 'program_artifacts/rust/[A-Za-z0-9_]+\.elf' prover/src prover/tests \ + @grep -rhoE 'program_artifacts/rust/[A-Za-z0-9_-]+\.elf' prover/src prover/tests \ | sed -E 's#.*/##; s#\.elf$$##' | sort -u \ | awk -v known="$(PROVER_TEST_GUESTS)" 'BEGIN { n = split(known, a, " "); for (i = 1; i <= n; i++) k[a[i]] = 1 } !($$0 in k) { if (!missing++) print "check-prover-test-elfs: prover tests read Rust guests missing from PROVER_TEST_GUESTS:"; print " " $$0 } END { if (missing) { print "Add them to PROVER_TEST_GUESTS in the Makefile."; exit 1 } }' From 03ed95828f80e87f7ae363b26cda85b1d992c630 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Wed, 2 Sep 2026 17:15:24 -0300 Subject: [PATCH 3/6] Make the prover-guest ELF guard fail in both directions instead of one check-prover-test-elfs only caught guests a prover test reads that are missing from PROVER_TEST_GUESTS. The mirror case is just as breaking and nothing flagged it: an entry left in the list after its guest directory is deleted makes compile-prover-test-elfs ask cargo to build a path that no longer exists, so it surfaces as a build error instead of a lint failure. feat/hint-arena (#942) deletes hint_min and hint_multi and adds prover reads of ecrecover_hints and hint_arena, so both halves of that collision are already scheduled. The guard now also fails when the grep matches nothing at all: an empty result means the artifact path moved or the tests build the guest name at runtime, and either way the check has silently stopped checking anything. prover/benches joins prover/src and prover/tests in the scan, since a bench reading a guest hits the same missing-ELF failure. --- Makefile | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 76444d1f7..f2d01ea39 100644 --- a/Makefile +++ b/Makefile @@ -183,16 +183,22 @@ compile-recursion-elfs: prepare-sysroot $(RECURSION_ARTIFACTS) $(RECURSION_VERIF # the Rust guests listed above, and the recursion guests. compile-prover-test-elfs: compile-programs-asm compile-recursion-elfs $(PROVER_TEST_ARTIFACTS) -# Guards the list against drift: a new prover test that reads a Rust guest not in -# $(PROVER_TEST_GUESTS) would pass in CI (which builds every guest) and panic with -# "elf not found" on a clean checkout running `make test-prover`. Run from `lint`. +# Guards the list against drift, in both directions. A prover test that reads a +# Rust guest missing from $(PROVER_TEST_GUESTS) passes in CI (which builds every +# guest) and panics with "elf not found" on a clean checkout running +# `make test-prover` — that is how this got here. An entry left behind after its +# guest is deleted is the mirror image: `compile-prover-test-elfs` then asks cargo +# to build a directory that no longer exists, and nothing flags it at review time. +# The empty-match case is a failure too: it means the path moved or the tests now +# build the name at runtime, either of which silently disarms the guard. +# Run from `lint`. # Rust guests only — the asm and recursion artifacts are built wholesale, so no # list can go stale for them. A prover test reading a bench guest (none do today) # would need its own entry here. check-prover-test-elfs: - @grep -rhoE 'program_artifacts/rust/[A-Za-z0-9_-]+\.elf' prover/src prover/tests \ + @grep -rhoE 'program_artifacts/rust/[A-Za-z0-9_-]+\.elf' prover/src prover/tests prover/benches \ | sed -E 's#.*/##; s#\.elf$$##' | sort -u \ - | awk -v known="$(PROVER_TEST_GUESTS)" 'BEGIN { n = split(known, a, " "); for (i = 1; i <= n; i++) k[a[i]] = 1 } !($$0 in k) { if (!missing++) print "check-prover-test-elfs: prover tests read Rust guests missing from PROVER_TEST_GUESTS:"; print " " $$0 } END { if (missing) { print "Add them to PROVER_TEST_GUESTS in the Makefile."; exit 1 } }' + | awk -v known="$(PROVER_TEST_GUESTS)" 'BEGIN { n = split(known, a, " "); for (i = 1; i <= n; i++) k[a[i]] = 1 } { seen[$$0] = 1 } !($$0 in k) { if (!missing++) print "check-prover-test-elfs: prover tests read Rust guests missing from PROVER_TEST_GUESTS:"; print " " $$0 } END { if (!NR) { print "check-prover-test-elfs: no program_artifacts/rust/*.elf reads found in prover/ — the grep pattern went stale, fix it or the guard checks nothing."; exit 1 } for (i = 1; i <= n; i++) if (!(a[i] in seen)) { if (!stale++) print "check-prover-test-elfs: PROVER_TEST_GUESTS entries no prover test reads:"; print " " a[i] } if (missing) print "Add them to PROVER_TEST_GUESTS in the Makefile."; if (stale) print "Remove them from PROVER_TEST_GUESTS in the Makefile."; if (missing || stale) exit 1 }' $(RECURSION_ARTIFACTS_DIR): mkdir -p $@ From 23ca6720037f12891b9f26610cb0e9c980b91586 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Wed, 2 Sep 2026 17:16:29 -0300 Subject: [PATCH 4/6] Give the remaining test and bench targets the guest ELFs they read The prover targets were not the only ones reading prebuilt artifacts without asking for them. Six more run tests that read ELFs off disk with no prerequisite to build them, so on a clean checkout they fail the same way: - test-flamegraph runs executor/tests/flamegraph.rs, which reads both asm and Rust guests. - test-cuda-integration, test-cuda-fallback, test-prover-comprehensive-cuda, bench-prover and bench-prover-cuda each prove a prebuilt asm guest through asm_elf_bytes (bench_single uses fib_iterative_1M). The GPU ones survive today only because scripts/gpu_test.sh happens to run compile-programs-asm and compile-programs-rust before calling them; invoked directly they fail like the rest. Adding compile-programs-asm is free when the artifacts are current: unlike the Rust guests, the asm rule has a real file prerequisite (%.elf: %.s), so make skips it instead of re-entering a compiler. --- Makefile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f2d01ea39..16d0b95ba 100644 --- a/Makefile +++ b/Makefile @@ -490,7 +490,8 @@ test-ethrex: compile-programs-rust ethrex-real-block-fixture test-ethrex-offline: compile-programs-rust cd tooling/ethrex-tests && cargo test --release -- --include-ignored --skip test_ethrex_real_block -test-flamegraph: +# executor/tests/flamegraph.rs reads prebuilt asm and Rust guests off disk. +test-flamegraph: compile-programs-asm compile-programs-rust cargo test -p executor --test flamegraph test-profile-recursion: test-profile-recursion-single test-profile-recursion-multi @@ -599,13 +600,15 @@ test-math-cuda: # Asserts the R1-R4 GPU dispatch counters fired on a real prove. # --test-threads=1: these tests reset and assert on process-global GPU call # counters, so they must run serially or one test's reset races another's read. -test-cuda-integration: +# compile-programs-asm: these tests prove a prebuilt asm guest (asm_elf_bytes). +test-cuda-integration: compile-programs-asm cargo test -p lambda-vm-prover --release --features cuda \ --test cuda_path_integration -- --ignored --nocapture --test-threads=1 # GPU error-path coverage (requires NVIDIA GPU + nvcc). # Forces cuda dispatch errors and asserts the CPU fallback still produces a verifying proof. -test-cuda-fallback: +# compile-programs-asm: these tests prove a prebuilt asm guest (asm_elf_bytes). +test-cuda-fallback: compile-programs-asm cargo test -p lambda-vm-prover --release --features test-cuda-faults \ --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1 @@ -622,7 +625,8 @@ test-prover-cuda: compile-prover-test-elfs # The comprehensive all-instructions prove (ignored by default) on the GPU path (requires # NVIDIA GPU + nvcc). GPU counterpart of the all-instructions half of CPU CI's merge-queue-only # comprehensive job (the CPU job also runs test_recursion_execute; recursion has no GPU leg yet). -test-prover-comprehensive-cuda: +# compile-programs-asm: all_instructions_64 is a prebuilt asm guest. +test-prover-comprehensive-cuda: compile-programs-asm cargo test --release -p lambda-vm-prover --features cuda \ test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture @@ -631,12 +635,13 @@ bench-math-cuda: cargo test -p math-cuda --release --test bench_quick -- --ignored --nocapture # Single-prove wall-time bench (warm-up + profiled run of fib_iterative_1M). -bench-prover: +# compile-programs-asm: fib_iterative_1M is a prebuilt asm guest (asm_elf_bytes). +bench-prover: compile-programs-asm cargo test -p lambda-vm-prover --release --test bench_single -- --ignored --nocapture # Single-prove wall-time bench with the GPU LDE path enabled. # Needs an NVIDIA GPU + CUDA toolkit/driver. -bench-prover-cuda: +bench-prover-cuda: compile-programs-asm cargo test -p lambda-vm-prover --release --features cuda --test bench_single -- --ignored --nocapture # Build all From 2b29a40b372821b66acdc1a4769060d4138d9e85 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Wed, 2 Sep 2026 17:17:48 -0300 Subject: [PATCH 5/6] Say what the prover guest subset actually saves The comment justified PROVER_TEST_GUESTS by the cost of re-entering cargo once per guest "on every test run". Measured warm on this tree that re-entry is about 0.1s per guest: `make compile-programs-rust` with all 35 artifacts up to date takes 3.35s against 2.53s for `make compile-prover-test-elfs`, so the subset is worth roughly a second per run and the framing oversells it. What the subset really buys is the case the target exists for, a clean checkout, where the full set compiles 27 extra guest crates from scratch. --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 16d0b95ba..171074d5b 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,11 @@ RUST_PROGRAMS := $(notdir $(basename $(RUST_PROGRAM_DIRS:%/=%))) RUST_ARTIFACTS := $(addprefix $(RUST_ARTIFACTS_DIR)/, $(addsuffix .elf, $(RUST_PROGRAMS))) # The Rust guests the prover test suite reads as prebuilt artifacts. The prover -# targets build these instead of all $(RUST_PROGRAMS), because the .elf rules are -# FORCE (see below) and would re-enter cargo once per guest on every test run. +# targets build these instead of all $(RUST_PROGRAMS) because the .elf rules are +# FORCE (see below), so the full set would compile 27 guest crates the prover +# suite never opens. What that buys is the first build on a clean checkout: once +# the target dir is warm the extra FORCE re-entries are about a second for all 35, +# so this list is not worth widening for anything but correctness. # `check-prover-test-elfs` fails if a prover test reads one that is missing here. PROVER_TEST_GUESTS := allocator commit_sum ecsm ef_io_demo ethrex hint_min hint_multi pure_commit PROVER_TEST_ARTIFACTS := $(addprefix $(RUST_ARTIFACTS_DIR)/, $(addsuffix .elf, $(PROVER_TEST_GUESTS))) From 6d3eef04f68c173f60b183090cd2306a032a2ed1 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Wed, 2 Sep 2026 18:21:18 -0300 Subject: [PATCH 6/6] Extend the ELF prerequisites to the GPU targets main added since MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two arrivals from main read prebuilt guests the same way the targets fixed earlier in this branch do. test-cuda-d1 proves all_instructions_64, an asm guest. test-cuda-fallback grew a second leg, gpu_force_downgrade, which reads executor/program_artifacts/rust/ethrex.elf — a Rust guest, so compile-programs-asm alone does not cover that target any more; naming $(RUST_ARTIFACTS_DIR)/ethrex.elf builds the one artifact it needs instead of pulling in the other seven through compile-prover-test-elfs. check-prover-test-elfs already covered the new read: gpu_force_downgrade lives in prover/tests and ethrex was in PROVER_TEST_GUESTS, so the guard stayed green across the merge without touching the list. --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bb4996784..2e3261dfe 100644 --- a/Makefile +++ b/Makefile @@ -643,15 +643,17 @@ test-cuda-integration: compile-programs-asm # # Its own binary + a process-wide env because gpu_lde_threshold() caches the value # on first read (OnceLock), so it must be set before any prove in the process. -test-cuda-d1: +# compile-programs-asm: the fixture (all_instructions_64) is a prebuilt asm guest. +test-cuda-d1: compile-programs-asm LAMBDA_VM_GPU_LDE_THRESHOLD=128 $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover \ --release --features cuda \ --test cuda_d1_path -- --ignored --nocapture --test-threads=1 # GPU error-path coverage (requires NVIDIA GPU + nvcc). # Forces cuda dispatch errors and asserts the CPU fallback still produces a verifying proof. -# compile-programs-asm: these tests prove a prebuilt asm guest (asm_elf_bytes). -test-cuda-fallback: compile-programs-asm +# cuda_fallback_tests proves a prebuilt asm guest (asm_elf_bytes); gpu_force_downgrade +# reads the ethrex Rust guest. Naming the one artifact keeps this off the other seven. +test-cuda-fallback: compile-programs-asm $(RUST_ARTIFACTS_DIR)/ethrex.elf $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \ --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1 $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features lambda-vm-prover/cuda \