diff --git a/Cargo.lock b/Cargo.lock index 93fd6b417..56cf0e6a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,6 +562,7 @@ dependencies = [ "ecsm", "k256", "lambda-vm-syscalls", + "proptest", "rustc-demangle", "serde", "serde_json", diff --git a/bin/cli/README.md b/bin/cli/README.md index 5ef3cf40d..733cbd6d7 100644 --- a/bin/cli/README.md +++ b/bin/cli/README.md @@ -41,7 +41,7 @@ cargo run -p cli --release -- execute [--private-input ] [-- |---|---| | `--private-input ` | Pass private input bytes to the guest (read via `get_private_input()`). | | `--flamegraph ` | Generate folded-stack flamegraph output. See [Guest Program Flamegraphs](#guest-program-flamegraphs). | -| `--cycles` | Count instructions during execution and print the dynamic instruction count. Also reports `Keccak calls` / `Ecsm calls` (accelerator syscall invocations). Combined with `--flamegraph`, the accelerator lines are omitted (the flamegraph path exposes no per-log data). | +| `--cycles` | Count instructions during execution and print the dynamic instruction count. Also reports `Keccak calls` / `Ecsm calls` / `Dma calls` (accelerator syscall invocations), plus `Dma bytes` copied and the `Dma rows` those copies add to the trace before its power-of-two padding. One guest `memcpy` is chunked into several DMA ecalls, so the byte and row lines, not the call count, are what the copies cost. The lines cover `memcpy`, `memmove` and `memset`, which share one ecall path and one table; the commit byte loop lives in that same table but is not tallied here. Combined with `--flamegraph`, the accelerator lines are omitted (the flamegraph path exposes no per-log data). | ### Prove diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index a04e920db..adf8bf86f 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -142,9 +142,12 @@ enum Commands { cycle_budget: Option, /// Print the dynamic instruction (cycle) count, plus `Keccak calls` / - /// `Ecsm calls` (accelerator syscall invocations). The accelerator lines - /// are omitted when combined with --flamegraph (that path has no per-log - /// data). + /// `Ecsm calls` / `Dma calls` (accelerator syscall invocations), and for + /// DMA the `Dma bytes` copied and the `Dma rows` those copies add to the + /// trace before its power-of-two padding. One `memcpy` is chunked into + /// several DMA ecalls, so the byte and row lines, not the call count, are + /// what the copies cost. The accelerator lines are omitted when combined + /// with --flamegraph (that path has no per-log data). #[arg(long)] cycles: bool, }, @@ -359,6 +362,40 @@ struct FlamegraphCliOptions { checkpoint_cycles: Option, } +/// One tally per [`Accelerator`] variant, printed by `execute --cycles`. +#[derive(Default)] +struct AccelCounts { + keccak: u64, + ecsm: u64, + dma: u64, + /// Bytes copied and DMA table rows those copies consume. Keccak and ECSM + /// cost the same per call, so DMA is the only accelerator whose report needs + /// a size next to its count: one `memcpy` becomes as many ecalls as the + /// guest stub chunks it into, which makes `dma` alone a poor cost proxy. + dma_bytes: u64, + dma_rows: u64, +} + +impl AccelCounts { + /// Exhaustive `match`: a new `Accelerator` variant is a compile error here, + /// so it cannot be executed without also being reported. `dst_val` is the + /// ECALL's logged operands: `dst_val` is the chunk's byte count for DMA and + /// unused for the others; `rows` is the MEMMOVE row count the executor derived + /// at the ecall, where it knows `src`, `dst` and `count` — the schedule reads + /// both ends' residues, so the count cannot be recovered from one address. + fn tally(&mut self, accelerator: Accelerator, dst_val: u64, rows: u64) { + match accelerator { + Accelerator::Keccak => self.keccak += 1, + Accelerator::Ecsm => self.ecsm += 1, + Accelerator::Dma => { + self.dma += 1; + self.dma_bytes += dst_val; + self.dma_rows += rows; + } + } + } +} + /// Classifies one executed instruction as an accelerator syscall invocation. /// /// Delegates to the executor's canonical `SyscallNumbers::accelerator()` so the @@ -412,7 +449,7 @@ fn cmd_execute( // below (the flamegraph path drives execution inside the executor and does // not expose per-log data). `None` means "not counted", so the accel lines // are omitted rather than printed as misleading zeros. - let mut accel_counts: Option<(u64, u64)> = None; + let mut accel_counts: Option = None; let cycle_count = if let Some(ref output_path) = flamegraph.path { // Shared execute+flamegraph path (executor::flamegraph) instead of @@ -478,14 +515,13 @@ fn cmd_execute( }; let mut cycle_count: u64 = 0; - let mut keccak_calls: u64 = 0; - let mut ecsm_calls: u64 = 0; - // Reused per chunk: `(current_pc, a7)` for logs whose a7 matches an - // accelerator syscall number. This is a cheap superset — a non-ECALL + let mut counts = AccelCounts::default(); + // Reused per chunk: `(current_pc, a7, dst_val)` for logs whose a7 matches + // an accelerator syscall number. This is a cheap superset — a non-ECALL // instruction can hold the same value in src1 — that `accelerator_of` // confirms below, once the chunk's `&Log` borrow (tied to the executor's // `&mut`) is released so the instruction cache can be read again. - let mut accel_candidates: Vec<(u64, u64)> = Vec::new(); + let mut accel_candidates: Vec<(u64, u64, u64, u64)> = Vec::new(); loop { let logs = match executor.resume_budgeted(cycle_count, cycle_budget) { Ok(logs) => logs, @@ -502,17 +538,20 @@ fn cmd_execute( .map(|s| s.accelerator().is_some()) .unwrap_or(false) { - accel_candidates.push((log.current_pc, log.src1_val)); + accel_candidates.push(( + log.current_pc, + log.src1_val, + log.dst_val, + log.src2_val, + )); } } } // `logs` is no longer used, so the executor's `&mut` borrow is free // and the instruction cache can be read to confirm each candidate. - for (pc, a7) in accel_candidates.drain(..) { - match accelerator_of(executor.instructions.get(pc), a7) { - Some(Accelerator::Keccak) => keccak_calls += 1, - Some(Accelerator::Ecsm) => ecsm_calls += 1, - None => {} + for (pc, a7, dst_val, rows) in accel_candidates.drain(..) { + if let Some(accelerator) = accelerator_of(executor.instructions.get(pc), a7) { + counts.tally(accelerator, dst_val, rows); } } if cycle_budget.is_some_and(|budget| cycle_count >= budget) { @@ -526,16 +565,19 @@ fn cmd_execute( } if cycles { - accel_counts = Some((keccak_calls, ecsm_calls)); + accel_counts = Some(counts); } cycle_count }; if cycles { println!("Cycles: {}", cycle_count); - if let Some((keccak_calls, ecsm_calls)) = accel_counts { - println!("Keccak calls: {}", keccak_calls); - println!("Ecsm calls: {}", ecsm_calls); + if let Some(counts) = accel_counts { + println!("Keccak calls: {}", counts.keccak); + println!("Ecsm calls: {}", counts.ecsm); + println!("Dma calls: {}", counts.dma); + println!("Dma bytes: {}", counts.dma_bytes); + println!("Dma rows: {}", counts.dma_rows); } } @@ -1102,43 +1144,113 @@ mod tests { assert_eq!(continuation_epoch_size(20).unwrap(), 1 << 20); } + /// The chip each syscall must drive, written out here rather than read back + /// from `SyscallNumbers::accelerator()`. Comparing the CLI against the + /// executor alone would pass if both agreed on the wrong answer — a chip + /// demoted to `None` has to fail somewhere, and this is that somewhere. + /// + /// Cross-checked row by row against `SyscallNumbers::ALL`, which the + /// executor's macro generates from the enum, so a new syscall fails the test + /// until it gets a row here. + const EXPECTED_ACCELERATORS: &[(SyscallNumbers, Option)] = &[ + (SyscallNumbers::KeccakPermute, Some(Accelerator::Keccak)), + (SyscallNumbers::Ecsm, Some(Accelerator::Ecsm)), + (SyscallNumbers::DmaMemcpy, Some(Accelerator::Dma)), + (SyscallNumbers::DmaMemset, Some(Accelerator::Dma)), + (SyscallNumbers::Print, None), + (SyscallNumbers::Panic, None), + (SyscallNumbers::Commit, None), + (SyscallNumbers::Halt, None), + // `hint` drives its own HINT table, but the executor maps it to no + // `Accelerator`: the ecall adds no correctness constraint, so there is no + // accelerated work to attribute. `execute --cycles` reports no hint line. + (SyscallNumbers::Hint, None), + ]; + // `accelerator_of` must match the prover's `CpuOperation::from_log`: count an // invocation only when the instruction is an ECALL AND a7 is the accelerator - // syscall number. Covers both accelerators, the non-accelerator syscalls, a - // non-ECALL whose src1 collides with an accelerator number, and a cache miss. + // syscall number. #[test] fn accelerator_of_mirrors_prover_classification() { - use executor::vm::instruction::execution::{ECSM_SYSCALL_NUMBER, KECCAK_SYSCALL_NUMBER}; - let ecall = Instruction::EcallEbreak; - assert_eq!( - accelerator_of(Some(&ecall), KECCAK_SYSCALL_NUMBER), - Some(Accelerator::Keccak) - ); - assert_eq!( - accelerator_of(Some(&ecall), ECSM_SYSCALL_NUMBER), - Some(Accelerator::Ecsm) - ); + for &syscall in SyscallNumbers::ALL { + let rows = EXPECTED_ACCELERATORS + .iter() + .filter(|(listed, _)| *listed == syscall) + .count(); + assert_eq!( + rows, 1, + "{syscall:?} needs exactly one row in EXPECTED_ACCELERATORS" + ); + } - // Non-accelerator syscalls (Commit=64, Halt=93) count as neither. - assert_eq!( - accelerator_of(Some(&ecall), SyscallNumbers::Commit as u64), - None - ); - assert_eq!( - accelerator_of(Some(&ecall), SyscallNumbers::Halt as u64), - None - ); + for &(syscall, expected) in EXPECTED_ACCELERATORS { + assert_eq!( + accelerator_of(Some(&ecall), syscall.raw()), + expected, + "ECALL with a7 of {syscall:?} must classify as {expected:?}" + ); + // A non-ECALL instruction whose src1 happens to equal a syscall a7 + // must not count — this is the `f.ecall &&` guard the prover applies. + assert_eq!( + accelerator_of(Some(&Instruction::Fence), syscall.raw()), + None, + "non-ECALL with a7 of {syscall:?} must not count" + ); + // No decoded instruction at the pc (cache miss) counts as neither. + assert_eq!(accelerator_of(None, syscall.raw()), None); + } + } - // A non-ECALL instruction whose src1 happens to equal an accelerator a7 - // must not count — this is the `f.ecall &&` guard the prover applies. - assert_eq!( - accelerator_of(Some(&Instruction::Fence), KECCAK_SYSCALL_NUMBER), - None - ); + // Every tallied accelerator gets its own counter: no two variants may share + // a field, and each must land in the one the report prints. + #[test] + fn accel_counts_tallies_each_accelerator_separately() { + for &(_, expected_accelerator) in EXPECTED_ACCELERATORS { + let Some(accelerator) = expected_accelerator else { + continue; + }; + let mut counts = AccelCounts::default(); + counts.tally(accelerator, 0, 0); + assert_eq!( + counts.keccak + counts.ecsm + counts.dma, + 1, + "{accelerator:?} must increment exactly one counter" + ); + let expected = match accelerator { + Accelerator::Keccak => counts.keccak, + Accelerator::Ecsm => counts.ecsm, + Accelerator::Dma => counts.dma, + }; + assert_eq!( + expected, 1, + "{accelerator:?} must increment its own counter" + ); + } + } + + // The byte and row lines are what make the DMA report a cost figure rather + // than a call count, so they must accumulate across chunked ecalls and use + // the executor's row formula — the same one trace generation sizes with. + #[test] + fn accel_counts_sizes_dma_calls() { + let mut counts = AccelCounts::default(); + // (bytes, rows) as the executor derives them at each ecall. + for (bytes, rows) in [(256, 33), (256, 33), (8, 2), (3, 4), (0, 1)] { + counts.tally(Accelerator::Dma, bytes, rows); + } - // No decoded instruction at the pc (cache miss) counts as neither. - assert_eq!(accelerator_of(None, KECCAK_SYSCALL_NUMBER), None); + assert_eq!(counts.dma, 5, "every DMA ecall counts as one call"); + assert_eq!(counts.dma_bytes, 523); + // The rows are the executor's, derived where src, dst and count are all known; + // the report only sums them. + assert_eq!(counts.dma_rows, 73); + + // The other accelerators must leave the DMA size lines alone. + let mut others = AccelCounts::default(); + others.tally(Accelerator::Keccak, 200, 0); + others.tally(Accelerator::Ecsm, 32, 0); + assert_eq!((others.dma, others.dma_bytes, others.dma_rows), (0, 0, 0)); } } diff --git a/docs/general_flow.md b/docs/general_flow.md index deee5e4fe..784d93eac 100644 --- a/docs/general_flow.md +++ b/docs/general_flow.md @@ -18,3 +18,17 @@ The Lambda VM proves correct execution of a RISC-V (RV64IM) program against an i 4. **Proof system** ([`crypto/stark/`](../crypto/stark/)) — commits to each table's trace via Merkle trees, samples challenges via Fiat-Shamir, and runs FRI for the low-degree test. Produces a `MultiProof`; the verifier replays the transcript and checks all AIR and lookup constraints. For a deeper dive into each component see the [proof system overview](./cryptography/proof_system.md). + +## Accelerated memory operations + +`memcpy` is accelerated: `lambda-vm-syscalls` exports it under its standard, unmangled C name, so both explicit calls and the copies the compiler emits implicitly (struct moves, slice copies, `Vec` growth) reach the DMA ecall with no guest source changes. Behaviour is identical to the C function for every input, including `n == 0` and any alignment of `dest`, `src` or `n`. `memmove` and `memset` are accelerated too, through the same chip: `memmove` reuses the copy ecall and chunks backwards when the ranges overlap, and `memset` is expressed as a *propagating* copy — the stub seeds eight bytes with ordinary stores and then calls the accelerator with `dst = seed_end, src = seed_start`, which the chip runs with the read/write timestamp order inverted so the seed walks across the range. Fills shorter than sixteen bytes take a plain store loop instead. `memcmp` is not accelerated and falls back to the toolchain's `compiler-builtins` definition. + +**Observability.** `cli execute --cycles` reports `Dma calls`, `Dma bytes` and `Dma rows`. The call line confirms the accelerator is engaged at all; the byte and row lines are the cost, since the guest stub chunks one `memcpy` into as many ecalls as it needs and each ecall adds one row per chunk plus a terminal row, where a chunk is eight bytes or one (see the alignment note below). `Dma rows` is the raw row count the copies contribute, before the DMA trace is padded to a power-of-two height — for a guest with few copies the padded table is larger than the reported figure. + +**Aligned vs misaligned.** The chunk width comes from the bytes remaining *and* from the alignment of the two ends, so both the row count and the cost per row depend on it. Each eight-byte chunk emits two width-8 memory operations, one reading the source and one writing the destination, each at its address as given, and the memory argument routes each one by that address: an 8-aligned window sharing one old timestamp reaches MEMW_A (29 columns, one ALU `LT` range check), and anything else falls to the general MEMW table (49 columns, eight `LT` rows). The two sides are independent, so a copy can take the fast path on one end and not the other. The schedule therefore walks one-byte rows until the destination reaches eight-alignment and takes eight-byte rows through the body — but only when the two ends share a residue mod 8, so that aligning one aligns both. When the residues differ, aligning the destination would push the source out of alignment on every row, which measured as a net loss, so the schedule takes eight-byte rows throughout and both ends stay misaligned. A misaligned copy therefore commits strictly more cells than an aligned copy of the same length, which is what makes the aligned/misaligned split the standard recommends informative here. It is not reported: the accelerator statistics are derived from `Log`, whose two operand slots are already taken (`dst_val = n`, and `src2_val` now carries the row count, which the executor has to derive at the ecall because the schedule reads both ends' residues and so cannot be recovered from `n` downstream). Reporting the aligned/misaligned split needs another slot or a dedicated counter. Left as a follow-up, and stated here rather than claimed as done. + +**Symbol resolution.** `compiler-builtins` defines `memcpy` *weakly*, and a linker extracts a static-archive member only to satisfy an *undefined* symbol — a weak definition already satisfies the reference, so a strong definition that lives in a member nothing else pulls in is dropped silently, with no duplicate-symbol diagnostic. Lambda VM therefore defines `memcpy` in [`syscalls/src/entrypoint.rs`](../syscalls/src/entrypoint.rs), the same object that defines `_start`, which every guest links unconditionally. That object is always extracted, so the strong definition is in the link graph from the start and overrides the weak one. No `--whole-archive` and no guest link flag is required, and resolution does not depend on archive order. + +**Deviation from the standard's scope clause.** The standard says the accelerated symbols "are exported from the vendor static library defined by the Static Library and Linker Script standard". Lambda VM has no such library: the guest interface is a Rust rlib (`lambda-vm-syscalls`), and `memcpy` is exported from its always-linked entrypoint object. The linking clause above is satisfied by mechanism (1); the packaging the scope clause assumes is not, and adopting it is a repo-wide decision rather than one this accelerator can make. + +Placing `memcpy` beside `_start` is insurance rather than a repair: defined in `syscalls.rs` it also resolved correctly, and not by luck — `_start` calls `sys_halt` from that module and it is not `#[inline]`, so every guest carries an undefined reference that forces the object out of the archive, whatever the guest itself names. What the move removes is the two things that guarantee rested on: `_start` continuing to call into `syscalls.rs`, and rustc's codegen-unit merging keeping the two modules together. Co-locating with `_start` — the one symbol the linker is obliged to resolve — makes the guarantee local instead, and `test_dma_memcpy_compiler_emitted_copies` (a guest that never names `memcpy`, asserting the DMA ecall count stays above zero) is what detects a regression, since a guest that falls back to the weak definition still produces correct output. diff --git a/executor/Cargo.toml b/executor/Cargo.toml index 91ae64ae9..b37bdc081 100644 --- a/executor/Cargo.toml +++ b/executor/Cargo.toml @@ -14,6 +14,7 @@ ecsm = { path = "../crypto/ecsm" } k256 = { version = "0.13", default-features = false, features = ["arithmetic", "expose-field"] } [dev-dependencies] +proptest = "1.9" # Test-only: the guest-side syscall crate re-declares the `hint` selectors as `usize` # and they must stay equal to the `u64` copies here (see `hint_selectors_match_the_guest`). # Unlike `crypto/crypto`'s and `ethrex-crypto`'s copies of this dep, it is NOT diff --git a/executor/programs/rust/dma_memcpy_cases/.cargo/config.toml b/executor/programs/rust/dma_memcpy_cases/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memcpy_cases/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memcpy_cases/Cargo.lock b/executor/programs/rust/dma_memcpy_cases/Cargo.lock new file mode 100644 index 000000000..5f1da6b2b --- /dev/null +++ b/executor/programs/rust/dma_memcpy_cases/Cargo.lock @@ -0,0 +1,294 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "const-default" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memcpy_cases" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-alloc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" +dependencies = [ + "const-default", + "critical-section", + "linked_list_allocator", + "rlsf", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "embedded-alloc", + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linked_list_allocator" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "rlsf" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" +dependencies = [ + "cfg-if", + "const-default", + "libc", + "rustversion", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memcpy_cases/Cargo.toml b/executor/programs/rust/dma_memcpy_cases/Cargo.toml new file mode 100644 index 000000000..86baaa9d2 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_cases/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memcpy_cases" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memcpy_cases/src/main.rs b/executor/programs/rust/dma_memcpy_cases/src/main.rs new file mode 100644 index 000000000..b8472eb96 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_cases/src/main.rs @@ -0,0 +1,77 @@ +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memcpy(dst: *mut u8, src: *const u8, count: usize) -> *mut u8; +} + +#[inline(never)] +fn dma_copy(dst: *mut u8, src: *const u8, count: usize) -> *mut u8 { + let count = core::hint::black_box(count); + unsafe { memcpy(dst, src, count) } +} + +fn fill_pattern(bytes: &mut [u8], seed: u8) { + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = (i as u8).wrapping_mul(37).wrapping_add(seed); + } +} + +pub fn main() { + let mut source = [0u8; 777]; + let mut destination = [0xA5u8; 777]; + fill_pattern(&mut source, 11); + + for count in [0usize, 1, 7, 8, 9, 31, 32, 33, 127, 128, 255, 256] { + destination.fill(0xA5); + let returned = dma_copy(destination.as_mut_ptr(), source.as_ptr(), count); + assert_eq!(returned, destination.as_mut_ptr()); + assert_eq!(&destination[..count], &source[..count]); + assert!(destination[count..].iter().all(|&byte| byte == 0xA5)); + } + + // More than one chunk: 777 bytes becomes four bounded DMA ecalls. + destination.fill(0); + dma_copy(destination.as_mut_ptr(), source.as_ptr(), source.len()); + assert_eq!(destination, source); + + // Snapshot semantics in both overlap directions. + let mut forward = [0u8; 320]; + fill_pattern(&mut forward, 23); + let forward_before = forward; + dma_copy( + unsafe { forward.as_mut_ptr().add(17) }, + forward.as_ptr(), + 256, + ); + assert_eq!(&forward[17..273], &forward_before[..256]); + + let mut backward = [0u8; 320]; + fill_pattern(&mut backward, 41); + let backward_before = backward; + dma_copy( + backward.as_mut_ptr(), + unsafe { backward.as_ptr().add(17) }, + 256, + ); + assert_eq!(&backward[..256], &backward_before[17..273]); + + // Force both operands to cross a 4 KiB page boundary. + let mut page_source = [0u8; 8192]; + let mut page_destination = [0u8; 8192]; + fill_pattern(&mut page_source, 67); + let src_to_boundary = 4096 - (page_source.as_ptr() as usize & 4095); + let dst_to_boundary = 4096 - (page_destination.as_ptr() as usize & 4095); + let src_offset = src_to_boundary.saturating_sub(3); + let dst_offset = dst_to_boundary.saturating_sub(5); + dma_copy( + unsafe { page_destination.as_mut_ptr().add(dst_offset) }, + unsafe { page_source.as_ptr().add(src_offset) }, + 256, + ); + assert_eq!( + &page_destination[dst_offset..dst_offset + 256], + &page_source[src_offset..src_offset + 256] + ); + + syscalls::syscalls::commit(b"dma-cases-ok"); +} diff --git a/executor/programs/rust/dma_memcpy_implicit/.cargo/config.toml b/executor/programs/rust/dma_memcpy_implicit/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memcpy_implicit/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memcpy_implicit/Cargo.lock b/executor/programs/rust/dma_memcpy_implicit/Cargo.lock new file mode 100644 index 000000000..3b4049770 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_implicit/Cargo.lock @@ -0,0 +1,294 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "const-default" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memcpy_implicit" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-alloc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" +dependencies = [ + "const-default", + "critical-section", + "linked_list_allocator", + "rlsf", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "embedded-alloc", + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linked_list_allocator" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "rlsf" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" +dependencies = [ + "cfg-if", + "const-default", + "libc", + "rustversion", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memcpy_implicit/Cargo.toml b/executor/programs/rust/dma_memcpy_implicit/Cargo.toml new file mode 100644 index 000000000..85068fca5 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_implicit/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memcpy_implicit" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memcpy_implicit/src/main.rs b/executor/programs/rust/dma_memcpy_implicit/src/main.rs new file mode 100644 index 000000000..d24903c9c --- /dev/null +++ b/executor/programs/rust/dma_memcpy_implicit/src/main.rs @@ -0,0 +1,34 @@ +//! Every copy here is emitted by the compiler: nothing declares or names +//! `memcpy`. The guest computes the same output whether or not the strong +//! `memcpy` symbol won the guest's link, so its DMA ecall count — not its +//! output — is what pins the symbol resolution. + +use lambda_vm_syscalls as syscalls; + +#[inline(never)] +fn copy_slice(destination: &mut [u8], source: &[u8]) { + destination.copy_from_slice(source); +} + +fn fill_pattern(bytes: &mut [u8], seed: u8) { + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = (i as u8).wrapping_mul(31).wrapping_add(seed); + } +} + +pub fn main() { + let mut source = [0u8; 512]; + fill_pattern(&mut source, 7); + // A runtime-sized length keeps LLVM from lowering the copies inline. + let length = core::hint::black_box(source.len()); + + let mut destination = [0u8; 512]; + copy_slice(&mut destination[..length], &source[..length]); + assert_eq!(destination, source); + + let mut grown = Vec::new(); + grown.extend_from_slice(&source[..length]); + assert_eq!(grown.as_slice(), &source[..]); + + syscalls::syscalls::commit(b"dma-implicit-ok"); +} diff --git a/executor/programs/rust/dma_memcpy_min/.cargo/config.toml b/executor/programs/rust/dma_memcpy_min/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memcpy_min/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memcpy_min/Cargo.lock b/executor/programs/rust/dma_memcpy_min/Cargo.lock new file mode 100644 index 000000000..06556e1d2 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_min/Cargo.lock @@ -0,0 +1,294 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "const-default" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memcpy_min" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-alloc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" +dependencies = [ + "const-default", + "critical-section", + "linked_list_allocator", + "rlsf", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "embedded-alloc", + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linked_list_allocator" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "rlsf" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07393724337be2ee43a9d86164df4505746874a3fa65913374bc6d6a92314362" +dependencies = [ + "cfg-if", + "const-default", + "libc", + "rustversion", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memcpy_min/Cargo.toml b/executor/programs/rust/dma_memcpy_min/Cargo.toml new file mode 100644 index 000000000..a791f7824 --- /dev/null +++ b/executor/programs/rust/dma_memcpy_min/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memcpy_min" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memcpy_min/src/main.rs b/executor/programs/rust/dma_memcpy_min/src/main.rs new file mode 100644 index 000000000..fb33e33fc --- /dev/null +++ b/executor/programs/rust/dma_memcpy_min/src/main.rs @@ -0,0 +1,16 @@ +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memcpy(dst: *mut u8, src: *const u8, count: usize) -> *mut u8; +} + +pub fn main() { + let source = *b"DMA copies eight-byte rows and a short tail"; + let mut destination = [0u8; 43]; + let count = core::hint::black_box(destination.len()); + + unsafe { + memcpy(destination.as_mut_ptr(), source.as_ptr(), count); + } + syscalls::syscalls::commit(&destination); +} diff --git a/executor/programs/rust/dma_memmove_cases/.cargo/config.toml b/executor/programs/rust/dma_memmove_cases/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memmove_cases/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memmove_cases/Cargo.lock b/executor/programs/rust/dma_memmove_cases/Cargo.lock new file mode 100644 index 000000000..3e0c1c056 --- /dev/null +++ b/executor/programs/rust/dma_memmove_cases/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memmove_cases" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memmove_cases/Cargo.toml b/executor/programs/rust/dma_memmove_cases/Cargo.toml new file mode 100644 index 000000000..b81ea25a9 --- /dev/null +++ b/executor/programs/rust/dma_memmove_cases/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memmove_cases" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memmove_cases/src/main.rs b/executor/programs/rust/dma_memmove_cases/src/main.rs new file mode 100644 index 000000000..45ecdb0de --- /dev/null +++ b/executor/programs/rust/dma_memmove_cases/src/main.rs @@ -0,0 +1,70 @@ +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memmove(dst: *mut u8, src: *const u8, count: usize) -> *mut u8; +} + +#[inline(never)] +fn dma_move(dst: *mut u8, src: *const u8, count: usize) -> *mut u8 { + let count = core::hint::black_box(count); + unsafe { memmove(dst, src, count) } +} + +fn fill_pattern(bytes: &mut [u8], seed: u8) { + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = (i as u8).wrapping_mul(37).wrapping_add(seed); + } +} + +pub fn main() { + // Disjoint regions behave like memcpy. + let mut source = [0u8; 777]; + let mut destination = [0xA5u8; 777]; + fill_pattern(&mut source, 11); + for count in [0usize, 1, 7, 8, 255, 256, 257, 777] { + destination.fill(0xA5); + let returned = dma_move(destination.as_mut_ptr(), source.as_ptr(), count); + assert_eq!(returned, destination.as_mut_ptr()); + assert_eq!(&destination[..count], &source[..count]); + assert!(destination[count..].iter().all(|&b| b == 0xA5)); + } + + // Forward overlap (dst inside [src, src+n)) is the case that needs BACKWARD + // chunking; a forward-chunked copy corrupts it once n exceeds one chunk. + // Offsets below and above 256 exercise both sides of the chunk boundary. + for (offset, count) in [(1usize, 600usize), (17, 600), (255, 600), (256, 600), (300, 700), (4, 8)] { + let mut buffer = [0u8; 1600]; + fill_pattern(&mut buffer, 23); + let before = buffer; + dma_move( + unsafe { buffer.as_mut_ptr().add(offset) }, + buffer.as_ptr(), + count, + ); + assert_eq!(&buffer[offset..offset + count], &before[..count]); + // Bytes below the destination must be untouched. + assert_eq!(&buffer[..offset], &before[..offset]); + } + + // Backward overlap (dst below src) stays forward-chunked. + for (offset, count) in [(1usize, 600usize), (17, 600), (300, 700)] { + let mut buffer = [0u8; 1600]; + fill_pattern(&mut buffer, 41); + let before = buffer; + dma_move( + buffer.as_mut_ptr(), + unsafe { buffer.as_ptr().add(offset) }, + count, + ); + assert_eq!(&buffer[..count], &before[offset..offset + count]); + } + + // Exact aliasing must be a no-op. + let mut same = [0u8; 300]; + fill_pattern(&mut same, 7); + let before = same; + dma_move(same.as_mut_ptr(), same.as_ptr(), 300); + assert_eq!(same, before); + + syscalls::syscalls::commit(b"dma-memmove-ok"); +} diff --git a/executor/programs/rust/dma_memset_cases/.cargo/config.toml b/executor/programs/rust/dma_memset_cases/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memset_cases/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memset_cases/Cargo.lock b/executor/programs/rust/dma_memset_cases/Cargo.lock new file mode 100644 index 000000000..471c82c00 --- /dev/null +++ b/executor/programs/rust/dma_memset_cases/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memset_cases" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memset_cases/Cargo.toml b/executor/programs/rust/dma_memset_cases/Cargo.toml new file mode 100644 index 000000000..de5dc5ede --- /dev/null +++ b/executor/programs/rust/dma_memset_cases/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memset_cases" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memset_cases/src/main.rs b/executor/programs/rust/dma_memset_cases/src/main.rs new file mode 100644 index 000000000..0625769fa --- /dev/null +++ b/executor/programs/rust/dma_memset_cases/src/main.rs @@ -0,0 +1,63 @@ +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memset(dst: *mut u8, fill: i32, count: usize) -> *mut u8; +} + +/// `black_box` on the count keeps LLVM from turning these into inline stores, +/// so every call really does reach the strong `memset` symbol and the DMA ecall. +#[inline(never)] +fn dma_set(dst: *mut u8, fill: i32, count: usize) -> *mut u8 { + let count = core::hint::black_box(count); + unsafe { memset(dst, fill, count) } +} + +pub fn main() { + let mut buffer = [0u8; 777]; + + // Every row-schedule boundary: empty, sub-tail, exact widths, the 256-byte + // per-ecall cap, and one length that forces several chunked ecalls. + for count in [0usize, 1, 7, 8, 9, 31, 32, 33, 127, 128, 255, 256] { + buffer.fill(0xA5); + let returned = dma_set(buffer.as_mut_ptr(), 0x3C, count); + assert_eq!(returned, buffer.as_mut_ptr()); + assert!(buffer[..count].iter().all(|&byte| byte == 0x3C)); + assert!(buffer[count..].iter().all(|&byte| byte == 0xA5)); + } + + // More than one chunk: 777 bytes becomes four bounded DMA ecalls. + buffer.fill(0); + dma_set(buffer.as_mut_ptr(), 0x5A, buffer.len()); + assert!(buffer.iter().all(|&byte| byte == 0x5A)); + + // Zero is the fill almost every real caller passes (`vec![0; n]` and the + // allocator's `alloc_zeroed`), and it is the one value a dropped write is + // indistinguishable from on a fresh buffer — so start from 0xA5. + buffer.fill(0xA5); + dma_set(buffer.as_mut_ptr(), 0, 100); + assert!(buffer[..100].iter().all(|&byte| byte == 0)); + assert!(buffer[100..].iter().all(|&byte| byte == 0xA5)); + + // The seeding `sb` writes the low byte of its source register, so a wide fill + // truncates as C's `memset(void*, int, size_t)` requires: `(unsigned char)c`. + buffer.fill(0); + dma_set(buffer.as_mut_ptr(), 0x1FF, 64); + assert!(buffer[..64].iter().all(|&byte| byte == 0xFF)); + + // A negative int sign-extends to 0xFFFF_FFFF_FFFF_FFFF under lp64, and `sb` + // takes its low byte, so the fill is 0xFF. + buffer.fill(0); + dma_set(buffer.as_mut_ptr(), -1, 32); + assert!(buffer[..32].iter().all(|&byte| byte == 0xFF)); + + // Unaligned destination that also crosses a 4 KiB page boundary. + let mut page_buffer = [0u8; 8192]; + let to_boundary = 4096 - (page_buffer.as_ptr() as usize & 4095); + let offset = to_boundary.saturating_sub(5); + dma_set(unsafe { page_buffer.as_mut_ptr().add(offset) }, 0x77, 256); + assert!(page_buffer[offset..offset + 256] + .iter() + .all(|&byte| byte == 0x77)); + + syscalls::syscalls::commit(b"dma-memset-ok"); +} diff --git a/executor/programs/rust/dma_memset_min/.cargo/config.toml b/executor/programs/rust/dma_memset_min/.cargo/config.toml new file mode 100644 index 000000000..8ef8239bb --- /dev/null +++ b/executor/programs/rust/dma_memset_min/.cargo/config.toml @@ -0,0 +1,9 @@ +[target.riscv64im-lambda-vm-elf] +rustflags = [ + "--cfg", "getrandom_backend=\"custom\"", + "-C", "passes=lower-atomic" +] + +[env] +CC_riscv64im_lambda_vm_elf = "clang" +CFLAGS_riscv64im_lambda_vm_elf = "--target=riscv64 -march=rv64im -mabi=lp64 --sysroot=/opt/lambda-vm-sysroot" diff --git a/executor/programs/rust/dma_memset_min/Cargo.lock b/executor/programs/rust/dma_memset_min/Cargo.lock new file mode 100644 index 000000000..45bef20ca --- /dev/null +++ b/executor/programs/rust/dma_memset_min/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "dma_memset_min" +version = "0.1.0" +dependencies = [ + "lambda-vm-syscalls", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "lambda-vm-syscalls" +version = "0.1.0" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "lazy_static", + "rand", + "riscv", + "thiserror", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "riscv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05cfa3f7b30c84536a9025150d44d26b8e1cc20ddf436448d74cd9591eefb25" +dependencies = [ + "critical-section", + "embedded-hal", + "paste", + "riscv-macros", + "riscv-pac", +] + +[[package]] +name = "riscv-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d323d13972c1b104aa036bc692cd08b822c8bbf23d79a27c526095856499799" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "riscv-pac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8188909339ccc0c68cfb5a04648313f09621e8b87dc03095454f1a11f6c5d436" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/executor/programs/rust/dma_memset_min/Cargo.toml b/executor/programs/rust/dma_memset_min/Cargo.toml new file mode 100644 index 000000000..3a98a947c --- /dev/null +++ b/executor/programs/rust/dma_memset_min/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "dma_memset_min" +version = "0.1.0" +edition = "2024" + +[dependencies] +lambda-vm-syscalls = { path = "../../../../syscalls" } diff --git a/executor/programs/rust/dma_memset_min/src/main.rs b/executor/programs/rust/dma_memset_min/src/main.rs new file mode 100644 index 000000000..1705064b6 --- /dev/null +++ b/executor/programs/rust/dma_memset_min/src/main.rs @@ -0,0 +1,17 @@ +use lambda_vm_syscalls as syscalls; + +unsafe extern "C" { + fn memset(dst: *mut u8, fill: i32, count: usize) -> *mut u8; +} + +pub fn main() { + // 43 bytes = five eight-byte rows plus a three-byte tail, so one call yields + // a first row, wide intermediate rows, tail rows and a terminal row. + let mut buffer = [0u8; 43]; + let count = core::hint::black_box(buffer.len()); + + unsafe { + memset(buffer.as_mut_ptr(), 0x3C, count); + } + syscalls::syscalls::commit(&buffer); +} diff --git a/executor/src/tests/dma_tests.rs b/executor/src/tests/dma_tests.rs new file mode 100644 index 000000000..7d4a685c0 --- /dev/null +++ b/executor/src/tests/dma_tests.rs @@ -0,0 +1,359 @@ +use crate::vm::instruction::decoding::Instruction; +use crate::vm::instruction::execution::{ + DMA_MEMCPY_MAX_BYTES, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_GAP, DMA_MEMSET_SYSCALL_NUMBER, + ExecutionError, dma_memset_crosses_limb_boundary, memmove_row_width, memmove_trace_rows, +}; +use crate::vm::memory::Memory; +use crate::vm::registers::Registers; +use proptest::prelude::*; + +fn run_dma(memory: &mut Memory, dst: u64, src: u64, count: u64) -> Result<(), ExecutionError> { + let mut registers = Registers::default(); + let mut pc = 0; + registers.write(17, DMA_MEMCPY_SYSCALL_NUMBER)?; + registers.write(10, dst)?; + registers.write(11, src)?; + registers.write(12, count)?; + Instruction::EcallEbreak.run(&mut pc, &mut registers, memory)?; + Ok(()) +} + +#[test] +fn dma_memcpy_copies_unaligned_body_and_tail() { + let mut memory = Memory::default(); + let input: Vec = (0..27).map(|i| (i * 7 + 3) as u8).collect(); + for (i, &byte) in input.iter().enumerate() { + memory.store_byte(0x1003 + i as u64, byte); + } + + run_dma(&mut memory, 0x2005, 0x1003, input.len() as u64).unwrap(); + assert_eq!( + memory.load_bytes(0x2005, input.len() as u64).unwrap(), + input + ); +} + +#[test] +fn dma_memcpy_has_snapshot_semantics_for_overlap() { + let mut memory = Memory::default(); + let input: Vec = (0..32).map(|i| i as u8).collect(); + for (i, &byte) in input.iter().enumerate() { + memory.store_byte(0x3000 + i as u64, byte); + } + + run_dma(&mut memory, 0x3004, 0x3000, 24).unwrap(); + assert_eq!( + memory.load_bytes(0x3004, 24).unwrap(), + input[..24], + "overlap must read the complete source snapshot before writing" + ); +} + +#[test] +fn dma_memcpy_rejects_wrapping_ranges() { + let mut memory = Memory::default(); + assert!(run_dma(&mut memory, 0x1000, u64::MAX - 3, 8).is_err()); + assert!(run_dma(&mut memory, u64::MAX - 3, 0x1000, 8).is_err()); +} + +#[test] +fn dma_memcpy_rejects_oversized_direct_ecall() { + let mut memory = Memory::default(); + assert!(matches!( + run_dma( + &mut memory, + 0x2000, + 0x1000, + DMA_MEMCPY_MAX_BYTES + 1 + ), + Err(ExecutionError::DmaChunkTooLarge(n)) + if n == DMA_MEMCPY_MAX_BYTES + 1 + )); +} + +/// The row helpers are what the trace builder sizes the DMA trace with and what +/// the CLI reports as the accelerator's cost, so pin them to the chunking rule +/// the trace builder actually walks rather than to the closed form itself. +#[test] +fn dma_row_helpers_match_the_chunk_loop() { + for count in 0..=DMA_MEMCPY_MAX_BYTES { + // The width no longer depends on either end's alignment, but the pairs are + // kept so the row count stays pinned if a future schedule reintroduces it. + for (src, dst) in [(0u64, 0u64), (5, 5), (7, 7), (0, 5), (2, 5), (8, 16)] { + let mut chunks = 0u64; + let mut remaining = count; + let mut offset = 0u64; + while remaining != 0 { + let width = u64::from(memmove_row_width(src, dst, offset, remaining, false)); + remaining -= width; + offset += width; + chunks += 1; + } + assert_eq!( + memmove_trace_rows(src, dst, count, false), + chunks + 1, + "src {src}, dst {dst}, count {count}: the terminal row is always emitted" + ); + } + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + /// Differentially compare the DMA snapshot semantics against a byte-vector + /// oracle. The generated ranges cover unaligned copies, both overlap + /// directions, zero/small/tail lengths, full chunks, and page crossings. + #[test] + fn dma_memcpy_matches_snapshot_oracle( + src_offset in 0usize..768, + dst_offset in 0usize..768, + count in 0usize..=DMA_MEMCPY_MAX_BYTES as usize, + seed in any::(), + ) { + const BASE: u64 = 0x0F00; + const REGION: usize = 1024; + + let mut initial = vec![0u8; REGION]; + let mut state = seed; + for byte in &mut initial { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = state as u8; + } + + let mut expected = initial.clone(); + let snapshot = expected[src_offset..src_offset + count].to_vec(); + expected[dst_offset..dst_offset + count].copy_from_slice(&snapshot); + + let mut memory = Memory::default(); + for (i, &byte) in initial.iter().enumerate() { + memory.store_byte(BASE + i as u64, byte); + } + run_dma( + &mut memory, + BASE + dst_offset as u64, + BASE + src_offset as u64, + count as u64, + ) + .unwrap(); + + let actual = memory.load_bytes(BASE, REGION as u64).unwrap(); + prop_assert_eq!(actual, expected); + } +} + +/// Drives the memset ecall directly. `a1` is a *source address* now, not a fill +/// byte: the accelerator performs a propagating copy, and the guest stub is what +/// seeds the first bytes. +fn run_memset(memory: &mut Memory, dst: u64, src: u64, count: u64) -> Result<(), ExecutionError> { + let mut registers = Registers::default(); + let mut pc = 0; + registers.write(17, DMA_MEMSET_SYSCALL_NUMBER)?; + registers.write(10, dst)?; + registers.write(11, src)?; + registers.write(12, count)?; + Instruction::EcallEbreak.run(&mut pc, &mut registers, memory)?; + Ok(()) +} + +/// What the guest stub does before the ecall: seed eight bytes with the fill. +fn seed(memory: &mut Memory, dst: u64, fill: u8) { + for i in 0..8 { + memory.store_byte(dst + i, fill); + } +} + +#[test] +fn dma_memset_fills_unaligned_body_and_tail() { + let mut memory = Memory::default(); + // 27 bytes at an unaligned base: the stub seeds eight and the ecall propagates + // the remaining nineteen from them. + seed(&mut memory, 0x2005, 0x3C); + run_memset(&mut memory, 0x2005 + 8, 0x2005, 27 - 8).unwrap(); + + assert_eq!(memory.load_bytes(0x2005, 27).unwrap(), vec![0x3Cu8; 27]); + // Neighbours must be untouched. + assert_eq!(memory.load_byte(0x2004), 0); + assert_eq!(memory.load_byte(0x2005 + 27), 0); +} + +#[test] +fn dma_memset_zero_count_writes_nothing() { + let mut memory = Memory::default(); + memory.store_byte(0x3000, 0x11); + run_memset(&mut memory, 0x3008, 0x3000, 0).unwrap(); + assert_eq!(memory.load_byte(0x3000), 0x11); +} + +#[test] +fn dma_memset_rejects_wrapping_range() { + let mut memory = Memory::default(); + assert!(run_memset(&mut memory, u64::MAX - 3, 0x2000, 8).is_err()); +} + +#[test] +fn dma_memset_rejects_oversized_chunk() { + let mut memory = Memory::default(); + assert!(matches!( + run_memset(&mut memory, 0x2000, 0x4000, DMA_MEMCPY_MAX_BYTES + 1), + Err(ExecutionError::DmaChunkTooLarge(n)) if n == DMA_MEMCPY_MAX_BYTES + 1 + )); +} + +#[test] +fn dma_memset_propagates_rather_than_filling() { + // The distinguishing property: the ecall is an overlapping *copy* walked + // forward, so whatever the stub seeded spreads across the range. Seeding a + // non-uniform pattern makes that visible — a real fill could not produce it. + let mut memory = Memory::default(); + for i in 0..8u64 { + memory.store_byte(0x2000 + i, i as u8); + } + run_memset(&mut memory, 0x2008, 0x2000, 16).unwrap(); + + assert_eq!( + memory.load_bytes(0x2000, 24).unwrap(), + (0..24u8).map(|i| i % 8).collect::>(), + "each step must observe the previous step's write" + ); +} + +proptest! { + /// The stub-plus-ecall pair must reproduce a reference fill for any length and + /// any destination alignment: the stub seeds `min(8, count)` bytes and the ecall + /// propagates the rest from them. + #[test] + fn dma_memset_matches_reference_fill( + dst_offset in 0usize..64, + count in 0usize..=(DMA_MEMCPY_MAX_BYTES as usize + 8), + fill in 0u8..=255, + ) { + const BASE: u64 = 0x9000; + const REGION: usize = 400; + + let mut expected = vec![0u8; REGION]; + expected[dst_offset..dst_offset + count].fill(fill); + + let mut memory = Memory::default(); + let dst = BASE + dst_offset as u64; + let seeded = count.min(8); + for i in 0..seeded as u64 { + memory.store_byte(dst + i, fill); + } + if count > seeded { + run_memset(&mut memory, dst + seeded as u64, dst, (count - seeded) as u64).unwrap(); + } + + let actual = memory.load_bytes(BASE, REGION as u64).unwrap(); + prop_assert_eq!(actual, expected); + } +} + +/// The operand contract is what pins `value` on an `is_set` row, so the executor +/// must accept exactly the shapes the AIR can prove -- no wider, or an honest +/// execution becomes unprovable, and no narrower, or the AIR admits executions +/// that never happened. +#[test] +fn dma_memset_rejects_every_gap_but_one() { + for (dst, src, why) in [ + ( + 0x2000u64, + 0x2000u64, + "dst == src leaves the value lanes unconstrained", + ), + (0x2004, 0x2000, "a gap under one row width"), + (0x2020, 0x2000, "a gap over one row width"), + (0x2000, 0x2008, "dst below src propagates the wrong way"), + ] { + let mut memory = Memory::default(); + assert!( + matches!( + run_memset(&mut memory, dst, src, 16), + Err(ExecutionError::DmaMemsetBadGap { .. }) + ), + "{why} (src {src:#x}, dst {dst:#x})" + ); + } + + // Rejected for count 0 too. A zero-length call with a *correct* gap is fine and + // provable (one row, first = end = 1, no read, no write); what this case exercises + // is `dst == src`, which the guard refuses regardless of count. + let mut memory = Memory::default(); + assert!(matches!( + run_memset(&mut memory, 0x2000, 0x2000, 0), + Err(ExecutionError::DmaMemsetBadGap { .. }) + )); + + // The AIR pins the gap limb-wise, so a `src` whose low limb sits within the gap + // of the boundary has no representable successor and must be refused here. + let mut memory = Memory::default(); + assert!(matches!( + run_memset(&mut memory, 0x1_0000_0000, 0xFFFF_FFF8, 8), + Err(ExecutionError::DmaMemsetBadGap { .. }) + )); + + // And the bound must cover the whole range, not just the first row. This chain + // starts clear of the boundary but walks into it: at offset 248 the row is + // `src = 0xFFFF_FFF8, dst = 0x1_0000_0000`, whose low limbs differ by + // `-0xFFFF_FFF8` rather than by the gap, so the AIR rejects that row. Accepting + // the call here would hand an honest guest a trace no prover can prove. + let mut memory = Memory::default(); + assert!( + matches!( + run_memset(&mut memory, 0xFFFF_FF08, 0xFFFF_FF00, 256), + Err(ExecutionError::DmaMemsetBadGap { .. }) + ), + "a memset whose range crosses the 2^32 limb boundary must be refused" + ); + + // The row just inside the boundary is still fine, so the bound is not blanket. + let mut memory = Memory::default(); + let src = 0xFFFF_FFFF - 256 - 8; + seed(&mut memory, src, 0x3C); + run_memset(&mut memory, src + 8, src, 256).unwrap(); + assert_eq!(memory.load_byte(src + 263), 0x3C); + + // And the one legal shape still works. + let mut memory = Memory::default(); + seed(&mut memory, 0x2000, 0x5A); + run_memset(&mut memory, 0x2008, 0x2000, 8).unwrap(); + for addr in 0x2000..0x2010 { + assert_eq!(memory.load_byte(addr), 0x5A, "byte at {addr:#x}"); + } +} + +/// The predicate the guest stub's fallback branch is written against. +/// +/// That branch is assembly and never runs on the host, so this is what pins its +/// logic: the stub takes the plain store loop exactly when this says the range +/// crosses, and the executor refuses the ecall on the same condition. If the two ever +/// disagree, an honest `memset` either aborts or produces an unprovable trace. +#[test] +fn the_memset_limb_boundary_predicate_matches_the_rows_the_air_can_pin() { + // Brute force: for every low limb near the boundary and every legal length, the + // predicate must agree with "some row of this chain has a carrying low limb". + for n in [0u64, 1, 8, 9, 255, 256] { + for delta in 0..600u64 { + let dst = 0x1_0000_0000u64.wrapping_sub(delta); + let any_row_carries = (0..=n).any(|offset| { + let row_dst = dst.wrapping_add(offset); + (row_dst & 0xFFFF_FFFF) + DMA_MEMSET_GAP > 0xFFFF_FFFF + }); + assert_eq!( + dma_memset_crosses_limb_boundary(dst, n), + any_row_carries, + "dst {dst:#x} (low {:#x}), n {n}", + dst & 0xFFFF_FFFF + ); + } + } + + // The stack top is the reachable case, and it is `main`'s own frame. + const STACK_TOP: u64 = 0xFFFF_FFFF_FFFF_FFF0; + assert!(dma_memset_crosses_limb_boundary(STACK_TOP - 8, 256)); + assert!(!dma_memset_crosses_limb_boundary(STACK_TOP - 264, 256)); + // Ordinary heap buffers are nowhere near it. + assert!(!dma_memset_crosses_limb_boundary(0x1_0000, 256)); +} diff --git a/executor/src/tests/mod.rs b/executor/src/tests/mod.rs index 244447b22..6cb04db7c 100644 --- a/executor/src/tests/mod.rs +++ b/executor/src/tests/mod.rs @@ -1,5 +1,7 @@ +pub mod dma_tests; pub mod ecsm_tests; pub mod flamegraph_tests; pub mod hint_tests; pub mod keccak_tests; pub mod memory_tests; +pub mod syscall_tests; diff --git a/executor/src/tests/syscall_tests.rs b/executor/src/tests/syscall_tests.rs new file mode 100644 index 000000000..31fa6e2d6 --- /dev/null +++ b/executor/src/tests/syscall_tests.rs @@ -0,0 +1,28 @@ +use crate::vm::instruction::execution::SyscallNumbers; + +/// `raw()` is the inverse of `TryFrom`: the number the guest puts in `a7` +/// must decode back to the variant it came from. Runs over `ALL`, so a syscall +/// whose `raw()` collides with another's is caught here rather than by a guest +/// silently taking the wrong ecall path. +#[test] +fn raw_round_trips_through_try_from() { + for &syscall in SyscallNumbers::ALL { + assert_eq!( + SyscallNumbers::try_from(syscall.raw()), + Ok(syscall), + "a7 = {} must decode back to {syscall:?}", + syscall.raw() + ); + } +} + +/// Two syscalls sharing an `a7` would make `TryFrom` pick one and leave the other +/// unreachable, and `ALL` is what the CLI parity test enumerates. +#[test] +fn every_syscall_has_a_distinct_a7() { + let mut raws: Vec = SyscallNumbers::ALL.iter().map(|s| s.raw()).collect(); + let listed = raws.len(); + raws.sort_unstable(); + raws.dedup(); + assert_eq!(raws.len(), listed, "two syscalls share an a7 value"); +} diff --git a/executor/src/vm/instruction/execution.rs b/executor/src/vm/instruction/execution.rs index 592af95e8..cd3fc8582 100644 --- a/executor/src/vm/instruction/execution.rs +++ b/executor/src/vm/instruction/execution.rs @@ -7,18 +7,44 @@ use crate::vm::{ const REGULAR_PC_UPDATE: u64 = 4; -pub enum SyscallNumbers { - // Placeholder discriminant. The actual syscall value is KECCAK_SYSCALL_NUMBER. +/// Declares `SyscallNumbers` and derives `ALL` from the same variant list, so a +/// syscall added to the enum is enumerated by everything driven off `ALL` (the +/// CLI's accelerator-parity test) without a second list to keep in sync. +macro_rules! syscall_numbers { + ($($(#[$meta:meta])* $variant:ident = $discriminant:literal,)+) => { + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub enum SyscallNumbers { + $($(#[$meta])* $variant = $discriminant,)+ + } + + impl SyscallNumbers { + /// Every variant, generated alongside the enum. + pub const ALL: &'static [SyscallNumbers] = &[$(SyscallNumbers::$variant,)+]; + } + }; +} + +syscall_numbers! { + /// Placeholder discriminant. The actual syscall value is `KECCAK_SYSCALL_NUMBER`. KeccakPermute = 0, Print = 1, Panic = 2, Commit = 64, Halt = 93, - // Placeholder discriminant. The actual syscall value is ECSM_SYSCALL_NUMBER. + /// Placeholder discriminant. The actual syscall value is `ECSM_SYSCALL_NUMBER`. Ecsm = 94, - // Placeholder discriminant. The actual syscall value is HINT_SYSCALL_NUMBER. - // Non-constraining hint (host computes modular inverse/sqrt, guest verifies). + /// Placeholder discriminant. The actual syscall value is + /// `HINT_SYSCALL_NUMBER`. Non-constraining hint (host computes modular + /// inverse/sqrt, guest verifies). Hint = 95, + /// Placeholder discriminant. The actual syscall value is + /// `DMA_MEMCPY_SYSCALL_NUMBER`. `memcpy` and `memmove` chunks are proven by + /// the MEMMOVE table. + DmaMemcpy = 96, + /// Placeholder discriminant. The actual syscall value is + /// `DMA_MEMSET_SYSCALL_NUMBER`. `memset` chunks are proven by the same + /// MEMMOVE table, which derives the inverted timestamp order from this number. + DmaMemset = 97, } /// Syscall number for KeccakPermute (u64::MAX - 1 = 0xFFFF_FFFF_FFFF_FFFE). @@ -34,6 +60,110 @@ const KECCAK_STATE_BYTES: u64 = 25 * 8; /// bus as `[lo32, hi32] = [2^32 - 11, 2^32 - 1]`. pub const ECSM_SYSCALL_NUMBER: u64 = u64::MAX - 10; +/// Syscall number for the copy accelerator, serving `memcpy` and `memmove`. +/// +/// The spec uses ECALL number `-30`, i.e. `u64::MAX - 29 = 0xFFFF_FFFF_FFFF_FFE2`, +/// which the MEMMOVE table puts on the `Ecall` bus as +/// `[lo32, hi32] = [2^32 - 30, 2^32 - 1]`. +/// +/// It starts a new group deliberately. `-1` through `-10` are reserved for hash +/// accelerators (`-1` SHA256, `-2` KECCAK today), and the earlier `-3`/`-4` pair sat +/// inside that range. Must match `syscalls/src/syscalls.rs`. +pub const DMA_MEMCPY_SYSCALL_NUMBER: u64 = u64::MAX - 29; +/// Maximum bytes accepted by one DMA ecall. The guest `memcpy` stub chunks +/// larger copies, and the prover enforces this bound on every first DMA row. +pub const DMA_MEMCPY_MAX_BYTES: u64 = 256; + +/// Width of one MEMMOVE row: eight bytes while at least eight remain, then one per +/// remaining byte. +/// +/// The AIR does not require this schedule. `tail` is a free bit, constrained only by +/// `(1 - tail) * lt8 = 0`, so a one-byte row is legal at any count and a prover may +/// walk one-byte rows to reach eight-byte alignment and keep the body on the cheaper +/// `MEMW_A` table. This function simply declines to. +/// +/// It used to do exactly that, splitting whenever `src` and `dst` shared a residue +/// mod 8. Measured on a real mainnet block, that cost more than it saved: prove time +/// 109.169s without the split against 111.780s with it, with row counts lower and +/// committed elements unchanged on three of four fixtures and lower on the fourth. +/// The reason is that the chip has only two widths. Aligning an end costs up to seven +/// one-byte rows at the head and shifts the tail into up to seven more, so buying two +/// or three `MEMW_A` rows costs a dozen rows on the wider `MEMW` table. A schedule +/// gated on a minimum body length could still win; the AIR already permits it, so it +/// can be added later without touching the constraint system or the proof format. +/// +/// `src`, `dst`, `offset` and `to_commit` stay in the signature for that reason: every +/// consumer — the executor's row count, the trace builder, the sizing pass and the CLI +/// report — already routes through here, so a future schedule needs no new plumbing. +pub fn memmove_row_width(src: u64, dst: u64, offset: u64, remaining: u64, to_commit: bool) -> u8 { + if remaining < 8 { + return 1; + } + let _ = (src, dst, offset, to_commit); + 8 +} + +/// Total MEMMOVE rows one ecall produces: its data rows plus the terminal row. +/// +/// A pure function of `(dst, count)` — the width now depends on the destination's +/// alignment, not on `count` alone. Every consumer that needs a row count (the +/// trace builder, the sizing pass, the CLI's accelerator report) goes through this +/// function, so none of them can drift from the trace the prover actually builds. +pub fn memmove_trace_rows(src: u64, dst: u64, count: u64, to_commit: bool) -> u64 { + let mut rows = 1; + let mut offset = 0u64; + let mut remaining = count; + while remaining != 0 { + let width = u64::from(memmove_row_width(src, dst, offset, remaining, to_commit)); + offset += width; + remaining -= width; + rows += 1; + } + rows +} +/// Syscall number for `memset`, the same accelerator run with the read/write +/// timestamp order inverted. +/// +/// ECALL number `-32`, i.e. `u64::MAX - 31`. It is not `-31` because +/// [`HINT_SYSCALL_NUMBER`] already holds that, so the copy group is `-30` and `-32` +/// with the hint wedged between. Must match `syscalls/src/syscalls.rs`. +/// +/// MEMMOVE decodes the functionality by receiving the syscall number as a line in +/// `is_set`, so any number on that line is reachable by some field element and no +/// particular neighbour is special. `IS_BIT(is_set)` is what carries the decoding +/// argument, and it does so whatever the numbering. +pub const DMA_MEMSET_SYSCALL_NUMBER: u64 = u64::MAX - 31; + +/// The one operand shape a DMA memset ecall may have: the destination trails the +/// source by exactly one wide row. +/// +/// This is not a convention, it is what makes the call sound. The accelerator runs +/// an `is_set` call with the write at `T+1` and the read at `T+2`, so a row's read +/// observes writes the same call already made — which is what propagates the seed. +/// That pins the copied value only while the read resolves to a *different*, +/// already-written address. With `dst == src` the read and the write address the +/// same cell at adjacent timestamps, the memory argument is satisfied by +/// `value == value`, and the eight value lanes become free field elements: a prover +/// could put anything it liked into RAM. The AIR therefore pins `dst = src + 8` on +/// every `is_set` row, and this constant is what both sides import so the two +/// bounds cannot drift. +pub const DMA_MEMSET_GAP: u64 = 8; + +/// Whether a memset over `[dst, dst + n)` would straddle the 2^32 limb boundary. +/// +/// The AIR pins `dst = src + 8` limb-wise on every `is_set` row, so a row whose low +/// limb carries has no representable successor. The executor refuses such a call and +/// the guest stub steers around it with a plain store loop; this is the single +/// predicate both sides are written against, so they cannot drift. +/// +/// It is not a theoretical case. The stack starts at `STACK_TOP = 0xFFFF_FFFF_FFFF_FFF0`, +/// whose low limb is `0xFFFF_FFF0`, so a buffer within `n + 8` bytes of the stack top +/// crosses — which is `main`'s own frame. +pub const fn dma_memset_crosses_limb_boundary(dst: u64, n: u64) -> bool { + // `n` is bounded by DMA_MEMCPY_MAX_BYTES on the ecall path, so this cannot wrap. + (dst & 0xFFFF_FFFF) + n + DMA_MEMSET_GAP > 0xFFFF_FFFF +} + /// Syscall number for the non-constraining `Hint` ecall. /// /// The host computes a modular inverse or square root and writes it back to the @@ -88,6 +218,8 @@ impl TryFrom for SyscallNumbers { 93 => Ok(SyscallNumbers::Halt), v if v == KECCAK_SYSCALL_NUMBER => Ok(SyscallNumbers::KeccakPermute), v if v == ECSM_SYSCALL_NUMBER => Ok(SyscallNumbers::Ecsm), + v if v == DMA_MEMCPY_SYSCALL_NUMBER => Ok(SyscallNumbers::DmaMemcpy), + v if v == DMA_MEMSET_SYSCALL_NUMBER => Ok(SyscallNumbers::DmaMemset), v if v == HINT_SYSCALL_NUMBER => Ok(SyscallNumbers::Hint), _ => Err(()), } @@ -99,9 +231,26 @@ impl TryFrom for SyscallNumbers { pub enum Accelerator { Keccak, Ecsm, + Dma, } impl SyscallNumbers { + /// The raw `a7` value this syscall is invoked with. The accelerator numbers + /// exceed `isize::MAX`, so they can't be enum discriminants. + pub fn raw(self) -> u64 { + match self { + SyscallNumbers::KeccakPermute => KECCAK_SYSCALL_NUMBER, + SyscallNumbers::Ecsm => ECSM_SYSCALL_NUMBER, + SyscallNumbers::DmaMemcpy => DMA_MEMCPY_SYSCALL_NUMBER, + SyscallNumbers::DmaMemset => DMA_MEMSET_SYSCALL_NUMBER, + SyscallNumbers::Hint => HINT_SYSCALL_NUMBER, + SyscallNumbers::Print => SyscallNumbers::Print as u64, + SyscallNumbers::Panic => SyscallNumbers::Panic as u64, + SyscallNumbers::Commit => SyscallNumbers::Commit as u64, + SyscallNumbers::Halt => SyscallNumbers::Halt as u64, + } + } + /// The accelerator this syscall drives, if any. Exhaustive `match self`: /// adding a `SyscallNumbers` variant is a compile error here, so a new /// accelerator can't be silently missed by counters that consume this. @@ -109,6 +258,7 @@ impl SyscallNumbers { match self { SyscallNumbers::KeccakPermute => Some(Accelerator::Keccak), SyscallNumbers::Ecsm => Some(Accelerator::Ecsm), + SyscallNumbers::DmaMemcpy | SyscallNumbers::DmaMemset => Some(Accelerator::Dma), SyscallNumbers::Print | SyscallNumbers::Panic | SyscallNumbers::Commit @@ -550,6 +700,78 @@ impl Instruction { src2_val = addr_xg; dst_val = addr_k; } + SyscallNumbers::DmaMemcpy => { + // memcpy(dst = x10, src = x11, n = x12). Snapshot the input + // before writing, which also gives this ecall well-defined + // memmove semantics when the regions overlap. The DMA trace + // authenticates the same read-at-T+1/write-at-T+2 relation. + let dst = registers.read(10)?; + let src = registers.read(11)?; + let n = registers.read(12)?; + if n > DMA_MEMCPY_MAX_BYTES { + return Err(ExecutionError::DmaChunkTooLarge(n)); + } + dst.checked_add(n).ok_or(MemoryError::AddressOverflow)?; + src.checked_add(n).ok_or(MemoryError::AddressOverflow)?; + + // The fixed-size scratch avoids a heap allocation on every + // hot-path ecall while preserving snapshot semantics. + let mut bytes = [0u8; DMA_MEMCPY_MAX_BYTES as usize]; + for (i, byte) in bytes[..n as usize].iter_mut().enumerate() { + *byte = memory.load_byte(src + i as u64); + } + for (i, &byte) in bytes[..n as usize].iter().enumerate() { + memory.store_byte(dst + i as u64, byte); + } + src2_val = memmove_trace_rows(src, dst, n, false); + dst_val = n; + } + SyscallNumbers::DmaMemset => { + // memset(dst = x10, src = x11, n = x12) — a *propagating* copy, + // not a fill. The stub seeds the first eight bytes with an + // ordinary store and calls with `dst = seed_end`, + // `src = seed_start`, so this is a plain overlapping memmove + // and only the timestamp order distinguishes it: the accelerator + // writes at T+1 and reads at T+2, so each step observes the + // previous step's write and the seed propagates across the range. + // A forward byte walk is exactly that semantics. + let dst = registers.read(10)?; + let src = registers.read(11)?; + let n = registers.read(12)?; + if n > DMA_MEMCPY_MAX_BYTES { + return Err(ExecutionError::DmaChunkTooLarge(n)); + } + dst.checked_add(n).ok_or(MemoryError::AddressOverflow)?; + src.checked_add(n).ok_or(MemoryError::AddressOverflow)?; + + // The operand contract, enforced unconditionally so that the + // executions this accepts are exactly the ones the AIR can + // prove. The low-limb condition is the second half of that: the + // AIR pins the gap limb-wise and so cannot express a carry out + // of the low limb, and rejecting the straddle here is cheaper + // than spending a carry column on an address range no guest + // reaches (cf. the HINT limb bounds below). + // + // The bound has to cover the whole range, not just the first + // row. `src` and `dst` both advance by the row width, and the + // AIR pins the gap on EVERY `is_set` row, so a chain that starts + // clear of the boundary can still walk into it: with + // `src = 0xFFFF_FF00, n = 256` the row at offset 248 has + // `SRC_0 = 0xFFFF_FFF8` against `DST_0 = 0`, which satisfies the + // gap in full 64-bit arithmetic but not limb-wise. Bounding the + // starting limb alone would accept an execution no prover can + // then prove. + if dma_memset_crosses_limb_boundary(src, n) || dst != src + DMA_MEMSET_GAP { + return Err(ExecutionError::DmaMemsetBadGap { src, dst }); + } + + for i in 0..n { + let byte = memory.load_byte(src + i); + memory.store_byte(dst + i, byte); + } + src2_val = memmove_trace_rows(src, dst, n, false); + dst_val = n; + } SyscallNumbers::Hint => { // Non-constraining hint: host computes a modular inverse/sqrt // and writes it to the guest, which verifies it (and falls back @@ -766,6 +988,13 @@ pub enum ExecutionError { EcsmAddressOverflow, #[error("ECSM xG and k operand ranges overlap")] EcsmOperandOverlap, + #[error("DMA chunk has {0} bytes; maximum per ecall is {DMA_MEMCPY_MAX_BYTES}")] + DmaChunkTooLarge(u64), + #[error( + "DMA memset needs dst == src + {DMA_MEMSET_GAP}, with src and src + n clear of \ + the 2^32 limb boundary; got src {src:#x}, dst {dst:#x}" + )] + DmaMemsetBadGap { src: u64, dst: u64 }, #[error("Hint address range overflows the lower 32-bit limb")] HintAddressOverflow, #[error("Unknown hint selector: {0}")] diff --git a/executor/src/vm/logs.rs b/executor/src/vm/logs.rs index de6b73d0b..d8a20dbc2 100644 --- a/executor/src/vm/logs.rs +++ b/executor/src/vm/logs.rs @@ -9,8 +9,18 @@ /// For ECALL instructions, these fields are repurposed (since decode sets read_register1/2=false, /// write_register=false, so src/dst are unconstrained): /// - `src1_val` = syscall number (from x17): 64=Commit, 93=Halt, etc. -/// - `src2_val` = buf_addr (x11) for Commit, 0 otherwise -/// - `dst_val` = count (x12) for Commit, 0 otherwise +/// - `src2_val` = Commit: buf_addr (x11); Keccak: state_addr; ECSM: addr_xG; +/// Hint: input addr; DMA memcpy and DMA memset: the number of MEMMOVE rows the +/// call produces. 0 for every other syscall. +/// - `dst_val` = Commit: count (x12); ECSM: addr_k; Hint: output addr; +/// DMA memcpy and DMA memset: byte count. 0 for every other syscall, Keccak +/// included. +/// +/// The row count is carried rather than recomputed downstream because it is not a +/// function of the byte count: a row is eight bytes or one, and the schedule reads +/// `src % 8` and `dst % 8` to decide. The executor is the only place that holds +/// `src`, `dst` and `count` at once, so it derives the count there and the CLI's +/// accelerator report just sums it. #[derive(Debug, Clone)] pub struct Log { /// PC before instruction execution (use this to look up the instruction) @@ -21,9 +31,9 @@ pub struct Log { /// For ECALL: syscall number from x17. pub src1_val: u64, /// Value of src2 register before execution (if used by the instruction). - /// For ECALL Commit: buf_addr from x11. + /// For ECALL: see the per-syscall table above. pub src2_val: u64, /// Value of dst register after execution (if used by the instruction). - /// For ECALL Commit: count from x12. + /// For ECALL: see the per-syscall table above. pub dst_val: u64, } diff --git a/executor/tests/rust.rs b/executor/tests/rust.rs index 1c13ad1a5..c2e8fa51e 100644 --- a/executor/tests/rust.rs +++ b/executor/tests/rust.rs @@ -1,6 +1,10 @@ use executor::{ elf::Elf, - vm::execution::{Executor, ReturnValues}, + vm::execution::{ExecutionResult, Executor, ReturnValues}, + vm::instruction::{ + decoding::Instruction, + execution::{DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER}, + }, }; // NOTE: These tests require 64-bit RISC-V ELF files (RV64IM). @@ -117,6 +121,112 @@ fn test_vector() { ); } +fn run_guest(path: &str) -> ExecutionResult { + let elf_data = std::fs::read(path).unwrap(); + let program = Elf::load(&elf_data).unwrap(); + Executor::new(&program, vec![]).unwrap().run().unwrap() +} + +/// DMA ecalls the guest actually executed. Zero means the copies were served by +/// `compiler_builtins` rather than by the accelerated `memcpy`. +fn dma_ecall_count(result: &ExecutionResult) -> usize { + result + .logs + .iter() + .filter(|log| { + log.src1_val == DMA_MEMCPY_SYSCALL_NUMBER + && matches!( + result.instructions.get(&log.current_pc), + Some(Instruction::EcallEbreak) + ) + }) + .count() +} + +#[test] +fn test_dma_memcpy() { + let result = run_guest("./program_artifacts/rust/dma_memcpy_min.elf"); + + assert_eq!( + result.return_values.memory_values, + b"DMA copies eight-byte rows and a short tail" + ); + assert!( + dma_ecall_count(&result) > 0, + "the strong memcpy symbol must execute at least one DMA ecall" + ); +} + +#[test] +fn test_dma_memcpy_cases() { + run_program_and_check_public_output( + "./program_artifacts/rust/dma_memcpy_cases.elf", + b"dma-cases-ok".to_vec(), + vec![], + ); +} + +/// The guests above declare `memcpy` themselves, which leaves the symbol +/// undefined in their objects and forces the linker to resolve it. This guest +/// never names `memcpy`: its copies are the ones the compiler emits, which is +/// the case that silently degrades if the strong definition ever stops winning +/// symbol resolution — the guest keeps producing the right output and only the +/// ecall count drops to zero. +#[test] +fn test_dma_memcpy_compiler_emitted_copies() { + let result = run_guest("./program_artifacts/rust/dma_memcpy_implicit.elf"); + + assert_eq!(result.return_values.memory_values, b"dma-implicit-ok"); + assert!( + dma_ecall_count(&result) > 0, + "compiler-emitted copies must reach the DMA ecall; a zero count means the \ + guest fell back to the weak compiler_builtins memcpy" + ); +} + +/// `memmove` shares the copy ecall with `memcpy`, so the only thing distinguishing it +/// is the stub's backward chunking when the ranges overlap with `dst` above `src`. +/// The guest walks both directions across the 256-byte chunk boundary; without this +/// the symbol was exercised only in `prove_elfs_tests`, where a failure reads as a +/// proving bug rather than an execution one. +#[test] +fn test_dma_memmove_cases() { + let elf_data = std::fs::read("./program_artifacts/rust/dma_memmove_cases.elf").unwrap(); + let program = Elf::load(&elf_data).unwrap(); + let result = Executor::new(&program, vec![]).unwrap().run().unwrap(); + + assert_eq!(result.return_values.memory_values, b"dma-memmove-ok"); + assert!( + result.logs.iter().any(|log| { + log.src1_val == DMA_MEMCPY_SYSCALL_NUMBER + && matches!( + result.instructions.get(&log.current_pc), + Some(Instruction::EcallEbreak) + ) + }), + "the strong memmove symbol must execute at least one copy ecall" + ); +} + +#[test] +fn test_dma_memset_cases() { + let elf_data = std::fs::read("./program_artifacts/rust/dma_memset_cases.elf").unwrap(); + let program = Elf::load(&elf_data).unwrap(); + let result = Executor::new(&program, vec![]).unwrap().run().unwrap(); + + assert_eq!(result.return_values.memory_values, b"dma-memset-ok"); + assert!( + result.logs.iter().any(|log| { + log.src1_val == DMA_MEMSET_SYSCALL_NUMBER + && matches!( + result.instructions.get(&log.current_pc), + Some(Instruction::EcallEbreak) + ) + }), + "the strong memset symbol must execute at least one DMA ecall" + ); +} + #[test] fn test_hashmap() { run_program_and_check_output("./program_artifacts/rust/hashmap.elf", 3, vec![]); diff --git a/prover/src/auto_storage.rs b/prover/src/auto_storage.rs index b4718974c..e7051216a 100644 --- a/prover/src/auto_storage.rs +++ b/prover/src/auto_storage.rs @@ -14,6 +14,9 @@ use crate::tables::dvrm::{bus_interactions as dvrm_buses, cols::NUM_COLUMNS as D use crate::tables::halt::{bus_interactions as halt_buses, cols::NUM_COLUMNS as HALT_COLS}; use crate::tables::load::{bus_interactions as load_buses, cols::NUM_COLUMNS as LOAD_COLS}; use crate::tables::lt::{bus_interactions as lt_buses, cols::NUM_COLUMNS as LT_COLS}; +use crate::tables::memmove::{ + bus_interactions as memmove_buses, cols::NUM_COLUMNS as MEMMOVE_COLS, +}; use crate::tables::memw::{bus_interactions as memw_buses, cols::NUM_COLUMNS as MEMW_COLS}; use crate::tables::memw_aligned::{ bus_interactions as memw_a_buses, cols::NUM_COLUMNS as MEMW_A_COLS, @@ -178,6 +181,12 @@ fn table_specs(lengths: &TableLengths) -> Vec { aux_cols(commit_buses().len()), 1, ), + ( + lengths.memmove_padded_rows, + MEMMOVE_COLS as u64, + aux_cols(memmove_buses().len()), + 1, + ), // BITWISE / DECODE / PAGE / REGISTER take the preprocessed-trace commit // path: it extracts ALL columns into the LDE and builds two Merkle trees // (precomputed_tree + mult_tree), so main_cols = full NUM_COLUMNS and diff --git a/prover/src/constraints/templates.rs b/prover/src/constraints/templates.rs index 04932eab8..99df7864e 100644 --- a/prover/src/constraints/templates.rs +++ b/prover/src/constraints/templates.rs @@ -372,3 +372,33 @@ pub fn emit_add_pair> let root_1 = bit(b, c1, carry_1); b.emit_base(idx + 1, root_1); } + +/// A 64-bit ADD that rejects unsigned overflow on active, non-terminal rows — +/// those where the `active_column` value minus the `end_column` value equals 1. +/// +/// The low-word carry remains boolean on every row. On active non-terminal +/// rows, the high-word carry is constrained to zero instead of merely boolean, +/// so `lhs + rhs` cannot wrap modulo `2^64`. Terminal and padding rows leave the +/// high carry unconstrained because their computed successor is not consumed. +pub fn emit_add_pair_no_overflow>( + b: &mut B, + idx: usize, + active_column: usize, + end_column: usize, + lhs: &AddOperand, + rhs: &AddOperand, + sum: &AddOperand, +) { + let inv_2_32 = b.const_base(INV_SHIFT_32); + let carry_0 = (add_operand_lo(b, lhs) + add_operand_lo(b, rhs) - add_operand_lo(b, sum)) + * inv_2_32.clone(); + let carry_1 = (add_operand_hi(b, lhs) + add_operand_hi(b, rhs) + carry_0.clone() + - add_operand_hi(b, sum)) + * inv_2_32; + + let one = b.one(); + b.emit_base(idx, carry_0.clone() * (one - carry_0)); + + let active = b.main(0, active_column) - b.main(0, end_column); + b.emit_base(idx + 1, active * carry_1); +} diff --git a/prover/src/continuation.rs b/prover/src/continuation.rs index df764ff18..60e18fd3d 100644 --- a/prover/src/continuation.rs +++ b/prover/src/continuation.rs @@ -1991,6 +1991,34 @@ mod tests { ); } + #[test] + fn test_dma_memcpy_across_continuation_epochs() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = std::fs::read( + workspace_root.join("executor/program_artifacts/rust/dma_memcpy_min.elf"), + ) + .expect("dma_memcpy_min.elf not found — build its make target"); + let opts = ProofOptions::default_test_options(); + + let bundle = prove_continuation(&elf_bytes, &[], 6, &opts) + .expect("DMA continuation proof generation"); + assert!( + bundle.num_epochs() > 1, + "64-cycle epochs must split the DMA guest" + ); + + let output = verify_continuation(&elf_bytes, &bundle, &opts) + .expect("DMA continuation verification") + .expect("honest DMA continuation must verify"); + assert_eq!( + output, b"DMA copies eight-byte rows and a short tail", + "continuation output must match the copied bytes" + ); + } + // Supplied genesis roots must verify identically to the trustless recompute, // and a tampered root (DECODE or a page) must be rejected. `data_page_touch` // touches a real ELF `.data` page, unlike this file's stack-only fixtures. diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 79ef4c715..a8ae20336 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -54,9 +54,9 @@ use crate::test_utils::{ E, F, VmAir, create_bitwise_air, create_branch_air, create_bytewise_air, create_commit_air, create_cpu_air, create_cpu32_air, create_decode_air, create_dvrm_air, create_ecdas_air, create_ecsm_air, create_eq_air, create_halt_air, create_hint_air, create_keccak_air, - create_keccak_rc_air, create_keccak_rnd_air, create_load_air, create_lt_air, create_memw_air, - create_memw_aligned_air, create_memw_register_air, create_mul_air, create_page_air, - create_register_air, create_shift_air, create_store_air, + create_keccak_rc_air, create_keccak_rnd_air, create_load_air, create_lt_air, + create_memmove_air, create_memw_air, create_memw_aligned_air, create_memw_register_air, + create_mul_air, create_page_air, create_register_air, create_shift_air, create_store_air, }; // Re-exported for downstream hosts and verifier guests (e.g. the in-VM @@ -82,8 +82,8 @@ pub struct RuntimePageRange { /// Number of tables that always contribute exactly one sub-proof, regardless /// of `TableCounts`: bitwise, decode, halt, commit, keccak, keccak_rnd, -/// keccak_rc, register, ecsm, ecdas, hint. -pub const FIXED_TABLE_COUNT: usize = 11; +/// keccak_rc, register, ecsm, ecdas, hint, memmove. +pub const FIXED_TABLE_COUNT: usize = 12; /// Number of chunks for each split table. /// The verifier needs this to reconstruct matching AIRs. @@ -523,6 +523,7 @@ pub(crate) struct VmAirs { pub ecsm: VmAir, pub ecdas: VmAir, pub hint: VmAir, + pub memmove: VmAir, pub register: VmAir, pub pages: Vec, pub memw_registers: Vec, @@ -549,6 +550,7 @@ impl VmAirs { (self.ecsm.as_ref(), &mut traces.ecsm, &()), (self.ecdas.as_ref(), &mut traces.ecdas, &()), (self.hint.as_ref(), &mut traces.hint, &()), + (self.memmove.as_ref(), &mut traces.memmove, &()), (self.register.as_ref(), &mut traces.register, &()), ]; if self.include_halt { @@ -624,6 +626,7 @@ impl VmAirs { self.ecsm.as_ref(), self.ecdas.as_ref(), self.hint.as_ref(), + self.memmove.as_ref(), self.register.as_ref(), ]; if self.include_halt { @@ -796,6 +799,7 @@ impl VmAirs { let ecsm: VmAir = Box::new(create_ecsm_air(proof_options)); let ecdas: VmAir = Box::new(create_ecdas_air(proof_options)); let hint: VmAir = Box::new(create_hint_air(proof_options)); + let memmove: VmAir = Box::new(create_memmove_air(proof_options)); let register: VmAir = if let Some((commitment, num_preprocessed_cols)) = register_preprocessed { Box::new( @@ -917,6 +921,7 @@ impl VmAirs { ecsm, ecdas, hint, + memmove, register, pages, memw_registers, @@ -935,6 +940,11 @@ impl VmAirs { /// Compute the bus balance offset for the COMMIT[index, value] bus. /// +/// The MEMMOVE chip commits eight bytes per row but sends them as eight +/// `(index, value)` pairs, one per byte, so this rebuild is independent of the +/// prover's row schedule — which it has to be: the schedule restarts at every +/// commit ECALL and the verifier sees only the concatenated `public_output`. +/// /// For each public output byte at index `i` with value `v`: /// `fingerprint = z - (BusId::Commit * α^0 + i * α^1 + v * α^2)` /// `term = +1 / fingerprint` diff --git a/prover/src/tables/commit.rs b/prover/src/tables/commit.rs index 65e74f182..9122283af 100644 --- a/prover/src/tables/commit.rs +++ b/prover/src/tables/commit.rs @@ -1,54 +1,52 @@ //! COMMIT (ECALL) table for writing bytes to stdout. //! //! This table handles the `write` syscall (ECALL #64): writing bytes from a memory -//! buffer to stdout. It uses a **recursive design** — each row commits one byte, -//! and rows are linked via a self-referencing "CommitNextByte" bus. +//! buffer to stdout. It is **one row per ECALL** — it accepts the syscall number, +//! reads the operand registers and advances the committed-length register, then +//! defers the byte copying itself to the MEMMOVE chip over `BusId::CommitDefer`. //! -//! Only the first row of each commit sequence receives from the CPU's ECALL bus; -//! subsequent rows receive from the previous commit row via the CommitNextByte bus. +//! The per-byte recursion this table used to run, and its self-referencing +//! `CommitNextByte` bus, are gone: MEMMOVE walks the buffer instead, and it — not +//! this table — is what sends the committed bytes on `BusId::Commit`, as eight +//! `(index, value)` pairs per row. That is the fact to keep in mind when reasoning +//! about the verifier, which rebuilds that bus from `public_output` +//! (`compute_commit_bus_offset`). //! -//! ## Columns (19 total) +//! ## Columns (8 total) //! - `timestamp`: DWordWL (2 cols) — timestamp of the ECALL -//! - `index`: BaseField (1 col) — global byte index for this committed value -//! - `address`: DWordWL (2 cols) — current buffer address -//! - `address_incr`: DWordHL (4 cols) — address + 1, as 4 halfwords -//! - `count`: DWordWL (2 cols) — remaining byte count -//! - `count_decr`: DWordHL (4 cols) — count - 1 as 4 halfwords (or all 0xFFFF when count=0) -//! - `first`: Bit — first row in a commit sequence -//! - `end`: Bit — last row (count was 0) -//! - `value`: Byte — the byte being committed +//! - `index`: BaseField (1 col) — global byte index the committed range starts at +//! - `address`: DWordWL (2 cols) — buffer address the committed range starts at +//! - `count`: DWordWL (2 cols) — number of bytes this ECALL commits //! - `mu`: Bit — multiplicity (1 for real rows, 0 for padding) //! -//! ## Bus Interactions (18 total) -//! - **Receiver**: Ecall bus — receives `[timestamp_lo, timestamp_hi, constant(64), constant(0)]` from CPU (mult = first) -//! - **Sender**: CommitNextByte bus — sends to next row (mult = mu - end) -//! - **Receiver**: CommitNextByte bus — receives from prev row (mult = mu - first) -//! - **Sender**: IsHalfword bus — range checks for count_decr halfwords (×4, mult = mu) -//! - **Sender**: IsHalfword bus — range checks for address_incr halfwords (×4, mult = mu) -//! - **Sender**: Zero bus — end detection via count_decr (mult = mu) -//! - **Sender**: Memw bus — read+write x10 register (fd=1→count) at ts (mult = first) -//! - **Sender**: Memw bus — read x11 register (buf_addr) at ts (mult = first) -//! - **Sender**: Memw bus — read x12 register (count) at ts (mult = first) -//! - **Sender**: Memw bus — read+write x254 commit index at ts (mult = first) -//! - **Sender**: Memw bus — read memory byte at ts (mult = mu - end) -//! - **Sender**: Commit bus — sends committed `(index, value)` pairs (mult = mu - end) +//! There is no `first` column: one row per ECALL means a real row is always the first +//! row of its commit, so `first` was identically `mu` and every multiplicity reads +//! `mu` instead. `address_incr`, `count_decr`, `end` and `value` modelled the per-byte +//! sequence and went with it. //! -//! ## Constraints (8 total) -//! - `range_first`: first * (1 - first) = 0 (degree 2) -//! - `range_end`: end * (1 - end) = 0 (degree 2) +//! ## Bus Interactions (6 total) +//! - **Receiver**: Ecall bus — receives `[timestamp_lo, timestamp_hi, constant(64), constant(0)]` from CPU (mult = mu) +//! - **Sender**: CommitDefer bus — hands the byte loop to MEMMOVE (mult = mu) +//! - **Sender**: Memw bus — read+write x10 register (fd=1→count) at ts (mult = mu) +//! - **Sender**: Memw bus — read x11 register (buf_addr) at ts (mult = mu) +//! - **Sender**: Memw bus — read x12 register (count) at ts (mult = mu) +//! - **Sender**: Memw bus — read+write x254 commit index at ts (mult = mu) +//! +//! The per-byte `Memw` read and the `Commit` `(index, value)` sender moved to MEMMOVE, +//! which sends the committed bytes itself. The eight `IsHalfword` range checks and the +//! `Zero` end-detection went with the columns they checked. `CommitNextByte` is retired +//! (bus id 20 is now a reserved hole). The count is pinned by +//! `commit_tests::test_bus_interactions_count`. +//! +//! ## Constraints (1 total) //! - `range_mu`: mu * (1 - mu) = 0 (degree 2) -//! - `first_or_end_implies_mu`: (first + end) * (1 - mu) = 0 (degree 2) -//! - `address_incr_carry_0`: ADD template carry_0 for address + 1 = address_incr (degree 2) -//! - `address_incr_carry_1`: ADD template carry_1 for address + 1 = address_incr (degree 2) -//! - `count_decr_carry_0`: SUB template carry_0 for count_decr + 1 = count (degree 2) -//! - `count_decr_carry_1`: SUB template carry_1 for count_decr + 1 = count (degree 2) //! use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; use stark::trace::TraceTable; use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; -use crate::constraints::templates::{AddOperand, emit_add_pair, emit_is_bit}; +use crate::constraints::templates::emit_is_bit; use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable}; @@ -68,7 +66,7 @@ pub mod cols { pub const TIMESTAMP_1: usize = 1; // Commit index (BaseField: 1 col) - /// index: global byte index of the committed value + /// index: global byte index the committed range starts at pub const INDEX: usize = 2; // Buffer address (DWordWL: 2 cols) @@ -77,49 +75,21 @@ pub mod cols { /// address[1]: high 32 bits pub const ADDRESS_1: usize = 4; - // address + 1 (DWordHL: 4 halfword cols) - /// address_incr[0]: halfword 0 (bits 0-15) - pub const ADDRESS_INCR_0: usize = 5; - /// address_incr[1]: halfword 1 (bits 16-31) - pub const ADDRESS_INCR_1: usize = 6; - /// address_incr[2]: halfword 2 (bits 32-47) - pub const ADDRESS_INCR_2: usize = 7; - /// address_incr[3]: halfword 3 (bits 48-63) - pub const ADDRESS_INCR_3: usize = 8; - - // Remaining byte count (DWordWL: 2 cols) + // Byte count (DWordWL: 2 cols) /// count[0]: low 32 bits - pub const COUNT_0: usize = 9; + pub const COUNT_0: usize = 5; /// count[1]: high 32 bits - pub const COUNT_1: usize = 10; - - // count - 1 (DWordHL: 4 halfword cols) - // When count > 0: count_decr = count - 1 - // When count = 0: count_decr = 0xFFFF_FFFF_FFFF_FFFF (all halfwords = 0xFFFF) - /// count_decr[0]: halfword 0 (bits 0-15) - pub const COUNT_DECR_0: usize = 11; - /// count_decr[1]: halfword 1 (bits 16-31) - pub const COUNT_DECR_1: usize = 12; - /// count_decr[2]: halfword 2 (bits 32-47) - pub const COUNT_DECR_2: usize = 13; - /// count_decr[3]: halfword 3 (bits 48-63) - pub const COUNT_DECR_3: usize = 14; - - // Control bits - /// first: 1 if this is the first row of a commit sequence - pub const FIRST: usize = 15; - /// end: 1 if this is the last row (count was 0) - pub const END: usize = 16; - - // Byte value being committed - /// value: the byte [0, 256) being committed at this row - pub const VALUE: usize = 17; + pub const COUNT_1: usize = 6; /// mu: multiplicity bit (1 for real rows, 0 for padding) - pub const MU: usize = 18; + /// + /// There is no `first` column any more. This table is one row per ECALL, so a + /// real row is always the first row of its commit, and `first` was identically + /// `mu`; every multiplicity that used to read `first` reads `mu` instead. + pub const MU: usize = 7; /// Total number of columns - pub const NUM_COLUMNS: usize = 19; + pub const NUM_COLUMNS: usize = 8; } // ========================================================================= @@ -128,24 +98,18 @@ pub mod cols { /// A single row in the COMMIT table. /// -/// Each row represents one byte being committed from a buffer. Rows are linked -/// via the CommitNextByte bus to form a chain for each commit ECALL. +/// One row per commit ECALL. It accepts the syscall number, reads the operands and +/// advances the committed-length register; MEMMOVE walks the buffer. #[derive(Debug, Clone)] pub struct CommitOperation { /// Timestamp of the originating ECALL pub timestamp: u64, - /// Global commit index for this byte + /// Global commit index the committed range starts at pub index: u64, - /// Current buffer address for this byte + /// Buffer address the committed range starts at pub address: u64, - /// Remaining byte count (including this byte, 0 on end row) + /// Number of bytes this ECALL commits pub count: u64, - /// Whether this is the first row of a commit sequence - pub first: bool, - /// Whether this is the end row (count was 0, no byte committed) - pub end: bool, - /// The byte value being committed (0 on end row) - pub value: u8, } // ========================================================================= @@ -170,53 +134,16 @@ pub fn generate_commit_trace( let table = &mut trace.main_table; for (row_idx, op) in ops.iter().enumerate() { - // Timestamp (DWordWL) table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); - - // Index (BaseField) table.set_u64(row_idx, cols::INDEX, op.index); - - // Address (DWordWL) table.set_dword_wl(row_idx, cols::ADDRESS_0, op.address); - - // address_incr = address + 1 (DWordHL: 4 halfwords) - let address_incr = op.address.wrapping_add(1); - table.set_dword_hl(row_idx, cols::ADDRESS_INCR_0, address_incr); - - // Count (DWordWL) table.set_dword_wl(row_idx, cols::COUNT_0, op.count); - - // count_decr: if count == 0, use 0xFFFF_FFFF_FFFF_FFFF; else count - 1 - let count_decr = if op.count == 0 { - u64::MAX - } else { - op.count - 1 - }; - table.set_dword_hl(row_idx, cols::COUNT_DECR_0, count_decr); - - // Control bits - table.set_bool(row_idx, cols::FIRST, op.first); - table.set_bool(row_idx, cols::END, op.end); - - // Value - table.set_byte(row_idx, cols::VALUE, op.value); - - // mu = 1 for all real rows (first, middle, and end rows) table.set_fe(row_idx, cols::MU, FE::one()); } - // Padding rows: spec requires count=1 and address_incr=[1,0,0,0] so - // the unconditional ADD/SUB templates have valid carry values. - // count=1 → count_decr=0 (all halfwords zero), address=0 → address_incr=1. - for row_idx in n..num_rows { - // count = 1 (low word) - table.set_fe(row_idx, cols::COUNT_0, FE::one()); - // address_incr halfword 0 = 1 (address=0, so address+1 = 1) - table.set_fe(row_idx, cols::ADDRESS_INCR_0, FE::one()); - // All other fields remain zero: timestamp=0, address=0, count_1=0, - // count_decr=[0,0,0,0], first=0, end=0, value=0, mu=0, - // address_incr_1..3=0 - } + // Padding rows are all-zero. The ADD/SUB templates that used to force a + // non-zero padding row went with `address_incr` and `count_decr`; the one + // surviving constraint is `IS_BIT(mu)`, which zero satisfies. trace } @@ -225,27 +152,19 @@ pub fn generate_commit_trace( // Bus interactions // ========================================================================= -/// Creates all bus interactions for the COMMIT table (18 total). +/// Creates all bus interactions for the COMMIT table (6 total). /// /// The COMMIT table: -/// - **Receives** Ecall from CPU with `[timestamp_lo, timestamp_hi, constant(64), constant(0)]` (mult = first) -/// - **Sends** to CommitNextByte with `[timestamp, index + 1, address_incr, count_decr]` (mult = mu - end) -/// - **Receives** from CommitNextByte with `[timestamp, index, address, count]` (mult = mu - first) -/// - **Sends** to IsHalfword for count_decr range checks (×4, mult = mu) -/// - **Sends** to IsHalfword for address_incr range checks (×4, mult = mu) -/// - **Sends** to Zero for end detection (mult = mu) -/// - **Sends** to Memw for register/memory accesses (×5, mult varies) +/// - **Receives** Ecall from CPU with `[timestamp_lo, timestamp_hi, constant(64), constant(0)]` (mult = mu) +/// - **Sends** to CommitDefer, handing the byte loop to MEMMOVE (mult = mu) +/// - **Sends** to Memw for register accesses (×4, mult = mu) pub fn bus_interactions() -> Vec { - // Reusable multiplicity expressions - let mu_minus_end = Multiplicity::Diff(cols::MU, cols::END); - let mu_minus_first = Multiplicity::Diff(cols::MU, cols::FIRST); - vec![ // 1. Receive ECALL from CPU (mult = first) // Payload: [timestamp_lo, timestamp_hi, syscall_lo32, syscall_hi32] BusInteraction::receiver( BusId::Ecall, - Multiplicity::Column(cols::FIRST), + Multiplicity::Column(cols::MU), vec![ BusValue::Packed { start_column: cols::TIMESTAMP_0, @@ -259,48 +178,12 @@ pub fn bus_interactions() -> Vec { BusValue::constant(0), // syscall number hi32 = 0 ], ), - // 2. Send to CommitNextByte (mult = mu - end) - // Sends: [timestamp, index + 1, address_incr(as DWordWL), count_decr(as DWordWL)] + // 2. Defer the byte loop to the MEMMOVE chip. COMMIT keeps the sys_write + // ecall number and the register-254 update; the copying is handed over. BusInteraction::sender( - BusId::CommitNextByte, - mu_minus_end.clone(), - vec![ - // timestamp (DWordWL: 2 Direct elements) - BusValue::Packed { - start_column: cols::TIMESTAMP_0, - packing: Packing::Direct, - }, - BusValue::Packed { - start_column: cols::TIMESTAMP_1, - packing: Packing::Direct, - }, - // index + 1 (BaseField) - BusValue::linear(vec![ - LinearTerm::Column { - coefficient: 1, - column: cols::INDEX, - }, - LinearTerm::Constant(1), - ]), - // address_incr (DWordHL → 2 bus elements via DWordHL packing) - BusValue::Packed { - start_column: cols::ADDRESS_INCR_0, - packing: Packing::DWordHL, - }, - // count_decr (DWordHL → 2 bus elements via DWordHL packing) - BusValue::Packed { - start_column: cols::COUNT_DECR_0, - packing: Packing::DWordHL, - }, - ], - ), - // 3. Receive from CommitNextByte (mult = mu - first) - // Receives: [timestamp, index, address, count] - BusInteraction::receiver( - BusId::CommitNextByte, - mu_minus_first, + BusId::CommitDefer, + Multiplicity::Column(cols::MU), vec![ - // timestamp (DWordWL) BusValue::Packed { start_column: cols::TIMESTAMP_0, packing: Packing::Direct, @@ -309,131 +192,29 @@ pub fn bus_interactions() -> Vec { start_column: cols::TIMESTAMP_1, packing: Packing::Direct, }, - // index (BaseField) - BusValue::Packed { - start_column: cols::INDEX, - packing: Packing::Direct, - }, - // address (DWordWL) BusValue::Packed { start_column: cols::ADDRESS_0, - packing: Packing::Direct, - }, - BusValue::Packed { - start_column: cols::ADDRESS_1, - packing: Packing::Direct, + packing: Packing::DWordWL, }, - // count (DWordWL → 2 bus elements) + // `dst` on the MEMMOVE side is a DWordWL, i.e. two bus elements; the + // COMMIT-domain address is the index, whose high word is always zero. + BusValue::linear(vec![LinearTerm::Column { + coefficient: 1, + column: cols::INDEX, + }]), + BusValue::constant(0), BusValue::Packed { start_column: cols::COUNT_0, packing: Packing::DWordWL, }, ], ), - // 4-7. IsHalfword for count_decr (×4, mult = mu) - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::COUNT_DECR_0, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::COUNT_DECR_1, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::COUNT_DECR_2, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::COUNT_DECR_3, - packing: Packing::Direct, - }], - ), - // 8-11. IsHalfword for address_incr (×4, mult = mu) - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::ADDRESS_INCR_0, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::ADDRESS_INCR_1, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::ADDRESS_INCR_2, - packing: Packing::Direct, - }], - ), - BusInteraction::sender( - BusId::IsHalfword, - Multiplicity::Column(cols::MU), - vec![BusValue::Packed { - start_column: cols::ADDRESS_INCR_3, - packing: Packing::Direct, - }], - ), - // 12. ZERO bus for end detection (mult = mu) - // Input: (65535 - cd_0) + (65535 - cd_1) + (65535 - cd_2) + (65535 - cd_3) - // Output: end (1 when all count_decr halfwords are 0xFFFF, i.e., count was 0) - BusInteraction::sender( - BusId::Zero, - Multiplicity::Column(cols::MU), - vec![ - BusValue::linear(vec![ - LinearTerm::Constant(4 * 65535), - LinearTerm::Column { - coefficient: -1, - column: cols::COUNT_DECR_0, - }, - LinearTerm::Column { - coefficient: -1, - column: cols::COUNT_DECR_1, - }, - LinearTerm::Column { - coefficient: -1, - column: cols::COUNT_DECR_2, - }, - LinearTerm::Column { - coefficient: -1, - column: cols::COUNT_DECR_3, - }, - ]), - BusValue::Packed { - start_column: cols::END, - packing: Packing::Direct, - }, - ], - ), // 13. MEMW read+write x10 (fd=1 → count) at ts (mult = first) // CO24 format: [old[8], is_register, base_addr[2], value[8], ts[2], w2, w4, w8] // old = [1,0,...,0] (asserts x10=1=fd), value = [count_0, count_1, 0,...,0] (writes count) BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::FIRST), + Multiplicity::Column(cols::MU), vec![ // old[0..7] = [1, 0, 0, 0, 0, 0, 0, 0] BusValue::constant(1), @@ -482,7 +263,7 @@ pub fn bus_interactions() -> Vec { // 14. MEMW read x11 (buf_addr) at ts (mult = first) BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::FIRST), + Multiplicity::Column(cols::MU), vec![ // old[0..7] = [ADDRESS_0, ADDRESS_1, 0, 0, 0, 0, 0, 0] BusValue::Packed { @@ -537,7 +318,7 @@ pub fn bus_interactions() -> Vec { // 15. MEMW read x12 (count) at ts (mult = first) BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::FIRST), + Multiplicity::Column(cols::MU), vec![ // old[0..7] = [COUNT_0, COUNT_1, 0, 0, 0, 0, 0, 0] BusValue::Packed { @@ -593,7 +374,7 @@ pub fn bus_interactions() -> Vec { // Single-word synthetic register per spec: width=1, base address 508. BusInteraction::sender( BusId::Memw, - Multiplicity::Column(cols::FIRST), + Multiplicity::Column(cols::MU), vec![ // old[0..7] = [INDEX, 0, 0, 0, 0, 0, 0, 0] BusValue::Packed { @@ -650,76 +431,6 @@ pub fn bus_interactions() -> Vec { BusValue::constant(0), ], ), - // 17. MEMW read byte at ts (mult = mu - end) - BusInteraction::sender( - BusId::Memw, - mu_minus_end.clone(), - vec![ - // old[0..7] = [VALUE, 0, 0, 0, 0, 0, 0, 0] - BusValue::Packed { - start_column: cols::VALUE, - packing: Packing::Direct, - }, - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - // is_register = 0 - BusValue::constant(0), - // base_address = [ADDRESS_0, ADDRESS_1] - BusValue::Packed { - start_column: cols::ADDRESS_0, - packing: Packing::Direct, - }, - BusValue::Packed { - start_column: cols::ADDRESS_1, - packing: Packing::Direct, - }, - // value[0..7] = [VALUE, 0, 0, 0, 0, 0, 0, 0] (read: same as old) - BusValue::Packed { - start_column: cols::VALUE, - packing: Packing::Direct, - }, - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - // timestamp = [TIMESTAMP_0, TIMESTAMP_1] - BusValue::Packed { - start_column: cols::TIMESTAMP_0, - packing: Packing::Direct, - }, - BusValue::Packed { - start_column: cols::TIMESTAMP_1, - packing: Packing::Direct, - }, - // w2=0, w4=0, w8=0 (width=1 byte) - BusValue::constant(0), - BusValue::constant(0), - BusValue::constant(0), - ], - ), - // 18. COMMIT[index, value] (mult = mu - end) - BusInteraction::sender( - BusId::Commit, - mu_minus_end, - vec![ - BusValue::Packed { - start_column: cols::INDEX, - packing: Packing::Direct, - }, - BusValue::Packed { - start_column: cols::VALUE, - packing: Packing::Direct, - }, - ], - ), ] } @@ -737,36 +448,11 @@ pub struct CommitConstraints; impl ConstraintSet for CommitConstraints { fn eval>(&self, b: &mut B) { - // idx 0-2: IS_BIT for first, end, mu - emit_is_bit(b, 0, cols::FIRST, None); - emit_is_bit(b, 1, cols::END, None); - emit_is_bit(b, 2, cols::MU, None); - - // idx 3: (first + end) * (1 - mu) - let one = b.one(); - let first = b.main(0, cols::FIRST); - let end = b.main(0, cols::END); - let mu = b.main(0, cols::MU); - b.emit_base(3, (first + end) * (one - mu)); - - // idx 4,5: ADD template for address + 1 = address_incr (unconditional) - emit_add_pair( - b, - 4, - &[], - &AddOperand::dword(cols::ADDRESS_0), - &AddOperand::constant(1), - &AddOperand::from_dword_hl(cols::ADDRESS_INCR_0), - ); - - // idx 6,7: SUB via ADD: count_decr + 1 = count (unconditional) - emit_add_pair( - b, - 6, - &[], - &AddOperand::from_dword_hl(cols::COUNT_DECR_0), - &AddOperand::constant(1), - &AddOperand::dword(cols::COUNT_0), - ); + // One constraint is all that is left. This table is one row per ECALL: it + // accepts the syscall number, reads the operands, advances x254 and hands the + // byte loop to MEMMOVE. Everything that modelled a per-byte sequence went with + // the loop — `first` (identically `mu` now), `end` and its `Zero` detection, + // and the `address_incr`/`count_decr` ADD pairs with their range checks. + emit_is_bit(b, 0, cols::MU, None); } } diff --git a/prover/src/tables/cpu.rs b/prover/src/tables/cpu.rs index fc4c2f976..cefa4c125 100644 --- a/prover/src/tables/cpu.rs +++ b/prover/src/tables/cpu.rs @@ -193,6 +193,12 @@ pub struct CpuOperation { /// addresses (x10/x11/x12) are recovered from the register state in the trace /// builder, exactly like ECSM. pub ecall_hint: bool, + + /// Whether this ECALL is a DMA memcpy. Operands are recovered from x10/x11/x12. + pub ecall_dma_memcpy: bool, + + /// Whether this ECALL is a DMA memset. Operands are recovered from x10/x11/x12. + pub ecall_dma_memset: bool, } impl CpuOperation { @@ -242,6 +248,10 @@ impl CpuOperation { f.ecall && log.src1_val == executor::vm::instruction::execution::ECSM_SYSCALL_NUMBER; let ecall_hint = f.ecall && log.src1_val == executor::vm::instruction::execution::HINT_SYSCALL_NUMBER; + let ecall_dma_memcpy = f.ecall + && log.src1_val == executor::vm::instruction::execution::DMA_MEMCPY_SYSCALL_NUMBER; + let ecall_dma_memset = f.ecall + && log.src1_val == executor::vm::instruction::execution::DMA_MEMSET_SYSCALL_NUMBER; // Word instructions are fully handled by CPU32; the main CPU row is a // delegate that only advances the PC and sends the CPU32 lookup. We still @@ -361,6 +371,8 @@ impl CpuOperation { keccak_state_addr, ecall_ecsm, ecall_hint, + ecall_dma_memcpy, + ecall_dma_memset, } } diff --git a/prover/src/tables/memmove.rs b/prover/src/tables/memmove.rs new file mode 100644 index 000000000..b1faf1c3c --- /dev/null +++ b/prover/src/tables/memmove.rs @@ -0,0 +1,949 @@ +//! MEMMOVE table — one streaming copy primitive for `memcpy`/`memmove`, `memset` +//! and the byte loop of `commit`. +//! +//! Replaces the separate DMA and DMA_SET tables and takes over COMMIT's looping. +//! A row copies `1` or `8` bytes from `src` to `dst` and chains through +//! [`BusId::MemmoveNext`] until a terminal row where `count == 0`. +//! +//! ## The three functionalities +//! +//! One-hot over `is_set` and `is_commit`; `is_cpy = mu - is_set - is_commit` is +//! linear and costs no column. Neither the memory domain nor the timestamp order +//! is chosen by the caller: both are *derived* from the selector, and the selector +//! is pinned to the ecall the row receives. +//! +//! | op | domains | timestamp order | arguments | +//! |---|---|---|---| +//! | memcpy / memmove | RAM → RAM | read `T+1`, write `T+2` | `x10`, `x11`, `x12` | +//! | memset | RAM → RAM | **write `T+1`, read `T+2`** | `x10`, `x11`, `x12` | +//! | commit | RAM → COMMIT | read `T+1`, write `T+2` | the COMMIT chip's defer bus | +//! +//! ## Timestamp order +//! +//! ```text +//! read_ts = T + 1 + is_set +//! write_ts = T + 2 - is_set +//! ``` +//! +//! Both are linear in a bit column. With the normal order every read of a call +//! happens at one timestamp and every write at a later one, so a whole chunk is a +//! snapshot — that is what gives `memmove` its overlap semantics for free. With the +//! order inverted, a row's read observes the *previous* row's write, so a self-copy +//! propagates its first bytes across the range: `memset`. `old_ts < ts` holds strictly +//! either way, so the memory argument is undisturbed. +//! +//! `memset` needs no special handling here at all. Its stub seeds the first eight +//! bytes with an ordinary store and calls with `(dst = seed_end, src = seed_start, +//! count = n - 8)`, so the chip sees a plain overlapping memmove. +//! +//! ## Width is chosen per row +//! +//! `tail` is free except that an eight-byte row is illegal when fewer than eight +//! bytes remain (`(1 - tail) * lt8 = 0`, with `lt8` pinned by the ALU). A schedule can +//! therefore walk one-byte rows until `dst` is eight-aligned and take eight-byte rows +//! through the body, which keeps those rows in MEMW_A rather than MEMW. +//! +//! ## Columns (38) +//! +//! - `timestamp` DWordWL (2), `src` DWordWL (2), `src_incr` DWordHL (4) +//! - `dst` DWordWL (2) — for `commit` this is the COMMIT-domain address, i.e. the +//! running global byte index — `dst_incr` DWordHL (4) +//! - `count` DWordWL (2), `count_decr` DWordHL (4) +//! - `first`, `end`, `tail`, `value[8]`, `mu` +//! - `is_set`, `is_commit` — the decoded functionality +//! - `lt8` — `count < 8`, pinned by the ALU +//! - `f_ncommit = first * (1 - is_commit)`, `mu_com = (mu - end) * is_commit`, +//! `mu_com_wide = mu_com * (1 - tail)` — multiplicities are strictly linear in this +//! framework, so an op-specific gate that is not already linear needs a column and a +//! degree-2 constraint. The RAM write needs none: it rides `mu - end - mu_com` +//! directly, which is `(mu - end) * (1 - is_commit)` expanded. +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use crate::constraints::templates::{ + AddLinearTerm, AddOperand, emit_add_pair, emit_add_pair_no_overflow, emit_is_bit, +}; + +use executor::vm::instruction::execution::{ + DMA_MEMCPY_MAX_BYTES as EXECUTOR_MAX_BYTES, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_GAP, + DMA_MEMSET_SYSCALL_NUMBER, +}; + +use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; + +const MEMCPY_LO32: u64 = DMA_MEMCPY_SYSCALL_NUMBER & 0xFFFF_FFFF; +const MEMCPY_HI32: u64 = DMA_MEMCPY_SYSCALL_NUMBER >> 32; +const MEMSET_LO32: u64 = DMA_MEMSET_SYSCALL_NUMBER & 0xFFFF_FFFF; +const MEMSET_HI32: u64 = DMA_MEMSET_SYSCALL_NUMBER >> 32; + +/// Maximum bytes one ecall may move, taken from the executor so the bound the AIR +/// proves cannot drift from the bound execution enforces. +pub const MEMMOVE_MAX_BYTES: u64 = EXECUTOR_MAX_BYTES; + +pub mod cols { + pub const TIMESTAMP_0: usize = 0; + pub const TIMESTAMP_1: usize = 1; + + pub const SRC_0: usize = 2; + pub const SRC_1: usize = 3; + + pub const SRC_INCR_0: usize = 4; + + pub const DST_0: usize = 8; + pub const DST_1: usize = 9; + + pub const DST_INCR_0: usize = 10; + + pub const COUNT_0: usize = 14; + pub const COUNT_1: usize = 15; + + pub const COUNT_DECR_0: usize = 16; + + pub const FIRST: usize = 20; + pub const END: usize = 21; + pub const TAIL: usize = 22; + pub const VALUE_0: usize = 23; + pub const VALUE: [usize; 8] = [ + VALUE_0, + VALUE_0 + 1, + VALUE_0 + 2, + VALUE_0 + 3, + VALUE_0 + 4, + VALUE_0 + 5, + VALUE_0 + 6, + VALUE_0 + 7, + ]; + pub const MU: usize = 31; + + /// Decoded functionality. `is_cpy = mu - is_set - is_commit` is implied. + pub const IS_SET: usize = 32; + pub const IS_COMMIT: usize = 33; + /// `count < 8`, pinned by the ALU; blocks an eight-byte row on a short count. + pub const LT8: usize = 34; + /// `first * (1 - is_commit)` — the ecall receive and the register reads. + pub const F_NCOMMIT: usize = 35; + /// `(mu - end) * is_commit` — the COMMIT-domain write. + pub const MU_COM: usize = 36; + /// `mu_com * (1 - tail)` — lanes 1..7 of the COMMIT-domain write. Without it a + /// one-byte commit row would send seven spurious `(index, 0)` pairs and corrupt + /// the public-output fingerprint. + pub const MU_COM_WIDE: usize = 37; + + /// The RAM write rides `mu - end - mu_com`, which is `(mu - end) * (1 - is_commit)` + /// expanded. It needs no column of its own: `Multiplicity::Linear` takes the + /// expression directly, and the product form was only ever a column because the + /// framework requires multiplicities to be linear. + pub const NUM_COLUMNS: usize = 38; +} + +/// Which functionality a row is running. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Functionality { + /// `memcpy` / `memmove`: RAM → RAM, snapshot order. + Copy, + /// `memset`: RAM → RAM, inverted order, so the fill propagates. + Set, + /// `commit`: RAM → COMMIT domain, snapshot order. + Commit, +} + +/// One row: `1` or `8` bytes, or the terminal row. +#[derive(Debug, Clone)] +pub struct MemmoveOperation { + pub timestamp: u64, + pub src: u64, + /// For `Commit` this is the COMMIT-domain address (the global byte index). + pub dst: u64, + /// Remaining byte count including this row's bytes; `0` on the terminal row. + pub count: u64, + pub width: u8, + pub first: bool, + pub end: bool, + pub functionality: Functionality, + /// The bytes moved, zero-padded past `width`. + pub value: [u8; 8], +} + +impl MemmoveOperation { + /// `read_ts = T + 1 + is_set`, `write_ts = T + 2 - is_set`. + pub fn read_timestamp(&self) -> u64 { + self.timestamp + 1 + u64::from(self.functionality == Functionality::Set) + } + + pub fn write_timestamp(&self) -> u64 { + self.timestamp + 2 - u64::from(self.functionality == Functionality::Set) + } +} + +/// Generates the MEMMOVE trace. One row per operation, padded to the next power of +/// two (min 4). Padding rows model an inactive one-byte copy so the unconditional +/// address/count relations still hold. +pub fn generate_memmove_trace( + ops: &[MemmoveOperation], +) -> TraceTable { + let n = ops.len(); + let num_rows = n.next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row_idx, op) in ops.iter().enumerate() { + let width = u64::from(op.width); + table.set_dword_wl(row_idx, cols::TIMESTAMP_0, op.timestamp); + + table.set_dword_wl(row_idx, cols::SRC_0, op.src); + table.set_dword_hl(row_idx, cols::SRC_INCR_0, op.src.wrapping_add(width)); + + table.set_dword_wl(row_idx, cols::DST_0, op.dst); + table.set_dword_hl(row_idx, cols::DST_INCR_0, op.dst.wrapping_add(width)); + + table.set_dword_wl(row_idx, cols::COUNT_0, op.count); + table.set_dword_hl(row_idx, cols::COUNT_DECR_0, op.count.wrapping_sub(width)); + + table.set_bool(row_idx, cols::FIRST, op.first); + table.set_bool(row_idx, cols::END, op.end); + table.set_bool(row_idx, cols::TAIL, op.width == 1); + for (column, &byte) in cols::VALUE.iter().zip(&op.value) { + table.set_byte(row_idx, *column, byte); + } + table.set_fe(row_idx, cols::MU, FE::one()); + + let is_set = op.functionality == Functionality::Set; + let is_commit = op.functionality == Functionality::Commit; + table.set_bool(row_idx, cols::IS_SET, is_set); + table.set_bool(row_idx, cols::IS_COMMIT, is_commit); + table.set_bool(row_idx, cols::LT8, op.count < 8); + table.set_bool(row_idx, cols::F_NCOMMIT, op.first && !is_commit); + table.set_bool(row_idx, cols::MU_COM, !op.end && is_commit); + table.set_bool( + row_idx, + cols::MU_COM_WIDE, + !op.end && is_commit && op.width == 8, + ); + } + + for row_idx in n..num_rows { + table.set_fe(row_idx, cols::COUNT_0, FE::one()); + table.set_fe(row_idx, cols::SRC_INCR_0, FE::one()); + table.set_fe(row_idx, cols::DST_INCR_0, FE::one()); + table.set_fe(row_idx, cols::TAIL, FE::one()); + table.set_fe(row_idx, cols::LT8, FE::one()); + } + + trace +} + +/// A MEMW register read (CO24, `is_register = 1`, width 2): `value == old ==` the +/// register's two 32-bit limbs, binding `x{reg}` to `(lo_col, hi_col)` at the ecall. +fn memw_register_read(reg_addr: u64, lo_col: usize, hi_col: usize) -> Vec { + let limbs = || { + vec![ + BusValue::Packed { + start_column: lo_col, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: hi_col, + packing: Packing::Direct, + }, + BusValue::constant(0), + BusValue::constant(0), + BusValue::constant(0), + BusValue::constant(0), + BusValue::constant(0), + BusValue::constant(0), + ] + }; + let mut tuple = limbs(); + tuple.push(BusValue::constant(1)); // is_register + tuple.push(BusValue::constant(reg_addr)); + tuple.push(BusValue::constant(0)); + tuple.extend(limbs()); + tuple.push(BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }); + tuple.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + tuple.push(BusValue::constant(1)); // w2 + tuple.push(BusValue::constant(0)); + tuple.push(BusValue::constant(0)); + tuple +} + +/// `T + offset + coefficient * is_set`, the timestamp-order customisation. +fn timestamp_with_order(offset: i64, is_set_coefficient: i64) -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::TIMESTAMP_0, + }, + LinearTerm::Column { + coefficient: is_set_coefficient, + column: cols::IS_SET, + }, + LinearTerm::Constant(offset), + ]) +} + +fn value_columns() -> Vec { + cols::VALUE + .iter() + .map(|&column| BusValue::Packed { + start_column: column, + packing: Packing::Direct, + }) + .collect() +} + +fn halfword(column: usize) -> BusInteraction { + BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::MU), + vec![BusValue::Packed { + start_column: column, + packing: Packing::Direct, + }], + ) +} + +/// The MEMMOVE bus interactions. +pub fn bus_interactions() -> Vec { + let mu_minus_end = Multiplicity::Diff(cols::MU, cols::END); + let mu_minus_first = Multiplicity::Diff(cols::MU, cols::FIRST); + // first * is_commit, without a column: first - f_ncommit. + let f_commit = Multiplicity::Diff(cols::FIRST, cols::F_NCOMMIT); + let w8 = || { + BusValue::linear(vec![ + LinearTerm::Constant(1), + LinearTerm::Column { + coefficient: -1, + column: cols::TAIL, + }, + ]) + }; + + let mut interactions = vec![ + // 1. Receive the ECALL for the two RAM-to-RAM functionalities. The syscall + // number is a linear function of the selector, so the decoded functionality + // is pinned to the ecall the guest actually made. + BusInteraction::receiver( + BusId::Ecall, + Multiplicity::Column(cols::F_NCOMMIT), + vec![ + BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }, + BusValue::linear(vec![ + LinearTerm::Constant(MEMCPY_LO32 as i64), + LinearTerm::Column { + coefficient: MEMSET_LO32 as i64 - MEMCPY_LO32 as i64, + column: cols::IS_SET, + }, + ]), + BusValue::linear(vec![ + LinearTerm::Constant(MEMCPY_HI32 as i64), + LinearTerm::Column { + coefficient: MEMSET_HI32 as i64 - MEMCPY_HI32 as i64, + column: cols::IS_SET, + }, + ]), + ], + ), + // 2. Receive the deferred loop from the COMMIT chip. + BusInteraction::receiver( + BusId::CommitDefer, + f_commit, + vec![ + BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::SRC_0, + packing: Packing::DWordWL, + }, + BusValue::Packed { + start_column: cols::DST_0, + packing: Packing::DWordWL, + }, + BusValue::Packed { + start_column: cols::COUNT_0, + packing: Packing::DWordWL, + }, + ], + ), + // 3. Chain forward. The selectors ride inside the tuple, so a chain cannot + // change functionality half way through it. + BusInteraction::sender( + BusId::MemmoveNext, + mu_minus_end.clone(), + vec![ + BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::SRC_INCR_0, + packing: Packing::DWordHL, + }, + BusValue::Packed { + start_column: cols::DST_INCR_0, + packing: Packing::DWordHL, + }, + BusValue::Packed { + start_column: cols::COUNT_DECR_0, + packing: Packing::DWordHL, + }, + BusValue::Packed { + start_column: cols::IS_SET, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::IS_COMMIT, + packing: Packing::Direct, + }, + ], + ), + // 4. Chain backward. + BusInteraction::receiver( + BusId::MemmoveNext, + mu_minus_first, + vec![ + BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::SRC_0, + packing: Packing::DWordWL, + }, + BusValue::Packed { + start_column: cols::DST_0, + packing: Packing::DWordWL, + }, + BusValue::Packed { + start_column: cols::COUNT_0, + packing: Packing::DWordWL, + }, + BusValue::Packed { + start_column: cols::IS_SET, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::IS_COMMIT, + packing: Packing::Direct, + }, + ], + ), + // 5-16. Halfword range checks. + halfword(cols::COUNT_DECR_0), + halfword(cols::COUNT_DECR_0 + 1), + halfword(cols::COUNT_DECR_0 + 2), + halfword(cols::COUNT_DECR_0 + 3), + halfword(cols::SRC_INCR_0), + halfword(cols::SRC_INCR_0 + 1), + halfword(cols::SRC_INCR_0 + 2), + halfword(cols::SRC_INCR_0 + 3), + halfword(cols::DST_INCR_0), + halfword(cols::DST_INCR_0 + 1), + halfword(cols::DST_INCR_0 + 2), + halfword(cols::DST_INCR_0 + 3), + // 17. `end == 1` iff every count_decr halfword is 0xFFFF. + BusInteraction::sender( + BusId::Zero, + Multiplicity::Column(cols::MU), + vec![ + BusValue::linear(vec![ + LinearTerm::Constant(4 * 65535), + LinearTerm::Column { + coefficient: -1, + column: cols::COUNT_DECR_0, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::COUNT_DECR_0 + 1, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::COUNT_DECR_0 + 2, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::COUNT_DECR_0 + 3, + }, + ]), + BusValue::Packed { + start_column: cols::END, + packing: Packing::Direct, + }, + ], + ), + // 18-20. Register reads, only for the ecall-driven functionalities. + BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::F_NCOMMIT), + memw_register_read(20, cols::DST_0, cols::DST_1), + ), + BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::F_NCOMMIT), + memw_register_read(22, cols::SRC_0, cols::SRC_1), + ), + BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::F_NCOMMIT), + memw_register_read(24, cols::COUNT_0, cols::COUNT_1), + ), + // 21. `lt8 = (count < 8)`. Width is otherwise the prover's choice. + BusInteraction::sender( + BusId::Alu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::COUNT_0, + packing: Packing::DWordWL, + }, + BusValue::constant(8), + BusValue::constant(0), + BusValue::constant(alu_op::LT as u64), + BusValue::Packed { + start_column: cols::LT8, + packing: Packing::Direct, + }, + BusValue::constant(0), + ], + ), + // 22. The first row of an ecall-driven call proves `count <= MEMMOVE_MAX_BYTES`. + // Commit is excluded: it arrives over CommitDefer, which the guest does not + // chunk. Note that nothing bounds a commit chain's length in-circuit -- the + // COMMIT chip range-checks no `count` either -- so a single `sys_write` can + // append rows here in proportion to its byte count. That is pre-existing + // (the deleted per-byte COMMIT loop had the same property) and it is not + // verifier-exploitable, since the COMMIT bus still has to balance against + // `public_output`; it is a prover-cost bound only, tracked separately. + BusInteraction::sender( + BusId::Alu, + Multiplicity::Column(cols::F_NCOMMIT), + vec![ + BusValue::Packed { + start_column: cols::COUNT_0, + packing: Packing::DWordWL, + }, + BusValue::constant(MEMMOVE_MAX_BYTES + 1), + BusValue::constant(0), + BusValue::constant(alu_op::LT as u64), + BusValue::constant(1), + BusValue::constant(0), + ], + ), + // 23. Read the source at `T + 1 + is_set`. + BusInteraction::sender(BusId::Memw, mu_minus_end.clone(), { + let mut values = value_columns(); + let mut tuple = Vec::with_capacity(24); + tuple.extend(values.iter().cloned()); + tuple.push(BusValue::constant(0)); // is_register + tuple.push(BusValue::Packed { + start_column: cols::SRC_0, + packing: Packing::Direct, + }); + tuple.push(BusValue::Packed { + start_column: cols::SRC_1, + packing: Packing::Direct, + }); + tuple.append(&mut values); + tuple.push(timestamp_with_order(1, 1)); + tuple.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + tuple.push(BusValue::constant(0)); // w2 + tuple.push(BusValue::constant(0)); // w4 + tuple.push(w8()); + tuple + }), + // 24. Write the destination at `T + 2 - is_set`, RAM domain only. + BusInteraction::sender( + BusId::Memw, + Multiplicity::Linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::MU, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::END, + }, + LinearTerm::Column { + coefficient: -1, + column: cols::MU_COM, + }, + ]), + { + let mut tuple = Vec::with_capacity(16); + tuple.push(BusValue::constant(0)); // is_register + tuple.push(BusValue::Packed { + start_column: cols::DST_0, + packing: Packing::Direct, + }); + tuple.push(BusValue::Packed { + start_column: cols::DST_1, + packing: Packing::Direct, + }); + tuple.extend(value_columns()); + tuple.push(timestamp_with_order(2, -1)); + tuple.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + tuple.push(BusValue::constant(0)); // w2 + tuple.push(BusValue::constant(0)); // w4 + tuple.push(w8()); + tuple + }, + ), + ]; + + // 25-32. Write the destination in the COMMIT domain, one `(index, value)` pair per + // byte. A row still carries eight bytes, but it sends them as eight separate pairs + // at `dst`, `dst + 1`, ..., rather than as one eight-lane tuple. + // + // The arity is what matters here, not the row width. The verifier rebuilds this bus + // from `public_output` alone, and it does not know where one commit ECALL ended and + // the next began — it sees only the concatenation. With one tuple per row the + // verifier would have to reproduce the prover's row schedule exactly, which it + // cannot: the schedule restarts at every ECALL, so a guest committing 4 bytes and + // then 4 more sends eight one-byte rows where the verifier, chunking the eight + // bytes it sees, would expect a single eight-byte row. An honest proof would be + // rejected. Addressing every byte by its own global index removes the grouping, and + // with it anything for the two sides to disagree about. + // + // Lane 0 is sent whenever the row copies (`mu_com`); lanes 1..7 only on an + // eight-byte row (`mu_com_wide`), so a one-byte row does not send seven spurious + // `(index, 0)` pairs. + let commit_pair = |lane: usize| { + vec![ + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::DST_0, + }, + LinearTerm::Constant(lane as i64), + ]), + BusValue::Packed { + start_column: cols::VALUE[lane], + packing: Packing::Direct, + }, + ] + }; + interactions.push(BusInteraction::sender( + BusId::Commit, + Multiplicity::Column(cols::MU_COM), + commit_pair(0), + )); + for lane in 1..8 { + interactions.push(BusInteraction::sender( + BusId::Commit, + Multiplicity::Column(cols::MU_COM_WIDE), + commit_pair(lane), + )); + } + + interactions +} + +/// The MEMMOVE constraints. +#[derive(Clone, Copy)] +pub struct MemmoveConstraints; + +impl ConstraintSet for MemmoveConstraints { + fn eval>(&self, b: &mut B) { + emit_is_bit(b, 0, cols::FIRST, None); + emit_is_bit(b, 1, cols::END, None); + emit_is_bit(b, 2, cols::TAIL, None); + emit_is_bit(b, 3, cols::MU, None); + emit_is_bit(b, 4, cols::IS_SET, None); + emit_is_bit(b, 5, cols::IS_COMMIT, None); + emit_is_bit(b, 6, cols::LT8, None); + emit_is_bit(b, 7, cols::F_NCOMMIT, None); + emit_is_bit(b, 8, cols::MU_COM, None); + emit_is_bit(b, 9, cols::MU_COM_WIDE, None); + + let one = b.one(); + let first = b.main(0, cols::FIRST); + let end = b.main(0, cols::END); + let mu = b.main(0, cols::MU); + let tail = b.main(0, cols::TAIL); + let lt8 = b.main(0, cols::LT8); + let is_set = b.main(0, cols::IS_SET); + let is_commit = b.main(0, cols::IS_COMMIT); + + // An active row is implied by first or end. + b.emit_base( + 10, + (first.clone() + end.clone()) * (one.clone() - mu.clone()), + ); + // The functionality is one-hot and only set on active rows. + b.emit_base(11, is_set.clone() * is_commit.clone()); + b.emit_base( + 12, + (is_set.clone() + is_commit.clone()) * (one.clone() - mu.clone()), + ); + // An eight-byte row is illegal when fewer than eight bytes remain. + b.emit_base(13, (one.clone() - tail.clone()) * lt8); + + // The two remaining gate columns. + b.emit_base( + 14, + b.main(0, cols::F_NCOMMIT) - first.clone() * (one.clone() - is_commit.clone()), + ); + b.emit_base( + 15, + b.main(0, cols::MU_COM) - (mu.clone() - end.clone()) * is_commit, + ); + let mu_com = b.main(0, cols::MU_COM); + b.emit_base( + 16, + b.main(0, cols::MU_COM_WIDE) - mu_com * (one.clone() - tail.clone()), + ); + + let step = AddOperand::linear( + &[ + AddLinearTerm::Constant(8), + AddLinearTerm::Column { + coefficient: -7, + column: cols::TAIL, + }, + ], + &[], + ); + + emit_add_pair_no_overflow( + b, + 17, + cols::MU, + cols::END, + &AddOperand::dword(cols::SRC_0), + &step, + &AddOperand::from_dword_hl(cols::SRC_INCR_0), + ); + emit_add_pair_no_overflow( + b, + 19, + cols::MU, + cols::END, + &AddOperand::dword(cols::DST_0), + &step, + &AddOperand::from_dword_hl(cols::DST_INCR_0), + ); + emit_add_pair( + b, + 21, + &[], + &AddOperand::from_dword_hl(cols::COUNT_DECR_0), + &step, + &AddOperand::dword(cols::COUNT_0), + ); + + // Unused lanes are zero on one-byte rows. + for (i, &column) in cols::VALUE.iter().enumerate().skip(1) { + b.emit_base(23 + i - 1, tail.clone() * b.main(0, column)); + } + + // memset's operand contract: `dst = src + 8`, limb-wise. + // + // This is what pins `value` on an `is_set` row. The inverted order puts the + // write at `T+1` and the read at `T+2`, so a row's read observes writes this + // call already made -- that is the propagation. It pins the copied value only + // while the read resolves to a *different* address, already written, with the + // recursion bottoming out in memory the call never wrote. `dst == src` is the + // one degenerate case: read and write address the same cell at adjacent + // timestamps, the memory argument closes on `value == value`, and all eight + // lanes become free field elements -- unconstrained RAM, chosen by the prover. + // Nothing else touches them (the lane constraints above only *zero* lanes 1..7 + // on narrow rows, and no `AreBytes` reaches them), so this is the only thing + // standing between the chip and an arbitrary memory write. + // + // `is_set` alone is the correct gate. `step` advances `src` and `dst` together + // (constraints 17 and 19), so `dst - src` is invariant along a chain and the + // relation holds on the terminal row as well; padding rows leave `is_set = 0`. + // Gating on the RAM-write multiplicity instead would exempt terminal rows at + // the cost of a degree, and the table is asserted to stay at degree 2. + // + // Pinning the exact gap rather than merely `src != dst` also settles the + // direction: `dst < src` is sound but propagates the wrong way, so the AIR + // would otherwise admit traces the executor's forward byte walk never produces. + // + // On what these two actually pin: taken together they force + // `packed(dst) - packed(src) = 8` in the field, and that holds whichever way a + // prover splits an address across the two limbs -- re-splitting as + // `(lo + 2^32, hi - 1)` cancels between the pair. So `DST_0 = SRC_0 + 8` always + // differs from `SRC_0`, and the aliasing forgery is dead unconditionally. What + // these constraints do NOT give on their own is the gap over the integers: + // getting from "gap of 8 in F" to "gap of 8 in Z" needs both limbs canonical, + // and MEMMOVE range-checks none of `SRC_0/SRC_1/DST_0/DST_1` (only the three + // `_INCR`/`_DECR` dwords get `IS_HALF`). Canonicality comes from the far end of + // the `Memory` bus instead: PAGE builds `address_lo` as `page_base_lo + OFFSET` + // from a preprocessed offset, and in continuations L2G must chain back to a + // GLOBAL_MEMORY genesis token of the same shape, so a non-canonical limb pair + // has no receiver. Worth knowing before adding another `Memw` producer or a + // non-PAGE `Memory` endpoint -- either would weaken this to the field statement. + // + // The executor additionally refuses a call whose range crosses the 2^32 limb + // boundary, so the honest trace never has to rely on that argument. + let gap = b.const_base(DMA_MEMSET_GAP); + b.emit_base( + 30, + is_set.clone() * (b.main(0, cols::DST_0) - b.main(0, cols::SRC_0) - gap), + ); + b.emit_base( + 31, + is_set.clone() * (b.main(0, cols::DST_1) - b.main(0, cols::SRC_1)), + ); + } +} + +#[cfg(test)] +mod shape_tests { + /// Pins the committed width of the table. + /// + /// The bus-interaction and constraint counts are asserted in `memmove_tests`, but + /// the column count was only ever printed, and it is the number readers check + /// against the spec. + /// + /// **The spec says 37 and this says 38, and both are right.** The spec types + /// `timestamp` as a `Word` — one column — where this code uses a `DWordWL`, which + /// is two. The high limb is provably zero (the CPU sends `constant(0)` in the + /// `Ecall` tuple and `MemmoveNext` propagates it), so the extra column carries no + /// information; it is a convention divergence, not a disagreement. The same `+1` + /// applies to COMMIT, where the spec says 7 and the code has 8, and to `memw.toml`. + /// Three readers have now reported this as a bug, so it is written down here. + /// + /// Unrelated trap for anyone grepping: `cpu_tests.rs` also asserts 38, for the CPU + /// table. Coincidence. + #[test] + fn the_committed_shape_is_pinned() { + assert_eq!( + super::cols::NUM_COLUMNS, + 38, + "MEMMOVE columns (spec: 37 + 1)" + ); + + let n = super::bus_interactions().len(); + println!( + "MEMMOVE: {} columns, {} bus interactions, aux {} -> weight {}", + super::cols::NUM_COLUMNS, + n, + n.div_ceil(2), + super::cols::NUM_COLUMNS + 3 * n.div_ceil(2) + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn op(functionality: Functionality, dst: u64, count: u64, width: u8) -> MemmoveOperation { + MemmoveOperation { + timestamp: 100, + src: 0x1000, + dst, + count, + width, + first: false, + end: false, + functionality, + value: [1, 2, 3, 4, 5, 6, 7, 8], + } + } + + #[test] + fn timestamp_order_is_inverted_only_for_memset() { + let copy = op(Functionality::Copy, 0x2000, 64, 8); + assert_eq!(copy.read_timestamp(), 101); + assert_eq!(copy.write_timestamp(), 102); + + let commit = op(Functionality::Commit, 0, 64, 8); + assert_eq!(commit.read_timestamp(), 101); + assert_eq!(commit.write_timestamp(), 102); + + // memset writes first, so a row's read observes the previous row's write and + // the seeded bytes propagate across the range. + let set = op(Functionality::Set, 0x2008, 64, 8); + assert_eq!(set.write_timestamp(), 101); + assert_eq!(set.read_timestamp(), 102); + } + + #[test] + fn gate_columns_follow_the_functionality() { + let rows = [ + op(Functionality::Copy, 0x2000, 64, 8), + op(Functionality::Commit, 0, 64, 8), + op(Functionality::Set, 0x2008, 64, 8), + ]; + let trace = generate_memmove_trace(&rows); + let table = &trace.main_table; + let get = |row: usize, column: usize| *table.get(row, column); + + // The RAM write rides `mu - end - mu_com` and has no column, so `mu_com` is + // what says which domain a row writes to. + // Copy: COMMIT write off, so the RAM write is on. + assert_eq!(get(0, cols::MU_COM), FE::zero()); + // Commit: the mirror image, and the wide lanes are open on an eight-byte row. + assert_eq!(get(1, cols::MU_COM), FE::one()); + assert_eq!(get(1, cols::MU_COM_WIDE), FE::one()); + // Set is a RAM-to-RAM copy like memcpy; only the order differs. + assert_eq!(get(2, cols::MU_COM), FE::zero()); + assert_eq!(get(2, cols::IS_SET), FE::one()); + } + + #[test] + fn a_one_byte_commit_row_closes_the_wide_lanes() { + // Otherwise it would send seven spurious `(index, 0)` pairs on the COMMIT bus + // and corrupt the public-output fingerprint. + let rows = [op(Functionality::Commit, 40, 3, 1)]; + let trace = generate_memmove_trace(&rows); + assert_eq!(*trace.main_table.get(0, cols::MU_COM), FE::one()); + assert_eq!(*trace.main_table.get(0, cols::MU_COM_WIDE), FE::zero()); + } + + #[test] + fn the_schedule_is_wide_until_the_tail_whatever_the_alignment() { + use super::super::trace_builder::memmove_row_width_for_test as w; + // Alignment no longer enters into it: matched residues, mismatched residues + // and both-aligned all take eight bytes while eight remain. + for (src, dst) in [(0x1005u64, 0x2005u64), (0x1002, 0x2005), (0x1000, 0x2000)] { + assert_eq!(w(src, dst, 0, 24, false), 8, "src {src:#x} dst {dst:#x}"); + assert_eq!(w(src, dst, 8, 16, false), 8, "src {src:#x} dst {dst:#x}"); + } + // A short remainder is one byte a row, which is the only case that narrows. + assert_eq!(w(0x1000, 0x2000, 16, 5, false), 1); + assert_eq!(w(0x1005, 0x2005, 0, 7, false), 1); + // Commit is unchanged. + assert_eq!(w(0x1002, 0x2005, 0, 24, true), 8); + } +} diff --git a/prover/src/tables/mod.rs b/prover/src/tables/mod.rs index f1a899f56..d187dc127 100644 --- a/prover/src/tables/mod.rs +++ b/prover/src/tables/mod.rs @@ -41,6 +41,7 @@ pub mod keccak_rnd; pub mod load; pub mod local_to_global; pub mod lt; +pub mod memmove; pub mod memw; pub mod memw_aligned; pub mod memw_register; diff --git a/prover/src/tables/trace_builder.rs b/prover/src/tables/trace_builder.rs index d3560826a..2080f0726 100644 --- a/prover/src/tables/trace_builder.rs +++ b/prover/src/tables/trace_builder.rs @@ -58,6 +58,7 @@ use super::keccak_rnd::{self, KeccakRoundOperation}; use super::load::{self, LoadOperation}; use super::local_to_global; use super::lt::{self, LtOperation}; +use super::memmove; use super::memw::{self, MemwOperation}; use super::memw_aligned; use super::memw_register::{self, RegRow}; @@ -550,6 +551,7 @@ fn collect_ops_from_cpu( Vec, Vec, Vec, + Vec, Vec, ) { let mut memw = MemwBuckets::with_register_capacity(cpu_ops.len() * 3); @@ -562,6 +564,7 @@ fn collect_ops_from_cpu( let mut cpu32_ops = Vec::new(); let mut ecsm_ops = Vec::new(); let mut ecdas_ops = Vec::new(); + let mut memmove_ops: Vec = Vec::new(); let mut hint_ops = Vec::new(); // Seed from the carried x254 (0 for a monolithic run or the first epoch) so a // continuation epoch indexes its commits globally, matching the x254 the @@ -604,6 +607,16 @@ fn collect_ops_from_cpu( )); let reg_commit_ops = collect_commit_memw_ops(op, register_state, memory_state); memw.extend_ops(reg_commit_ops); + let (commit_memw, commit_rows) = collect_memmove_ops( + memmove::Functionality::Commit, + op.timestamp, + op.commit_buf_addr, + current_commit_index as u64, + op.commit_count, + memory_state, + ); + memw.extend_ops(commit_memw); + memmove_ops.extend(commit_rows); let count = u32::try_from(op.commit_count).expect("commit_count exceeds u32 range"); current_commit_index = current_commit_index .checked_add(count) @@ -657,6 +670,66 @@ fn collect_ops_from_cpu( ecdas_ops.extend(ecdas_rows); } + // DMA memcpy: authenticate x10/x11/x12, snapshot all source bytes at + // T+1, then write all destination bytes at T+2. + if op.ecall_dma_memcpy { + let dst = register_state.read(10).0; + let src = register_state.read(11).0; + let count = register_state.read(12).0; + for (reg, value) in [(10u8, dst), (11u8, src), (12u8, count)] { + let packed = pack_register_value(value); + let (_old_value, old_ts) = register_state.read(reg); + memw.extend_ops(vec![ + MemwOperation::new(true, 2 * reg as u64, packed, op.timestamp, 2, true) + .with_old(packed, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ]); + register_state.write(reg, value, op.timestamp); + } + let (mm_memw, rows) = collect_memmove_ops( + memmove::Functionality::Copy, + op.timestamp, + src, + dst, + count, + memory_state, + ); + memw.extend_ops(mm_memw); + memmove_ops.extend(rows); + } + + // DMA memset: authenticate x10/x11/x12, then run the copy primitive with the + // read/write order inverted — write at T+1, read at T+2. There IS a source + // phase, and the self-overlap is the point: the stub seeds eight bytes and + // calls with `dst = src + 8`, so each row's read observes the write eight + // bytes back and the seed propagates. That is what constraints 30/31 pin. + if op.ecall_dma_memset { + // memset is a memmove call whose only distinguishing feature is the + // inverted timestamp order; the stub already seeded the first eight + // bytes and passed dst = seed_end, src = seed_start, count = n - 8. + let dst = register_state.read(10).0; + let src = register_state.read(11).0; + let count = register_state.read(12).0; + for (reg, value) in [(10u8, dst), (11u8, src), (12u8, count)] { + let packed = pack_register_value(value); + let (_old_value, old_ts) = register_state.read(reg); + memw.extend_ops(vec![ + MemwOperation::new(true, 2 * reg as u64, packed, op.timestamp, 2, true) + .with_old(packed, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]), + ]); + register_state.write(reg, value, op.timestamp); + } + let (memset_memw, rows) = collect_memmove_ops( + memmove::Functionality::Set, + op.timestamp, + src, + dst, + count, + memory_state, + ); + memw.extend_ops(memset_memw); + memmove_ops.extend(rows); + } + // Collect Hint ecall operations (the 32-byte output write). if op.ecall_hint { let (hint_memw, hint_op) = collect_hint_ops(op, memory_state, register_state); @@ -700,12 +773,22 @@ fn collect_ops_from_cpu( bitwise_ops.extend(op.collect_bitwise_ops()); } - // Each ecall generates count+1 operations (count real rows + 1 end row). - // Count only this epoch's rows, so subtract the carried start index. + // COMMIT is one row per ecall now: the sys_write number, the fd check and the + // x254 update. The byte loop lives on the MEMMOVE chip, so the committed length + // is checked against the rows that actually move the bytes. debug_assert_eq!( commit_ops.len(), - (current_commit_index - start_commit_index) as usize + commit_ecall_count as usize, - "commit_ops count should match accumulated commit index plus end rows" + commit_ecall_count as usize, + "COMMIT should hold exactly one row per commit ecall" + ); + debug_assert_eq!( + memmove_ops + .iter() + .filter(|op| op.functionality == memmove::Functionality::Commit && !op.end) + .map(|op| op.width as u64) + .sum::(), + (current_commit_index - start_commit_index) as u64, + "MEMMOVE should move exactly the committed byte count" ); ( @@ -719,6 +802,7 @@ fn collect_ops_from_cpu( cpu32_ops, ecsm_ops, ecdas_ops, + memmove_ops, hint_ops, ) } @@ -959,6 +1043,161 @@ fn collect_ecsm_ops( (memw_ops, ecsm_op, ecdas_ops) } +/// Replays one ecall through the unified memmove primitive. +/// +/// Register operands are read at `T`. For a copy, every source chunk is read at `T+1` +/// before any destination chunk is written at `T+2`, which is the executor's snapshot +/// semantics even when the regions overlap; a memset inverts that order so each row's +/// read observes the previous row's write. Chunks are eight bytes while at least eight +/// remain, then one byte per tail row — width is a per-row choice the AIR permits at +/// any count, and this is simply the schedule the builder takes. +/// +/// Timestamp order follows the functionality: `Copy` and `Commit` read every chunk +/// at `T+1` and write at `T+2`, which snapshots the whole source range and gives +/// overlapping regions memmove semantics; `Set` inverts it, writing at `T+1` and +/// reading at `T+2`, so each row observes the previous row's write and the seeded +/// bytes propagate across the range. `Commit` writes into the COMMIT domain, so it +/// emits no MEMW write at all. +fn collect_memmove_ops( + functionality: memmove::Functionality, + timestamp: u64, + src: u64, + dst: u64, + count: u64, + memory_state: &mut MemoryState, +) -> (Vec, Vec) { + let mut memw_ops = Vec::new(); + let mut rows = Vec::new(); + let inverted = functionality == memmove::Functionality::Set; + let to_commit_domain = functionality == memmove::Functionality::Commit; + let read_ts = timestamp + 1 + u64::from(inverted); + let write_ts = timestamp + 2 - u64::from(inverted); + + let mut offset = 0u64; + let mut remaining = count; + let mut first = true; + // Snapshot order needs every read to land before any write, so the writes of a + // non-inverted call are held back to a second pass. + let mut deferred_writes = Vec::new(); + + while remaining != 0 { + let width = executor::vm::instruction::execution::memmove_row_width( + src, + dst, + offset, + remaining, + to_commit_domain, + ); + let source_addr = src.wrapping_add(offset); + let destination_addr = dst.wrapping_add(offset); + let (value, old_timestamps) = memory_state.read_bytes(source_addr, width as usize); + let bytes = value.map(|byte| byte as u8); + let dword = u64::from_le_bytes(bytes); + + if inverted { + // Write first, at the earlier timestamp, so this row's read sees the + // previous row's write. + let (old_values, old_dst_timestamps) = + memory_state.read_bytes(destination_addr, width as usize); + memw_ops.push( + MemwOperation::new(false, destination_addr, value, write_ts, width, false) + .with_old(old_values, old_dst_timestamps), + ); + memory_state.write_bytes(destination_addr, dword, width as usize, write_ts); + memw_ops.push( + MemwOperation::new(false, source_addr, value, read_ts, width, true) + .with_old(value, old_timestamps), + ); + memory_state.write_bytes(source_addr, dword, width as usize, read_ts); + } else { + memw_ops.push( + MemwOperation::new(false, source_addr, value, read_ts, width, true) + .with_old(value, old_timestamps), + ); + memory_state.write_bytes(source_addr, dword, width as usize, read_ts); + if !to_commit_domain { + deferred_writes.push((destination_addr, width, value, dword)); + } + } + + rows.push(memmove::MemmoveOperation { + timestamp, + src: source_addr, + dst: destination_addr, + count: remaining, + width, + first, + end: false, + functionality, + value: bytes, + }); + + first = false; + offset += u64::from(width); + remaining -= u64::from(width); + } + + for (destination_addr, width, value, dword) in deferred_writes { + let (old_values, old_timestamps) = + memory_state.read_bytes(destination_addr, width as usize); + memw_ops.push( + MemwOperation::new(false, destination_addr, value, write_ts, width, false) + .with_old(old_values, old_timestamps), + ); + memory_state.write_bytes(destination_addr, dword, width as usize, write_ts); + } + + rows.push(memmove::MemmoveOperation { + timestamp, + src: src.wrapping_add(count), + dst: dst.wrapping_add(count), + count: 0, + width: 1, + first, + end: true, + functionality, + value: [0; 8], + }); + + (memw_ops, rows) +} + +/// Sizing-pass replay of one MEMMOVE-driven ecall. +/// +/// Deliberately delegates to [`collect_memmove_ops`] rather than re-deriving the +/// schedule: the two used to be separate implementations pinned together by an +/// assertion, and the schedule is now a function of `dst` as well as `count`, which +/// is exactly the kind of thing that drifts. The cost is one allocation per ecall. +#[cfg(feature = "disk-spill")] +fn replay_memmove_for_sizing( + functionality: memmove::Functionality, + timestamp: u64, + src: u64, + dst: u64, + count: u64, + memory_state: &mut MemoryState, + mut visit_memw: impl FnMut(&MemwOperation), +) -> u64 { + let (memw_ops, rows) = + collect_memmove_ops(functionality, timestamp, src, dst, count, memory_state); + for op in &memw_ops { + visit_memw(op); + } + rows.len() as u64 +} + +/// Test hook for the schedule, so the MEMMOVE unit tests can pin it. +#[cfg(test)] +pub fn memmove_row_width_for_test( + src: u64, + dst: u64, + offset: u64, + remaining: u64, + to_commit: bool, +) -> u8 { + executor::vm::instruction::execution::memmove_row_width(src, dst, offset, remaining, to_commit) +} + /// Collects the memory operations for a `Hint` ecall. /// /// The `hint` ecall writes a 32-byte value (a modular inverse / sqrt) to guest @@ -1285,7 +1524,7 @@ fn cpu32_chip_op( fn collect_commit_memw_ops( op: &CpuOperation, register_state: &mut RegisterState, - memory_state: &mut MemoryState, + _memory_state: &mut MemoryState, ) -> Vec { let ts = op.timestamp; let buf_addr = op.commit_buf_addr; @@ -1359,18 +1598,7 @@ fn collect_commit_memw_ops( register_state.write_index(new_index, ts); } - // Memory byte reads at ts - for i in 0..count { - let addr = buf_addr.wrapping_add(i); - let (byte_val, old_ts) = memory_state.read_byte(addr); - let value = [byte_val as u32, 0, 0, 0, 0, 0, 0, 0]; - let old_timestamps = [old_ts, 0, 0, 0, 0, 0, 0, 0]; - let memw_op = - MemwOperation::new(false, addr, value, ts, 1, true).with_old(value, old_timestamps); - memw_ops.push(memw_op); - memory_state.write_byte(addr, byte_val, ts); - } - + // The byte reads moved to the MEMMOVE chip, eight at a time. memw_ops } @@ -2249,88 +2477,50 @@ fn collect_bitwise_from_page( /// at the moment the ECALL executes. fn expand_commit_operations_for_ecall( ecall: &CpuOperation, - memory_state: &MemoryState, + _memory_state: &MemoryState, start_index: u64, ) -> Vec { - let mut ops = Vec::new(); - - let timestamp = ecall.timestamp; - let buf_addr = ecall.commit_buf_addr; + // One row per ecall now: the sys_write number, the fd check and the x254 update. + // The byte loop is deferred to the MEMMOVE chip over `BusId::CommitDefer`. let count = ecall.commit_count; - - for i in 0..=count { - let remaining = count - i; - let is_end = remaining == 0; - let value = if !is_end { - let (byte_val, _ts) = memory_state.read_byte(buf_addr.wrapping_add(i)); - byte_val - } else { - 0 - }; - ops.push(CommitOperation { - timestamp, - index: start_index.wrapping_add(i), - address: buf_addr.wrapping_add(i), - count: remaining, - first: i == 0, - end: is_end, - value, - }); - } - - ops + vec![CommitOperation { + timestamp: ecall.timestamp, + index: start_index, + address: ecall.commit_buf_addr, + count, + }] } -/// Collect bitwise lookups from COMMIT operations. -/// -/// The COMMIT table sends: -/// - IsHalfword for count_decr components (4 per real row, mult = mu) -/// - IsHalfword for address_incr halfwords (4 per real row, mult = mu) -/// - Zero for end detection (1 per real row, mult = mu) -/// -/// Note: AreBytes for value is intentionally omitted per spec. -fn collect_bitwise_from_commit(commit_ops: &[CommitOperation]) -> Vec { - let mut lookups = Vec::new(); - - for op in commit_ops { - // IsHalfword for count_decr components (4 halfwords, mult = mu) - let count_decr = if op.count == 0 { - u64::MAX - } else { - op.count - 1 - }; - for shift in [0, 16, 32, 48] { - let half = ((count_decr >> shift) & 0xFFFF) as u16; - lookups.push(BitwiseOperation::halfword( - BitwiseOperationType::IsHalf, - (half & 0xFF) as u8, - ((half >> 8) & 0xFF) as u8, - )); - } +/// BITWISE lookups sent by the MEMMOVE table: twelve `IS_HALF` for the three +/// incremented dwords plus the `ZERO` end detection, one set per row. +fn collect_bitwise_from_memmove(ops: &[memmove::MemmoveOperation]) -> Vec { + let mut lookups = Vec::with_capacity(ops.len() * 13); + for op in ops { + let width = u64::from(op.width); + let count_decr = op.count.wrapping_sub(width); + let src_incr = op.src.wrapping_add(width); + let dst_incr = op.dst.wrapping_add(width); - // IsHalfword for address_incr halfwords (4 halfwords, mult = mu) - // All real rows send these, matching the spec's unconditional mult = mu. - let address_incr = op.address.wrapping_add(1); - for shift in [0, 16, 32, 48] { - let half = ((address_incr >> shift) & 0xFFFF) as u16; - lookups.push(BitwiseOperation::halfword( - BitwiseOperationType::IsHalf, - (half & 0xFF) as u8, - ((half >> 8) & 0xFF) as u8, - )); + for value in [count_decr, src_incr, dst_incr] { + for shift in [0, 16, 32, 48] { + let half = ((value >> shift) & 0xFFFF) as u16; + lookups.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + (half >> 8) as u8, + )); + } } - // Zero bus for end detection (mult = mu) - // Input: (65535 - cd_0) + (65535 - cd_1) + (65535 - cd_2) + (65535 - cd_3) - // When count_decr = 0xFFFF_FFFF_FFFF_FFFF (count=0), sum = 0 → end=1 - let cd_0 = (count_decr & 0xFFFF) as u32; - let cd_1 = ((count_decr >> 16) & 0xFFFF) as u32; - let cd_2 = ((count_decr >> 32) & 0xFFFF) as u32; - let cd_3 = ((count_decr >> 48) & 0xFFFF) as u32; - let zero_input = (65535 - cd_0) + (65535 - cd_1) + (65535 - cd_2) + (65535 - cd_3); + let halves = [ + (count_decr & 0xFFFF) as u32, + ((count_decr >> 16) & 0xFFFF) as u32, + ((count_decr >> 32) & 0xFFFF) as u32, + ((count_decr >> 48) & 0xFFFF) as u32, + ]; + let zero_input = halves.into_iter().map(|half| 65535 - half).sum(); lookups.push(BitwiseOperation::zero(zero_input)); } - lookups } @@ -2870,6 +3060,10 @@ pub struct Traces { /// ECDAS double/add table (variable rows per ecall) pub ecdas: TraceTable, + /// Unified MEMMOVE table: one streaming copy primitive for memcpy/memmove, + /// memset and the commit byte loop, selected by decoded functionality columns. + pub memmove: TraceTable, + /// HINT table (one row per non-constraining hint ecall). pub hint: TraceTable, @@ -2915,6 +3109,8 @@ struct CollectedOps { // EC scalar-multiplication accelerator chips. ecsm_ops: Vec, ecdas_ops: Vec, + // Unified memmove rows: memcpy/memmove, memset and the commit byte loop. + memmove_ops: Vec, // Non-constraining hint ecall. hint_ops: Vec, } @@ -2971,6 +3167,7 @@ fn collect_all_ops( cpu32_ops: Vec, ecsm_ops: Vec, ecdas_ops: Vec, + memmove_ops: Vec, hint_ops: Vec, register_state: &mut RegisterState, is_final: bool, @@ -3115,6 +3312,7 @@ fn collect_all_ops( ecsm_ops, ecdas_ops, hint_ops, + memmove_ops, } } @@ -3158,6 +3356,7 @@ fn build_traces( cpu32_ops, ecsm_ops, ecdas_ops, + memmove_ops, hint_ops, } = ops; @@ -3166,6 +3365,18 @@ fn build_traces( // ===================================================================== lt_ops.extend(collect_lt_from_memw(&memw_ops)); lt_ops.extend(collect_lt_from_memw_aligned(&memw_aligned_ops)); + // MEMMOVE: `lt8` on every row, and the per-ecall byte bound on the first row. + lt_ops.extend( + memmove_ops + .iter() + .map(|op| LtOperation::new(op.count, 8, false)), + ); + lt_ops.extend( + memmove_ops + .iter() + .filter(|op| op.first && op.functionality != memmove::Functionality::Commit) + .map(|op| LtOperation::new(op.count, memmove::MEMMOVE_MAX_BYTES + 1, false)), + ); // HINT range-checks: selector < 3 and both address low limbs < 2^32 - 31 (matching // the executor's HintUnknownSelector / HintAddressOverflow rejections). Three LT ops // per hint call; the HINT table sends the matching ALU LT interactions. @@ -3183,11 +3394,19 @@ fn build_traces( #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p4_bitwise_collect"); - let public_output_bytes: Vec = commit_ops - .iter() - .filter(|op| !op.end) - .map(|op| op.value) - .collect(); + // The committed bytes now flow through the MEMMOVE chip, so the public output is + // read off its COMMIT-domain rows rather than off COMMIT's (one row per ecall). + let public_output_bytes: Vec = { + let mut rows: Vec<&memmove::MemmoveOperation> = memmove_ops + .iter() + .filter(|op| op.functionality == memmove::Functionality::Commit && !op.end) + .collect(); + // `dst` is the COMMIT-domain address, i.e. the running global byte index. + rows.sort_by_key(|op| op.dst); + rows.iter() + .flat_map(|op| op.value[..op.width as usize].iter().copied()) + .collect() + }; // CPU padding rows send ARE_BYTES with all-zero values. // Add corresponding ops so the bitwise table multiplicities balance. @@ -3241,7 +3460,7 @@ fn build_traces( } }), Box::new(|h| h.add_ops(&collect_bitwise_from_memw_aligned(&memw_aligned_ops))), - Box::new(|h| h.add_ops(&collect_bitwise_from_commit(&commit_ops))), + Box::new(|h| h.add_ops(&collect_bitwise_from_memmove(&memmove_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_keccak(&keccak_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_ecsm(&ecsm_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_ecdas(&ecdas_ops))), @@ -3532,6 +3751,7 @@ fn build_traces( // ECSM accelerator traces (empty/all-padding for programs that do not use ECSM). let gen_ecsm = || ecsm::generate_ecsm_trace(&ecsm_ops); let gen_ecdas = || ecdas::generate_ecdas_trace(&ecdas_ops); + let gen_memmove = || memmove::generate_memmove_trace(&memmove_ops); // HINT table (all-padding for programs that make no hint ecalls). let gen_hint = || hint::generate_hint_trace(&hint_ops); @@ -3546,6 +3766,7 @@ fn build_traces( let (mut eqs_slot, mut bytewises_slot, mut stores_slot, mut cpu32s_slot) = (None, None, None, None); let (mut ecsm_slot, mut ecdas_slot) = (None, None); + let mut memmove_slot = None; let mut hint_slot = None; #[cfg(feature = "disk-spill")] @@ -3588,6 +3809,7 @@ fn build_traces( spawn_into!(cpu32s_slot, gen_cpu32s); spawn_into!(ecsm_slot, gen_ecsm); spawn_into!(ecdas_slot, gen_ecdas); + spawn_into!(memmove_slot, gen_memmove); spawn_into!(hint_slot, gen_hint); }); } else { @@ -3616,6 +3838,7 @@ fn build_traces( cpu32s_slot = Some(gen_cpu32s()); ecsm_slot = Some(gen_ecsm()); ecdas_slot = Some(gen_ecdas()); + memmove_slot = Some(gen_memmove()); hint_slot = Some(gen_hint()); } @@ -3651,6 +3874,7 @@ fn build_traces( let mut halt_trace = halt_slot.expect(PHASE5_RAN); let ecsm_trace = ecsm_slot.expect(PHASE5_RAN); let ecdas_trace = ecdas_slot.expect(PHASE5_RAN); + let memmove_trace = memmove_slot.expect(PHASE5_RAN); let hint_trace = hint_slot.expect(PHASE5_RAN); // Fixed-size and per-page tables aren't built through `chunk_and_generate`, @@ -3719,6 +3943,7 @@ fn build_traces( keccak_rc: keccak_rc_trace, ecsm: ecsm_trace, ecdas: ecdas_trace, + memmove: memmove_trace, hint: hint_trace, memw_registers, local_to_global, @@ -3763,6 +3988,7 @@ pub struct TableLengths { pub dvrm_padded_rows: u64, pub branch_padded_rows: u64, pub commit_padded_rows: u64, + pub memmove_padded_rows: u64, pub decode_rows: u64, pub unique_page_count: u64, pub cycle_count: u64, @@ -3802,6 +4028,7 @@ pub fn count_table_lengths( let mut dvrm_count = 0usize; let mut branch_count = 0usize; let mut commit_count = 0usize; + let mut memmove_count = 0usize; let mut current_commit_index = 0u32; let partition_memw = |op: &MemwOperation, @@ -3871,11 +4098,8 @@ pub fn count_table_lengths( // ECALL Commit if cpu_op.ecall_commit { - // Match `expand_commit_operations_for_ecall`'s `0..=count` loop - // without building the op vector. - commit_count += (cpu_op.commit_count as usize) - .checked_add(1) - .ok_or_else(|| Error::Execution("commit_count overflows usize".into()))?; + // COMMIT is one row per ecall now; the byte loop is MEMMOVE's. + commit_count += 1; let reg_commit_ops = collect_commit_memw_ops(&cpu_op, &mut register_state, &mut memory_state); for memw_op in ®_commit_ops { @@ -3886,6 +4110,24 @@ pub fn count_table_lengths( &mut memw_register_count, ); } + let rows = replay_memmove_for_sizing( + memmove::Functionality::Commit, + cpu_op.timestamp, + cpu_op.commit_buf_addr, + current_commit_index as u64, + cpu_op.commit_count, + &mut memory_state, + |memw_op| { + partition_memw( + memw_op, + &mut memw_by_width, + &mut memw_aligned_count, + &mut memw_register_count, + ); + }, + ); + memmove_count += rows as usize; + lt_count += rows as usize; let count = u32::try_from(cpu_op.commit_count) .map_err(|_| Error::Execution("commit_count exceeds u32 range".into()))?; current_commit_index = current_commit_index @@ -3893,6 +4135,50 @@ pub fn count_table_lengths( .ok_or_else(|| Error::Execution("commit index exceeds u32 range".into()))?; } + if cpu_op.ecall_dma_memcpy || cpu_op.ecall_dma_memset { + let functionality = if cpu_op.ecall_dma_memset { + memmove::Functionality::Set + } else { + memmove::Functionality::Copy + }; + let dst = register_state.read(10).0; + let src = register_state.read(11).0; + let count = register_state.read(12).0; + for (reg, value) in [(10u8, dst), (11u8, src), (12u8, count)] { + let packed = pack_register_value(value); + let (_old_value, old_ts) = register_state.read(reg); + let reg_op = + MemwOperation::new(true, 2 * reg as u64, packed, cpu_op.timestamp, 2, true) + .with_old(packed, [old_ts, old_ts, 0, 0, 0, 0, 0, 0]); + partition_memw( + ®_op, + &mut memw_by_width, + &mut memw_aligned_count, + &mut memw_register_count, + ); + register_state.write(reg, value, cpu_op.timestamp); + } + let rows = replay_memmove_for_sizing( + functionality, + cpu_op.timestamp, + src, + dst, + count, + &mut memory_state, + |memw_op| { + partition_memw( + memw_op, + &mut memw_by_width, + &mut memw_aligned_count, + &mut memw_register_count, + ); + }, + ); + memmove_count += rows as usize; + // One LT per row pins `lt8`, plus one per ecall for the chunk cap. + lt_count += rows as usize + 1; + } + if cpu_op.ecall_hint { // Mirror `collect_hint_ops`: three register reads (a0/a1/a2) and four // 8-byte output writes go through the memory argument, plus the three LT @@ -3971,6 +4257,7 @@ pub fn count_table_lengths( .checked_next_power_of_two() .unwrap_or(usize::MAX) .max(4) as u64, + memmove_padded_rows: memmove_count.next_power_of_two().max(4) as u64, decode_rows, unique_page_count, cycle_count, @@ -4037,6 +4324,11 @@ impl Traces { tables.push(&mut self.keccak_rnd); tables.push(&mut self.ecsm); tables.push(&mut self.ecdas); + // MEMMOVE qualifies on the same criterion as ECSM/ECDAS: of the tables left + // out, it is the only one whose height scales with the workload rather than + // being one row per event. Nothing mutates it after the build, so unlike + // BITWISE it cannot go stale on the device. + tables.push(&mut self.memmove); let bytes_of = |t: &TraceTable| { t.num_rows() * t.num_main_columns * 8 @@ -4118,6 +4410,7 @@ impl Traces { ecsm, ecdas, hint, + memmove, memw_registers, eqs, bytewises, @@ -4185,6 +4478,7 @@ impl Traces { } total += (ecsm.num_rows() * ECSM_COLS) as u64; total += (ecdas.num_rows() * ECDAS_COLS) as u64; + total += (memmove.num_rows() * super::memmove::cols::NUM_COLUMNS) as u64; total += (hint.num_rows() * HINT_COLS) as u64; total } @@ -4227,6 +4521,7 @@ impl Traces { let n_cpu32 = aux_cols(super::cpu32::bus_interactions().len()); let n_ecsm = aux_cols(super::ecsm::bus_interactions().len()); let n_ecdas = aux_cols(super::ecdas::bus_interactions().len()); + let n_memmove = aux_cols(super::memmove::bus_interactions().len()); let n_hint = aux_cols(super::hint::bus_interactions().len()); let Traces { @@ -4251,6 +4546,7 @@ impl Traces { ecsm, ecdas, hint, + memmove, memw_registers, eqs, bytewises, @@ -4318,6 +4614,7 @@ impl Traces { } total += (ecsm.num_rows() * n_ecsm) as u64; total += (ecdas.num_rows() * n_ecdas) as u64; + total += (memmove.num_rows() * n_memmove) as u64; total += (hint.num_rows() * n_hint) as u64; total } @@ -4673,6 +4970,7 @@ impl Traces { ecsm_ops, ecdas_ops, hint_ops, + memmove_ops, ) = collect_ops_from_cpu(&cpu_ops, &mut memory_state, &mut register_state); #[cfg(feature = "instruments")] drop(__sp); @@ -4692,6 +4990,7 @@ impl Traces { ecsm_ops, ecdas_ops, hint_ops, + memmove_ops, &mut register_state, is_final, ); @@ -4786,6 +5085,7 @@ impl Traces { ecsm_ops, ecdas_ops, hint_ops, + memmove_ops, ) = collect_ops_from_cpu(&cpu_ops, &mut memory_state, &mut register_state); let ops = collect_all_ops( @@ -4801,6 +5101,7 @@ impl Traces { ecsm_ops, ecdas_ops, hint_ops, + memmove_ops, &mut register_state, true, ); diff --git a/prover/src/tables/types.rs b/prover/src/tables/types.rs index fab4aabff..99bb1db0a 100644 --- a/prover/src/tables/types.rs +++ b/prover/src/tables/types.rs @@ -309,8 +309,8 @@ pub enum BusId { Decode = 18, /// System call handling (CPU → HALT/COMMIT for all ECALLs) Ecall = 19, - /// COMMIT self-referencing recursive bus (row N → row N+1) - CommitNextByte = 20, + // ID 20 is reserved for the removed CommitNextByte bus: COMMIT's per-byte + // recursion moved to MEMMOVE, which chains over [`BusId::MemmoveNext`]. /// COMMIT output bus: verifier computes the receiver contribution externally /// from `VmProof.public_output` using the shared LogUp challenges Commit = 21, @@ -353,12 +353,35 @@ pub enum BusId { /// and sends Bit[ts, idx_k] for the MSB (mult = μ). Bit = 30, + // ========================================================================= + // DMA memcpy accelerator + // ========================================================================= + // IDs 29 and 32 are reserved for the removed DmaNext and DmaSetNext buses: + // the DMA and DMA_SET tables they chained are both replaced by MEMMOVE, which + // chains over [`BusId::MemmoveNext`] and carries the functionality selectors + // inside the tuple rather than separating the paths by bus id. + // ========================================================================= // Continuations // ========================================================================= /// Cross-epoch memory bus: the local-to-global table's per-cell init/fini /// boundary claims, matched across epochs by the final aggregation LogUp. GlobalMemory = 31, + + // ========================================================================= + // Unified memmove primitive + // ========================================================================= + /// MEMMOVE self-referential streaming bus. A row sends + /// `(timestamp, src_incr, dst_incr, count_decr, is_set, is_commit)` to the next + /// row and receives `(timestamp, src, dst, count, is_set, is_commit)` from the + /// previous one. The functionality selectors travel inside the tuple, so a chain + /// cannot change operation half way through it — the guarantee that the three + /// removed DmaNext/DmaSetNext/CommitNextByte buses used to give structurally. + MemmoveNext = 33, + /// COMMIT → MEMMOVE hand-off: COMMIT keeps the `sys_write` ecall number and the + /// register-254 update, and defers its byte loop here as + /// `(timestamp, buf_addr, start_index, count)`. + CommitDefer = 34, } impl BusId { @@ -377,8 +400,9 @@ impl BusId { BusId::Branch => "Branch", BusId::Decode => "Decode", BusId::Ecall => "Ecall", - BusId::CommitNextByte => "CommitNextByte", BusId::Commit => "Commit", + BusId::MemmoveNext => "MemmoveNext", + BusId::CommitDefer => "CommitDefer", BusId::Keccak => "Keccak", BusId::KeccakRc => "KeccakRc", BusId::ByteAlu => "ByteAlu", @@ -409,7 +433,6 @@ impl TryFrom for BusId { 17 => Ok(BusId::Branch), 18 => Ok(BusId::Decode), 19 => Ok(BusId::Ecall), - 20 => Ok(BusId::CommitNextByte), 21 => Ok(BusId::Commit), 22 => Ok(BusId::Keccak), 23 => Ok(BusId::KeccakRc), @@ -420,6 +443,8 @@ impl TryFrom for BusId { 28 => Ok(BusId::Ecdas), 30 => Ok(BusId::Bit), 31 => Ok(BusId::GlobalMemory), + 33 => Ok(BusId::MemmoveNext), + 34 => Ok(BusId::CommitDefer), other => Err(other), } } diff --git a/prover/src/test_utils.rs b/prover/src/test_utils.rs index d6a8b8608..877240668 100644 --- a/prover/src/test_utils.rs +++ b/prover/src/test_utils.rs @@ -912,6 +912,21 @@ pub fn create_hint_air(proof_options: &ProofOptions) -> ConcreteVmAir ConcreteVmAir { + build_air( + crate::tables::memmove::cols::NUM_COLUMNS, + crate::tables::memmove::bus_interactions(), + proof_options, + 1, + crate::tables::memmove::MemmoveConstraints, + "MEMMOVE", + ) +} + /// Create COMMIT AIR with constraints and bus interactions. pub fn create_commit_air(proof_options: &ProofOptions) -> ConcreteVmAir { build_air( diff --git a/prover/src/tests/commit_tests.rs b/prover/src/tests/commit_tests.rs index fdaf4d2cd..f744fa15f 100644 --- a/prover/src/tests/commit_tests.rs +++ b/prover/src/tests/commit_tests.rs @@ -1,435 +1,160 @@ //! Tests for the COMMIT (ECALL) table. //! -//! Covers trace generation, constraint formula verification, and edge cases. +//! COMMIT is now one row per `sys_write` ECALL: it accepts the syscall number, reads +//! the operand registers, advances the committed-length register x254, and hands the +//! byte loop to MEMMOVE over `BusId::CommitDefer`. Everything that modelled a +//! per-byte sequence — `first`, `end`, `value`, `address_incr`, `count_decr` and +//! their range checks — went with the loop, so the tests that exercised that +//! machinery went with it too. What is left is the row shape, the padding, and the +//! interaction and constraint inventory. -use crate::constraints::templates::INV_SHIFT_32; use crate::tables::commit::{CommitOperation, cols, generate_commit_trace}; -use crate::tables::types::FE; +use crate::tables::types::{FE, VmTable}; +use crate::test_utils::{busless_air, validate_busless}; -// ========================================================================= -// Helper: build a commit row -// ========================================================================= - -fn op( - timestamp: u64, - index: u64, - address: u64, - count: u64, - first: bool, - end: bool, - value: u8, -) -> CommitOperation { +fn op(timestamp: u64, index: u64, address: u64, count: u64) -> CommitOperation { CommitOperation { timestamp, index, address, count, - first, - end, - value, } } // ========================================================================= -// Trace generation tests +// Trace generation // ========================================================================= #[test] -fn test_commit_single_byte() { - // count=1: first row (first=1, count=1, value=0x41) + end row (end=1, count=0) - let ops = vec![ - op(100, 0, 0x1000, 1, true, false, 0x41), - op(100, 1, 0x1001, 0, false, true, 0), - ]; - let trace = generate_commit_trace(&ops); - - // Row 0: first=1, end=0, count=1, value=0x41, mu=1 - let r0 = trace.main_table.get_row(0); - assert_eq!(r0[cols::FIRST], FE::one()); - assert_eq!(r0[cols::END], FE::zero()); - assert_eq!(r0[cols::COUNT_0], FE::one()); - assert_eq!(r0[cols::COUNT_1], FE::zero()); - assert_eq!(r0[cols::VALUE], FE::from(0x41u64)); - assert_eq!(r0[cols::MU], FE::one()); - assert_eq!(r0[cols::TIMESTAMP_0], FE::from(100u64)); - assert_eq!(r0[cols::INDEX], FE::zero()); - - // Row 0: address = 0x1000 - assert_eq!(r0[cols::ADDRESS_0], FE::from(0x1000u64)); - assert_eq!(r0[cols::ADDRESS_1], FE::zero()); +fn a_commit_ecall_is_one_row_carrying_its_operands() { + let trace = generate_commit_trace(&[op(0x1234_5678_9ABC, 42, 0x2000, 7)]); + let r = trace.main_table.get_row(0); - // Row 0: address_incr = 0x1001 - assert_eq!(r0[cols::ADDRESS_INCR_0], FE::from(0x1001u64)); - assert_eq!(r0[cols::ADDRESS_INCR_1], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_2], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_3], FE::zero()); - - // Row 0: count_decr = 0 (count=1 → count-1=0) - assert_eq!(r0[cols::COUNT_DECR_0], FE::zero()); - assert_eq!(r0[cols::COUNT_DECR_1], FE::zero()); - assert_eq!(r0[cols::COUNT_DECR_2], FE::zero()); - assert_eq!(r0[cols::COUNT_DECR_3], FE::zero()); - - // Row 1: first=0, end=1, count=0, value=0, mu=1 - let r1 = trace.main_table.get_row(1); - assert_eq!(r1[cols::FIRST], FE::zero()); - assert_eq!(r1[cols::END], FE::one()); - assert_eq!(r1[cols::COUNT_0], FE::zero()); - assert_eq!(r1[cols::VALUE], FE::zero()); - assert_eq!(r1[cols::MU], FE::one()); - assert_eq!(r1[cols::INDEX], FE::one()); - - // Row 1: count_decr = all 0xFFFF (count=0 → underflow) - assert_eq!(r1[cols::COUNT_DECR_0], FE::from(0xFFFFu64)); - assert_eq!(r1[cols::COUNT_DECR_1], FE::from(0xFFFFu64)); - assert_eq!(r1[cols::COUNT_DECR_2], FE::from(0xFFFFu64)); - assert_eq!(r1[cols::COUNT_DECR_3], FE::from(0xFFFFu64)); + assert_eq!(r[cols::TIMESTAMP_0], FE::from(0x5678_9ABCu64)); + assert_eq!(r[cols::TIMESTAMP_1], FE::from(0x1234u64)); + assert_eq!(r[cols::INDEX], FE::from(42u64)); + assert_eq!(r[cols::ADDRESS_0], FE::from(0x2000u64)); + assert_eq!(r[cols::ADDRESS_1], FE::zero()); + assert_eq!(r[cols::COUNT_0], FE::from(7u64)); + assert_eq!(r[cols::COUNT_1], FE::zero()); + assert_eq!(r[cols::MU], FE::one()); } #[test] -fn test_commit_multi_byte() { - // count=3: 3 data rows + 1 end row = 4 rows - let ops = vec![ - op(200, 10, 0x2000, 3, true, false, b'H'), - op(200, 11, 0x2001, 2, false, false, b'i'), - op(200, 12, 0x2002, 1, false, false, b'!'), - op(200, 13, 0x2003, 0, false, true, 0), - ]; - let trace = generate_commit_trace(&ops); - - // Row 0: first=1 - let r0 = trace.main_table.get_row(0); - assert_eq!(r0[cols::FIRST], FE::one()); - assert_eq!(r0[cols::END], FE::zero()); - assert_eq!(r0[cols::COUNT_0], FE::from(3u64)); - assert_eq!(r0[cols::VALUE], FE::from(b'H' as u64)); - assert_eq!(r0[cols::INDEX], FE::from(10u64)); +fn several_ecalls_are_several_rows_and_nothing_chains_them() { + // Two commits: 3 bytes from index 0, then 9 bytes from index 3. Under the old + // per-byte design this was 3 + 1 + 9 + 1 rows linked by CommitNextByte; it is now + // exactly two independent rows. + let trace = generate_commit_trace(&[op(100, 0, 0x2000, 3), op(200, 3, 0x3000, 9)]); - // Row 1: middle row, count decrement 3→2 - let r1 = trace.main_table.get_row(1); - assert_eq!(r1[cols::FIRST], FE::zero()); - assert_eq!(r1[cols::END], FE::zero()); - assert_eq!(r1[cols::COUNT_0], FE::from(2u64)); - assert_eq!(r1[cols::VALUE], FE::from(b'i' as u64)); - assert_eq!(r1[cols::INDEX], FE::from(11u64)); + let first = trace.main_table.get_row(0); + assert_eq!(first[cols::INDEX], FE::zero()); + assert_eq!(first[cols::COUNT_0], FE::from(3u64)); - // Row 2: middle row, count decrement 2→1 - let r2 = trace.main_table.get_row(2); - assert_eq!(r2[cols::COUNT_0], FE::from(1u64)); - assert_eq!(r2[cols::VALUE], FE::from(b'!' as u64)); - assert_eq!(r2[cols::INDEX], FE::from(12u64)); - - // Row 3: end row - let r3 = trace.main_table.get_row(3); - assert_eq!(r3[cols::FIRST], FE::zero()); - assert_eq!(r3[cols::END], FE::one()); - assert_eq!(r3[cols::COUNT_0], FE::zero()); - assert_eq!(r3[cols::INDEX], FE::from(13u64)); - - // All rows share timestamp and mu=1 - for row in 0..4 { - let r = trace.main_table.get_row(row); - assert_eq!(r[cols::TIMESTAMP_0], FE::from(200u64)); - assert_eq!(r[cols::MU], FE::one()); - } - - // Address chain: 0x2000, 0x2001, 0x2002, 0x2003 - for (row, addr) in (0x2000u64..=0x2003).enumerate() { - let r = trace.main_table.get_row(row); - assert_eq!(r[cols::ADDRESS_0], FE::from(addr)); - } + let second = trace.main_table.get_row(1); + assert_eq!(second[cols::TIMESTAMP_0], FE::from(200u64)); + assert_eq!(second[cols::INDEX], FE::from(3u64)); + assert_eq!(second[cols::COUNT_0], FE::from(9u64)); } #[test] -fn test_commit_zero_count() { - // count=0: single row with first=1 AND end=1 - let ops = vec![op(50, 7, 0x3000, 0, true, true, 0)]; - let trace = generate_commit_trace(&ops); - let r0 = trace.main_table.get_row(0); - - assert_eq!(r0[cols::FIRST], FE::one()); - assert_eq!(r0[cols::END], FE::one()); - assert_eq!(r0[cols::COUNT_0], FE::zero()); - assert_eq!(r0[cols::MU], FE::one()); - assert_eq!(r0[cols::INDEX], FE::from(7u64)); - - // count_decr = all 0xFFFF when count=0 - assert_eq!(r0[cols::COUNT_DECR_0], FE::from(0xFFFFu64)); - assert_eq!(r0[cols::COUNT_DECR_1], FE::from(0xFFFFu64)); - assert_eq!(r0[cols::COUNT_DECR_2], FE::from(0xFFFFu64)); - assert_eq!(r0[cols::COUNT_DECR_3], FE::from(0xFFFFu64)); +fn a_zero_length_commit_is_still_a_row() { + // The ECALL happened and x254 still has to be read and written, so the row exists + // even though MEMMOVE will copy nothing. + let trace = generate_commit_trace(&[op(100, 5, 0x2000, 0)]); + let r = trace.main_table.get_row(0); + assert_eq!(r[cols::COUNT_0], FE::zero()); + assert_eq!(r[cols::MU], FE::one()); } #[test] -fn test_commit_trace_padding() { - // 1 real row → padded to 4 (minimum power of 2) - let ops = vec![op(10, 0, 0x100, 0, true, true, 0)]; - let trace = generate_commit_trace(&ops); +fn padding_rows_are_all_zero() { + // The ADD/SUB templates that forced a non-zero padding row are gone with + // `address_incr` and `count_decr`, so padding is plain zero now. + let trace = generate_commit_trace(&[op(100, 0, 0x2000, 1)]); assert_eq!(trace.num_rows(), 4); - - // Padding rows (1..4): mu=0, count=1, address_incr_0=1 - for row in 1..4 { - let r = trace.main_table.get_row(row); - assert_eq!(r[cols::MU], FE::zero()); - assert_eq!(r[cols::COUNT_0], FE::one()); - assert_eq!(r[cols::ADDRESS_INCR_0], FE::one()); - assert_eq!(r[cols::FIRST], FE::zero()); - assert_eq!(r[cols::END], FE::zero()); - assert_eq!(r[cols::VALUE], FE::zero()); - assert_eq!(r[cols::ADDRESS_0], FE::zero()); - assert_eq!(r[cols::TIMESTAMP_0], FE::zero()); - assert_eq!(r[cols::INDEX], FE::zero()); + for row_idx in 1..4 { + let r = trace.main_table.get_row(row_idx); + for (col, value) in r.iter().enumerate().take(cols::NUM_COLUMNS) { + assert_eq!(*value, FE::zero(), "padding row {row_idx}, column {col}"); + } } } #[test] -fn test_commit_trace_dimensions() { - // 5 rows → next power of 2 = 8 - let ops: Vec<_> = (0..5) - .map(|i| op(300, i, 0x4000 + i, 5 - i, i == 0, i == 4, (0x60 + i) as u8)) - .collect(); - let trace = generate_commit_trace(&ops); - - assert_eq!(trace.num_rows(), 8); - assert_eq!(cols::NUM_COLUMNS, 19); +fn the_table_pads_to_a_power_of_two_with_a_floor_of_four() { + assert_eq!(generate_commit_trace(&[]).num_rows(), 4); + assert_eq!(generate_commit_trace(&[op(1, 0, 0x2000, 1)]).num_rows(), 4); + let five: Vec<_> = (0..5).map(|i| op(i, i, 0x2000, 1)).collect(); + assert_eq!(generate_commit_trace(&five).num_rows(), 8); + assert_eq!( + generate_commit_trace(&[op(1, 0, 0x2000, 1)]) + .main_table + .get_row(0) + .len(), + cols::NUM_COLUMNS + ); } -// ========================================================================= -// Constraint formula tests (field arithmetic) -// ========================================================================= - #[test] -fn test_is_bit_constraints() { - // x * (1 - x) = 0 for x in {0, 1} - for x_val in [FE::zero(), FE::one()] { - let result = x_val * (FE::one() - x_val); - assert_eq!(result, FE::zero()); - } - // x=2 should fail - let x = FE::from(2u64); - assert_ne!(x * (FE::one() - x), FE::zero()); -} - -#[test] -fn test_first_or_end_implies_mu() { - // (first + end) * (1 - mu) = 0 - // Valid combos: (0,0,0), (0,0,1), (1,0,1), (0,1,1), (1,1,1) - let valid = [ - (0u64, 0u64, 0u64), - (0, 0, 1), - (1, 0, 1), - (0, 1, 1), - (1, 1, 1), - ]; - for (f, e, m) in valid { - let first = FE::from(f); - let end = FE::from(e); - let mu = FE::from(m); - let result = (first + end) * (FE::one() - mu); - assert_eq!( - result, - FE::zero(), - "Should pass for first={f}, end={e}, mu={m}" - ); - } - - // Invalid: first=1, mu=0 - let result = (FE::one() + FE::zero()) * (FE::one() - FE::zero()); - assert_ne!(result, FE::zero()); - - // Invalid: end=1, mu=0 - let result = (FE::zero() + FE::one()) * (FE::one() - FE::zero()); - assert_ne!(result, FE::zero()); -} - -#[test] -fn test_add_constraint_address() { - // address + 1 = address_incr - // carry_0 = (addr_lo + 1 - incr_lo) * 2^(-32) - let inv_2_32 = FE::from(INV_SHIFT_32); - - // Case 1: no carry. address=0x1000, address+1=0x1001 - let addr_lo = FE::from(0x1000u64); - let incr_lo = FE::from(0x1001u64); - let carry_0 = (addr_lo + FE::one() - incr_lo) * inv_2_32; - assert_eq!(carry_0, FE::zero()); - assert_eq!(carry_0 * (FE::one() - carry_0), FE::zero()); - - // carry_1 = (addr_hi + carry_0 - incr_hi) * 2^(-32) - let carry_1 = (FE::zero() + carry_0 - FE::zero()) * inv_2_32; - assert_eq!(carry_1, FE::zero()); - - // Case 2: carry at 32-bit boundary. address=0x0000_0000_FFFF_FFFF - // address+1 = 0x0000_0001_0000_0000 - // DWordHL halfwords: [0x0000, 0x0000, 0x0001, 0x0000] - // incr_lo = h[0] + 2^16*h[1] = 0 - // incr_hi = h[2] + 2^16*h[3] = 1 - let addr_lo_2 = FE::from(0xFFFF_FFFFu64); - let incr_lo_2 = FE::zero(); - let incr_hi_2 = FE::one(); - let carry_0_2 = (addr_lo_2 + FE::one() - incr_lo_2) * inv_2_32; - assert_eq!(carry_0_2, FE::one()); - let carry_1_2 = (FE::zero() + carry_0_2 - incr_hi_2) * inv_2_32; - assert_eq!(carry_1_2, FE::zero()); -} - -#[test] -fn test_sub_constraint_count() { - // SUB via reversed ADD: count_decr + 1 = count - // carry_0 = (count_decr_lo + 1 - count_lo) * 2^(-32) - let inv_2_32 = FE::from(INV_SHIFT_32); - - // Case 1: count=3, count_decr=2 - let cd_lo = FE::from(2u64); - let count_lo = FE::from(3u64); - let carry_0 = (cd_lo + FE::one() - count_lo) * inv_2_32; - assert_eq!(carry_0, FE::zero()); - - // Case 2: count=0, count_decr=0xFFFF_FFFF_FFFF_FFFF - // count_decr_lo = 0xFFFF + 0xFFFF*2^16 = 0xFFFF_FFFF - let cd_lo_0 = FE::from(0xFFFF_FFFFu64); - let cd_hi_0 = FE::from(0xFFFF_FFFFu64); - // carry_0 = (0xFFFF_FFFF + 1 - 0) * 2^(-32) = 1 - let carry_0_0 = (cd_lo_0 + FE::one() - FE::zero()) * inv_2_32; - assert_eq!(carry_0_0, FE::one()); - // carry_1 = (0xFFFF_FFFF + 1 - 0) * 2^(-32) = 1 - let carry_1_0 = (cd_hi_0 + carry_0_0 - FE::zero()) * inv_2_32; - assert_eq!(carry_1_0, FE::one()); - // Both carries are valid bits - assert_eq!(carry_1_0 * (FE::one() - carry_1_0), FE::zero()); -} - -#[test] -fn test_padding_satisfies_constraints() { - // Padding row: first=0, end=0, mu=0, count=1, address=0, address_incr=[1,0,0,0] - // count_decr=[0,0,0,0] (count=1 -> count-1=0) - let inv_2_32 = FE::from(INV_SHIFT_32); - let one = FE::one(); - let zero = FE::zero(); - - // C0-2: IS_BIT for first=0, end=0, mu=0 - assert_eq!(zero * (one - zero), zero); - - // C3: (first + end) * (1 - mu) = (0+0)*(1-0) = 0 - assert_eq!((zero + zero) * (one - zero), zero); - - // C4-5: address + 1 = address_incr - // addr_lo=0, incr_lo=1 -> carry_0 = (0+1-1)*inv = 0 - let carry_0 = (zero + one - one) * inv_2_32; - assert_eq!(carry_0, zero); - assert_eq!(carry_0 * (one - carry_0), zero); - let carry_1 = (zero + carry_0 - zero) * inv_2_32; - assert_eq!(carry_1, zero); - assert_eq!(carry_1 * (one - carry_1), zero); - - // C6-7: count_decr + 1 = count - // cd_lo=0, count_lo=1 -> carry_0 = (0+1-1)*inv = 0 - let carry_0_sub = (zero + one - one) * inv_2_32; - assert_eq!(carry_0_sub, zero); - assert_eq!(carry_0_sub * (one - carry_0_sub), zero); - let carry_1_sub = (zero + carry_0_sub - zero) * inv_2_32; - assert_eq!(carry_1_sub, zero); - assert_eq!(carry_1_sub * (one - carry_1_sub), zero); +fn a_full_width_timestamp_survives_the_limb_split() { + let trace = generate_commit_trace(&[op(u64::MAX, 0, 0x2000, 1)]); + let r = trace.main_table.get_row(0); + assert_eq!(r[cols::TIMESTAMP_0], FE::from(0xFFFF_FFFFu64)); + assert_eq!(r[cols::TIMESTAMP_1], FE::from(0xFFFF_FFFFu64)); } // ========================================================================= -// Edge case tests +// Constraints // ========================================================================= #[test] -fn test_count_decr_at_zero() { - // count=0 -> count_decr halfwords all 0xFFFF - let ops = vec![op(1, 0, 0, 0, true, true, 0)]; - let trace = generate_commit_trace(&ops); - let r0 = trace.main_table.get_row(0); - - for col in [ - cols::COUNT_DECR_0, - cols::COUNT_DECR_1, - cols::COUNT_DECR_2, - cols::COUNT_DECR_3, - ] { - assert_eq!(r0[col], FE::from(0xFFFFu64)); - } -} - -#[test] -fn test_address_incr_overflow() { - // address = 0xFFFF_FFFF_FFFF_FFFF -> address+1 wraps to 0 - let ops = vec![op(1, 0, u64::MAX, 1, true, false, 0xFF)]; - let trace = generate_commit_trace(&ops); - let r0 = trace.main_table.get_row(0); +fn mu_must_be_a_bit_and_that_is_the_only_constraint() { + let air = busless_air(cols::NUM_COLUMNS, crate::tables::commit::CommitConstraints); - // address = [0xFFFF_FFFF, 0xFFFF_FFFF] - assert_eq!(r0[cols::ADDRESS_0], FE::from(0xFFFF_FFFFu64)); - assert_eq!(r0[cols::ADDRESS_1], FE::from(0xFFFF_FFFFu64)); + let honest = generate_commit_trace(&[op(100, 0, 0x2000, 4)]); + assert!(validate_busless(&air, &honest), "an honest row must pass"); - // address_incr = 0 (all halfwords zero) - assert_eq!(r0[cols::ADDRESS_INCR_0], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_1], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_2], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_3], FE::zero()); - - // Verify ADD constraint holds for the wrapped case - let inv_2_32 = FE::from(INV_SHIFT_32); - let addr_lo = FE::from(0xFFFF_FFFFu64); - let addr_hi = FE::from(0xFFFF_FFFFu64); - // carry_0 = (0xFFFF_FFFF + 1 - 0) * inv = 1 - let carry_0 = (addr_lo + FE::one() - FE::zero()) * inv_2_32; - assert_eq!(carry_0, FE::one()); - // carry_1 = (0xFFFF_FFFF + 1 - 0) * inv = 1 - let carry_1 = (addr_hi + carry_0 - FE::zero()) * inv_2_32; - assert_eq!(carry_1, FE::one()); - assert_eq!(carry_0 * (FE::one() - carry_0), FE::zero()); - assert_eq!(carry_1 * (FE::one() - carry_1), FE::zero()); + let mut forged = honest.clone(); + forged.main_table.set_fe(0, cols::MU, FE::from(2u64)); + assert!( + !validate_busless(&air, &forged), + "mu must be constrained to a bit" + ); } -#[test] -fn test_large_timestamp() { - // Timestamp with both hi and lo words populated - let ts: u64 = 0x0000_0001_0000_0064; // hi=1, lo=100 - let ops = vec![op(ts, 0, 0x5000, 1, true, false, 0xAB)]; - let trace = generate_commit_trace(&ops); - let r0 = trace.main_table.get_row(0); - - assert_eq!(r0[cols::TIMESTAMP_0], FE::from(ts & 0xFFFF_FFFF)); - assert_eq!(r0[cols::TIMESTAMP_1], FE::from(ts >> 32)); -} +// ========================================================================= +// Inventory — these pin the deferral, so a regression shows up here first +// ========================================================================= #[test] -fn test_minimum_table_size() { - // Empty ops -> still 4 rows (minimum) - let trace = generate_commit_trace(&[]); - assert_eq!(trace.num_rows(), 4); - - // All padding rows - for row in 0..4 { - let r = trace.main_table.get_row(row); - assert_eq!(r[cols::MU], FE::zero()); - assert_eq!(r[cols::COUNT_0], FE::one()); +fn test_bus_interactions_count() { + use crate::tables::commit::bus_interactions; + // Ecall receive, CommitDefer send, and four register accesses (x10 read+write, + // x11 read, x12 read, x254 read+write). The eight IsHalfword range checks and the + // Zero end-detection went with the byte loop. + assert_eq!(bus_interactions().len(), 6); + // Every one of them now rides `mu`: with one row per ECALL, `first` was + // identically `mu` and the column is gone. + use stark::lookup::Multiplicity; + for (i, interaction) in bus_interactions().iter().enumerate() { + assert!( + matches!(interaction.multiplicity, Multiplicity::Column(c) if c == cols::MU), + "interaction {i} should ride mu" + ); } } +/// Pins the committed width of the table. +/// +/// **The spec says 7 and this says 8, and both are right** — the spec types +/// `timestamp` as a `Word` (one column) where this code uses a `DWordWL` (two). The +/// high limb is provably zero, so the extra column carries no information. The same +/// `+1` applies to MEMMOVE, where the spec says 37 and the code has 38. See +/// `memmove::shape_tests::the_committed_shape_is_pinned`. #[test] -fn test_address_incr_halfword_carry() { - // address = 0xFFFF -> address+1 = 0x10000 - // Tests carry propagation across halfwords within the low 32-bit word - let ops = vec![op(1, 0, 0xFFFF, 1, true, false, 0)]; - let trace = generate_commit_trace(&ops); - let r0 = trace.main_table.get_row(0); - - // address_incr = 0x10000: h[0]=0x0000, h[1]=0x0001, h[2]=0, h[3]=0 - assert_eq!(r0[cols::ADDRESS_INCR_0], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_1], FE::one()); - assert_eq!(r0[cols::ADDRESS_INCR_2], FE::zero()); - assert_eq!(r0[cols::ADDRESS_INCR_3], FE::zero()); -} - -#[test] -fn test_bus_interactions_count() { - use crate::tables::commit::bus_interactions; - let interactions = bus_interactions(); - assert_eq!(interactions.len(), 18); +fn the_committed_shape_is_pinned() { + assert_eq!(cols::NUM_COLUMNS, 8, "COMMIT columns (spec: 7 + 1)"); } #[test] @@ -437,11 +162,9 @@ fn test_constraints_count_and_indices() { use crate::tables::commit::CommitConstraints; use stark::constraints::builder::ConstraintSet; let meta = CommitConstraints.meta(); - assert_eq!(meta.len(), 8); - // Dense, idx-ordered. + assert_eq!(meta.len(), 1); for (i, m) in meta.iter().enumerate() { assert_eq!(m.constraint_idx, i); } - // All constraints are degree 2 (unconditional). assert_eq!(CommitConstraints.max_degree(), 2); } diff --git a/prover/src/tests/compute_commit_bus_offset_tests.rs b/prover/src/tests/compute_commit_bus_offset_tests.rs index ca6aab272..b58f1e199 100644 --- a/prover/src/tests/compute_commit_bus_offset_tests.rs +++ b/prover/src/tests/compute_commit_bus_offset_tests.rs @@ -6,6 +6,8 @@ use math::field::element::FieldElement; +use executor::vm::instruction::execution::memmove_row_width; + use crate::compute_commit_bus_offset; use crate::tables::types::{BusId, GoldilocksExtension}; @@ -122,3 +124,85 @@ fn test_zero_fingerprint_in_middle_returns_none() { None, ); } + +/// The COMMIT tuples the MEMMOVE chip actually sends, walking the production row +/// schedule one commit ECALL at a time. +/// +/// This is the prover side, not a second copy of the verifier: the widths come from +/// `memmove_row_width`, the same function the trace builder and the sizing pass use, +/// and the tuple shape is the chip's — one `(global index, byte)` pair per byte. +/// `commits` is the per-ECALL split of the public output, which is exactly the thing +/// the verifier never learns. +fn prover_offset( + commits: &[&[u8]], + start_index: u64, + z: &FieldElement, + alpha: &FieldElement, +) -> Option> { + let bus_id = FieldElement::::from(BusId::Commit as u64); + let alpha_sq = alpha * alpha; + let mut total = FieldElement::::zero(); + let mut index = start_index; + + for bytes in commits { + let base = index; + let mut offset = 0u64; + let mut remaining = bytes.len() as u64; + while remaining != 0 { + let width = u64::from(memmove_row_width(0, base, offset, remaining, true)); + for lane in 0..width { + let byte = bytes[(offset + lane) as usize]; + let lc = bus_id + + (FieldElement::::from(base + offset + lane) * alpha) + + (FieldElement::::from(byte as u64) * alpha_sq); + total += (z - lc).inv().ok()?; + } + offset += width; + remaining -= width; + } + index += bytes.len() as u64; + } + + Some(total) +} + +/// The verifier rebuilds the COMMIT bus from the concatenated `public_output` and +/// never learns where one commit ECALL ended and the next began. So the prover's +/// tuples must not depend on that split. +/// +/// This is the regression test for the eight-lane tuple: with one tuple per row, +/// `[&[..4], &[..4]]` sent eight one-byte tuples while the verifier, chunking the +/// eight bytes it sees, expected a single eight-byte one — an honest proof rejected. +#[test] +fn test_prover_tuples_are_independent_of_the_ecall_split() { + let z = FieldElement::::from(9_876_543_211u64); + let alpha = FieldElement::::from(1_357u64); + + let splits: &[&[&[u8]]] = &[ + // One ECALL, sub-eight, exact eight, and a wide body with a tail. + &[&[1, 2, 3, 4]], + &[&[1, 2, 3, 4, 5, 6, 7, 8]], + &[&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]], + // Several ECALLs. Only the last may be a multiple of eight without the + // schedule and the verifier's chunking drifting apart. + &[&[1, 2, 3, 4], &[5, 6, 7, 8]], + &[&[1, 2, 3], &[4, 5, 6, 7, 8, 9, 10, 11, 12]], + &[&[1, 2, 3, 4, 5, 6, 7, 8], &[9], &[10, 11, 12, 13, 14]], + &[&[1], &[2], &[3], &[4], &[5], &[6], &[7], &[8], &[9]], + ]; + + for (case, commits) in splits.iter().enumerate() { + for &start_index in &[0u64, 1, 7, 8, 4_294_967_290] { + let concatenated: Vec = commits.concat(); + let prover = prover_offset(commits, start_index, &z, &alpha) + .expect("no fingerprint collision on the prover side"); + let verifier = compute_commit_bus_offset(&concatenated, start_index, &z, &alpha) + .expect("no fingerprint collision on the verifier side"); + assert_eq!( + prover, verifier, + "case {case} at start_index {start_index}: the COMMIT bus does not \ + balance, so an honest proof would be rejected" + ); + } + } +} diff --git a/prover/src/tests/constraint_program_device_tests.rs b/prover/src/tests/constraint_program_device_tests.rs index a29a7cb49..70a7322fe 100644 --- a/prover/src/tests/constraint_program_device_tests.rs +++ b/prover/src/tests/constraint_program_device_tests.rs @@ -157,6 +157,7 @@ fn all_table_programs_lower_and_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); check_air_device(&create_cpu_air(&opts), "CPU"); + check_air_device(&create_memmove_air(&opts), "memmove"); check_air_device(&create_bitwise_air(&opts), "BITWISE"); check_air_device(&create_lt_air(&opts), "LT"); check_air_device(&create_shift_air(&opts), "SHIFT"); diff --git a/prover/src/tests/constraint_program_tests.rs b/prover/src/tests/constraint_program_tests.rs index e227da53d..67c97eeaa 100644 --- a/prover/src/tests/constraint_program_tests.rs +++ b/prover/src/tests/constraint_program_tests.rs @@ -155,6 +155,7 @@ fn all_table_programs_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); check_air(&create_cpu_air(&opts), "CPU"); + check_air(&create_memmove_air(&opts), "memmove"); check_air(&create_bitwise_air(&opts), "BITWISE"); check_air(&create_lt_air(&opts), "LT"); check_air(&create_shift_air(&opts), "SHIFT"); diff --git a/prover/src/tests/constraint_set_tests_b.rs b/prover/src/tests/constraint_set_tests_b.rs index a7f68ecfd..8adfbd3d4 100644 --- a/prover/src/tests/constraint_set_tests_b.rs +++ b/prover/src/tests/constraint_set_tests_b.rs @@ -241,6 +241,20 @@ mod commit { } } +// ============================================================================= +// dma.rs +// ============================================================================= + +mod memmove { + use super::*; + use crate::tables::memmove::{MemmoveConstraints, cols}; + + #[test] + fn memmove_constraint_set_folder_capture_agree() { + check_table("memmove", &MemmoveConstraints, cols::NUM_COLUMNS); + } +} + // ============================================================================= // keccak.rs // ============================================================================= diff --git a/prover/src/tests/count_table_lengths_drift_tests.rs b/prover/src/tests/count_table_lengths_drift_tests.rs index 7337f0790..10097e6d8 100644 --- a/prover/src/tests/count_table_lengths_drift_tests.rs +++ b/prover/src/tests/count_table_lengths_drift_tests.rs @@ -5,6 +5,7 @@ use crate::tables::trace_builder::{Traces, count_table_lengths}; use crate::test_utils::run_asm_elf; use executor::elf::Elf; use executor::vm::execution::Executor; +use executor::vm::instruction::decoding::Instruction; use executor::vm::logs::Log; fn assert_count_table_lengths_matches(elf: &Elf, logs: &[Log]) { @@ -50,6 +51,10 @@ fn assert_count_table_lengths_matches(elf: &Elf, logs: &[Log]) { predicted.commit_padded_rows, traces.commit.main_table.height as u64, "commit" ); + assert_eq!( + predicted.memmove_padded_rows, traces.memmove.main_table.height as u64, + "memmove" + ); assert_eq!( predicted.decode_rows, traces.decode.main_table.height as u64, "decode" @@ -99,6 +104,51 @@ fn count_table_lengths_matches_traces() { assert_count_table_lengths_matches(&elf, &logs); } +/// Runs one Rust DMA guest and asserts the sizing pass matches the built traces. +/// Each ecall has two hand-maintained replays — `collect_dma_*_ops` for +/// generation and `replay_dma_*_for_sizing` for counting — and they must agree, +/// so the fixtures cover a single chunk plus the multi-chunk / overlapping / +/// near-`MAX_DATA_ROWS` cases of `dma_memcpy_cases`, and the same schedule +/// driven through the memset table. +fn assert_dma_fixture_counts(elf_name: &str, syscall_number: u64) { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join(format!("executor/program_artifacts/rust/{elf_name}"))) + .unwrap_or_else(|_| panic!("{elf_name} not found — build its make target")); + let elf = Elf::load(&elf_bytes).expect("valid DMA guest ELF"); + let result = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("DMA guest execution"); + + assert!( + result.logs.iter().any(|log| { + log.src1_val == syscall_number + && matches!( + result.instructions.get(&log.current_pc), + Some(Instruction::EcallEbreak) + ) + }), + "fixture must contain a DMA ecall" + ); + assert_count_table_lengths_matches(&elf, &result.logs); +} + +#[test] +fn count_table_lengths_matches_nonempty_dma_trace() { + use executor::vm::instruction::execution::{ + DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER, + }; + + assert_dma_fixture_counts("dma_memcpy_min.elf", DMA_MEMCPY_SYSCALL_NUMBER); + assert_dma_fixture_counts("dma_memcpy_cases.elf", DMA_MEMCPY_SYSCALL_NUMBER); + assert_dma_fixture_counts("dma_memset_min.elf", DMA_MEMSET_SYSCALL_NUMBER); + assert_dma_fixture_counts("dma_memset_cases.elf", DMA_MEMSET_SYSCALL_NUMBER); +} + /// The `hint` ecall routes three register reads (`a0`/`a1`/`a2`) and four output /// writes through the memory argument, plus two LT range-checks (selector, in_addr). /// `count_table_lengths` must replay all of that exactly, or `memw_register` (an diff --git a/prover/src/tests/memmove_tests.rs b/prover/src/tests/memmove_tests.rs new file mode 100644 index 000000000..894d42181 --- /dev/null +++ b/prover/src/tests/memmove_tests.rs @@ -0,0 +1,555 @@ +use crate::tables::memmove::{MemmoveOperation, cols, generate_memmove_trace}; +use crate::tables::types::{FE, VmTable}; +use crate::test_utils::{busless_air, validate_busless}; + +/// A memset row. The contract the AIR pins is `dst = src + 8`, so build it that +/// way by default and let the tests below break it deliberately. +fn set_row(count: u64, first: bool, end: bool, src: u64, dst: u64) -> MemmoveOperation { + MemmoveOperation { + width: if count < 8 { 1 } else { 8 }, + functionality: crate::tables::memmove::Functionality::Set, + timestamp: 100, + src, + dst, + count, + first, + end, + // A narrow row must leave lanes 1..7 clear (constraints 25-31), and the + // terminal row copies nothing at all. + value: if end { + [0; 8] + } else if count < 8 { + [0xAB, 0, 0, 0, 0, 0, 0, 0] + } else { + [0xAB; 8] + }, + } +} + +/// A row whose width is chosen independently of `count`, so a test can express +/// `tail != (count < 8)`. `row()` and `set_row()` both derive width from count, which +/// makes `(1 - tail) * lt8` identically zero and constraint 13 impossible to state. +fn row_of_width( + functionality: crate::tables::memmove::Functionality, + count: u64, + width: u8, + first: bool, + end: bool, + src: u64, + dst: u64, +) -> MemmoveOperation { + MemmoveOperation { + width, + functionality, + timestamp: 100, + src, + dst, + count, + first, + end, + value: if width == 1 { + [0xAB, 0, 0, 0, 0, 0, 0, 0] + } else { + [0xAB; 8] + }, + } +} + +fn row(count: u64, first: bool, end: bool, value: [u8; 8]) -> MemmoveOperation { + MemmoveOperation { + width: if count < 8 { 1 } else { 8 }, + functionality: crate::tables::memmove::Functionality::Copy, + timestamp: 100, + src: 0x1000, + dst: 0x2000, + count, + first, + end, + value, + } +} + +#[test] +fn memmove_trace_uses_eight_byte_rows_then_a_byte_tail() { + let trace = generate_memmove_trace(&[ + row(10, true, false, *b"abcdefgh"), + row(2, false, false, [b'i', 0, 0, 0, 0, 0, 0, 0]), + row(1, false, false, [b'j', 0, 0, 0, 0, 0, 0, 0]), + row(0, false, true, [0; 8]), + ]); + + let wide = trace.main_table.get_row(0); + assert_eq!(wide[cols::TAIL], FE::zero()); + assert_eq!(wide[cols::SRC_INCR_0], FE::from(0x1008u64)); + assert_eq!(wide[cols::COUNT_DECR_0], FE::from(2u64)); + for (i, &byte) in b"abcdefgh".iter().enumerate() { + assert_eq!(wide[cols::VALUE[i]], FE::from(byte as u64)); + } + + let tail = trace.main_table.get_row(1); + assert_eq!(tail[cols::TAIL], FE::one()); + assert_eq!(tail[cols::SRC_INCR_0], FE::from(0x1001u64)); + assert_eq!(tail[cols::COUNT_DECR_0], FE::one()); + assert_eq!(tail[cols::VALUE[0]], FE::from(b'i' as u64)); + assert!(cols::VALUE[1..].iter().all(|&c| tail[c] == FE::zero())); + + let terminal = trace.main_table.get_row(3); + assert_eq!(terminal[cols::END], FE::one()); + assert_eq!(terminal[cols::TAIL], FE::one()); + assert_eq!(terminal[cols::COUNT_DECR_0], FE::from(0xFFFFu64)); + assert_eq!(terminal[cols::COUNT_DECR_0 + 1], FE::from(0xFFFFu64)); + assert_eq!(terminal[cols::COUNT_DECR_0 + 2], FE::from(0xFFFFu64)); + assert_eq!(terminal[cols::COUNT_DECR_0 + 3], FE::from(0xFFFFu64)); +} + +#[test] +fn empty_memmove_call_is_a_single_first_and_terminal_row() { + let trace = generate_memmove_trace(&[row(0, true, true, [0; 8])]); + let first = trace.main_table.get_row(0); + assert_eq!(first[cols::FIRST], FE::one()); + assert_eq!(first[cols::END], FE::one()); + assert_eq!(first[cols::MU], FE::one()); +} + +#[test] +fn memmove_constraints_accept_valid_rows_and_reject_nonzero_tail_lanes() { + let mut trace = generate_memmove_trace(&[ + row(2, true, false, [b'a', 0, 0, 0, 0, 0, 0, 0]), + row(1, false, false, [b'b', 0, 0, 0, 0, 0, 0, 0]), + row(0, false, true, [0; 8]), + ]); + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + assert!(validate_busless(&air, &trace)); + + trace.main_table.set(0, cols::VALUE[1], FE::one()); + assert!( + !validate_busless(&air, &trace), + "a one-byte row must not smuggle additional copied lanes" + ); +} + +#[test] +fn memmove_constraints_reject_active_source_or_destination_wrap() { + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + + let source_wrap = generate_memmove_trace(&[MemmoveOperation { + width: 8, + functionality: crate::tables::memmove::Functionality::Copy, + timestamp: 100, + src: u64::MAX - 3, + dst: 0x2000, + count: 8, + first: true, + end: false, + value: [0; 8], + }]); + assert!( + !validate_busless(&air, &source_wrap), + "an active source increment must not wrap modulo 2^64" + ); + + let destination_wrap = generate_memmove_trace(&[MemmoveOperation { + width: 8, + functionality: crate::tables::memmove::Functionality::Copy, + timestamp: 100, + src: 0x1000, + dst: u64::MAX - 3, + count: 8, + first: true, + end: false, + value: [0; 8], + }]); + assert!( + !validate_busless(&air, &destination_wrap), + "an active destination increment must not wrap modulo 2^64" + ); +} + +#[test] +fn memmove_terminal_row_may_wrap_unused_successor_columns() { + let trace = generate_memmove_trace(&[MemmoveOperation { + width: 1, + functionality: crate::tables::memmove::Functionality::Copy, + timestamp: 100, + src: u64::MAX, + dst: u64::MAX, + count: 0, + first: true, + end: true, + value: [0; 8], + }]); + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + assert!( + validate_busless(&air, &trace), + "terminal successors are not consumed and may wrap" + ); +} + +#[test] +fn memmove_bus_interactions_count() { + use crate::tables::memmove::bus_interactions; + // 23 on the DMA table this replaces, plus the CommitDefer receive and eight + // COMMIT-domain sends — one `(index, value)` pair per byte, lane 0 at `mu_com` + // and lanes 1..7 at `mu_com_wide`. One tuple per row would be two sends and + // three fewer aux columns (the count is `ceil(interactions / 2)`), but it would + // make the verifier's rebuild depend on the prover's row schedule, which + // restarts at every commit ECALL while the verifier sees only the concatenated + // `public_output`. Per-byte pairs are what make the two sides agree. + assert_eq!(bus_interactions().len(), 32); +} + +#[test] +fn memmove_constraints_count_and_indices() { + use crate::tables::memmove::MemmoveConstraints; + use stark::constraints::builder::ConstraintSet; + let meta = MemmoveConstraints.meta(); + assert_eq!(meta.len(), 32); + // Dense, idx-ordered. + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i); + } + // All constraints are degree 2 (no over-degree slips in a template change). + assert_eq!(MemmoveConstraints.max_degree(), 2); +} + +#[test] +fn memmove_padding_row_cannot_claim_first_or_end() { + // Constraint 4, `(first + end) * (1 - mu) = 0`, is the sole guard that a + // padding row (mu = 0) cannot masquerade as the first or terminal row of a + // copy — bitness alone accepts first = 1 or end = 1, so nothing else rejects + // it. A padding row claiming `first` would forge an ECALL receive; claiming + // `end` would forge a copy's terminal row. + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + let base = generate_memmove_trace(&[ + row(2, true, false, [b'a', 0, 0, 0, 0, 0, 0, 0]), + row(1, false, false, [b'b', 0, 0, 0, 0, 0, 0, 0]), + row(0, false, true, [0; 8]), + ]); + // Row 3 is padding: mu = 0, first = end = 0, and the trace validates. + assert_eq!(base.main_table.get_row(3)[cols::MU], FE::zero()); + assert!(validate_busless(&air, &base)); + + let mut forge_first = base.clone(); + forge_first.main_table.set(3, cols::FIRST, FE::one()); + assert!( + !validate_busless(&air, &forge_first), + "a padding row (mu = 0) must not claim to be a copy's first row" + ); + + let mut forge_end = base; + forge_end.main_table.set(3, cols::END, FE::one()); + assert!( + !validate_busless(&air, &forge_end), + "a padding row (mu = 0) must not claim to be a copy's terminal row" + ); +} + +/// Pins the shape of the COMMIT-domain sends: eight `(index, value)` pairs, one per +/// byte, indexed off `dst` — not one eight-lane tuple per row. +/// +/// The verifier rebuilds this bus from `public_output` alone and never learns where +/// one commit ECALL ended and the next began. Per-byte pairs are what make its +/// rebuild independent of the prover's row schedule, which restarts at every ECALL; +/// `test_prover_tuples_are_independent_of_the_ecall_split` is the arithmetic half of +/// the same argument, and this is the half that keeps it anchored to the real chip. +#[test] +fn memmove_commit_sends_one_pair_per_byte() { + use crate::tables::memmove::bus_interactions; + use crate::tables::types::BusId; + use stark::lookup::{BusValue, LinearTerm, Multiplicity, Packing}; + + let commit_sends: Vec<_> = bus_interactions() + .into_iter() + .filter(|interaction| interaction.bus_id == BusId::Commit as u64) + .collect(); + + assert_eq!(commit_sends.len(), 8, "one COMMIT send per byte lane"); + + for (lane, interaction) in commit_sends.iter().enumerate() { + assert!(interaction.is_sender, "lane {lane} must send"); + + // Lane 0 rides every copying row; lanes 1..7 only an eight-byte row, so a + // one-byte row sends no spurious `(index, 0)` pairs. + let expected_multiplicity = if lane == 0 { + cols::MU_COM + } else { + cols::MU_COM_WIDE + }; + assert!( + matches!(interaction.multiplicity, Multiplicity::Column(column) if column == expected_multiplicity), + "lane {lane} has the wrong multiplicity" + ); + + assert_eq!( + interaction.values.len(), + 2, + "lane {lane} is an (index, value) pair" + ); + + match &interaction.values[0] { + BusValue::Linear(terms) => { + assert!( + matches!( + terms.as_slice(), + [ + LinearTerm::Column { coefficient: 1, column }, + LinearTerm::Constant(offset), + ] if *column == cols::DST_0 && *offset == lane as i64 + ), + "lane {lane} must be indexed at dst + {lane}" + ); + } + _ => panic!("lane {lane}'s index must be a linear combination"), + } + + assert!( + matches!( + interaction.values[1], + BusValue::Packed { + start_column, + packing: Packing::Direct, + } if start_column == cols::VALUE[lane] + ), + "lane {lane} must carry value[{lane}]" + ); + } +} + +/// The memset operand contract, in the direction that matters. +/// +/// `dst == src` is the degenerate case: read and write address the same cell at +/// adjacent timestamps, so the memory argument closes on `value == value` and every +/// value lane becomes a free field element. Constraints 32 and 33 are the only thing +/// that rejects it, so both are tested here in both directions, and a `Copy` row is +/// tested to confirm the gate is `is_set` and does not leak onto the copy path (a +/// memcpy with `dst == src` is harmless -- its read is at `T+1` and pins `value` to +/// live memory). +#[test] +fn memmove_constraints_pin_the_memset_gap() { + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + + // Honest: dst = src + 8, on a wide row, a narrow row and the terminal row. + let honest = generate_memmove_trace(&[ + set_row(16, true, false, 0x1000, 0x1008), + set_row(8, false, false, 0x1008, 0x1010), + set_row(0, false, true, 0x1010, 0x1018), + ]); + assert!( + validate_busless(&air, &honest), + "a memset chain with dst = src + 8 must be accepted" + ); + + // The forgery: dst == src leaves `value` unconstrained. + let degenerate = generate_memmove_trace(&[ + set_row(16, true, false, 0x1000, 0x1000), + set_row(0, false, true, 0x1000, 0x1000), + ]); + assert!( + !validate_busless(&air, °enerate), + "dst == src must be rejected: it makes every value lane a free field element" + ); + + // Wrong gap, and the wrong direction, are both out of contract too. + for (src, dst, why) in [ + (0x1000u64, 0x1004u64, "a gap under one row width"), + (0x1000, 0x1020, "a gap over one row width"), + ( + 0x1008, + 0x1000, + "dst below src, which propagates the wrong way", + ), + ] { + let trace = generate_memmove_trace(&[ + set_row(16, true, false, src, dst), + set_row(0, false, true, src, dst), + ]); + assert!( + !validate_busless(&air, &trace), + "{why} must be rejected (src {src:#x}, dst {dst:#x})" + ); + } + + // The high limb is pinned as well, so the gap cannot be forged across limbs. + let straddle = generate_memmove_trace(&[ + set_row(16, true, false, 0x1000, 0x1_0000_1008), + set_row(0, false, true, 0x1000, 0x1_0000_1008), + ]); + assert!( + !validate_busless(&air, &straddle), + "a gap of 8 in the low limb but not the high one must be rejected" + ); + + // And the gate really is `is_set`. This has to be a genuinely aliased copy — + // `row()` hardcodes src 0x1000 / dst 0x2000, so using it here would only show + // that the constraints tolerate a gap of 0x1000, not that they are off for Copy. + // A memcpy with dst == src is harmless: its read is at T+1 and pins `value` to + // live memory, which is exactly why the pin is gated on `is_set`. + let copy = crate::tables::memmove::Functionality::Copy; + let copy_aliased = generate_memmove_trace(&[ + row_of_width(copy, 8, 8, true, false, 0x1000, 0x1000), + row_of_width(copy, 0, 1, false, true, 0x1008, 0x1008), + ]); + assert!( + validate_busless(&air, ©_aliased), + "constraints 30-31 must not fire on Copy rows, even with dst == src" + ); +} + +/// Constraint 13, `(1 - tail) * lt8 = 0`, in both directions. +/// +/// This is the constraint that replaced the old DMA table's hard pin of +/// `tail = (count < 8)`. Erik asked for exactly this relaxation so the prover may +/// take one-byte rows at any count and reach the aligned `MEMW_A` path, so both +/// directions matter: the narrow-at-high-count row must be ACCEPTED (it is the +/// alignment prologue `memmove_row_width` emits), and the wide-at-low-count row must +/// be REJECTED (it would move eight bytes where fewer were authorised). +/// +/// Neither case is expressible through `row()` or `set_row()`, which derive width +/// from count and so can only ever produce `tail == lt8`. +#[test] +fn memmove_constraint_14_frees_narrow_rows_but_not_wide_ones() { + use crate::tables::memmove::Functionality::Copy; + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + + // ACCEPTED: a one-byte row with eight bytes still to go — the prologue that + // walks `dst` up to eight-byte alignment. `tail = 1`, `lt8 = 0`. + let prologue = generate_memmove_trace(&[ + row_of_width(Copy, 16, 1, true, false, 0x1001, 0x2001), + row_of_width(Copy, 15, 1, false, false, 0x1002, 0x2002), + row_of_width(Copy, 14, 8, false, false, 0x1003, 0x2003), + row_of_width(Copy, 6, 1, false, false, 0x100B, 0x200B), + row_of_width(Copy, 0, 1, false, true, 0x100C, 0x200C), + ]); + assert!( + validate_busless(&air, &prologue), + "a one-byte row at count >= 8 is legal: it is the alignment prologue" + ); + + // REJECTED: an eight-byte row with fewer than eight bytes left. `tail = 0`, + // `lt8 = 1`, so `(1 - tail) * lt8 = 1`. + let overrun = generate_memmove_trace(&[ + row_of_width(Copy, 7, 8, true, false, 0x1000, 0x2000), + row_of_width(Copy, 0, 1, false, true, 0x1008, 0x2008), + ]); + assert!( + !validate_busless(&air, &overrun), + "an eight-byte row must be illegal when only seven bytes remain" + ); +} + +/// The `Commit` functionality at constraint level, which had no negative coverage +/// at all: memcpy rows have four forgery tests and memset five, commit none. The +/// `prove_elfs` forgery helper excludes it by construction (`is_copy = !IS_SET && +/// !IS_COMMIT`), so the accepting direction is exercised end to end but nothing ever +/// tried to break the COMMIT-domain gating. +/// +/// Covers constraints 12 (one-hot), 13 (no selector on a padding row) and 18 +/// (`mu_com_wide = mu_com * (1 - tail)`), the last being the structural successor of +/// the deleted DMA_SET `FILL_WIDE`, which had four negative tests and lost them all. +#[test] +fn memmove_constraints_gate_the_commit_functionality() { + use crate::tables::memmove::Functionality::{Commit, Copy}; + let air = busless_air( + cols::NUM_COLUMNS, + crate::tables::memmove::MemmoveConstraints, + ); + + // Baseline: an honest commit chain. `dst` is the global byte index, so it starts + // at 0 and the gap pin must not fire here — commit is not `is_set`. + let honest = generate_memmove_trace(&[ + row_of_width(Commit, 12, 8, true, false, 0x3000, 0), + row_of_width(Commit, 4, 1, false, false, 0x3008, 8), + row_of_width(Commit, 3, 1, false, false, 0x3009, 9), + row_of_width(Commit, 2, 1, false, false, 0x300A, 10), + row_of_width(Commit, 1, 1, false, false, 0x300B, 11), + row_of_width(Commit, 0, 1, false, true, 0x300C, 12), + ]); + assert!( + validate_busless(&air, &honest), + "an honest commit chain must be accepted" + ); + + // Constraint 11: a row cannot claim two functionalities. Setting `is_set` on a + // commit row would buy the inverted timestamp order on a chain the COMMIT chip + // authorised. + // + // This case has to be built on a chain whose addresses already satisfy the memset + // gap pin (constraints 30-31), or those reject it first and the assertion passes + // for the wrong reason — verified by mutation: neutering 12 alone left an earlier + // version of this test green. + let gap_clean = generate_memmove_trace(&[ + row_of_width(Commit, 8, 8, true, false, 0x3000, 0x3008), + row_of_width(Commit, 0, 1, false, true, 0x3008, 0x3010), + ]); + assert!( + validate_busless(&air, &gap_clean), + "the gap-clean commit baseline must itself be accepted" + ); + let mut one_hot = gap_clean.clone(); + one_hot.main_table.set_fe(0, cols::IS_SET, FE::one()); + assert!( + !validate_busless(&air, &one_hot), + "is_set and is_commit must not both be set (constraint 11)" + ); + + // Constraint 16: widen a one-byte commit row. `mu_com_wide` is what stops it + // broadcasting seven spurious `(index, 0)` pairs onto the COMMIT bus, which the + // verifier rebuilds from `public_output` — so a forgery here corrupts the output + // fingerprint rather than merely wasting a row. + let mut trace = honest.clone(); + trace.main_table.set_fe(1, cols::MU_COM_WIDE, FE::one()); + assert!( + !validate_busless(&air, &trace), + "a one-byte commit row must not claim the wide lanes (constraint 16)" + ); + + // Constraint 12: no selector on a padding row. The chain above is six rows, so + // the trace pads to eight and row 7 is padding with mu = 0. + let mut trace = honest.clone(); + assert_eq!( + trace.main_table.get_row(7)[cols::MU], + FE::zero(), + "row 7 is expected to be padding" + ); + trace.main_table.set_fe(7, cols::IS_COMMIT, FE::one()); + assert!( + !validate_busless(&air, &trace), + "a padding row must not carry a functionality selector (constraint 12)" + ); + + // There is deliberately no "commit row also claims the RAM write" case here any + // more. That forgery needed `mu_ram` to be a witness column; the RAM write now + // rides the linear multiplicity `mu - end - mu_com` directly, so a commit row + // (`mu_com = mu - end`) drives it to zero by construction and there is nothing + // left to forge. + + // Control: the same forgeries on a Copy chain are a different matter — this only + // establishes that the honest Copy baseline is clean, so the failures above are + // attributable to the commit gating rather than to the row shapes. + let copy_ok = generate_memmove_trace(&[ + row_of_width(Copy, 8, 8, true, false, 0x1000, 0x2000), + row_of_width(Copy, 0, 1, false, true, 0x1008, 0x2008), + ]); + assert!( + validate_busless(&air, ©_ok), + "Copy baseline must be clean" + ); +} diff --git a/prover/src/tests/mod.rs b/prover/src/tests/mod.rs index 9288cf2ac..84eb2382c 100644 --- a/prover/src/tests/mod.rs +++ b/prover/src/tests/mod.rs @@ -61,6 +61,8 @@ pub mod lt_bus_tests; #[cfg(test)] pub mod lt_tests; #[cfg(test)] +pub mod memmove_tests; +#[cfg(test)] pub mod memw_aligned_tests; #[cfg(test)] pub mod memw_register_tests; diff --git a/prover/src/tests/ood_window_ir_tests.rs b/prover/src/tests/ood_window_ir_tests.rs index 29d224627..9fd960f7f 100644 --- a/prover/src/tests/ood_window_ir_tests.rs +++ b/prover/src/tests/ood_window_ir_tests.rs @@ -90,6 +90,7 @@ fn all_table_windows_match_captured_ir() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); assert_ood_window_matches_ir(&create_cpu_air(&opts), true, "CPU"); + assert_ood_window_matches_ir(&create_memmove_air(&opts), true, "memmove"); assert_ood_window_matches_ir(&create_bitwise_air(&opts), true, "BITWISE"); assert_ood_window_matches_ir(&create_lt_air(&opts), true, "LT"); assert_ood_window_matches_ir(&create_shift_air(&opts), true, "SHIFT"); diff --git a/prover/src/tests/prove_elfs_tests.rs b/prover/src/tests/prove_elfs_tests.rs index e45c7b927..dcfef65d5 100644 --- a/prover/src/tests/prove_elfs_tests.rs +++ b/prover/src/tests/prove_elfs_tests.rs @@ -1212,6 +1212,370 @@ fn test_prove_ecsm_rust_guest() { ); } +#[test] +fn test_prove_dma_memcpy_rust_guest() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memcpy_min.elf")) + .expect("dma_memcpy_min.elf not found — build its make target"); + + let proof = prove_vm_minimal(&elf_bytes, &[], &Default::default()); + assert!( + verify_vm_minimal(&proof, &elf_bytes), + "DMA memcpy guest should verify" + ); + assert_eq!( + proof.public_output, + b"DMA copies eight-byte rows and a short tail" + ); +} + +/// Positive control for the fixture the memset forgery tests tamper with. Those +/// tests assert that verification FAILS, so without this they would also pass if +/// the untampered trace never verified in the first place. +#[test] +fn test_prove_dma_memset_min_rust_guest() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memset_min.elf")) + .expect("dma_memset_min.elf not found — build its make target"); + + let proof = prove_vm_minimal(&elf_bytes, &[], &Default::default()); + assert!( + verify_vm_minimal(&proof, &elf_bytes), + "DMA memset guest should verify" + ); + assert_eq!(proof.public_output, [0x3Cu8; 43]); +} + +/// End-to-end memset: the guest exercises every row-schedule boundary (empty, +/// sub-tail, exact widths, the per-ecall cap, multi-chunk, a wide fill truncated +/// to its low byte, and an unaligned page-crossing destination), so a passing +/// proof covers the memset rows, their bus balance, and the operand-gap pin +/// together. +#[test] +fn test_prove_dma_memset_cases_rust_guest() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memset_cases.elf")) + .expect("dma_memset_cases.elf not found — build its make target"); + + let proof = prove_vm_minimal(&elf_bytes, &[], &Default::default()); + assert!( + verify_vm_minimal(&proof, &elf_bytes), + "DMA memset guest should verify" + ); + assert_eq!(proof.public_output, b"dma-memset-ok"); +} + +/// memmove rides the memcpy ecall unchanged. The interesting case is a forward +/// overlap longer than one 256-byte chunk: the stub must walk chunks backwards, +/// or an earlier chunk clobbers source bytes a later one still needs. +#[test] +fn test_prove_dma_memmove_cases_rust_guest() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memmove_cases.elf")) + .expect("dma_memmove_cases.elf not found — build its make target"); + + let proof = prove_vm_minimal(&elf_bytes, &[], &Default::default()); + assert!( + verify_vm_minimal(&proof, &elf_bytes), + "DMA memmove guest should verify" + ); + assert_eq!(proof.public_output, b"dma-memmove-ok"); +} + +#[test] +fn test_prove_dma_memcpy_cases_rust_guest() { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memcpy_cases.elf")) + .expect("dma_memcpy_cases.elf not found — build its make target"); + + let proof = prove_vm_minimal(&elf_bytes, &[], &Default::default()); + assert!( + verify_vm_minimal(&proof, &elf_bytes), + "DMA differential cases guest should verify" + ); + assert_eq!(proof.public_output, b"dma-cases-ok"); +} + +#[test] +fn test_prove_dma_memcpy_forged_value_rejected() { + use crate::tables::memmove::cols as dma_cols; + + let (elf, mut traces) = dma_memcpy_fixture(); + let forged_row = dma_row_matching(&traces, |_, end, _tail| !end); + let original = *traces + .memmove + .main_table + .get(forged_row, dma_cols::VALUE[0]); + traces.memmove.main_table.set( + forged_row, + dma_cols::VALUE[0], + original + FieldElement::::one(), + ); + + assert_dma_forgery_rejected( + &elf, + &mut traces, + "changing the structurally shared copied byte must unbalance MEMW", + ); +} + +#[test] +fn test_prove_dma_memcpy_forged_intermediate_source_rejected() { + use crate::tables::memmove::cols as dma_cols; + + let (elf, mut traces) = dma_memcpy_fixture(); + let forged_row = dma_row_matching(&traces, |first, end, _tail| !first && !end); + + // Shift both the current source and its locally-consistent successor. The + // row's ADD remains valid, but the predecessor's MemmoveNext tuple and the + // source-memory read no longer match. + let src_lo = *traces.memmove.main_table.get(forged_row, dma_cols::SRC_0); + let src_incr_lo = *traces + .memmove + .main_table + .get(forged_row, dma_cols::SRC_INCR_0); + traces.memmove.main_table.set( + forged_row, + dma_cols::SRC_0, + src_lo + FieldElement::from(8u64), + ); + traces.memmove.main_table.set( + forged_row, + dma_cols::SRC_INCR_0, + src_incr_lo + FieldElement::from(8u64), + ); + + assert_dma_forgery_rejected( + &elf, + &mut traces, + "an intermediate source row must remain chained to its predecessor", + ); +} + +#[test] +fn test_prove_dma_memcpy_forged_early_end_rejected() { + use crate::tables::memmove::cols as dma_cols; + + let (elf, mut traces) = dma_memcpy_fixture(); + let forged_row = dma_row_matching(&traces, |_first, end, _tail| !end); + traces + .memmove + .main_table + .set(forged_row, dma_cols::END, FieldElement::one()); + + assert_dma_forgery_rejected(&elf, &mut traces, "END must be equivalent to count == 0"); +} + +#[test] +fn test_prove_dma_memcpy_forged_wide_tail_rejected() { + use crate::tables::memmove::cols as dma_cols; + + let (elf, mut traces) = dma_memcpy_fixture(); + let forged_row = dma_row_matching(&traces, |_first, end, tail| !end && !tail); + traces + .memmove + .main_table + .set(forged_row, dma_cols::TAIL, FieldElement::one()); + + assert_dma_forgery_rejected(&elf, &mut traces, "a row's width must match its step"); +} + +#[test] +fn test_prove_dma_memset_forged_intermediate_fill_rejected() { + use crate::tables::memmove::cols as dma_set_cols; + + let (elf, mut traces) = dma_memset_fixture(); + // `!tail` matters: on a one-byte row `fill_wide` must stay zero, so shifting + // both lanes there would trip constraint 9 locally and the test would prove + // something else. + let forged_row = dma_set_row_matching(&traces, |first, end, tail| !first && !end && !tail); + // Shift both lanes so the row stays internally consistent (constraint 10 + // still holds); only the chain token and the MEMW write disagree. + for column in [dma_set_cols::VALUE[0], dma_set_cols::VALUE[1]] { + let original = *traces.memmove.main_table.get(forged_row, column); + traces.memmove.main_table.set( + forged_row, + column, + original + FieldElement::::one(), + ); + } + + assert_dma_forgery_rejected( + &elf, + &mut traces, + "an intermediate row must keep the fill byte its predecessor sent", + ); +} + +#[test] +fn test_prove_dma_memset_forged_early_end_rejected() { + use crate::tables::memmove::cols as dma_set_cols; + + let (elf, mut traces) = dma_memset_fixture(); + let forged_row = dma_set_row_matching(&traces, |_first, end, _tail| !end); + traces + .memmove + .main_table + .set(forged_row, dma_set_cols::END, FieldElement::one()); + + assert_dma_forgery_rejected(&elf, &mut traces, "END must be equivalent to count == 0"); +} + +/// Flipping `tail` rewrites `step` from 8 to 1, so the row's own address and +/// count arithmetic stop holding. Note this is rejected locally by the ADD +/// carries, NOT by the ALU LT that pins `tail = (count < 8)` — that bus has no +/// negative coverage here, the same gap the memcpy sibling has. +#[test] +fn test_prove_dma_memset_forged_wide_tail_rejected() { + use crate::tables::memmove::cols as dma_set_cols; + + let (elf, mut traces) = dma_memset_fixture(); + let forged_row = dma_set_row_matching(&traces, |_first, end, tail| !end && !tail); + traces + .memmove + .main_table + .set(forged_row, dma_set_cols::TAIL, FieldElement::one()); + + assert_dma_forgery_rejected(&elf, &mut traces, "a row's width must match its step"); +} + +/// Soundness: the timestamp order is what makes a memset a memset. Clearing `IS_SET` +/// on a chain turns the row back into an ordinary snapshot copy, which reads at `T+1` +/// instead of `T+2` — so the read no longer observes the previous row's write and the +/// MEMW tuples stop matching the memory the executor produced. This is the one piece of +/// the unified chip with no ancestor in either of the tables it replaces. +#[test] +fn test_prove_dma_memset_forged_order_bit_rejected() { + use crate::tables::memmove::cols as mm_cols; + + let (elf, mut traces) = dma_memset_fixture(); + let forged_row = dma_set_row_matching(&traces, |_first, end, _tail| !end); + traces + .memmove + .main_table + .set(forged_row, mm_cols::IS_SET, FieldElement::zero()); + + assert_dma_forgery_rejected( + &elf, + &mut traces, + "clearing the order bit must break the propagation the fill depends on", + ); +} + +#[test] +fn test_prove_dma_memset_forged_intermediate_destination_rejected() { + use crate::tables::memmove::cols as dma_set_cols; + + let (elf, mut traces) = dma_memset_fixture(); + let forged_row = dma_set_row_matching(&traces, |first, end, tail| !first && !end && !tail); + + // Shift both the current destination and its locally-consistent successor. + // The row's ADD stays valid; the predecessor's MemmoveNext tuple and the + // memory write no longer match. + for column in [dma_set_cols::DST_0, dma_set_cols::DST_INCR_0] { + let original = *traces.memmove.main_table.get(forged_row, column); + traces + .memmove + .main_table + .set(forged_row, column, original + FieldElement::from(8u64)); + } + + assert_dma_forgery_rejected( + &elf, + &mut traces, + "an intermediate row must stay chained to its predecessor's address", + ); +} + +fn dma_memset_fixture() -> (Elf, Traces) { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memset_min.elf")) + .expect("dma_memset_min.elf not found — build its make target"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let result = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("execution"); + let traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + (elf, traces) +} + +fn dma_set_row_matching(traces: &Traces, predicate: impl Fn(bool, bool, bool) -> bool) -> usize { + use crate::tables::memmove::cols as mm_cols; + + let one = FieldElement::::one(); + (0..traces.memmove.num_rows()) + .find(|&row| { + let get = |column| *traces.memmove.main_table.get(row, column) == one; + get(mm_cols::MU) + && get(mm_cols::IS_SET) + && predicate(get(mm_cols::FIRST), get(mm_cols::END), get(mm_cols::TAIL)) + }) + .expect("guest must contain the requested real memset row") +} + +fn dma_memcpy_fixture() -> (Elf, Traces) { + let workspace_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") + .to_path_buf(); + let elf_bytes = + std::fs::read(workspace_root.join("executor/program_artifacts/rust/dma_memcpy_min.elf")) + .expect("dma_memcpy_min.elf not found — build its make target"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let result = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("execution"); + let traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + (elf, traces) +} + +fn dma_row_matching(traces: &Traces, predicate: impl Fn(bool, bool, bool) -> bool) -> usize { + use crate::tables::memmove::cols as mm_cols; + + let one = FieldElement::::one(); + (0..traces.memmove.num_rows()) + .find(|&row| { + let get = |column| *traces.memmove.main_table.get(row, column) == one; + // memcpy rows only: memset and commit have their own forgery surface. + let is_copy = !get(mm_cols::IS_SET) && !get(mm_cols::IS_COMMIT); + get(mm_cols::MU) + && is_copy + && predicate(get(mm_cols::FIRST), get(mm_cols::END), get(mm_cols::TAIL)) + }) + .expect("guest must contain the requested real MEMMOVE copy row") +} + +fn assert_dma_forgery_rejected(elf: &Elf, traces: &mut Traces, reason: &str) { + assert!(!prove_and_verify_vm_minimal(elf, traces), "{reason}"); +} /// End-to-end prove→verify for the non-constraining `Hint` ecall: the minimal Rust /// guest does one `hint` call (secp256k1 base-field inverse of 3) and commits the result. /// This exercises the whole HINT table bus surface (Ecall receive, the x10/x11/x12 @@ -3052,7 +3416,13 @@ fn test_prove_ef_io_demo_concatenates() { let elf_bytes = std::fs::read(workspace_root.join("executor/program_artifacts/rust/ef_io_demo.elf")) .expect("ef_io_demo.elf not found — run `make compile-programs-rust`"); - let input: &[u8] = b"hello world!"; + // 25 bytes, so `ef_io_demo`'s `buf_size / 2` split gives commits of 12 and 13. + // Both exceed eight, so each ecall emits a WIDE commit row, and the second is + // based at global index 12 -- not 8-aligned. That is the configuration where a + // prover-side row schedule and the verifier's `public_output` rebuild would drift + // apart if the COMMIT bus were grouped per row rather than per byte. The old + // input, `b"hello world!"`, split 6 + 6 and so produced only one-byte commit rows. + let input: &[u8] = b"hello world, and hello ef"; let proof = crate::prove_with_inputs(&elf_bytes, input).expect("prove should succeed"); assert!( crate::verify(&proof, &elf_bytes).expect("verify should not error"), diff --git a/prover/tests/gpu_constraint_interp_real.rs b/prover/tests/gpu_constraint_interp_real.rs index 4446fb446..a14232dd6 100644 --- a/prover/tests/gpu_constraint_interp_real.rs +++ b/prover/tests/gpu_constraint_interp_real.rs @@ -271,5 +271,6 @@ fn all_table_programs_gpu_match_cpu_oracle() { check_air(&create_keccak_rc_air(&opts), "KECCAK_RC"); check_air(&create_ecsm_air(&opts), "ECSM"); check_air(&create_ecdas_air(&opts), "ECDAS"); + check_air(&create_memmove_air(&opts), "memmove"); check_air(&create_hint_air(&opts), "HINT"); } diff --git a/syscalls/src/allocator.rs b/syscalls/src/allocator.rs index 920d7e1c3..c22afac8b 100644 --- a/syscalls/src/allocator.rs +++ b/syscalls/src/allocator.rs @@ -1,6 +1,17 @@ use riscv as _; const MAX_MEMORY_SIZE: usize = 0xC000_0000; + +/// The accelerated `memset` depends on this. +/// +/// MEMMOVE pins `dst = src + 8` limb-wise on every `is_set` row, so a row whose low +/// 32-bit limb carries has no representable successor and the executor refuses the +/// call. Keeping every non-stack address below 2^32 is half of why no well-formed +/// object can produce such a row; the other half is that a stack object satisfies +/// `buf + n <= STACK_TOP`, whose low limb is `0xFFFF_FFF0`, leaving 15 bytes of +/// headroom for the 8-byte gap. Raise this ceiling past 2^32 and accelerated `memset` +/// starts failing on heap buffers. +const _: () = assert!(MAX_MEMORY_SIZE < 1 << 32); const WORD_SIZE: usize = 4; // Guest global allocator, selectable at build time. The default was chosen on measured A/Bs diff --git a/syscalls/src/entrypoint.rs b/syscalls/src/entrypoint.rs index 2e4f89a3b..487e72393 100644 --- a/syscalls/src/entrypoint.rs +++ b/syscalls/src/entrypoint.rs @@ -1,4 +1,11 @@ -use crate::{allocator::init_allocator, syscalls::sys_halt}; +use core::arch::global_asm; + +use crate::{ + allocator::init_allocator, + syscalls::{ + DMA_MEMCPY_MAX_BYTES, DMA_MEMCPY_SYSCALL_NUMBER, DMA_MEMSET_SYSCALL_NUMBER, sys_halt, + }, +}; /// # Safety /// @@ -14,3 +21,223 @@ pub unsafe extern "C" fn _start() -> ! { sys_halt(); } } + +// --------------------------------------------------------------------------- +// DMA memcpy symbol override +// +// `memcpy` is defined next to `_start` on purpose, and not in `syscalls.rs`. +// `compiler_builtins` defines `memcpy` weakly, and a linker extracts an archive +// member only to satisfy an undefined symbol — a weak definition already +// satisfies it, so a strong definition sitting in a member nothing else pulls in +// is silently dropped, with no duplicate-symbol diagnostic. The object defining +// `_start` is always extracted, so co-locating the symbol makes it win +// resolution without `--whole-archive` or any guest link flag. This is the +// "always-linked runtime" mechanism the accelerated-memory-operations standard +// requires vendors to pick and document; see `docs/general_flow.md`. +// +// This placement is insurance, not a repair for an observed failure: in +// `syscalls.rs` the symbol also won resolution, and not by luck — `_start` calls +// `sys_halt` from that module and it is not `#[inline]`, so every guest carries +// an undefined reference that forces the object out of the archive, whatever the +// guest itself names. What the move buys is not depending on that: neither on +// `_start` continuing to call into `syscalls.rs`, nor on rustc's codegen-unit +// merging keeping the two modules together. Only same-module items are +// guaranteed to share an object (partitioning places them together and merging +// never splits), so co-locating with `_start` — the one symbol the linker is +// obliged to resolve — makes the guarantee local. +// `test_dma_memcpy_compiler_emitted_copies` is what detects a regression: a guest +// that falls back still produces correct output, only its ecall count drops. +// +// A Rust `#[no_mangle] fn memcpy` did not reliably override compiler-builtins in +// optimized guests: the final ELF still jumped to compiler_builtins' +// implementation. LLVM still inlines statically-sized tiny copies. Remaining +// out-of-line copies are split into bounded DMA ecalls so a single guest +// instruction cannot create an unbounded continuation trace. +// +// `.p2align 2` is load-bearing: a bare `.section` gives sh_addralign = 1, so the +// linker is free to place `memcpy` at an address that is not a multiple of 4 and +// the VM, which fetches one 4-byte instruction per pc, could not decode it. +// --------------------------------------------------------------------------- + +global_asm!( + r#" + .section .text.memcpy,"ax",@progbits + .p2align 2 + .globl memcpy + .type memcpy,@function +memcpy: + mv t0, a0 + mv t1, a2 + beqz t1, .Ldma_memcpy_done +.Ldma_memcpy_loop: + li a2, {max_bytes} + bgeu t1, a2, .Ldma_memcpy_call + mv a2, t1 +.Ldma_memcpy_call: + li a7, {syscall} + ecall + sub t1, t1, a2 + add a0, a0, a2 + add a1, a1, a2 + bnez t1, .Ldma_memcpy_loop +.Ldma_memcpy_done: + mv a0, t0 + ret + .size memcpy, .-memcpy +"#, + syscall = const DMA_MEMCPY_SYSCALL_NUMBER, + max_bytes = const DMA_MEMCPY_MAX_BYTES, +); + +// --------------------------------------------------------------------------- +// DMA memmove symbol override +// +// Here rather than in `syscalls.rs` for the same reason as `memcpy` above: +// `compiler_builtins` defines `memmove` weakly too, so the strong definition has +// to sit in the object the linker is obliged to extract. +// +// Reuses the memcpy ecall unchanged — no new table, no new syscall. Each ecall +// already snapshots its whole source range before writing (all reads at T+1, +// all writes at T+2), so a single chunk has memmove semantics for free. +// +// Chunking is what breaks it: copying [0,256) -> [4,260) clobbers source bytes +// that a later forward chunk still needs. So when the destination starts inside +// the source range (src < dst < src+n) the chunks are walked from the END +// backwards; every chunk then reads bytes no earlier chunk has written yet. +// Otherwise (disjoint, or dst below src) forward chunking is already safe. +// --------------------------------------------------------------------------- + +global_asm!( + r#" + .section .text.memmove,"ax",@progbits + .p2align 2 + .globl memmove + .type memmove,@function +memmove: + mv t0, a0 + beqz a2, .Ldma_memmove_done + bgeu a1, a0, .Ldma_memmove_fwd // src >= dst: forward is safe + add t2, a1, a2 + bgeu a0, t2, .Ldma_memmove_fwd // dst >= src+n: disjoint + // Overlapping with dst inside [src, src+n): walk chunks from the end. + add a0, a0, a2 + add a1, a1, a2 + mv t1, a2 +.Ldma_memmove_back_loop: + li a2, {max_bytes} + bgeu t1, a2, .Ldma_memmove_back_call + mv a2, t1 +.Ldma_memmove_back_call: + sub a0, a0, a2 + sub a1, a1, a2 + li a7, {syscall} + ecall + sub t1, t1, a2 + bnez t1, .Ldma_memmove_back_loop + j .Ldma_memmove_done +.Ldma_memmove_fwd: + mv t1, a2 +.Ldma_memmove_fwd_loop: + li a2, {max_bytes} + bgeu t1, a2, .Ldma_memmove_fwd_call + mv a2, t1 +.Ldma_memmove_fwd_call: + li a7, {syscall} + ecall + sub t1, t1, a2 + add a0, a0, a2 + add a1, a1, a2 + bnez t1, .Ldma_memmove_fwd_loop +.Ldma_memmove_done: + mv a0, t0 + ret + .size memmove, .-memmove +"#, + syscall = const DMA_MEMCPY_SYSCALL_NUMBER, + max_bytes = const DMA_MEMCPY_MAX_BYTES, +); + +// --------------------------------------------------------------------------- +// DMA memset symbol override +// +// Here rather than in `syscalls.rs` for the same reason as `memcpy` above. +// +// memset is expressed as a *propagating* memmove, so it needs no accelerator of its +// own: the stub seeds the first eight bytes with an ordinary store and then calls the +// copy accelerator with `dst = seed_end`, `src = seed_start`. The chip runs that call +// with the read/write timestamp order inverted — it writes at T+1 and reads at T+2 — +// so every step observes the previous step's write and the seed propagates across the +// range. The ecall number is what selects the order; the guest never chooses it. +// +// `a1` therefore carries a source address here, not the fill byte, and it is only ever +// read by the `sb`s that lay down the seed. +// +// No range check for the 2^32 limb boundary here, and none is needed. The accelerator +// pins `dst = src + 8` limb-wise, so a row whose low limb carries has no representable +// successor and the executor refuses the call. No well-formed object can produce one: +// every non-stack address lives below `MAX_MEMORY_SIZE` (0xC000_0000), and a stack +// object satisfies `buf + n <= STACK_TOP`, whose low limb is 0xFFFF_FFF0 — so +// `low(buf) + n <= 0xFFFF_FFF0`, leaving 15 bytes of headroom for the 8-byte gap. The +// margin is STACK_TOP's 16-byte ABI alignment. A guest that computes a pointer past +// the stack top is already undefined behaviour in C, and gets a clean executor error +// rather than an unprovable trace. `sb` writes the low byte of its source, so +// C's `(unsigned char)c` truncation comes for free and needs no masking of its own -- +// a wide or negative `int` fill lands as the right byte either way. +// +// Fills shorter than sixteen bytes take a plain store loop: they cannot amortise the +// seed, and below eight bytes there is nothing left to propagate. +// --------------------------------------------------------------------------- + +global_asm!( + r#" + .section .text.memset,"ax",@progbits + .p2align 2 + .globl memset + .type memset,@function +memset: + mv t0, a0 + beqz a2, .Ldma_memset_done + li t2, 16 + bltu a2, t2, .Ldma_memset_bytewise + // Seed the first eight bytes one at a time. A doubleword store would be shorter + // but would assume an alignment `dst` does not have: a byte array on the stack is + // 1-aligned, and seeding it with `sd` is silently wrong there. + sb a1, 0(a0) + sb a1, 1(a0) + sb a1, 2(a0) + sb a1, 3(a0) + sb a1, 4(a0) + sb a1, 5(a0) + sb a1, 6(a0) + sb a1, 7(a0) + mv t1, a2 + addi t1, t1, -8 + mv a1, a0 + addi a0, a0, 8 +.Ldma_memset_loop: + li a2, {max_bytes} + bgeu t1, a2, .Ldma_memset_call + mv a2, t1 +.Ldma_memset_call: + li a7, {syscall} + ecall + sub t1, t1, a2 + add a0, a0, a2 + add a1, a1, a2 + bnez t1, .Ldma_memset_loop + j .Ldma_memset_done +.Ldma_memset_bytewise: + mv t1, a2 +.Ldma_memset_byte_loop: + sb a1, 0(a0) + addi a0, a0, 1 + addi t1, t1, -1 + bnez t1, .Ldma_memset_byte_loop +.Ldma_memset_done: + mv a0, t0 + ret + .size memset, .-memset +"#, + syscall = const DMA_MEMSET_SYSCALL_NUMBER, + max_bytes = const DMA_MEMCPY_MAX_BYTES, +); diff --git a/syscalls/src/syscalls.rs b/syscalls/src/syscalls.rs index 5228455ea..e4f1c7e7e 100644 --- a/syscalls/src/syscalls.rs +++ b/syscalls/src/syscalls.rs @@ -33,6 +33,20 @@ const KECCAK_SYSCALL_NUMBER: usize = usize::MAX - 1; #[cfg(target_arch = "riscv64")] const ECSM_SYSCALL_NUMBER: usize = usize::MAX - 10; +/// Copy-accelerator syscall number, serving `memcpy` and `memmove` (-30 as usize). +/// Must match the executor. +#[cfg(target_arch = "riscv64")] +pub(crate) const DMA_MEMCPY_SYSCALL_NUMBER: usize = usize::MAX - 29; +/// Maximum bytes sent in one DMA ecall. Larger `memcpy` calls are split by the +/// strong assembly stub so continuation table height remains bounded by cycles. +#[cfg(target_arch = "riscv64")] +pub(crate) const DMA_MEMCPY_MAX_BYTES: usize = 256; + +/// `memset` syscall number (-32 as usize; -31 is the hint ecall). Must match the +/// executor. +#[cfg(target_arch = "riscv64")] +pub(crate) const DMA_MEMSET_SYSCALL_NUMBER: usize = usize::MAX - 31; + /// Syscall number for the non-constraining Hint ecall. /// Must match `executor::...::execution::HINT_SYSCALL_NUMBER` (u64::MAX - 30). #[cfg(target_arch = "riscv64")]