From 1a82b97c88b0852815be886d4ffb7d098978b0dd Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 8 Sep 2026 16:18:16 -0300 Subject: [PATCH] refactor(lfm,stark): delete the preprocessed round root and the mixed-height MMCS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `prep_root` and `prep_widths` leave `LfmRegistryEntry` and `LfmArtifacts`, and the machinery behind them goes with them: `PREP_ROUND_SLOTS`, `prep_round_dims`, `pinned_prep_widths` (method and free function), `prep_round_shape`, `slot_of_table`, `commit.rs`'s `PrepRoundBuilder`, `stark::batched` (`shape.rs` and its `mod.rs`) and `stark::fri::mmcs`. They were one thing: a SECOND commitment over the preprocessed matrices the per-slot `roots` already commit individually, gathered into one mixed-height tree. The machine proves and verifies a per-table `MultiProof` whose openings authenticate against `roots`, so nothing read the round — `verify_against_ artifacts` said so in its own doc. With no reader, the round was a pinned constant that every `build_artifacts` call paid a tree for and six registry entries carried. `slot_of_table` goes because `pinned_prep_widths` was its only caller: it mapped an epoch TABLE index back to a registry SLOT across the chunking and chip-mask shifts, and that translation existed to index the round's width slice. `par::par_for_each_mut_indexed` goes for the same reason — its only caller was `fri/mmcs.rs:776`. No blessed value is hand-edited. The six entries lose two FIELDS; every root, `program_id` and height in them is byte-identical, and `lfm_program_id` never took the round as an argument, so no digest moves. `compute_lfm_registry` stops emitting the two fields, so a regenerated table matches the struct. Tests: the seven M-6 prep-round tests in `machine_tests.rs` and the twelve drift assertions that paired `entry.prep_root`/`prep_widths` against the artifacts, plus `blake3_chip_tests::the_prep_round_expands_with_the_blake3_ chunks`. `verify_against_artifacts_agrees_with_the_registry_path` sits inside the same banner and STAYS — it is M-7's honest-path control for a function that stays. `stark::batched::shape` carried no tests; `fri/mmcs.rs` carried fourteen, whose subject was the mixed-height tree itself. --- crypto/stark/src/batched/mod.rs | 9 - crypto/stark/src/batched/shape.rs | 374 ----- crypto/stark/src/fri/mmcs.rs | 1826 ------------------------ crypto/stark/src/fri/mod.rs | 1 - crypto/stark/src/lib.rs | 1 - crypto/stark/src/par.rs | 23 - prover/src/bin/compute_lfm_registry.rs | 8 - prover/src/lfm/blake3_chip_tests.rs | 77 - prover/src/lfm/commit.rs | 86 +- prover/src/lfm/machine_tests.rs | 372 ----- prover/src/lfm/mod.rs | 2 +- prover/src/lfm/proof.rs | 16 +- prover/src/lfm/registry.rs | 372 +---- 13 files changed, 16 insertions(+), 3151 deletions(-) delete mode 100644 crypto/stark/src/batched/mod.rs delete mode 100644 crypto/stark/src/batched/shape.rs delete mode 100644 crypto/stark/src/fri/mmcs.rs diff --git a/crypto/stark/src/batched/mod.rs b/crypto/stark/src/batched/mod.rs deleted file mode 100644 index c3c880238..000000000 --- a/crypto/stark/src/batched/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Shape derivation for a multi-matrix preprocessed round. -//! -//! [`shape`] records which table contributes which matrix to which round, -//! derived from the AIR set rather than read from a proof. The LFM registry -//! pins a preprocessed round root over several slots' matrices and needs that -//! description to say which widths the round covers -//! (`prover/src/lfm/registry.rs::pinned_prep_widths`). - -pub mod shape; diff --git a/crypto/stark/src/batched/shape.rs b/crypto/stark/src/batched/shape.rs deleted file mode 100644 index ab0249eb8..000000000 --- a/crypto/stark/src/batched/shape.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! A round's committed shape — which table contributes a matrix to it, at what -//! height and width. -//! -//! Every number here is derived from the AIR set and the per-table trace -//! lengths, never read out of a proof. That is what lets a verifier rebuild the -//! shape it must pass to [`crate::fri::mmcs::MixedMmcs::verify_batch`] instead -//! of trusting the prover's word for it (`fri/mmcs.rs`, "Width binding"). -//! -//! # Why one type and not a list per round -//! -//! Each round (preprocessed, main, aux, composition parts) has a DIFFERENT -//! participation list: only preprocessed tables contribute a preprocessed -//! matrix, only tables with a RAP contribute an aux matrix. The index a matrix -//! has inside its round is therefore NOT its table index, and the two are easy -//! to confuse — a confusion that shows up as an opening authenticated at the -//! wrong leaf rather than as a compile error. [`RoundShape`] keeps the mapping -//! in one place so every reader takes it from the same code. - -use crate::config::Commitment; -use crate::traits::AIR; - -/// The preprocessed round's pinned shape: the root a program's registry entry -/// commits, and the widths the leaf parse depends on. -/// -/// # Why the widths travel with the root -/// -/// Under a per-slot scheme a group's width is implied by its own root plus its -/// AIR. Under one tree over several matrices the widths decide how each leaf is -/// *parsed*, so a comparison of roots alone is only equivalent to the per-slot -/// comparisons it replaces if the parse is pinned too. They are carried here -/// rather than derived at the -/// comparison site so that a caller holding entry A but an AIR set built for -/// entry B is rejected as a width disagreement rather than as an unexplained -/// root mismatch. -/// -/// # Absence means different things to a producer and a checker -/// -/// Held as an `Option`, and the two sides do NOT mean the same thing by the -/// absence: -/// -/// - **Producer — permissive.** `None` is how the root is generated in the first -/// place (registry regeneration has nothing to compare against yet). Supplying -/// it buys a fail-fast: a stale preprocessed constant is caught at build time -/// rather than by every future checker. -/// - **Checker — fails closed.** `None` is accepted only for an AIR set with no -/// preprocessed table at all. A set that HAS a preprocessed round and no -/// pinned root is rejected, because the only root left to check against would -/// be the one whoever built the matrices chose. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct PinnedPrep<'a> { - pub root: &'a Commitment, - /// One width per contributing matrix, in [`RoundShape::tables`] order. - pub widths: &'a [usize], -} - -/// Which tables contribute a matrix to one batched round, and with what shape. -#[derive(Clone, Debug, PartialEq, Eq, Default)] -pub struct RoundShape { - /// Contributing table indices, ascending. Position `i` in this vector is - /// matrix `i` of the round — the order the MMCS concatenates leaves in, and - /// the order openings are presented in. - pub tables: Vec, - /// `(log_height, width)` per contributing matrix, in `tables` order. - pub dims: Vec<(usize, usize)>, -} - -impl RoundShape { - pub fn is_empty(&self) -> bool { - self.tables.is_empty() - } - - pub fn heights(&self) -> Vec { - self.dims.iter().map(|(h, _)| *h).collect() - } - - pub fn widths(&self) -> Vec { - self.dims.iter().map(|(_, w)| *w).collect() - } - - /// The round's own tallest matrix — the index space - /// [`crate::fri::mmcs::MixedMmcs::verify_batch`] accepts. `None` for an - /// empty round. - pub fn h_max(&self) -> Option { - self.dims.iter().map(|(h, _)| *h).max() - } -} - -/// The one table whose MAIN matrix is committed as its own standalone row-pair -/// tree instead of contributing a matrix to the shared main round. -/// -/// This is the L2G carve-out: a continuation epoch's LOCAL_TO_GLOBAL table -/// keeps a per-table main commitment so the cross-epoch root-equality binding -/// (`verify_l2g_commitment_binding_view`) reads the SAME root from a batched -/// epoch as from a per-table one. The carved tree is built by the same -/// committer the per-table prover uses (`commit_rows_bit_reversed_subset` over -/// the full committed-main range), so the two roots are byte-identical for the -/// same trace. The carved table's aux and composition-parts matrices stay in -/// the shared rounds; only its main commitment moves. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct CarvedMain { - /// The carved table's index in the AIR set. - pub table: usize, - /// The carved matrix's width — the table's committed main columns. - pub width: usize, -} - -/// The shape of every batched round in one epoch. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct EpochShape { - /// `log2` of each table's LDE length, in table order. This is the FRI's - /// shape: query indices live in the tallest of these domains. - pub heights: Vec, - /// Preprocessed columns. Empty when no table is preprocessed. - pub prep: RoundShape, - /// Main trace columns — every table except a carved one. For a - /// preprocessed table this is the MULTIPLICITY columns only, matching the - /// per-table path's split (`commit_main_trace`: `[0, num_precomputed)` is - /// the preprocessed matrix, `[num_precomputed, total)` the committed main - /// one). - pub main: RoundShape, - /// Auxiliary (RAP) columns. Empty when no table has a RAP. - pub aux: RoundShape, - /// Composition-polynomial parts — every table. - pub parts: RoundShape, - /// The table (at most one) whose main matrix is committed standalone. - /// `None` for an ordinary epoch. See [`CarvedMain`]. - pub carved_main: Option, -} - -/// Why an epoch cannot be proved (or verified) with one batched instance. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ShapeError { - /// No tables at all. - Empty, - /// A table's LDE length is not a power of two, is 1, or overflows a shift. - /// Heights come from proof-supplied trace lengths on the verifier's side, so - /// this is a rejection, never a panic. - BadHeight { table: usize, lde_size: usize }, - /// A table declares zero committed main columns, so it has no matrix to - /// contribute and no leaf to open. - NoMainColumns { table: usize }, - /// The batched path commits ONE FRI instance for the whole epoch, so every - /// table must agree on the parameters that instance is defined by. The - /// per-table path has no such requirement, which is exactly why this is - /// checked rather than assumed. - MixedProofOptions { table: usize, field: &'static str }, - /// The carved-main table index does not name a table of this epoch. - CarvedOutOfRange { table: usize }, - /// A preprocessed table cannot be carved: its main-round matrix is the - /// multiplicity columns only, while the per-table root the carve must - /// reproduce commits the full main range. The one production carve (L2G) - /// has no preprocessed columns, so this is rejected rather than supported. - CarvedTablePreprocessed { table: usize }, -} - -impl core::fmt::Display for ShapeError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - ShapeError::Empty => write!(f, "an epoch needs at least one table"), - ShapeError::BadHeight { table, lde_size } => write!( - f, - "table {table}: LDE length {lde_size} is not a power of two greater than 1" - ), - ShapeError::NoMainColumns { table } => { - write!(f, "table {table} commits no main columns") - } - ShapeError::MixedProofOptions { table, field } => write!( - f, - "table {table} disagrees with table 0 on `{field}`; one batched FRI \ - instance needs one set of parameters" - ), - ShapeError::CarvedOutOfRange { table } => { - write!(f, "carved-main table {table} is not a table of this epoch") - } - ShapeError::CarvedTablePreprocessed { table } => write!( - f, - "carved-main table {table} is preprocessed; the carve commits the full \ - main range and cannot reproduce a preprocessed table's per-table root" - ), - } - } -} - -/// The epoch-wide FRI parameters, once every table has been checked to agree. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct EpochFriParams { - pub blowup_log: u32, - pub coset_offset: u64, - pub grinding_factor: u8, - pub num_queries: usize, - pub final_poly_log_degree: u32, -} - -impl EpochShape { - /// Derive the shape from the AIR set and each table's interpolation-domain - /// size (`trace_length`). - /// - /// The prover passes the trace lengths it is about to prove; the verifier - /// passes the ones the proof declares. Both then hold the same `EpochShape` - /// without either having read it from the other. - pub fn derive( - airs: &[&dyn AIR], - trace_lengths: &[usize], - ) -> Result<(Self, EpochFriParams), ShapeError> - where - F: math::field::traits::IsFFTField - + math::field::traits::IsSubFieldOf - + Send - + Sync - + 'static, - E: math::field::traits::IsField + Send + Sync + 'static, - { - Self::derive_carved(airs, trace_lengths, None) - } - - /// As [`EpochShape::derive`], with one table's main matrix carved into a - /// standalone commitment ([`CarvedMain`]). Both sides pass the SAME - /// `carved_main`: it is verifier-owned configuration (like the AIR set), - /// never read from a proof. - pub fn derive_carved( - airs: &[&dyn AIR], - trace_lengths: &[usize], - carved_main: Option, - ) -> Result<(Self, EpochFriParams), ShapeError> - where - F: math::field::traits::IsFFTField - + math::field::traits::IsSubFieldOf - + Send - + Sync - + 'static, - E: math::field::traits::IsField + Send + Sync + 'static, - { - if airs.is_empty() || airs.len() != trace_lengths.len() { - return Err(ShapeError::Empty); - } - if let Some(c) = carved_main { - if c >= airs.len() { - return Err(ShapeError::CarvedOutOfRange { table: c }); - } - if airs[c].is_preprocessed() { - return Err(ShapeError::CarvedTablePreprocessed { table: c }); - } - } - - let first = airs[0].options(); - let params = EpochFriParams { - blowup_log: (first.blowup_factor as usize).trailing_zeros(), - coset_offset: first.coset_offset, - grinding_factor: first.grinding_factor, - num_queries: first.fri_number_of_queries, - final_poly_log_degree: first.fri_final_poly_log_degree as u32, - }; - - let mut heights = Vec::with_capacity(airs.len()); - let mut prep = RoundShape::default(); - let mut main = RoundShape::default(); - let mut aux = RoundShape::default(); - let mut parts = RoundShape::default(); - let mut carved = None; - - for (table, (air, &trace_length)) in airs.iter().zip(trace_lengths).enumerate() { - let options = air.options(); - for (field, same) in [ - ( - "blowup_factor", - options.blowup_factor == first.blowup_factor, - ), - ("coset_offset", options.coset_offset == first.coset_offset), - ( - "grinding_factor", - options.grinding_factor == first.grinding_factor, - ), - ( - "fri_number_of_queries", - options.fri_number_of_queries == first.fri_number_of_queries, - ), - ( - "fri_final_poly_log_degree", - options.fri_final_poly_log_degree == first.fri_final_poly_log_degree, - ), - ] { - if !same { - return Err(ShapeError::MixedProofOptions { table, field }); - } - } - - let lde_size = trace_length - .checked_mul(options.blowup_factor as usize) - .ok_or(ShapeError::BadHeight { - table, - lde_size: usize::MAX, - })?; - if !lde_size.is_power_of_two() || lde_size < 2 || lde_size.trailing_zeros() >= u32::BITS - { - return Err(ShapeError::BadHeight { table, lde_size }); - } - let h = lde_size.trailing_zeros() as usize; - heights.push(h); - - let (total_main_cols, aux_cols) = air.trace_layout(); - let num_precomputed = if air.is_preprocessed() { - air.num_precomputed_columns() - } else { - 0 - }; - let committed_main = total_main_cols - .checked_sub(num_precomputed) - .ok_or(ShapeError::NoMainColumns { table })?; - if committed_main == 0 { - return Err(ShapeError::NoMainColumns { table }); - } - - if num_precomputed > 0 { - prep.tables.push(table); - prep.dims.push((h, num_precomputed)); - } - if carved_main == Some(table) { - carved = Some(CarvedMain { - table, - width: committed_main, - }); - } else { - main.tables.push(table); - main.dims.push((h, committed_main)); - } - if aux_cols > 0 && air.has_aux_trace() { - aux.tables.push(table); - aux.dims.push((h, aux_cols)); - } - let num_parts = air.composition_poly_degree_bound(trace_length) / trace_length; - parts.tables.push(table); - parts.dims.push((h, num_parts.max(1))); - } - - Ok(( - Self { - heights, - prep, - main, - aux, - parts, - carved_main: carved, - }, - params, - )) - } - - /// The epoch's tallest LDE — the domain query indices are drawn in. - pub fn h_max(&self) -> usize { - self.heights.iter().copied().max().unwrap_or(0) - } - - /// One width per table, in table order, summing every matrix that table - /// contributes across all four rounds. - /// - /// Summing rather than listing per round is deliberate: a table's total - /// committed width moves whenever ANY of its four matrices changes width, so - /// the sum separates exactly the shapes four separate lists would, while - /// staying one entry per table and needing no agreed round ordering. - pub fn total_widths(&self) -> Vec { - let mut widths = vec![0usize; self.heights.len()]; - for round in [&self.prep, &self.main, &self.aux, &self.parts] { - for (&table, (_, w)) in round.tables.iter().zip(round.dims.iter()) { - widths[table] += *w; - } - } - // A carved main matrix is committed outside the shared rounds but is - // still committed width: the histogram binds it like any other. - if let Some(c) = &self.carved_main { - widths[c.table] += c.width; - } - widths - } -} diff --git a/crypto/stark/src/fri/mmcs.rs b/crypto/stark/src/fri/mmcs.rs deleted file mode 100644 index 3f8074037..000000000 --- a/crypto/stark/src/fri/mmcs.rs +++ /dev/null @@ -1,1826 +0,0 @@ -//! Mixed-height, row-pair MMCS (Merkle Mixed Commitment Scheme). -//! -//! Commits ALL of an epoch's matrices (one per table, of possibly different -//! heights) into ONE mixed-height Merkle tree, so a single query opens ONE -//! authentication path that covers every table's row at that query — the -//! proof-size / opening-path win of the unified-shard design (SP1 / OpenVM / -//! Plonky3). Mirrors Plonky3's `MerkleTreeMmcs`, adapted to the [`StarkHash`] -//! commitment configuration and to the row-pair `(x, -x)` leaf layout (#735). -//! -//! This is a standalone primitive: the prover and verifier do not build epoch -//! commitments with it yet. The leaf and injection layout documented below is -//! the single source of truth for whoever wires it in. -//! -//! # Inputs -//! -//! [`MixedMmcs::commit`] reads matrices through a [`LeafSource`], which reports -//! each matrix's `(log_height, width)` and serves its rows on demand: -//! - `log_height`: `log2` of the row count; the matrix has `2^log_height` rows. -//! - `width`: number of committed columns. -//! - rows are addressed by **bit-reversed** LDE position (the same layout the -//! per-table trace commit produces internally). -//! -//! # Row-pair leaves -//! -//! Leaf `k` of a matrix groups LDE positions `2k` and `2k+1` (the FRI fold pair -//! `x` and `-x`), all `width` columns batched. A matrix of `log_height h` has -//! `2^(h-1)` leaves. In [`MixedMmcs::open_batch`] / [`PolynomialOpenings`]: -//! `evaluations` = row `2k`, `evaluations_sym` = row `2k+1`. -//! -//! # Tree layout (the soundness-relevant contract) -//! -//! Let `h_max = max(log_height)`. The base digest layer (layer 0) has -//! `N0 = 2^(h_max-1)` nodes. Layer `i` has `N0 >> i` nodes; the root is the sole -//! node of layer `h_max-1`. A matrix of `log_height h` is *injected* at layer -//! index `i = h_max - h` (so the tallest matrices, `h == h_max`, populate the -//! base layer; shorter matrices enter where the layer width matches their leaf -//! count `2^(h-1)`). -//! -//! Hashing (`H = >::hash_data` over a `Vec` of field elements; -//! `C = >::hash_new_parent`, the 2-input compression — the same -//! two functions, on the same backend, that the existing per-table tree uses): -//! -//! - **Base layer** node `k` (`k in [0, N0)`): -//! `layer0[k] = H( CONCAT_{m : h_m == h_max} (row_m(2k) || row_m(2k+1)) )` -//! where matrices of height `h_max` are concatenated in INPUT order. -//! - **Climb** from layer `i` to layer `i+1` (`j in [0, N_{i+1})`): -//! `parent = C(layer_i[2j], layer_i[2j+1])`. Let `inject_h = h_max - 1 - i`. If -//! any matrix has `h_m == inject_h`, then -//! `layer_{i+1}[j] = C( parent, H( CONCAT_{m : h_m == inject_h} (row_m(2j) || row_m(2j+1)) ) )` -//! (injecting matrices concatenated in INPUT order); otherwise -//! `layer_{i+1}[j] = parent`. -//! - `root = layer_{h_max-1}[0]`. -//! -//! Because the leaf and parent hashes come from `H::Batched` — the backend the -//! per-table row-pair tree already commits with — a single-matrix `MixedMmcs` is -//! byte-identical to that tree by construction, not by coincidence. There is no -//! second encoding of a leaf to keep in step. -//! -//! # Query opening -//! -//! For query `iota in [0, N0)`, matrix `m` is opened at leaf -//! `k_m = iota >> (h_max - h_m)` (`= iota >> i_m`). The shared authentication -//! path holds, for each level `level in [0, h_max-1)`, the sibling -//! `layer_level[(iota >> level) ^ 1]`. ONE path authenticates all matrices. -//! The per-matrix [`PolynomialOpenings::proof`] fields are empty; the single -//! [`MixedOpening::proof`] is the authenticator. -//! -//! # ★ Index convention — a HARD PRECONDITION on the caller -//! -//! `iota` is a leaf index **in THIS tree**: it must be drawn from -//! `[0, 2^(h_max-1))` where `h_max` is *this MMCS's* tallest matrix. -//! [`MixedMmcs::verify_batch`] walks the path with `(iota >> level) & 1`, i.e. it -//! consumes the **low** `h_max - 1` bits, while a shorter matrix inside the tree -//! is located by `iota >> (h_max - h_m)`, i.e. by the **high** bits. Both are -//! consistent only when the two `h_max` agree. -//! -//! A caller that batches several rounds under one shared FRI query index must -//! therefore reduce a global index before calling in: -//! -//! ```text -//! iota_round = iota_fri >> (h_max_fri - h_max_round) -//! ``` -//! -//! Passing the un-reduced `iota_fri` to a round whose `h_max` is below the FRI's -//! is not a loud error — prover and verifier share this routine, so a wrong -//! convention is self-consistent: honest proofs still verify and the failure is -//! that short matrices end up authenticated at positions the FRI join never -//! checks. [`MixedMmcs::verify_batch`] rejects an `iota` outside `[0, 2^(h_max-1))` -//! to turn most of that class of misuse into a rejection rather than a silent -//! mis-binding, but the reduction remains the caller's obligation: an index that -//! happens to land in range is accepted at the wrong leaf. -//! `short_round_low_bit_convention_is_exercised` is the control on this. -//! -//! # Width binding (soundness) -//! -//! [`MixedMmcs::verify_batch`] takes per-matrix `widths` alongside `heights`. -//! Within a height group the leaf hash is over the FLAT concatenation of every -//! matrix's opened row pair (`A.eval ‖ A.eval_sym ‖ B.eval ‖ B.eval_sym ‖ …`), -//! which does NOT by itself record where each matrix's columns end. Fixing -//! `widths[m]` (matrix `m`'s column count) makes those boundaries unambiguous: -//! without it a prover could shift a boundary — e.g. lengthen one matrix's -//! `evaluations` by one element and shorten its `evaluations_sym` by one — -//! leaving the flat bytes (and therefore the group hash) identical while feeding -//! a corrupted row downstream. Consumers MUST pass the committed public -//! per-table column counts, in the same INPUT order as `heights`, derived from -//! the AIR set rather than read out of the proof. -//! -//! `heights` and `widths` must ALSO be bound into the Fiat-Shamir transcript by -//! the consumer, before any challenge that depends on the shape. -//! -//! # Determinism -//! -//! The tree is a pure function of `(matrices, input order)`. Grouping within a -//! height (base batching and injection) follows INPUT order; the prover and -//! verifier MUST pass matrices and `heights` in the same per-epoch order. -//! -//! # Memory: what the caller may drop, and when -//! -//! The MMCS owns no evaluations. It stores the digest layers -//! (`O(2^(h_max-1))` nodes) plus each matrix's `(log_height, width)`; rows are -//! pulled through [`LeafSource`] both at commit and at open time. Two properties -//! follow, and `commit_reads_each_height_group_in_one_contiguous_phase` is the -//! control on the second: -//! -//! - `commit` reads matrix `m`'s rows **only while building level -//! `h_max - h_m`**, and levels are built in descending height order. A caller -//! may therefore produce a height group's LDEs, commit, and drop them before -//! the next group is needed. -//! - Within one height group the leaf is a single `hash_data` over the group's -//! concatenated rows, so `commit` reads every matrix of that height at every -//! leaf: their access windows overlap, and a caller serving them from in-RAM -//! LDE buffers holds the whole group at once. Since the tallest group is most -//! of a real epoch's tables, that is `O(N)` resident at the base layer. -//! -//! [`StreamingMmcsBuilder`] is the escape, and it is the one a batched prover -//! must use for the base group. It keeps one incremental leaf hasher per leaf -//! (`IsLeafHasher`) and absorbs matrices as they arrive, so a caller produces one -//! matrix's LDE, absorbs it and drops it — retained state is -//! `O(leaves x hasher_state)`, independent of how many matrices there are and how -//! wide they get. `streaming_builder_serves_the_base_group_without_holding_it` -//! traces both halves: that `commit`'s base-group windows overlap, and that the -//! builder's are pairwise disjoint at every height. A `LeafSource` serving from -//! disk, device memory or recomputation remains a second, orthogonal escape. - -use core::marker::PhantomData; - -use crypto::merkle_tree::proof::Proof; -use crypto::merkle_tree::traits::{IsLeafHasher, IsMerkleTreeBackend, IsStreamingLeafBackend}; -use math::fft::bit_reversing::reverse_index; -use math::field::element::FieldElement; -use math::field::traits::IsField; -use math::traits::AsBytes; - -use crate::config::{Commitment, StarkHash}; -use crate::proof::stark::PolynomialOpenings; - -/// On-demand supplier of committed matrix rows, so [`MixedMmcs`] builds its -/// digests and serves openings WITHOUT owning a copy of the (large) LDE buffers. -/// Both [`MixedMmcs::commit`] and [`MixedMmcs::open_batch`] read every leaf -/// through this trait, so the root and opened rows are byte-identical to those a -/// matrix-owning MMCS would produce — the prover keeps only the LDE buffers it -/// already retains for DEEP, and each MMCS stores just digests. -/// -/// Rows are addressed in each matrix's committed row-pair layout: `append_row(m, -/// r, out)` appends matrix `m`'s row at **bit-reversed** LDE position `r` (its -/// `width(m)` committed columns, in column order). This is the same `r`-indexing -/// the module's "Tree layout" section uses; an implementor holding the -/// natural-order LDE maps `r` to `reverse_index(r, 2^log_height(m))`. -pub trait LeafSource { - /// Number of committed matrices, in canonical input order. - fn num_matrices(&self) -> usize; - /// `log2` of matrix `m`'s row count. Row-pair leaves require `>= 1`. - fn log_height(&self, m: usize) -> usize; - /// Matrix `m`'s committed column count. - fn width(&self, m: usize) -> usize; - /// Append matrix `m`'s bit-reversed LDE row `bitrev_row` (its `width(m)` - /// committed columns) to `out`. `bitrev_row in [0, 2^log_height(m))`. - fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>); -} - -/// One committed matrix borrowed from a retained LDE buffer. Resolves each -/// bit-reversed row on demand (mapping through `reverse_index`) so the MMCS owns -/// no copy of the evaluations. See [`LeafSource`]. -pub enum BorrowedMatrix<'a, E: IsField> { - /// A `stride`-wide, row-major, NATURAL-order LDE buffer (the main / aux LDE - /// retained in `Round1::lde_trace`). This matrix occupies columns - /// `[col_start, col_start + width)`; its bit-reversed row `r` lives at - /// natural-order row `reverse_index(r, 2^log_height)`. - RowMajorNatural { - data: &'a [FieldElement], - stride: usize, - col_start: usize, - width: usize, - log_height: usize, - }, - /// Column-major NATURAL-order columns (the composition-poly LDE retained in - /// `Round2::lde_composition_poly_evaluations`): `cols[c][nat]` is column `c` - /// at natural-order row `nat`. Every committed column is used. - ColMajorNatural { - cols: &'a [Vec>], - log_height: usize, - }, -} - -impl BorrowedMatrix<'_, E> { - fn log_height(&self) -> usize { - match self { - BorrowedMatrix::RowMajorNatural { log_height, .. } - | BorrowedMatrix::ColMajorNatural { log_height, .. } => *log_height, - } - } - - fn width(&self) -> usize { - match self { - BorrowedMatrix::RowMajorNatural { width, .. } => *width, - BorrowedMatrix::ColMajorNatural { cols, .. } => cols.len(), - } - } - - fn append_row(&self, bitrev_row: usize, out: &mut Vec>) { - match self { - BorrowedMatrix::RowMajorNatural { - data, - stride, - col_start, - width, - log_height, - } => { - let nat = reverse_index(bitrev_row, 1u64 << log_height); - let base = nat * stride + col_start; - out.extend_from_slice(&data[base..base + width]); - } - BorrowedMatrix::ColMajorNatural { cols, log_height } => { - let nat = reverse_index(bitrev_row, 1u64 << log_height); - for col in cols.iter() { - out.push(col[nat].clone()); - } - } - } - } -} - -impl LeafSource for Vec> { - fn num_matrices(&self) -> usize { - self.len() - } - fn log_height(&self, m: usize) -> usize { - self[m].log_height() - } - fn width(&self, m: usize) -> usize { - self[m].width() - } - fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { - self[m].append_row(bitrev_row, out); - } -} - -/// A committed mixed-height, row-pair MMCS under the commitment configuration -/// `H`. Stores ONLY the digest layers (to serve the shared authentication path) -/// plus each matrix's `(log_height, width)` (to locate leaves). The row DATA is -/// served on demand by the caller's [`LeafSource`] — the MMCS never owns a copy -/// of the LDE. -pub struct MixedMmcs { - root: Commitment, - /// `layers[0]` is the base digest layer; `layers[h_max-1] == [root]`. - layers: Vec>, - /// Per committed matrix, in input order: `(log_height, width)`. - dims: Vec<(usize, usize)>, - h_max: usize, - _marker: PhantomData<(E, H)>, -} - -/// The opening of ALL matrices at one query index, authenticated by a single -/// shared Merkle path. -#[derive( - Debug, - Clone, - serde::Serialize, - serde::Deserialize, - rkyv::Archive, - rkyv::Serialize, - rkyv::Deserialize, -)] -#[serde(bound = "")] -pub struct MixedOpening { - /// The one authentication path covering every matrix's row at the query. - pub proof: Proof, - /// Per-matrix row pair (in the same INPUT order as `commit`). Each entry's - /// own `proof` is empty — [`MixedOpening::proof`] is the authenticator. - pub per_matrix: Vec>, -} - -/// Hash the row pair `(row(2*leaf), row(2*leaf+1))` of every matrix whose index -/// is in `group` (in the given order), all columns batched, into one digest. -/// Rows are pulled from `source` — the MMCS owns no copy. -fn hash_group_leaf(source: &S, group: &[usize], leaf: usize) -> Commitment -where - E: IsField + 'static, - H: StarkHash, - S: LeafSource, - FieldElement: AsBytes + Sync + Send, -{ - let mut buf: Vec> = Vec::new(); - for &m in group { - source.append_row(m, 2 * leaf, &mut buf); - source.append_row(m, 2 * leaf + 1, &mut buf); - } - as IsMerkleTreeBackend>::hash_data(&buf) -} - -/// ★ The ELEMENT SEQUENCE one group leaf covers, in hashing order: per matrix, -/// all of `evaluations` and then all of `evaluations_sym`, matrices in the -/// group's given (round INPUT) order. -/// -/// Split out of [`hash_group_openings`], its only production caller, so a -/// differential can compare the SEQUENCE two implementations feed rather than -/// only the digests they end up disagreeing on. A digest differential says THAT -/// a re-derivation disagrees; this says WHERE. Callers must not restate the -/// order themselves — that is the point of exporting it. -pub fn group_opening_felts(group: &[&PolynomialOpenings]) -> Vec> { - let mut buf: Vec> = Vec::new(); - for o in group { - buf.extend_from_slice(&o.evaluations); - buf.extend_from_slice(&o.evaluations_sym); - } - buf -} - -/// Verifier-side analogue of [`hash_group_leaf`]: hash the opened row pairs of a -/// group of openings (in the given order) into one digest. -pub fn hash_group_openings(group: &[&PolynomialOpenings]) -> Commitment -where - E: IsField + 'static, - H: StarkHash, - FieldElement: AsBytes + Sync + Send, -{ - as IsMerkleTreeBackend>::hash_data(&group_opening_felts(group)) -} - -#[inline] -fn compress(left: &Commitment, right: &Commitment) -> Commitment -where - E: IsField + 'static, - H: StarkHash, - FieldElement: AsBytes + Sync + Send, -{ - as IsMerkleTreeBackend>::hash_new_parent(left, right) -} - -impl MixedMmcs -where - E: IsField + 'static, - H: StarkHash, - FieldElement: AsBytes + Sync + Send, -{ - /// Commit the matrices supplied by `source` into one mixed-height row-pair - /// tree, storing only the digest layers. See the module docs for the exact - /// leaf/injection layout. `source` provides each matrix's dimensions and its - /// bit-reversed rows on demand; no copy of the evaluations is retained. - /// - /// Leaf hashing (the base layer and each injected climb layer) is parallel - /// across leaves via [`crate::par::par_map_collect`]; the per-level output is - /// index-ordered, so the root and layers are byte-identical to a sequential - /// build. `S: Sync` lets leaf closures read `source` from worker threads. - /// - /// Levels are built in descending height order and matrix `m` is read only - /// while its own level is built, so the caller may release a height group's - /// buffers once the next level starts — see the module's memory section. - pub fn commit + Sync>(source: &S) -> Self { - let num_matrices = source.num_matrices(); - assert!( - num_matrices > 0, - "MixedMmcs::commit requires at least one matrix" - ); - - let dims: Vec<(usize, usize)> = (0..num_matrices) - .map(|m| { - let log_height = source.log_height(m); - assert!( - log_height >= 1, - "log_height must be >= 1 (row-pair leaves need at least 2 rows)" - ); - (log_height, source.width(m)) - }) - .collect(); - - let h_max = dims - .iter() - .map(|(log_height, _)| *log_height) - .max() - .expect("dims is non-empty"); - - // Per-height group leaf digests, built in descending height order — the - // order that makes the memory claim in the module header true. Index `h` - // is `Some` exactly when some matrix has that height. - let mut group_digests: Vec>> = vec![None; h_max + 1]; - for h in (1..=h_max).rev() { - let group: Vec = (0..num_matrices).filter(|&m| dims[m].0 == h).collect(); - if group.is_empty() { - continue; - } - // 2^(h-1) independent group-leaf hashes; at `h == h_max` that is the - // bulk of the tree's hashing (half of all nodes). Parallel across - // leaves. - group_digests[h] = Some(crate::par::par_map_collect(0..1usize << (h - 1), |k| { - hash_group_leaf::(source, &group, k) - })); - } - - Self::from_group_digests(dims, h_max, group_digests) - } - - /// Build the tree from each height group's already-hashed leaf digests. - /// - /// The single climb implementation. [`Self::commit`] reaches it having hashed - /// every group leaf in one pass; [`StreamingMmcsBuilder`] reaches it having - /// hashed them incrementally, matrix by matrix. That the two produce the same - /// tree is therefore a property of calling one function, not a coincidence - /// two code paths have to be shown to share. - fn from_group_digests( - dims: Vec<(usize, usize)>, - h_max: usize, - mut group_digests: Vec>>, - ) -> Self { - let mut layers: Vec> = Vec::with_capacity(h_max); - layers.push( - group_digests[h_max] - .take() - .expect("the tallest height group is occupied by construction"), - ); - - // Climb, compressing pairs and injecting shorter matrices where the layer - // width matches their leaf count. Each level's nodes are independent - // (they read only the previous, already-materialized layer), so parallel - // across nodes; levels stay sequential. - let mut i = 0usize; - while layers[i].len() > 1 { - let next_len = layers[i].len() / 2; - let injected = group_digests[h_max - 1 - i].take(); - - let cur = &layers[i]; - let next: Vec = crate::par::par_map_collect(0..next_len, |j| { - let parent = compress::(&cur[2 * j], &cur[2 * j + 1]); - match &injected { - Some(digests) => compress::(&parent, &digests[j]), - None => parent, - } - }); - layers.push(next); - i += 1; - } - - let root = layers.last().expect("at least the base layer exists")[0]; - - MixedMmcs { - root, - layers, - dims, - h_max, - _marker: PhantomData, - } - } - - /// The committed root. - pub fn root(&self) -> Commitment { - self.root - } - - /// `log2` of the tallest committed matrix. The query index this MMCS accepts - /// lives in `[0, 2^(h_max-1))` — see the module's index-convention section. - pub fn h_max(&self) -> usize { - self.h_max - } - - /// Per committed matrix, in input order: `(log_height, width)`. The verifier - /// is expected to rebuild these from the AIR set rather than read them here; - /// this accessor exists so a prover can bind the shape it actually committed. - pub fn dims(&self) -> &[(usize, usize)] { - &self.dims - } - - /// The leaf of matrix `m` that query `iota` opens: `iota >> (h_max - h_m)`. - /// `None` when `m` is not a committed matrix or `iota` is out of this tree's - /// index space. - /// - /// Exposed alongside [`Self::auth_path`] so a prover can assemble a - /// [`MixedOpening`] ONE MATRIX AT A TIME. [`Self::open_batch`] wants a - /// `LeafSource` describing the whole round, which means every matrix's rows - /// readable at once — the same `O(N)` residency [`StreamingMmcsBuilder`] - /// exists to keep out of the commit. Query indices are only known after the - /// FRI, so without these two the win would be given back at opening time. - pub fn row_pair_leaf(&self, iota: usize, m: usize) -> Option { - if iota >= 1usize << (self.h_max - 1) { - return None; - } - let (log_height, _) = *self.dims.get(m)?; - Some(iota >> (self.h_max - log_height)) - } - - /// The shared authentication path for `iota`, reading no matrix rows at all. - /// `None` when `iota` is outside this tree's index space. - pub fn auth_path(&self, iota: usize) -> Option> { - if iota >= 1usize << (self.h_max - 1) { - return None; - } - let mut merkle_path = Vec::with_capacity(self.h_max - 1); - for level in 0..(self.h_max - 1) { - merkle_path.push(self.layers[level][(iota >> level) ^ 1]); - } - Some(Proof { merkle_path }) - } - - /// Open all matrices at query `iota in [0, 2^(h_max-1))`, returning each - /// matrix's row pair plus one shared authentication path. Row data is served - /// by `source`, which MUST describe the same matrices (same order and - /// dimensions) as the one passed to [`Self::commit`]. - pub fn open_batch>(&self, iota: usize, source: &S) -> MixedOpening { - let n0 = 1usize << (self.h_max - 1); - assert!(iota < n0, "iota {iota} out of range (n0 = {n0})"); - debug_assert_eq!( - source.num_matrices(), - self.dims.len(), - "leaf source matrix count must match the committed tree" - ); - - let per_matrix: Vec> = (0..self.dims.len()) - .map(|m| { - let (log_height, width) = self.dims[m]; - debug_assert_eq!(source.log_height(m), log_height); - debug_assert_eq!(source.width(m), width); - let k = iota >> (self.h_max - log_height); - let mut evaluations = Vec::with_capacity(width); - source.append_row(m, 2 * k, &mut evaluations); - let mut evaluations_sym = Vec::with_capacity(width); - source.append_row(m, 2 * k + 1, &mut evaluations_sym); - PolynomialOpenings { - proof: Proof { - merkle_path: Vec::new(), - }, - evaluations, - evaluations_sym, - } - }) - .collect(); - - let mut merkle_path = Vec::with_capacity(self.h_max - 1); - for level in 0..(self.h_max - 1) { - let sibling = (iota >> level) ^ 1; - merkle_path.push(self.layers[level][sibling]); - } - - MixedOpening { - proof: Proof { merkle_path }, - per_matrix, - } - } - - /// Verify a batched opening at `iota` against `root`. `heights[m]` is the - /// `log_height` of matrix `m` and `widths[m]` its column count, both in the - /// SAME order as `opening.per_matrix`, and both supplied by the verifier from - /// the AIR set rather than read out of the proof. - /// - /// `widths` binds each matrix's boundary inside the per-height-group leaf - /// hash (see the module `# Width binding` section): the group leaf hashes the - /// FLAT concatenation of every matrix's `evaluations ‖ evaluations_sym`, so - /// without fixed widths a prover could shift a matrix boundary while keeping - /// the flat bytes — and thus the hash — identical. Pinning `widths` makes the - /// boundaries unambiguous and closes that forgery. - /// - /// `iota` must already be reduced to this tree's index space — see the - /// module's index-convention section. Out-of-range indices are rejected here, - /// but that check is a backstop, not a substitute for the reduction. - /// - /// Returns `false` on every malformed input; it never panics, so a verifier - /// can call it on adversarial data. - pub fn verify_batch( - root: &Commitment, - iota: usize, - opening: &MixedOpening, - heights: &[usize], - widths: &[usize], - ) -> bool { - if opening.per_matrix.len() != heights.len() - || heights.len() != widths.len() - || heights.is_empty() - { - return false; - } - // Bind per-matrix boundaries: every opened matrix must present exactly - // `widths[m]` columns in BOTH rows of its pair. A boundary shift keeps the - // flat per-group concatenation identical but changes these lengths. - for (o, w) in opening.per_matrix.iter().zip(widths.iter()) { - if o.evaluations.len() != *w || o.evaluations_sym.len() != *w { - return false; - } - } - let Some(&h_max) = heights.iter().max() else { - return false; - }; - // Honest heights are >= 1 (row-pair leaves need >= 2 rows) and far below - // the shift width; guard both ends rather than trust the proof's shape. - if h_max == 0 || h_max >= usize::BITS as usize { - return false; - } - // Only the low `h_max - 1` bits of `iota` are consumed (one per level), so - // an index from a taller domain would authenticate the short matrices at a - // position nothing else checks. Reject it instead. - if iota >= 1usize << (h_max - 1) { - return false; - } - if opening.proof.merkle_path.len() != h_max - 1 { - return false; - } - - // Base node: batch all tallest matrices' opened row pairs (input order). - let base_group: Vec<&PolynomialOpenings> = opening - .per_matrix - .iter() - .zip(heights.iter()) - .filter(|(_, h)| **h == h_max) - .map(|(o, _)| o) - .collect(); - let mut acc = hash_group_openings::(&base_group); - - for level in 0..(h_max - 1) { - let sibling = &opening.proof.merkle_path[level]; - let bit = (iota >> level) & 1; - let mut parent = if bit == 0 { - compress::(&acc, sibling) - } else { - compress::(sibling, &acc) - }; - - // Inject matrices whose leaf count matches this (halved) layer, in - // INPUT order — mirroring `commit`'s climb exactly. - let inject_h = h_max - 1 - level; - let inject_group: Vec<&PolynomialOpenings> = opening - .per_matrix - .iter() - .zip(heights.iter()) - .filter(|(_, h)| **h == inject_h) - .map(|(o, _)| o) - .collect(); - if !inject_group.is_empty() { - let inj = hash_group_openings::(&inject_group); - parent = compress::(&parent, &inj); - } - acc = parent; - } - - &acc == root - } -} - -/// One leaf hasher of the commitment configuration's batched leaf backend. -type LeafHasherOf = <::Batched as IsStreamingLeafBackend>::LeafHasher; - -/// Builds a [`MixedMmcs`] by absorbing matrices ONE AT A TIME, so a prover never -/// has to hold a height group's LDE buffers simultaneously. -/// -/// # Why this exists -/// -/// [`MixedMmcs::commit`] reads matrix `m` only while building level -/// `h_max - h_m`, so a caller may drop a height group before the next is needed. -/// That is not enough for the group that matters. Within one height the leaf is a -/// single hash over the concatenation of every matrix's row pair, so `commit` -/// needs them all readable at once — and the tallest group is most of an epoch's -/// tables. A caller serving those rows from full in-RAM LDE buffers is back to -/// `O(N)` at the base layer, which is the whole memory win given back. -/// -/// This builder inverts the loop: it keeps one incremental leaf hasher per leaf -/// ([`IsLeafHasher`]) and absorbs matrices into them as they arrive, so the -/// caller produces one matrix's LDE, absorbs it, and drops it. Retained state is -/// `O(leaves × hasher_state)` — bounded by the epoch's tallest height and -/// independent of how many matrices there are or how wide they get. -/// -/// # Contract -/// -/// The shape is declared up front and matrices arrive in that order: the leaf -/// concatenation binds input order (see the module's determinism section), and a -/// builder that let matrices arrive out of order would commit a different tree -/// than [`MixedMmcs::commit`] over the same input. The resulting tree IS that -/// tree — both finish through one climb — which is what makes the two -/// interchangeable rather than merely tested to agree. -pub struct StreamingMmcsBuilder -where - FieldElement: AsBytes + Sync + Send, -{ - dims: Vec<(usize, usize)>, - h_max: usize, - /// Indexed by height: the in-progress leaf hashers of that height group, - /// present from construction until the group's last matrix is absorbed. - pending: Vec>>>, - /// Indexed by height: the group's finalized leaf digests. - group_digests: Vec>>, - /// Matrices of each height still to arrive. A height reaching zero is what - /// releases that group's hashers. - remaining: Vec, - next: usize, -} - -impl StreamingMmcsBuilder -where - E: IsField + 'static, - H: StarkHash, - FieldElement: AsBytes + Sync + Send, -{ - /// Declare the epoch's shape: `(log_height, width)` per matrix, in the order - /// the matrices will be absorbed and in the order the verifier will present - /// their openings. - pub fn new(dims: &[(usize, usize)]) -> Self { - assert!( - !dims.is_empty(), - "StreamingMmcsBuilder requires at least one matrix" - ); - assert!( - dims.iter().all(|(log_height, _)| *log_height >= 1), - "log_height must be >= 1 (row-pair leaves need at least 2 rows)" - ); - let h_max = dims - .iter() - .map(|(log_height, _)| *log_height) - .max() - .expect("dims is non-empty"); - - let mut remaining = vec![0usize; h_max + 1]; - for (log_height, _) in dims { - remaining[*log_height] += 1; - } - - let pending = (0..=h_max) - .map(|h| { - (remaining[h] > 0).then(|| { - (0..1usize << (h - 1)) - .map(|_| as IsStreamingLeafBackend>::leaf_hasher()) - .collect() - }) - }) - .collect(); - - Self { - dims: dims.to_vec(), - h_max, - pending, - group_digests: vec![None; h_max + 1], - remaining, - next: 0, - } - } - - /// Absorb the next declared matrix, reading its rows from `source` at index - /// `m`. The caller may drop that matrix's buffers as soon as this returns. - /// - /// Panics when the arriving matrix's shape disagrees with what was declared — - /// a prover-side programming error, not proof data. - pub fn absorb + Sync>(&mut self, source: &S, m: usize) { - let index = self.next; - assert!( - index < self.dims.len(), - "absorbed more matrices ({}) than were declared ({})", - index + 1, - self.dims.len() - ); - let (log_height, width) = self.dims[index]; - assert_eq!( - (source.log_height(m), source.width(m)), - (log_height, width), - "matrix {index} arrived with a shape the builder was not declared for" - ); - - let hashers = self.pending[log_height] - .as_mut() - .expect("a height with matrices outstanding still holds its hashers"); - // One update per leaf, parallel across leaves — the same shape, and the - // same cost, as `commit`'s one-shot group hash. - crate::par::par_for_each_mut_indexed(hashers, |leaf, hasher| { - let mut row_pair = Vec::with_capacity(2 * width); - source.append_row(m, 2 * leaf, &mut row_pair); - source.append_row(m, 2 * leaf + 1, &mut row_pair); - hasher.update(&row_pair); - }); - - self.next += 1; - self.remaining[log_height] -= 1; - if self.remaining[log_height] == 0 { - let hashers = self.pending[log_height] - .take() - .expect("the group was present a moment ago"); - self.group_digests[log_height] = - Some(hashers.into_iter().map(IsLeafHasher::finalize).collect()); - } - } - - /// Finish the tree. Panics if a declared matrix never arrived — the digests - /// would silently commit to a leaf that absorbed less than it claims. - pub fn finish(self) -> MixedMmcs { - assert_eq!( - self.next, - self.dims.len(), - "{} of {} declared matrices were absorbed", - self.next, - self.dims.len() - ); - MixedMmcs::from_group_digests(self.dims, self.h_max, self.group_digests) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::commitment::commit_bit_reversed; - use crate::config::DefaultStarkHash; - use math::field::element::FieldElement; - use math::field::goldilocks::GoldilocksField; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - type FE = FieldElement; - type Mmcs = MixedMmcs; - - /// Reference [`LeafSource`] owning bit-reversed row-major matrices. Every - /// test commits/opens through this, so the byte-parity assertion against - /// `commit_bit_reversed` pins the tree contract; `borrowed_sources_match_ - /// owned_reference` cross-checks it against the borrowed (natural-order) - /// sources a prover would use. - struct OwnedMatrices { - /// Each entry: `(bit-reversed row-major data, log_height, width)`. - mats: Vec<(Vec>, usize, usize)>, - } - - impl LeafSource for OwnedMatrices { - fn num_matrices(&self) -> usize { - self.mats.len() - } - fn log_height(&self, m: usize) -> usize { - self.mats[m].1 - } - fn width(&self, m: usize) -> usize { - self.mats[m].2 - } - fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { - let (data, _log_height, width) = &self.mats[m]; - out.extend_from_slice(&data[bitrev_row * width..(bitrev_row + 1) * width]); - } - } - - fn owned(mats: Vec<(Vec, usize, usize)>) -> OwnedMatrices { - OwnedMatrices { mats } - } - - /// Build a row-major, bit-reversed flat vec from column-major natural-order - /// `columns`, matching the layout the existing trace commit consumes: row `j` - /// of the output = `[col_0[br(j)], ..., col_{w-1}[br(j)]]` with - /// `br = reverse_index(., num_rows)`. - fn row_major_bit_reversed(columns: &[Vec], num_rows: usize) -> Vec { - let width = columns.len(); - let mut out = vec![FE::from(0u64); num_rows * width]; - for (r, chunk) in out.chunks_exact_mut(width).enumerate() { - let br = reverse_index(r, num_rows as u64); - for (c, col) in columns.iter().enumerate() { - chunk[c] = col[br]; - } - } - out - } - - /// Build a row-major flat vec in NATURAL order (no bit reversal): row `r` = - /// `[col_0[r], ..., col_{w-1}[r]]`. This is the layout the prover's - /// `BorrowedMatrix::RowMajorNatural` reads (the retained main/aux LDE buffer). - fn row_major_natural(columns: &[Vec], num_rows: usize) -> Vec { - let width = columns.len(); - let mut out = vec![FE::from(0u64); num_rows * width]; - for (r, chunk) in out.chunks_exact_mut(width).enumerate() { - for (c, col) in columns.iter().enumerate() { - chunk[c] = col[r]; - } - } - out - } - - fn make_columns(width: usize, num_rows: usize, seed: u64) -> Vec> { - (0..width) - .map(|c| { - (0..num_rows) - .map(|r| { - FE::from(seed.wrapping_mul(31) + (c as u64) * 1009 + (r as u64) * 7 + 1) - }) - .collect() - }) - .collect() - } - - #[test] - fn single_matrix_commit_open_verify_and_tamper() { - let log_height = 2usize; - let num_rows = 1usize << log_height; - let width = 3usize; - let columns = make_columns(width, num_rows, 5); - let data = row_major_bit_reversed(&columns, num_rows); - - let src = owned(vec![(data.clone(), log_height, width)]); - let mmcs = Mmcs::commit(&src); - let heights = [log_height]; - let widths = [width]; - let n0 = 1usize << (log_height - 1); - - for iota in 0..n0 { - let opening = mmcs.open_batch(iota, &src); - assert_eq!(opening.per_matrix.len(), 1); - let k = iota; - let row_2k = data[(2 * k) * width..(2 * k + 1) * width].to_vec(); - let row_2k1 = data[(2 * k + 1) * width..(2 * k + 2) * width].to_vec(); - assert_eq!(opening.per_matrix[0].evaluations, row_2k); - assert_eq!(opening.per_matrix[0].evaluations_sym, row_2k1); - assert!(Mmcs::verify_batch( - &mmcs.root(), - iota, - &opening, - &heights, - &widths - )); - } - - let mut opening = mmcs.open_batch(0, &src); - opening.per_matrix[0].evaluations[0] = - &opening.per_matrix[0].evaluations[0] + &FE::from(1u64); - assert!(!Mmcs::verify_batch( - &mmcs.root(), - 0, - &opening, - &heights, - &widths - )); - } - - /// ★ The [`StarkHash`] backward-compatibility statement: a single-matrix MMCS - /// IS the existing per-table row-pair tree. It holds by construction — both - /// go through `H::Batched`'s `hash_data` / `hash_new_parent` — and this - /// pins that no second leaf encoding crept in. - /// - /// Both sides have to be the SAME `H` for that to mean anything, which is - /// why this module commits under `DefaultStarkHash`: `commit_bit_reversed` - /// is alias-pinned, so naming a fixed hash here compares two configurations - /// and reports a hash difference as a layout difference. It did exactly that - /// at the P-a flip, when the alias moved and this side did not. - #[test] - fn single_matrix_root_matches_existing_row_pair_tree() { - let log_height = 3usize; - let num_rows = 1usize << log_height; - let width = 4usize; - let columns = make_columns(width, num_rows, 9); - - let (_, existing_root) = - commit_bit_reversed(&columns, 2).expect("non-empty columns build a tree"); - - let data = row_major_bit_reversed(&columns, num_rows); - let mmcs = Mmcs::commit(&owned(vec![(data, log_height, width)])); - - assert_eq!(mmcs.root(), existing_root); - } - - #[test] - fn mixed_height_open_positions_verify_and_tamper() { - // Three matrices, log_heights {5, 5, 3}, widths {2, 1, 4}. - let (ha, hb, hc) = (5usize, 5usize, 3usize); - let (wa, wb, wc) = (2usize, 1usize, 4usize); - let a = row_major_bit_reversed(&make_columns(wa, 1 << ha, 1), 1 << ha); - let b = row_major_bit_reversed(&make_columns(wb, 1 << hb, 2), 1 << hb); - let c = row_major_bit_reversed(&make_columns(wc, 1 << hc, 3), 1 << hc); - - let src = owned(vec![ - (a.clone(), ha, wa), - (b.clone(), hb, wb), - (c.clone(), hc, wc), - ]); - let mmcs = Mmcs::commit(&src); - let heights = [ha, hb, hc]; - let widths = [wa, wb, wc]; - let h_max = 5usize; - let n0 = 1usize << (h_max - 1); // 16 - - let row = |data: &[FE], w: usize, r: usize| data[r * w..(r + 1) * w].to_vec(); - - for iota in [0usize, 1, 2, 3, 7, 8, 13, n0 - 1] { - let opening = mmcs.open_batch(iota, &src); - assert_eq!(opening.per_matrix.len(), 3); - - // Tall matrices open at k = iota >> 0 = iota. - assert_eq!(opening.per_matrix[0].evaluations, row(&a, wa, 2 * iota)); - assert_eq!( - opening.per_matrix[0].evaluations_sym, - row(&a, wa, 2 * iota + 1) - ); - assert_eq!(opening.per_matrix[1].evaluations, row(&b, wb, 2 * iota)); - - // Height-3 matrix opens at k = iota >> (5 - 3) = iota >> 2. - let kc = iota >> (h_max - hc); - assert_eq!(opening.per_matrix[2].evaluations, row(&c, wc, 2 * kc)); - assert_eq!( - opening.per_matrix[2].evaluations_sym, - row(&c, wc, 2 * kc + 1) - ); - - assert!( - Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), - "honest opening at iota={iota} must verify" - ); - } - - // Tamper the height-3 matrix's opened row -> rejection (proves the short - // matrix is bound by the shared path via injection). - let iota = 6usize; - let mut opening = mmcs.open_batch(iota, &src); - opening.per_matrix[2].evaluations[0] = - &opening.per_matrix[2].evaluations[0] + &FE::from(1u64); - assert!( - !Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), - "tampered height-3 row must be rejected" - ); - - // Tamper a tall-matrix row too -> rejection. - let mut opening2 = mmcs.open_batch(iota, &src); - opening2.per_matrix[0].evaluations[0] = - &opening2.per_matrix[0].evaluations[0] + &FE::from(1u64); - assert!( - !Mmcs::verify_batch(&mmcs.root(), iota, &opening2, &heights, &widths), - "tampered tall-matrix row must be rejected" - ); - } - - /// Vector test: hand-compute the root for `{log_height 2, log_height 1}` - /// matrices per the documented layout and assert equality. Pins the - /// leaf/injection contract, plus determinism. - #[test] - fn vector_root_layout_contract_and_determinism() { - // A: log_height 2 (4 rows), width 2 ; B: log_height 1 (2 rows), width 3. - let a_data = row_major_bit_reversed(&make_columns(2, 4, 3), 4); - let b_data = row_major_bit_reversed(&make_columns(3, 2, 8), 2); - - let src = owned(vec![(a_data.clone(), 2, 2), (b_data.clone(), 1, 3)]); - let mmcs = Mmcs::commit(&src); - - // Hand recomputation via the backend primitives, in the documented order. - let arow = |r: usize| a_data[r * 2..(r + 1) * 2].to_vec(); - let brow = |r: usize| b_data[r * 3..(r + 1) * 3].to_vec(); - let h = |v: Vec| { - <::Batched as IsMerkleTreeBackend>::hash_data(&v) - }; - - // Base layer (matrix A only): leaf k = H(A.row(2k) || A.row(2k+1)). - let mut leaf0 = arow(0); - leaf0.extend(arow(1)); - let mut leaf1 = arow(2); - leaf1.extend(arow(3)); - let l00 = h(leaf0); - let l01 = h(leaf1); - - // Climb to layer 1 (root): compress the base pair, then inject B (h=1). - let parent = compress::(&l00, &l01); - let mut binj = brow(0); - binj.extend(brow(1)); - let inj = h(binj); - let expected_root = compress::(&parent, &inj); - - assert_eq!( - mmcs.root(), - expected_root, - "root must match the hand-computed mixed-height layout" - ); - - // Determinism: a second commit over the same inputs yields the same root. - let mmcs2 = Mmcs::commit(&owned(vec![(a_data, 2, 2), (b_data, 1, 3)])); - assert_eq!(mmcs.root(), mmcs2.root(), "commit must be deterministic"); - - for iota in 0..2usize { - let opening = mmcs.open_batch(iota, &src); - // heights {2, 1}, widths {2, 3}. - assert!(Mmcs::verify_batch( - &mmcs.root(), - iota, - &opening, - &[2, 1], - &[2, 3] - )); - } - } - - /// Two SAME-HEIGHT matrices share one base-group leaf, whose hash is over the - /// FLAT concatenation `A.eval ‖ A.eval_sym ‖ B.eval ‖ B.eval_sym`. A malicious - /// prover can shift the A|A_sym boundary (move one element from A's - /// `evaluations_sym` into A's `evaluations`) leaving that flat concatenation — - /// and hence the leaf hash — byte-identical, so a width-blind `verify_batch` - /// would accept it. The per-matrix width binding rejects the shift. - #[test] - fn boundary_shift_forgery_rejected() { - let h = 2usize; - let num_rows = 1usize << h; - let (wa, wb) = (2usize, 1usize); // wA >= 2 so we can steal one column. - let a = row_major_bit_reversed(&make_columns(wa, num_rows, 11), num_rows); - let b = row_major_bit_reversed(&make_columns(wb, num_rows, 22), num_rows); - - let src = owned(vec![(a, h, wa), (b, h, wb)]); - let mmcs = Mmcs::commit(&src); - let heights = [h, h]; - let widths = [wa, wb]; - - let iota = 0usize; - let opening = mmcs.open_batch(iota, &src); - assert!( - Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), - "honest opening must verify" - ); - - // Forge: lengthen A.evaluations by one element taken from A.evaluations_sym. - let mut forged = mmcs.open_batch(iota, &src); - let moved = forged.per_matrix[0].evaluations_sym.remove(0); - forged.per_matrix[0].evaluations.push(moved); - - // The FLAT per-group concatenation is byte-identical to the honest one, so - // the group leaf hash is UNCHANGED — the rejection must come from the width - // check, not from a differing hash. - let flat = |o: &MixedOpening| -> Vec { - let mut v = Vec::new(); - for m in &o.per_matrix { - v.extend_from_slice(&m.evaluations); - v.extend_from_slice(&m.evaluations_sym); - } - v - }; - assert_eq!( - flat(&opening), - flat(&forged), - "the flat concatenation must be byte-identical (boundary-only shift)" - ); - - assert!( - !Mmcs::verify_batch(&mmcs.root(), iota, &forged, &heights, &widths), - "boundary-shift forgery must be rejected by the width binding" - ); - } - - /// Extension-field (Fp3) coverage: the aux and composition matrices an epoch - /// batches are cubic-extension. Byte-parity cross-check of a single Fp3 matrix - /// against the existing per-table row-pair tree, plus an open/verify/tamper - /// roundtrip over the extension path. - #[test] - fn single_matrix_fp3_root_matches_existing_row_pair_tree() { - use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Fp3; - type F3 = FieldElement; - - let log_height = 3usize; - let num_rows = 1usize << log_height; - let width = 3usize; - - // Populate ALL three components so the 24-byte extension serialization is - // exercised (not just the embedded-base subset). - let columns: Vec> = (0..width) - .map(|c| { - (0..num_rows) - .map(|r| { - F3::new([ - FE::from((c as u64) * 7 + r as u64 + 1), - FE::from((r as u64) * 13 + 2), - FE::from((c as u64) * 5 + (r as u64) * 3 + 4), - ]) - }) - .collect() - }) - .collect(); - - let (_, existing_root) = - commit_bit_reversed(&columns, 2).expect("non-empty columns build a tree"); - - // Row-major bit-reversed equivalent of the same column-major data. - let mut data = vec![F3::zero(); num_rows * width]; - for (r, chunk) in data.chunks_exact_mut(width).enumerate() { - let br = reverse_index(r, num_rows as u64); - for (c, col) in columns.iter().enumerate() { - chunk[c] = col[br]; - } - } - - let src = OwnedMatrices { - mats: vec![(data, log_height, width)], - }; - let mmcs = MixedMmcs::::commit(&src); - assert_eq!( - mmcs.root(), - existing_root, - "Fp3 single-matrix root must match the existing row-pair tree" - ); - - let heights = [log_height]; - let widths = [width]; - for iota in 0..(1usize << (log_height - 1)) { - let opening = mmcs.open_batch(iota, &src); - assert!(MixedMmcs::::verify_batch( - &mmcs.root(), - iota, - &opening, - &heights, - &widths - )); - } - - let mut opening = mmcs.open_batch(0, &src); - opening.per_matrix[0].evaluations[0] = &opening.per_matrix[0].evaluations[0] + &F3::one(); - assert!(!MixedMmcs::::verify_batch( - &mmcs.root(), - 0, - &opening, - &heights, - &widths - )); - } - - /// Equivalence (the soundness contract a batched prover relies on): the - /// digest-only MMCS built from borrowed, NATURAL-order leaf sources yields the - /// SAME root and the SAME opened rows as the reference owning source over the - /// bit-reversed data — for the row-major (main / aux) layout, the column-major - /// (composition) layout, AND a main-split column sub-range (`col_start > 0`). - /// Only the leaf-byte source changes; nothing the verifier sees does. - #[test] - fn borrowed_sources_match_owned_reference() { - // Mixed heights {5, 5, 3}; the height-3 matrix exercises injection. - let specs = [(5usize, 3usize, 100u64), (5, 1, 200), (3, 4, 300)]; - - // Column-major natural-order columns per matrix. - let cols: Vec>> = specs - .iter() - .map(|&(lh, w, seed)| make_columns(w, 1 << lh, seed)) - .collect(); - - // Reference: owned, bit-reversed row-major. - let owned_src = owned( - specs - .iter() - .zip(cols.iter()) - .map(|(&(lh, w, _), c)| (row_major_bit_reversed(c, 1 << lh), lh, w)) - .collect(), - ); - - // Borrowed row-major NATURAL (the retained main / aux LDE buffer). - let rm_natural: Vec> = specs - .iter() - .zip(cols.iter()) - .map(|(&(lh, _, _), c)| row_major_natural(c, 1 << lh)) - .collect(); - let rm_src: Vec> = specs - .iter() - .zip(rm_natural.iter()) - .map(|(&(lh, w, _), data)| BorrowedMatrix::RowMajorNatural { - data: data.as_slice(), - stride: w, - col_start: 0, - width: w, - log_height: lh, - }) - .collect(); - - // Borrowed column-major NATURAL (the retained composition-poly LDE). - let cm_src: Vec> = specs - .iter() - .zip(cols.iter()) - .map(|(&(lh, _, _), c)| BorrowedMatrix::ColMajorNatural { - cols: c.as_slice(), - log_height: lh, - }) - .collect(); - - let owned_mmcs = Mmcs::commit(&owned_src); - let rm_mmcs = Mmcs::commit(&rm_src); - let cm_mmcs = Mmcs::commit(&cm_src); - assert_eq!( - owned_mmcs.root(), - rm_mmcs.root(), - "row-major natural root must match the owned reference" - ); - assert_eq!( - owned_mmcs.root(), - cm_mmcs.root(), - "column-major natural root must match the owned reference" - ); - - let n0 = 1usize << (5 - 1); - for iota in 0..n0 { - let o = owned_mmcs.open_batch(iota, &owned_src); - let rm = rm_mmcs.open_batch(iota, &rm_src); - let cm = cm_mmcs.open_batch(iota, &cm_src); - assert_eq!(o.proof.merkle_path, rm.proof.merkle_path); - assert_eq!(o.proof.merkle_path, cm.proof.merkle_path); - for i in 0..specs.len() { - assert_eq!(o.per_matrix[i].evaluations, rm.per_matrix[i].evaluations); - assert_eq!( - o.per_matrix[i].evaluations_sym, - rm.per_matrix[i].evaluations_sym - ); - assert_eq!(o.per_matrix[i].evaluations, cm.per_matrix[i].evaluations); - assert_eq!( - o.per_matrix[i].evaluations_sym, - cm.per_matrix[i].evaluations_sym - ); - } - } - - // Main-split sub-range: a RowMajorNatural over a wider buffer with a - // leading prefix (`col_start = prefix`) must match an owned matrix built - // over ONLY the committed trailing columns. - let (lh, prefix, w) = (4usize, 2usize, 3usize); - let num_rows = 1usize << lh; - let full = make_columns(prefix + w, num_rows, 42); - let full_natural = row_major_natural(&full, num_rows); - let sub_cols: Vec> = full[prefix..].to_vec(); - let sub_owned = owned(vec![(row_major_bit_reversed(&sub_cols, num_rows), lh, w)]); - let split_src: Vec> = - vec![BorrowedMatrix::RowMajorNatural { - data: full_natural.as_slice(), - stride: prefix + w, - col_start: prefix, - width: w, - log_height: lh, - }]; - let sub_owned_mmcs = Mmcs::commit(&sub_owned); - let split_mmcs = Mmcs::commit(&split_src); - assert_eq!( - sub_owned_mmcs.root(), - split_mmcs.root(), - "main-split (col_start>0) root must match the owned sub-range" - ); - for iota in 0..(1usize << (lh - 1)) { - let a = sub_owned_mmcs.open_batch(iota, &sub_owned); - let b = split_mmcs.open_batch(iota, &split_src); - assert_eq!(a.per_matrix[0].evaluations, b.per_matrix[0].evaluations); - assert_eq!( - a.per_matrix[0].evaluations_sym, - b.per_matrix[0].evaluations_sym - ); - } - } - - /// ★ The index-convention control (the module's "HARD PRECONDITION" section). - /// - /// A round whose tallest matrix is SHORTER than the FRI's tallest is the case - /// where the two index conventions disagree: `verify_batch` consumes the LOW - /// `h_max_round - 1` bits of whatever index it is handed, while a matrix - /// inside the tree is located by the HIGH bits of the FRI index. This asserts - /// three things about that case: - /// - /// 1. honest-path control — the correctly reduced index verifies; - /// 2. a tampered row of a SHORT (injected) matrix is rejected, so the low-bits - /// walk really does authenticate the short matrices at the reduced index; - /// 3. handing the un-reduced FRI index straight in is rejected — the misuse is - /// detectable, not silently accepted at some other leaf. - /// - /// A tamper control on the tallest matrix alone would pass under either - /// convention and catch none of this. - #[test] - fn short_round_low_bit_convention_is_exercised() { - // A hypothetical FRI over a 2^6 domain: iota_fri in [0, 2^5). - let h_max_fri = 6usize; - // This round's matrices are shorter: heights {4, 2}. - let (h_tall, h_short) = (4usize, 2usize); - let (w_tall, w_short) = (3usize, 2usize); - let tall = row_major_bit_reversed(&make_columns(w_tall, 1 << h_tall, 77), 1 << h_tall); - let short = row_major_bit_reversed(&make_columns(w_short, 1 << h_short, 88), 1 << h_short); - - let src = owned(vec![(tall, h_tall, w_tall), (short, h_short, w_short)]); - let mmcs = Mmcs::commit(&src); - let heights = [h_tall, h_short]; - let widths = [w_tall, w_short]; - assert_eq!(mmcs.h_max(), h_tall, "the round's h_max is below the FRI's"); - - // The reduction the caller owes: iota_round = iota_fri >> (h_fri - h_round). - let shift = h_max_fri - h_tall; - // Pick a FRI index whose low bits differ from the reduced index's, so the - // two conventions genuinely disagree here. - let iota_fri = 0b10110usize; - let iota_round = iota_fri >> shift; - assert_ne!( - iota_fri & ((1 << (h_tall - 1)) - 1), - iota_round, - "the test index must distinguish the low-bit and high-bit conventions" - ); - - // (1) Honest-path control at the reduced index. - let opening = mmcs.open_batch(iota_round, &src); - assert!( - Mmcs::verify_batch(&mmcs.root(), iota_round, &opening, &heights, &widths), - "the correctly reduced index must verify" - ); - - // (2) Tamper the SHORT (injected) matrix — the matrix a tall-only control - // would never touch, and the one the disagreeing conventions move. - let mut tampered = mmcs.open_batch(iota_round, &src); - tampered.per_matrix[1].evaluations[0] = - &tampered.per_matrix[1].evaluations[0] + &FE::from(1u64); - assert!( - !Mmcs::verify_batch(&mmcs.root(), iota_round, &tampered, &heights, &widths), - "a tampered SHORT-matrix row must be rejected at the reduced index" - ); - - // (3) The misuse: hand the un-reduced FRI index in. It is out of this - // tree's range, so the range guard rejects it rather than walking to some - // unrelated leaf. - assert!( - iota_fri >= 1usize << (h_tall - 1), - "the un-reduced index is outside this round's leaf range" - ); - assert!( - !Mmcs::verify_batch(&mmcs.root(), iota_fri, &opening, &heights, &widths), - "an un-reduced FRI index must be rejected, not accepted at another leaf" - ); - - // And an in-range index that is simply the wrong leaf is rejected too, so - // the guard is not the only thing standing between the two conventions. - let wrong_but_in_range = iota_fri & ((1 << (h_tall - 1)) - 1); - assert!( - !Mmcs::verify_batch( - &mmcs.root(), - wrong_but_in_range, - &opening, - &heights, - &widths - ), - "an opening replayed at the wrong in-range leaf must be rejected" - ); - } - - /// The malformed-input surface of `verify_batch`: every shape error returns - /// `false` rather than panicking, since a verifier calls this on proof data. - #[test] - fn verify_batch_rejects_malformed_shapes_without_panicking() { - let h = 3usize; - let w = 2usize; - let data = row_major_bit_reversed(&make_columns(w, 1 << h, 4), 1 << h); - let src = owned(vec![(data, h, w)]); - let mmcs = Mmcs::commit(&src); - let root = mmcs.root(); - let opening = mmcs.open_batch(1, &src); - - assert!(Mmcs::verify_batch(&root, 1, &opening, &[h], &[w])); - // Mismatched metadata lengths. - assert!(!Mmcs::verify_batch(&root, 1, &opening, &[h, h], &[w])); - assert!(!Mmcs::verify_batch(&root, 1, &opening, &[h], &[w, w])); - // Empty metadata. - assert!(!Mmcs::verify_batch(&root, 1, &opening, &[], &[])); - // A height that would overflow the level shift. - assert!(!Mmcs::verify_batch( - &root, - 1, - &opening, - &[usize::BITS as usize], - &[w] - )); - // An index past this tree's leaf count. - assert!(!Mmcs::verify_batch( - &root, - 1usize << (h - 1), - &opening, - &[h], - &[w] - )); - // A path of the wrong length. - let mut short_path = opening.clone(); - short_path.proof.merkle_path.pop(); - assert!(!Mmcs::verify_batch(&root, 1, &short_path, &[h], &[w])); - } - - /// Wraps a source and records, per matrix, the first and last global access - /// sequence number, plus a residency model the caller drives. `Mutex` / - /// atomics (not `Cell`) because both `commit` and the streaming builder read - /// the source from rayon workers. - struct Tracing<'a, E: IsField> { - inner: &'a OwnedMatrices, - clock: AtomicUsize, - window: Mutex>, - /// The residency model: which matrices the caller says it is holding. - resident: Vec, - live: AtomicUsize, - peak: AtomicUsize, - /// Rows served for a matrix the caller had already dropped. Any nonzero - /// count means the access pattern does not fit the residency policy. - reads_while_dropped: AtomicUsize, - } - - impl<'a, E: IsField> Tracing<'a, E> { - fn new(inner: &'a OwnedMatrices) -> Self { - let n = inner.num_matrices(); - Self { - inner, - clock: AtomicUsize::new(0), - window: Mutex::new(vec![(usize::MAX, 0); n]), - resident: (0..n).map(|_| AtomicBool::new(false)).collect(), - live: AtomicUsize::new(0), - peak: AtomicUsize::new(0), - reads_while_dropped: AtomicUsize::new(0), - } - } - - /// Declare every matrix held for the whole build — the only policy - /// `MixedMmcs::commit` can be served under. - fn materialize_all(&self) { - for m in 0..self.inner.num_matrices() { - self.materialize(m); - } - } - - fn materialize(&self, m: usize) { - if !self.resident[m].swap(true, Ordering::SeqCst) { - let live = self.live.fetch_add(1, Ordering::SeqCst) + 1; - self.peak.fetch_max(live, Ordering::SeqCst); - } - } - - fn drop_matrix(&self, m: usize) { - if self.resident[m].swap(false, Ordering::SeqCst) { - self.live.fetch_sub(1, Ordering::SeqCst); - } - } - - fn windows(self) -> (Vec<(usize, usize)>, usize, usize) { - let peak = self.peak.load(Ordering::SeqCst); - let dropped_reads = self.reads_while_dropped.load(Ordering::SeqCst); - let windows = self.window.into_inner().expect("uncontended after commit"); - (windows, peak, dropped_reads) - } - } - - impl LeafSource for Tracing<'_, E> { - fn num_matrices(&self) -> usize { - self.inner.num_matrices() - } - fn log_height(&self, m: usize) -> usize { - self.inner.log_height(m) - } - fn width(&self, m: usize) -> usize { - self.inner.width(m) - } - fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { - if !self.resident[m].load(Ordering::SeqCst) { - self.reads_while_dropped.fetch_add(1, Ordering::SeqCst); - } - let t = self.clock.fetch_add(1, Ordering::SeqCst); - let mut w = self.window.lock().expect("no test thread panics here"); - w[m].0 = w[m].0.min(t); - w[m].1 = w[m].1.max(t); - drop(w); - self.inner.append_row(m, bitrev_row, out); - } - } - - /// Heights {5, 5, 3, 2}: two matrices share the TALLEST height, so the base - /// group actually batches — which is the group the memory claim is about. - fn residency_fixture() -> ([(usize, usize, u64); 4], OwnedMatrices) { - let specs = [(5usize, 2usize, 1u64), (5, 3, 2), (3, 1, 3), (2, 4, 4)]; - let inner = owned( - specs - .iter() - .map(|&(lh, w, seed)| { - ( - row_major_bit_reversed(&make_columns(w, 1 << lh, seed), 1 << lh), - lh, - w, - ) - }) - .collect(), - ); - (specs, inner) - } - - /// The streaming builder is not a second implementation of the tree: it - /// finishes through the same climb `commit` does. This pins the consequence — - /// same root, same layers, same openings — so a future change that forked the - /// two would fail here rather than at a verifier three modules away. - #[test] - fn streaming_builder_commits_the_same_tree_as_commit() { - let (specs, inner) = residency_fixture(); - let dims: Vec<(usize, usize)> = specs.iter().map(|&(lh, w, _)| (lh, w)).collect(); - - let mut builder = StreamingMmcsBuilder::::new(&dims); - for m in 0..dims.len() { - builder.absorb(&inner, m); - } - let streamed = builder.finish(); - let reference = Mmcs::commit(&inner); - - assert_eq!( - streamed.root(), - reference.root(), - "the streamed root must equal the one-shot root" - ); - assert_eq!(streamed.h_max(), reference.h_max()); - assert_eq!(streamed.dims(), reference.dims()); - - let heights: Vec = specs.iter().map(|&(lh, _, _)| lh).collect(); - let widths: Vec = specs.iter().map(|&(_, w, _)| w).collect(); - for iota in 0..1usize << (streamed.h_max() - 1) { - let opening = streamed.open_batch(iota, &inner); - assert!( - Mmcs::verify_batch(&streamed.root(), iota, &opening, &heights, &widths), - "an opening of the streamed tree must verify at iota {iota}" - ); - assert_eq!( - opening.proof.merkle_path, - reference.open_batch(iota, &inner).proof.merkle_path, - "the authentication path at iota {iota} must be the same path" - ); - } - } - - /// ★ The acceptance test for the batched commit's memory claim. - /// - /// `commit`'s contract is per height GROUP: it reads a group inside one - /// contiguous phase, so a caller may drop the group before the next. That is - /// not enough. Within the tallest group the leaf is one hash over every - /// matrix's concatenated row pair, so `commit` reads all of them at every - /// leaf — their access windows OVERLAP, and a caller has to hold the whole - /// group. On a real epoch the tallest group is most of the tables, so that is - /// `O(N)` resident at the base layer: the memory batching exists to remove, - /// given back. - /// - /// The streaming builder's windows are pairwise disjoint across ALL matrices, - /// same-height ones included, so the residency policy "materialize, absorb, - /// drop" serves it with exactly ONE matrix live. Both halves are traced here; - /// the second is the property the batched R1 / aux / parts commits must be - /// built on, and the first is what makes it a real difference rather than a - /// restatement. - #[test] - fn streaming_builder_serves_the_base_group_without_holding_it() { - let (specs, inner) = residency_fixture(); - let dims: Vec<(usize, usize)> = specs.iter().map(|&(lh, w, _)| (lh, w)).collect(); - let base_group: Vec = (0..specs.len()).filter(|&m| specs[m].0 == 5).collect(); - assert!( - base_group.len() > 1, - "the fixture must batch more than one matrix at the tallest height" - ); - - // --- What `commit` requires: the whole group resident at once. --- - let tracing = Tracing::new(&inner); - tracing.materialize_all(); - let commit_root = Mmcs::commit(&tracing).root(); - let (commit_windows, commit_peak, commit_dropped_reads) = tracing.windows(); - assert_eq!(commit_dropped_reads, 0, "the control held everything"); - assert_eq!( - commit_peak, - specs.len(), - "serving `commit` needs every matrix resident" - ); - for (i, &m) in base_group.iter().enumerate() { - for &n in &base_group[i + 1..] { - let (fm, lm) = commit_windows[m]; - let (fn_, ln) = commit_windows[n]; - assert!( - fm <= ln && fn_ <= lm, - "matrices {m} and {n} share the base height, so `commit` must \ - read them in OVERLAPPING windows [{fm},{lm}] / [{fn_},{ln}] — \ - if this ever stops holding, the escape below is no longer the \ - thing that buys the memory" - ); - } - } - - // --- What the streaming builder requires: one matrix at a time. --- - let tracing = Tracing::new(&inner); - let mut builder = StreamingMmcsBuilder::::new(&dims); - for m in 0..dims.len() { - tracing.materialize(m); - builder.absorb(&tracing, m); - tracing.drop_matrix(m); - } - let streamed_root = builder.finish().root(); - let (streamed_windows, streamed_peak, streamed_dropped_reads) = tracing.windows(); - - assert_eq!( - streamed_root, commit_root, - "the escape must not change what is committed" - ); - assert_eq!( - streamed_dropped_reads, 0, - "no row may be read after the caller dropped its matrix" - ); - assert_eq!( - streamed_peak, - 1, - "the base height group must be served with ONE matrix resident, not \ - {} — this is the batched commit's whole memory claim", - specs.len() - ); - for (m, &(first, last)) in streamed_windows.iter().enumerate() { - assert!(first <= last, "matrix {m} was never read"); - for (n, &(fn_, ln)) in streamed_windows.iter().enumerate().skip(m + 1) { - assert!( - last < fn_ || ln < first, - "matrices {m} and {n} were read in overlapping windows \ - [{first},{last}] / [{fn_},{ln}] — the builder must finish one \ - matrix before the next is needed, at EVERY height" - ); - } - } - } - - /// A declared matrix that never arrives would leave its group's leaves having - /// absorbed less than the shape says, committing a tree no verifier rebuilds. - /// The builder refuses rather than producing it. - #[test] - #[should_panic(expected = "of 4 declared matrices were absorbed")] - fn finishing_with_a_matrix_missing_panics() { - let (specs, inner) = residency_fixture(); - let dims: Vec<(usize, usize)> = specs.iter().map(|&(lh, w, _)| (lh, w)).collect(); - let mut builder = StreamingMmcsBuilder::::new(&dims); - for m in 0..dims.len() - 1 { - builder.absorb(&inner, m); - } - builder.finish(); - } - - /// The incremental leaf hasher's whole contract: where the updates fall must - /// not show. Checked at every split point of a leaf, and for the extension - /// field the aux and composition matrices actually use — a framing bug that - /// only appeared at an element boundary would slip past a base-field check. - #[test] - fn leaf_hasher_splits_anywhere_and_matches_hash_data() { - use crypto::merkle_tree::traits::IsLeafHasher; - use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Fp3; - - fn check(leaf: Vec>) - where - FieldElement: AsBytes + Sync + Send, - { - let expected = - <::Batched as IsMerkleTreeBackend>::hash_data( - &leaf, - ); - for split in 0..=leaf.len() { - let mut hasher = - <::Batched as IsStreamingLeafBackend>::leaf_hasher(); - hasher.update(&leaf[..split]); - hasher.update(&leaf[split..]); - assert_eq!( - hasher.finalize(), - expected, - "splitting the leaf at {split} changed the digest" - ); - } - // Three updates, so an implementation that only ever saw two would not - // pass by accident. - let mut hasher = - <::Batched as IsStreamingLeafBackend>::leaf_hasher(); - for element in &leaf { - hasher.update(core::slice::from_ref(element)); - } - assert_eq!(hasher.finalize(), expected, "element-at-a-time must agree"); - } - - check::((1u64..=9).map(FE::from).collect()); - check::( - (1u64..=9) - .map(|i| { - FieldElement::::new([FE::from(i), FE::from(i * 7 + 1), FE::from(i * 13)]) - }) - .collect(), - ); - } - - /// The memory contract from the module's "what the caller may drop" section, - /// made falsifiable: `commit` reads each height group's rows inside ONE - /// contiguous window of the build, and the windows run in descending height - /// order. A rewrite that materialized every matrix up front, or that revisited - /// a group after moving on, would fail here. - #[test] - fn commit_reads_each_height_group_in_one_contiguous_phase() { - // Heights {5, 5, 3, 2}: two groups sharing the base layer, two injected. - let specs = [(5usize, 2usize, 1u64), (5, 3, 2), (3, 1, 3), (2, 4, 4)]; - let inner = owned( - specs - .iter() - .map(|&(lh, w, seed)| { - ( - row_major_bit_reversed(&make_columns(w, 1 << lh, seed), 1 << lh), - lh, - w, - ) - }) - .collect(), - ); - let tracing = Tracing::new(&inner); - tracing.materialize_all(); - - let traced_root = Mmcs::commit(&tracing).root(); - assert_eq!( - traced_root, - Mmcs::commit(&inner).root(), - "tracing must not change what is committed" - ); - - let (windows, _peak, _dropped) = tracing.windows(); - for (m, (first, last)) in windows.iter().enumerate() { - assert!(*first <= *last, "matrix {m} was never read"); - } - - // Same-height matrices share a window; different heights must not overlap, - // and taller groups must come first. - for (m, &(fm, lm)) in windows.iter().enumerate() { - for (n, &(fn_, ln)) in windows.iter().enumerate() { - if specs[m].0 <= specs[n].0 { - continue; - } - assert!( - lm < fn_ || ln < fm, - "matrices {m} (h={}) and {n} (h={}) were read in overlapping \ - windows [{fm},{lm}] / [{fn_},{ln}] — a height group must be \ - readable and then droppable", - specs[m].0, - specs[n].0 - ); - assert!( - lm < fn_, - "the taller matrix {m} (h={}) must be read before the shorter \ - {n} (h={})", - specs[m].0, - specs[n].0 - ); - } - } - } -} diff --git a/crypto/stark/src/fri/mod.rs b/crypto/stark/src/fri/mod.rs index 2241a79c0..0458b9b93 100644 --- a/crypto/stark/src/fri/mod.rs +++ b/crypto/stark/src/fri/mod.rs @@ -1,7 +1,6 @@ pub mod fri_commitment; pub mod fri_decommit; pub(crate) mod fri_functions; -pub mod mmcs; pub(crate) mod terminal; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; diff --git a/crypto/stark/src/lib.rs b/crypto/stark/src/lib.rs index a7edb7996..49e196482 100644 --- a/crypto/stark/src/lib.rs +++ b/crypto/stark/src/lib.rs @@ -3,7 +3,6 @@ #[cfg(all(target_arch = "wasm32", feature = "disk-spill"))] compile_error!("the `disk-spill` feature requires memmap2, which does not compile on wasm32"); -pub mod batched; #[cfg(feature = "debug-checks")] pub mod bus_debug; pub mod commitment; diff --git a/crypto/stark/src/par.rs b/crypto/stark/src/par.rs index 1c2863059..cee693e3f 100644 --- a/crypto/stark/src/par.rs +++ b/crypto/stark/src/par.rs @@ -92,26 +92,3 @@ pub(crate) fn par_try_for_each_mut( slice.iter_mut().try_for_each(f) } } - -/// Run `f(i, &mut item)` for each element of `slice` with its index. Parallel -/// when `feature = "parallel"`, sequential otherwise. -pub(crate) fn par_for_each_mut_indexed( - slice: &mut [T], - f: impl Fn(usize, &mut T) + Sync + Send, -) { - #[cfg(feature = "parallel")] - { - use rayon::prelude::*; - slice - .par_iter_mut() - .enumerate() - .for_each(|(i, item)| f(i, item)); - } - #[cfg(not(feature = "parallel"))] - { - slice - .iter_mut() - .enumerate() - .for_each(|(i, item)| f(i, item)); - } -} diff --git a/prover/src/bin/compute_lfm_registry.rs b/prover/src/bin/compute_lfm_registry.rs index ddc06ef70..79a69a4e9 100644 --- a/prover/src/bin/compute_lfm_registry.rs +++ b/prover/src/bin/compute_lfm_registry.rs @@ -82,14 +82,6 @@ fn main() { artifacts.chip_set.keccak, artifacts.chip_set.blake3 ); println!(" program_id: {},", fmt_bytes(&artifacts.program_id)); - println!(" prep_root: {},", fmt_bytes(&artifacts.prep_root)); - let widths = artifacts - .prep_widths - .iter() - .map(u16::to_string) - .collect::>() - .join(", "); - println!(" prep_widths: [{widths}],"); println!(" }},"); } } diff --git a/prover/src/lfm/blake3_chip_tests.rs b/prover/src/lfm/blake3_chip_tests.rs index 726b6ac88..c263f5249 100644 --- a/prover/src/lfm/blake3_chip_tests.rs +++ b/prover/src/lfm/blake3_chip_tests.rs @@ -1897,83 +1897,6 @@ fn the_census_counts_every_blake3_chunk() { } } -/// The batched preprocessed round expands with the chunks: eleven fixed slot -/// matrices, then ONE per `LFM_BLAKE3` chunk at that chunk's own LDE height — -/// and the shape a verifier reads back rebuilds the pinned root. -/// -/// The round is absorbed in slot order and this chip is the last slot in it, so -/// the chunks land at the end; getting the count or an individual height wrong -/// is not loud (the tree still builds), which is why the rebuild is the -/// assertion rather than the shape alone. -#[test] -fn the_prep_round_expands_with_the_blake3_chunks() { - use super::commit::{PrepRoundBuilder, group_columns, lde_columns}; - use super::registry::PREP_ROUND_SLOTS; - - let opts = options(); - let program = chunked_chain_program(); - let artifacts = build_artifacts(&program, &opts); - let (heights, widths) = artifacts.prep_round_shape(opts.blowup_factor); - - assert_eq!(heights.len(), widths.len()); - assert_eq!( - heights.len(), - PREP_ROUND_SLOTS.len() - 1 + 3, - "eleven fixed slots plus one matrix per chunk" - ); - let blowup_log = (opts.blowup_factor as usize).trailing_zeros() as usize; - for (i, slot) in PREP_ROUND_SLOTS.take(super::airs::BLAKE3_SLOT).enumerate() { - assert_eq!( - heights[i], - artifacts.log_heights[slot] as usize + blowup_log - ); - } - for (c, h) in artifacts.blake3_chunk_log_heights.iter().enumerate() { - assert_eq!( - heights[super::airs::BLAKE3_SLOT + c], - *h as usize + blowup_log, - "chunk {c}: the round's height must be the chunk's LDE height" - ); - assert_eq!( - widths[super::airs::BLAKE3_SLOT + c], - program.groups.blake3.width - ); - } - - let range = super::trace::range_group(); - let fixed = [ - &program.groups.const_, - &program.groups.balu, - &program.groups.xalu, - &program.groups.select, - &program.groups.bitdec, - &program.groups.hash, - &program.groups.keccak, - &program.groups.lanes, - &program.groups.hint, - &program.groups.public, - &range, - ]; - let dims: Vec<(usize, usize)> = heights - .iter() - .copied() - .zip(widths.iter().copied()) - .collect(); - let mut round = PrepRoundBuilder::new(&dims); - for g in fixed.iter() { - round.absorb(&lde_columns(&group_columns(g), &opts)); - } - for c in 0..artifacts.blake3_chunks() { - let g = program.blake3_chunk_group(c); - round.absorb(&lde_columns(&group_columns(&g), &opts)); - } - assert_eq!( - round.finish(), - artifacts.prep_root, - "the shape a verifier reads back must rebuild the pinned root" - ); -} - /// The knob's whole path: a variable VALUE becomes a policy, the policy becomes /// chunks, and the chunked program proves and verifies. The parse itself is /// tested in [`super::chunking`]; this is what says the value reaches the diff --git a/prover/src/lfm/commit.rs b/prover/src/lfm/commit.rs index 866ea7060..1e7881a0d 100644 --- a/prover/src/lfm/commit.rs +++ b/prover/src/lfm/commit.rs @@ -10,7 +10,6 @@ use math::polynomial::Polynomial; use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed_with}; use stark::config::Commitment; -use stark::fri::mmcs::{BorrowedMatrix, StreamingMmcsBuilder}; use stark::proof::options::ProofOptions; use stark::prover::evaluate_polynomial_on_lde_domain; @@ -20,13 +19,9 @@ use super::compiler::ColumnGroup; /// The coset LDE of a column matrix, column-major and in NATURAL order. /// -/// Split out of [`commit_columns`] because two things now consume it: the -/// per-slot row-pair commitment below, and the batched preprocessed round -/// ([`prep_round_root`]), which reads exactly this shape through -/// `BorrowedMatrix::ColMajorNatural`. Computing it once and handing it to both -/// is what keeps the batched root a commitment to *the same* evaluations the -/// per-slot root commits to, rather than to a second, independently built copy -/// of them. +/// Split out of [`commit_columns`] so a caller that needs the evaluations for +/// something else can expand once and commit from the same copy, rather than +/// building a second, independently expanded one. pub fn lde_columns(columns: &[Vec], options: &ProofOptions) -> Vec> { let num_rows = columns.first().map_or(0, Vec::len); let polys: Vec> = columns @@ -82,78 +77,3 @@ pub fn group_columns(group: &ColumnGroup) -> Vec> { pub fn commit_group(group: &ColumnGroup, options: &ProofOptions) -> Commitment { commit_columns(&group_columns(group), options) } - -/// The batched preprocessed round's root: ONE mixed-height MMCS over several -/// slots' LDE matrices, in slot order. -/// -/// # What this is for -/// -/// Under the batched commitment path a query opens ONE authentication path -/// covering every preprocessed matrix, instead of one path per slot. This is -/// the root such a verifier compares against -/// ([`stark::fri::mmcs::MixedMmcs::verify_batch`]), and the registry pins it -/// alongside the per-slot roots it does not replace. -/// -/// # Streaming, deliberately -/// -/// Absorbing through [`StreamingMmcsBuilder`] rather than `MixedMmcs::commit` -/// is what lets the caller expand one slot's LDE, commit it, absorb it and drop -/// it. `commit` reads every matrix of a height group at once, which for the -/// registry builder would mean holding all twelve groups' LDEs simultaneously — -/// a memory regression in a function the king gate and a dozen tests call. -/// -/// # Determinism -/// -/// The tree is a pure function of the matrices AND their order, so the caller -/// must absorb in the same slot order a verifier will present openings in. The -/// heights are LDE heights (`log2(rows * blowup)`), not trace heights — the -/// registry's own `log_heights` are trace heights, and the two differ by -/// `log2(blowup)`. -pub struct PrepRoundBuilder { - builder: StreamingMmcsBuilder, -} - -impl PrepRoundBuilder { - /// Declare the round's shape: `(log_height, width)` per participating slot, - /// in absorption order. `log_height` is the LDE height. - pub fn new(dims: &[(usize, usize)]) -> Self { - Self { - builder: StreamingMmcsBuilder::new(dims), - } - } - - /// Absorb one slot's LDE matrix. The caller may drop it as soon as this - /// returns. - /// - /// # Panics - /// - /// On an empty matrix, or a column length that is not a power of two. - /// Deriving the height as `len.trailing_zeros()` is only the height when the - /// length is a power of two — for anything else it silently reports a - /// SMALLER height (a length of 12 reads as 4), and the round would then - /// commit a tree over a shape nobody declared. This runs at program-build - /// and registry-regeneration time, never on a verify path, so an unusable - /// input is a caller bug and asserting is correct here (unlike on the - /// verifier, where the house rule is to reject rather than panic). - pub fn absorb(&mut self, lde_columns: &[Vec]) { - let len = lde_columns - .first() - .map(Vec::len) - .expect("a participating slot has at least one column"); - assert!( - len.is_power_of_two(), - "an LDE column length must be a power of two, got {len}" - ); - let log_height = len.trailing_zeros() as usize; - let source = vec![BorrowedMatrix::ColMajorNatural { - cols: lde_columns, - log_height, - }]; - self.builder.absorb(&source, 0); - } - - /// The round's root. - pub fn finish(self) -> Commitment { - self.builder.finish().root() - } -} diff --git a/prover/src/lfm/machine_tests.rs b/prover/src/lfm/machine_tests.rs index cc7b48a5d..ec92da94f 100644 --- a/prover/src/lfm/machine_tests.rs +++ b/prover/src/lfm/machine_tests.rs @@ -132,14 +132,6 @@ fn registry_drift_trivial_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } #[test] @@ -252,14 +244,6 @@ fn registry_drift_fri_toy_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } /// The kill-risk-3 instrument on the first real verification program. @@ -556,14 +540,6 @@ fn registry_drift_keccak_chain_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } /// The kill-risk-3 instrument with the keccak family in the set. @@ -833,14 +809,6 @@ fn registry_drift_keccak_sponge_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } #[test] @@ -1509,14 +1477,6 @@ fn registry_drift_transcript_replay_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } /// Pins the emitted SHAPE, which the value tests would only catch indirectly: @@ -2408,14 +2368,6 @@ fn registry_drift_statement_replay_v0_blowup2() { ); assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); - assert_eq!( - entry.prep_root, artifacts.prep_root, - "batched preprocessed-round root drifted" - ); - assert_eq!( - entry.prep_widths, artifacts.prep_widths, - "batched preprocessed-round widths drifted" - ); } #[test] @@ -4995,330 +4947,6 @@ fn every_registry_mask_is_the_programs_own_usage() { } } -// =========================================================================== -// The batched preprocessed round (M-6) -// =========================================================================== - -/// The round's membership is a scope decision, so it is pinned rather than left -/// to be inferred from whichever slots happened to have columns. Widening it — -/// to cover `KECCAK_RC` and `BITWISE` — is a real option with a real cost (every -/// `build_artifacts` call would expand `bitwise`'s 2^20 x 11 table instead of -/// reading a pinned constant), and this test is what makes taking it deliberate. -#[test] -fn the_prep_round_covers_exactly_the_program_groups() { - use crate::lfm::airs::{KECCAK_RND_SLOT, NUM_LFM_CHIPS}; - use crate::lfm::registry::PREP_ROUND_SLOTS; - - let artifacts = build_artifacts(&trivial_program(), &options()); - - assert_eq!( - PREP_ROUND_SLOTS, - 0..12, - "the round covers the twelve program-dependent groups" - ); - assert!( - !PREP_ROUND_SLOTS.contains(&KECCAK_RND_SLOT), - "KECCAK_RND has no preprocessed columns, so it has no leaf in the round" - ); - - // Membership is read from PREP_ROUND_SLOTS and NOWHERE ELSE. An earlier - // draft asserted `inside == (width > 0)` and described a zero width as "how - // a verifier reads 'not in this round'". That is a SECOND, independent - // derivation of a fact the slot list already states, and it is the failure - // shape MMCS-PLAN §3.3 warns about: if the two ever disagreed — a genuinely - // zero-width group, or a non-member slot carrying a width — a prover and a - // verifier would both derive the same wrong round and honest proofs would - // keep verifying with nothing failing. - // - // The widths are still checked, but as an ENCODING property (members are - // non-empty, non-members carry nothing), never as the definition. - for slot in PREP_ROUND_SLOTS { - assert!( - artifacts.prep_widths[slot] > 0, - "slot {slot} is in the round, so it must contribute a non-empty matrix" - ); - } - for slot in (0..NUM_LFM_CHIPS).filter(|s| !PREP_ROUND_SLOTS.contains(s)) { - assert_eq!( - artifacts.prep_widths[slot], 0, - "slot {slot} is outside the round, so the entry carries no width for it — \ - an encoding check, NOT the definition of membership" - ); - } - assert_ne!( - artifacts.prep_root, [0u8; 32], - "the round must actually commit something" - ); -} - -/// ★ The property that makes `prep_root` a commitment to the SAME evaluations -/// the per-slot roots commit to, rather than to an independently built copy. -/// -/// A mixed-height MMCS over ONE matrix is the per-table row-pair tree, by -/// construction and not by coincidence — both finish through the same leaf hash -/// and the same climb. Checking it here pins that the registry's two commitment -/// paths share a leaf encoding; if they ever stopped, `prep_root` would be -/// binding a different parse of the same columns and nothing else would say so. -#[test] -fn a_single_slot_prep_round_equals_that_slots_own_root() { - use crate::lfm::commit::{PrepRoundBuilder, commit_lde_columns, group_columns, lde_columns}; - - let opts = options(); - let program = trivial_program(); - let lde = lde_columns(&group_columns(&program.groups.const_), &opts); - let log_height = lde[0].len().trailing_zeros() as usize; - - let mut round = PrepRoundBuilder::new(&[(log_height, lde.len())]); - round.absorb(&lde); - - assert_eq!( - round.finish(), - commit_lde_columns(&lde), - "a one-matrix batched round must equal the per-slot row-pair tree" - ); -} - -/// Falsification: the round must be sensitive to the data it covers. A root -/// that never moved would satisfy every equality test above while binding -/// nothing. -#[test] -fn a_changed_group_moves_the_prep_root() { - use crate::lfm::commit::{PrepRoundBuilder, group_columns, lde_columns}; - - let opts = options(); - let program = trivial_program(); - let mut columns = group_columns(&program.groups.const_); - - let lde = lde_columns(&columns, &opts); - let log_height = lde[0].len().trailing_zeros() as usize; - let dims = [(log_height, lde.len())]; - let mut round = PrepRoundBuilder::new(&dims); - round.absorb(&lde); - let honest = round.finish(); - - columns[0][0] += crate::tables::types::FE::one(); - let tampered_lde = lde_columns(&columns, &opts); - let mut round = PrepRoundBuilder::new(&dims); - round.absorb(&tampered_lde); - - assert_ne!( - round.finish(), - honest, - "a changed preprocessed value must move the batched round's root" - ); -} - -/// ★ The slot-to-table map is NOT the identity, and the registry cannot show -/// that on its own. -/// -/// Every registered program has `keccak_rnd_chunks == 1`, which makes slot and -/// table indices coincide for all fifteen slots. A map hard-coded to the -/// identity would therefore pass every registry-derived test in this file. This -/// drives it at a chunk count above one, which is the only place the shift is -/// observable. -#[test] -fn the_slot_to_table_map_is_not_the_identity_beyond_one_chunk() { - use crate::lfm::airs::{ChipSet, KECCAK_RND_SLOT}; - use crate::lfm::registry::slot_of_table; - - // The FULL set: this test drives the chunking axis alone; the chip-mask - // axis (absent families leaving holes) is driven by the masked epoch in - // `a_batched_lfm_epoch_is_refused_for_the_round_coverage_gap`. - const FULL: ChipSet = ChipSet::FULL; - - // One chunk: the degenerate case the whole registry lives in. - assert_eq!(slot_of_table(12, 1, 1, FULL), Some(KECCAK_RND_SLOT)); - assert_eq!(slot_of_table(13, 1, 1, FULL), Some(13)); - assert_eq!(slot_of_table(14, 1, 1, FULL), Some(14)); - assert_eq!( - slot_of_table(15, 1, 1, FULL), - None, - "past the end of the set" - ); - - // Three chunks: KECCAK_RC moves from table 13 to table 15. An identity map - // would answer 13 here and be wrong by exactly the off-by-one this exists - // to catch. - for table in 12..15 { - assert_eq!( - slot_of_table(table, 3, 1, FULL), - Some(KECCAK_RND_SLOT), - "table {table} is a KECCAK_RND copy at three chunks" - ); - } - assert_eq!( - slot_of_table(15, 3, 1, FULL), - Some(13), - "KECCAK_RC shifted by chunks" - ); - assert_eq!( - slot_of_table(16, 3, 1, FULL), - Some(14), - "BITWISE shifted by chunks" - ); - assert_ne!( - slot_of_table(13, 3, 1, FULL), - Some(13), - "the map must not be the identity once more than one chunk exists" - ); -} - -/// ★ The compaction refuses the real LFM epoch, loudly, because the round is -/// partial. -/// -/// `KECCAK_RC` and `BITWISE` are preprocessed AIRs, so an LFM epoch's prep round -/// has fourteen contributing matrices while `PREP_ROUND_SLOTS` covers twelve. -/// Returning a twelve-entry slice for a fourteen-matrix round would describe a -/// different round than `prep_root` commits — so the answer must be `None`, and -/// the honest-path arm below shows `None` is discrimination and not a stub. -#[test] -fn the_width_compaction_rejects_a_round_it_does_not_cover() { - use crate::lfm::airs::ChipSet; - use crate::lfm::registry::{PREP_ROUND_SLOTS, pinned_prep_widths}; - use stark::batched::shape::RoundShape; - - let artifacts = build_artifacts(&trivial_program(), &options()); - - // The shape a FULL-set LFM epoch has: the twelve groups plus KECCAK_RC and - // BITWISE, at one KECCAK_RND chunk. - let real = RoundShape { - tables: (0..12).chain([13, 14]).collect(), - dims: (0..14).map(|_| (4usize, 1usize)).collect(), - }; - assert_eq!( - pinned_prep_widths(&real, &artifacts.prep_widths, 1, 1, ChipSet::FULL), - None, - "the round does not cover KECCAK_RC/BITWISE, so it must refuse rather than \ - hand back a slice describing a different round" - ); - - // Honest-path control: restricted to the slots the round DOES cover, the - // compaction succeeds and reproduces the entry's widths in table order. - let covered = RoundShape { - tables: PREP_ROUND_SLOTS.collect(), - dims: PREP_ROUND_SLOTS.map(|_| (4usize, 1usize)).collect(), - }; - let expected: Vec = PREP_ROUND_SLOTS - .map(|s| artifacts.prep_widths[s] as usize) - .collect(); - assert_eq!( - pinned_prep_widths(&covered, &artifacts.prep_widths, 1, 1, ChipSet::FULL), - Some(expected), - "honest-path control: a round inside PREP_ROUND_SLOTS must compact cleanly" - ); -} - -/// The compaction indexes `tables`, so reordering the round reorders the slice. -/// A filter-the-zeros implementation would return ascending slot order whatever -/// `tables` said, and would pass every other test in this file. -#[test] -fn the_width_compaction_follows_table_order_not_slot_order() { - use crate::lfm::registry::pinned_prep_widths; - use stark::batched::shape::RoundShape; - - let artifacts = build_artifacts(&trivial_program(), &options()); - - let forward: Vec = (0..4).collect(); - let reversed: Vec = (0..4).rev().collect(); - let dims: Vec<(usize, usize)> = (0..4).map(|_| (4usize, 1usize)).collect(); - - let a = pinned_prep_widths( - &RoundShape { - tables: forward, - dims: dims.clone(), - }, - &artifacts.prep_widths, - 1, - 1, - crate::lfm::airs::ChipSet::FULL, - ) - .expect("slots 0..4 are covered"); - let b = pinned_prep_widths( - &RoundShape { - tables: reversed, - dims, - }, - &artifacts.prep_widths, - 1, - 1, - crate::lfm::airs::ChipSet::FULL, - ) - .expect("slots 0..4 are covered"); - - let mut a_rev = a.clone(); - a_rev.reverse(); - assert_eq!(b, a_rev, "the slice must follow `tables` order"); - assert_ne!( - a, b, - "the fixture's first four slots must have distinct widths, or this test \ - cannot tell the two orders apart" - ); -} - -/// The shape a batched verifier reads back must be the shape the round was -/// built with. Two derivations of the same thing are how the LDE-vs-trace -/// height distinction gets lost: `prep_round_dims` is one function with two -/// callers precisely so this can be asserted rather than hoped for. -#[test] -fn the_prep_round_shape_matches_what_was_committed() { - use crate::lfm::commit::PrepRoundBuilder; - use crate::lfm::commit::{group_columns, lde_columns}; - use crate::lfm::registry::PREP_ROUND_SLOTS; - - let opts = options(); - let program = trivial_program(); - let artifacts = build_artifacts(&program, &opts); - let (heights, widths) = artifacts.prep_round_shape(opts.blowup_factor); - - assert_eq!(heights.len(), widths.len()); - assert_eq!( - heights.len(), - PREP_ROUND_SLOTS.len(), - "every program group participates in this fixture" - ); - - // Heights are LDE heights, not trace heights — the distinction this shape - // exists to get right. - let blowup_log = (opts.blowup_factor as usize).trailing_zeros() as usize; - for (i, slot) in PREP_ROUND_SLOTS.enumerate() { - assert_eq!( - heights[i], - artifacts.log_heights[slot] as usize + blowup_log, - "slot {slot}: the round's height must be the LDE height" - ); - } - - // And rebuilding the round from that shape reproduces the pinned root. - let groups = [ - &program.groups.const_, - &program.groups.balu, - &program.groups.xalu, - &program.groups.select, - &program.groups.bitdec, - &program.groups.hash, - &program.groups.keccak, - &program.groups.lanes, - &program.groups.hint, - &program.groups.public, - &crate::lfm::trace::range_group(), - &program.groups.blake3, - ]; - let dims: Vec<(usize, usize)> = heights - .iter() - .copied() - .zip(widths.iter().copied()) - .collect(); - let mut round = PrepRoundBuilder::new(&dims); - for g in groups.iter() { - round.absorb(&lde_columns(&group_columns(g), &opts)); - } - assert_eq!( - round.finish(), - artifacts.prep_root, - "the shape a verifier reads back must rebuild the pinned root" - ); -} - /// M-7's entry point must agree with the seven-argument form it delegates to — /// on an honest proof, and on a tampered digest. Without the negative this /// would pass for a function that returned `true` unconditionally. diff --git a/prover/src/lfm/mod.rs b/prover/src/lfm/mod.rs index 4104cabff..bf7f1e4ae 100644 --- a/prover/src/lfm/mod.rs +++ b/prover/src/lfm/mod.rs @@ -60,7 +60,7 @@ pub mod word; pub use airs::{LfmAirs, NUM_LFM_CHIPS, num_lfm_airs}; pub use builder::{ArenaSchema, LfmBuilder, LfmProgramSource}; pub use chunking::{KECCAK_RND_MAX_CHUNK_ROWS, KeccakChunking}; -pub use commit::{PrepRoundBuilder, commit_columns, commit_group, commit_lde_columns, lde_columns}; +pub use commit::{commit_columns, commit_group, commit_lde_columns, lde_columns}; pub use compiler::{ColumnGroup, LfmColumnGroups, LfmProgram, compile}; pub use executor::{LfmExecError, LfmExecution, LfmRecords, execute}; pub use hash::{HasherKind, LfmHasher, TestPermutation}; diff --git a/prover/src/lfm/proof.rs b/prover/src/lfm/proof.rs index 08ae5fa42..6b7d70c28 100644 --- a/prover/src/lfm/proof.rs +++ b/prover/src/lfm/proof.rs @@ -236,20 +236,8 @@ pub fn lfm_verify( /// `verify_against` takes seven separate pieces of program shape, so every new /// thing the registry pins would change its signature and every call site with /// it. Taking the struct means a field added to `LfmArtifacts` reaches the -/// verifier without moving anyone — `prep_root` and `prep_widths` (M-6) were the -/// first, and `prover/tests/d0_king_gate.rs` compiles unchanged across their -/// arrival because of it. -/// -/// # ⚠ What it does NOT do yet -/// -/// It does not check `prep_root`. The LFM machine proves and verifies a -/// per-table [`MultiProof`], whose openings are authenticated against the -/// per-slot `roots`; `prep_root` is a second commitment over those same -/// preprocessed matrices, gathered into one multi-matrix round, and nothing -/// reads it. It is plumbing with no consumer, and saying otherwise would -/// overstate what a passing verification means. -/// -/// The shape it commits to is [`LfmArtifacts::prep_round_shape`]. +/// verifier without moving anyone, and `prover/tests/d0_king_gate.rs` compiles +/// unchanged across such an arrival because of it. pub fn verify_against_artifacts( artifacts: &LfmArtifacts, proof: &MultiProof, diff --git a/prover/src/lfm/registry.rs b/prover/src/lfm/registry.rs index a61cf9f16..b3eabacb6 100644 --- a/prover/src/lfm/registry.rs +++ b/prover/src/lfm/registry.rs @@ -17,13 +17,9 @@ use stark::proof::options::ProofOptions; use crate::tables::{bitwise, keccak_rc}; -use stark::batched::shape::RoundShape; +use super::airs::{BLAKE3_SLOT, ChipSet, NUM_LFM_CHIPS, blake3_chunk_rows}; -use super::airs::{ - BLAKE3_SLOT, ChipSet, KECCAK_RC_SLOT, KECCAK_RND_SLOT, KECCAK_SLOT, NUM_LFM_CHIPS, - blake3_chunk_rows, -}; -use super::commit::{PrepRoundBuilder, commit_lde_columns, group_columns, lde_columns}; +use super::commit::{commit_lde_columns, group_columns, lde_columns}; use super::compiler::LfmProgram; use super::hash::HasherKind; use super::statement::lfm_program_id; @@ -96,15 +92,6 @@ pub struct LfmRegistryEntry { /// the proof. See [`ChipSet`]. pub chip_set: ChipSet, pub program_id: Commitment, - /// The batched preprocessed round's root — ONE mixed-height MMCS over the - /// participating slots' matrices. See [`PREP_ROUND_SLOTS`] for which, and - /// `LfmArtifacts::prep_root` for what it does and does not replace. - pub prep_root: Commitment, - /// Committed column count per slot, `0` for a slot outside the round. This - /// is the `widths` a batched verifier must pass to - /// `MixedMmcs::verify_batch`, and it is program shape — derived here, never - /// read off a proof. - pub prep_widths: [u16; NUM_LFM_CHIPS], } impl LfmRegistryEntry { @@ -126,8 +113,6 @@ impl LfmRegistryEntry { hasher: self.hasher, chip_set: self.chip_set, program_id: self.program_id, - prep_root: self.prep_root, - prep_widths: self.prep_widths, } } } @@ -158,57 +143,9 @@ pub struct LfmArtifacts { /// compiled groups at bless time. See [`ChipSet`]. pub chip_set: ChipSet, pub program_id: Commitment, - /// The batched preprocessed round's root: ONE mixed-height MMCS over the - /// [`PREP_ROUND_SLOTS`] matrices, committing the SAME evaluations `roots` - /// commits individually. - /// - /// ⚠ **It replaces nothing yet.** The LFM machine proves and verifies - /// through `multi_prove` / `multi_verify_views`, which read `roots`. This is - /// the value a batched verifier would compare against once that switch is - /// made; until then it is pinned and drift-tested, and nothing consumes it. - /// - /// ⚠ **It does not cover every slot.** See [`PREP_ROUND_SLOTS`]: slots - /// outside it keep their individual `roots` entry as the only thing binding - /// them, and a batched verifier must go on checking those separately. - /// Consolidating a per-table check into one comparison is exactly where - /// coverage goes missing (MMCS-PLAN §3.3). - /// - /// ⚠ **It is NOT folded into `program_id`.** Doing so would move all six - /// blessed digests, which this change is required not to do. The - /// consequence is that the recursion statement does not yet attest to it; - /// folding it in belongs to the next deliberate re-bless. - pub prep_root: Commitment, - /// Committed column count per slot, `0` outside the round. - pub prep_widths: [u16; NUM_LFM_CHIPS], } impl LfmArtifacts { - /// The widths a batched verifier needs for `prep`, or `None` when this - /// program's round does not cover it. - /// - /// This is the whole bridge between the registry's per-slot storage and the - /// contributing-matrix slice `stark::batched::shape::PinnedPrep` takes. The - /// slice is returned owned rather than as a `PinnedPrep`, because that type - /// borrows its widths and the caller has to own them for the duration of the - /// verify: - /// - /// ```ignore - /// let widths = artifacts.pinned_prep_widths(&shape.prep)?; - /// let pin = PinnedPrep { root: &artifacts.prep_root, widths: &widths }; - /// ``` - /// - /// ⚠ Today this returns `None` for every real LFM epoch — see - /// [`PREP_ROUND_SLOTS`]. That is the honest answer, not a stub. - pub fn pinned_prep_widths(&self, prep: &RoundShape) -> Option> { - pinned_prep_widths( - prep, - &self.prep_widths, - self.keccak_rnd_chunks, - self.blake3_chunks(), - self.chip_set, - ) - } - /// `LFM_BLAKE3` instances this program COMMITS — never zero, since slot 11's /// group is committed even for a program that never compresses. /// @@ -223,228 +160,6 @@ impl LfmArtifacts { } } -/// The slots the batched preprocessed round covers: the twelve -/// program-dependent column groups (0–11). -/// -/// ★ Slot 11 contributes one MATRIX PER `LFM_BLAKE3` CHUNK, so a chunked -/// program's round has eleven fixed matrices plus `n` — see -/// [`prep_round_dims`], which expands the slot in place. The membership rule is -/// still this range and nothing else. -/// -/// # Why not all fifteen -/// -/// - **Slot 12 (`KECCAK_RND`)** has no preprocessed columns at all — there is -/// nothing to commit, and a mixed-height MMCS has no leaf for a height-0 -/// matrix. -/// - **Slots 13–14 (`KECCAK_RC`, `BITWISE`)** are owned by `tables/`, and their -/// commitments are STATICALLY PINNED precisely so nothing recomputes them: -/// `bitwise` is 2^20 rows by 11 columns, so putting it in this round would -/// make every `build_artifacts` call — including the king gate's and a dozen -/// tests' — expand a ~2^21 x 11 LDE it currently gets for free from a -/// constant. The round therefore covers exactly the groups `build_artifacts` -/// already materializes, and costs nothing extra. -/// -/// This is a scope decision, not a law: a round covering all fourteen -/// committing slots is implementable, and what it costs is one full expansion -/// of the two production tables per call. `the_prep_round_covers_exactly_the_program_groups` -/// pins the current set so widening it is a deliberate act. -/// -/// ⚠ **Consequence, and it is the reason [`pinned_prep_widths`] exists.** The -/// BATCHED path derives its preprocessed round from the AIR SET, and there -/// `KECCAK_RC` and `BITWISE` are preprocessed AIRs (9 and 11 precomputed -/// columns), so an LFM epoch's prep round has FOURTEEN contributing matrices -/// while this round covers twelve. [`prep_root`](LfmArtifacts::prep_root) is -/// therefore NOT the epoch's batched preprocessed root and must not be handed to -/// `stark::batched::shape::PinnedPrep` as one. Widening this range to cover the -/// two production tables is the prerequisite for that, and it is M-8's, not -/// M-6's. -pub const PREP_ROUND_SLOTS: core::ops::Range = 0..12; - -/// The batched preprocessed round's `(log_height, width)` per participating -/// slot, in slot order — the shape both the builder and a verifier need. -/// -/// ★ **The heights are LDE heights, `log_heights + log2(blowup)`.** The -/// registry's `log_heights` are TRACE heights, and a mixed-height MMCS is -/// indexed by the committed matrix's height, which is the LDE's. Getting this -/// wrong is not loud: every height would be uniformly too small, the tree would -/// still build, and openings would authenticate at leaves the FRI join never -/// checks. One derivation — used by `build_artifacts_with_hasher` to declare the -/// round and by [`LfmArtifacts::prep_round_shape`] to describe it — is what -/// stops the two from disagreeing. -pub fn prep_round_dims( - log_heights: &[u8; NUM_LFM_CHIPS], - prep_widths: &[u16; NUM_LFM_CHIPS], - blowup_factor: u8, - blake3_chunk_log_heights: &[u8], -) -> Vec<(usize, usize)> { - let blowup_log = (blowup_factor as usize).trailing_zeros() as usize; - PREP_ROUND_SLOTS - .flat_map(|i| { - // Membership is PREP_ROUND_SLOTS and nothing else. An earlier draft - // wrote `.filter(|&i| prep_widths[i] > 0)` here, which is a SECOND - // derivation of the round's membership competing with the slot range - // above and with `pinned_prep_widths`'s indexing of - // `RoundShape::tables`. A member with no columns is a broken registry - // entry, not a slot to skip quietly: skipping it would shorten the - // declared round, shift every later matrix's position in the tree, - // and still build — the failure mode MMCS-PLAN §3.3 warns about, - // where prover and verifier agree on the same wrong shape. - assert!( - prep_widths[i] > 0, - "slot {i} is in PREP_ROUND_SLOTS but carries no committed columns" - ); - let width = prep_widths[i] as usize; - // `LFM_BLAKE3` contributes one matrix PER CHUNK, at each chunk's own - // height and at the shared group width. Expanded in place rather - // than appended, because the round is absorbed in slot order and the - // chip's slot is 11 — the last of the round. A single-chunk program - // yields exactly the one entry this used to emit, so the round's - // shape (and its root) do not move when chunking is off. - if i == BLAKE3_SLOT { - blake3_chunk_log_heights - .iter() - .map(|h| (*h as usize + blowup_log, width)) - .collect::>() - } else { - vec![(log_heights[i] as usize + blowup_log, width)] - } - }) - .collect() -} - -/// The registry slot an epoch TABLE index belongs to, in -/// [`LfmAirs::air_refs`](crate::lfm::airs::LfmAirs::air_refs) order. -/// -/// The two orders diverge on two independent axes, and both must be walked -/// here exactly as `air_refs` emits them: -/// -/// - **Chunking**: `KECCAK_RND` appears `keccak_rnd_chunks` times and -/// `LFM_BLAKE3` appears `blake3_chunks` times, so `KECCAK_RC` sits that many -/// tables after the last always-on slot, not at a fixed index. -/// - **The chip mask**: an absent family's slots are not emitted at all — -/// `KECCAK_SLOT` (6), the `KECCAK_RND` copies and `KECCAK_RC` leave with the -/// keccak family, `BLAKE3_SLOT` (11) with the blake3 one — and every table -/// after a hole shifts down. A map written against the full set attributes a -/// masked epoch's `BITWISE` to a program-group slot, which is exactly the -/// agree-on-the-same-wrong-shape failure §3.3 warns about. -/// -/// ★ Every registered program today has `keccak_rnd_chunks <= 1`, which hides -/// the first axis. The second is live in the table itself — four of the six -/// registry entries mask out at least one family, and no entry is FULL. The -/// tests drive both: `the_slot_to_table_map_is_not_the_identity_beyond_one_chunk` -/// at three chunks, and the masked TrivialV0 epoch in -/// `a_batched_lfm_epoch_is_refused_for_the_round_coverage_gap` for the holes. -/// -/// `None` for a table index past the end of the set. -pub fn slot_of_table( - table: usize, - keccak_rnd_chunks: usize, - blake3_chunks: usize, - chip_set: ChipSet, -) -> Option { - let mut t = table; - // Slots 0..=5, always present. - if t < 6 { - return Some(t); - } - t -= 6; - if chip_set.keccak { - if t == 0 { - return Some(KECCAK_SLOT); - } - t -= 1; - } - // Slots 7..=10, always present. - if t < 4 { - return Some(7 + t); - } - t -= 4; - if chip_set.blake3 { - if t < blake3_chunks { - return Some(BLAKE3_SLOT); - } - t -= blake3_chunks; - } - if chip_set.keccak { - if t < keccak_rnd_chunks { - return Some(KECCAK_RND_SLOT); - } - t -= keccak_rnd_chunks; - if t == 0 { - return Some(KECCAK_RC_SLOT); - } - t -= 1; - } - // BITWISE, always present, always last. - if t == 0 { Some(14) } else { None } -} - -/// Compact a registry entry's per-slot widths into the contributing-matrix slice -/// `stark::batched::shape::PinnedPrep` takes. -/// -/// # One derivation, not two -/// -/// The slice is built by INDEXING `prep.tables` — the round's own list of -/// contributing table indices — and never by filtering the per-slot array for -/// non-zero entries. Those are two independent derivations of the same fact, and -/// if they ever disagreed (a genuinely zero-width group, or a non-member slot -/// carrying a width) a prover and a verifier would both compact the same wrong -/// way and honest proofs would keep verifying with nothing failing. That is the -/// failure shape MMCS-PLAN §3.3's closing warning describes, one level up from -/// the root comparison itself. -/// -/// # `None` is the loud half -/// -/// Returns `None` when the epoch's preprocessed round contains a matrix -/// [`PREP_ROUND_SLOTS`] does not cover. **Today that is every real LFM epoch**, -/// because `KECCAK_RC` and `BITWISE` are preprocessed AIRs and the round is not -/// widened yet. A rejection is the correct answer: the alternative is handing a -/// verifier a width slice that describes a different round than the root does. -pub fn pinned_prep_widths( - prep: &RoundShape, - prep_widths: &[u16; NUM_LFM_CHIPS], - keccak_rnd_chunks: usize, - blake3_chunks: usize, - chip_set: ChipSet, -) -> Option> { - prep.tables - .iter() - .map(|&table| { - let slot = slot_of_table(table, keccak_rnd_chunks, blake3_chunks, chip_set)?; - if !PREP_ROUND_SLOTS.contains(&slot) { - return None; - } - match prep_widths[slot] { - 0 => None, - w => Some(w as usize), - } - }) - .collect() -} - -impl LfmArtifacts { - /// The batched preprocessed round's shape, as - /// `stark::fri::mmcs::MixedMmcs::verify_batch` wants it: `(heights, widths)` - /// over the participating slots, in slot order. - /// - /// `blowup_factor` is taken rather than stored because it is a property of - /// the proof options the artifacts were built under; - /// `the_prep_round_shape_matches_what_was_committed` pins that passing the - /// options a caller committed with reproduces the declared shape. - pub fn prep_round_shape(&self, blowup_factor: u8) -> (Vec, Vec) { - let dims = prep_round_dims( - &self.log_heights, - &self.prep_widths, - blowup_factor, - &self.blake3_chunk_log_heights, - ); - ( - dims.iter().map(|(h, _)| *h).collect(), - dims.iter().map(|(_, w)| *w).collect(), - ) - } -} - /// Commits every instruction column group (plus the fixed tables) at the given /// options and derives the program digest. Host-side, seconds — there is no /// keygen in this framework. @@ -596,67 +311,41 @@ pub fn build_artifacts_with_hasher( ]; let mut roots = [[0u8; 32]; NUM_LFM_CHIPS]; let mut log_heights = [0u8; NUM_LFM_CHIPS]; - let mut prep_widths = [0u16; NUM_LFM_CHIPS]; - // Metadata first, so the round's shape comes from the SAME derivation a - // verifier will use (`prep_round_dims`) rather than from a second walk of - // the groups that could drift from it. + // Heights first, from the compiled groups: the LDE walk below commits them + // and the digest binds them, so both read one derivation. for (i, g) in groups.iter().enumerate() { log_heights[i] = g.padded_rows.trailing_zeros() as u8; - if PREP_ROUND_SLOTS.contains(&i) { - prep_widths[i] = - u16::try_from(g.width).expect("a chip group is far under 65535 columns"); - } } - // The chunk heights are arithmetic (`blake3_chunk_rows`), so the round's - // shape is declared without materializing a single chunk group. Slot 11's - // own entries are chunk 0's — the two arrays stay the shape a single-table - // program has always had. + // The chunk heights are arithmetic (`blake3_chunk_rows`), so they are known + // without materializing a single chunk group. Slot 11's own entry is chunk + // 0's — the array stays the shape a single-table program has. let blake3_chunk_log_heights: Vec = blake3_chunk_rows(program) .into_iter() .map(|rows| rows.trailing_zeros() as u8) .collect(); log_heights[BLAKE3_SLOT] = blake3_chunk_log_heights[0]; - prep_widths[BLAKE3_SLOT] = u16::try_from(program.groups.blake3.width) - .expect("a chip group is far under 65535 columns"); - - let prep_dims = prep_round_dims( - &log_heights, - &prep_widths, - options.blowup_factor, - &blake3_chunk_log_heights, - ); - let mut prep = PrepRoundBuilder::new(&prep_dims); for (i, g) in groups.iter().enumerate() { - // One expansion per group, consumed twice: by the per-slot root and by - // the batched round. `commit_group` used to do its own expansion and - // throw it away; going through `lde_columns` keeps the batched root a - // commitment to the SAME evaluations rather than to a second copy. let lde = lde_columns(&group_columns(g), options); roots[i] = commit_lde_columns(&lde); - if PREP_ROUND_SLOTS.contains(&i) { - prep.absorb(&lde); - } - // Dropped here — peak residency is one group's LDE, exactly as before. + // Dropped here — peak residency is one group's LDE. drop(lde); } // Then `LFM_BLAKE3`, one chunk at a time: materialize the chunk's group, - // expand it, commit it, absorb it, drop both. Peak residency stays one - // chunk's LDE — which is the whole point of chunking this chip. + // expand it, commit it, drop both. Peak residency stays one chunk's LDE — + // which is the whole point of chunking this chip. let blake3_chunk_roots: Vec = (0..blake3_chunk_log_heights.len()) .map(|c| { let group = program.blake3_chunk_group(c); let lde = lde_columns(&group_columns(&group), options); drop(group); let root = commit_lde_columns(&lde); - prep.absorb(&lde); drop(lde); root }) .collect(); roots[BLAKE3_SLOT] = blake3_chunk_roots[0]; - let prep_root = prep.finish(); // Slot 12 (KECCAK_RND) keeps the all-zero sentinel installed above. roots[13] = keccak_rc::preprocessed_commitment(options); log_heights[13] = keccak_rc::NUM_ROWS.trailing_zeros() as u8; @@ -682,9 +371,6 @@ pub fn build_artifacts_with_hasher( // and the mask decides what a proof carries, exactly where it always did: // `ChipSet::num_airs` and `LfmAirs::air_refs`. - // `prep_root` and `prep_widths` are deliberately NOT arguments here: the - // batched-round pins ride the entry, not the digest. Folding them in - // belongs to the next deliberate re-bless. See `LfmArtifacts::prep_root`. let program_id = lfm_program_id( &roots, &log_heights, @@ -703,8 +389,6 @@ pub fn build_artifacts_with_hasher( hasher, chip_set, program_id, - prep_root, - prep_widths, } } @@ -886,12 +570,6 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0xda, 0x00, 0x7d, 0x2e, 0xc7, 0x6d, 0xaa, 0x6e, 0x38, 0x96, 0x64, 0x81, 0xde, 0xed, 0x27, 0xfd, 0x68, 0xde, ], - prep_root: [ - 0x23, 0x57, 0x15, 0x53, 0x9b, 0xdb, 0xf1, 0x9e, 0x9e, 0x6f, 0x9b, 0xce, 0x1d, 0x51, - 0x9e, 0x57, 0x28, 0x28, 0x47, 0x36, 0x03, 0x3b, 0x0b, 0x78, 0xd9, 0xdb, 0x7b, 0x1b, - 0x80, 0x2b, 0xf9, 0xac, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, LfmRegistryEntry { kind: LfmProgramKind::FriToyV0, @@ -985,12 +663,6 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0x01, 0xc2, 0x3e, 0x0b, 0x93, 0xda, 0x00, 0xc6, 0xb4, 0x6d, 0x99, 0xd5, 0x7e, 0xc0, 0x6b, 0xb6, 0x49, 0xf4, ], - prep_root: [ - 0x81, 0x31, 0x40, 0x97, 0xdc, 0x51, 0x37, 0x09, 0x39, 0x04, 0x60, 0x51, 0xe7, 0x3c, - 0x35, 0x58, 0x21, 0x69, 0xdd, 0x0e, 0x5f, 0xbf, 0x0f, 0x69, 0x1d, 0xb4, 0xff, 0x7a, - 0xae, 0x80, 0x43, 0x5c, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, LfmRegistryEntry { kind: LfmProgramKind::KeccakChainV0, @@ -1084,12 +756,6 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0x06, 0xff, 0x4e, 0x56, 0x57, 0x50, 0x3c, 0x9e, 0x51, 0xc9, 0xe0, 0x40, 0x3c, 0xc8, 0x58, 0xd4, 0x2a, 0x33, ], - prep_root: [ - 0x17, 0xb9, 0x2d, 0x29, 0xbd, 0x27, 0x65, 0xb1, 0x9f, 0xb3, 0xe7, 0x4e, 0x89, 0xb8, - 0x89, 0x66, 0xc6, 0xd1, 0xc5, 0x63, 0x0f, 0x8f, 0x12, 0x0b, 0x4e, 0xff, 0x73, 0x86, - 0x1f, 0x03, 0xf2, 0x5b, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, LfmRegistryEntry { kind: LfmProgramKind::KeccakSpongeV0, @@ -1183,12 +849,6 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0xa9, 0x91, 0xaf, 0x2c, 0xd1, 0xc7, 0xb2, 0xff, 0xbb, 0xf9, 0x14, 0xed, 0x40, 0xde, 0xf6, 0x54, 0x80, 0xf0, ], - prep_root: [ - 0xa6, 0x14, 0xdf, 0x60, 0xda, 0x68, 0x8c, 0xfc, 0x67, 0x5d, 0x4b, 0x31, 0xaa, 0xce, - 0xa4, 0x82, 0x1e, 0xf0, 0xfb, 0x02, 0x08, 0xf4, 0x0e, 0x4b, 0xd4, 0x6f, 0xba, 0x2e, - 0x85, 0x07, 0xb8, 0xe2, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, LfmRegistryEntry { kind: LfmProgramKind::TranscriptReplayV0, @@ -1282,12 +942,6 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0x08, 0x77, 0xa5, 0x67, 0x4e, 0x53, 0x1e, 0x36, 0xdd, 0x56, 0x89, 0xc9, 0x4c, 0xc9, 0x88, 0x15, 0x64, 0x4f, ], - prep_root: [ - 0x77, 0x62, 0x5c, 0x36, 0x2d, 0xd7, 0xe8, 0xbf, 0xbf, 0x58, 0x2e, 0xdd, 0x42, 0x73, - 0x72, 0x7c, 0x5d, 0xf0, 0x74, 0x17, 0xb2, 0xdb, 0xa3, 0xbf, 0x11, 0x8f, 0x30, 0xfe, - 0x20, 0xab, 0x63, 0x4e, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, LfmRegistryEntry { kind: LfmProgramKind::StatementReplayV0, @@ -1381,11 +1035,5 @@ pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ 0x97, 0x07, 0x99, 0xf6, 0x54, 0xa3, 0x87, 0x07, 0x34, 0xa5, 0x14, 0x8a, 0xd5, 0x63, 0x21, 0xd6, 0xa6, 0x4d, ], - prep_root: [ - 0x47, 0x89, 0xd7, 0x30, 0x6e, 0x18, 0xd5, 0x29, 0x48, 0x34, 0x27, 0x88, 0x91, 0x55, - 0x33, 0x26, 0x90, 0xf5, 0x33, 0x4d, 0x88, 0xe8, 0xdb, 0x90, 0x73, 0xff, 0x38, 0xc1, - 0xae, 0xc4, 0xf4, 0xc9, - ], - prep_widths: [6, 10, 11, 8, 134, 13, 56, 12, 2, 3, 1, 20, 0, 0, 0], }, ];