From fa881c6d1591d0f54a22f056711926b403454eee Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 8 Apr 2026 11:54:51 +1200 Subject: [PATCH 01/48] Lock-free store state and apply_block --- crates/store/src/blocks.rs | 4 +- crates/store/src/db/mod.rs | 13 - crates/store/src/errors.rs | 10 +- crates/store/src/server/block_producer.rs | 62 ++-- crates/store/src/server/mod.rs | 3 +- crates/store/src/state/apply_block.rs | 301 ------------------ crates/store/src/state/mod.rs | 361 +++++++++++----------- crates/store/src/state/sync_state.rs | 7 +- crates/store/src/state/writer.rs | 247 +++++++++++++++ crates/store/src/state/writer_guard.rs | 66 ++++ 10 files changed, 527 insertions(+), 547 deletions(-) delete mode 100644 crates/store/src/state/apply_block.rs create mode 100644 crates/store/src/state/writer.rs create mode 100644 crates/store/src/state/writer_guard.rs diff --git a/crates/store/src/blocks.rs b/crates/store/src/blocks.rs index 749ef0289..4ed251bd5 100644 --- a/crates/store/src/blocks.rs +++ b/crates/store/src/blocks.rs @@ -83,9 +83,9 @@ impl BlockStore { #[instrument( target = COMPONENT, name = "store.block_store.save_block", - skip(self, data), + skip_all, err, - fields(block_size = data.len()) + fields(block.number = block_num.as_u32(), block.size = data.len()) )] pub async fn save_block(&self, block_num: BlockNumber, data: &[u8]) -> std::io::Result<()> { let (epoch_path, block_path) = self.epoch_block_path(block_num)?; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 3bd4d11a8..61cf62c06 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -25,7 +25,6 @@ use miden_protocol::note::{ }; use miden_protocol::transaction::{InputNoteCommitment, TransactionId}; use miden_protocol::utils::serde::{Deserializable, Serializable}; -use tokio::sync::oneshot; use tracing::{info, instrument}; use crate::COMPONENT; @@ -536,25 +535,13 @@ impl Db { #[instrument(target = COMPONENT, skip_all, err)] pub async fn apply_block( &self, - allow_acquire: oneshot::Sender<()>, - acquire_done: oneshot::Receiver<()>, signed_block: SignedBlock, notes: Vec<(NoteRecord, Option)>, proving_inputs: Option, ) -> Result<()> { self.transact("apply block", move |conn| -> Result<()> { models::queries::apply_block(conn, &signed_block, ¬es, proving_inputs)?; - - // XXX FIXME TODO free floating mutex MUST NOT exist - // it doesn't bind it properly to the data locked! - if allow_acquire.send(()).is_err() { - tracing::warn!(target: COMPONENT, "failed to send notification for successful block application, potential deadlock"); - } - models::queries::prune_history(conn, signed_block.header().block_num())?; - - acquire_done.blocking_recv()?; - Ok(()) }) .await diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 1e008b593..4c01aa350 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -201,14 +201,16 @@ pub enum ApplyBlockError { // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("block applying was cancelled because of closed channel on database side")] - ClosedChannel(#[from] RecvError), - #[error("concurrent write detected")] - ConcurrentWrite, #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, #[error("database update failed: {0}")] DbUpdateTaskFailed(String), + #[error("failed to send block to writer task (channel closed)")] + WriterTaskSendFailed( + #[source] Box>, + ), + #[error("writer task dropped the result channel")] + WriterTaskRecvFailed(#[from] tokio::sync::oneshot::error::RecvError), } impl From for Status { diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index 0102a1928..32970cb19 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -14,7 +14,7 @@ use miden_protocol::batch::OrderedBatches; use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; use tonic::{Request, Response, Status}; -use tracing::{Instrument, error}; +use tracing::error; use crate::errors::ApplyBlockError; use crate::server::api::{ @@ -89,44 +89,28 @@ impl block_producer_server::BlockProducer for StoreApi { block_inputs, }; - // We perform the apply block work in a separate task. This prevents the caller - // cancelling the request and thereby cancelling the task at an arbitrary point of - // execution. - // - // Normally this shouldn't be a problem, however our apply_block isn't quite ACID compliant - // so things get a bit messy. This is more a temporary hack-around to minimize this risk. - let this = self.clone(); - tokio::spawn( - async move { - let block_num = header.block_num(); - let signed_block = SignedBlock::new(header, body, signature) - .map_err(|err| Status::new(tonic::Code::Internal, err.as_report()))?; - // Note: This is an internal endpoint, so its safe to expose the full error - // report. - this.state - .apply_block(signed_block, Some(proving_inputs)) - .await - .inspect(|_| { - if let Err(err) = this.chain_tip_sender.send(block_num) { - error!("Failed to send chain tip: {:?}", err); - } - }) - .map_err(|err| { - span.set_error(&err); - let code = match err { - ApplyBlockError::InvalidBlockError(_) => tonic::Code::InvalidArgument, - _ => tonic::Code::Internal, - }; - Status::new(code, err.as_report()) - }) - } - .in_current_span(), - ) - .await - .map_err(|err| { - tonic::Status::internal(err.as_report_context("joining apply_block task failed")) - }) - .flatten()?; + let block_num = header.block_num(); + let signed_block = SignedBlock::new(header, body, signature) + .map_err(|err| Status::new(tonic::Code::Internal, err.as_report()))?; + + // Apply the block. + self.state + .apply_block(signed_block, Some(proving_inputs)) + .await + .inspect(|_| { + if let Err(err) = self.chain_tip_sender.send(block_num) { + error!("Failed to send chain tip: {:?}", err); + } + }) + .map_err(|err| { + span.set_error(&err); + let code = match err { + ApplyBlockError::InvalidBlockError(_) => tonic::Code::InvalidArgument, + _ => tonic::Code::Internal, + }; + Status::new(code, err.as_report()) + })?; + Ok(Response::new(())) } diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 34f983775..a4b7e09be 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -176,14 +176,13 @@ impl Store { /// Spawns the gRPC servers and the DB maintenance background task. fn spawn_grpc_servers( - state: State, + state: Arc, chain_tip_sender: watch::Sender, grpc_options: GrpcOptionsInternal, rpc_listener: TcpListener, ntx_builder_listener: TcpListener, block_producer_listener: TcpListener, ) -> anyhow::Result>> { - let state = Arc::new(state); let rpc_service = store::rpc_server::RpcServer::new(api::StoreApi { state: Arc::clone(&state), chain_tip_sender: chain_tip_sender.clone(), diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/apply_block.rs deleted file mode 100644 index d8aab21e7..000000000 --- a/crates/store/src/state/apply_block.rs +++ /dev/null @@ -1,301 +0,0 @@ -use std::sync::Arc; - -use miden_node_proto::BlockProofRequest; -use miden_node_utils::ErrorReport; -use miden_protocol::account::delta::AccountUpdateDetails; -use miden_protocol::block::SignedBlock; -use miden_protocol::note::NoteDetails; -use miden_protocol::transaction::OutputNote; -use miden_protocol::utils::serde::Serializable; -use tokio::sync::oneshot; -use tracing::{Instrument, info, info_span, instrument}; - -use crate::db::NoteRecord; -use crate::errors::{ApplyBlockError, InvalidBlockError}; -use crate::state::State; -use crate::{COMPONENT, HistoricalError}; - -impl State { - /// Apply changes of a new block to the DB and in-memory data structures. - /// - /// ## Note on state consistency - /// - /// The server contains in-memory representations of the existing trees, the in-memory - /// representation must be kept consistent with the committed data, this is necessary so to - /// provide consistent results for all endpoints. In order to achieve consistency, the - /// following steps are used: - /// - /// - the request data is validated, prior to starting any modifications. - /// - block is being saved into the store in parallel with updating the DB, but before - /// committing. This block is considered as candidate and not yet available for reading - /// because the latest block pointer is not updated yet. - /// - a transaction is open in the DB and the writes are started. - /// - while the transaction is not committed, concurrent reads are allowed, both the DB and the - /// in-memory representations, which are consistent at this stage. - /// - prior to committing the changes to the DB, an exclusive lock to the in-memory data is - /// acquired, preventing concurrent reads to the in-memory data, since that will be - /// out-of-sync w.r.t. the DB. - /// - the DB transaction is committed, and requests that read only from the DB can proceed to - /// use the fresh data. - /// - the in-memory structures are updated, including the latest block pointer and the lock is - /// released. - /// - /// # Errors - /// - /// Returns an error if `proving_inputs` is `None` and the block is not the genesis block. - // TODO: This span is logged in a root span, we should connect it to the parent span. - #[expect(clippy::too_many_lines)] - #[instrument(target = COMPONENT, skip_all, err)] - pub async fn apply_block( - &self, - signed_block: SignedBlock, - proving_inputs: Option, - ) -> Result<(), ApplyBlockError> { - let _lock = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; - - let header = signed_block.header(); - let body = signed_block.body(); - - // Validate that header and body match. - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); - } - - let block_num = header.block_num(); - let block_commitment = header.commitment(); - - // Validate that the applied block is the next block in sequence. - let prev_block = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, - } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - - // Save the block to the block store. In a case of a rolled-back DB transaction, the - // in-memory state will be unchanged, but the block might still be written into the - // block store. Thus, such block should be considered as block candidates, but not - // finalized blocks. So we should check for the latest block when getting block from - // the store. - let signed_block_bytes = signed_block.to_bytes(); - let store = Arc::clone(&self.block_store); - let block_save_task = tokio::spawn( - async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), - ); - - // Scope to read in-memory data, compute mutations required for updating account - // and nullifier trees, and validate the request. - let ( - nullifier_tree_old_root, - nullifier_tree_update, - account_tree_old_root, - account_tree_update, - ) = { - let inner = self.inner.read().await; - - let _span = info_span!(target: COMPONENT, "update_in_memory_structs").entered(); - - // nullifiers can be produced only once - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } - - // compute updates for the in-memory data structures - - // new_block.chain_root must be equal to the chain MMR root prior to the update - let peaks = inner.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } - - // compute update for nullifier tree - let nullifier_tree_update = inner - .nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - // We do our best here to notify the serve routine, if it doesn't care (dropped the - // receiver) we can't do much. - let _ = self.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidNullifierRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } - - // compute update for account tree - let account_tree_update = inner - .account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - let _ = self.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidAccountRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } - - ( - inner.nullifier_tree.root(), - nullifier_tree_update, - inner.account_tree.root_latest(), - account_tree_update, - ) - }; - - // Build note tree. - let note_tree = body.compute_block_note_tree(); - if note_tree.root() != header.note_root() { - return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); - } - - let notes = body - .output_notes() - .map(|(note_index, note)| { - let (details, nullifier) = match note { - OutputNote::Public(note) => { - (Some(NoteDetails::from(note.as_note())), Some(note.as_note().nullifier())) - }, - OutputNote::Private(_) => (None, None), - }; - - let inclusion_path = note_tree.open(note_index); - - let note_record = NoteRecord { - block_num, - note_index, - note_id: note.id().as_word(), - note_commitment: note.to_commitment(), - metadata: note.metadata().clone(), - details, - inclusion_path, - }; - - Ok((note_record, nullifier)) - }) - .collect::, InvalidBlockError>>()?; - - // Signals the transaction is ready to be committed, and the write lock can be acquired. - let (allow_acquire, acquired_allowed) = oneshot::channel::<()>(); - // Signals the write lock has been acquired, and the transaction can be committed. - let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - - // Extract public account updates with deltas before block is moved into async task. - // Private accounts are filtered out since they don't expose their state changes. - let account_deltas = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Delta(delta) => Some(delta.clone()), - AccountUpdateDetails::Private => None, - }, - )); - - // The DB and in-memory state updates need to be synchronized and are partially - // overlapping. Namely, the DB transaction only proceeds after this task acquires the - // in-memory write lock. This requires the DB update to run concurrently, so a new task is - // spawned. - let db = Arc::clone(&self.db); - let db_update_task = tokio::spawn( - async move { - db.apply_block(allow_acquire, acquire_done, signed_block, notes, proving_inputs) - .await - } - .in_current_span(), - ); - - // Wait for the message from the DB update task, that we ready to commit the DB transaction. - acquired_allowed.await.map_err(ApplyBlockError::ClosedChannel)?; - - // Awaiting the block saving task to complete without errors. - block_save_task.await??; - - // Scope to update the in-memory data. - async move { - // We need to hold the write lock here to prevent inconsistency between the in-memory - // state and the DB state. Thus, we need to wait for the DB update task to complete - // successfully. - let mut inner = self.inner.write().await; - - // We need to check that neither the nullifier tree nor the account tree have changed - // while we were waiting for the DB preparation task to complete. If either of them - // did change, we do not proceed with in-memory and database updates, since it may - // lead to an inconsistent state. - if inner.nullifier_tree.root() != nullifier_tree_old_root - || inner.account_tree.root_latest() != account_tree_old_root - { - return Err(ApplyBlockError::ConcurrentWrite); - } - - // Notify the DB update task that the write lock has been acquired, so it can commit - // the DB transaction. - inform_acquire_done - .send(()) - .map_err(|_| ApplyBlockError::DbUpdateTaskFailed("Receiver was dropped".into()))?; - - // TODO: shutdown #91 - // Await for successful commit of the DB transaction. If the commit fails, we mustn't - // change in-memory state, so we return a block applying error and don't proceed with - // in-memory updates. - db_update_task - .await? - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // Update the in-memory data structures after successful commit of the DB transaction - inner - .nullifier_tree - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: old nullifier tree root must be checked before this step"); - inner - .account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: old account tree root must be checked before this step"); - - inner.blockchain.push(block_commitment); - - Ok(()) - } - .in_current_span() - .await?; - - self.forest.write().await.apply_block_updates(block_num, account_deltas)?; - - info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); - - Ok(()) - } -} diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 079825120..18e81989a 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -1,13 +1,21 @@ -//! Abstraction to synchronize state modifications. +//! Lock-free state management for the Miden store. //! -//! The [State] provides data access and modifications methods, its main purpose is to ensure that -//! data is atomically written, and that reads are consistent. +//! The [State] provides data access and modification methods. Reads are lock-free: readers access +//! shared in-memory structures directly via [`WriterGuard`]. A single writer task, serialized by +//! a channel, applies block mutations and then atomically advances a block counter +//! ([`committed_block_num`](State::committed_block_num)) to publish the new state to readers. +//! +//! The `Release`/`Acquire` ordering on the atomic counter establishes a happens-before +//! relationship, guaranteeing that readers who observe a given block number will see all +//! mutations up to and including that block. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use miden_node_proto::BlockProofRequest; use miden_node_proto::domain::account::{ AccountDetailRequest, AccountDetails, @@ -29,12 +37,12 @@ use miden_protocol::account::{AccountId, StorageMapKey, StorageMapWitness, Stora use miden_protocol::asset::{AssetVaultKey, AssetWitness}; use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; +use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, SignedBlock}; use miden_protocol::crypto::merkle::mmr::{MmrPeaks, MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtProof, SmtStorage}; +use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtProof}; use miden_protocol::note::{NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{mpsc, oneshot}; use tracing::{info, instrument}; use crate::account_state_forest::{AccountStateForest, WitnessError}; @@ -67,8 +75,11 @@ use loader::{ verify_tree_consistency, }; -mod apply_block; mod sync_state; +pub(crate) mod writer; +mod writer_guard; + +pub(crate) use writer_guard::WriterGuard; // FINALITY // ================================================================================================ @@ -93,51 +104,44 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } -/// Container for state that needs to be updated atomically. -struct InnerState -where - S: SmtStorage, -{ - nullifier_tree: NullifierTree>, - blockchain: Blockchain, - account_tree: AccountTreeWithHistory, -} - -impl InnerState { - /// Returns the latest block number. - fn latest_block_num(&self) -> BlockNumber { - self.blockchain - .chain_tip() - .expect("chain should always have at least the genesis block") - } -} - // CHAIN STATE // ================================================================================================ /// The rollup state. +/// +/// All in-memory state is accessed lock-free. A single writer task (serialized by a channel) +/// mutates the state and then advances [`committed_block_num`](Self::committed_block_num) with +/// `Release` ordering. Readers load the counter with `Acquire` ordering before accessing shared +/// structures, guaranteeing they see a consistent snapshot. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. - db: Arc, + pub(crate) db: Arc, /// The block store which stores full block contents for all blocks. - block_store: Arc, + pub(crate) block_store: Arc, - /// Read-write lock used to prevent writing to a structure while it is being used. - /// - /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock>, + /// Atomic block counter — the version pointer for readers. + /// Advanced by the writer AFTER all in-memory mutations are complete. + pub(crate) committed_block_num: AtomicU32, + + /// Nullifier tree — append-only, backed by `RocksDB`. + pub(crate) nullifier_tree: WriterGuard>>, + + /// Account tree with historical overlay support (50-block history). + pub(crate) account_tree: WriterGuard>, - /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. - forest: RwLock, + /// Chain MMR (Merkle Mountain Range of block commitments). + pub(crate) blockchain: WriterGuard, - /// To allow readers to access the tree data while an update in being performed, and prevent - /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. - writer: Mutex<()>, + /// Forest state for account storage maps and vault witnesses. + pub(crate) forest: WriterGuard, + + /// Channel to the single writer task. + writer_tx: mpsc::Sender, /// Request termination of the process due to a fatal internal state error. - termination_ask: tokio::sync::mpsc::Sender, + pub(crate) termination_ask: tokio::sync::mpsc::Sender, /// The latest proven-in-sequence block number, updated by the proof scheduler. proven_tip: ProvenTipReader, @@ -148,12 +152,16 @@ impl State { // -------------------------------------------------------------------------------------------- /// Loads the state from the data directory. + /// + /// The returned `Arc` is ready to use. The writer task is spawned internally and + /// holds a clone of the `Arc`. Dropping all external clones and closing the writer channel + /// will terminate the writer task. #[instrument(target = COMPONENT, skip_all)] pub async fn load( data_path: &Path, storage_options: StorageOptions, termination_ask: tokio::sync::mpsc::Sender, - ) -> Result<(Self, ProvenTipWriter), StateInitializationError> { + ) -> Result<(Arc, ProvenTipWriter), StateInitializationError> { let data_directory = DataDirectory::load(data_path.to_path_buf()) .map_err(StateInitializationError::DataDirectoryLoadError)?; @@ -185,18 +193,12 @@ impl State { let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; // Verify that tree roots match the expected roots from the database. - // This catches any divergence between persistent storage and the database caused by - // corruption or incomplete shutdown. verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); let forest = load_smt_forest(&mut db, latest_block_num).await?; - let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); - - let forest = RwLock::new(forest); - let writer = Mutex::new(()); let db = Arc::new(db); // Initialize the proven tip from database. @@ -204,18 +206,27 @@ impl State { db.proven_chain_tip().await.map_err(StateInitializationError::DatabaseError)?; let (proven_tip_writer, proven_tip) = ProvenTipWriter::new(proven_tip); - Ok(( - Self { - db, - block_store, - inner, - forest, - writer, - termination_ask, - proven_tip, - }, - proven_tip_writer, - )) + // Create the writer channel. Buffer size of 1: only one block can be in flight. + let (writer_tx, writer_rx) = mpsc::channel(1); // TODO(currentpr): Probably want some buffering. + + let state = Arc::new(Self { + db, + block_store, + committed_block_num: AtomicU32::new(latest_block_num.as_u32()), + nullifier_tree: WriterGuard::new(nullifier_tree), + account_tree: WriterGuard::new(account_tree), + blockchain: WriterGuard::new(blockchain), + forest: WriterGuard::new(forest), + writer_tx, + termination_ask, + proven_tip, + }); + + // Spawn the single writer task. + let writer_state = Arc::clone(&state); + tokio::spawn(writer::writer_loop(writer_rx, writer_state)); + + Ok((state, proven_tip_writer)) } /// Returns the database. @@ -228,9 +239,49 @@ impl State { Arc::clone(&self.block_store) } - // STATE ACCESSORS + // BLOCK APPLICATION + // -------------------------------------------------------------------------------------------- + + /// Apply changes of a new block to the DB and in-memory data structures. + /// + /// This sends the block to the single writer task via a channel and awaits the result. + /// The writer task handles all validation, DB writes, and in-memory mutations. + #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] + pub async fn apply_block( + &self, + signed_block: SignedBlock, + proving_inputs: Option, + ) -> Result<(), ApplyBlockError> { + let (result_tx, result_rx) = oneshot::channel(); + self.writer_tx + .send(writer::WriteRequest { signed_block, proving_inputs, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(Box::new(e)))?; + result_rx.await? + } + + // STATE ACCESSORS (lock-free) // -------------------------------------------------------------------------------------------- + /// Returns the committed block number (atomic load with Acquire ordering). + fn committed_block_num(&self) -> BlockNumber { + BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)) + } + + /// Returns the effective chain tip for the given finality level. + /// + /// - [`Finality::Committed`]: returns the latest committed block number from the atomic + /// counter. + /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch + /// channel, updated by the proof scheduler). + #[expect(clippy::unused_async)] + pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { + match finality { + Finality::Committed => self.committed_block_num(), + Finality::Proven => self.proven_tip.read(), + } + } + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. /// /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is @@ -244,8 +295,7 @@ impl State { let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { - let inner = self.inner.read().await; - let mmr_proof = inner.blockchain.open(header.block_num())?; + let mmr_proof = self.blockchain.as_ref().open(header.block_num())?; Some(mmr_proof) } else { None @@ -262,10 +312,10 @@ impl State { /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret)] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { - let inner = self.inner.read().await; + let tree = self.nullifier_tree.as_ref(); nullifiers .iter() - .map(|n| inner.nullifier_tree.open(n)) + .map(|n| tree.open(n)) .map(NullifierWitness::into_proof) .collect() } @@ -287,7 +337,7 @@ impl State { &self, block_num: Option, ) -> Result, GetCurrentBlockchainDataError> { - let blockchain = &self.inner.read().await.blockchain; + let blockchain = self.blockchain.as_ref(); if let Some(number) = block_num && number == self.chain_tip(Finality::Committed).await { @@ -354,41 +404,33 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - // Scoped block to automatically drop the read lock guard as soon as we're done. - // We also avoid accessing the db in the block as this would delay dropping the guard. - let (batch_reference_block, partial_mmr) = { - let inner_state = self.inner.read().await; + let blockchain = self.blockchain.as_ref(); + let latest_block_num = self.committed_block_num(); - let latest_block_num = inner_state.latest_block_num(); + let highest_block_num = + *blocks.last().expect("we should have checked for empty block references"); + if highest_block_num > latest_block_num { + return Err(GetBatchInputsError::UnknownTransactionBlockReference { + highest_block_num, + latest_block_num, + }); + } - let highest_block_num = - *blocks.last().expect("we should have checked for empty block references"); - if highest_block_num > latest_block_num { - return Err(GetBatchInputsError::UnknownTransactionBlockReference { - highest_block_num, - latest_block_num, - }); - } + // Remove the latest block from the to-be-tracked blocks as it will be the reference + // block for the batch itself and thus added to the MMR within the batch kernel, so + // there is no need to prove its inclusion. + blocks.remove(&latest_block_num); - // Remove the latest block from the to-be-tracked blocks as it will be the reference - // block for the batch itself and thus added to the MMR within the batch kernel, so - // there is no need to prove its inclusion. - blocks.remove(&latest_block_num); - - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = inner_state - .blockchain - .partial_mmr_from_blocks(&blocks, latest_block_num) - .expect("latest block num should exist and all blocks in set should be < than latest block"); - - (latest_block_num, partial_mmr) - }; + // SAFETY: + // - The latest block num was retrieved from the committed counter and the blockchain is + // guaranteed to have been updated to at least that block (happens-before). + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. + let partial_mmr = blockchain.partial_mmr_from_blocks(&blocks, latest_block_num).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + let batch_reference_block = latest_block_num; // Fetch the reference block of the batch as part of this query, so we can avoid looking it // up in a separate DB access. @@ -437,8 +479,6 @@ impl State { reference_blocks: BTreeSet, ) -> Result { // Get the note inclusion proofs from the DB. - // We do this first so we have to acquire the lock to the state just once. There we need the - // reference blocks of the note proofs to get their authentication paths in the chain MMR. let unauthenticated_note_proofs = self .db .select_note_inclusion_proofs(unauthenticated_note_commitments) @@ -454,7 +494,7 @@ impl State { blocks.extend(note_proof_reference_blocks); let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, account_ids, nullifiers).await?; + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; // Fetch the block headers for all blocks in the partial MMR plus the latest one which will // be used as the previous block header of the block being built. @@ -477,14 +517,6 @@ impl State { // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. let latest_block_header = headers.swap_remove(latest_block_header_index); - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block header's block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) .expect("partial mmr and block headers should be consistent"); @@ -497,17 +529,15 @@ impl State { )) } - /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as + /// Get account and nullifier witnesses for the requested account IDs and nullifiers as well as /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its /// number is removed from `blocks` and returned separately. - /// - /// This method acquires the lock to the inner state and does not access the DB so we release - /// the lock asap. - async fn get_block_inputs_witnesses( + #[expect(clippy::type_complexity)] + fn get_block_inputs_witnesses( &self, blocks: &mut BTreeSet, - account_ids: Vec, - nullifiers: Vec, + account_ids: &[AccountId], + nullifiers: &[Nullifier], ) -> Result< ( BlockNumber, @@ -517,9 +547,11 @@ impl State { ), GetBlockInputsError, > { - let inner = self.inner.read().await; + let blockchain = self.blockchain.as_ref(); + let account_tree = self.account_tree.as_ref(); + let nullifier_tree = self.nullifier_tree.as_ref(); - let latest_block_number = inner.latest_block_num(); + let latest_block_number = self.committed_block_num(); // If `blocks` is empty, use the latest block number which will never trigger the error. let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); @@ -534,26 +566,15 @@ impl State { // inclusion in the chain. blocks.remove(&latest_block_number); - // Fetch the partial MMR at the state of the latest block with authentication paths for the - // provided set of blocks. - // - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will also - // retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block numbers - // smaller than latest block num remain in the set. Therefore all the block numbers are - // guaranteed to exist in the chain state at latest block num. - let partial_mmr = - inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); + let partial_mmr = blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); // Fetch witnesses for all accounts. let account_witnesses = account_ids .iter() .copied() - .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) + .map(|account_id| (account_id, account_tree.open_latest(account_id))) .collect::>(); // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent or @@ -561,7 +582,7 @@ impl State { let nullifier_witnesses: BTreeMap = nullifiers .iter() .copied() - .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) + .map(|nullifier| (nullifier, nullifier_tree.open(&nullifier))) .collect(); Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) @@ -577,12 +598,13 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); - let inner = self.inner.read().await; + let account_tree = self.account_tree.as_ref(); + let nullifier_tree = self.nullifier_tree.as_ref(); - let account_commitment = inner.account_tree.get_latest_commitment(account_id); + let account_commitment = account_tree.get_latest_commitment(account_id); let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + Some(!account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) } else { None }; @@ -599,7 +621,7 @@ impl State { .iter() .map(|nullifier| NullifierInfo { nullifier: *nullifier, - block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + block_num: nullifier_tree.get_block_num(nullifier).unwrap_or_default(), }) .collect(); @@ -631,12 +653,6 @@ impl State { /// Returns network account IDs within the specified block range (based on account creation /// block). - /// - /// The function may return fewer accounts than exist in the range if the result would exceed - /// `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` rows. In this case, the result is - /// truncated at a block boundary to ensure all accounts from included blocks are returned. - /// - /// The response includes the last block number that was fully included in the result. pub async fn get_all_network_accounts( &self, block_range: RangeInclusive, @@ -663,7 +679,7 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; + let (block_num, witness) = self.get_account_witness(block_num, account_id)?; let details = if let Some(request) = details { Some(self.fetch_public_account_details(account_id, block_num, request).await?) @@ -678,30 +694,29 @@ impl State { /// /// If `block_num` is provided, returns the witness at that historical block; /// otherwise, returns the witness at the latest block. - async fn get_account_witness( + fn get_account_witness( &self, block_num: Option, account_id: AccountId, ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - let inner_state = self.inner.read().await; + let account_tree = self.account_tree.as_ref(); // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { // Historical query: use the account tree with history - let witness = - inner_state.account_tree.open_at(account_id, requested_block).ok_or_else(|| { - let latest_block = inner_state.account_tree.block_number_latest(); - if requested_block > latest_block { - GetAccountError::UnknownBlock(requested_block) - } else { - GetAccountError::BlockPruned(requested_block) - } - })?; + let witness = account_tree.open_at(account_id, requested_block).ok_or_else(|| { + let latest_block = account_tree.block_number_latest(); + if requested_block > latest_block { + GetAccountError::UnknownBlock(requested_block) + } else { + GetAccountError::BlockPruned(requested_block) + } + })?; (requested_block, witness) } else { // Latest query: use the latest state - let block_num = inner_state.account_tree.block_number_latest(); - let witness = inner_state.account_tree.open_latest(account_id); + let block_num = account_tree.block_number_latest(); + let witness = account_tree.open_latest(account_id); (block_num, witness) }; @@ -734,14 +749,10 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - // Validate block exists in the blockchain before querying the database - { - let inner = self.inner.read().await; - let latest_block_num = inner.latest_block_num(); - - if block_num > latest_block_num { - return Err(GetAccountError::UnknownBlock(block_num)); - } + // Validate block exists in the blockchain before querying the database. + let latest_block_num = self.committed_block_num(); + if block_num > latest_block_num { + return Err(GetAccountError::UnknownBlock(block_num)); } // Query account header and storage header together in a single DB call @@ -796,9 +807,9 @@ impl State { let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; if !map_keys_requests.is_empty() { - let forest_guard = self.forest.read().await; + let forest = self.forest.as_ref(); for (index, slot_name, keys) in map_keys_requests { - let details = forest_guard + let details = forest .get_storage_map_details_for_keys( account_id, slot_name.clone(), @@ -856,18 +867,6 @@ impl State { }) } - /// Returns the effective chain tip for the given finality level. - /// - /// - [`Finality::Committed`]: returns the latest committed block number (from in-memory MMR). - /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch - /// channel, updated by the proof scheduler). - pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { - match finality { - Finality::Committed => self.inner.read().await.latest_block_num(), - Finality::Proven => self.proven_tip.read(), - } - } - /// Loads a block from the block store. Return `Ok(None)` if the block is not found. pub async fn load_block( &self, @@ -915,6 +914,7 @@ impl State { } /// Returns vault asset witnesses for the specified account and block number. + #[expect(clippy::unused_async)] pub async fn get_vault_asset_witnesses( &self, account_id: AccountId, @@ -923,8 +923,7 @@ impl State { ) -> Result, WitnessError> { let witnesses = self .forest - .read() - .await + .as_ref() .get_vault_asset_witnesses(account_id, block_num, vault_keys)?; Ok(witnesses) } @@ -934,6 +933,7 @@ impl State { /// /// Note that the `raw_key` is the raw, user-provided key that needs to be hashed in order to /// get the actual key into the storage map. + #[expect(clippy::unused_async)] pub async fn get_storage_map_witness( &self, account_id: AccountId, @@ -943,8 +943,7 @@ impl State { ) -> Result { let witness = self .forest - .read() - .await + .as_ref() .get_storage_map_witness(account_id, slot_name, block_num, raw_key)?; Ok(witness) } diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index cda17e6ab..5f4a58617 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -78,10 +78,8 @@ impl State { let to_forest = block_to.as_usize(); let mmr_delta = self - .inner - .read() - .await .blockchain + .as_ref() .as_mmr() .get_delta(Forest::new(from_forest), Forest::new(to_forest)) .map_err(StateSyncError::FailedToBuildMmrDelta)?; @@ -129,8 +127,7 @@ impl State { // SAFETY: it is ensured that block_end <= chain_tip, and the blockchain MMR always has // at least chain_tip + 1 leaves. let mmr_checkpoint = block_end + 1; - let mmr_proof = - self.inner.read().await.blockchain.open_at(block_num, mmr_checkpoint)?; + let mmr_proof = self.blockchain.as_ref().open_at(block_num, mmr_checkpoint)?; results.push((note_sync, mmr_proof)); current_from = block_num + 1; diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs new file mode 100644 index 000000000..bb36b9d6d --- /dev/null +++ b/crates/store/src/state/writer.rs @@ -0,0 +1,247 @@ +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use miden_node_proto::BlockProofRequest; +use miden_node_utils::ErrorReport; +use miden_protocol::account::delta::AccountUpdateDetails; +use miden_protocol::block::SignedBlock; +use miden_protocol::note::NoteDetails; +use miden_protocol::transaction::OutputNote; +use miden_protocol::utils::serde::Serializable; +use tokio::sync::{mpsc, oneshot}; +use tracing::{Instrument, info, info_span, instrument}; + +use crate::db::NoteRecord; +use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::State; +use crate::{COMPONENT, HistoricalError}; + +/// A request to apply a new block, sent through the writer channel. +pub struct WriteRequest { + pub signed_block: SignedBlock, + pub proving_inputs: Option, + pub result_tx: oneshot::Sender>, +} + +/// Runs the single writer loop. Receives blocks through the channel and applies them +/// sequentially. Channel serialization guarantees no concurrent writers — no mutex needed. +pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc) { + while let Some(req) = rx.recv().await { + let result = + Box::pin(apply_block_inner(&state, req.signed_block, req.proving_inputs)).await; + let _ = req.result_tx.send(result); + } +} + +/// Apply changes of a new block to the DB and in-memory data structures. +/// +/// ## Consistency model +/// +/// This function is the sole writer to all in-memory state. Readers access the same structures +/// concurrently without locks because: +/// +/// - The data structures are append-only or overlay-based (keyed by block number). +/// - All mutations are completed before the atomic block counter is advanced (`Release`). +/// - Readers load the counter (`Acquire`) before querying, establishing happens-before. +/// +/// The DB transaction is committed independently. Readers gate visibility through the atomic +/// block counter, so the brief window where the DB has block N+1 but the counter still says N +/// is invisible to readers. +#[expect(clippy::too_many_lines)] +#[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] +async fn apply_block_inner( + state: &State, + signed_block: SignedBlock, + proving_inputs: Option, +) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + + // Validate that header and body match. + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); + } + + let block_num = header.block_num(); + let block_commitment = header.commitment(); + + // Validate that the applied block is the next block in sequence. + let prev_block = state + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } + + // Save the block to the block store concurrently. + // In a case of a rolled-back DB transaction, the in-memory state will be unchanged, but + // the block might still be written into the block store. Such blocks should be considered + // as candidates, not finalized blocks. + let signed_block_bytes = signed_block.to_bytes(); + let store = Arc::clone(&state.block_store); + let block_save_task = tokio::spawn( + async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), + ); + + // Compute mutations required for updating account and nullifier trees. + // SAFETY: This is the single writer task, serialized by the channel. No concurrent + // mutations to these structures are possible. + let (nullifier_tree_update, account_tree_update) = { + let nullifier_tree = unsafe { state.nullifier_tree.as_mut() }; + let account_tree = unsafe { state.account_tree.as_mut() }; + let blockchain = state.blockchain.as_ref(); + + let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); + + // Nullifiers can be produced only once. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // new_block.chain_root must be equal to the chain MMR root prior to the update. + let peaks = blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // Compute update for nullifier tree. + let nullifier_tree_update = nullifier_tree + .compute_mutations( + body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidNullifierRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // Compute update for account tree. + let account_tree_update = account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidAccountRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } + + (nullifier_tree_update, account_tree_update) + }; + + // Build note tree. + let note_tree = body.compute_block_note_tree(); + if note_tree.root() != header.note_root() { + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); + } + + let notes = body + .output_notes() + .map(|(note_index, note)| { + let (details, nullifier) = match note { + OutputNote::Public(note) => { + (Some(NoteDetails::from(note.as_note())), Some(note.as_note().nullifier())) + }, + OutputNote::Private(_) => (None, None), + }; + + let inclusion_path = note_tree.open(note_index); + + let note_record = NoteRecord { + block_num, + note_index, + note_id: note.id().as_word(), + note_commitment: note.to_commitment(), + metadata: note.metadata().clone(), + details, + inclusion_path, + }; + + Ok((note_record, nullifier)) + }) + .collect::, InvalidBlockError>>()?; + + // Extract public account deltas before block is moved into the DB task. + let account_deltas = + Vec::from_iter(body.updated_accounts().iter().filter_map( + |update| match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + }, + )); + + // Commit to DB. No oneshot synchronization dance needed — readers gate visibility + // through the atomic block counter, not through DB transaction timing. + state + .db + .apply_block(signed_block, notes, proving_inputs) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // Await the block store save task. + block_save_task.await??; + + // Apply in-memory mutations. + // SAFETY: This is the single writer task, no concurrent mutations. + unsafe { + state + .nullifier_tree + .as_mut() + .apply_mutations(nullifier_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + state + .account_tree + .as_mut() + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + state.blockchain.as_mut().push(block_commitment); + + state.forest.as_mut().apply_block_updates(block_num, account_deltas)?; + } + + // PUBLISH: Advance the atomic block counter with Release ordering. + // All mutations above are guaranteed visible to any reader that subsequently loads this + // counter with Acquire ordering. + state.committed_block_num.store(block_num.as_u32(), Ordering::Release); + + info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); + + Ok(()) +} diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs new file mode 100644 index 000000000..4ad761bd2 --- /dev/null +++ b/crates/store/src/state/writer_guard.rs @@ -0,0 +1,66 @@ +use std::cell::UnsafeCell; + +/// A single-writer / multi-reader wrapper that provides lock-free access to shared state. +/// +/// This type enables a pattern where one dedicated writer task mutates data while many reader +/// tasks concurrently access it, without any locks. +/// +/// # Safety Contract +/// +/// 1. **Single writer**: Only one task (the writer, serialized by a channel) may call +/// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type +/// system. +/// 2. **Publish barrier**: After completing all mutations, the writer performs an atomic store with +/// [`Release`](std::sync::atomic::Ordering::Release) ordering on a shared block counter. +/// 3. **Subscribe barrier**: Before calling [`as_ref()`](Self::as_ref), readers perform an atomic +/// load with [`Acquire`](std::sync::atomic::Ordering::Acquire) ordering on the same counter. +/// 4. The `Release`/`Acquire` pair establishes a *happens-before* relationship, guaranteeing that +/// all mutations performed before the `Release` store are visible to any reader that observes +/// the updated counter value. +/// +/// Because the wrapped data structures are append-only or overlay-based (keyed by block number), +/// readers that observe an older counter value will simply query at that older block number, +/// which is safe. +pub struct WriterGuard { + inner: UnsafeCell, +} + +// SAFETY: The single-writer invariant is enforced by the channel-based writer task architecture. +// Readers only call `as_ref()` which returns `&T`. The writer completes all mutations before +// advancing the atomic block counter (`Release`), and readers load the counter (`Acquire`) +// before accessing the data. This guarantees no data races. +unsafe impl Send for WriterGuard {} +unsafe impl Sync for WriterGuard {} + +impl WriterGuard { + /// Creates a new `WriterGuard` wrapping the given value. + pub fn new(value: T) -> Self { + Self { inner: UnsafeCell::new(value) } + } + + /// Returns a shared reference to the wrapped value. + /// + /// Safe for any reader thread. The data is guaranteed to be in a consistent state because + /// the caller must have loaded the atomic block counter with `Acquire` ordering before + /// calling this method, establishing a happens-before relationship with the writer's + /// `Release` store. + pub fn as_ref(&self) -> &T { + // SAFETY: The writer completes all mutations before the Release store on the block + // counter. The reader loads the counter with Acquire before calling this. The + // Acquire/Release pair ensures all writes are visible. + unsafe { &*self.inner.get() } + } + + /// Returns an exclusive mutable reference to the wrapped value. + /// + /// # Safety + /// + /// Must only be called from the single writer task. The caller must ensure: + /// - No other calls to `as_mut()` are concurrent (enforced by channel serialization). + /// - All mutations through the returned reference are completed before performing a `Release` + /// store on the shared block counter. + #[expect(clippy::mut_from_ref)] + pub unsafe fn as_mut(&self) -> &mut T { + unsafe { &mut *self.inner.get() } + } +} From b389c591eef68ed02a3db8630527ea4002e6ca3e Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 8 Apr 2026 12:17:39 +1200 Subject: [PATCH 02/48] Add ReadSnapshot --- crates/store/src/state/mod.rs | 152 ++++++++++++++++--------- crates/store/src/state/sync_state.rs | 7 +- crates/store/src/state/writer_guard.rs | 2 +- 3 files changed, 103 insertions(+), 58 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 18e81989a..919c70736 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -147,6 +147,32 @@ pub struct State { proven_tip: ProvenTipReader, } +// READ SNAPSHOT +// ================================================================================================ + +/// A consistent read view of the in-memory state. +/// +/// Created by [`State::snapshot()`], which performs an atomic load with `Acquire` ordering on the +/// committed block counter. This establishes a happens-before relationship with the writer's +/// `Release` store, guaranteeing that all tree references obtained through this snapshot reflect +/// a fully consistent state at [`block_num`](Self::block_num). +/// +/// This is the **only** way for readers to obtain references to the in-memory trees. The +/// [`WriterGuard::as_ref()`] method is `pub(super)`, preventing direct access from outside the +/// `state` module without going through the snapshot barrier. +pub(crate) struct ReadSnapshot<'a> { + /// The committed block number at the time the snapshot was taken. + pub block_num: BlockNumber, + /// Nullifier tree — append-only. + pub nullifier_tree: &'a NullifierTree>, + /// Account tree with historical overlay support. + pub account_tree: &'a AccountTreeWithHistory, + /// Chain MMR. + pub blockchain: &'a Blockchain, + /// Forest state for account storage maps and vault witnesses. + pub forest: &'a AccountStateForest, +} + impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- @@ -207,7 +233,7 @@ impl State { let (proven_tip_writer, proven_tip) = ProvenTipWriter::new(proven_tip); // Create the writer channel. Buffer size of 1: only one block can be in flight. - let (writer_tx, writer_rx) = mpsc::channel(1); // TODO(currentpr): Probably want some buffering. + let (writer_tx, writer_rx) = mpsc::channel(1); let state = Arc::new(Self { db, @@ -263,21 +289,39 @@ impl State { // STATE ACCESSORS (lock-free) // -------------------------------------------------------------------------------------------- - /// Returns the committed block number (atomic load with Acquire ordering). - fn committed_block_num(&self) -> BlockNumber { - BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)) + /// Takes a consistent snapshot of all in-memory state. + /// + /// Performs an `Acquire` load on the committed block counter, then obtains references to all + /// trees. The `Acquire` ordering guarantees that all writer mutations completed before the + /// corresponding `Release` store are visible through these references. + /// + /// This is the **only** sanctioned way for readers to access the in-memory trees. + fn snapshot(&self) -> ReadSnapshot<'_> { + let block_num = BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)); + ReadSnapshot { + block_num, + nullifier_tree: self.nullifier_tree.as_ref(), + account_tree: self.account_tree.as_ref(), + blockchain: self.blockchain.as_ref(), + forest: self.forest.as_ref(), + } } /// Returns the effective chain tip for the given finality level. /// - /// - [`Finality::Committed`]: returns the latest committed block number from the atomic - /// counter. + /// - [`Finality::Committed`]: returns the latest committed block number from the atomic counter + /// (Acquire ordering). /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch /// channel, updated by the proof scheduler). + /// + /// This method only returns a block number — it does not access any trees, so the Acquire + /// load here is sufficient without a full snapshot. #[expect(clippy::unused_async)] pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { match finality { - Finality::Committed => self.committed_block_num(), + Finality::Committed => { + BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)) + }, Finality::Proven => self.proven_tip.read(), } } @@ -292,10 +336,11 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { + let snapshot = self.snapshot(); let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { - let mmr_proof = self.blockchain.as_ref().open(header.block_num())?; + let mmr_proof = snapshot.blockchain.open(header.block_num())?; Some(mmr_proof) } else { None @@ -312,10 +357,10 @@ impl State { /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret)] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { - let tree = self.nullifier_tree.as_ref(); + let snapshot = self.snapshot(); nullifiers .iter() - .map(|n| tree.open(n)) + .map(|n| snapshot.nullifier_tree.open(n)) .map(NullifierWitness::into_proof) .collect() } @@ -337,9 +382,9 @@ impl State { &self, block_num: Option, ) -> Result, GetCurrentBlockchainDataError> { - let blockchain = self.blockchain.as_ref(); + let snapshot = self.snapshot(); if let Some(number) = block_num - && number == self.chain_tip(Finality::Committed).await + && number == snapshot.block_num { return Ok(None); } @@ -352,7 +397,8 @@ impl State { .await .map_err(GetCurrentBlockchainDataError::ErrorRetrievingBlockHeader)? .unwrap(); - let peaks = blockchain + let peaks = snapshot + .blockchain .peaks_at(block_header.block_num()) .map_err(GetCurrentBlockchainDataError::InvalidPeaks)?; @@ -404,8 +450,8 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - let blockchain = self.blockchain.as_ref(); - let latest_block_num = self.committed_block_num(); + let snapshot = self.snapshot(); + let latest_block_num = snapshot.block_num; let highest_block_num = *blocks.last().expect("we should have checked for empty block references"); @@ -426,9 +472,10 @@ impl State { // guaranteed to have been updated to at least that block (happens-before). // - We have checked that no block number in the blocks set is greater than latest block // number *and* latest block num was removed from the set. - let partial_mmr = blockchain.partial_mmr_from_blocks(&blocks, latest_block_num).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); + let partial_mmr = + snapshot.blockchain.partial_mmr_from_blocks(&blocks, latest_block_num).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); let batch_reference_block = latest_block_num; @@ -547,11 +594,8 @@ impl State { ), GetBlockInputsError, > { - let blockchain = self.blockchain.as_ref(); - let account_tree = self.account_tree.as_ref(); - let nullifier_tree = self.nullifier_tree.as_ref(); - - let latest_block_number = self.committed_block_num(); + let snapshot = self.snapshot(); + let latest_block_number = snapshot.block_num; // If `blocks` is empty, use the latest block number which will never trigger the error. let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); @@ -566,15 +610,16 @@ impl State { // inclusion in the chain. blocks.remove(&latest_block_number); - let partial_mmr = blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); + let partial_mmr = + snapshot.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); // Fetch witnesses for all accounts. let account_witnesses = account_ids .iter() .copied() - .map(|account_id| (account_id, account_tree.open_latest(account_id))) + .map(|account_id| (account_id, snapshot.account_tree.open_latest(account_id))) .collect::>(); // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent or @@ -582,7 +627,7 @@ impl State { let nullifier_witnesses: BTreeMap = nullifiers .iter() .copied() - .map(|nullifier| (nullifier, nullifier_tree.open(&nullifier))) + .map(|nullifier| (nullifier, snapshot.nullifier_tree.open(&nullifier))) .collect(); Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) @@ -598,13 +643,12 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); - let account_tree = self.account_tree.as_ref(); - let nullifier_tree = self.nullifier_tree.as_ref(); + let snapshot = self.snapshot(); - let account_commitment = account_tree.get_latest_commitment(account_id); + let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + Some(!snapshot.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) } else { None }; @@ -621,7 +665,7 @@ impl State { .iter() .map(|nullifier| NullifierInfo { nullifier: *nullifier, - block_num: nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + block_num: snapshot.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), }) .collect(); @@ -699,24 +743,25 @@ impl State { block_num: Option, account_id: AccountId, ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - let account_tree = self.account_tree.as_ref(); + let snapshot = self.snapshot(); // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { // Historical query: use the account tree with history - let witness = account_tree.open_at(account_id, requested_block).ok_or_else(|| { - let latest_block = account_tree.block_number_latest(); - if requested_block > latest_block { - GetAccountError::UnknownBlock(requested_block) - } else { - GetAccountError::BlockPruned(requested_block) - } - })?; + let witness = + snapshot.account_tree.open_at(account_id, requested_block).ok_or_else(|| { + let latest_block = snapshot.account_tree.block_number_latest(); + if requested_block > latest_block { + GetAccountError::UnknownBlock(requested_block) + } else { + GetAccountError::BlockPruned(requested_block) + } + })?; (requested_block, witness) } else { // Latest query: use the latest state - let block_num = account_tree.block_number_latest(); - let witness = account_tree.open_latest(account_id); + let block_num = snapshot.account_tree.block_number_latest(); + let witness = snapshot.account_tree.open_latest(account_id); (block_num, witness) }; @@ -750,8 +795,8 @@ impl State { } // Validate block exists in the blockchain before querying the database. - let latest_block_num = self.committed_block_num(); - if block_num > latest_block_num { + let snapshot = self.snapshot(); + if block_num > snapshot.block_num { return Err(GetAccountError::UnknownBlock(block_num)); } @@ -807,9 +852,9 @@ impl State { let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; if !map_keys_requests.is_empty() { - let forest = self.forest.as_ref(); for (index, slot_name, keys) in map_keys_requests { - let details = forest + let details = snapshot + .forest .get_storage_map_details_for_keys( account_id, slot_name.clone(), @@ -921,10 +966,9 @@ impl State { block_num: BlockNumber, vault_keys: BTreeSet, ) -> Result, WitnessError> { - let witnesses = self - .forest - .as_ref() - .get_vault_asset_witnesses(account_id, block_num, vault_keys)?; + let snapshot = self.snapshot(); + let witnesses = + snapshot.forest.get_vault_asset_witnesses(account_id, block_num, vault_keys)?; Ok(witnesses) } @@ -941,9 +985,9 @@ impl State { block_num: BlockNumber, raw_key: StorageMapKey, ) -> Result { - let witness = self + let snapshot = self.snapshot(); + let witness = snapshot .forest - .as_ref() .get_storage_map_witness(account_id, slot_name, block_num, raw_key)?; Ok(witness) } diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 5f4a58617..66da87abd 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -77,9 +77,9 @@ impl State { let from_forest = (block_from + 1).as_usize(); let to_forest = block_to.as_usize(); - let mmr_delta = self + let snapshot = self.snapshot(); + let mmr_delta = snapshot .blockchain - .as_ref() .as_mmr() .get_delta(Forest::new(from_forest), Forest::new(to_forest)) .map_err(StateSyncError::FailedToBuildMmrDelta)?; @@ -101,6 +101,7 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { + let snapshot = self.snapshot(); let block_end = *block_range.end(); let note_tags: Arc<[u32]> = note_tags.into(); @@ -127,7 +128,7 @@ impl State { // SAFETY: it is ensured that block_end <= chain_tip, and the blockchain MMR always has // at least chain_tip + 1 leaves. let mmr_checkpoint = block_end + 1; - let mmr_proof = self.blockchain.as_ref().open_at(block_num, mmr_checkpoint)?; + let mmr_proof = snapshot.blockchain.open_at(block_num, mmr_checkpoint)?; results.push((note_sync, mmr_proof)); current_from = block_num + 1; diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs index 4ad761bd2..814acf46a 100644 --- a/crates/store/src/state/writer_guard.rs +++ b/crates/store/src/state/writer_guard.rs @@ -44,7 +44,7 @@ impl WriterGuard { /// the caller must have loaded the atomic block counter with `Acquire` ordering before /// calling this method, establishing a happens-before relationship with the writer's /// `Release` store. - pub fn as_ref(&self) -> &T { + pub(super) fn as_ref(&self) -> &T { // SAFETY: The writer completes all mutations before the Release store on the block // counter. The reader loads the counter with Acquire before calling this. The // Acquire/Release pair ensures all writes are visible. From 13c2e3f6e85e72177525d20cab06fc27291c4cf3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 8 Apr 2026 13:01:52 +1200 Subject: [PATCH 03/48] RWLock for in-memory fields --- crates/store/src/state/mod.rs | 127 +++++++++++++++------------ crates/store/src/state/sync_state.rs | 4 +- crates/store/src/state/writer.rs | 38 ++++---- 3 files changed, 93 insertions(+), 76 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 919c70736..4d604e26d 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -1,13 +1,13 @@ -//! Lock-free state management for the Miden store. +//! State management for the Miden store. //! -//! The [State] provides data access and modification methods. Reads are lock-free: readers access -//! shared in-memory structures directly via [`WriterGuard`]. A single writer task, serialized by -//! a channel, applies block mutations and then atomically advances a block counter -//! ([`committed_block_num`](State::committed_block_num)) to publish the new state to readers. +//! The [State] provides data access and modification methods. A single writer task, serialized by +//! a channel, applies block mutations. In-memory structures (`account_tree`, `blockchain`, +//! `forest`) are protected by [`RwLock`] because their internal collections (`BTreeMap`, `Vec`, +//! `HashMap`) are not safe for concurrent mutation during reads. The `RocksDB`-backed nullifier +//! tree uses a lock-free [`WriterGuard`] because `RocksDB` MVCC provides snapshot isolation. //! -//! The `Release`/`Acquire` ordering on the atomic counter establishes a happens-before -//! relationship, guaranteeing that readers who observe a given block number will see all -//! mutations up to and including that block. +//! Readers obtain a [`ReadSnapshot`] via [`State::snapshot()`], which acquires read locks on the +//! in-memory structures. The writer acquires write locks only during the brief mutation phase. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; @@ -42,7 +42,7 @@ use miden_protocol::crypto::merkle::mmr::{MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtProof}; use miden_protocol::note::{NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{RwLock, mpsc, oneshot}; use tracing::{info, instrument}; use crate::account_state_forest::{AccountStateForest, WitnessError}; @@ -109,10 +109,9 @@ pub struct TransactionInputs { /// The rollup state. /// -/// All in-memory state is accessed lock-free. A single writer task (serialized by a channel) -/// mutates the state and then advances [`committed_block_num`](Self::committed_block_num) with -/// `Release` ordering. Readers load the counter with `Acquire` ordering before accessing shared -/// structures, guaranteeing they see a consistent snapshot. +/// A single writer task (serialized by a channel) mutates the state. In-memory structures are +/// protected by [`RwLock`]; the `RocksDB`-backed nullifier tree is lock-free via [`WriterGuard`]. +/// Readers obtain a [`ReadSnapshot`] which holds read locks on the in-memory structures. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -126,16 +125,28 @@ pub struct State { pub(crate) committed_block_num: AtomicU32, /// Nullifier tree — append-only, backed by `RocksDB`. + /// + /// Lock-free via [`WriterGuard`]: `RocksDB` MVCC provides safe concurrent read access + /// during writes. pub(crate) nullifier_tree: WriterGuard>>, /// Account tree with historical overlay support (50-block history). - pub(crate) account_tree: WriterGuard>, + /// + /// Protected by [`RwLock`]: the in-memory `BTreeMap` overlays are not safe for concurrent + /// mutation during reads. + pub(crate) account_tree: RwLock>, /// Chain MMR (Merkle Mountain Range of block commitments). - pub(crate) blockchain: WriterGuard, + /// + /// Protected by [`RwLock`]: the in-memory `Vec` backing the MMR is not safe for concurrent + /// mutation during reads. + pub(crate) blockchain: RwLock, /// Forest state for account storage maps and vault witnesses. - pub(crate) forest: WriterGuard, + /// + /// Protected by [`RwLock`]: the in-memory `HashMap` backing the forest is not safe for + /// concurrent mutation during reads. + pub(crate) forest: RwLock, /// Channel to the single writer task. writer_tx: mpsc::Sender, @@ -152,25 +163,25 @@ pub struct State { /// A consistent read view of the in-memory state. /// -/// Created by [`State::snapshot()`], which performs an atomic load with `Acquire` ordering on the -/// committed block counter. This establishes a happens-before relationship with the writer's -/// `Release` store, guaranteeing that all tree references obtained through this snapshot reflect -/// a fully consistent state at [`block_num`](Self::block_num). +/// Created by [`State::snapshot()`], which acquires read locks on the in-memory structures and +/// performs an atomic load with `Acquire` ordering on the committed block counter. The read locks +/// guarantee the writer cannot mutate the in-memory structures while any reader holds a snapshot. +/// +/// The nullifier tree is accessed lock-free via [`WriterGuard`] because its `RocksDB` MVCC +/// backend provides safe concurrent read access during writes. /// -/// This is the **only** way for readers to obtain references to the in-memory trees. The -/// [`WriterGuard::as_ref()`] method is `pub(super)`, preventing direct access from outside the -/// `state` module without going through the snapshot barrier. +/// This is the **only** way for readers to obtain references to the in-memory trees. pub(crate) struct ReadSnapshot<'a> { /// The committed block number at the time the snapshot was taken. pub block_num: BlockNumber, - /// Nullifier tree — append-only. + /// Nullifier tree — lock-free (`RocksDB` MVCC). pub nullifier_tree: &'a NullifierTree>, - /// Account tree with historical overlay support. - pub account_tree: &'a AccountTreeWithHistory, - /// Chain MMR. - pub blockchain: &'a Blockchain, - /// Forest state for account storage maps and vault witnesses. - pub forest: &'a AccountStateForest, + /// Account tree with historical overlay support — read-locked. + pub account_tree: tokio::sync::RwLockReadGuard<'a, AccountTreeWithHistory>, + /// Chain MMR — read-locked. + pub blockchain: tokio::sync::RwLockReadGuard<'a, Blockchain>, + /// Forest state for account storage maps and vault witnesses — read-locked. + pub forest: tokio::sync::RwLockReadGuard<'a, AccountStateForest>, } impl State { @@ -240,9 +251,9 @@ impl State { block_store, committed_block_num: AtomicU32::new(latest_block_num.as_u32()), nullifier_tree: WriterGuard::new(nullifier_tree), - account_tree: WriterGuard::new(account_tree), - blockchain: WriterGuard::new(blockchain), - forest: WriterGuard::new(forest), + account_tree: RwLock::new(account_tree), + blockchain: RwLock::new(blockchain), + forest: RwLock::new(forest), writer_tx, termination_ask, proven_tip, @@ -286,24 +297,27 @@ impl State { result_rx.await? } - // STATE ACCESSORS (lock-free) + // STATE ACCESSORS // -------------------------------------------------------------------------------------------- /// Takes a consistent snapshot of all in-memory state. /// - /// Performs an `Acquire` load on the committed block counter, then obtains references to all - /// trees. The `Acquire` ordering guarantees that all writer mutations completed before the - /// corresponding `Release` store are visible through these references. + /// Acquires read locks on the three in-memory structures (account tree, blockchain, forest) + /// and performs an `Acquire` load on the committed block counter. The read locks guarantee + /// the writer cannot mutate these structures while any snapshot is held. + /// + /// The nullifier tree is accessed lock-free via [`WriterGuard`] because its `RocksDB` MVCC + /// backend provides safe concurrent read access during writes. /// /// This is the **only** sanctioned way for readers to access the in-memory trees. - fn snapshot(&self) -> ReadSnapshot<'_> { + async fn snapshot(&self) -> ReadSnapshot<'_> { let block_num = BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)); ReadSnapshot { block_num, nullifier_tree: self.nullifier_tree.as_ref(), - account_tree: self.account_tree.as_ref(), - blockchain: self.blockchain.as_ref(), - forest: self.forest.as_ref(), + account_tree: self.account_tree.read().await, + blockchain: self.blockchain.read().await, + forest: self.forest.read().await, } } @@ -336,7 +350,7 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { @@ -357,7 +371,7 @@ impl State { /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret)] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; nullifiers .iter() .map(|n| snapshot.nullifier_tree.open(n)) @@ -382,7 +396,7 @@ impl State { &self, block_num: Option, ) -> Result, GetCurrentBlockchainDataError> { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; if let Some(number) = block_num && number == snapshot.block_num { @@ -450,7 +464,7 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let latest_block_num = snapshot.block_num; let highest_block_num = @@ -541,7 +555,7 @@ impl State { blocks.extend(note_proof_reference_blocks); let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers).await?; // Fetch the block headers for all blocks in the partial MMR plus the latest one which will // be used as the previous block header of the block being built. @@ -579,8 +593,7 @@ impl State { /// Get account and nullifier witnesses for the requested account IDs and nullifiers as well as /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its /// number is removed from `blocks` and returned separately. - #[expect(clippy::type_complexity)] - fn get_block_inputs_witnesses( + async fn get_block_inputs_witnesses( &self, blocks: &mut BTreeSet, account_ids: &[AccountId], @@ -594,7 +607,7 @@ impl State { ), GetBlockInputsError, > { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let latest_block_number = snapshot.block_num; // If `blocks` is empty, use the latest block number which will never trigger the error. @@ -643,7 +656,7 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); @@ -723,7 +736,7 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_account_witness(block_num, account_id)?; + let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; let details = if let Some(request) = details { Some(self.fetch_public_account_details(account_id, block_num, request).await?) @@ -738,12 +751,12 @@ impl State { /// /// If `block_num` is provided, returns the witness at that historical block; /// otherwise, returns the witness at the latest block. - fn get_account_witness( + async fn get_account_witness( &self, block_num: Option, account_id: AccountId, ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { @@ -795,7 +808,7 @@ impl State { } // Validate block exists in the blockchain before querying the database. - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; if block_num > snapshot.block_num { return Err(GetAccountError::UnknownBlock(block_num)); } @@ -959,14 +972,13 @@ impl State { } /// Returns vault asset witnesses for the specified account and block number. - #[expect(clippy::unused_async)] pub async fn get_vault_asset_witnesses( &self, account_id: AccountId, block_num: BlockNumber, vault_keys: BTreeSet, ) -> Result, WitnessError> { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let witnesses = snapshot.forest.get_vault_asset_witnesses(account_id, block_num, vault_keys)?; Ok(witnesses) @@ -977,7 +989,6 @@ impl State { /// /// Note that the `raw_key` is the raw, user-provided key that needs to be hashed in order to /// get the actual key into the storage map. - #[expect(clippy::unused_async)] pub async fn get_storage_map_witness( &self, account_id: AccountId, @@ -985,7 +996,7 @@ impl State { block_num: BlockNumber, raw_key: StorageMapKey, ) -> Result { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let witness = snapshot .forest .get_storage_map_witness(account_id, slot_name, block_num, raw_key)?; diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 66da87abd..36455fd27 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -77,7 +77,7 @@ impl State { let from_forest = (block_from + 1).as_usize(); let to_forest = block_to.as_usize(); - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let mmr_delta = snapshot .blockchain .as_mmr() @@ -101,7 +101,7 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { - let snapshot = self.snapshot(); + let snapshot = self.snapshot().await; let block_end = *block_range.end(); let note_tags: Arc<[u32]> = note_tags.into(); diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index bb36b9d6d..e4c0da296 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -37,12 +37,10 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// ## Consistency model /// -/// This function is the sole writer to all in-memory state. Readers access the same structures -/// concurrently without locks because: -/// -/// - The data structures are append-only or overlay-based (keyed by block number). -/// - All mutations are completed before the atomic block counter is advanced (`Release`). -/// - Readers load the counter (`Acquire`) before querying, establishing happens-before. +/// This function is the sole writer to all in-memory state. The nullifier tree (backed by +/// `RocksDB` with MVCC) is accessed lock-free via [`WriterGuard`]. The in-memory structures +/// (`account_tree`, `blockchain`, `forest`) are protected by [`RwLock`]: the writer acquires +/// read locks during validation and write locks during the brief mutation phase. /// /// The DB transaction is committed independently. Readers gate visibility through the atomic /// block counter, so the brief window where the DB has block N+1 but the counter still says N @@ -99,12 +97,12 @@ async fn apply_block_inner( ); // Compute mutations required for updating account and nullifier trees. - // SAFETY: This is the single writer task, serialized by the channel. No concurrent - // mutations to these structures are possible. + // The nullifier tree uses WriterGuard (RocksDB MVCC — safe for concurrent access). + // The account tree and blockchain use RwLock (in-memory — need read lock for validation). let (nullifier_tree_update, account_tree_update) = { let nullifier_tree = unsafe { state.nullifier_tree.as_mut() }; - let account_tree = unsafe { state.account_tree.as_mut() }; - let blockchain = state.blockchain.as_ref(); + let account_tree = state.account_tree.read().await; + let blockchain = state.blockchain.read().await; let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); @@ -218,22 +216,30 @@ async fn apply_block_inner( block_save_task.await??; // Apply in-memory mutations. - // SAFETY: This is the single writer task, no concurrent mutations. + + // Nullifier tree: lock-free via WriterGuard (RocksDB MVCC). + // SAFETY: This is the single writer task, serialized by the channel. unsafe { state .nullifier_tree .as_mut() .apply_mutations(nullifier_tree_update) .expect("Unreachable: mutations were computed from the current tree state"); - state - .account_tree - .as_mut() + } + + // In-memory structures: acquire write locks for the brief mutation phase. + { + let mut account_tree = state.account_tree.write().await; + let mut blockchain = state.blockchain.write().await; + let mut forest = state.forest.write().await; + + account_tree .apply_mutations(account_tree_update) .expect("Unreachable: mutations were computed from the current tree state"); - state.blockchain.as_mut().push(block_commitment); + blockchain.push(block_commitment); - state.forest.as_mut().apply_block_updates(block_num, account_deltas)?; + forest.apply_block_updates(block_num, account_deltas)?; } // PUBLISH: Advance the atomic block counter with Release ordering. From 2f71740ac288e45d9841429bf4de4bf85ed6ad7e Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 8 Apr 2026 13:09:00 +1200 Subject: [PATCH 04/48] Lock and DB commit --- crates/store/src/state/writer.rs | 42 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index e4c0da296..1e3636f78 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -40,11 +40,12 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// This function is the sole writer to all in-memory state. The nullifier tree (backed by /// `RocksDB` with MVCC) is accessed lock-free via [`WriterGuard`]. The in-memory structures /// (`account_tree`, `blockchain`, `forest`) are protected by [`RwLock`]: the writer acquires -/// read locks during validation and write locks during the brief mutation phase. +/// read locks during validation and write locks during the commit phase. /// -/// The DB transaction is committed independently. Readers gate visibility through the atomic -/// block counter, so the brief window where the DB has block N+1 but the counter still says N -/// is invisible to readers. +/// The write locks are acquired **before** the DB commit and held through both the DB commit +/// and in-memory mutations. This ensures readers never observe a state where the DB has +/// block N+1 but the in-memory structures still reflect block N. The atomic block counter is +/// advanced before the write locks are released. #[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( @@ -204,8 +205,15 @@ async fn apply_block_inner( }, )); - // Commit to DB. No oneshot synchronization dance needed — readers gate visibility - // through the atomic block counter, not through DB transaction timing. + // Acquire write locks BEFORE committing to DB. This ensures readers cannot observe a + // state where the DB has block N+1 but the in-memory structures still reflect block N. + // The write locks block all readers (via snapshot()) for the duration of the DB commit + // and in-memory mutations, guaranteeing atomicity from the reader's perspective. + let mut account_tree = state.account_tree.write().await; + let mut blockchain = state.blockchain.write().await; + let mut forest = state.forest.write().await; + + // Commit to DB while holding write locks. state .db .apply_block(signed_block, notes, proving_inputs) @@ -227,24 +235,16 @@ async fn apply_block_inner( .expect("Unreachable: mutations were computed from the current tree state"); } - // In-memory structures: acquire write locks for the brief mutation phase. - { - let mut account_tree = state.account_tree.write().await; - let mut blockchain = state.blockchain.write().await; - let mut forest = state.forest.write().await; - - account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); + account_tree + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); - blockchain.push(block_commitment); + blockchain.push(block_commitment); - forest.apply_block_updates(block_num, account_deltas)?; - } + forest.apply_block_updates(block_num, account_deltas)?; - // PUBLISH: Advance the atomic block counter with Release ordering. - // All mutations above are guaranteed visible to any reader that subsequently loads this - // counter with Acquire ordering. + // Advance the atomic block counter before releasing write locks, so readers that + // acquire read locks immediately after see the updated block number. state.committed_block_num.store(block_num.as_u32(), Ordering::Release); info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); From 8404f436a073d203b75ef410f4b3727d4d2d58d8 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 09:43:05 +1200 Subject: [PATCH 05/48] Remove RWLocks and use ArcSwap --- Cargo.lock | 22 ++- Cargo.toml | 4 + crates/store/Cargo.toml | 1 + crates/store/src/account_state_forest/mod.rs | 1 + crates/store/src/accounts/mod.rs | 2 +- crates/store/src/server/ntx_builder.rs | 2 - crates/store/src/state/mod.rs | 188 +++++++++---------- crates/store/src/state/sync_state.rs | 4 +- crates/store/src/state/writer.rs | 63 ++++--- crates/store/src/state/writer_guard.rs | 38 ++-- 10 files changed, 161 insertions(+), 164 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bae361aa3..17914a71a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -2955,8 +2964,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0a034a460e27723dcfdf25effffab84331c3b46b13e7a1bd674197cc71bfe" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" dependencies = [ "blake3", "cc", @@ -2998,8 +3006,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8bf6ebde028e79bcc61a3632d2f375a5cc64caa17d014459f75015238cb1e08" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" dependencies = [ "quote", "syn 2.0.117", @@ -3026,8 +3033,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38011348f4fb4c9e5ce1f471203d024721c00e3b60a91aa91aaefe6738d8b5ea" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3344,6 +3350,7 @@ name = "miden-node-store" version = "0.15.0" dependencies = [ "anyhow", + "arc-swap", "assert_matches", "build-rs", "criterion", @@ -3643,8 +3650,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff78082e9b4ca89863e68da01b35f8a4029ee6fd912e39fa41fde4273a7debab" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" dependencies = [ "p3-field", "p3-goldilocks", diff --git a/Cargo.toml b/Cargo.toml index 7bc7d1ede..4b4fa8939 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,3 +142,7 @@ should_panic_without_expect = "allow" # We don't care about the specific panic # Configure `cargo-typos` [workspace.metadata.typos] files.extend-exclude = ["*.svg"] # Ignore SVG files. + +[patch.crates-io] +miden-crypto = { git = "https://github.com/0xmiden/crypto", branch = "sergerad-clone" } +miden-serde-utils = { git = "https://github.com/0xmiden/crypto", branch = "sergerad-clone" } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index aa8cfc3a2..10462e0e1 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -15,6 +15,7 @@ version.workspace = true workspace = true [dependencies] +arc-swap = "1" anyhow = { workspace = true } deadpool = { default-features = false, features = ["managed", "rt_tokio_1"], version = "0.12" } deadpool-diesel = { features = ["sqlite"], version = "0.6" } diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 558c79df7..9cbc64a48 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -64,6 +64,7 @@ pub enum WitnessError { // ================================================================================================ /// Container for forest-related state that needs to be updated atomically. +#[derive(Clone)] pub(crate) struct AccountStateForest { /// `LargeSmtForest` for efficient account storage reconstruction. /// Populated during block import with storage and vault SMTs. diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index f9815190b..298bb232a 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -115,7 +115,7 @@ impl HistoricalOverlay { /// This structure maintains a sliding window of historical account states by storing /// reversion data (mutations that undo changes). Historical witnesses are reconstructed /// by starting from the latest state and applying reversion overlays backwards in time. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct AccountTreeWithHistory { /// The current block number (latest state). block_number: BlockNumber, diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index ff1b6eb33..763e5b9b3 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -259,7 +259,6 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let asset_witnesses = self .state .get_vault_asset_witnesses(account_id, block_num, vault_keys) - .await .map_err(internal_error)?; // Convert AssetWitness to protobuf format by extracting witness data. @@ -313,7 +312,6 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let storage_witness = self .state .get_storage_map_witness(account_id, &slot_name, block_num, map_key) - .await .map_err(internal_error)?; // Convert StorageMapWitness to protobuf format by extracting witness data. diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 4d604e26d..fed435698 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -2,19 +2,19 @@ //! //! The [State] provides data access and modification methods. A single writer task, serialized by //! a channel, applies block mutations. In-memory structures (`account_tree`, `blockchain`, -//! `forest`) are protected by [`RwLock`] because their internal collections (`BTreeMap`, `Vec`, -//! `HashMap`) are not safe for concurrent mutation during reads. The `RocksDB`-backed nullifier -//! tree uses a lock-free [`WriterGuard`] because `RocksDB` MVCC provides snapshot isolation. +//! `forest`) are held in an [`Arc`] behind an [`ArcSwap`](arc_swap::ArcSwap), providing wait-free +//! reads with no lock contention. The `RocksDB`-backed nullifier tree uses a lock-free +//! [`WriterGuard`] because `RocksDB` MVCC provides snapshot isolation. //! -//! Readers obtain a [`ReadSnapshot`] via [`State::snapshot()`], which acquires read locks on the -//! in-memory structures. The writer acquires write locks only during the brief mutation phase. +//! Readers obtain an `Arc` via [`State::snapshot()`] (wait-free, no locks). +//! The writer clones the current state, mutates the clone, and atomically swaps the pointer. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; use std::path::Path; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use arc_swap::ArcSwap; use miden_node_proto::BlockProofRequest; use miden_node_proto::domain::account::{ AccountDetailRequest, @@ -42,7 +42,7 @@ use miden_protocol::crypto::merkle::mmr::{MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtProof}; use miden_protocol::note::{NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{RwLock, mpsc, oneshot}; +use tokio::sync::{mpsc, oneshot}; use tracing::{info, instrument}; use crate::account_state_forest::{AccountStateForest, WitnessError}; @@ -104,14 +104,43 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } +// IN-MEMORY STATE +// ================================================================================================ + +/// A consistent, immutable snapshot of all in-memory state at a given block. +/// +/// Held behind an [`ArcSwap`] in [`State`]. +/// +/// ## Performance +/// +/// - **Readers** obtain an `Arc` via [`State::snapshot()`], which calls +/// `ArcSwap::load_full()` — a wait-free atomic refcount bump with no data cloning. The returned +/// `Arc` is a frozen view: even if the writer swaps in a new state, readers continue to see their +/// snapshot unchanged until they drop the `Arc`. +/// +/// - **Writer** (once per block) deep-clones this struct via `InMemoryState::clone()` to produce a +/// mutable copy, applies mutations, and atomically swaps the pointer via `ArcSwap::store()`. This +/// is the only place where a deep clone occurs. +#[derive(Clone)] +pub(crate) struct InMemoryState { + /// The committed block number for this snapshot. + pub block_num: BlockNumber, + /// Account tree with historical overlay support. + pub account_tree: AccountTreeWithHistory, + /// Chain MMR (Merkle Mountain Range of block commitments). + pub blockchain: Blockchain, + /// Forest state for account storage maps and vault witnesses. + pub forest: AccountStateForest, +} + // CHAIN STATE // ================================================================================================ /// The rollup state. /// /// A single writer task (serialized by a channel) mutates the state. In-memory structures are -/// protected by [`RwLock`]; the `RocksDB`-backed nullifier tree is lock-free via [`WriterGuard`]. -/// Readers obtain a [`ReadSnapshot`] which holds read locks on the in-memory structures. +/// held in an `Arc` behind an [`ArcSwap`], providing wait-free reads. The +/// `RocksDB`-backed nullifier tree is lock-free via [`WriterGuard`]. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -120,33 +149,18 @@ pub struct State { /// The block store which stores full block contents for all blocks. pub(crate) block_store: Arc, - /// Atomic block counter — the version pointer for readers. - /// Advanced by the writer AFTER all in-memory mutations are complete. - pub(crate) committed_block_num: AtomicU32, - /// Nullifier tree — append-only, backed by `RocksDB`. /// /// Lock-free via [`WriterGuard`]: `RocksDB` MVCC provides safe concurrent read access /// during writes. pub(crate) nullifier_tree: WriterGuard>>, - /// Account tree with historical overlay support (50-block history). + /// All in-memory state (account tree, blockchain MMR, forest) held atomically. /// - /// Protected by [`RwLock`]: the in-memory `BTreeMap` overlays are not safe for concurrent - /// mutation during reads. - pub(crate) account_tree: RwLock>, - - /// Chain MMR (Merkle Mountain Range of block commitments). - /// - /// Protected by [`RwLock`]: the in-memory `Vec` backing the MMR is not safe for concurrent - /// mutation during reads. - pub(crate) blockchain: RwLock, - - /// Forest state for account storage maps and vault witnesses. - /// - /// Protected by [`RwLock`]: the in-memory `HashMap` backing the forest is not safe for - /// concurrent mutation during reads. - pub(crate) forest: RwLock, + /// Readers call `snapshot()` which returns `Arc` via a wait-free atomic + /// refcount bump — no data cloning. The writer deep-clones once per block, mutates the + /// copy, and atomically swaps via `ArcSwap::store()`. + pub(crate) in_memory: ArcSwap, /// Channel to the single writer task. writer_tx: mpsc::Sender, @@ -158,32 +172,6 @@ pub struct State { proven_tip: ProvenTipReader, } -// READ SNAPSHOT -// ================================================================================================ - -/// A consistent read view of the in-memory state. -/// -/// Created by [`State::snapshot()`], which acquires read locks on the in-memory structures and -/// performs an atomic load with `Acquire` ordering on the committed block counter. The read locks -/// guarantee the writer cannot mutate the in-memory structures while any reader holds a snapshot. -/// -/// The nullifier tree is accessed lock-free via [`WriterGuard`] because its `RocksDB` MVCC -/// backend provides safe concurrent read access during writes. -/// -/// This is the **only** way for readers to obtain references to the in-memory trees. -pub(crate) struct ReadSnapshot<'a> { - /// The committed block number at the time the snapshot was taken. - pub block_num: BlockNumber, - /// Nullifier tree — lock-free (`RocksDB` MVCC). - pub nullifier_tree: &'a NullifierTree>, - /// Account tree with historical overlay support — read-locked. - pub account_tree: tokio::sync::RwLockReadGuard<'a, AccountTreeWithHistory>, - /// Chain MMR — read-locked. - pub blockchain: tokio::sync::RwLockReadGuard<'a, Blockchain>, - /// Forest state for account storage maps and vault witnesses — read-locked. - pub forest: tokio::sync::RwLockReadGuard<'a, AccountStateForest>, -} - impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- @@ -246,14 +234,18 @@ impl State { // Create the writer channel. Buffer size of 1: only one block can be in flight. let (writer_tx, writer_rx) = mpsc::channel(1); + let in_memory = ArcSwap::from_pointee(InMemoryState { + block_num: latest_block_num, + account_tree, + blockchain, + forest, + }); + let state = Arc::new(Self { db, block_store, - committed_block_num: AtomicU32::new(latest_block_num.as_u32()), nullifier_tree: WriterGuard::new(nullifier_tree), - account_tree: RwLock::new(account_tree), - blockchain: RwLock::new(blockchain), - forest: RwLock::new(forest), + in_memory, writer_tx, termination_ask, proven_tip, @@ -302,40 +294,28 @@ impl State { /// Takes a consistent snapshot of all in-memory state. /// - /// Acquires read locks on the three in-memory structures (account tree, blockchain, forest) - /// and performs an `Acquire` load on the committed block counter. The read locks guarantee - /// the writer cannot mutate these structures while any snapshot is held. + /// Returns an `Arc` via a wait-free `ArcSwap::load_full()`. This performs + /// only an atomic refcount increment — **no data is cloned**. No locks are acquired. /// - /// The nullifier tree is accessed lock-free via [`WriterGuard`] because its `RocksDB` MVCC - /// backend provides safe concurrent read access during writes. + /// The returned `Arc` is a frozen view: it keeps the snapshot alive for as long as needed, + /// even if the writer swaps in a new state in the meantime. Readers holding the old `Arc` + /// are completely unaffected by the swap. /// - /// This is the **only** sanctioned way for readers to access the in-memory trees. - async fn snapshot(&self) -> ReadSnapshot<'_> { - let block_num = BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)); - ReadSnapshot { - block_num, - nullifier_tree: self.nullifier_tree.as_ref(), - account_tree: self.account_tree.read().await, - blockchain: self.blockchain.read().await, - forest: self.forest.read().await, - } + /// The nullifier tree is accessed separately via [`WriterGuard`] (lock-free, `RocksDB` MVCC). + fn snapshot(&self) -> Arc { + self.in_memory.load_full() } /// Returns the effective chain tip for the given finality level. /// - /// - [`Finality::Committed`]: returns the latest committed block number from the atomic counter - /// (Acquire ordering). + /// - [`Finality::Committed`]: returns the latest committed block number from the in-memory + /// state snapshot (wait-free via `ArcSwap`). /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch /// channel, updated by the proof scheduler). - /// - /// This method only returns a block number — it does not access any trees, so the Acquire - /// load here is sufficient without a full snapshot. #[expect(clippy::unused_async)] pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { match finality { - Finality::Committed => { - BlockNumber::from(self.committed_block_num.load(Ordering::Acquire)) - }, + Finality::Committed => self.snapshot().block_num, Finality::Proven => self.proven_tip.read(), } } @@ -350,7 +330,7 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { @@ -371,10 +351,10 @@ impl State { /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret)] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { - let snapshot = self.snapshot().await; + let nullifier_tree = self.nullifier_tree.as_ref(); nullifiers .iter() - .map(|n| snapshot.nullifier_tree.open(n)) + .map(|n| nullifier_tree.open(n)) .map(NullifierWitness::into_proof) .collect() } @@ -396,7 +376,7 @@ impl State { &self, block_num: Option, ) -> Result, GetCurrentBlockchainDataError> { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); if let Some(number) = block_num && number == snapshot.block_num { @@ -464,7 +444,7 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let latest_block_num = snapshot.block_num; let highest_block_num = @@ -482,8 +462,8 @@ impl State { blocks.remove(&latest_block_num); // SAFETY: - // - The latest block num was retrieved from the committed counter and the blockchain is - // guaranteed to have been updated to at least that block (happens-before). + // - The latest block num was retrieved from the snapshot and the blockchain within the + // snapshot is guaranteed to be consistent with that block number. // - We have checked that no block number in the blocks set is greater than latest block // number *and* latest block num was removed from the set. let partial_mmr = @@ -555,7 +535,7 @@ impl State { blocks.extend(note_proof_reference_blocks); let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers).await?; + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; // Fetch the block headers for all blocks in the partial MMR plus the latest one which will // be used as the previous block header of the block being built. @@ -593,7 +573,7 @@ impl State { /// Get account and nullifier witnesses for the requested account IDs and nullifiers as well as /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its /// number is removed from `blocks` and returned separately. - async fn get_block_inputs_witnesses( + fn get_block_inputs_witnesses( &self, blocks: &mut BTreeSet, account_ids: &[AccountId], @@ -607,7 +587,7 @@ impl State { ), GetBlockInputsError, > { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let latest_block_number = snapshot.block_num; // If `blocks` is empty, use the latest block number which will never trigger the error. @@ -637,10 +617,11 @@ impl State { // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent or // not as this is done as part of proposing the block. + let nullifier_tree = self.nullifier_tree.as_ref(); let nullifier_witnesses: BTreeMap = nullifiers .iter() .copied() - .map(|nullifier| (nullifier, snapshot.nullifier_tree.open(&nullifier))) + .map(|nullifier| (nullifier, nullifier_tree.open(&nullifier))) .collect(); Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) @@ -656,7 +637,7 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); @@ -674,11 +655,12 @@ impl State { }); } + let nullifier_tree = self.nullifier_tree.as_ref(); let nullifiers = nullifiers .iter() .map(|nullifier| NullifierInfo { nullifier: *nullifier, - block_num: snapshot.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + block_num: nullifier_tree.get_block_num(nullifier).unwrap_or_default(), }) .collect(); @@ -736,7 +718,7 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; + let (block_num, witness) = self.get_account_witness(block_num, account_id)?; let details = if let Some(request) = details { Some(self.fetch_public_account_details(account_id, block_num, request).await?) @@ -751,12 +733,12 @@ impl State { /// /// If `block_num` is provided, returns the witness at that historical block; /// otherwise, returns the witness at the latest block. - async fn get_account_witness( + fn get_account_witness( &self, block_num: Option, account_id: AccountId, ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { @@ -808,7 +790,7 @@ impl State { } // Validate block exists in the blockchain before querying the database. - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); if block_num > snapshot.block_num { return Err(GetAccountError::UnknownBlock(block_num)); } @@ -972,13 +954,13 @@ impl State { } /// Returns vault asset witnesses for the specified account and block number. - pub async fn get_vault_asset_witnesses( + pub fn get_vault_asset_witnesses( &self, account_id: AccountId, block_num: BlockNumber, vault_keys: BTreeSet, ) -> Result, WitnessError> { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let witnesses = snapshot.forest.get_vault_asset_witnesses(account_id, block_num, vault_keys)?; Ok(witnesses) @@ -989,14 +971,14 @@ impl State { /// /// Note that the `raw_key` is the raw, user-provided key that needs to be hashed in order to /// get the actual key into the storage map. - pub async fn get_storage_map_witness( + pub fn get_storage_map_witness( &self, account_id: AccountId, slot_name: &StorageSlotName, block_num: BlockNumber, raw_key: StorageMapKey, ) -> Result { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let witness = snapshot .forest .get_storage_map_witness(account_id, slot_name, block_num, raw_key)?; diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 36455fd27..66da87abd 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -77,7 +77,7 @@ impl State { let from_forest = (block_from + 1).as_usize(); let to_forest = block_to.as_usize(); - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let mmr_delta = snapshot .blockchain .as_mmr() @@ -101,7 +101,7 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { - let snapshot = self.snapshot().await; + let snapshot = self.snapshot(); let block_end = *block_range.end(); let note_tags: Arc<[u32]> = note_tags.into(); diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 1e3636f78..46afd9e16 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -1,5 +1,4 @@ use std::sync::Arc; -use std::sync::atomic::Ordering; use miden_node_proto::BlockProofRequest; use miden_node_utils::ErrorReport; @@ -13,7 +12,7 @@ use tracing::{Instrument, info, info_span, instrument}; use crate::db::NoteRecord; use crate::errors::{ApplyBlockError, InvalidBlockError}; -use crate::state::State; +use crate::state::{InMemoryState, State}; use crate::{COMPONENT, HistoricalError}; /// A request to apply a new block, sent through the writer channel. @@ -39,13 +38,19 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// This function is the sole writer to all in-memory state. The nullifier tree (backed by /// `RocksDB` with MVCC) is accessed lock-free via [`WriterGuard`]. The in-memory structures -/// (`account_tree`, `blockchain`, `forest`) are protected by [`RwLock`]: the writer acquires -/// read locks during validation and write locks during the commit phase. +/// (`account_tree`, `blockchain`, `forest`) are held in an `Arc` behind an +/// `ArcSwap`. The writer loads the current state, validates against it, commits to DB (no locks +/// held), then deep-clones the state, applies mutations, and atomically swaps the pointer. /// -/// The write locks are acquired **before** the DB commit and held through both the DB commit -/// and in-memory mutations. This ensures readers never observe a state where the DB has -/// block N+1 but the in-memory structures still reflect block N. The atomic block counter is -/// advanced before the write locks are released. +/// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an +/// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either +/// the old or new state, never a partial update. Readers holding an `Arc` to the old state are +/// completely unaffected by the swap. +/// +/// ## Performance +/// +/// The only deep clone of `InMemoryState` occurs once per block in this function. Readers pay +/// only an atomic refcount bump per `snapshot()` call. #[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( @@ -97,13 +102,14 @@ async fn apply_block_inner( async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), ); + // Load the current in-memory state snapshot for validation (wait-free, no locks). + let snapshot = state.in_memory.load_full(); + // Compute mutations required for updating account and nullifier trees. // The nullifier tree uses WriterGuard (RocksDB MVCC — safe for concurrent access). - // The account tree and blockchain use RwLock (in-memory — need read lock for validation). + // The account tree and blockchain are read from the snapshot (no locks needed). let (nullifier_tree_update, account_tree_update) = { let nullifier_tree = unsafe { state.nullifier_tree.as_mut() }; - let account_tree = state.account_tree.read().await; - let blockchain = state.blockchain.read().await; let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); @@ -119,7 +125,7 @@ async fn apply_block_inner( } // new_block.chain_root must be equal to the chain MMR root prior to the update. - let peaks = blockchain.peaks(); + let peaks = snapshot.blockchain.peaks(); if peaks.hash_peaks() != header.chain_commitment() { return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); } @@ -139,7 +145,8 @@ async fn apply_block_inner( } // Compute update for account tree. - let account_tree_update = account_tree + let account_tree_update = snapshot + .account_tree .compute_mutations( body.updated_accounts() .iter() @@ -205,15 +212,8 @@ async fn apply_block_inner( }, )); - // Acquire write locks BEFORE committing to DB. This ensures readers cannot observe a - // state where the DB has block N+1 but the in-memory structures still reflect block N. - // The write locks block all readers (via snapshot()) for the duration of the DB commit - // and in-memory mutations, guaranteeing atomicity from the reader's perspective. - let mut account_tree = state.account_tree.write().await; - let mut blockchain = state.blockchain.write().await; - let mut forest = state.forest.write().await; - - // Commit to DB while holding write locks. + // Commit to DB. No locks are held — this is the key advantage over the previous design. + // Readers continue to see the old in-memory state (via their Arc) while the DB commits. state .db .apply_block(signed_block, notes, proving_inputs) @@ -223,7 +223,9 @@ async fn apply_block_inner( // Await the block store save task. block_save_task.await??; - // Apply in-memory mutations. + // Deep-clone the in-memory state to produce an owned mutable copy for applying mutations. + // This is the only deep clone per block — readers pay only an atomic refcount bump. + let mut new_state = InMemoryState::clone(&snapshot); // Nullifier tree: lock-free via WriterGuard (RocksDB MVCC). // SAFETY: This is the single writer task, serialized by the channel. @@ -235,17 +237,20 @@ async fn apply_block_inner( .expect("Unreachable: mutations were computed from the current tree state"); } - account_tree + new_state + .account_tree .apply_mutations(account_tree_update) .expect("Unreachable: mutations were computed from the current tree state"); - blockchain.push(block_commitment); + new_state.blockchain.push(block_commitment); + + new_state.forest.apply_block_updates(block_num, account_deltas)?; - forest.apply_block_updates(block_num, account_deltas)?; + new_state.block_num = block_num; - // Advance the atomic block counter before releasing write locks, so readers that - // acquire read locks immediately after see the updated block number. - state.committed_block_num.store(block_num.as_u32(), Ordering::Release); + // Atomically publish the new state. Readers that call snapshot() after this point + // will see the updated state. Readers holding the old Arc continue unaffected. + state.in_memory.store(Arc::new(new_state)); info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs index 814acf46a..6661bfadd 100644 --- a/crates/store/src/state/writer_guard.rs +++ b/crates/store/src/state/writer_guard.rs @@ -10,25 +10,25 @@ use std::cell::UnsafeCell; /// 1. **Single writer**: Only one task (the writer, serialized by a channel) may call /// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type /// system. -/// 2. **Publish barrier**: After completing all mutations, the writer performs an atomic store with -/// [`Release`](std::sync::atomic::Ordering::Release) ordering on a shared block counter. -/// 3. **Subscribe barrier**: Before calling [`as_ref()`](Self::as_ref), readers perform an atomic -/// load with [`Acquire`](std::sync::atomic::Ordering::Acquire) ordering on the same counter. -/// 4. The `Release`/`Acquire` pair establishes a *happens-before* relationship, guaranteeing that -/// all mutations performed before the `Release` store are visible to any reader that observes -/// the updated counter value. +/// 2. **Publish barrier**: After completing all mutations, the writer performs an `ArcSwap::store` +/// on the shared in-memory state, which includes a `Release` memory barrier. +/// 3. **Subscribe barrier**: Before calling [`as_ref()`](Self::as_ref), readers perform an +/// `ArcSwap::load_full` which includes an `Acquire` memory barrier. +/// 4. The barrier pair establishes a *happens-before* relationship, guaranteeing that all mutations +/// performed before the store are visible to any reader that observes the updated state. /// /// Because the wrapped data structures are append-only or overlay-based (keyed by block number), -/// readers that observe an older counter value will simply query at that older block number, -/// which is safe. +/// readers that observe an older state will simply query at that older block number, which is +/// safe. pub struct WriterGuard { inner: UnsafeCell, } // SAFETY: The single-writer invariant is enforced by the channel-based writer task architecture. // Readers only call `as_ref()` which returns `&T`. The writer completes all mutations before -// advancing the atomic block counter (`Release`), and readers load the counter (`Acquire`) -// before accessing the data. This guarantees no data races. +// performing an `ArcSwap::store` (which includes a Release barrier), and readers perform an +// `ArcSwap::load_full` (which includes an Acquire barrier) before accessing the data. +// This guarantees no data races. unsafe impl Send for WriterGuard {} unsafe impl Sync for WriterGuard {} @@ -41,13 +41,13 @@ impl WriterGuard { /// Returns a shared reference to the wrapped value. /// /// Safe for any reader thread. The data is guaranteed to be in a consistent state because - /// the caller must have loaded the atomic block counter with `Acquire` ordering before - /// calling this method, establishing a happens-before relationship with the writer's - /// `Release` store. + /// the caller accesses shared state through `ArcSwap::load_full` (which includes an + /// `Acquire` barrier), establishing a happens-before relationship with the writer's + /// `ArcSwap::store` (which includes a `Release` barrier). pub(super) fn as_ref(&self) -> &T { - // SAFETY: The writer completes all mutations before the Release store on the block - // counter. The reader loads the counter with Acquire before calling this. The - // Acquire/Release pair ensures all writes are visible. + // SAFETY: The writer completes all mutations before the ArcSwap::store (Release barrier). + // The reader performs ArcSwap::load_full (Acquire barrier) before calling this. + // The barrier pair ensures all writes are visible. unsafe { &*self.inner.get() } } @@ -57,8 +57,8 @@ impl WriterGuard { /// /// Must only be called from the single writer task. The caller must ensure: /// - No other calls to `as_mut()` are concurrent (enforced by channel serialization). - /// - All mutations through the returned reference are completed before performing a `Release` - /// store on the shared block counter. + /// - All mutations through the returned reference are completed before performing an + /// `ArcSwap::store` on the shared in-memory state. #[expect(clippy::mut_from_ref)] pub unsafe fn as_mut(&self) -> &mut T { unsafe { &mut *self.inner.get() } From 2b0c1accbd843b2af0afbc419c164fd1a9859923 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 09:50:02 +1200 Subject: [PATCH 06/48] Fix lint --- Cargo.toml | 4 ++-- crates/store/Cargo.toml | 2 +- crates/store/src/state/mod.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4b4fa8939..ca74beb7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,5 +144,5 @@ should_panic_without_expect = "allow" # We don't care about the specific panic files.extend-exclude = ["*.svg"] # Ignore SVG files. [patch.crates-io] -miden-crypto = { git = "https://github.com/0xmiden/crypto", branch = "sergerad-clone" } -miden-serde-utils = { git = "https://github.com/0xmiden/crypto", branch = "sergerad-clone" } +miden-crypto = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } +miden-serde-utils = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 10462e0e1..e6230a082 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -15,8 +15,8 @@ version.workspace = true workspace = true [dependencies] -arc-swap = "1" anyhow = { workspace = true } +arc-swap = "1" deadpool = { default-features = false, features = ["managed", "rt_tokio_1"], version = "0.12" } deadpool-diesel = { features = ["sqlite"], version = "0.6" } diesel = { features = ["numeric", "sqlite"], version = "2.3" } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index fed435698..ec18f8512 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -573,6 +573,7 @@ impl State { /// Get account and nullifier witnesses for the requested account IDs and nullifiers as well as /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its /// number is removed from `blocks` and returned separately. + #[expect(clippy::type_complexity)] fn get_block_inputs_witnesses( &self, blocks: &mut BTreeSet, From 30249a8ef18b82410a4f76ae2eff2d1fe075b467 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 12:36:52 +1200 Subject: [PATCH 07/48] Drop snapshot sooner --- crates/store/src/state/mod.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index ec18f8512..a24d8be15 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -330,10 +330,10 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - let snapshot = self.snapshot(); let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { + let snapshot = self.snapshot(); let mmr_proof = snapshot.blockchain.open(header.block_num())?; Some(mmr_proof) } else { @@ -638,14 +638,23 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); - let snapshot = self.snapshot(); + let (new_account_id_prefix_is_unique, account_commitment) = { + let snapshot = self.snapshot(); - let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); + let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); - let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!snapshot.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) - } else { - None + if account_commitment.is_empty() { + ( + Some( + !snapshot + .account_tree + .contains_account_id_prefix_in_latest(account_id.prefix()), + ), + account_commitment, + ) + } else { + (None, account_commitment) + } }; // Non-unique account Id prefixes for new accounts are not allowed. From 43ee1cfade97aca9094e6d328fef929847122369 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 13:27:33 +1200 Subject: [PATCH 08/48] Drop snapshot earlier in get_block_inputs_witnesses --- crates/store/src/state/mod.rs | 62 ++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index a24d8be15..a99294aa1 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -588,33 +588,41 @@ impl State { ), GetBlockInputsError, > { - let snapshot = self.snapshot(); - let latest_block_number = snapshot.block_num; - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } - - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove its - // inclusion in the chain. - blocks.remove(&latest_block_number); - - let partial_mmr = - snapshot.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); + // Take a snapshot and extract everything we need, then drop it so readers of newer + // snapshots aren't held up by this Arc. + let (latest_block_number, partial_mmr, account_witnesses) = { + let snapshot = self.snapshot(); + let latest_block_number = snapshot.block_num; + + // If `blocks` is empty, use the latest block number which will never trigger the error. + let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); + if highest_block_number > latest_block_number { + return Err(GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number, + latest_block_number, + }); + } - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, snapshot.account_tree.open_latest(account_id))) - .collect::>(); + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + let partial_mmr = snapshot + .blockchain + .partial_mmr_from_blocks(blocks, latest_block_number) + .expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, snapshot.account_tree.open_latest(account_id))) + .collect::>(); + + (latest_block_number, partial_mmr, account_witnesses) + }; // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent or // not as this is done as part of proposing the block. @@ -638,6 +646,8 @@ impl State { ) -> Result { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); + // Take a snapshot and extract everything we need, then drop it so readers of newer + // snapshots aren't held up by this Arc. let (new_account_id_prefix_is_unique, account_commitment) = { let snapshot = self.snapshot(); From 28e80705920ff06926fcb921f3e2b279d79a1fd2 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 13:31:36 +1200 Subject: [PATCH 09/48] Tidy error variants --- crates/store/src/errors.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 4c01aa350..f5cd1f0e7 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -24,11 +24,13 @@ use miden_protocol::errors::{ use miden_protocol::note::{NoteId, Nullifier}; use miden_protocol::transaction::OutputNote; use thiserror::Error; +use tokio::sync::mpsc::error::SendError; use tokio::sync::oneshot::error::RecvError; use tonic::Status; use crate::account_state_forest::{AccountStateForestError, WitnessError}; use crate::db::models::conv::DatabaseTypeConversionError; +use crate::state::writer::WriteRequest; // PROOF SCHEDULER ERRORS // ================================================================================================= @@ -198,6 +200,10 @@ pub enum ApplyBlockError { InvalidBlockError(#[from] InvalidBlockError), #[error("account state forest error")] AccountStateForestError(#[from] AccountStateForestError), + #[error("failed to send block to writer task (channel closed)")] + WriterTaskSendFailed(#[from] Box>), + #[error("writer task dropped the result channel")] + WriterTaskRecvFailed(#[from] RecvError), // OTHER ERRORS // --------------------------------------------------------------------------------------------- @@ -205,12 +211,6 @@ pub enum ApplyBlockError { DbBlockHeaderEmpty, #[error("database update failed: {0}")] DbUpdateTaskFailed(String), - #[error("failed to send block to writer task (channel closed)")] - WriterTaskSendFailed( - #[source] Box>, - ), - #[error("writer task dropped the result channel")] - WriterTaskRecvFailed(#[from] tokio::sync::oneshot::error::RecvError), } impl From for Status { From 5a8b843d6a4da34fbcf55016902965f8c41e8440 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 16:30:29 +1200 Subject: [PATCH 10/48] Fix comments and reduce pub fields to super --- crates/store/src/state/mod.rs | 10 +++---- crates/store/src/state/writer_guard.rs | 39 +++++++++++--------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index a99294aa1..257f7c79b 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -144,29 +144,29 @@ pub(crate) struct InMemoryState { pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. - pub(crate) db: Arc, + pub(super) db: Arc, /// The block store which stores full block contents for all blocks. - pub(crate) block_store: Arc, + pub(super) block_store: Arc, /// Nullifier tree — append-only, backed by `RocksDB`. /// /// Lock-free via [`WriterGuard`]: `RocksDB` MVCC provides safe concurrent read access /// during writes. - pub(crate) nullifier_tree: WriterGuard>>, + pub(super) nullifier_tree: WriterGuard>>, /// All in-memory state (account tree, blockchain MMR, forest) held atomically. /// /// Readers call `snapshot()` which returns `Arc` via a wait-free atomic /// refcount bump — no data cloning. The writer deep-clones once per block, mutates the /// copy, and atomically swaps via `ArcSwap::store()`. - pub(crate) in_memory: ArcSwap, + pub(super) in_memory: ArcSwap, /// Channel to the single writer task. writer_tx: mpsc::Sender, /// Request termination of the process due to a fatal internal state error. - pub(crate) termination_ask: tokio::sync::mpsc::Sender, + pub(super) termination_ask: tokio::sync::mpsc::Sender, /// The latest proven-in-sequence block number, updated by the proof scheduler. proven_tip: ProvenTipReader, diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs index 6661bfadd..eccd0a08e 100644 --- a/crates/store/src/state/writer_guard.rs +++ b/crates/store/src/state/writer_guard.rs @@ -10,25 +10,23 @@ use std::cell::UnsafeCell; /// 1. **Single writer**: Only one task (the writer, serialized by a channel) may call /// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type /// system. -/// 2. **Publish barrier**: After completing all mutations, the writer performs an `ArcSwap::store` -/// on the shared in-memory state, which includes a `Release` memory barrier. -/// 3. **Subscribe barrier**: Before calling [`as_ref()`](Self::as_ref), readers perform an -/// `ArcSwap::load_full` which includes an `Acquire` memory barrier. -/// 4. The barrier pair establishes a *happens-before* relationship, guaranteeing that all mutations -/// performed before the store are visible to any reader that observes the updated state. +/// 2. **Concurrent read safety**: The wrapped type (currently the `RocksDB`-backed nullifier tree) +/// provides its own MVCC / snapshot isolation, making concurrent reads during writes safe at the +/// storage layer. +/// 3. **Append-only data**: The wrapped data structures are append-only (keyed by block number), so +/// readers observing an older state simply query at that older block number, which is safe. /// -/// Because the wrapped data structures are append-only or overlay-based (keyed by block number), -/// readers that observe an older state will simply query at that older block number, which is -/// safe. +/// Concurrent read safety relies on the guarantees of the underlying storage engine (e.g. +/// `RocksDB` MVCC). pub struct WriterGuard { inner: UnsafeCell, } // SAFETY: The single-writer invariant is enforced by the channel-based writer task architecture. -// Readers only call `as_ref()` which returns `&T`. The writer completes all mutations before -// performing an `ArcSwap::store` (which includes a Release barrier), and readers perform an -// `ArcSwap::load_full` (which includes an Acquire barrier) before accessing the data. -// This guarantees no data races. +// Readers only call `as_ref()` which returns `&T`. Concurrent read safety during writes is +// guaranteed by the underlying storage engine (RocksDB MVCC / snapshot isolation), not by +// acquire/release barriers. The data structures are append-only, so readers see a consistent +// view at their query's block number. unsafe impl Send for WriterGuard {} unsafe impl Sync for WriterGuard {} @@ -40,14 +38,13 @@ impl WriterGuard { /// Returns a shared reference to the wrapped value. /// - /// Safe for any reader thread. The data is guaranteed to be in a consistent state because - /// the caller accesses shared state through `ArcSwap::load_full` (which includes an - /// `Acquire` barrier), establishing a happens-before relationship with the writer's - /// `ArcSwap::store` (which includes a `Release` barrier). + /// Safe for any reader thread. Concurrent read safety is provided by the underlying storage + /// engine (e.g. `RocksDB` MVCC / snapshot isolation). The data structures are append-only, + /// so readers see a consistent view at their query's block number. pub(super) fn as_ref(&self) -> &T { - // SAFETY: The writer completes all mutations before the ArcSwap::store (Release barrier). - // The reader performs ArcSwap::load_full (Acquire barrier) before calling this. - // The barrier pair ensures all writes are visible. + // SAFETY: Single-writer is enforced by the channel-based writer task. Concurrent reads + // are safe because the underlying storage engine (RocksDB) provides MVCC / snapshot + // isolation, and the data is append-only. unsafe { &*self.inner.get() } } @@ -57,8 +54,6 @@ impl WriterGuard { /// /// Must only be called from the single writer task. The caller must ensure: /// - No other calls to `as_mut()` are concurrent (enforced by channel serialization). - /// - All mutations through the returned reference are completed before performing an - /// `ArcSwap::store` on the shared in-memory state. #[expect(clippy::mut_from_ref)] pub unsafe fn as_mut(&self) -> &mut T { unsafe { &mut *self.inner.get() } From 42932ff351e30c956a5750988b5858a1eaee14dc Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 9 Apr 2026 16:32:21 +1200 Subject: [PATCH 11/48] Remove mention of previous design from comments --- crates/store/src/state/writer.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 46afd9e16..968e4cd98 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -39,8 +39,8 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// This function is the sole writer to all in-memory state. The nullifier tree (backed by /// `RocksDB` with MVCC) is accessed lock-free via [`WriterGuard`]. The in-memory structures /// (`account_tree`, `blockchain`, `forest`) are held in an `Arc` behind an -/// `ArcSwap`. The writer loads the current state, validates against it, commits to DB (no locks -/// held), then deep-clones the state, applies mutations, and atomically swaps the pointer. +/// `ArcSwap`. The writer loads the current state, validates against it, commits to DB, then +/// deep-clones the state, applies mutations, and atomically swaps the pointer. /// /// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an /// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either @@ -102,7 +102,7 @@ async fn apply_block_inner( async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), ); - // Load the current in-memory state snapshot for validation (wait-free, no locks). + // Load the current in-memory state snapshot for validation (wait-free). let snapshot = state.in_memory.load_full(); // Compute mutations required for updating account and nullifier trees. @@ -212,8 +212,8 @@ async fn apply_block_inner( }, )); - // Commit to DB. No locks are held — this is the key advantage over the previous design. - // Readers continue to see the old in-memory state (via their Arc) while the DB commits. + // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while + // the DB commits. state .db .apply_block(signed_block, notes, proving_inputs) From 39ae2a20ec4cd3a900fe8bde12ff569de797fa1d Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 15:43:35 +1200 Subject: [PATCH 12/48] Integrate Snapshot / Readers --- Cargo.lock | 64 +- Cargo.toml | 11 +- crates/large-smt-backend-rocksdb/src/lib.rs | 6 +- .../large-smt-backend-rocksdb/src/rocksdb.rs | 779 ++++++++++++------ crates/store/src/accounts/mod.rs | 46 +- crates/store/src/state/loader.rs | 15 +- crates/store/src/state/mod.rs | 57 +- crates/store/src/state/writer.rs | 86 +- 8 files changed, 730 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17914a71a..621badce0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -188,7 +188,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1653,7 +1653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2263,7 +2263,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -2839,8 +2839,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a12734b3bf5e8c88580c4c173368865d1efd6f51f48365df76b59f12d8dc53" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "alloy-sol-types", "fs-err", @@ -2915,8 +2914,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "334340b0b2e6993f6b71af359a77cf8730c4a19ac366f77aee734a5faa993897" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2964,7 +2962,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" dependencies = [ "blake3", "cc", @@ -3006,7 +3004,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" dependencies = [ "quote", "syn 2.0.117", @@ -3033,7 +3031,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3533,8 +3531,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595b3d43ceb562d05e248a6c52bdc2992dc270b60d6d0e8c0a6480aa30672b8e" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "bech32", "fs-err", @@ -3563,8 +3560,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c287782953c94452c2f7431feff137e4fe8b8d1eb5aa9112eab83a6f11c8f2" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "proc-macro2", "quote", @@ -3650,7 +3646,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#71e667f6383751586c078f8723d79e59489a2e66" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" dependencies = [ "p3-field", "p3-goldilocks", @@ -3659,8 +3655,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f63cd9264dc1f9f124fe644ee631828cc9bfd71022a75cd5bc1678f3ba7b56" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "fs-err", "miden-assembly", @@ -3677,8 +3672,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2761dd1f8baa744c91d1ff143d954c623d5fb1d2dfec8c8ab1dac2254343d7cd" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3700,8 +3694,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e894e952e2819545e9351f7427779f82538e51553dfaca3294301ff308086497" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "miden-processor", "miden-protocol", @@ -3714,8 +3707,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e7aac0d511aa412138ea7dfa4a6d8c76340c6028fa91313727b7ad3614711b" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" dependencies = [ "miden-protocol", "miden-tx", @@ -3929,7 +3921,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4794,7 +4786,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph 0.8.3", @@ -4815,7 +4807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -4937,7 +4929,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.37", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -4975,7 +4967,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -5344,7 +5336,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5415,7 +5407,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5822,7 +5814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6037,7 +6029,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6046,7 +6038,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6065,7 +6057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7075,7 +7067,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ca74beb7b..43e3e83fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ miden-node-rocksdb-cxx-linkage-fix = { path = "crates/rocksdb-cxx-linkage-fix", # miden-protocol dependencies. These should be updated in sync. miden-agglayer = { version = "0.14" } miden-block-prover = { version = "0.14" } -miden-protocol = { default-features = false, version = "0.14" } +miden-protocol = { default-features = false, version = "=0.14.3" } miden-standards = { version = "0.14" } miden-testing = { version = "0.14" } miden-tx = { default-features = false, version = "0.14" } @@ -146,3 +146,12 @@ files.extend-exclude = ["*.svg"] # Ignore SVG files. [patch.crates-io] miden-crypto = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } miden-serde-utils = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } + +miden-agglayer = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-block-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-protocol = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-protocol-macros = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-standards = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-testing = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-tx = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-tx-batch-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } diff --git a/crates/large-smt-backend-rocksdb/src/lib.rs b/crates/large-smt-backend-rocksdb/src/lib.rs index 563439c9f..839289d03 100644 --- a/crates/large-smt-backend-rocksdb/src/lib.rs +++ b/crates/large-smt-backend-rocksdb/src/lib.rs @@ -27,6 +27,8 @@ mod helpers; #[expect(clippy::doc_markdown, clippy::inline_always)] mod rocksdb; // Re-export from miden-protocol. +/// Re-export of `rocksdb::DB` for consumers that need the raw database handle type. +pub use ::rocksdb::DB; pub use miden_protocol::crypto::merkle::smt::{ InnerNode, LargeSmt, @@ -39,6 +41,8 @@ pub use miden_protocol::crypto::merkle::smt::{ SmtLeafError, SmtProof, SmtStorage, + SmtStorageReader, + SmtStorageWriter, StorageError, StorageUpdateParts, StorageUpdates, @@ -56,4 +60,4 @@ pub use miden_protocol::{ merkle::{EmptySubtreeRoots, InnerNodeInfo, MerkleError, NodeIndex, SparseMerklePath}, }, }; -pub use rocksdb::{RocksDbConfig, RocksDbStorage}; +pub use rocksdb::{RocksDbConfig, RocksDbSnapshotStorage, RocksDbStorage}; diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 1a54feb47..0e91a2006 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -22,7 +22,14 @@ use rocksdb::{ WriteBatch, }; -use super::{SmtStorage, StorageError, StorageUpdateParts, StorageUpdates, SubtreeUpdate}; +use super::{ + SmtStorageReader, + SmtStorageWriter, + StorageError, + StorageUpdateParts, + StorageUpdates, + SubtreeUpdate, +}; use crate::helpers::{insert_into_leaf, map_rocksdb_err, remove_from_leaf}; use crate::{EMPTY_WORD, Word}; @@ -67,7 +74,7 @@ const ENTRY_COUNT_KEY: &[u8] = b"entry_count"; /// `NodeIndex`. /// - `METADATA_CF` ("metadata"): Stores overall SMT metadata such as the current root hash, total /// leaf count, and total entry count. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RocksDbStorage { db: Arc, } @@ -259,9 +266,19 @@ impl RocksDbStorage { let name = cf_for_depth(index.depth()); self.cf_handle(name).expect("CF handle missing") } + + /// Returns a reference to the inner `Arc`. + pub fn db(&self) -> &Arc { + &self.db + } + + /// Creates a new [`RocksDbSnapshotStorage`] from this storage's database. + pub fn snapshot_storage(&self) -> RocksDbSnapshotStorage { + RocksDbSnapshotStorage::new(Arc::clone(&self.db)) + } } -impl SmtStorage for RocksDbStorage { +impl SmtStorageReader for RocksDbStorage { /// Retrieves the total count of non-empty leaves from the `METADATA_CF` column family. /// Returns 0 if the count is not found. /// @@ -308,125 +325,6 @@ impl SmtStorage for RocksDbStorage { }) } - /// Inserts a key-value pair into the SMT leaf at the specified logical `index`. - /// - /// This operation involves: - /// 1. Retrieving the current leaf (if any) at `index`. - /// 2. Inserting the new key-value pair into the leaf. - /// 3. Updating the leaf and entry counts in the metadata column family. - /// 4. Writing all changes (leaf data, counts) to RocksDB in a single batch. - /// - /// Note: This only updates the leaf. Callers are responsible for recomputing and - /// persisting the corresponding inner nodes. - /// - /// # Errors - /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. - /// - `StorageError::DeserializationError`: If existing leaf data is corrupt. - #[expect(clippy::single_match_else)] - fn insert_value( - &mut self, - index: u64, - key: Word, - value: Word, - ) -> Result, StorageError> { - debug_assert_ne!(value, EMPTY_WORD); - - let mut batch = WriteBatch::default(); - - // Fetch initial counts. - let mut current_leaf_count = self.leaf_count()?; - let mut current_entry_count = self.entry_count()?; - - let leaves_cf = self.cf_handle(LEAVES_CF)?; - let db_key = Self::index_db_key(index); - - let maybe_leaf = self.get_leaf(index)?; - - let value_to_return: Option = match maybe_leaf { - Some(mut existing_leaf) => { - let old_value = insert_into_leaf(&mut existing_leaf, key, value)?; - // Determine if the overall SMT entry_count needs to change. - // entry_count increases if: - // 1. The key was not present in this leaf before (`old_value` is `None`). - // 2. The key was present but held `EMPTY_WORD` (`old_value` is - // `Some(EMPTY_WORD)`). - if old_value.is_none_or(|old_v| old_v == EMPTY_WORD) { - current_entry_count += 1; - } - // current_leaf_count does not change because the leaf itself already existed. - batch.put_cf(leaves_cf, db_key, existing_leaf.to_bytes()); - old_value - }, - None => { - // Leaf at `index` does not exist, so create a new one. - let new_leaf = SmtLeaf::Single((key, value)); - // A new leaf is created. - current_leaf_count += 1; - // This new leaf contains one new SMT entry. - current_entry_count += 1; - batch.put_cf(leaves_cf, db_key, new_leaf.to_bytes()); - // No previous value, as the leaf (and thus the key in it) was new. - None - }, - }; - - // Add updated metadata counts to the batch. - let metadata_cf = self.cf_handle(METADATA_CF)?; - batch.put_cf(metadata_cf, LEAF_COUNT_KEY, current_leaf_count.to_be_bytes()); - batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, current_entry_count.to_be_bytes()); - - // Atomically write all changes (leaf data and metadata counts). - self.db.write(batch).map_err(map_rocksdb_err)?; - - Ok(value_to_return) - } - - /// Removes a key-value pair from the SMT leaf at the specified logical `index`. - /// - /// This operation involves: - /// 1. Retrieving the leaf at `index`. - /// 2. Removing the `key` from the leaf. If the leaf becomes empty, it's deleted from RocksDB. - /// 3. Updating the leaf and entry counts in the metadata column family. - /// 4. Writing all changes (leaf data/deletion, counts) to RocksDB in a single batch. - /// - /// Returns `Ok(None)` if the leaf at `index` does not exist or the `key` is not found. - /// - /// Note: This only updates the leaf. Callers are responsible for recomputing and - /// persisting the corresponding inner nodes. - /// - /// # Errors - /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. - /// - `StorageError::DeserializationError`: If existing leaf data is corrupt. - fn remove_value(&mut self, index: u64, key: Word) -> Result, StorageError> { - let Some(mut leaf) = self.get_leaf(index)? else { - return Ok(None); - }; - - let mut batch = WriteBatch::default(); - let cf = self.cf_handle(LEAVES_CF)?; - let metadata_cf = self.cf_handle(METADATA_CF)?; - let db_key = Self::index_db_key(index); - let mut entry_count = self.entry_count()?; - let mut leaf_count = self.leaf_count()?; - - let (current_value, is_empty) = remove_from_leaf(&mut leaf, key); - if let Some(current_value) = current_value - && current_value != EMPTY_WORD - { - entry_count -= 1; - } - if is_empty { - leaf_count -= 1; - batch.delete_cf(cf, db_key); - } else { - batch.put_cf(cf, db_key, leaf.to_bytes()); - } - batch.put_cf(metadata_cf, LEAF_COUNT_KEY, leaf_count.to_be_bytes()); - batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, entry_count.to_be_bytes()); - self.db.write(batch).map_err(map_rocksdb_err)?; - Ok(current_value) - } - /// Retrieves a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. /// /// # Errors @@ -444,59 +342,6 @@ impl SmtStorage for RocksDbStorage { } } - /// Sets or updates multiple SMT leaf nodes in the `LEAVES_CF` column family. - /// - /// This method performs a batch write to RocksDB. It also updates the global - /// leaf and entry counts in the `METADATA_CF` based on the provided `leaves` map, - /// overwriting any previous counts. - /// - /// Note: This method assumes the provided `leaves` map represents the entirety - /// of leaves to be stored or that counts are being explicitly reset. - /// Note: This only updates the leaves. Callers are responsible for recomputing and - /// persisting the corresponding inner nodes. - /// - /// # Errors - /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. - fn set_leaves(&mut self, leaves: Map) -> Result<(), StorageError> { - let cf = self.cf_handle(LEAVES_CF)?; - let leaf_count: usize = leaves.len(); - let entry_count: usize = leaves.values().map(|leaf| leaf.entries().len()).sum(); - let mut batch = WriteBatch::default(); - for (idx, leaf) in leaves { - let key = Self::index_db_key(idx); - let value = leaf.to_bytes(); - batch.put_cf(cf, key, &value); - } - let metadata_cf = self.cf_handle(METADATA_CF)?; - batch.put_cf(metadata_cf, LEAF_COUNT_KEY, leaf_count.to_be_bytes()); - batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, entry_count.to_be_bytes()); - self.db.write(batch).map_err(map_rocksdb_err)?; - Ok(()) - } - - /// Removes a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. - /// - /// Important: This method currently *does not* update the global leaf and entry counts - /// in the metadata. Callers are responsible for managing these counts separately - /// if using this method directly, or preferably use `apply` or `remove_value` which handle - /// counts. - /// - /// Note: This only removes the leaf. Callers are responsible for recomputing and - /// persisting the corresponding inner nodes. - /// - /// # Errors - /// - `StorageError::Backend`: If the leaves column family is missing or a RocksDB error occurs. - /// - `StorageError::DeserializationError`: If the retrieved (to be returned) leaf data is - /// corrupt. - fn remove_leaf(&mut self, index: u64) -> Result, StorageError> { - let key = Self::index_db_key(index); - let cf = self.cf_handle(LEAVES_CF)?; - let old_bytes = self.db.get_cf(cf, key).map_err(map_rocksdb_err)?; - self.db.delete_cf(cf, key).map_err(map_rocksdb_err)?; - Ok(old_bytes - .map(|bytes| SmtLeaf::read_from_bytes(&bytes).expect("failed to deserialize leaf"))) - } - /// Retrieves multiple SMT leaf nodes by their logical `indices` using RocksDB's `multi_get_cf`. /// /// # Errors @@ -633,48 +478,299 @@ impl SmtStorage for RocksDbStorage { Ok(results) } - /// Stores a single subtree in RocksDB and optionally updates the depth-24 root cache. + /// Retrieves a single inner node (non-leaf node) from within a Subtree. /// - /// The subtree is serialized and written to its corresponding column family. - /// If it's a depth-24 subtree, the root node’s hash is also stored in the - /// dedicated `DEPTH_24_CF` cache to support top-level reconstruction. + /// This method is intended for accessing nodes at depths greater than or equal to + /// `IN_MEMORY_DEPTH`. It first finds the appropriate Subtree containing the `index`, then + /// delegates to `Subtree::get_inner_node()`. /// - /// # Parameters - /// - `subtree`: A reference to the subtree to be stored. + /// # Errors + /// - `StorageError::Backend`: If `index.depth() < IN_MEMORY_DEPTH`, or if RocksDB errors occur. + /// - `StorageError::Value`: If the containing Subtree data is corrupt. + fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { + if index.depth() < IN_MEMORY_DEPTH { + return Err(StorageError::Unsupported( + "Cannot get inner node from upper part of the tree".into(), + )); + } + let subtree_root_index = Subtree::find_subtree_root(index); + Ok(self + .get_subtree(subtree_root_index)? + .and_then(|subtree| subtree.get_inner_node(index))) + } + + /// Returns an iterator over all (logical u64 index, `SmtLeaf`) pairs in the `LEAVES_CF`. + /// + /// The iterator uses a RocksDB snapshot for consistency and iterates in lexicographical + /// order of the keys (leaf indices). Errors during iteration (e.g., deserialization issues) + /// cause the iterator to skip the problematic item and attempt to continue. /// /// # Errors - /// - Returns `StorageError` if column family lookup, serialization, or the write operation - /// fails. - fn set_subtree(&mut self, subtree: &Subtree) -> Result<(), StorageError> { - let subtrees_cf = self.subtree_cf(subtree.root_index()); - let mut batch = WriteBatch::default(); + /// - `StorageError::Backend`: If the leaves column family is missing or a RocksDB error occurs + /// during iterator creation. + fn iter_leaves(&self) -> Result + '_>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let mut read_opts = ReadOptions::default(); + read_opts.set_total_order_seek(true); + let db_iter = self.db.iterator_cf_opt(cf, read_opts, IteratorMode::Start); - let key = Self::subtree_db_key(subtree.root_index()); - let value = subtree.to_vec(); - batch.put_cf(subtrees_cf, key, value); + Ok(Box::new(RocksDbDirectLeafIterator { iter: db_iter })) + } - // Also update level 24 hash cache if this is a level 24 subtree - if subtree.root_index().depth() == IN_MEMORY_DEPTH { - let root_hash = subtree - .get_inner_node(subtree.root_index()) - .ok_or_else(|| StorageError::Unsupported("Subtree root node not found".into()))? - .hash(); + /// Returns an iterator over all `Subtree` instances across all subtree column families. + /// + /// The iterator uses a RocksDB snapshot and iterates in lexicographical order of keys + /// (subtree root NodeIndex) across all depth column families (24, 32, 40, 48, 56). + /// Errors during iteration (e.g., deserialization issues) cause the iterator to skip + /// the problematic item and attempt to continue. + /// + /// # Errors + /// - `StorageError::Backend`: If any subtree column family is missing or a RocksDB error occurs + /// during iterator creation. + fn iter_subtrees(&self) -> Result + '_>, StorageError> { + // All subtree column family names in order + const SUBTREE_CFS: [&str; 5] = + [SUBTREE_24_CF, SUBTREE_32_CF, SUBTREE_40_CF, SUBTREE_48_CF, SUBTREE_56_CF]; - let depth24_cf = self.cf_handle(DEPTH_24_CF)?; - let hash_key = Self::index_db_key(subtree.root_index().position()); - batch.put_cf(depth24_cf, hash_key, root_hash.to_bytes()); + let mut cf_handles = Vec::new(); + for cf_name in SUBTREE_CFS { + cf_handles.push(self.cf_handle(cf_name)?); } - self.db.write(batch).map_err(map_rocksdb_err)?; - Ok(()) + Ok(Box::new(RocksDbSubtreeIterator::new(&self.db, cf_handles))) } - /// Bulk-writes subtrees to storage (bypassing WAL). - /// - /// This method writes a vector of serialized `Subtree` objects directly to their - /// corresponding RocksDB column families based on their root index. + /// Retrieves all depth 24 hashes for fast tree rebuilding. /// - /// ⚠️ **Warning:** This function should only be used during **initial SMT construction**. + /// # Errors + /// - `StorageError::Backend`: If the depth24 column family is missing or a RocksDB error + /// occurs. + /// - `StorageError::Value`: If any hash bytes are corrupt. + fn get_depth24(&self) -> Result, StorageError> { + let cf = self.cf_handle(DEPTH_24_CF)?; + let iter = self.db.iterator_cf(cf, IteratorMode::Start); + let mut hashes = Vec::new(); + + for item in iter { + let (key_bytes, value_bytes) = item.map_err(map_rocksdb_err)?; + + let index = index_from_key_bytes(&key_bytes)?; + let hash = Word::read_from_bytes(&value_bytes)?; + + hashes.push((index, hash)); + } + + Ok(hashes) + } +} + +impl SmtStorageWriter for RocksDbStorage { + /// Inserts a key-value pair into the SMT leaf at the specified logical `index`. + /// + /// This operation involves: + /// 1. Retrieving the current leaf (if any) at `index`. + /// 2. Inserting the new key-value pair into the leaf. + /// 3. Updating the leaf and entry counts in the metadata column family. + /// 4. Writing all changes (leaf data, counts) to RocksDB in a single batch. + /// + /// Note: This only updates the leaf. Callers are responsible for recomputing and + /// persisting the corresponding inner nodes. + /// + /// # Errors + /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. + /// - `StorageError::DeserializationError`: If existing leaf data is corrupt. + #[expect(clippy::single_match_else)] + fn insert_value( + &mut self, + index: u64, + key: Word, + value: Word, + ) -> Result, StorageError> { + debug_assert_ne!(value, EMPTY_WORD); + + let mut batch = WriteBatch::default(); + + // Fetch initial counts. + let mut current_leaf_count = self.leaf_count()?; + let mut current_entry_count = self.entry_count()?; + + let leaves_cf = self.cf_handle(LEAVES_CF)?; + let db_key = Self::index_db_key(index); + + let maybe_leaf = self.get_leaf(index)?; + + let value_to_return: Option = match maybe_leaf { + Some(mut existing_leaf) => { + let old_value = insert_into_leaf(&mut existing_leaf, key, value)?; + // Determine if the overall SMT entry_count needs to change. + // entry_count increases if: + // 1. The key was not present in this leaf before (`old_value` is `None`). + // 2. The key was present but held `EMPTY_WORD` (`old_value` is + // `Some(EMPTY_WORD)`). + if old_value.is_none_or(|old_v| old_v == EMPTY_WORD) { + current_entry_count += 1; + } + // current_leaf_count does not change because the leaf itself already existed. + batch.put_cf(leaves_cf, db_key, existing_leaf.to_bytes()); + old_value + }, + None => { + // Leaf at `index` does not exist, so create a new one. + let new_leaf = SmtLeaf::Single((key, value)); + // A new leaf is created. + current_leaf_count += 1; + // This new leaf contains one new SMT entry. + current_entry_count += 1; + batch.put_cf(leaves_cf, db_key, new_leaf.to_bytes()); + // No previous value, as the leaf (and thus the key in it) was new. + None + }, + }; + + // Add updated metadata counts to the batch. + let metadata_cf = self.cf_handle(METADATA_CF)?; + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, current_leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, current_entry_count.to_be_bytes()); + + // Atomically write all changes (leaf data and metadata counts). + self.db.write(batch).map_err(map_rocksdb_err)?; + + Ok(value_to_return) + } + + /// Removes a key-value pair from the SMT leaf at the specified logical `index`. + /// + /// This operation involves: + /// 1. Retrieving the leaf at `index`. + /// 2. Removing the `key` from the leaf. If the leaf becomes empty, it's deleted from RocksDB. + /// 3. Updating the leaf and entry counts in the metadata column family. + /// 4. Writing all changes (leaf data/deletion, counts) to RocksDB in a single batch. + /// + /// Returns `Ok(None)` if the leaf at `index` does not exist or the `key` is not found. + /// + /// Note: This only updates the leaf. Callers are responsible for recomputing and + /// persisting the corresponding inner nodes. + /// + /// # Errors + /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. + /// - `StorageError::DeserializationError`: If existing leaf data is corrupt. + fn remove_value(&mut self, index: u64, key: Word) -> Result, StorageError> { + let Some(mut leaf) = self.get_leaf(index)? else { + return Ok(None); + }; + + let mut batch = WriteBatch::default(); + let cf = self.cf_handle(LEAVES_CF)?; + let metadata_cf = self.cf_handle(METADATA_CF)?; + let db_key = Self::index_db_key(index); + let mut entry_count = self.entry_count()?; + let mut leaf_count = self.leaf_count()?; + + let (current_value, is_empty) = remove_from_leaf(&mut leaf, key); + if let Some(current_value) = current_value + && current_value != EMPTY_WORD + { + entry_count -= 1; + } + if is_empty { + leaf_count -= 1; + batch.delete_cf(cf, db_key); + } else { + batch.put_cf(cf, db_key, leaf.to_bytes()); + } + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, entry_count.to_be_bytes()); + self.db.write(batch).map_err(map_rocksdb_err)?; + Ok(current_value) + } + + /// Sets or updates multiple SMT leaf nodes in the `LEAVES_CF` column family. + /// + /// This method performs a batch write to RocksDB. It also updates the global + /// leaf and entry counts in the `METADATA_CF` based on the provided `leaves` map, + /// overwriting any previous counts. + /// + /// # Errors + /// - `StorageError::Backend`: If column families are missing or a RocksDB error occurs. + fn set_leaves(&mut self, leaves: Map) -> Result<(), StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let leaf_count: usize = leaves.len(); + let entry_count: usize = leaves.values().map(|leaf| leaf.entries().len()).sum(); + let mut batch = WriteBatch::default(); + for (idx, leaf) in leaves { + let key = Self::index_db_key(idx); + let value = leaf.to_bytes(); + batch.put_cf(cf, key, &value); + } + let metadata_cf = self.cf_handle(METADATA_CF)?; + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, entry_count.to_be_bytes()); + self.db.write(batch).map_err(map_rocksdb_err)?; + Ok(()) + } + + /// Removes a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. + /// + /// Important: This method currently *does not* update the global leaf and entry counts + /// in the metadata. Callers are responsible for managing these counts separately + /// if using this method directly, or preferably use `apply` or `remove_value` which handle + /// counts. + /// + /// # Errors + /// - `StorageError::Backend`: If the leaves column family is missing or a RocksDB error occurs. + /// - `StorageError::DeserializationError`: If the retrieved (to be returned) leaf data is + /// corrupt. + fn remove_leaf(&mut self, index: u64) -> Result, StorageError> { + let key = Self::index_db_key(index); + let cf = self.cf_handle(LEAVES_CF)?; + let old_bytes = self.db.get_cf(cf, key).map_err(map_rocksdb_err)?; + self.db.delete_cf(cf, key).map_err(map_rocksdb_err)?; + Ok(old_bytes + .map(|bytes| SmtLeaf::read_from_bytes(&bytes).expect("failed to deserialize leaf"))) + } + + /// Stores a single subtree in RocksDB and optionally updates the depth-24 root cache. + /// + /// The subtree is serialized and written to its corresponding column family. + /// If it's a depth-24 subtree, the root node’s hash is also stored in the + /// dedicated `DEPTH_24_CF` cache to support top-level reconstruction. + /// + /// # Parameters + /// - `subtree`: A reference to the subtree to be stored. + /// + /// # Errors + /// - Returns `StorageError` if column family lookup, serialization, or the write operation + /// fails. + fn set_subtree(&mut self, subtree: &Subtree) -> Result<(), StorageError> { + let subtrees_cf = self.subtree_cf(subtree.root_index()); + let mut batch = WriteBatch::default(); + + let key = Self::subtree_db_key(subtree.root_index()); + let value = subtree.to_vec(); + batch.put_cf(subtrees_cf, key, value); + + // Also update level 24 hash cache if this is a level 24 subtree + if subtree.root_index().depth() == IN_MEMORY_DEPTH { + let root_hash = subtree + .get_inner_node(subtree.root_index()) + .ok_or_else(|| StorageError::Unsupported("Subtree root node not found".into()))? + .hash(); + + let depth24_cf = self.cf_handle(DEPTH_24_CF)?; + let hash_key = Self::index_db_key(subtree.root_index().position()); + batch.put_cf(depth24_cf, hash_key, root_hash.to_bytes()); + } + + self.db.write(batch).map_err(map_rocksdb_err)?; + Ok(()) + } + + /// Bulk-writes subtrees to storage (bypassing WAL). + /// + /// This method writes a vector of serialized `Subtree` objects directly to their + /// corresponding RocksDB column families based on their root index. + /// + /// ⚠️ **Warning:** This function should only be used during **initial SMT construction**. /// It disables the WAL, meaning writes are **not crash-safe** and can result in data loss /// if the process terminates unexpectedly. /// @@ -728,27 +824,6 @@ impl SmtStorage for RocksDbStorage { Ok(()) } - /// Retrieves a single inner node (non-leaf node) from within a Subtree. - /// - /// This method is intended for accessing nodes at depths greater than or equal to - /// `IN_MEMORY_DEPTH`. It first finds the appropriate Subtree containing the `index`, then - /// delegates to `Subtree::get_inner_node()`. - /// - /// # Errors - /// - `StorageError::Backend`: If `index.depth() < IN_MEMORY_DEPTH`, or if RocksDB errors occur. - /// - `StorageError::Value`: If the containing Subtree data is corrupt. - fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { - if index.depth() < IN_MEMORY_DEPTH { - return Err(StorageError::Unsupported( - "Cannot get inner node from upper part of the tree".into(), - )); - } - let subtree_root_index = Subtree::find_subtree_root(index); - Ok(self - .get_subtree(subtree_root_index)? - .and_then(|subtree| subtree.get_inner_node(index))) - } - /// Sets or updates a single inner node (non-leaf node) within a Subtree. /// /// This method is intended for `index.depth() >= IN_MEMORY_DEPTH`. @@ -918,37 +993,239 @@ impl SmtStorage for RocksDbStorage { Ok(()) } +} - /// Returns an iterator over all (logical u64 index, `SmtLeaf`) pairs in the `LEAVES_CF`. +// SNAPSHOT STORAGE +// -------------------------------------------------------------------------------------------- + +/// Inner state shared by all clones of a snapshot storage. +/// +/// # Safety +/// +/// `snapshot` borrows from `db`. Fields are dropped in declaration order in Rust, +/// so `snapshot` is dropped before `db`'s refcount is decremented. The `Arc` +/// ensures the `DB` lives at least as long as any `SnapshotInner`. +struct SnapshotInner { + // IMPORTANT: field order matters for drop order. + // `snapshot` must be declared before `db` so it is dropped first. + snapshot: rocksdb::Snapshot<'static>, // actually borrows from `db` + db: Arc, +} + +/// A read-only, `Clone`-able RocksDB storage that reads from a point-in-time snapshot. +/// +/// All clones share the same snapshot via `Arc`, providing a consistent view of +/// the database at the time the snapshot was created. +/// +/// Implements [`SmtStorageReader`] but not [`SmtStorageWriter`]. +#[derive(Clone)] +pub struct RocksDbSnapshotStorage { + inner: Arc, +} + +impl std::fmt::Debug for RocksDbSnapshotStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RocksDbSnapshotStorage").finish_non_exhaustive() + } +} + +impl RocksDbSnapshotStorage { + /// Creates a new snapshot storage from the given database. /// - /// The iterator uses a RocksDB snapshot for consistency and iterates in lexicographical - /// order of the keys (leaf indices). Errors during iteration (e.g., deserialization issues) - /// cause the iterator to skip the problematic item and attempt to continue. + /// # Safety /// - /// # Errors - /// - `StorageError::Backend`: If the leaves column family is missing or a RocksDB error occurs - /// during iterator creation. + /// We use `Arc::as_ptr` to get a reference without borrowing `db`, allowing `db` + /// to be moved into `SnapshotInner`. The `Arc` stored alongside the snapshot + /// guarantees the DB outlives the snapshot (snapshot field is dropped before db field + /// due to declaration order). + pub fn new(db: Arc) -> Self { + // SAFETY: See struct-level safety documentation. + let db_ref: &DB = unsafe { &*Arc::as_ptr(&db) }; + let snapshot = db_ref.snapshot(); + let snapshot: rocksdb::Snapshot<'static> = unsafe { std::mem::transmute(snapshot) }; + Self { + inner: Arc::new(SnapshotInner { snapshot, db }), + } + } + + fn cf_handle(&self, name: &str) -> Result<&rocksdb::ColumnFamily, StorageError> { + self.inner + .db + .cf_handle(name) + .ok_or_else(|| StorageError::Unsupported(format!("unknown column family `{name}`"))) + } + + #[inline(always)] + fn subtree_cf(&self, index: NodeIndex) -> &rocksdb::ColumnFamily { + let name = cf_for_depth(index.depth()); + self.cf_handle(name).expect("CF handle missing") + } +} + +impl SmtStorageReader for RocksDbSnapshotStorage { + fn leaf_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.inner.snapshot.get_cf(cf, LEAF_COUNT_KEY).map_err(map_rocksdb_err)?.map_or( + Ok(0), + |bytes| { + let arr: [u8; 8] = + bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "leaf count", + expected: 8, + found: bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) + }, + ) + } + + fn entry_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.inner + .snapshot + .get_cf(cf, ENTRY_COUNT_KEY) + .map_err(map_rocksdb_err)? + .map_or(Ok(0), |bytes| { + let arr: [u8; 8] = + bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "entry count", + expected: 8, + found: bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) + }) + } + + fn get_leaf(&self, index: u64) -> Result, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let key = RocksDbStorage::index_db_key(index); + match self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)? { + Some(bytes) => { + let leaf = SmtLeaf::read_from_bytes(&bytes)?; + Ok(Some(leaf)) + }, + None => Ok(None), + } + } + + fn get_leaves(&self, indices: &[u64]) -> Result>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let db_keys: Vec<[u8; 8]> = + indices.iter().map(|&idx| RocksDbStorage::index_db_key(idx)).collect(); + let results = self.inner.snapshot.multi_get_cf(db_keys.iter().map(|k| (cf, k.as_ref()))); + + results + .into_iter() + .map(|result| match result { + Ok(Some(bytes)) => Ok(Some(SmtLeaf::read_from_bytes(&bytes)?)), + Ok(None) => Ok(None), + Err(e) => Err(map_rocksdb_err(e)), + }) + .collect() + } + + fn has_leaves(&self) -> Result { + Ok(self.leaf_count()? > 0) + } + + fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { + let cf = self.subtree_cf(index); + let key = RocksDbStorage::subtree_db_key(index); + match self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)? { + Some(bytes) => { + let subtree = Subtree::from_vec(index, &bytes)?; + Ok(Some(subtree)) + }, + None => Ok(None), + } + } + + fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { + use rayon::prelude::*; + + let mut depth_buckets: [Vec<(usize, NodeIndex)>; 5] = Default::default(); + + for (original_index, &node_index) in indices.iter().enumerate() { + let depth = node_index.depth(); + let bucket_index = match depth { + 56 => 0, + 48 => 1, + 40 => 2, + 32 => 3, + 24 => 4, + _ => { + return Err(StorageError::Unsupported(format!( + "unsupported subtree depth {depth}" + ))); + }, + }; + depth_buckets[bucket_index].push((original_index, node_index)); + } + let mut results = vec![None; indices.len()]; + + let bucket_results: Result, StorageError> = depth_buckets + .into_par_iter() + .enumerate() + .filter(|(_, bucket)| !bucket.is_empty()) + .map( + |(bucket_index, bucket)| -> Result)>, StorageError> { + let depth = SUBTREE_DEPTHS[bucket_index]; + let cf = self.cf_handle(cf_for_depth(depth))?; + let keys: Vec<_> = bucket + .iter() + .map(|(_, idx)| RocksDbStorage::subtree_db_key(*idx)) + .collect(); + + let db_results = + self.inner.snapshot.multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))); + + bucket + .into_iter() + .zip(db_results) + .map(|((original_index, node_index), db_result)| { + let subtree = match db_result { + Ok(Some(bytes)) => Some(Subtree::from_vec(node_index, &bytes)?), + Ok(None) => None, + Err(e) => return Err(map_rocksdb_err(e)), + }; + Ok((original_index, subtree)) + }) + .collect() + }, + ) + .collect(); + + for bucket_result in bucket_results? { + for (original_index, subtree) in bucket_result { + results[original_index] = subtree; + } + } + + Ok(results) + } + + fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { + if index.depth() < IN_MEMORY_DEPTH { + return Err(StorageError::Unsupported( + "Cannot get inner node from upper part of the tree".into(), + )); + } + let subtree_root_index = Subtree::find_subtree_root(index); + Ok(self + .get_subtree(subtree_root_index)? + .and_then(|subtree| subtree.get_inner_node(index))) + } + fn iter_leaves(&self) -> Result + '_>, StorageError> { let cf = self.cf_handle(LEAVES_CF)?; let mut read_opts = ReadOptions::default(); read_opts.set_total_order_seek(true); - let db_iter = self.db.iterator_cf_opt(cf, read_opts, IteratorMode::Start); + let db_iter = self.inner.snapshot.iterator_cf_opt(cf, read_opts, IteratorMode::Start); Ok(Box::new(RocksDbDirectLeafIterator { iter: db_iter })) } - /// Returns an iterator over all `Subtree` instances across all subtree column families. - /// - /// The iterator uses a RocksDB snapshot and iterates in lexicographical order of keys - /// (subtree root NodeIndex) across all depth column families (24, 32, 40, 48, 56). - /// Errors during iteration (e.g., deserialization issues) cause the iterator to skip - /// the problematic item and attempt to continue. - /// - /// # Errors - /// - `StorageError::Backend`: If any subtree column family is missing or a RocksDB error occurs - /// during iterator creation. fn iter_subtrees(&self) -> Result + '_>, StorageError> { - // All subtree column family names in order const SUBTREE_CFS: [&str; 5] = [SUBTREE_24_CF, SUBTREE_32_CF, SUBTREE_40_CF, SUBTREE_48_CF, SUBTREE_56_CF]; @@ -957,21 +1234,15 @@ impl SmtStorage for RocksDbStorage { cf_handles.push(self.cf_handle(cf_name)?); } - Ok(Box::new(RocksDbSubtreeIterator::new(&self.db, cf_handles))) + Ok(Box::new(RocksDbSubtreeIterator::new(&self.inner.db, cf_handles))) } - /// Retrieves all depth 24 hashes for fast tree rebuilding. - /// - /// # Errors - /// - `StorageError::Backend`: If the depth24 column family is missing or a RocksDB error - /// occurs. - /// - `StorageError::Value`: If any hash bytes are corrupt. fn get_depth24(&self) -> Result, StorageError> { let cf = self.cf_handle(DEPTH_24_CF)?; - let iter = self.db.iterator_cf(cf, IteratorMode::Start); + let db_iter = self.inner.snapshot.iterator_cf(cf, IteratorMode::Start); let mut hashes = Vec::new(); - for item in iter { + for item in db_iter { let (key_bytes, value_bytes) = item.map_err(map_rocksdb_err)?; let index = index_from_key_bytes(&key_bytes)?; diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 298bb232a..d0f491309 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -15,6 +15,7 @@ use miden_protocol::crypto::merkle::smt::{ SMT_DEPTH, SmtLeaf, SmtStorage, + SmtStorageReader, }; use miden_protocol::crypto::merkle::{ EmptySubtreeRoots, @@ -68,7 +69,7 @@ enum HistoricalSelector { /// Captures reversion state for historical queries at a specific block. #[derive(Debug, Clone)] -struct HistoricalOverlay { +pub(crate) struct HistoricalOverlay { block_number: BlockNumber, root: Word, node_mutations: HashMap, @@ -116,7 +117,7 @@ impl HistoricalOverlay { /// reversion data (mutations that undo changes). Historical witnesses are reconstructed /// by starting from the latest state and applying reversion overlays backwards in time. #[derive(Debug, Clone)] -pub struct AccountTreeWithHistory { +pub struct AccountTreeWithHistory { /// The current block number (latest state). block_number: BlockNumber, /// The latest account tree state. @@ -125,7 +126,7 @@ pub struct AccountTreeWithHistory { overlays: BTreeMap, } -impl AccountTreeWithHistory { +impl AccountTreeWithHistory { /// Maximum number of historical blocks to maintain. pub const MAX_HISTORY: usize = 50; @@ -339,19 +340,29 @@ impl AccountTreeWithHistory { Some((path, leaf)) } - // PUBLIC MUTATORS + // PUBLIC ACCESSORS (continued) // -------------------------------------------------------------------------------------------- - /// Computes and applies mutations in one operation. + /// Returns a reference to the historical overlays. + pub(crate) fn overlays(&self) -> &BTreeMap { + &self.overlays + } + + /// Creates an `AccountTreeWithHistory` from its constituent parts. /// - /// This is a convenience method primarily for testing. - pub fn compute_and_apply_mutations( - &mut self, - account_commitments: impl IntoIterator, - ) -> Result<(), HistoricalError> { - let mutations = self.compute_mutations(account_commitments)?; - self.apply_mutations(mutations) + /// This is used by the writer to construct a snapshot-backed read-only copy. + pub(crate) fn from_parts( + latest: AccountTree>, + block_number: BlockNumber, + overlays: BTreeMap, + ) -> Self { + Self { block_number, latest, overlays } } +} + +impl AccountTreeWithHistory { + // MUTATORS + // -------------------------------------------------------------------------------------------- /// Computes mutations relative to the latest state. pub fn compute_mutations( @@ -361,6 +372,17 @@ impl AccountTreeWithHistory { Ok(self.latest.compute_mutations(account_commitments)?) } + /// Computes and applies mutations in one operation. + /// + /// This is a convenience method primarily for testing. + pub fn compute_and_apply_mutations( + &mut self, + account_commitments: impl IntoIterator, + ) -> Result<(), HistoricalError> { + let mutations = self.compute_mutations(account_commitments)?; + self.apply_mutations(mutations) + } + /// Applies mutations and advances to the next block. /// /// This method: diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index 3863f4afb..9439640f0 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -14,7 +14,7 @@ use std::path::Path; use miden_crypto::merkle::mmr::Mmr; #[cfg(feature = "rocksdb")] -use miden_large_smt_backend_rocksdb::RocksDbStorage; +use miden_large_smt_backend_rocksdb::{RocksDbStorage, SmtStorageReader}; use miden_node_utils::clap::RocksDbOptions; use miden_protocol::block::account_tree::{AccountIdKey, AccountTree}; use miden_protocol::block::nullifier_tree::NullifierTree; @@ -57,12 +57,21 @@ const PUBLIC_ACCOUNT_IDS_PAGE_SIZE: NonZeroUsize = NonZeroUsize::new(1_000).unwr // STORAGE TYPE ALIAS // ================================================================================================ -/// The storage backend for trees. +/// The writable storage backend for trees. #[cfg(feature = "rocksdb")] pub type TreeStorage = RocksDbStorage; #[cfg(not(feature = "rocksdb"))] pub type TreeStorage = MemoryStorage; +/// The read-only storage backend used by `InMemoryState` for lock-free reads. +/// +/// With `rocksdb`, this is a snapshot-backed read-only storage (`RocksDbSnapshotStorage`). +/// Without `rocksdb`, this is the same as `TreeStorage` (in-memory, already `Clone`). +#[cfg(feature = "rocksdb")] +pub type SnapshotTreeStorage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage; +#[cfg(not(feature = "rocksdb"))] +pub type SnapshotTreeStorage = MemoryStorage; + // ERROR CONVERSION // ================================================================================================ @@ -340,7 +349,7 @@ impl StorageLoader for RocksDbStorage { /// Loads an SMT from persistent storage. #[cfg(feature = "rocksdb")] -pub fn load_smt(storage: S) -> Result, StateInitializationError> { +pub fn load_smt(storage: S) -> Result, StateInitializationError> { LargeSmt::load(storage).map_err(account_tree_large_smt_error_to_init_error) } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 257f7c79b..8390e774e 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -68,6 +68,7 @@ mod loader; use loader::{ ACCOUNT_TREE_STORAGE_DIR, NULLIFIER_TREE_STORAGE_DIR, + SnapshotTreeStorage, StorageLoader, TreeStorage, load_mmr, @@ -125,8 +126,8 @@ pub struct TransactionInputs { pub(crate) struct InMemoryState { /// The committed block number for this snapshot. pub block_num: BlockNumber, - /// Account tree with historical overlay support. - pub account_tree: AccountTreeWithHistory, + /// Account tree with historical overlay support (read-only, snapshot-backed). + pub account_tree: AccountTreeWithHistory, /// Chain MMR (Merkle Mountain Range of block commitments). pub blockchain: Blockchain, /// Forest state for account storage maps and vault witnesses. @@ -155,6 +156,19 @@ pub struct State { /// during writes. pub(super) nullifier_tree: WriterGuard>>, + /// Writable account tree — owned by the single writer task. + /// + /// Lock-free via [`WriterGuard`]: the writer applies mutations here, then creates + /// a snapshot-backed read-only copy for `InMemoryState`. + pub(super) account_tree: WriterGuard>, + + /// Handle to the `RocksDB` database used for account tree storage. + /// + /// Used to create snapshot storage instances for read-only snapshots in `InMemoryState`. + /// Stored as `Arc` obtained from `RocksDbStorage::db()`. + #[cfg(feature = "rocksdb")] + pub(super) account_db: std::sync::Arc, + /// All in-memory state (account tree, blockchain MMR, forest) held atomically. /// /// Readers call `snapshot()` which returns `Arc` via a wait-free atomic @@ -208,6 +222,11 @@ impl State { &storage_options.account_tree.into(), ACCOUNT_TREE_STORAGE_DIR, )?; + + // Grab the DB handle before loading (needed for creating snapshots). + #[cfg(feature = "rocksdb")] + let account_db = std::sync::Arc::clone(account_storage.db()); + let account_tree = account_storage.load_account_tree(&mut db).await?; let nullifier_storage = TreeStorage::create( @@ -220,7 +239,34 @@ impl State { // Verify that tree roots match the expected roots from the database. verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; - let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + // Create the writable account tree with history (owned by the writer). + let account_tree_with_history = AccountTreeWithHistory::new(account_tree, latest_block_num); + + // Create a snapshot-backed read-only account tree for InMemoryState. + let snapshot_account_tree = { + #[cfg(feature = "rocksdb")] + { + use miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage; + + let snapshot_storage = RocksDbSnapshotStorage::new(Arc::clone(&account_db)); + let snapshot_smt = loader::load_smt(snapshot_storage) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + // SAFETY: The snapshot reads from the same DB that the writable tree + // was just loaded and validated from. No need to re-validate. + let snapshot_tree = + miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); + AccountTreeWithHistory::from_parts( + snapshot_tree, + account_tree_with_history.block_number_latest(), + account_tree_with_history.overlays().clone(), + ) + } + #[cfg(not(feature = "rocksdb"))] + { + // In memory mode, the trees are the same type, just clone. + account_tree_with_history.clone() + } + }; let forest = load_smt_forest(&mut db, latest_block_num).await?; @@ -236,7 +282,7 @@ impl State { let in_memory = ArcSwap::from_pointee(InMemoryState { block_num: latest_block_num, - account_tree, + account_tree: snapshot_account_tree, blockchain, forest, }); @@ -245,6 +291,9 @@ impl State { db, block_store, nullifier_tree: WriterGuard::new(nullifier_tree), + account_tree: WriterGuard::new(account_tree_with_history), + #[cfg(feature = "rocksdb")] + account_db, in_memory, writer_tx, termination_ask, diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 968e4cd98..bcc697840 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -36,11 +36,10 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// ## Consistency model /// -/// This function is the sole writer to all in-memory state. The nullifier tree (backed by -/// `RocksDB` with MVCC) is accessed lock-free via [`WriterGuard`]. The in-memory structures -/// (`account_tree`, `blockchain`, `forest`) are held in an `Arc` behind an -/// `ArcSwap`. The writer loads the current state, validates against it, commits to DB, then -/// deep-clones the state, applies mutations, and atomically swaps the pointer. +/// This function is the sole writer to all state. Both the nullifier tree and account tree +/// (backed by `RocksDB` with MVCC) are accessed lock-free via [`WriterGuard`]. The writer +/// applies mutations to the writable trees, then creates a new `InMemoryState` with a +/// snapshot-backed read-only account tree and atomically swaps the pointer. /// /// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an /// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either @@ -49,8 +48,8 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// ## Performance /// -/// The only deep clone of `InMemoryState` occurs once per block in this function. Readers pay -/// only an atomic refcount bump per `snapshot()` call. +/// No deep clone of account tree data occurs. The snapshot-backed account tree reads directly +/// from a `RocksDB` snapshot. Readers pay only an atomic refcount bump per `snapshot()` call. #[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( @@ -107,9 +106,11 @@ async fn apply_block_inner( // Compute mutations required for updating account and nullifier trees. // The nullifier tree uses WriterGuard (RocksDB MVCC — safe for concurrent access). - // The account tree and blockchain are read from the snapshot (no locks needed). + // The account tree uses WriterGuard — the writer owns the writable copy. let (nullifier_tree_update, account_tree_update) = { let nullifier_tree = unsafe { state.nullifier_tree.as_mut() }; + // SAFETY: This is the single writer task, serialized by the channel. + let account_tree = unsafe { state.account_tree.as_mut() }; let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); @@ -144,9 +145,8 @@ async fn apply_block_inner( return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); } - // Compute update for account tree. - let account_tree_update = snapshot - .account_tree + // Compute update for account tree from the writable tree (always in sync with DB). + let account_tree_update = account_tree .compute_mutations( body.updated_accounts() .iter() @@ -223,11 +223,7 @@ async fn apply_block_inner( // Await the block store save task. block_save_task.await??; - // Deep-clone the in-memory state to produce an owned mutable copy for applying mutations. - // This is the only deep clone per block — readers pay only an atomic refcount bump. - let mut new_state = InMemoryState::clone(&snapshot); - - // Nullifier tree: lock-free via WriterGuard (RocksDB MVCC). + // Apply mutations to the writable trees (writes to RocksDB). // SAFETY: This is the single writer task, serialized by the channel. unsafe { state @@ -237,16 +233,60 @@ async fn apply_block_inner( .expect("Unreachable: mutations were computed from the current tree state"); } - new_state - .account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); + // SAFETY: This is the single writer task, serialized by the channel. + unsafe { + state + .account_tree + .as_mut() + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + } - new_state.blockchain.push(block_commitment); + // Build a new read-only InMemoryState with a snapshot-backed account tree. + // The snapshot captures the RocksDB state after mutations have been applied. + let snapshot_account_tree = { + #[cfg(feature = "rocksdb")] + { + use crate::accounts::AccountTreeWithHistory; + use crate::state::loader; + + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&state.account_db), + ); + let snapshot_smt = loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + let snapshot_tree = + miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); + + // SAFETY: Single writer — safe to read overlays and block number. + let writable_tree = unsafe { state.account_tree.as_mut() }; + AccountTreeWithHistory::from_parts( + snapshot_tree, + writable_tree.block_number_latest(), + writable_tree.overlays().clone(), + ) + } + #[cfg(not(feature = "rocksdb"))] + { + // In memory mode, clone the writable tree (which already has mutations applied). + // SAFETY: Single writer — safe to read from the writable tree. + let writable_tree = unsafe { state.account_tree.as_mut() }; + writable_tree.clone() + } + }; + + let mut new_blockchain = snapshot.blockchain.clone(); + new_blockchain.push(block_commitment); - new_state.forest.apply_block_updates(block_num, account_deltas)?; + let mut new_forest = snapshot.forest.clone(); + new_forest.apply_block_updates(block_num, account_deltas)?; - new_state.block_num = block_num; + let new_state = InMemoryState { + block_num, + account_tree: snapshot_account_tree, + blockchain: new_blockchain, + forest: new_forest, + }; // Atomically publish the new state. Readers that call snapshot() after this point // will see the updated state. Readers holding the old Arc continue unaffected. From 407bce1ee1b7239b5e64bde20ead4978a18b3429 Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 16:06:08 +1200 Subject: [PATCH 13/48] Reduce WriterGuard --- Cargo.lock | 16 +-- crates/store/src/state/mod.rs | 165 ++++++++++++++----------- crates/store/src/state/writer.rs | 28 ++++- crates/store/src/state/writer_guard.rs | 38 ++---- 4 files changed, 133 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 621badce0..b271e6d50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2839,7 +2839,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "alloy-sol-types", "fs-err", @@ -2914,7 +2914,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -3531,7 +3531,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "bech32", "fs-err", @@ -3560,7 +3560,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "proc-macro2", "quote", @@ -3655,7 +3655,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "fs-err", "miden-assembly", @@ -3672,7 +3672,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3694,7 +3694,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "miden-processor", "miden-protocol", @@ -3707,7 +3707,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a16534c873596cc6da0b59cca61a69348be88834" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" dependencies = [ "miden-protocol", "miden-tx", diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 8390e774e..04052b9ad 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -1,13 +1,13 @@ //! State management for the Miden store. //! //! The [State] provides data access and modification methods. A single writer task, serialized by -//! a channel, applies block mutations. In-memory structures (`account_tree`, `blockchain`, -//! `forest`) are held in an [`Arc`] behind an [`ArcSwap`](arc_swap::ArcSwap), providing wait-free -//! reads with no lock contention. The `RocksDB`-backed nullifier tree uses a lock-free -//! [`WriterGuard`] because `RocksDB` MVCC provides snapshot isolation. +//! a channel, applies block mutations. All reader-visible state (trees, blockchain MMR, forest) is +//! held in an [`Arc`] behind an [`ArcSwap`](arc_swap::ArcSwap), providing wait-free +//! reads with no lock contention. //! //! Readers obtain an `Arc` via [`State::snapshot()`] (wait-free, no locks). -//! The writer clones the current state, mutates the clone, and atomically swaps the pointer. +//! The writer applies mutations to its own writable trees (behind [`WriterGuard`]), then builds a +//! new `InMemoryState` with snapshot-backed read-only copies and atomically swaps the pointer. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; @@ -126,6 +126,8 @@ pub struct TransactionInputs { pub(crate) struct InMemoryState { /// The committed block number for this snapshot. pub block_num: BlockNumber, + /// Nullifier tree (read-only, snapshot-backed). + pub nullifier_tree: NullifierTree>, /// Account tree with historical overlay support (read-only, snapshot-backed). pub account_tree: AccountTreeWithHistory, /// Chain MMR (Merkle Mountain Range of block commitments). @@ -139,9 +141,10 @@ pub(crate) struct InMemoryState { /// The rollup state. /// -/// A single writer task (serialized by a channel) mutates the state. In-memory structures are -/// held in an `Arc` behind an [`ArcSwap`], providing wait-free reads. The -/// `RocksDB`-backed nullifier tree is lock-free via [`WriterGuard`]. +/// A single writer task (serialized by a channel) mutates the state. All trees, the blockchain +/// MMR, and the forest are held in an `Arc` behind an [`ArcSwap`], providing +/// wait-free reads. The writer keeps its own writable copies of the trees behind [`WriterGuard`] +/// and creates snapshot-backed read-only copies for `InMemoryState` after each block. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -150,30 +153,33 @@ pub struct State { /// The block store which stores full block contents for all blocks. pub(super) block_store: Arc, - /// Nullifier tree — append-only, backed by `RocksDB`. + /// Writable nullifier tree — owned by the single writer task. /// - /// Lock-free via [`WriterGuard`]: `RocksDB` MVCC provides safe concurrent read access - /// during writes. + /// Writer-only via [`WriterGuard`]. Readers access the snapshot-backed copy in + /// `InMemoryState` instead. pub(super) nullifier_tree: WriterGuard>>, /// Writable account tree — owned by the single writer task. /// - /// Lock-free via [`WriterGuard`]: the writer applies mutations here, then creates - /// a snapshot-backed read-only copy for `InMemoryState`. + /// Writer-only via [`WriterGuard`]. Readers access the snapshot-backed copy in + /// `InMemoryState` instead. pub(super) account_tree: WriterGuard>, - /// Handle to the `RocksDB` database used for account tree storage. - /// + /// Handle to the RocksDB database used for account tree storage. /// Used to create snapshot storage instances for read-only snapshots in `InMemoryState`. - /// Stored as `Arc` obtained from `RocksDbStorage::db()`. #[cfg(feature = "rocksdb")] pub(super) account_db: std::sync::Arc, - /// All in-memory state (account tree, blockchain MMR, forest) held atomically. + /// Handle to the RocksDB database used for nullifier tree storage. + /// Used to create snapshot storage instances for read-only snapshots in `InMemoryState`. + #[cfg(feature = "rocksdb")] + pub(super) nullifier_db: std::sync::Arc, + + /// All in-memory state held atomically behind an `ArcSwap`. /// /// Readers call `snapshot()` which returns `Arc` via a wait-free atomic - /// refcount bump — no data cloning. The writer deep-clones once per block, mutates the - /// copy, and atomically swaps via `ArcSwap::store()`. + /// refcount bump — no data cloning. The writer builds a new `InMemoryState` with + /// snapshot-backed trees after each block and atomically swaps via `ArcSwap::store()`. pub(super) in_memory: ArcSwap, /// Channel to the single writer task. @@ -234,6 +240,11 @@ impl State { &storage_options.nullifier_tree.into(), NULLIFIER_TREE_STORAGE_DIR, )?; + + // Grab the DB handle before loading (needed for creating snapshots). + #[cfg(feature = "rocksdb")] + let nullifier_db = std::sync::Arc::clone(nullifier_storage.db()); + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; // Verify that tree roots match the expected roots from the database. @@ -268,6 +279,24 @@ impl State { } }; + // Create a snapshot-backed read-only nullifier tree for InMemoryState. + let snapshot_nullifier_tree = { + #[cfg(feature = "rocksdb")] + { + use miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage; + + let snapshot_storage = + RocksDbSnapshotStorage::new(std::sync::Arc::clone(&nullifier_db)); + let snapshot_smt = loader::load_smt(snapshot_storage) + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.to_string()))?; + NullifierTree::new_unchecked(snapshot_smt) + } + #[cfg(not(feature = "rocksdb"))] + { + nullifier_tree.clone() + } + }; + let forest = load_smt_forest(&mut db, latest_block_num).await?; let db = Arc::new(db); @@ -282,6 +311,7 @@ impl State { let in_memory = ArcSwap::from_pointee(InMemoryState { block_num: latest_block_num, + nullifier_tree: snapshot_nullifier_tree, account_tree: snapshot_account_tree, blockchain, forest, @@ -294,6 +324,8 @@ impl State { account_tree: WriterGuard::new(account_tree_with_history), #[cfg(feature = "rocksdb")] account_db, + #[cfg(feature = "rocksdb")] + nullifier_db, in_memory, writer_tx, termination_ask, @@ -400,10 +432,10 @@ impl State { /// Note: these proofs are invalidated once the nullifier tree is modified, i.e. on a new block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret)] pub async fn check_nullifiers(&self, nullifiers: &[Nullifier]) -> Vec { - let nullifier_tree = self.nullifier_tree.as_ref(); + let snapshot = self.snapshot(); nullifiers .iter() - .map(|n| nullifier_tree.open(n)) + .map(|n| snapshot.nullifier_tree.open(n)) .map(NullifierWitness::into_proof) .collect() } @@ -637,49 +669,41 @@ impl State { ), GetBlockInputsError, > { - // Take a snapshot and extract everything we need, then drop it so readers of newer - // snapshots aren't held up by this Arc. - let (latest_block_number, partial_mmr, account_witnesses) = { - let snapshot = self.snapshot(); - let latest_block_number = snapshot.block_num; - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } + // Take a snapshot and extract everything we need from it. + let snapshot = self.snapshot(); + let latest_block_number = snapshot.block_num; + + // If `blocks` is empty, use the latest block number which will never trigger the error. + let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); + if highest_block_number > latest_block_number { + return Err(GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number, + latest_block_number, + }); + } - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove - // its inclusion in the chain. - blocks.remove(&latest_block_number); - - let partial_mmr = snapshot - .blockchain - .partial_mmr_from_blocks(blocks, latest_block_number) - .expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); - - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, snapshot.account_tree.open_latest(account_id))) - .collect::>(); - - (latest_block_number, partial_mmr, account_witnesses) - }; + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + let partial_mmr = + snapshot.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, snapshot.account_tree.open_latest(account_id))) + .collect::>(); // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent or // not as this is done as part of proposing the block. - let nullifier_tree = self.nullifier_tree.as_ref(); let nullifier_witnesses: BTreeMap = nullifiers .iter() .copied() - .map(|nullifier| (nullifier, nullifier_tree.open(&nullifier))) + .map(|nullifier| (nullifier, snapshot.nullifier_tree.open(&nullifier))) .collect(); Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) @@ -697,23 +721,14 @@ impl State { // Take a snapshot and extract everything we need, then drop it so readers of newer // snapshots aren't held up by this Arc. - let (new_account_id_prefix_is_unique, account_commitment) = { - let snapshot = self.snapshot(); + let snapshot = self.snapshot(); - let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); + let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); - if account_commitment.is_empty() { - ( - Some( - !snapshot - .account_tree - .contains_account_id_prefix_in_latest(account_id.prefix()), - ), - account_commitment, - ) - } else { - (None, account_commitment) - } + let new_account_id_prefix_is_unique = if account_commitment.is_empty() { + Some(!snapshot.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + } else { + None }; // Non-unique account Id prefixes for new accounts are not allowed. @@ -724,15 +739,17 @@ impl State { }); } - let nullifier_tree = self.nullifier_tree.as_ref(); let nullifiers = nullifiers .iter() .map(|nullifier| NullifierInfo { nullifier: *nullifier, - block_num: nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + block_num: snapshot.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), }) .collect(); + // Drop snapshot before the async DB call. + drop(snapshot); + let found_unauthenticated_notes = self .db .select_existing_note_commitments(unauthenticated_note_commitments) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index bcc697840..bbf595ac5 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -242,8 +242,31 @@ async fn apply_block_inner( .expect("Unreachable: mutations were computed from the current tree state"); } - // Build a new read-only InMemoryState with a snapshot-backed account tree. - // The snapshot captures the RocksDB state after mutations have been applied. + // Build a new read-only InMemoryState with snapshot-backed trees. + // The snapshots capture the RocksDB state after mutations have been applied. + let snapshot_nullifier_tree = { + #[cfg(feature = "rocksdb")] + { + use miden_protocol::block::nullifier_tree::NullifierTree; + + use crate::state::loader; + + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&state.nullifier_db), + ); + let snapshot_smt = loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + NullifierTree::new_unchecked(snapshot_smt) + } + #[cfg(not(feature = "rocksdb"))] + { + // In memory mode, clone the writable tree (which already has mutations applied). + // SAFETY: Single writer — safe to read from the writable tree. + let writable_tree = unsafe { state.nullifier_tree.as_mut() }; + writable_tree.clone() + } + }; + let snapshot_account_tree = { #[cfg(feature = "rocksdb")] { @@ -283,6 +306,7 @@ async fn apply_block_inner( let new_state = InMemoryState { block_num, + nullifier_tree: snapshot_nullifier_tree, account_tree: snapshot_account_tree, blockchain: new_blockchain, forest: new_forest, diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs index eccd0a08e..4ebf61c27 100644 --- a/crates/store/src/state/writer_guard.rs +++ b/crates/store/src/state/writer_guard.rs @@ -1,32 +1,22 @@ use std::cell::UnsafeCell; -/// A single-writer / multi-reader wrapper that provides lock-free access to shared state. +/// A single-writer wrapper that provides interior mutability for the writer task. /// -/// This type enables a pattern where one dedicated writer task mutates data while many reader -/// tasks concurrently access it, without any locks. +/// This type enables a pattern where one dedicated writer task mutates data stored in a shared +/// `Arc`, without any locks. Readers do NOT access the wrapped data — they use +/// snapshot-backed copies in `InMemoryState` via `ArcSwap` instead. /// /// # Safety Contract /// -/// 1. **Single writer**: Only one task (the writer, serialized by a channel) may call -/// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type -/// system. -/// 2. **Concurrent read safety**: The wrapped type (currently the `RocksDB`-backed nullifier tree) -/// provides its own MVCC / snapshot isolation, making concurrent reads during writes safe at the -/// storage layer. -/// 3. **Append-only data**: The wrapped data structures are append-only (keyed by block number), so -/// readers observing an older state simply query at that older block number, which is safe. -/// -/// Concurrent read safety relies on the guarantees of the underlying storage engine (e.g. -/// `RocksDB` MVCC). +/// **Single writer**: Only one task (the writer, serialized by a channel) may call +/// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type +/// system. No reader access is provided. pub struct WriterGuard { inner: UnsafeCell, } // SAFETY: The single-writer invariant is enforced by the channel-based writer task architecture. -// Readers only call `as_ref()` which returns `&T`. Concurrent read safety during writes is -// guaranteed by the underlying storage engine (RocksDB MVCC / snapshot isolation), not by -// acquire/release barriers. The data structures are append-only, so readers see a consistent -// view at their query's block number. +// No reader access is provided — readers use snapshot-backed copies in InMemoryState instead. unsafe impl Send for WriterGuard {} unsafe impl Sync for WriterGuard {} @@ -36,18 +26,6 @@ impl WriterGuard { Self { inner: UnsafeCell::new(value) } } - /// Returns a shared reference to the wrapped value. - /// - /// Safe for any reader thread. Concurrent read safety is provided by the underlying storage - /// engine (e.g. `RocksDB` MVCC / snapshot isolation). The data structures are append-only, - /// so readers see a consistent view at their query's block number. - pub(super) fn as_ref(&self) -> &T { - // SAFETY: Single-writer is enforced by the channel-based writer task. Concurrent reads - // are safe because the underlying storage engine (RocksDB) provides MVCC / snapshot - // isolation, and the data is append-only. - unsafe { &*self.inner.get() } - } - /// Returns an exclusive mutable reference to the wrapped value. /// /// # Safety From b3d11d429f97475226e5d5d47f6981ba979f6e40 Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 16:14:40 +1200 Subject: [PATCH 14/48] Remove WriterGuard --- crates/store/src/state/mod.rs | 41 +++++-------- crates/store/src/state/writer.rs | 80 ++++++++++++-------------- crates/store/src/state/writer_guard.rs | 39 ------------- 3 files changed, 50 insertions(+), 110 deletions(-) delete mode 100644 crates/store/src/state/writer_guard.rs diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 04052b9ad..ea863f491 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -6,8 +6,8 @@ //! reads with no lock contention. //! //! Readers obtain an `Arc` via [`State::snapshot()`] (wait-free, no locks). -//! The writer applies mutations to its own writable trees (behind [`WriterGuard`]), then builds a -//! new `InMemoryState` with snapshot-backed read-only copies and atomically swaps the pointer. +//! The writer applies mutations to its own writable trees (owned directly, no locks), then builds +//! a new `InMemoryState` with snapshot-backed read-only copies and atomically swaps the pointer. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; @@ -78,9 +78,6 @@ use loader::{ mod sync_state; pub(crate) mod writer; -mod writer_guard; - -pub(crate) use writer_guard::WriterGuard; // FINALITY // ================================================================================================ @@ -143,8 +140,9 @@ pub(crate) struct InMemoryState { /// /// A single writer task (serialized by a channel) mutates the state. All trees, the blockchain /// MMR, and the forest are held in an `Arc` behind an [`ArcSwap`], providing -/// wait-free reads. The writer keeps its own writable copies of the trees behind [`WriterGuard`] -/// and creates snapshot-backed read-only copies for `InMemoryState` after each block. +/// wait-free reads. The writer owns writable copies of the trees directly (passed as owned +/// values to [`writer::writer_loop`]) and creates snapshot-backed read-only copies for +/// `InMemoryState` after each block. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -153,25 +151,13 @@ pub struct State { /// The block store which stores full block contents for all blocks. pub(super) block_store: Arc, - /// Writable nullifier tree — owned by the single writer task. - /// - /// Writer-only via [`WriterGuard`]. Readers access the snapshot-backed copy in - /// `InMemoryState` instead. - pub(super) nullifier_tree: WriterGuard>>, - - /// Writable account tree — owned by the single writer task. - /// - /// Writer-only via [`WriterGuard`]. Readers access the snapshot-backed copy in - /// `InMemoryState` instead. - pub(super) account_tree: WriterGuard>, - /// Handle to the RocksDB database used for account tree storage. - /// Used to create snapshot storage instances for read-only snapshots in `InMemoryState`. + /// Used by the writer to create snapshot storage instances for `InMemoryState`. #[cfg(feature = "rocksdb")] pub(super) account_db: std::sync::Arc, /// Handle to the RocksDB database used for nullifier tree storage. - /// Used to create snapshot storage instances for read-only snapshots in `InMemoryState`. + /// Used by the writer to create snapshot storage instances for `InMemoryState`. #[cfg(feature = "rocksdb")] pub(super) nullifier_db: std::sync::Arc, @@ -320,8 +306,6 @@ impl State { let state = Arc::new(Self { db, block_store, - nullifier_tree: WriterGuard::new(nullifier_tree), - account_tree: WriterGuard::new(account_tree_with_history), #[cfg(feature = "rocksdb")] account_db, #[cfg(feature = "rocksdb")] @@ -332,9 +316,14 @@ impl State { proven_tip, }); - // Spawn the single writer task. + // Spawn the single writer task with owned writable trees. let writer_state = Arc::clone(&state); - tokio::spawn(writer::writer_loop(writer_rx, writer_state)); + tokio::spawn(writer::writer_loop( + writer_rx, + writer_state, + nullifier_tree, + account_tree_with_history, + )); Ok((state, proven_tip_writer)) } @@ -381,8 +370,6 @@ impl State { /// The returned `Arc` is a frozen view: it keeps the snapshot alive for as long as needed, /// even if the writer swaps in a new state in the meantime. Readers holding the old `Arc` /// are completely unaffected by the swap. - /// - /// The nullifier tree is accessed separately via [`WriterGuard`] (lock-free, `RocksDB` MVCC). fn snapshot(&self) -> Arc { self.in_memory.load_full() } diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index bbf595ac5..b97298bff 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -4,14 +4,18 @@ use miden_node_proto::BlockProofRequest; use miden_node_utils::ErrorReport; use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::block::SignedBlock; +use miden_protocol::block::nullifier_tree::NullifierTree; +use miden_protocol::crypto::merkle::smt::LargeSmt; use miden_protocol::note::NoteDetails; use miden_protocol::transaction::OutputNote; use miden_protocol::utils::serde::Serializable; use tokio::sync::{mpsc, oneshot}; use tracing::{Instrument, info, info_span, instrument}; +use crate::accounts::AccountTreeWithHistory; use crate::db::NoteRecord; use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::loader::TreeStorage; use crate::state::{InMemoryState, State}; use crate::{COMPONENT, HistoricalError}; @@ -23,11 +27,22 @@ pub struct WriteRequest { } /// Runs the single writer loop. Receives blocks through the channel and applies them -/// sequentially. Channel serialization guarantees no concurrent writers — no mutex needed. -pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc) { +/// sequentially. The writer owns the writable trees — no locks or interior mutability needed. +pub(crate) async fn writer_loop( + mut rx: mpsc::Receiver, + state: Arc, + mut nullifier_tree: NullifierTree>, + mut account_tree: AccountTreeWithHistory, +) { while let Some(req) = rx.recv().await { - let result = - Box::pin(apply_block_inner(&state, req.signed_block, req.proving_inputs)).await; + let result = Box::pin(apply_block_inner( + &state, + &mut nullifier_tree, + &mut account_tree, + req.signed_block, + req.proving_inputs, + )) + .await; let _ = req.result_tx.send(result); } } @@ -36,10 +51,9 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// ## Consistency model /// -/// This function is the sole writer to all state. Both the nullifier tree and account tree -/// (backed by `RocksDB` with MVCC) are accessed lock-free via [`WriterGuard`]. The writer -/// applies mutations to the writable trees, then creates a new `InMemoryState` with a -/// snapshot-backed read-only account tree and atomically swaps the pointer. +/// This function is the sole writer to all state. The writer owns the writable trees directly +/// (no locks or interior mutability). It applies mutations, then creates a new `InMemoryState` +/// with snapshot-backed read-only copies and atomically swaps the pointer. /// /// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an /// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either @@ -48,12 +62,14 @@ pub(crate) async fn writer_loop(mut rx: mpsc::Receiver, state: Arc /// /// ## Performance /// -/// No deep clone of account tree data occurs. The snapshot-backed account tree reads directly -/// from a `RocksDB` snapshot. Readers pay only an atomic refcount bump per `snapshot()` call. +/// No deep clone of tree data occurs in RocksDB mode. The snapshot-backed trees read directly +/// from RocksDB snapshots. Readers pay only an atomic refcount bump per `snapshot()` call. #[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( state: &State, + nullifier_tree: &mut NullifierTree>, + account_tree: &mut AccountTreeWithHistory, signed_block: SignedBlock, proving_inputs: Option, ) -> Result<(), ApplyBlockError> { @@ -105,13 +121,7 @@ async fn apply_block_inner( let snapshot = state.in_memory.load_full(); // Compute mutations required for updating account and nullifier trees. - // The nullifier tree uses WriterGuard (RocksDB MVCC — safe for concurrent access). - // The account tree uses WriterGuard — the writer owns the writable copy. let (nullifier_tree_update, account_tree_update) = { - let nullifier_tree = unsafe { state.nullifier_tree.as_mut() }; - // SAFETY: This is the single writer task, serialized by the channel. - let account_tree = unsafe { state.account_tree.as_mut() }; - let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); // Nullifiers can be produced only once. @@ -224,23 +234,13 @@ async fn apply_block_inner( block_save_task.await??; // Apply mutations to the writable trees (writes to RocksDB). - // SAFETY: This is the single writer task, serialized by the channel. - unsafe { - state - .nullifier_tree - .as_mut() - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - } + nullifier_tree + .apply_mutations(nullifier_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); - // SAFETY: This is the single writer task, serialized by the channel. - unsafe { - state - .account_tree - .as_mut() - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - } + account_tree + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); // Build a new read-only InMemoryState with snapshot-backed trees. // The snapshots capture the RocksDB state after mutations have been applied. @@ -260,10 +260,7 @@ async fn apply_block_inner( } #[cfg(not(feature = "rocksdb"))] { - // In memory mode, clone the writable tree (which already has mutations applied). - // SAFETY: Single writer — safe to read from the writable tree. - let writable_tree = unsafe { state.nullifier_tree.as_mut() }; - writable_tree.clone() + nullifier_tree.clone() } }; @@ -281,20 +278,15 @@ async fn apply_block_inner( let snapshot_tree = miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); - // SAFETY: Single writer — safe to read overlays and block number. - let writable_tree = unsafe { state.account_tree.as_mut() }; AccountTreeWithHistory::from_parts( snapshot_tree, - writable_tree.block_number_latest(), - writable_tree.overlays().clone(), + account_tree.block_number_latest(), + account_tree.overlays().clone(), ) } #[cfg(not(feature = "rocksdb"))] { - // In memory mode, clone the writable tree (which already has mutations applied). - // SAFETY: Single writer — safe to read from the writable tree. - let writable_tree = unsafe { state.account_tree.as_mut() }; - writable_tree.clone() + account_tree.clone() } }; diff --git a/crates/store/src/state/writer_guard.rs b/crates/store/src/state/writer_guard.rs deleted file mode 100644 index 4ebf61c27..000000000 --- a/crates/store/src/state/writer_guard.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::cell::UnsafeCell; - -/// A single-writer wrapper that provides interior mutability for the writer task. -/// -/// This type enables a pattern where one dedicated writer task mutates data stored in a shared -/// `Arc`, without any locks. Readers do NOT access the wrapped data — they use -/// snapshot-backed copies in `InMemoryState` via `ArcSwap` instead. -/// -/// # Safety Contract -/// -/// **Single writer**: Only one task (the writer, serialized by a channel) may call -/// [`as_mut()`](Self::as_mut). This invariant is enforced architecturally, not by the type -/// system. No reader access is provided. -pub struct WriterGuard { - inner: UnsafeCell, -} - -// SAFETY: The single-writer invariant is enforced by the channel-based writer task architecture. -// No reader access is provided — readers use snapshot-backed copies in InMemoryState instead. -unsafe impl Send for WriterGuard {} -unsafe impl Sync for WriterGuard {} - -impl WriterGuard { - /// Creates a new `WriterGuard` wrapping the given value. - pub fn new(value: T) -> Self { - Self { inner: UnsafeCell::new(value) } - } - - /// Returns an exclusive mutable reference to the wrapped value. - /// - /// # Safety - /// - /// Must only be called from the single writer task. The caller must ensure: - /// - No other calls to `as_mut()` are concurrent (enforced by channel serialization). - #[expect(clippy::mut_from_ref)] - pub unsafe fn as_mut(&self) -> &mut T { - unsafe { &mut *self.inner.get() } - } -} From a1e2cd5fddfeed47c1175c0162e722acbfd0bd81 Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 16:16:13 +1200 Subject: [PATCH 15/48] Lint --- crates/store/src/state/mod.rs | 4 ++-- crates/store/src/state/writer.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index ea863f491..a3820de43 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -151,12 +151,12 @@ pub struct State { /// The block store which stores full block contents for all blocks. pub(super) block_store: Arc, - /// Handle to the RocksDB database used for account tree storage. + /// Handle to the `RocksDB` database used for account tree storage. /// Used by the writer to create snapshot storage instances for `InMemoryState`. #[cfg(feature = "rocksdb")] pub(super) account_db: std::sync::Arc, - /// Handle to the RocksDB database used for nullifier tree storage. + /// Handle to the `RocksDB` database used for nullifier tree storage. /// Used by the writer to create snapshot storage instances for `InMemoryState`. #[cfg(feature = "rocksdb")] pub(super) nullifier_db: std::sync::Arc, diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index b97298bff..f2a791990 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -62,8 +62,8 @@ pub(crate) async fn writer_loop( /// /// ## Performance /// -/// No deep clone of tree data occurs in RocksDB mode. The snapshot-backed trees read directly -/// from RocksDB snapshots. Readers pay only an atomic refcount bump per `snapshot()` call. +/// No deep clone of tree data occurs in `RocksDB` mode. The snapshot-backed trees read directly +/// from `RocksDB` snapshots. Readers pay only an atomic refcount bump per `snapshot()` call. #[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( From 9fc06c8b842f4972b452e2fec871064939b5d6ce Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 16:35:20 +1200 Subject: [PATCH 16/48] Fix ntx builder --- crates/ntx-builder/src/builder.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/ntx-builder/src/builder.rs b/crates/ntx-builder/src/builder.rs index 3cb2d60a4..bca8ce5fd 100644 --- a/crates/ntx-builder/src/builder.rs +++ b/crates/ntx-builder/src/builder.rs @@ -297,6 +297,14 @@ impl NetworkTransactionBuilder { async fn update_chain_tip(&mut self, tip: BlockHeader) { let mut chain_state = self.chain_state.write().await; + // Skip if this block is already accounted for. This can happen during initialization: + // the mempool subscription is created before the chain state is fetched from the store, + // so BlockCommitted events for blocks that occurred in between are already reflected in + // the initial chain state. + if tip.block_num() <= chain_state.chain_tip_header.block_num() { + return; + } + // Update MMR which lags by one block. let mmr_tip = chain_state.chain_tip_header.clone(); Arc::make_mut(&mut chain_state.chain_mmr).add_block(&mmr_tip, true); From 029477329a8423dab5cec2f9e456d150caf7d18a Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 10 Apr 2026 20:40:55 +1200 Subject: [PATCH 17/48] Scope db queries by state snapshot block num --- crates/store/src/account_state_forest/mod.rs | 2 ++ crates/store/src/errors.rs | 5 ++++ crates/store/src/state/mod.rs | 28 +++++++++++++++----- crates/store/src/state/writer.rs | 3 ++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 9cbc64a48..58eb77191 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -58,6 +58,8 @@ pub enum WitnessError { StorageMapError(#[from] StorageMapError), #[error("failed to construct asset")] AssetError(#[from] AssetError), + #[error("block {0} is unknown")] + UnknownBlock(BlockNumber), } // ACCOUNT STATE FOREST diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index f5cd1f0e7..d5b7d6bf4 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -87,6 +87,8 @@ pub enum DatabaseError { AccountsNotFoundInDb(Vec), #[error("account {0} is not on the chain")] AccountNotPublic(AccountId), + #[error("block {0} is unknown")] + UnknownBlock(BlockNumber), #[error("invalid block parameters: block_from ({from}) > block_to ({to})")] InvalidBlockRange { from: BlockNumber, to: BlockNumber }, #[error("data corrupted: {0}")] @@ -231,6 +233,9 @@ pub enum GetBlockHeaderError { #[error("error retrieving the merkle proof for the block")] #[grpc(internal)] MmrError(#[from] MmrError), + #[error("block {0} is unknown")] + #[grpc(invalid_argument)] + UnknownBlock(BlockNumber), } #[derive(Error, Debug)] diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index a3820de43..e05ef6272 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -398,10 +398,16 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - let block_header = self.db.select_block_header_by_block_num(block_num).await?; + // Scope the DB query to the snapshot's block number to ensure consistency between + // the block header (from SQLite) and the MMR proof (from the snapshot). + let snapshot = self.snapshot(); + let block_num = block_num.unwrap_or(snapshot.block_num); + if block_num > snapshot.block_num { + return Err(GetBlockHeaderError::UnknownBlock(block_num)); + } + let block_header = self.db.select_block_header_by_block_num(Some(block_num)).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { - let snapshot = self.snapshot(); let mmr_proof = snapshot.blockchain.open(header.block_num())?; Some(mmr_proof) } else { @@ -451,14 +457,14 @@ impl State { return Ok(None); } - // SAFETY: `select_block_header_by_block_num` will always return `Some(chain_tip_header)` - // when `None` is passed + // Scope the DB query to the snapshot's block number to ensure consistency between + // the block header (from SQLite) and the blockchain peaks (from the snapshot). let block_header: BlockHeader = self .db - .select_block_header_by_block_num(None) + .select_block_header_by_block_num(Some(snapshot.block_num)) .await .map_err(GetCurrentBlockchainDataError::ErrorRetrievingBlockHeader)? - .unwrap(); + .expect("block header for snapshot block number must exist in DB"); let peaks = snapshot .blockchain .peaks_at(block_header.block_num()) @@ -1015,6 +1021,10 @@ impl State { block_num: BlockNumber, page: Page, ) -> Result<(Vec, Page), DatabaseError> { + let snapshot = self.snapshot(); + if block_num > snapshot.block_num { + return Err(DatabaseError::UnknownBlock(block_num)); + } self.db.select_unconsumed_network_notes(account_id, block_num, page).await } @@ -1034,6 +1044,9 @@ impl State { vault_keys: BTreeSet, ) -> Result, WitnessError> { let snapshot = self.snapshot(); + if block_num > snapshot.block_num { + return Err(WitnessError::UnknownBlock(block_num)); + } let witnesses = snapshot.forest.get_vault_asset_witnesses(account_id, block_num, vault_keys)?; Ok(witnesses) @@ -1052,6 +1065,9 @@ impl State { raw_key: StorageMapKey, ) -> Result { let snapshot = self.snapshot(); + if block_num > snapshot.block_num { + return Err(WitnessError::UnknownBlock(block_num)); + } let witness = snapshot .forest .get_storage_map_witness(account_id, slot_name, block_num, raw_key)?; diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index f2a791990..c6c1bac0e 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -223,7 +223,8 @@ async fn apply_block_inner( )); // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while - // the DB commits. + // the DB commits. We ensure consistency by scoping all RPC queries that hit DB data by the + // block number that is Arc swapped at the end of this function. state .db .apply_block(signed_block, notes, proving_inputs) From 6b6c5c126e6ce2e6dab023546c9e56657764a4eb Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 13 Apr 2026 12:56:22 +1200 Subject: [PATCH 18/48] Update upstream deps and user readers properly --- Cargo.lock | 40 +++++++++--------- Cargo.toml | 14 +++--- bin/stress-test/src/seeding/mod.rs | 2 +- crates/block-producer/src/test_utils/batch.rs | 2 +- crates/large-smt-backend-rocksdb/src/lib.rs | 1 - .../large-smt-backend-rocksdb/src/rocksdb.rs | 6 +-- crates/store/src/db/models/queries/notes.rs | 2 +- .../samples/02-with-account-files/bridge.mac | Bin 31163 -> 31178 bytes crates/validator/src/server/tests.rs | 2 +- 9 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b271e6d50..eef890824 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2838,8 +2838,8 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "alloy-sol-types", "fs-err", @@ -2913,8 +2913,8 @@ dependencies = [ [[package]] name = "miden-block-prover" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2962,7 +2962,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" dependencies = [ "blake3", "cc", @@ -3004,7 +3004,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" dependencies = [ "quote", "syn 2.0.117", @@ -3031,7 +3031,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3530,8 +3530,8 @@ dependencies = [ [[package]] name = "miden-protocol" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "bech32", "fs-err", @@ -3559,8 +3559,8 @@ dependencies = [ [[package]] name = "miden-protocol-macros" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "proc-macro2", "quote", @@ -3646,7 +3646,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#17e071c17b6343b5ca8c4a6c803f23c429fbab8e" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" dependencies = [ "p3-field", "p3-goldilocks", @@ -3654,8 +3654,8 @@ dependencies = [ [[package]] name = "miden-standards" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "fs-err", "miden-assembly", @@ -3671,8 +3671,8 @@ dependencies = [ [[package]] name = "miden-testing" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3693,8 +3693,8 @@ dependencies = [ [[package]] name = "miden-tx" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "miden-processor", "miden-protocol", @@ -3706,8 +3706,8 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" -version = "0.14.3" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#a56b359904e5f9918eb1bcc8c4fdf429cb3ea78e" +version = "0.15.0" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" dependencies = [ "miden-protocol", "miden-tx", diff --git a/Cargo.toml b/Cargo.toml index 43e3e83fc..5c91269cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,13 +61,13 @@ miden-remote-prover-client = { path = "crates/remote-prover-client", versio miden-node-rocksdb-cxx-linkage-fix = { path = "crates/rocksdb-cxx-linkage-fix", version = "0.15" } # miden-protocol dependencies. These should be updated in sync. -miden-agglayer = { version = "0.14" } -miden-block-prover = { version = "0.14" } -miden-protocol = { default-features = false, version = "=0.14.3" } -miden-standards = { version = "0.14" } -miden-testing = { version = "0.14" } -miden-tx = { default-features = false, version = "0.14" } -miden-tx-batch-prover = { version = "0.14" } +miden-agglayer = { version = "0.15" } +miden-block-prover = { version = "0.15" } +miden-protocol = { default-features = false, version = "0.15" } +miden-standards = { version = "0.15" } +miden-testing = { version = "0.15" } +miden-tx = { default-features = false, version = "0.15" } +miden-tx-batch-prover = { version = "0.15" } # Other miden dependencies. These should align with those expected by miden-protocol. miden-crypto = { version = "0.23" } diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 79b461848..2a0a5a6ea 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -370,7 +370,7 @@ fn create_batch(txs: &[ProvenTransaction], block_ref: &BlockHeader) -> ProvenBat .collect(); let input_notes = txs.iter().flat_map(|tx| tx.input_notes().iter().cloned()).collect(); let output_notes = txs.iter().flat_map(|tx| tx.output_notes().iter().cloned()).collect(); - ProvenBatch::new( + ProvenBatch::new_unchecked( BatchId::from_transactions(txs.iter()), block_ref.commitment(), block_ref.block_num(), diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index ca705e241..df97d3ea4 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -59,7 +59,7 @@ impl TransactionBatchConstructor for ProvenBatch { output_notes.extend(tx.output_notes().iter().cloned()); } - ProvenBatch::new( + ProvenBatch::new_unchecked( BatchId::from_transactions(txs.iter().copied()), Word::empty(), BlockNumber::GENESIS, diff --git a/crates/large-smt-backend-rocksdb/src/lib.rs b/crates/large-smt-backend-rocksdb/src/lib.rs index 839289d03..0da257ca3 100644 --- a/crates/large-smt-backend-rocksdb/src/lib.rs +++ b/crates/large-smt-backend-rocksdb/src/lib.rs @@ -42,7 +42,6 @@ pub use miden_protocol::crypto::merkle::smt::{ SmtProof, SmtStorage, SmtStorageReader, - SmtStorageWriter, StorageError, StorageUpdateParts, StorageUpdates, diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 0e91a2006..3158fb56c 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -23,8 +23,8 @@ use rocksdb::{ }; use super::{ + SmtStorage, SmtStorageReader, - SmtStorageWriter, StorageError, StorageUpdateParts, StorageUpdates, @@ -564,7 +564,7 @@ impl SmtStorageReader for RocksDbStorage { } } -impl SmtStorageWriter for RocksDbStorage { +impl SmtStorage for RocksDbStorage { /// Inserts a key-value pair into the SMT leaf at the specified logical `index`. /// /// This operation involves: @@ -1017,7 +1017,7 @@ struct SnapshotInner { /// All clones share the same snapshot via `Arc`, providing a consistent view of /// the database at the time the snapshot was created. /// -/// Implements [`SmtStorageReader`] but not [`SmtStorageWriter`]. +/// Implements [`SmtStorageReader`] only (read-only snapshot). #[derive(Clone)] pub struct RocksDbSnapshotStorage { inner: Arc, diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index 266ef7f1a..1bb605125 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -804,7 +804,7 @@ impl TryInto for NoteMetadataRawRow { type Error = DatabaseError; fn try_into(self) -> Result { let sender = AccountId::read_from_bytes(&self.sender[..])?; - let note_type = NoteType::try_from(self.note_type as u32) + let note_type = NoteType::try_from(self.note_type as u8) .map_err(miden_node_db::DatabaseError::conversiont_from_sql::)?; let tag = NoteTag::new(self.tag as u32); let attachment = NoteAttachment::read_from_bytes(&self.attachment)?; diff --git a/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac b/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac index 4e6b5be52471d2afec0aabbab404007ca447c77c..f1e539b01e322f17b9ba6170038ee2307aa6c747 100644 GIT binary patch delta 1218 zcmajd`A<|=6bJA*uU17|TCEEf1w?$%R<_30u>uZdpbupp1F{Sw+5w~)*&K?zqW&PF z5E2T-Z;)zjUN90Jm&V~~mDV65(aBg9#T6^ExYS95IwEehwO9KG^#1V4$xY6=_nvcq z9ir1ibg+%1o62#nlB11E?Atxud&X&;>dhKOzP}AT@<=h;F}etMAZ(*Wz<#u9(LtU?yG^B z&9#FUy!WeA?Gso(>(@KXy3B)T2O@lakP_=V=Ql zNhAgOB#G>BCdqUa=4757U7sw`0(dMH={O9`)37deaH`3;9x(^srzFxOxKduf|J_uc z^iVS^VM=w-UIeAdbOg>ciB3V2&c801F41lDGatcV5INNp>@}RCX|QC7q=r4iK`~0T z(TDuhbvj)@SYmBQ16`RlZ)@~^A?j28#=6F4*T%A{p`cl5a4zEsa5K-qE0d=_Xfh=V zL2IT=Q7|kQ1N)}F`A?xk6SXfNI4f?Bz38pFG2?i5U2f0^YWK9RDHj4N8%r++;{5VZ z;e6a=@mk{0F8}*G_ulPuUMgri;khZN+3yhiXTH<)*=dec?NO}W8GJlwn^mrQ%!d?K z{}^1;w#U=cT-jK7qoKIWOK8lW4`GEwTG&_Y77mndev*1IYQ~PKt?ibGto5CHv~&O7 zIx^(a_6>*6R+U&iNu`QlHAgJ#YAAoJV(qP(7gxupcU*VeO^VbNMn9jMmz*-GA;C8T zwl713r)6B;!y7Fr;o~%Q!>QlAf9mCudzEKG=i5}hdF=(+J>%I87qV*ecK9s_FEy>p zI(O1M=H9;MzP0+c1((+i{;}{*YMq+)b46$Iru*fDNqK3el-x(q{sNKarbkZ3ym3SUpu zUdU5CT!kVvLtW&cX9}(E|6aer&Z(s((6%vKT#sO4(A{ko_fdqb833mZ`o1m<_f-4IIsN3>wA7+D# zB3jO*a;U*@<`I~?q80koC{Q;yc%}oany{3#YN(5$d`l%JsSE!=_lM7%U+%f*o^$WD zF=!csrb`sm^Ovbg3M%<{_dd#9G_^3&;|;N=?{qwki7)h!&RWpdxla@Q)W1!WbNFND zH*syxHLiVj^O2&SBbp&eR>_QugQ0xe+=-CjNmqXNi8HrTUKX^;UaoT!cXa;S@nr8< z^nA0oYNF^|cLUyC#e#sU)im5f6Y+Pn5Er0*H4A|#m$9Hgt856;v1tth1yykje1^(6 z8XB=Qj)O}md6xwz9wr8$M?3=qsEVf{2TjClXdyRDIEYFGOO1Nr&x&O5 zM@a$;^YL&32TM_!$UqXBh*q>Gve1w6Bo^kPHHm{5)UIQo6q|_0s**Y2u`Zc`UvM~? zrJl{?^B6yXK;EPb1Vv;-grOvrfhNo)wqSFrK#k1C@(o6C!|@GMR}b1qgAC z4lL+MW8phgs_3b7Di&_zF!2w}-NaBMbMX63?cj+fH3Lh~t`?{WKm0O!5nk4e2k(mL zuW$&qiROV`Ys@)YWwAc4>$b?hw_)`3S@D736gfIJ(@;^BPQyi9OuT{0bQXf}YB~o} z%*|jH-|(M)P14d;H(H*B{$Li@)3QAQLl64%?zMKdtjN-JpDw(<N(ZMYv;4~sqUn#jJwN5nJbcB^!5}`AhNC``MiFdpRsn{&^C?uM0tSe%Egc_ zbK}TQy5G2k^0_CyT@|6I+QP!yXy0O1JZ?;V=$jR3Q@aSAyfLtmb}et+UU0 zeI{>-@dA-3mgKB{o?|H6?4@3D?5|m=HRp2^>fhQwLvNuZ*(0GFJ)IMYXw`-*M)N=U zd)}G4^MPw0DPQl36?8`%c%$v$Q0YMX^@i9py)#LLF(lh^nWX1tUB_g2z_Q1+N3BmI zGtSeT4pg;LFxxjM~b=1#H?%LpeG~T#w334!*|WOa|<@O2@)OY}Rq4nW`)f zmg9I91K*-Nn}!e3kj;?kT1k8kUBqtm$zk9N)aI}-hz_D3uY5m^0xeqdXedTI@l_np zW8fyr^J#d12I4DNlh1$-UBu%UwM~HKRoeyf|E_N5pdE8}Fz^VQcd+1vQvK9G?RpyC z!ErqUpQHQ(0S0& Date: Mon, 13 Apr 2026 13:21:15 +1200 Subject: [PATCH 19/48] toml --- Cargo.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5c91269cf..9ac2bada0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,11 +147,11 @@ files.extend-exclude = ["*.svg"] # Ignore SVG files. miden-crypto = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } miden-serde-utils = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } -miden-agglayer = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-block-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-protocol = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-protocol-macros = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-standards = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-testing = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-tx = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } -miden-tx-batch-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-agglayer = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-block-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-protocol = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-protocol-macros = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-standards = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-testing = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-tx = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } +miden-tx-batch-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } From 3614454130fddf8e0b4f2fb4be0ce6e0b29a0bca Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 13 Apr 2026 14:05:33 +1200 Subject: [PATCH 20/48] Dedupe entry and leaf counting --- .../large-smt-backend-rocksdb/src/helpers.rs | 22 ++++++++ .../large-smt-backend-rocksdb/src/rocksdb.rs | 55 +++++-------------- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/helpers.rs b/crates/large-smt-backend-rocksdb/src/helpers.rs index 23f3c8d88..582b56044 100644 --- a/crates/large-smt-backend-rocksdb/src/helpers.rs +++ b/crates/large-smt-backend-rocksdb/src/helpers.rs @@ -81,3 +81,25 @@ pub(crate) fn remove_from_leaf(leaf: &mut SmtLeaf, key: Word) -> (Option, }, } } + +#[expect(clippy::needless_pass_by_value, reason = "simplifies chaining")] +pub(crate) fn count_leaves(leaf_count_bytes: Vec) -> Result { + let arr: [u8; 8] = + leaf_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "leaf count", + expected: 8, + found: leaf_count_bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) +} + +#[expect(clippy::needless_pass_by_value, reason = "simplifies chaining")] +pub(crate) fn count_entries(entry_count_bytes: Vec) -> Result { + let arr: [u8; 8] = + entry_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "entry count", + expected: 8, + found: entry_count_bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) +} diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 3158fb56c..edfeba9fc 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -30,7 +30,13 @@ use super::{ StorageUpdates, SubtreeUpdate, }; -use crate::helpers::{insert_into_leaf, map_rocksdb_err, remove_from_leaf}; +use crate::helpers::{ + count_entries, + count_leaves, + insert_into_leaf, + map_rocksdb_err, + remove_from_leaf, +}; use crate::{EMPTY_WORD, Word}; const IN_MEMORY_DEPTH: u8 = 24; @@ -291,15 +297,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, LEAF_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), |bytes| { - let arr: [u8; 8] = - bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "leaf count", - expected: 8, - found: bytes.len(), - })?; - Ok(usize::from_be_bytes(arr)) - }) + .map_or(Ok(0), count_leaves) } /// Retrieves the total count of key-value entries from the `METADATA_CF` column family. @@ -314,15 +312,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), |bytes| { - let arr: [u8; 8] = - bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "entry count", - expected: 8, - found: bytes.len(), - })?; - Ok(usize::from_be_bytes(arr)) - }) + .map_or(Ok(0), count_entries) } /// Retrieves a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. @@ -1065,18 +1055,11 @@ impl RocksDbSnapshotStorage { impl SmtStorageReader for RocksDbSnapshotStorage { fn leaf_count(&self) -> Result { let cf = self.cf_handle(METADATA_CF)?; - self.inner.snapshot.get_cf(cf, LEAF_COUNT_KEY).map_err(map_rocksdb_err)?.map_or( - Ok(0), - |bytes| { - let arr: [u8; 8] = - bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "leaf count", - expected: 8, - found: bytes.len(), - })?; - Ok(usize::from_be_bytes(arr)) - }, - ) + self.inner + .snapshot + .get_cf(cf, LEAF_COUNT_KEY) + .map_err(map_rocksdb_err)? + .map_or(Ok(0), count_leaves) } fn entry_count(&self) -> Result { @@ -1085,15 +1068,7 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .snapshot .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), |bytes| { - let arr: [u8; 8] = - bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "entry count", - expected: 8, - found: bytes.len(), - })?; - Ok(usize::from_be_bytes(arr)) - }) + .map_or(Ok(0), count_entries) } fn get_leaf(&self, index: u64) -> Result, StorageError> { From 98c7fc49167676de453cb20a9f2a6688c7e489c3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 13 Apr 2026 14:16:44 +1200 Subject: [PATCH 21/48] read_leaf and rename other helpers --- .../large-smt-backend-rocksdb/src/helpers.rs | 11 +++++-- .../large-smt-backend-rocksdb/src/rocksdb.rs | 33 ++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/helpers.rs b/crates/large-smt-backend-rocksdb/src/helpers.rs index 582b56044..e2cbf644a 100644 --- a/crates/large-smt-backend-rocksdb/src/helpers.rs +++ b/crates/large-smt-backend-rocksdb/src/helpers.rs @@ -1,4 +1,5 @@ use miden_crypto::merkle::smt::{MAX_LEAF_ENTRIES, SmtLeaf, SmtLeafError}; +use miden_crypto::utils::Deserializable; use miden_crypto::word::LexicographicWord; use rocksdb::Error as RocksDbError; @@ -83,7 +84,7 @@ pub(crate) fn remove_from_leaf(leaf: &mut SmtLeaf, key: Word) -> (Option, } #[expect(clippy::needless_pass_by_value, reason = "simplifies chaining")] -pub(crate) fn count_leaves(leaf_count_bytes: Vec) -> Result { +pub(crate) fn read_leaf_count(leaf_count_bytes: Vec) -> Result { let arr: [u8; 8] = leaf_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { what: "leaf count", @@ -94,7 +95,7 @@ pub(crate) fn count_leaves(leaf_count_bytes: Vec) -> Result) -> Result { +pub(crate) fn read_entry_count(entry_count_bytes: Vec) -> Result { let arr: [u8; 8] = entry_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { what: "entry count", @@ -103,3 +104,9 @@ pub(crate) fn count_entries(entry_count_bytes: Vec) -> Result) -> Result, StorageError> { + let leaf = SmtLeaf::read_from_bytes(&leaf_bytes)?; + Ok(Some(leaf)) +} diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index edfeba9fc..93d1a936b 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -31,10 +31,11 @@ use super::{ SubtreeUpdate, }; use crate::helpers::{ - count_entries, - count_leaves, insert_into_leaf, map_rocksdb_err, + read_entry_count, + read_leaf, + read_leaf_count, remove_from_leaf, }; use crate::{EMPTY_WORD, Word}; @@ -297,7 +298,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, LEAF_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), count_leaves) + .map_or(Ok(0), read_leaf_count) } /// Retrieves the total count of key-value entries from the `METADATA_CF` column family. @@ -312,7 +313,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), count_entries) + .map_or(Ok(0), read_entry_count) } /// Retrieves a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. @@ -323,13 +324,7 @@ impl SmtStorageReader for RocksDbStorage { fn get_leaf(&self, index: u64) -> Result, StorageError> { let cf = self.cf_handle(LEAVES_CF)?; let key = Self::index_db_key(index); - match self.db.get_cf(cf, key).map_err(map_rocksdb_err)? { - Some(bytes) => { - let leaf = SmtLeaf::read_from_bytes(&bytes)?; - Ok(Some(leaf)) - }, - None => Ok(None), - } + self.db.get_cf(cf, key).map_err(map_rocksdb_err)?.map_or(Ok(None), read_leaf) } /// Retrieves multiple SMT leaf nodes by their logical `indices` using RocksDB's `multi_get_cf`. @@ -1059,7 +1054,7 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .snapshot .get_cf(cf, LEAF_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), count_leaves) + .map_or(Ok(0), read_leaf_count) } fn entry_count(&self) -> Result { @@ -1068,19 +1063,17 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .snapshot .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), count_entries) + .map_or(Ok(0), read_entry_count) } fn get_leaf(&self, index: u64) -> Result, StorageError> { let cf = self.cf_handle(LEAVES_CF)?; let key = RocksDbStorage::index_db_key(index); - match self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)? { - Some(bytes) => { - let leaf = SmtLeaf::read_from_bytes(&bytes)?; - Ok(Some(leaf)) - }, - None => Ok(None), - } + self.inner + .snapshot + .get_cf(cf, key) + .map_err(map_rocksdb_err)? + .map_or(Ok(None), read_leaf) } fn get_leaves(&self, indices: &[u64]) -> Result>, StorageError> { From 259bc975115de69912d7f8b9d2c67564d38ae644 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 13 Apr 2026 15:12:17 +1200 Subject: [PATCH 22/48] Add read_leaves --- .../large-smt-backend-rocksdb/src/helpers.rs | 12 ++++++++++ .../large-smt-backend-rocksdb/src/rocksdb.rs | 23 ++++++++----------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/helpers.rs b/crates/large-smt-backend-rocksdb/src/helpers.rs index e2cbf644a..9b86c9432 100644 --- a/crates/large-smt-backend-rocksdb/src/helpers.rs +++ b/crates/large-smt-backend-rocksdb/src/helpers.rs @@ -110,3 +110,15 @@ pub(crate) fn read_leaf(leaf_bytes: Vec) -> Result, StorageE let leaf = SmtLeaf::read_from_bytes(&leaf_bytes)?; Ok(Some(leaf)) } + +pub(crate) fn read_leaves( + leaves: Vec>>, +) -> Result>, StorageError> { + leaves + .into_iter() + .map(|leaf| match leaf { + Some(bytes) => Ok(Some(SmtLeaf::read_from_bytes(&bytes)?)), + None => Ok(None), + }) + .collect() +} diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 93d1a936b..536a02abb 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -36,6 +36,7 @@ use crate::helpers::{ read_entry_count, read_leaf, read_leaf_count, + read_leaves, remove_from_leaf, }; use crate::{EMPTY_WORD, Word}; @@ -337,14 +338,11 @@ impl SmtStorageReader for RocksDbStorage { let db_keys: Vec<[u8; 8]> = indices.iter().map(|&idx| Self::index_db_key(idx)).collect(); let results = self.db.multi_get_cf(db_keys.iter().map(|k| (cf, k.as_ref()))); - results + let leaves = results .into_iter() - .map(|result| match result { - Ok(Some(bytes)) => Ok(Some(SmtLeaf::read_from_bytes(&bytes)?)), - Ok(None) => Ok(None), - Err(e) => Err(map_rocksdb_err(e)), - }) - .collect() + .collect::>>, rocksdb::Error>>() + .map_err(map_rocksdb_err)?; + read_leaves(leaves) } /// Returns true if the storage has any leaves. @@ -1082,14 +1080,11 @@ impl SmtStorageReader for RocksDbSnapshotStorage { indices.iter().map(|&idx| RocksDbStorage::index_db_key(idx)).collect(); let results = self.inner.snapshot.multi_get_cf(db_keys.iter().map(|k| (cf, k.as_ref()))); - results + let leaves = results .into_iter() - .map(|result| match result { - Ok(Some(bytes)) => Ok(Some(SmtLeaf::read_from_bytes(&bytes)?)), - Ok(None) => Ok(None), - Err(e) => Err(map_rocksdb_err(e)), - }) - .collect() + .collect::>>, rocksdb::Error>>() + .map_err(map_rocksdb_err)?; + read_leaves(leaves) } fn has_leaves(&self) -> Result { From 73643b59551ab15cec57e11d550ecf7c0dbb7be4 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 13 Apr 2026 20:29:59 +1200 Subject: [PATCH 23/48] ManuallyDrop snapshot --- .../large-smt-backend-rocksdb/src/rocksdb.rs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 536a02abb..e5085c94d 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -1,5 +1,6 @@ use alloc::boxed::Box; use alloc::vec::Vec; +use std::mem::ManuallyDrop; use std::path::PathBuf; use std::sync::Arc; @@ -985,16 +986,22 @@ impl SmtStorage for RocksDbStorage { /// /// # Safety /// -/// `snapshot` borrows from `db`. Fields are dropped in declaration order in Rust, -/// so `snapshot` is dropped before `db`'s refcount is decremented. The `Arc` -/// ensures the `DB` lives at least as long as any `SnapshotInner`. +/// `snapshot` borrows from `db` so we must ensure that `snapshot` is dropped before `db`'s refcount +/// is decremented. The `Arc` ensures the `DB` lives at least as long as any `SnapshotInner`. struct SnapshotInner { - // IMPORTANT: field order matters for drop order. - // `snapshot` must be declared before `db` so it is dropped first. - snapshot: rocksdb::Snapshot<'static>, // actually borrows from `db` + snapshot: ManuallyDrop>, db: Arc, } +impl Drop for SnapshotInner { + fn drop(&mut self) { + // Ensure that the snapshot is dropped before the database reference count is decremented. + unsafe { + ManuallyDrop::drop(&mut self.snapshot); + } + } +} + /// A read-only, `Clone`-able RocksDB storage that reads from a point-in-time snapshot. /// /// All clones share the same snapshot via `Arc`, providing a consistent view of @@ -1027,7 +1034,10 @@ impl RocksDbSnapshotStorage { let snapshot = db_ref.snapshot(); let snapshot: rocksdb::Snapshot<'static> = unsafe { std::mem::transmute(snapshot) }; Self { - inner: Arc::new(SnapshotInner { snapshot, db }), + inner: Arc::new(SnapshotInner { + snapshot: ManuallyDrop::new(snapshot), + db, + }), } } From c62d577b36e4181e9989a28f9d8bf3acf4473d2a Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 08:41:08 +1200 Subject: [PATCH 24/48] RM as_ptr and clean up comments --- .../large-smt-backend-rocksdb/src/rocksdb.rs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index e5085c94d..6bb498e4e 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -984,10 +984,14 @@ impl SmtStorage for RocksDbStorage { /// Inner state shared by all clones of a snapshot storage. /// +/// This struct pairs a RocksDB snapshot with an `Arc` to ensure the database +/// lives for as long as the snapshot that references it. This allows for long-lived snapshots. +/// /// # Safety /// -/// `snapshot` borrows from `db` so we must ensure that `snapshot` is dropped before `db`'s refcount -/// is decremented. The `Arc` ensures the `DB` lives at least as long as any `SnapshotInner`. +/// `snapshot` borrows from `db`, so `snapshot` must be dropped before `db`'s refcount +/// is decremented. This is enforced by the `Drop` impl, which manually drops the +/// snapshot before the compiler auto-drops the `Arc`. struct SnapshotInner { snapshot: ManuallyDrop>, db: Arc, @@ -1021,17 +1025,10 @@ impl std::fmt::Debug for RocksDbSnapshotStorage { impl RocksDbSnapshotStorage { /// Creates a new snapshot storage from the given database. - /// - /// # Safety - /// - /// We use `Arc::as_ptr` to get a reference without borrowing `db`, allowing `db` - /// to be moved into `SnapshotInner`. The `Arc` stored alongside the snapshot - /// guarantees the DB outlives the snapshot (snapshot field is dropped before db field - /// due to declaration order). pub fn new(db: Arc) -> Self { - // SAFETY: See struct-level safety documentation. - let db_ref: &DB = unsafe { &*Arc::as_ptr(&db) }; - let snapshot = db_ref.snapshot(); + // SAFETY: We can transmute the snapshot to a static lifetime because we know that + // the database will outlive the snapshot. + let snapshot = db.snapshot(); let snapshot: rocksdb::Snapshot<'static> = unsafe { std::mem::transmute(snapshot) }; Self { inner: Arc::new(SnapshotInner { From 38e1705806dcc91e08f18402e4f87980234ce0e3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 08:55:44 +1200 Subject: [PATCH 25/48] Dedupe more with helpers --- .../large-smt-backend-rocksdb/src/helpers.rs | 115 +++++++++--- .../large-smt-backend-rocksdb/src/rocksdb.rs | 169 +++--------------- 2 files changed, 124 insertions(+), 160 deletions(-) diff --git a/crates/large-smt-backend-rocksdb/src/helpers.rs b/crates/large-smt-backend-rocksdb/src/helpers.rs index 9b86c9432..634232344 100644 --- a/crates/large-smt-backend-rocksdb/src/helpers.rs +++ b/crates/large-smt-backend-rocksdb/src/helpers.rs @@ -1,4 +1,8 @@ -use miden_crypto::merkle::smt::{MAX_LEAF_ENTRIES, SmtLeaf, SmtLeafError}; +use alloc::boxed::Box; +use alloc::vec::Vec; + +use miden_crypto::merkle::NodeIndex; +use miden_crypto::merkle::smt::{MAX_LEAF_ENTRIES, SmtLeaf, SmtLeafError, Subtree}; use miden_crypto::utils::Deserializable; use miden_crypto::word::LexicographicWord; use rocksdb::Error as RocksDbError; @@ -83,25 +87,12 @@ pub(crate) fn remove_from_leaf(leaf: &mut SmtLeaf, key: Word) -> (Option, } } -#[expect(clippy::needless_pass_by_value, reason = "simplifies chaining")] -pub(crate) fn read_leaf_count(leaf_count_bytes: Vec) -> Result { - let arr: [u8; 8] = - leaf_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "leaf count", - expected: 8, - found: leaf_count_bytes.len(), - })?; - Ok(usize::from_be_bytes(arr)) -} - -#[expect(clippy::needless_pass_by_value, reason = "simplifies chaining")] -pub(crate) fn read_entry_count(entry_count_bytes: Vec) -> Result { - let arr: [u8; 8] = - entry_count_bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { - what: "entry count", - expected: 8, - found: entry_count_bytes.len(), - })?; +pub(crate) fn read_count(what: &'static str, bytes: &[u8]) -> Result { + let arr: [u8; 8] = bytes.try_into().map_err(|_| StorageError::BadValueLen { + what, + expected: 8, + found: bytes.len(), + })?; Ok(usize::from_be_bytes(arr)) } @@ -122,3 +113,87 @@ pub(crate) fn read_leaves( }) .collect() } + +pub(crate) fn read_subtree( + index: NodeIndex, + db_result: Option>, +) -> Result, StorageError> { + match db_result { + Some(bytes) => { + let subtree = Subtree::from_vec(index, &bytes)?; + Ok(Some(subtree)) + }, + None => Ok(None), + } +} + +/// Deserializes a batch of raw `multi_get` results into subtrees, preserving the original +/// indices for reassembly. +pub(crate) fn read_subtree_batch( + bucket: Vec<(usize, NodeIndex)>, + db_results: Vec>, RocksDbError>>, +) -> Result)>, StorageError> { + bucket + .into_iter() + .zip(db_results) + .map(|((original_index, node_index), db_result)| { + let subtree = match db_result { + Ok(Some(bytes)) => Some(Subtree::from_vec(node_index, &bytes)?), + Ok(None) => None, + Err(e) => return Err(map_rocksdb_err(e)), + }; + Ok((original_index, subtree)) + }) + .collect() +} + +/// Buckets subtree node indices by depth for batched column family lookups. +/// +/// Returns an array of 5 buckets (for depths 56, 48, 40, 32, 24), where each bucket +/// contains `(original_index, NodeIndex)` pairs. +pub(crate) fn bucket_by_depth( + indices: &[NodeIndex], +) -> Result<[Vec<(usize, NodeIndex)>; 5], StorageError> { + let mut depth_buckets: [Vec<(usize, NodeIndex)>; 5] = Default::default(); + + for (original_index, &node_index) in indices.iter().enumerate() { + let depth = node_index.depth(); + let bucket_index = match depth { + 56 => 0, + 48 => 1, + 40 => 2, + 32 => 3, + 24 => 4, + _ => { + return Err(StorageError::Unsupported(format!( + "unsupported subtree depth {depth}" + ))); + }, + }; + depth_buckets[bucket_index].push((original_index, node_index)); + } + + Ok(depth_buckets) +} + +pub(crate) fn read_depth24_entries( + iter: impl Iterator, Box<[u8]>), RocksDbError>>, +) -> Result, StorageError> { + let mut hashes = Vec::new(); + for item in iter { + let (key_bytes, value_bytes) = item.map_err(map_rocksdb_err)?; + let index = index_from_key_bytes(&key_bytes)?; + let hash = Word::read_from_bytes(&value_bytes)?; + hashes.push((index, hash)); + } + Ok(hashes) +} + +pub(crate) fn index_from_key_bytes(key_bytes: &[u8]) -> Result { + if key_bytes.len() != 8 { + return Err(StorageError::BadKeyLen { expected: 8, found: key_bytes.len() }); + } + let mut arr = [0u8; 8]; + arr.copy_from_slice(key_bytes); + Ok(u64::from_be_bytes(arr)) +} diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 6bb498e4e..2a22d583f 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -32,12 +32,16 @@ use super::{ SubtreeUpdate, }; use crate::helpers::{ + bucket_by_depth, + index_from_key_bytes, insert_into_leaf, map_rocksdb_err, - read_entry_count, + read_count, + read_depth24_entries, read_leaf, - read_leaf_count, read_leaves, + read_subtree, + read_subtree_batch, remove_from_leaf, }; use crate::{EMPTY_WORD, Word}; @@ -300,7 +304,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, LEAF_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), read_leaf_count) + .map_or(Ok(0), |b| read_count("leaf count", &b)) } /// Retrieves the total count of key-value entries from the `METADATA_CF` column family. @@ -315,7 +319,7 @@ impl SmtStorageReader for RocksDbStorage { self.db .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), read_entry_count) + .map_or(Ok(0), |b| read_count("entry count", &b)) } /// Retrieves a single SMT leaf node by its logical `index` from the `LEAVES_CF` column family. @@ -375,53 +379,15 @@ impl SmtStorageReader for RocksDbStorage { fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { let cf = self.subtree_cf(index); let key = Self::subtree_db_key(index); - match self.db.get_cf(cf, key).map_err(map_rocksdb_err)? { - Some(bytes) => { - let subtree = Subtree::from_vec(index, &bytes)?; - Ok(Some(subtree)) - }, - None => Ok(None), - } + read_subtree(index, self.db.get_cf(cf, key).map_err(map_rocksdb_err)?) } - /// Batch-retrieves multiple subtrees from RocksDB by their node indices. - /// - /// This method groups requests by subtree depth into column family buckets, - /// then performs parallel `multi_get` operations to efficiently retrieve - /// all subtrees. Results are deserialized and placed in the same order as - /// the input indices. - /// - /// # Parameters - /// - `indices`: A slice of subtree root indices to retrieve. - /// - /// # Returns - /// - A `Vec>` where each index corresponds to the original input. - /// - `Ok(...)` if all fetches succeed. - /// - `Err(StorageError)` if any RocksDB access or deserialization fails. fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { use rayon::prelude::*; - let mut depth_buckets: [Vec<(usize, NodeIndex)>; 5] = Default::default(); - - for (original_index, &node_index) in indices.iter().enumerate() { - let depth = node_index.depth(); - let bucket_index = match depth { - 56 => 0, - 48 => 1, - 40 => 2, - 32 => 3, - 24 => 4, - _ => { - return Err(StorageError::Unsupported(format!( - "unsupported subtree depth {depth}" - ))); - }, - }; - depth_buckets[bucket_index].push((original_index, node_index)); - } + let depth_buckets = bucket_by_depth(indices)?; let mut results = vec![None; indices.len()]; - // Process depth buckets in parallel let bucket_results: Result, StorageError> = depth_buckets .into_par_iter() .enumerate() @@ -433,26 +399,17 @@ impl SmtStorageReader for RocksDbStorage { let keys: Vec<_> = bucket.iter().map(|(_, idx)| Self::subtree_db_key(*idx)).collect(); - let db_results = self.db.multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))); - - // Process results for this bucket - bucket + let db_results: Vec<_> = self + .db + .multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))) .into_iter() - .zip(db_results) - .map(|((original_index, node_index), db_result)| { - let subtree = match db_result { - Ok(Some(bytes)) => Some(Subtree::from_vec(node_index, &bytes)?), - Ok(None) => None, - Err(e) => return Err(map_rocksdb_err(e)), - }; - Ok((original_index, subtree)) - }) - .collect() + .collect(); + + read_subtree_batch(bucket, db_results) }, ) .collect(); - // Flatten results and place them in correct positions for bucket_result in bucket_results? { for (original_index, subtree) in bucket_result { results[original_index] = subtree; @@ -532,19 +489,7 @@ impl SmtStorageReader for RocksDbStorage { /// - `StorageError::Value`: If any hash bytes are corrupt. fn get_depth24(&self) -> Result, StorageError> { let cf = self.cf_handle(DEPTH_24_CF)?; - let iter = self.db.iterator_cf(cf, IteratorMode::Start); - let mut hashes = Vec::new(); - - for item in iter { - let (key_bytes, value_bytes) = item.map_err(map_rocksdb_err)?; - - let index = index_from_key_bytes(&key_bytes)?; - let hash = Word::read_from_bytes(&value_bytes)?; - - hashes.push((index, hash)); - } - - Ok(hashes) + read_depth24_entries(self.db.iterator_cf(cf, IteratorMode::Start)) } } @@ -1059,7 +1004,7 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .snapshot .get_cf(cf, LEAF_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), read_leaf_count) + .map_or(Ok(0), |b| read_count("leaf count", &b)) } fn entry_count(&self) -> Result { @@ -1068,7 +1013,7 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .snapshot .get_cf(cf, ENTRY_COUNT_KEY) .map_err(map_rocksdb_err)? - .map_or(Ok(0), read_entry_count) + .map_or(Ok(0), |b| read_count("entry count", &b)) } fn get_leaf(&self, index: u64) -> Result, StorageError> { @@ -1101,36 +1046,13 @@ impl SmtStorageReader for RocksDbSnapshotStorage { fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { let cf = self.subtree_cf(index); let key = RocksDbStorage::subtree_db_key(index); - match self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)? { - Some(bytes) => { - let subtree = Subtree::from_vec(index, &bytes)?; - Ok(Some(subtree)) - }, - None => Ok(None), - } + read_subtree(index, self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)?) } fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { use rayon::prelude::*; - let mut depth_buckets: [Vec<(usize, NodeIndex)>; 5] = Default::default(); - - for (original_index, &node_index) in indices.iter().enumerate() { - let depth = node_index.depth(); - let bucket_index = match depth { - 56 => 0, - 48 => 1, - 40 => 2, - 32 => 3, - 24 => 4, - _ => { - return Err(StorageError::Unsupported(format!( - "unsupported subtree depth {depth}" - ))); - }, - }; - depth_buckets[bucket_index].push((original_index, node_index)); - } + let depth_buckets = bucket_by_depth(indices)?; let mut results = vec![None; indices.len()]; let bucket_results: Result, StorageError> = depth_buckets @@ -1146,21 +1068,14 @@ impl SmtStorageReader for RocksDbSnapshotStorage { .map(|(_, idx)| RocksDbStorage::subtree_db_key(*idx)) .collect(); - let db_results = - self.inner.snapshot.multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))); - - bucket + let db_results: Vec<_> = self + .inner + .snapshot + .multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))) .into_iter() - .zip(db_results) - .map(|((original_index, node_index), db_result)| { - let subtree = match db_result { - Ok(Some(bytes)) => Some(Subtree::from_vec(node_index, &bytes)?), - Ok(None) => None, - Err(e) => return Err(map_rocksdb_err(e)), - }; - Ok((original_index, subtree)) - }) - .collect() + .collect(); + + read_subtree_batch(bucket, db_results) }, ) .collect(); @@ -1209,19 +1124,7 @@ impl SmtStorageReader for RocksDbSnapshotStorage { fn get_depth24(&self) -> Result, StorageError> { let cf = self.cf_handle(DEPTH_24_CF)?; - let db_iter = self.inner.snapshot.iterator_cf(cf, IteratorMode::Start); - let mut hashes = Vec::new(); - - for item in db_iter { - let (key_bytes, value_bytes) = item.map_err(map_rocksdb_err)?; - - let index = index_from_key_bytes(&key_bytes)?; - let hash = Word::read_from_bytes(&value_bytes)?; - - hashes.push((index, hash)); - } - - Ok(hashes) + read_depth24_entries(self.inner.snapshot.iterator_cf(cf, IteratorMode::Start)) } } @@ -1475,20 +1378,6 @@ impl AsRef<[u8]> for KeyBytes { // HELPERS // -------------------------------------------------------------------------------------------- -/// Deserializes an index (u64) from a RocksDB key byte slice. -/// Expects `key_bytes` to be exactly 8 bytes long. -/// -/// # Errors -/// - `StorageError::BadKeyLen`: If `key_bytes` is not 8 bytes long or conversion fails. -fn index_from_key_bytes(key_bytes: &[u8]) -> Result { - if key_bytes.len() != 8 { - return Err(StorageError::BadKeyLen { expected: 8, found: key_bytes.len() }); - } - let mut arr = [0u8; 8]; - arr.copy_from_slice(key_bytes); - Ok(u64::from_be_bytes(arr)) -} - /// Reconstructs a `NodeIndex` from the variable-length subtree key stored in `RocksDB`. /// /// * `key_bytes` is the big-endian tail of the 64-bit value: From c658b838a226ec96a8ac8b3947a4a5e61e6414d5 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 08:59:56 +1200 Subject: [PATCH 26/48] Comments --- crates/large-smt-backend-rocksdb/src/helpers.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/large-smt-backend-rocksdb/src/helpers.rs b/crates/large-smt-backend-rocksdb/src/helpers.rs index 634232344..d7057265b 100644 --- a/crates/large-smt-backend-rocksdb/src/helpers.rs +++ b/crates/large-smt-backend-rocksdb/src/helpers.rs @@ -87,6 +87,8 @@ pub(crate) fn remove_from_leaf(leaf: &mut SmtLeaf, key: Word) -> (Option, } } +/// Deserializes a big-endian `usize` count from exactly 8 bytes. `what` is used in the error +/// message if the length is wrong. pub(crate) fn read_count(what: &'static str, bytes: &[u8]) -> Result { let arr: [u8; 8] = bytes.try_into().map_err(|_| StorageError::BadValueLen { what, @@ -96,12 +98,14 @@ pub(crate) fn read_count(what: &'static str, bytes: &[u8]) -> Result) -> Result, StorageError> { let leaf = SmtLeaf::read_from_bytes(&leaf_bytes)?; Ok(Some(leaf)) } +/// Deserializes a batch of optional raw byte vectors into optional SMT leaves. pub(crate) fn read_leaves( leaves: Vec>>, ) -> Result>, StorageError> { @@ -114,6 +118,7 @@ pub(crate) fn read_leaves( .collect() } +/// Deserializes a single subtree from an optional raw byte vector. pub(crate) fn read_subtree( index: NodeIndex, db_result: Option>, @@ -176,6 +181,7 @@ pub(crate) fn bucket_by_depth( Ok(depth_buckets) } +/// Deserializes depth-24 hash entries from a database iterator into `(index, Word)` pairs. pub(crate) fn read_depth24_entries( iter: impl Iterator, Box<[u8]>), RocksDbError>>, ) -> Result, StorageError> { @@ -189,6 +195,7 @@ pub(crate) fn read_depth24_entries( Ok(hashes) } +/// Deserializes a `u64` index from an 8-byte big-endian key. pub(crate) fn index_from_key_bytes(key_bytes: &[u8]) -> Result { if key_bytes.len() != 8 { return Err(StorageError::BadKeyLen { expected: 8, found: key_bytes.len() }); From 163ccfe08e9326aa1782bc560b6f5e328e8d9dae Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 09:03:54 +1200 Subject: [PATCH 27/48] Update deps --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3947ed13..4b20cb5f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2845,7 +2845,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "alloy-sol-types", "fs-err", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" dependencies = [ "blake3", "cc", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" dependencies = [ "quote", "syn 2.0.117", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "bech32", "fs-err", @@ -3566,7 +3566,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "proc-macro2", "quote", @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#ded92df7616b724d1c4f36fed968ee341d602fda" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" dependencies = [ "p3-field", "p3-goldilocks", @@ -3661,7 +3661,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "fs-err", "miden-assembly", @@ -3678,7 +3678,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3700,7 +3700,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "miden-processor", "miden-protocol", @@ -3713,7 +3713,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b6dcab32565289d5772b45230f0df1118d16a9db" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" dependencies = [ "miden-protocol", "miden-tx", From bdb815635dc6e2c588c03a325578682d83282694 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 09:10:33 +1200 Subject: [PATCH 28/48] Reorder AccountTreeWithHistory fns --- crates/store/src/accounts/mod.rs | 35 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index d0f491309..3a2d2be60 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -149,6 +149,17 @@ impl AccountTreeWithHistory { } } + /// Creates an `AccountTreeWithHistory` from its constituent parts. + /// + /// This is used by the writer to construct a snapshot-backed read-only copy. + pub(crate) fn from_parts( + latest: AccountTree>, + block_number: BlockNumber, + overlays: BTreeMap, + ) -> Self { + Self { block_number, latest, overlays } + } + // PUBLIC ACCESSORS // -------------------------------------------------------------------------------------------- @@ -226,6 +237,11 @@ impl AccountTreeWithHistory { self.latest.contains_account_id_prefix(prefix) } + /// Returns a reference to the historical overlays. + pub(crate) fn overlays(&self) -> &BTreeMap { + &self.overlays + } + // PRIVATE HELPERS - HISTORICAL RECONSTRUCTION // -------------------------------------------------------------------------------------------- @@ -339,25 +355,6 @@ impl AccountTreeWithHistory { let path = SparseMerklePath::try_from(path).ok()?; Some((path, leaf)) } - - // PUBLIC ACCESSORS (continued) - // -------------------------------------------------------------------------------------------- - - /// Returns a reference to the historical overlays. - pub(crate) fn overlays(&self) -> &BTreeMap { - &self.overlays - } - - /// Creates an `AccountTreeWithHistory` from its constituent parts. - /// - /// This is used by the writer to construct a snapshot-backed read-only copy. - pub(crate) fn from_parts( - latest: AccountTree>, - block_number: BlockNumber, - overlays: BTreeMap, - ) -> Self { - Self { block_number, latest, overlays } - } } impl AccountTreeWithHistory { From 95aa192ba9d075fb64f69b814a91c07b3e7a39be Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 09:17:39 +1200 Subject: [PATCH 29/48] Split rocksdb.rs --- crates/large-smt-backend-rocksdb/src/lib.rs | 5 +- .../large-smt-backend-rocksdb/src/rocksdb.rs | 250 ++---------------- .../src/rocksdb_snapshot.rs | 243 +++++++++++++++++ 3 files changed, 270 insertions(+), 228 deletions(-) create mode 100644 crates/large-smt-backend-rocksdb/src/rocksdb_snapshot.rs diff --git a/crates/large-smt-backend-rocksdb/src/lib.rs b/crates/large-smt-backend-rocksdb/src/lib.rs index 0da257ca3..7a97183b7 100644 --- a/crates/large-smt-backend-rocksdb/src/lib.rs +++ b/crates/large-smt-backend-rocksdb/src/lib.rs @@ -26,6 +26,8 @@ extern crate alloc; mod helpers; #[expect(clippy::doc_markdown, clippy::inline_always)] mod rocksdb; +#[expect(clippy::doc_markdown, clippy::inline_always)] +mod rocksdb_snapshot; // Re-export from miden-protocol. /// Re-export of `rocksdb::DB` for consumers that need the raw database handle type. pub use ::rocksdb::DB; @@ -59,4 +61,5 @@ pub use miden_protocol::{ merkle::{EmptySubtreeRoots, InnerNodeInfo, MerkleError, NodeIndex, SparseMerklePath}, }, }; -pub use rocksdb::{RocksDbConfig, RocksDbSnapshotStorage, RocksDbStorage}; +pub use rocksdb::{RocksDbConfig, RocksDbStorage}; +pub use rocksdb_snapshot::RocksDbSnapshotStorage; diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 2a22d583f..22a6f931b 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -1,6 +1,5 @@ use alloc::boxed::Box; use alloc::vec::Vec; -use std::mem::ManuallyDrop; use std::path::PathBuf; use std::sync::Arc; @@ -46,28 +45,28 @@ use crate::helpers::{ }; use crate::{EMPTY_WORD, Word}; -const IN_MEMORY_DEPTH: u8 = 24; +pub(crate) const IN_MEMORY_DEPTH: u8 = 24; /// The name of the `RocksDB` column family used for storing SMT leaves. -const LEAVES_CF: &str = "leaves"; +pub(crate) const LEAVES_CF: &str = "leaves"; /// The names of the `RocksDB` column families used for storing SMT subtrees (deep nodes). -const SUBTREE_24_CF: &str = "st24"; -const SUBTREE_32_CF: &str = "st32"; -const SUBTREE_40_CF: &str = "st40"; -const SUBTREE_48_CF: &str = "st48"; -const SUBTREE_56_CF: &str = "st56"; -const SUBTREE_DEPTHS: [u8; 5] = [56, 48, 40, 32, 24]; +pub(crate) const SUBTREE_24_CF: &str = "st24"; +pub(crate) const SUBTREE_32_CF: &str = "st32"; +pub(crate) const SUBTREE_40_CF: &str = "st40"; +pub(crate) const SUBTREE_48_CF: &str = "st48"; +pub(crate) const SUBTREE_56_CF: &str = "st56"; +pub(crate) const SUBTREE_DEPTHS: [u8; 5] = [56, 48, 40, 32, 24]; /// The name of the `RocksDB` column family used for storing metadata (e.g., root, counts). -const METADATA_CF: &str = "metadata"; +pub(crate) const METADATA_CF: &str = "metadata"; /// The name of the `RocksDB` column family used for storing level 24 hashes for fast tree /// rebuilding. -const DEPTH_24_CF: &str = "depth24"; +pub(crate) const DEPTH_24_CF: &str = "depth24"; /// The key used in the `METADATA_CF` column family to store the total count of non-empty leaves. -const LEAF_COUNT_KEY: &[u8] = b"leaf_count"; +pub(crate) const LEAF_COUNT_KEY: &[u8] = b"leaf_count"; /// The key used in the `METADATA_CF` column family to store the total count of key-value entries. -const ENTRY_COUNT_KEY: &[u8] = b"entry_count"; +pub(crate) const ENTRY_COUNT_KEY: &[u8] = b"entry_count"; /// A `RocksDB`-backed persistent storage implementation for a Sparse Merkle Tree (SMT). /// @@ -243,14 +242,14 @@ impl RocksDbStorage { /// Converts an index (u64) into a fixed-size byte array for use as a `RocksDB` key. #[inline(always)] - fn index_db_key(index: u64) -> [u8; 8] { + pub(crate) fn index_db_key(index: u64) -> [u8; 8] { index.to_be_bytes() } /// Converts a `NodeIndex` (for a subtree root) into a `KeyBytes` for use as a `RocksDB` key. /// The `KeyBytes` is a wrapper around a 8-byte value with a variable-length prefix. #[inline(always)] - fn subtree_db_key(index: NodeIndex) -> KeyBytes { + pub(crate) fn subtree_db_key(index: NodeIndex) -> KeyBytes { let keep = match index.depth() { 24 => 3, 32 => 4, @@ -285,9 +284,10 @@ impl RocksDbStorage { &self.db } - /// Creates a new [`RocksDbSnapshotStorage`] from this storage's database. - pub fn snapshot_storage(&self) -> RocksDbSnapshotStorage { - RocksDbSnapshotStorage::new(Arc::clone(&self.db)) + /// Creates a new [`crate::rocksdb_snapshot::RocksDbSnapshotStorage`] from this storage's + /// database. + pub fn snapshot_storage(&self) -> crate::rocksdb_snapshot::RocksDbSnapshotStorage { + crate::rocksdb_snapshot::RocksDbSnapshotStorage::new(Arc::clone(&self.db)) } } @@ -924,210 +924,6 @@ impl SmtStorage for RocksDbStorage { } } -// SNAPSHOT STORAGE -// -------------------------------------------------------------------------------------------- - -/// Inner state shared by all clones of a snapshot storage. -/// -/// This struct pairs a RocksDB snapshot with an `Arc` to ensure the database -/// lives for as long as the snapshot that references it. This allows for long-lived snapshots. -/// -/// # Safety -/// -/// `snapshot` borrows from `db`, so `snapshot` must be dropped before `db`'s refcount -/// is decremented. This is enforced by the `Drop` impl, which manually drops the -/// snapshot before the compiler auto-drops the `Arc`. -struct SnapshotInner { - snapshot: ManuallyDrop>, - db: Arc, -} - -impl Drop for SnapshotInner { - fn drop(&mut self) { - // Ensure that the snapshot is dropped before the database reference count is decremented. - unsafe { - ManuallyDrop::drop(&mut self.snapshot); - } - } -} - -/// A read-only, `Clone`-able RocksDB storage that reads from a point-in-time snapshot. -/// -/// All clones share the same snapshot via `Arc`, providing a consistent view of -/// the database at the time the snapshot was created. -/// -/// Implements [`SmtStorageReader`] only (read-only snapshot). -#[derive(Clone)] -pub struct RocksDbSnapshotStorage { - inner: Arc, -} - -impl std::fmt::Debug for RocksDbSnapshotStorage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RocksDbSnapshotStorage").finish_non_exhaustive() - } -} - -impl RocksDbSnapshotStorage { - /// Creates a new snapshot storage from the given database. - pub fn new(db: Arc) -> Self { - // SAFETY: We can transmute the snapshot to a static lifetime because we know that - // the database will outlive the snapshot. - let snapshot = db.snapshot(); - let snapshot: rocksdb::Snapshot<'static> = unsafe { std::mem::transmute(snapshot) }; - Self { - inner: Arc::new(SnapshotInner { - snapshot: ManuallyDrop::new(snapshot), - db, - }), - } - } - - fn cf_handle(&self, name: &str) -> Result<&rocksdb::ColumnFamily, StorageError> { - self.inner - .db - .cf_handle(name) - .ok_or_else(|| StorageError::Unsupported(format!("unknown column family `{name}`"))) - } - - #[inline(always)] - fn subtree_cf(&self, index: NodeIndex) -> &rocksdb::ColumnFamily { - let name = cf_for_depth(index.depth()); - self.cf_handle(name).expect("CF handle missing") - } -} - -impl SmtStorageReader for RocksDbSnapshotStorage { - fn leaf_count(&self) -> Result { - let cf = self.cf_handle(METADATA_CF)?; - self.inner - .snapshot - .get_cf(cf, LEAF_COUNT_KEY) - .map_err(map_rocksdb_err)? - .map_or(Ok(0), |b| read_count("leaf count", &b)) - } - - fn entry_count(&self) -> Result { - let cf = self.cf_handle(METADATA_CF)?; - self.inner - .snapshot - .get_cf(cf, ENTRY_COUNT_KEY) - .map_err(map_rocksdb_err)? - .map_or(Ok(0), |b| read_count("entry count", &b)) - } - - fn get_leaf(&self, index: u64) -> Result, StorageError> { - let cf = self.cf_handle(LEAVES_CF)?; - let key = RocksDbStorage::index_db_key(index); - self.inner - .snapshot - .get_cf(cf, key) - .map_err(map_rocksdb_err)? - .map_or(Ok(None), read_leaf) - } - - fn get_leaves(&self, indices: &[u64]) -> Result>, StorageError> { - let cf = self.cf_handle(LEAVES_CF)?; - let db_keys: Vec<[u8; 8]> = - indices.iter().map(|&idx| RocksDbStorage::index_db_key(idx)).collect(); - let results = self.inner.snapshot.multi_get_cf(db_keys.iter().map(|k| (cf, k.as_ref()))); - - let leaves = results - .into_iter() - .collect::>>, rocksdb::Error>>() - .map_err(map_rocksdb_err)?; - read_leaves(leaves) - } - - fn has_leaves(&self) -> Result { - Ok(self.leaf_count()? > 0) - } - - fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { - let cf = self.subtree_cf(index); - let key = RocksDbStorage::subtree_db_key(index); - read_subtree(index, self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)?) - } - - fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { - use rayon::prelude::*; - - let depth_buckets = bucket_by_depth(indices)?; - let mut results = vec![None; indices.len()]; - - let bucket_results: Result, StorageError> = depth_buckets - .into_par_iter() - .enumerate() - .filter(|(_, bucket)| !bucket.is_empty()) - .map( - |(bucket_index, bucket)| -> Result)>, StorageError> { - let depth = SUBTREE_DEPTHS[bucket_index]; - let cf = self.cf_handle(cf_for_depth(depth))?; - let keys: Vec<_> = bucket - .iter() - .map(|(_, idx)| RocksDbStorage::subtree_db_key(*idx)) - .collect(); - - let db_results: Vec<_> = self - .inner - .snapshot - .multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))) - .into_iter() - .collect(); - - read_subtree_batch(bucket, db_results) - }, - ) - .collect(); - - for bucket_result in bucket_results? { - for (original_index, subtree) in bucket_result { - results[original_index] = subtree; - } - } - - Ok(results) - } - - fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { - if index.depth() < IN_MEMORY_DEPTH { - return Err(StorageError::Unsupported( - "Cannot get inner node from upper part of the tree".into(), - )); - } - let subtree_root_index = Subtree::find_subtree_root(index); - Ok(self - .get_subtree(subtree_root_index)? - .and_then(|subtree| subtree.get_inner_node(index))) - } - - fn iter_leaves(&self) -> Result + '_>, StorageError> { - let cf = self.cf_handle(LEAVES_CF)?; - let mut read_opts = ReadOptions::default(); - read_opts.set_total_order_seek(true); - let db_iter = self.inner.snapshot.iterator_cf_opt(cf, read_opts, IteratorMode::Start); - - Ok(Box::new(RocksDbDirectLeafIterator { iter: db_iter })) - } - - fn iter_subtrees(&self) -> Result + '_>, StorageError> { - const SUBTREE_CFS: [&str; 5] = - [SUBTREE_24_CF, SUBTREE_32_CF, SUBTREE_40_CF, SUBTREE_48_CF, SUBTREE_56_CF]; - - let mut cf_handles = Vec::new(); - for cf_name in SUBTREE_CFS { - cf_handles.push(self.cf_handle(cf_name)?); - } - - Ok(Box::new(RocksDbSubtreeIterator::new(&self.inner.db, cf_handles))) - } - - fn get_depth24(&self) -> Result, StorageError> { - let cf = self.cf_handle(DEPTH_24_CF)?; - read_depth24_entries(self.inner.snapshot.iterator_cf(cf, IteratorMode::Start)) - } -} - /// Syncs the RocksDB database to disk before dropping the storage. /// /// This ensures that all data is persisted to disk before the storage is dropped. @@ -1150,8 +946,8 @@ impl Drop for RocksDbStorage { /// Wraps a `DBIteratorWithThreadMode` and handles deserialization of keys to `u64` (leaf index) /// and values to `SmtLeaf`. Skips items that fail to deserialize or if a RocksDB error occurs /// for an item, attempting to continue iteration. -struct RocksDbDirectLeafIterator<'a> { - iter: DBIteratorWithThreadMode<'a, DB>, +pub(crate) struct RocksDbDirectLeafIterator<'a> { + pub(crate) iter: DBIteratorWithThreadMode<'a, DB>, } impl Iterator for RocksDbDirectLeafIterator<'_> { @@ -1171,7 +967,7 @@ impl Iterator for RocksDbDirectLeafIterator<'_> { /// /// Iterates through all subtree column families (24, 32, 40, 48, 56) sequentially. /// When one column family is exhausted, it moves to the next one. -struct RocksDbSubtreeIterator<'a> { +pub(crate) struct RocksDbSubtreeIterator<'a> { db: &'a DB, cf_handles: Vec<&'a rocksdb::ColumnFamily>, current_cf_index: usize, @@ -1179,7 +975,7 @@ struct RocksDbSubtreeIterator<'a> { } impl<'a> RocksDbSubtreeIterator<'a> { - fn new(db: &'a DB, cf_handles: Vec<&'a rocksdb::ColumnFamily>) -> Self { + pub(crate) fn new(db: &'a DB, cf_handles: Vec<&'a rocksdb::ColumnFamily>) -> Self { let mut iterator = Self { db, cf_handles, @@ -1413,7 +1209,7 @@ fn subtree_root_from_key_bytes(key_bytes: &[u8], depth: u8) -> Result &'static str { +pub(crate) fn cf_for_depth(depth: u8) -> &'static str { match depth { 24 => SUBTREE_24_CF, 32 => SUBTREE_32_CF, diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb_snapshot.rs b/crates/large-smt-backend-rocksdb/src/rocksdb_snapshot.rs new file mode 100644 index 000000000..fc4e0e1b6 --- /dev/null +++ b/crates/large-smt-backend-rocksdb/src/rocksdb_snapshot.rs @@ -0,0 +1,243 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; +use std::mem::ManuallyDrop; +use std::sync::Arc; + +use miden_crypto::merkle::NodeIndex; +use miden_crypto::merkle::smt::{InnerNode, SmtLeaf, Subtree}; +use rocksdb::{DB, IteratorMode, ReadOptions}; + +use super::{SmtStorageReader, StorageError}; +use crate::Word; +use crate::helpers::{ + bucket_by_depth, + map_rocksdb_err, + read_count, + read_depth24_entries, + read_leaf, + read_leaves, + read_subtree, + read_subtree_batch, +}; +use crate::rocksdb::{ + DEPTH_24_CF, + ENTRY_COUNT_KEY, + IN_MEMORY_DEPTH, + LEAF_COUNT_KEY, + LEAVES_CF, + METADATA_CF, + RocksDbDirectLeafIterator, + RocksDbStorage, + RocksDbSubtreeIterator, + SUBTREE_24_CF, + SUBTREE_32_CF, + SUBTREE_40_CF, + SUBTREE_48_CF, + SUBTREE_56_CF, + SUBTREE_DEPTHS, + cf_for_depth, +}; + +// SNAPSHOT STORAGE +// -------------------------------------------------------------------------------------------- + +/// Inner state shared by all clones of a snapshot storage. +/// +/// This struct pairs a RocksDB snapshot with an `Arc` to ensure the database +/// lives for as long as the snapshot that references it. +/// +/// # Safety +/// +/// `snapshot` borrows from `db`, so `snapshot` must be dropped before `db`'s refcount +/// is decremented. This is enforced by the `Drop` impl, which manually drops the +/// snapshot before the compiler auto-drops the `Arc`. +struct SnapshotInner { + snapshot: ManuallyDrop>, + db: Arc, +} + +impl Drop for SnapshotInner { + fn drop(&mut self) { + // Ensure that the snapshot is dropped before the database reference count is decremented. + unsafe { + ManuallyDrop::drop(&mut self.snapshot); + } + } +} + +/// A read-only, `Clone`-able RocksDB storage that reads from a point-in-time snapshot. +/// +/// All clones share the same snapshot via `Arc`, providing a consistent view of +/// the database at the time the snapshot was created. +/// +/// Implements [`SmtStorageReader`] only (read-only snapshot). +#[derive(Clone)] +pub struct RocksDbSnapshotStorage { + inner: Arc, +} + +impl std::fmt::Debug for RocksDbSnapshotStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RocksDbSnapshotStorage").finish_non_exhaustive() + } +} + +impl RocksDbSnapshotStorage { + /// Creates a new snapshot storage from the given database. + pub fn new(db: Arc) -> Self { + // SAFETY: We can transmute the snapshot to a static lifetime because we know that + // the database will outlive the snapshot. + let snapshot = db.snapshot(); + let snapshot: rocksdb::Snapshot<'static> = unsafe { std::mem::transmute(snapshot) }; + Self { + inner: Arc::new(SnapshotInner { + snapshot: ManuallyDrop::new(snapshot), + db, + }), + } + } + + fn cf_handle(&self, name: &str) -> Result<&rocksdb::ColumnFamily, StorageError> { + self.inner + .db + .cf_handle(name) + .ok_or_else(|| StorageError::Unsupported(format!("unknown column family `{name}`"))) + } + + #[inline(always)] + fn subtree_cf(&self, index: NodeIndex) -> &rocksdb::ColumnFamily { + let name = cf_for_depth(index.depth()); + self.cf_handle(name).expect("CF handle missing") + } +} + +impl SmtStorageReader for RocksDbSnapshotStorage { + fn leaf_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.inner + .snapshot + .get_cf(cf, LEAF_COUNT_KEY) + .map_err(map_rocksdb_err)? + .map_or(Ok(0), |b| read_count("leaf count", &b)) + } + + fn entry_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.inner + .snapshot + .get_cf(cf, ENTRY_COUNT_KEY) + .map_err(map_rocksdb_err)? + .map_or(Ok(0), |b| read_count("entry count", &b)) + } + + fn get_leaf(&self, index: u64) -> Result, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let key = RocksDbStorage::index_db_key(index); + self.inner + .snapshot + .get_cf(cf, key) + .map_err(map_rocksdb_err)? + .map_or(Ok(None), read_leaf) + } + + fn get_leaves(&self, indices: &[u64]) -> Result>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let db_keys: Vec<[u8; 8]> = + indices.iter().map(|&idx| RocksDbStorage::index_db_key(idx)).collect(); + let results = self.inner.snapshot.multi_get_cf(db_keys.iter().map(|k| (cf, k.as_ref()))); + + let leaves = results + .into_iter() + .collect::>>, rocksdb::Error>>() + .map_err(map_rocksdb_err)?; + read_leaves(leaves) + } + + fn has_leaves(&self) -> Result { + Ok(self.leaf_count()? > 0) + } + + fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { + let cf = self.subtree_cf(index); + let key = RocksDbStorage::subtree_db_key(index); + read_subtree(index, self.inner.snapshot.get_cf(cf, key).map_err(map_rocksdb_err)?) + } + + fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { + use rayon::prelude::*; + + let depth_buckets = bucket_by_depth(indices)?; + let mut results = vec![None; indices.len()]; + + let bucket_results: Result, StorageError> = depth_buckets + .into_par_iter() + .enumerate() + .filter(|(_, bucket)| !bucket.is_empty()) + .map( + |(bucket_index, bucket)| -> Result)>, StorageError> { + let depth = SUBTREE_DEPTHS[bucket_index]; + let cf = self.cf_handle(cf_for_depth(depth))?; + let keys: Vec<_> = bucket + .iter() + .map(|(_, idx)| RocksDbStorage::subtree_db_key(*idx)) + .collect(); + + let db_results: Vec<_> = self + .inner + .snapshot + .multi_get_cf(keys.iter().map(|k| (cf, k.as_ref()))) + .into_iter() + .collect(); + + read_subtree_batch(bucket, db_results) + }, + ) + .collect(); + + for bucket_result in bucket_results? { + for (original_index, subtree) in bucket_result { + results[original_index] = subtree; + } + } + + Ok(results) + } + + fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { + if index.depth() < IN_MEMORY_DEPTH { + return Err(StorageError::Unsupported( + "Cannot get inner node from upper part of the tree".into(), + )); + } + let subtree_root_index = Subtree::find_subtree_root(index); + Ok(self + .get_subtree(subtree_root_index)? + .and_then(|subtree| subtree.get_inner_node(index))) + } + + fn iter_leaves(&self) -> Result + '_>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let mut read_opts = ReadOptions::default(); + read_opts.set_total_order_seek(true); + let db_iter = self.inner.snapshot.iterator_cf_opt(cf, read_opts, IteratorMode::Start); + + Ok(Box::new(RocksDbDirectLeafIterator { iter: db_iter })) + } + + fn iter_subtrees(&self) -> Result + '_>, StorageError> { + const SUBTREE_CFS: [&str; 5] = + [SUBTREE_24_CF, SUBTREE_32_CF, SUBTREE_40_CF, SUBTREE_48_CF, SUBTREE_56_CF]; + + let mut cf_handles = Vec::new(); + for cf_name in SUBTREE_CFS { + cf_handles.push(self.cf_handle(cf_name)?); + } + + Ok(Box::new(RocksDbSubtreeIterator::new(&self.inner.db, cf_handles))) + } + + fn get_depth24(&self) -> Result, StorageError> { + let cf = self.cf_handle(DEPTH_24_CF)?; + read_depth24_entries(self.inner.snapshot.iterator_cf(cf, IteratorMode::Start)) + } +} From 3de334034fec17ee8267d3aa59c527189af5a78b Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 09:48:10 +1200 Subject: [PATCH 30/48] Split apply_block_inner impl and move store calls --- crates/store/src/state/writer.rs | 384 +++++++++++++++++-------------- 1 file changed, 207 insertions(+), 177 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index c6c1bac0e..c3c010523 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -1,21 +1,23 @@ use std::sync::Arc; +use miden_large_smt_backend_rocksdb::RocksDbStorage; use miden_node_proto::BlockProofRequest; use miden_node_utils::ErrorReport; use miden_protocol::account::delta::AccountUpdateDetails; -use miden_protocol::block::SignedBlock; -use miden_protocol::block::nullifier_tree::NullifierTree; +use miden_protocol::block::account_tree::AccountMutationSet; +use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; +use miden_protocol::block::{BlockBody, BlockHeader, SignedBlock}; use miden_protocol::crypto::merkle::smt::LargeSmt; use miden_protocol::note::NoteDetails; use miden_protocol::transaction::OutputNote; use miden_protocol::utils::serde::Serializable; use tokio::sync::{mpsc, oneshot}; -use tracing::{Instrument, info, info_span, instrument}; +use tracing::{info, instrument}; use crate::accounts::AccountTreeWithHistory; use crate::db::NoteRecord; use crate::errors::{ApplyBlockError, InvalidBlockError}; -use crate::state::loader::TreeStorage; +use crate::state::loader::{SnapshotTreeStorage, TreeStorage}; use crate::state::{InMemoryState, State}; use crate::{COMPONENT, HistoricalError}; @@ -64,7 +66,6 @@ pub(crate) async fn writer_loop( /// /// No deep clone of tree data occurs in `RocksDB` mode. The snapshot-backed trees read directly /// from `RocksDB` snapshots. Readers pay only an atomic refcount bump per `snapshot()` call. -#[expect(clippy::too_many_lines)] #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( state: &State, @@ -75,8 +76,87 @@ async fn apply_block_inner( ) -> Result<(), ApplyBlockError> { let header = signed_block.header(); let body = signed_block.body(); + let block_num = header.block_num(); + let block_commitment = header.commitment(); + + validate_block_header(state, header, body).await?; + + // Load the current in-memory state snapshot for validation (wait-free). + let snapshot = state.in_memory.load_full(); + + // Compute mutations required for updating account and nullifier trees. + let (nullifier_tree_update, account_tree_update) = + compute_tree_mutations(state, &snapshot, header, body, nullifier_tree, account_tree)?; + + let notes = build_note_records(header, body)?; + + // Extract public account deltas before block is moved into the DB task. + let account_deltas = + Vec::from_iter(body.updated_accounts().iter().filter_map( + |update| match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + }, + )); + + // Apply mutations to the writable trees (writes to RocksDB). + nullifier_tree + .apply_mutations(nullifier_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + account_tree + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + // Build new read-only snapshot-backed trees for the new in-memory state. + let snapshot_nullifier_tree = build_snapshot_nullifier_tree(state, nullifier_tree); + let snapshot_account_tree = build_snapshot_account_tree(state, account_tree); + + let mut new_blockchain = snapshot.blockchain.clone(); + new_blockchain.push(block_commitment); + + let mut new_forest = snapshot.forest.clone(); + new_forest.apply_block_updates(block_num, account_deltas)?; + + let new_state = InMemoryState { + block_num, + nullifier_tree: snapshot_nullifier_tree, + account_tree: snapshot_account_tree, + blockchain: new_blockchain, + forest: new_forest, + }; + + // We have completed all in-memory mutations on the new clone of in-memory state. Now commit to + // storage before swapping the Arc. + + // Save the block to the block store. + let signed_block_bytes = signed_block.to_bytes(); + state.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while + // the DB commits. We ensure consistency by scoping all RPC queries that hit DB data by the + // block number that is Arc swapped at the end of this function. + state + .db + .apply_block(signed_block, notes, proving_inputs) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - // Validate that header and body match. + // Atomically publish the new state. Readers that call snapshot() after this point + // will see the updated state. Readers holding the old Arc continue unaffected. + state.in_memory.store(Arc::new(new_state)); + + info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); + + Ok(()) +} + +/// Validates that the block header is consistent with the body and follows the previous block. +async fn validate_block_header( + state: &State, + header: &BlockHeader, + body: &BlockBody, +) -> Result<(), ApplyBlockError> { let tx_commitment = body.transactions().commitment(); if header.tx_commitment() != tx_commitment { return Err(InvalidBlockError::InvalidBlockTxCommitment { @@ -87,9 +167,6 @@ async fn apply_block_inner( } let block_num = header.block_num(); - let block_commitment = header.commitment(); - - // Validate that the applied block is the next block in sequence. let prev_block = state .db .select_block_header_by_block_num(None) @@ -107,88 +184,90 @@ async fn apply_block_inner( return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); } - // Save the block to the block store concurrently. - // In a case of a rolled-back DB transaction, the in-memory state will be unchanged, but - // the block might still be written into the block store. Such blocks should be considered - // as candidates, not finalized blocks. - let signed_block_bytes = signed_block.to_bytes(); - let store = Arc::clone(&state.block_store); - let block_save_task = tokio::spawn( - async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), - ); + Ok(()) +} - // Load the current in-memory state snapshot for validation (wait-free). - let snapshot = state.in_memory.load_full(); +/// Compute mutations for the nullifier tree and account tree. +fn compute_tree_mutations( + state: &State, + snapshot: &Arc, + header: &BlockHeader, + body: &BlockBody, + nullifier_tree: &mut NullifierTree>, + account_tree: &mut AccountTreeWithHistory, +) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + // Nullifiers can be produced only once. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } - // Compute mutations required for updating account and nullifier trees. - let (nullifier_tree_update, account_tree_update) = { - let _span = info_span!(target: COMPONENT, "compute_tree_mutations").entered(); - - // Nullifiers can be produced only once. - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } + // new_block.chain_root must be equal to the chain MMR root prior to the update. + let peaks = snapshot.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } - // new_block.chain_root must be equal to the chain MMR root prior to the update. - let peaks = snapshot.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } + // Compute update for nullifier tree. + let nullifier_tree_update = nullifier_tree + .compute_mutations( + body.created_nullifiers() + .iter() + .map(|nullifier| (*nullifier, header.block_num())), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidNullifierRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } - // Compute update for nullifier tree. - let nullifier_tree_update = nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidNullifierRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } + // Compute update for account tree from the writable tree (always in sync with DB). + let account_tree_update = account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; - // Compute update for account tree from the writable tree (always in sync with DB). - let account_tree_update = account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; + if account_tree_update.as_mutation_set().root() != header.account_root() { + let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidAccountRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } - if account_tree_update.as_mutation_set().root() != header.account_root() { - let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidAccountRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } + Ok((nullifier_tree_update, account_tree_update)) +} - (nullifier_tree_update, account_tree_update) - }; +/// Builds the note tree, validates its root against the header, and collects note records. +fn build_note_records( + header: &BlockHeader, + body: &BlockBody, +) -> Result)>, ApplyBlockError> { + let block_num = header.block_num(); - // Build note tree. let note_tree = body.compute_block_note_tree(); if note_tree.root() != header.note_root() { return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); } - let notes = body - .output_notes() + body.output_notes() .map(|(note_index, note)| { let (details, nullifier) = match note { OutputNote::Public(note) => { @@ -211,105 +290,56 @@ async fn apply_block_inner( Ok((note_record, nullifier)) }) - .collect::, InvalidBlockError>>()?; - - // Extract public account deltas before block is moved into the DB task. - let account_deltas = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Delta(delta) => Some(delta.clone()), - AccountUpdateDetails::Private => None, - }, - )); - - // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while - // the DB commits. We ensure consistency by scoping all RPC queries that hit DB data by the - // block number that is Arc swapped at the end of this function. - state - .db - .apply_block(signed_block, notes, proving_inputs) - .await - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // Await the block store save task. - block_save_task.await??; - - // Apply mutations to the writable trees (writes to RocksDB). - nullifier_tree - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - // Build a new read-only InMemoryState with snapshot-backed trees. - // The snapshots capture the RocksDB state after mutations have been applied. - let snapshot_nullifier_tree = { - #[cfg(feature = "rocksdb")] - { - use miden_protocol::block::nullifier_tree::NullifierTree; - - use crate::state::loader; - - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&state.nullifier_db), - ); - let snapshot_smt = loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - NullifierTree::new_unchecked(snapshot_smt) - } - #[cfg(not(feature = "rocksdb"))] - { - nullifier_tree.clone() - } - }; - - let snapshot_account_tree = { - #[cfg(feature = "rocksdb")] - { - use crate::accounts::AccountTreeWithHistory; - use crate::state::loader; - - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&state.account_db), - ); - let snapshot_smt = loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - let snapshot_tree = - miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); - - AccountTreeWithHistory::from_parts( - snapshot_tree, - account_tree.block_number_latest(), - account_tree.overlays().clone(), - ) - } - #[cfg(not(feature = "rocksdb"))] - { - account_tree.clone() - } - }; - - let mut new_blockchain = snapshot.blockchain.clone(); - new_blockchain.push(block_commitment); - - let mut new_forest = snapshot.forest.clone(); - new_forest.apply_block_updates(block_num, account_deltas)?; - - let new_state = InMemoryState { - block_num, - nullifier_tree: snapshot_nullifier_tree, - account_tree: snapshot_account_tree, - blockchain: new_blockchain, - forest: new_forest, - }; - - // Atomically publish the new state. Readers that call snapshot() after this point - // will see the updated state. Readers holding the old Arc continue unaffected. - state.in_memory.store(Arc::new(new_state)); + .collect::, InvalidBlockError>>() + .map_err(Into::into) +} - info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); +/// Builds a snapshot-backed nullifier tree for the new in-memory state. +fn build_snapshot_nullifier_tree( + state: &State, + nullifier_tree: &NullifierTree>, +) -> NullifierTree> { + #[cfg(feature = "rocksdb")] + { + let _ = nullifier_tree; + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&state.nullifier_db), + ); + let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + NullifierTree::new_unchecked(snapshot_smt) + } + #[cfg(not(feature = "rocksdb"))] + { + let _ = state; + nullifier_tree.clone() + } +} - Ok(()) +/// Builds a snapshot-backed account tree for the new in-memory state. +fn build_snapshot_account_tree( + state: &State, + account_tree: &AccountTreeWithHistory, +) -> AccountTreeWithHistory { + #[cfg(feature = "rocksdb")] + { + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&state.account_db), + ); + let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + let snapshot_tree = + miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); + + AccountTreeWithHistory::from_parts( + snapshot_tree, + account_tree.block_number_latest(), + account_tree.overlays().clone(), + ) + } + #[cfg(not(feature = "rocksdb"))] + { + let _ = state; + account_tree.clone() + } } From 607b9928be6248d0ea9310c3aa96dc84d7920281 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 10:12:26 +1200 Subject: [PATCH 31/48] Revert bridge.mac --- .../samples/02-with-account-files/bridge.mac | Bin 31178 -> 31167 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac b/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac index f1e539b01e322f17b9ba6170038ee2307aa6c747..4eaf26f63c4b06e65cc268e1050241e3d1d15fec 100644 GIT binary patch delta 1303 zcmajeeNNGc5uLEt3ZCU7U9Ox9$;KvBT)wTmYYEcp=a zl)8sHq}4Ssp)wCRwIP1&xbPB&P7xTEf`J_niHRW`zv>11zxR*ddH#6c=lyz~)`xK6 zA=ICz;Dqr4RY5_8F>2Z^YMNkjV36Y*&Q5cu;wkjTJjYMwK$=0A=MYE+<_UcRoTk5+ zuvmHPO1U!RFZX6;#=hM>$0M5`DF@1}wfpnC_A5s`G=;Oy-4_{~g^zu_pAT&5JlfC^ z|17sz{4B=7x$Vk7ZNsGxLl)J}Ry@u>*;#|V5iIbiSV==W>WR0pj<^`DD_K~G5-|%> zG>b<-h4rf#$TdVV@EOV@Y4{$CA~`sZ9`CczgZqe{=orPoAS$A0$Ur@@73+vWXpQ1P zY}Bsq0r(i@QkEK>iN8o=!5uwhEG)u(G7bVTa18^|s3)4yx`u^*lti;2L~}F;p{R~w zpa|=UUm9XLFyf(D25z7|mZipAjG4@QfQ3fAd?84KHhMYC^Y6<2)N(&6vG6-fuyXIE zT_M$$_64Ius<*RCj|l|k+2kpUpD%05IlDKZw!~vcz_nt9RLq2%Dr zpj9}qmWEPvjbq>ls^eHVfwnjfTCgadCR-nhXTcvm5*Vn*&BP`&CGgaPdHB&f9XQ~` zy2-11t*1eZ5+aM*^*kKJ;6x7U(3Z%;PbgQ=lj#&JbfBHshnqJr)B`tc+;9n;P@lwr zA6k=mYQ=o~Cd3ynC?~u>T+v@<<1LR(gWcxPla`8toXD#k0{8Bk(PJl^E4||-X#0SM zGJ}eSbGVdv4dp5pys$;Zl?1v^)li@xW*yBLa2e|!+O{jK+$3^bF8lGxoww>=A8wd& z*RxXgTp)-DCsVI)%(9$~>N_G-_ zir?eIBYWB4+Op{J?yg)41m?-Tt&NA3alS3Ddgq1;Yx3)c-z@aB#V`vrd1J-P#Z_Nu z0#K36LI_%uYo!r?XdgA!_^dsBBi!xpWJ8zF(mj>;`dqDTr=D6Gs$ce+2?9YF$?17p z)pPl^8Fok4Ry`I3`}-W}3HvH2QEie|jr;oT-NB$)y-b?=v(vSwEx$Hal=lDNldcm6 zeLg-ff-HGLJ-F!|H(kUl zB1sVjr?7Aa>r-ebM;nQcqCAy@2DGP67H>ov3-{2J#=#O)XgCPN2@OL^Tar#gF>2Ep z;LuEB6Aln>qH6{NU!yvM1v}b^LZf_BA_W?(%cMzjScz^pk;y=$KHUmRw%4T6U2Ig?E9IZJtgyTdG0|!vDjfY{QtM)4S y6zt8T$q^m%S=fpCd>(pHQoxV~tuElnGIb#j6}VK#gT-jqX(_M&dmYqom;DPymKvx4 delta 1318 zcmajddrVVT7y$75-3d4W*>s{PD2Vlnu7GZC>MRw;g%xct4+}`GPF{vbi!#d{Xfg$X zxljP`{17xa zJI=u>0RRQKr%CtD04HK^CuRS2*-$x9eGJf_r$6@Z(_D$-`{*>v=8Mw~U7?sElM_5Di2pHW3$~ zC6b3wltuB7img!(As@|Oa*XaQ{K&GPU7%ZYRF70ba0Dq|Qhpn-TD zn}{)Ji4h<{%!=&?Xh2rM)8K{e3Kh&o?>HXjV|knaOHs0%BiA($d(g6+hhdbh;2{KC zR|v2i)k+R3(M&vuE~Nmc(WK(wHab*1J;p~|!}$V)iiY?Zpvd&!uW0bvs0>V(i}Gq$ zY+6m_jrms`*>47?o2#7{{SQ%8rw@6|+AY=`mG+2jFhVh|O1<~dwfk4^PgvISaj#w1 z?CCl-F%j(v4Ah}pA_pR>6M6U*ZHWS0MO_j@Qk#-^Sc2XwIp{zw@hmp4w9?chyd|5B zjp}%qh)(tB{U;?ekbp8`1!g5%;Q&fg1UQbi6dt;fO=U(yrt)wH9mL0|UB%IqFCJdi z4O7sN#=&B=q*)2=4w<`nXiBf|MM*_{(WN=>ikHG0YeEzkVkdFe;)nMHKiGZX?!cML zy7tqa+cV8U$1r&M2S0srmIkVGH1|3D#Cde)HtGozrr*&NTGhVK(=(@{UVqb6SnQ{$ z&tHVfwLGwBS-V%!TvYxv`I0=KadKPdMp^ono_(zJ)h_q2%MDvwZIzq$djdttNX#*- zdrjNkD=WEO_2TOI)UMyHcM~I2`l#p9yd?ERlQJ+3TfYudJS(R29^GtFFYu16aV38{ zt^4JQdll!x7agVg^E!1I_VMI|OG&hajX_Hm6s_Bwe%_KZ?%qN3KuJRTlD^H(KbJZp zW{pVil%aGT4;D19V_+xRNdG9Z=>nWYM>+@nxG;kU7dB@I3o#I#IUM|fvRnq9U>4C2 zjkz2YqMazBG|x)XxQO|9Dqny;)ap2Rip@G6rlLeIfE+D)2IA1E=fI4z0xOJ&ZX0@N z@Wj4ChHQRS$b%6L-&o-e3sDB3^={@?SUp*Zeebrv2v From 354f1cd80102964d74c88fac7c87f256d78f9bd3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 10:16:57 +1200 Subject: [PATCH 32/48] Remove unnecessary async --- .../samples/02-with-account-files/bridge.mac | Bin 31167 -> 31178 bytes crates/store/src/server/block_producer.rs | 2 +- crates/store/src/server/mod.rs | 7 +++---- crates/store/src/server/ntx_builder.rs | 10 +++++----- crates/store/src/server/rpc_api.rs | 18 +++++++++--------- crates/store/src/state/mod.rs | 7 +++---- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac b/crates/store/src/genesis/config/samples/02-with-account-files/bridge.mac index 4eaf26f63c4b06e65cc268e1050241e3d1d15fec..f1e539b01e322f17b9ba6170038ee2307aa6c747 100644 GIT binary patch delta 1318 zcmajddrVVT7y$75-3d4W*>s{PD2Vlnu7GZC>MRw;g%xct4+}`GPF{vbi!#d{Xfg$X zxljP`{17xa zJI=u>0RRQKr%CtD04HK^CuRS2*-$x9eGJf_r$6@Z(_D$-`{*>v=8Mw~U7?sElM_5Di2pHW3$~ zC6b3wltuB7img!(As@|Oa*XaQ{K&GPU7%ZYRF70ba0Dq|Qhpn-TD zn}{)Ji4h<{%!=&?Xh2rM)8K{e3Kh&o?>HXjV|knaOHs0%BiA($d(g6+hhdbh;2{KC zR|v2i)k+R3(M&vuE~Nmc(WK(wHab*1J;p~|!}$V)iiY?Zpvd&!uW0bvs0>V(i}Gq$ zY+6m_jrms`*>47?o2#7{{SQ%8rw@6|+AY=`mG+2jFhVh|O1<~dwfk4^PgvISaj#w1 z?CCl-F%j(v4Ah}pA_pR>6M6U*ZHWS0MO_j@Qk#-^Sc2XwIp{zw@hmp4w9?chyd|5B zjp}%qh)(tB{U;?ekbp8`1!g5%;Q&fg1UQbi6dt;fO=U(yrt)wH9mL0|UB%IqFCJdi z4O7sN#=&B=q*)2=4w<`nXiBf|MM*_{(WN=>ikHG0YeEzkVkdFe;)nMHKiGZX?!cML zy7tqa+cV8U$1r&M2S0srmIkVGH1|3D#Cde)HtGozrr*&NTGhVK(=(@{UVqb6SnQ{$ z&tHVfwLGwBS-V%!TvYxv`I0=KadKPdMp^ono_(zJ)h_q2%MDvwZIzq$djdttNX#*- zdrjNkD=WEO_2TOI)UMyHcM~I2`l#p9yd?ERlQJ+3TfYudJS(R29^GtFFYu16aV38{ zt^4JQdll!x7agVg^E!1I_VMI|OG&hajX_Hm6s_Bwe%_KZ?%qN3KuJRTlD^H(KbJZp zW{pVil%aGT4;D19V_+xRNdG9Z=>nWYM>+@nxG;kU7dB@I3o#I#IUM|fvRnq9U>4C2 zjkz2YqMazBG|x)XxQO|9Dqny;)ap2Rip@G6rlLeIfE+D)2IA1E=fI4z0xOJ&ZX0@N z@Wj4ChHQRS$b%6L-&o-e3sDB3^={@?SUp*Zeebrv2v delta 1303 zcmajeeNNGc5uLEt3ZCU7U9Ox9$;KvBT)wTmYYEcp=a zl)8sHq}4Ssp)wCRwIP1&xbPB&P7xTEf`J_niHRW`zv>11zxR*ddH#6c=lyz~)`xK6 zA=ICz;Dqr4RY5_8F>2Z^YMNkjV36Y*&Q5cu;wkjTJjYMwK$=0A=MYE+<_UcRoTk5+ zuvmHPO1U!RFZX6;#=hM>$0M5`DF@1}wfpnC_A5s`G=;Oy-4_{~g^zu_pAT&5JlfC^ z|17sz{4B=7x$Vk7ZNsGxLl)J}Ry@u>*;#|V5iIbiSV==W>WR0pj<^`DD_K~G5-|%> zG>b<-h4rf#$TdVV@EOV@Y4{$CA~`sZ9`CczgZqe{=orPoAS$A0$Ur@@73+vWXpQ1P zY}Bsq0r(i@QkEK>iN8o=!5uwhEG)u(G7bVTa18^|s3)4yx`u^*lti;2L~}F;p{R~w zpa|=UUm9XLFyf(D25z7|mZipAjG4@QfQ3fAd?84KHhMYC^Y6<2)N(&6vG6-fuyXIE zT_M$$_64Ius<*RCj|l|k+2kpUpD%05IlDKZw!~vcz_nt9RLq2%Dr zpj9}qmWEPvjbq>ls^eHVfwnjfTCgadCR-nhXTcvm5*Vn*&BP`&CGgaPdHB&f9XQ~` zy2-11t*1eZ5+aM*^*kKJ;6x7U(3Z%;PbgQ=lj#&JbfBHshnqJr)B`tc+;9n;P@lwr zA6k=mYQ=o~Cd3ynC?~u>T+v@<<1LR(gWcxPla`8toXD#k0{8Bk(PJl^E4||-X#0SM zGJ}eSbGVdv4dp5pys$;Zl?1v^)li@xW*yBLa2e|!+O{jK+$3^bF8lGxoww>=A8wd& z*RxXgTp)-DCsVI)%(9$~>N_G-_ zir?eIBYWB4+Op{J?yg)41m?-Tt&NA3alS3Ddgq1;Yx3)c-z@aB#V`vrd1J-P#Z_Nu z0#K36LI_%uYo!r?XdgA!_^dsBBi!xpWJ8zF(mj>;`dqDTr=D6Gs$ce+2?9YF$?17p z)pPl^8Fok4Ry`I3`}-W}3HvH2QEie|jr;oT-NB$)y-b?=v(vSwEx$Hal=lDNldcm6 zeLg-ff-HGLJ-F!|H(kUl zB1sVjr?7Aa>r-ebM;nQcqCAy@2DGP67H>ov3-{2J#=#O)XgCPN2@OL^Tar#gF>2Ep z;LuEB6Aln>qH6{NU!yvM1v}b^LZf_BA_W?(%cMzjScz^pk;y=$KHUmRw%4T6U2Ig?E9IZJtgyTdG0|!vDjfY{QtM)4S y6zt8T$q^m%S=fpCd>(pHQoxV~tuElnGIb#j6}VK#gT-jqX(_M&dmYqom;DPymKvx4 diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index 32970cb19..e9526f143 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -192,7 +192,7 @@ impl block_producer_server::BlockProducer for StoreApi { .inspect_err(|err| tracing::Span::current().set_error(err)) .map_err(|err| tonic::Status::internal(err.as_report()))?; - let block_height = self.state.chain_tip(Finality::Committed).await.as_u32(); + let block_height = self.state.chain_tip(Finality::Committed).as_u32(); Ok(Response::new(proto::store::TransactionInputs { account_state: Some(proto::store::transaction_inputs::AccountTransactionInputRecord { diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index a4b7e09be..88b43083d 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -108,8 +108,7 @@ impl Store { self.block_prover_url, self.max_concurrent_proofs, tx_proven_tip, - ) - .await; + ); // Spawn gRPC Servers. let mut join_set = Self::spawn_grpc_servers( @@ -144,7 +143,7 @@ impl Store { /// /// Returns the scheduler task handle and the chain tip sender (needed by gRPC services to /// notify the scheduler of new blocks). - async fn spawn_proof_scheduler( + fn spawn_proof_scheduler( state: &State, block_prover_url: Option, max_concurrent_proofs: NonZeroUsize, @@ -159,7 +158,7 @@ impl Store { Arc::new(BlockProver::local()) }; - let chain_tip = state.chain_tip(crate::state::Finality::Committed).await; + let chain_tip = state.chain_tip(crate::state::Finality::Committed); let (chain_tip_tx, chain_tip_rx) = watch::channel(chain_tip); let handle = proof_scheduler::spawn( diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 763e5b9b3..71dec0595 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -146,7 +146,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let mut chain_tip = self.state.chain_tip(Finality::Committed).await; + let mut chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(Some(request), "GetNetworkAccountIds")? .into_inclusive_range::(&chain_tip)?; @@ -160,7 +160,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { last_block_included = chain_tip; } - chain_tip = self.state.chain_tip(Finality::Committed).await; + chain_tip = self.state.chain_tip(Finality::Committed); Ok(Response::new(proto::store::NetworkAccountIdList { account_ids, @@ -252,7 +252,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let block_num = if let Some(num) = request.block_num { num.into() } else { - self.state.chain_tip(Finality::Committed).await + self.state.chain_tip(Finality::Committed) }; // Retrieve the asset witnesses. @@ -305,7 +305,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let block_num = if let Some(num) = request.block_num { num.into() } else { - self.state.chain_tip(Finality::Committed).await + self.state.chain_tip(Finality::Committed) }; // Retrieve the storage map witness. @@ -321,7 +321,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { key: Some(map_key.into()), proof: Some(proof.into()), }), - block_num: self.state.chain_tip(Finality::Committed).await.as_u32(), + block_num: self.state.chain_tip(Finality::Committed).as_u32(), })) } } diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index 87055936f..9012f895d 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -94,7 +94,7 @@ impl rpc_server::Rpc for StoreApi { return Err(SyncNullifiersError::InvalidPrefixLength(request.prefix_len).into()); } - let chain_tip = self.state.chain_tip(Finality::Committed).await; + let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(request.block_range, "SyncNullifiersRequest")? .into_inclusive_range::(&chain_tip)?; @@ -129,7 +129,7 @@ impl rpc_server::Rpc for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let chain_tip = self.state.chain_tip(Finality::Committed).await; + let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(request.block_range, "SyncNotesRequest")? .into_inclusive_range::(&chain_tip)?; @@ -180,10 +180,10 @@ impl rpc_server::Rpc for StoreApi { let block_to = match sync_target { SyncTarget::BlockNumber(block_num) => { - block_num.min(self.state.chain_tip(Finality::Committed).await) + block_num.min(self.state.chain_tip(Finality::Committed)) }, - SyncTarget::CommittedChainTip => self.state.chain_tip(Finality::Committed).await, - SyncTarget::ProvenChainTip => self.state.chain_tip(Finality::Proven).await, + SyncTarget::CommittedChainTip => self.state.chain_tip(Finality::Committed), + SyncTarget::ProvenChainTip => self.state.chain_tip(Finality::Proven), }; if block_from > block_to { @@ -274,7 +274,7 @@ impl rpc_server::Rpc for StoreApi { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let chain_tip = self.state.chain_tip(Finality::Committed).await; + let chain_tip = self.state.chain_tip(Finality::Committed); let account_id: AccountId = read_account_id::< proto::rpc::SyncAccountVaultRequest, @@ -336,7 +336,7 @@ impl rpc_server::Rpc for StoreApi { Err(SyncAccountStorageMapsError::AccountNotPublic(account_id))?; } - let chain_tip = self.state.chain_tip(Finality::Committed).await; + let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::( request.block_range, "SyncAccountStorageMapsRequest", @@ -376,7 +376,7 @@ impl rpc_server::Rpc for StoreApi { Ok(Response::new(proto::rpc::StoreStatus { version: env!("CARGO_PKG_VERSION").to_string(), status: "connected".to_string(), - chain_tip: self.state.chain_tip(Finality::Committed).await.as_u32(), + chain_tip: self.state.chain_tip(Finality::Committed).as_u32(), })) } @@ -408,7 +408,7 @@ impl rpc_server::Rpc for StoreApi { let request = request.into_inner(); - let chain_tip = self.state.chain_tip(Finality::Committed).await; + let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::( request.block_range, "SyncTransactionsRequest", diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index e05ef6272..983330de4 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -380,8 +380,7 @@ impl State { /// state snapshot (wait-free via `ArcSwap`). /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch /// channel, updated by the proof scheduler). - #[expect(clippy::unused_async)] - pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { + pub fn chain_tip(&self, finality: Finality) -> BlockNumber { match finality { Finality::Committed => self.snapshot().block_num, Finality::Proven => self.proven_tip.read(), @@ -991,7 +990,7 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed).await { + if block_num > self.chain_tip(Finality::Committed) { return Ok(None); } self.block_store.load_block(block_num).await.map_err(Into::into) @@ -1002,7 +1001,7 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven).await { + if block_num > self.chain_tip(Finality::Proven) { return Ok(None); } self.block_store.load_proof(block_num).await.map_err(Into::into) From 4aa598b6b7db6215544214764d91e5d401381d73 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 10:38:42 +1200 Subject: [PATCH 33/48] More chain tip checks --- crates/store/src/errors.rs | 9 ++++----- crates/store/src/server/rpc_api.rs | 20 +++++++++++++++++--- crates/store/src/state/mod.rs | 3 +++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index d5b7d6bf4..510ab4585 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -267,11 +267,8 @@ pub enum StateSyncError { pub enum SyncChainMmrError { #[error("invalid block range")] InvalidBlockRange(#[source] InvalidBlockRange), - #[error("start block is not known")] - FutureBlock { - chain_tip: BlockNumber, - block_from: BlockNumber, - }, + #[error("block {0} is unknown")] + UnknownBlock(BlockNumber), #[error("malformed block number")] DeserializationFailed(#[source] ConversionError), #[error("database error")] @@ -355,6 +352,8 @@ pub enum SyncNullifiersError { InvalidPrefixLength(u32), #[error("malformed nullifier prefix")] DeserializationFailed(#[from] ConversionError), + #[error("block {0} is unknown")] + UnknownBlock(BlockNumber), } // SYNC ACCOUNT VAULT ERRORS diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index 9012f895d..02f88c33e 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -1,5 +1,5 @@ use miden_node_proto::convert; -use miden_node_proto::domain::block::SyncTarget; +use miden_node_proto::domain::block::{InvalidBlockRange, SyncTarget}; use miden_node_proto::generated::store::rpc_server; use miden_node_proto::generated::{self as proto}; use miden_node_utils::limiter::{ @@ -99,6 +99,10 @@ impl rpc_server::Rpc for StoreApi { read_block_range::(request.block_range, "SyncNullifiersRequest")? .into_inclusive_range::(&chain_tip)?; + if block_range.end() > &chain_tip { + return Err(SyncNullifiersError::UnknownBlock(chain_tip).into()); + } + let (nullifiers, block_num) = self .state .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) @@ -133,7 +137,7 @@ impl rpc_server::Rpc for StoreApi { let block_range = read_block_range::(request.block_range, "SyncNotesRequest")? .into_inclusive_range::(&chain_tip)?; - if *block_range.end() > chain_tip { + if block_range.end() > &chain_tip { Err(NoteSyncError::FutureBlock { chain_tip, block_to: *block_range.end() })?; } @@ -186,9 +190,19 @@ impl rpc_server::Rpc for StoreApi { SyncTarget::ProvenChainTip => self.state.chain_tip(Finality::Proven), }; + // Check range sanity. if block_from > block_to { - Err(SyncChainMmrError::FutureBlock { chain_tip: block_to, block_from })?; + Err(SyncChainMmrError::InvalidBlockRange(InvalidBlockRange::StartGreaterThanEnd { + start: block_from, + end: block_to, + }))?; } + + // Check range does not go beyond current tip. + if block_to > self.state.chain_tip(Finality::Committed) { + Err(SyncChainMmrError::UnknownBlock(block_to))?; + } + let block_range = block_from..=block_to; let (mmr_delta, block_header) = self.state.sync_chain_mmr(block_range.clone()).await.map_err(internal_error)?; diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 983330de4..46e0a2f47 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -820,6 +820,9 @@ impl State { // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { + if requested_block > snapshot.block_num { + return Err(GetAccountError::UnknownBlock(requested_block)); + } // Historical query: use the account tree with history let witness = snapshot.account_tree.open_at(account_id, requested_block).ok_or_else(|| { From 619df560e0b4fe95f98e99d00957bfa33d288bd4 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 12:36:48 +1200 Subject: [PATCH 34/48] Return block height from get_transactions_input (fix) --- crates/store/src/server/block_producer.rs | 7 ++--- crates/store/src/state/mod.rs | 34 ++++++++++++++--------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index e9526f143..5f21b09bb 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -26,7 +26,6 @@ use crate::server::api::{ validate_note_commitments, validate_nullifiers, }; -use crate::state::Finality; // BLOCK PRODUCER ENDPOINTS // ================================================================================================ @@ -185,15 +184,13 @@ impl block_producer_server::BlockProducer for StoreApi { let unauthenticated_note_commitments = validate_note_commitments(&request.unauthenticated_notes)?; - let tx_inputs = self + let (tx_inputs, block_height) = self .state .get_transaction_inputs(account_id, &nullifiers, unauthenticated_note_commitments) .await .inspect_err(|err| tracing::Span::current().set_error(err)) .map_err(|err| tonic::Status::internal(err.as_report()))?; - let block_height = self.state.chain_tip(Finality::Committed).as_u32(); - Ok(Response::new(proto::store::TransactionInputs { account_state: Some(proto::store::transaction_inputs::AccountTransactionInputRecord { account_id: Some(account_id.into()), @@ -215,7 +212,7 @@ impl block_producer_server::BlockProducer for StoreApi { .map(Into::into) .collect(), new_account_id_prefix_is_unique: tx_inputs.new_account_id_prefix_is_unique, - block_height, + block_height: block_height.as_u32(), })) } } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 46e0a2f47..3530a9f01 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -701,19 +701,21 @@ impl State { Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) } - /// Returns data needed by the block producer to verify transactions validity. + /// Returns data needed by the block producer to verify transactions validity and the + /// corresponding block height. #[instrument(target = COMPONENT, skip_all, ret)] pub async fn get_transaction_inputs( &self, account_id: AccountId, nullifiers: &[Nullifier], unauthenticated_note_commitments: Vec, - ) -> Result { + ) -> Result<(TransactionInputs, BlockNumber), DatabaseError> { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); // Take a snapshot and extract everything we need, then drop it so readers of newer // snapshots aren't held up by this Arc. let snapshot = self.snapshot(); + let block_height = snapshot.block_num; let account_commitment = snapshot.account_tree.get_latest_commitment(account_id); @@ -725,10 +727,13 @@ impl State { // Non-unique account Id prefixes for new accounts are not allowed. if let Some(false) = new_account_id_prefix_is_unique { - return Ok(TransactionInputs { - new_account_id_prefix_is_unique, - ..Default::default() - }); + return Ok(( + TransactionInputs { + new_account_id_prefix_is_unique, + ..Default::default() + }, + block_height, + )); } let nullifiers = nullifiers @@ -739,7 +744,7 @@ impl State { }) .collect(); - // Drop snapshot before the async DB call. + // Drop snapshot immediately after using it. drop(snapshot); let found_unauthenticated_notes = self @@ -747,12 +752,15 @@ impl State { .select_existing_note_commitments(unauthenticated_note_commitments) .await?; - Ok(TransactionInputs { - account_commitment, - nullifiers, - found_unauthenticated_notes, - new_account_id_prefix_is_unique, - }) + Ok(( + TransactionInputs { + account_commitment, + nullifiers, + found_unauthenticated_notes, + new_account_id_prefix_is_unique, + }, + block_height, + )) } /// Returns details for public (on-chain) account. From dfed797bfad427c005b76e1a6c2915425d12bbb2 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 12:49:56 +1200 Subject: [PATCH 35/48] block scope select_existing_note_commitments --- crates/store/src/db/mod.rs | 6 ++++-- crates/store/src/db/models/queries/notes.rs | 6 +++++- crates/store/src/state/mod.rs | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 61cf62c06..29549b0a8 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -503,14 +503,16 @@ impl Db { .await } - /// Returns all note commitments from the DB that match the provided ones. + /// Returns all note commitments from the DB that match the provided ones, scoped to notes + /// committed at or before the given block number. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_existing_note_commitments( &self, note_commitments: Vec, + block_num: BlockNumber, ) -> Result> { self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments(conn, note_commitments.as_slice()) + queries::select_existing_note_commitments(conn, note_commitments.as_slice(), block_num) }) .await } diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index 1bb605125..eb28438a4 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -233,7 +233,8 @@ pub(crate) fn select_notes_by_id( Ok(records) } -/// Select the subset of note commitments that already exist in the notes table +/// Select the subset of note commitments that already exist in the notes table, scoped to notes +/// committed at or before the given block number. /// /// # Raw SQL /// @@ -242,10 +243,12 @@ pub(crate) fn select_notes_by_id( /// notes.note_commitment /// FROM notes /// WHERE note_commitment IN (?1) +/// AND committed_at <= ?2 /// ``` pub(crate) fn select_existing_note_commitments( conn: &mut SqliteConnection, note_commitments: &[Word], + block_num: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -253,6 +256,7 @@ pub(crate) fn select_existing_note_commitments( let raw_commitments = SelectDsl::select(schema::notes::table, schema::notes::note_commitment) .filter(schema::notes::note_commitment.eq_any(¬e_commitments)) + .filter(schema::notes::committed_at.le(block_num.to_raw_sql())) .load::>(conn)?; let commitments = raw_commitments diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 3530a9f01..7b1c97d60 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -749,7 +749,7 @@ impl State { let found_unauthenticated_notes = self .db - .select_existing_note_commitments(unauthenticated_note_commitments) + .select_existing_note_commitments(unauthenticated_note_commitments, block_height) .await?; Ok(( From c2e37e0cb27c1280f27bbc17318d9991b8196929 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 13:12:31 +1200 Subject: [PATCH 36/48] Remove unnecessary tip checks and update comments --- crates/store/src/server/rpc_api.rs | 3 ++- crates/store/src/state/mod.rs | 37 +++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index 02f88c33e..e0b34ab2b 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -94,11 +94,12 @@ impl rpc_server::Rpc for StoreApi { return Err(SyncNullifiersError::InvalidPrefixLength(request.prefix_len).into()); } + // Ensure that the block range is scoped by the snapshot because we will return the + // snapshot's block number as the tip of the chain in the response below. let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(request.block_range, "SyncNullifiersRequest")? .into_inclusive_range::(&chain_tip)?; - if block_range.end() > &chain_tip { return Err(SyncNullifiersError::UnknownBlock(chain_tip).into()); } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 7b1c97d60..dbb372f1d 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -138,11 +138,32 @@ pub(crate) struct InMemoryState { /// The rollup state. /// -/// A single writer task (serialized by a channel) mutates the state. All trees, the blockchain -/// MMR, and the forest are held in an `Arc` behind an [`ArcSwap`], providing -/// wait-free reads. The writer owns writable copies of the trees directly (passed as owned -/// values to [`writer::writer_loop`]) and creates snapshot-backed read-only copies for -/// `InMemoryState` after each block. +/// State is comprised of three data sets: +/// +/// 1. **In-memory** ([`InMemoryState`]): nullifier tree, account tree, blockchain MMR, and account +/// state forest. Held behind an [`ArcSwap`] for wait-free reads. +/// 2. **SQLite**: block headers, notes, nullifiers, accounts, transactions, and other relational +/// data. +/// 3. **File-based** ([`BlockStore`]): serialized blocks and proofs stored on disk. +/// +/// A single writer task (serialized by a channel) mutates all three data sets. The writer owns +/// writable copies of the in-memory trees directly (passed as owned values to +/// [`writer::writer_loop`]) and creates snapshot-backed read-only copies for [`InMemoryState`] +/// after each block. The writer commits to SQLite and the block store *before* swapping the +/// in-memory pointer, so there is a window where the DB/files are ahead of the in-memory state. +/// +/// ## Consistency rules for reader methods +/// +/// Any method that combines in-memory and SQLite data **must** take a snapshot and use its +/// `block_num` to scope all DB queries. This ensures the two data sets are consistent even +/// during the window described above. Concretely, such a method must either: +/// +/// - Reject requests where the caller-supplied block number exceeds the snapshot's chain tip, or +/// - Inherently limit its DB query scope to `<= snapshot.block_num`. +/// +/// Methods that operate purely on SQLite or file-based data (e.g. loading a block by number, +/// querying account details by a caller-supplied block number that was already validated) are +/// free to access those stores directly without taking a snapshot. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -1001,9 +1022,6 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed) { - return Ok(None); - } self.block_store.load_block(block_num).await.map_err(Into::into) } @@ -1012,9 +1030,6 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven) { - return Ok(None); - } self.block_store.load_proof(block_num).await.map_err(Into::into) } From d5a39bc9f322b2787e8fb4838eea40080b5f30c4 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 13:43:27 +1200 Subject: [PATCH 37/48] Return chain tip and avoid using snapshot out of state --- crates/store/src/errors.rs | 2 + crates/store/src/server/rpc_api.rs | 21 ++------- crates/store/src/state/mod.rs | 37 +++++++--------- crates/store/src/state/sync_state.rs | 64 ++++++++++++++++++++++------ 4 files changed, 72 insertions(+), 52 deletions(-) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 510ab4585..e0c670a28 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -261,6 +261,8 @@ pub enum StateSyncError { EmptyBlockHeadersTable, #[error("failed to build MMR delta")] FailedToBuildMmrDelta(#[from] MmrError), + #[error("block {0} is unknown")] + UnknownBlock(BlockNumber), } #[derive(Error, Debug, GrpcError)] diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index e0b34ab2b..043b66f39 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -94,17 +94,12 @@ impl rpc_server::Rpc for StoreApi { return Err(SyncNullifiersError::InvalidPrefixLength(request.prefix_len).into()); } - // Ensure that the block range is scoped by the snapshot because we will return the - // snapshot's block number as the tip of the chain in the response below. let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(request.block_range, "SyncNullifiersRequest")? .into_inclusive_range::(&chain_tip)?; - if block_range.end() > &chain_tip { - return Err(SyncNullifiersError::UnknownBlock(chain_tip).into()); - } - let (nullifiers, block_num) = self + let (nullifiers, block_num, chain_tip) = self .state .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) .await @@ -138,14 +133,11 @@ impl rpc_server::Rpc for StoreApi { let block_range = read_block_range::(request.block_range, "SyncNotesRequest")? .into_inclusive_range::(&chain_tip)?; - if block_range.end() > &chain_tip { - Err(NoteSyncError::FutureBlock { chain_tip, block_to: *block_range.end() })?; - } // Validate note tags count check::(request.note_tags.len())?; - let (results, last_block_checked) = + let (results, last_block_checked, chain_tip) = self.state.sync_notes(request.note_tags, block_range).await?; let blocks = results @@ -199,11 +191,6 @@ impl rpc_server::Rpc for StoreApi { }))?; } - // Check range does not go beyond current tip. - if block_to > self.state.chain_tip(Finality::Committed) { - Err(SyncChainMmrError::UnknownBlock(block_to))?; - } - let block_range = block_from..=block_to; let (mmr_delta, block_header) = self.state.sync_chain_mmr(block_range.clone()).await.map_err(internal_error)?; @@ -306,7 +293,7 @@ impl rpc_server::Rpc for StoreApi { )? .into_inclusive_range::(&chain_tip)?; - let (last_included_block, updates) = self + let (last_included_block, updates, chain_tip) = self .state .sync_account_vault(account_id, block_range) .await @@ -358,7 +345,7 @@ impl rpc_server::Rpc for StoreApi { )? .into_inclusive_range::(&chain_tip)?; - let storage_maps_page = self + let (storage_maps_page, chain_tip) = self .state .sync_account_storage_maps(account_id, block_range) .await diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index dbb372f1d..e9522f469 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -825,10 +825,17 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_account_witness(block_num, account_id)?; + let snapshot = self.snapshot(); + + let (block_num, witness) = Self::get_account_witness(&snapshot, block_num, account_id)?; let details = if let Some(request) = details { - Some(self.fetch_public_account_details(account_id, block_num, request).await?) + Some( + Self::fetch_public_account_details( + &self.db, &snapshot, account_id, block_num, request, + ) + .await?, + ) } else { None }; @@ -841,12 +848,10 @@ impl State { /// If `block_num` is provided, returns the witness at that historical block; /// otherwise, returns the witness at the latest block. fn get_account_witness( - &self, + snapshot: &Arc, block_num: Option, account_id: AccountId, ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - let snapshot = self.snapshot(); - // Determine which block to query let (block_num, witness) = if let Some(requested_block) = block_num { if requested_block > snapshot.block_num { @@ -884,7 +889,8 @@ impl State { /// All-entries queries (`SlotData::All`) use the forest to request all entries database. #[expect(clippy::too_many_lines)] async fn fetch_public_account_details( - &self, + db: &Arc, + snapshot: &Arc, account_id: AccountId, block_num: BlockNumber, detail_request: AccountDetailRequest, @@ -899,15 +905,8 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } - // Validate block exists in the blockchain before querying the database. - let snapshot = self.snapshot(); - if block_num > snapshot.block_num { - return Err(GetAccountError::UnknownBlock(block_num)); - } - // Query account header and storage header together in a single DB call - let (account_header, storage_header) = self - .db + let (account_header, storage_header) = db .select_account_header_with_storage_header_at_block(account_id, block_num) .await? .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; @@ -915,9 +914,7 @@ impl State { let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, Some(_) => { - self.db - .select_account_code_by_commitment(account_header.code_commitment()) - .await? + db.select_account_code_by_commitment(account_header.code_commitment()).await? }, None => None, }; @@ -927,8 +924,7 @@ impl State { AccountVaultDetails::empty() }, Some(_) => { - let vault_assets = - self.db.select_account_vault_at_block(account_id, block_num).await?; + let vault_assets = db.select_account_vault_at_block(account_id, block_num).await?; AccountVaultDetails::from_assets(vault_assets) }, None => AccountVaultDetails::empty(), @@ -978,8 +974,7 @@ impl State { // TODO parallelize the read requests for (index, slot_name) in all_entries_requests { - let details = self - .db + let details = db .reconstruct_storage_map_from_db( account_id, slot_name.clone(), diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 66da87abd..13c4e58ac 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -44,11 +44,14 @@ impl State { &self, block_range: RangeInclusive, ) -> Result<(MmrDelta, BlockHeader), StateSyncError> { + let snapshot = self.snapshot(); + let chain_tip = snapshot.block_num; let block_from = *block_range.start(); let block_to = *block_range.end(); + if block_to > chain_tip { + return Err(StateSyncError::UnknownBlock(block_to)); + } - // SAFETY: block_to has been validated to be <= the effective tip (chain tip or latest - // proven block) by the caller, so it must exist in the database. let block_header = self .db .select_block_header_by_block_num(Some(block_to)) @@ -77,7 +80,6 @@ impl State { let from_forest = (block_from + 1).as_usize(); let to_forest = block_to.as_usize(); - let snapshot = self.snapshot(); let mmr_delta = snapshot .blockchain .as_mmr() @@ -100,9 +102,15 @@ impl State { &self, note_tags: Vec, block_range: RangeInclusive, - ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { + ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber, BlockNumber), NoteSyncError> { + // Ensure the requested block range is within the chain's current tip. let snapshot = self.snapshot(); + let chain_tip = snapshot.block_num; let block_end = *block_range.end(); + if block_range.end() > &chain_tip { + Err(NoteSyncError::FutureBlock { chain_tip, block_to: *block_range.end() })?; + } + let note_tags: Arc<[u32]> = note_tags.into(); let mut results = Vec::new(); @@ -138,38 +146,66 @@ impl State { let last_block_checked = results.last().map_or(block_end, |(update, _)| update.block_header.block_num()); - Ok((results, last_block_checked)) + Ok((results, last_block_checked, chain_tip)) } + /// Returns nullifiers matching the given prefixes within the block range. + /// + /// The block range is validated against the snapshot's chain tip. Returns the matching + /// nullifiers, the last block included, and the chain tip at the time of the query. pub async fn sync_nullifiers( &self, prefix_len: u32, nullifier_prefixes: Vec, block_range: RangeInclusive, - ) -> Result<(Vec, BlockNumber), DatabaseError> { - self.db + ) -> Result<(Vec, BlockNumber, BlockNumber), DatabaseError> { + // Ensure the db query is scoped by the snapshot's chain tip. + let chain_tip = self.snapshot().block_num; + if block_range.end() > &chain_tip { + return Err(DatabaseError::UnknownBlock(*block_range.end())); + } + + let (nullifiers, block_num) = self + .db .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) - .await + .await?; + + Ok((nullifiers, block_num, chain_tip)) } // ACCOUNT STATE SYNCHRONIZATION // -------------------------------------------------------------------------------------------- - /// Returns account vault updates for specified account within a block range. + /// Returns account vault updates for specified account within a block range, including the last + /// included block and the chain tip. pub async fn sync_account_vault( &self, account_id: AccountId, block_range: RangeInclusive, - ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.get_account_vault_sync(account_id, block_range).await + ) -> Result<(BlockNumber, Vec, BlockNumber), DatabaseError> { + // Ensure the db query is scoped by the snapshot's chain tip. + let chain_tip = self.snapshot().block_num; + if block_range.end() > &chain_tip { + return Err(DatabaseError::UnknownBlock(*block_range.end())); + } + let (last_included_block, vault_updates) = + self.db.get_account_vault_sync(account_id, block_range).await?; + Ok((last_included_block, vault_updates, chain_tip)) } - /// Returns storage map values for syncing within a block range. + /// Returns storage map values for syncing within a block range including the chain tip. pub async fn sync_account_storage_maps( &self, account_id: AccountId, block_range: RangeInclusive, - ) -> Result { - self.db.select_storage_map_sync_values(account_id, block_range, None).await + ) -> Result<(StorageMapValuesPage, BlockNumber), DatabaseError> { + // Ensure the db query is scoped by the snapshot's chain tip. + let chain_tip = self.snapshot().block_num; + if block_range.end() > &chain_tip { + return Err(DatabaseError::UnknownBlock(*block_range.end())); + } + let storage_map_values = + self.db.select_storage_map_sync_values(account_id, block_range, None).await?; + Ok((storage_map_values, chain_tip)) } } From ee7461709793121e8e2a30560b12c86cb368f90c Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 14:28:32 +1200 Subject: [PATCH 38/48] Add scoped result --- crates/store/src/db/mod.rs | 6 +- crates/store/src/db/models/queries/notes.rs | 31 ++-------- crates/store/src/server/block_producer.rs | 4 +- crates/store/src/server/rpc_api.rs | 21 +++++-- crates/store/src/state/mod.rs | 67 +++++++++++++++------ crates/store/src/state/sync_state.rs | 31 ++++++---- 6 files changed, 94 insertions(+), 66 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 29549b0a8..cdd80a067 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -517,14 +517,16 @@ impl Db { .await } - /// Loads inclusion proofs for notes matching the given note commitments. + /// Loads inclusion proofs for notes matching the given note commitments, scoped to notes + /// committed at or before the given block number. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_note_inclusion_proofs( &self, note_commitments: BTreeSet, + block_num: BlockNumber, ) -> Result> { self.transact("block note inclusion proofs by commitment", move |conn| { - models::queries::select_note_inclusion_proofs(conn, ¬e_commitments) + models::queries::select_note_inclusion_proofs(conn, ¬e_commitments, block_num) }) .await } diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index eb28438a4..db966378e 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -315,36 +315,12 @@ pub(crate) fn select_all_notes( Ok(records) } -/// Select note inclusion proofs matching the note commitments. -/// -/// # Parameters -/// * `note_ids`: Set of note IDs to query -/// - Limit: 0 <= count <= 1000 -/// -/// # Returns -/// -/// - Empty map if no matching `note`. -/// - Otherwise, note inclusion proofs, which `note_id` matches the `NoteId` as bytes. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// committed_at, -/// note_id, -/// batch_index, -/// note_index, -/// inclusion_path -/// FROM -/// notes -/// WHERE -/// note_id IN (?1) -/// ORDER BY -/// committed_at ASC -/// ``` +/// Select note inclusion proofs for notes matching the given commitments, scoped to notes +/// committed at or before the given block number. pub(crate) fn select_note_inclusion_proofs( conn: &mut SqliteConnection, note_commitments: &BTreeSet, + block_num: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -361,6 +337,7 @@ pub(crate) fn select_note_inclusion_proofs( ), ) .filter(schema::notes::note_commitment.eq_any(note_commitments)) + .filter(schema::notes::committed_at.le(block_num.to_raw_sql())) .order_by(schema::notes::committed_at.asc()) .load::<(i64, Vec, i32, i32, Vec)>(conn)?; diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index 5f21b09bb..95ae51ec7 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -184,12 +184,14 @@ impl block_producer_server::BlockProducer for StoreApi { let unauthenticated_note_commitments = validate_note_commitments(&request.unauthenticated_notes)?; - let (tx_inputs, block_height) = self + let result = self .state .get_transaction_inputs(account_id, &nullifiers, unauthenticated_note_commitments) .await .inspect_err(|err| tracing::Span::current().set_error(err)) .map_err(|err| tonic::Status::internal(err.as_report()))?; + let block_height = result.chain_tip(); + let tx_inputs = result.inner; Ok(Response::new(proto::store::TransactionInputs { account_state: Some(proto::store::transaction_inputs::AccountTransactionInputRecord { diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index 043b66f39..bb0f35c3c 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -99,11 +99,13 @@ impl rpc_server::Rpc for StoreApi { read_block_range::(request.block_range, "SyncNullifiersRequest")? .into_inclusive_range::(&chain_tip)?; - let (nullifiers, block_num, chain_tip) = self + let result = self .state .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) .await .map_err(SyncNullifiersError::from)?; + let chain_tip = result.chain_tip(); + let (nullifiers, block_num) = result.inner; let nullifiers = nullifiers .into_iter() @@ -137,8 +139,9 @@ impl rpc_server::Rpc for StoreApi { // Validate note tags count check::(request.note_tags.len())?; - let (results, last_block_checked, chain_tip) = - self.state.sync_notes(request.note_tags, block_range).await?; + let result = self.state.sync_notes(request.note_tags, block_range).await?; + let chain_tip = result.chain_tip(); + let (results, last_block_checked) = result.inner; let blocks = results .into_iter() @@ -293,11 +296,13 @@ impl rpc_server::Rpc for StoreApi { )? .into_inclusive_range::(&chain_tip)?; - let (last_included_block, updates, chain_tip) = self + let result = self .state .sync_account_vault(account_id, block_range) .await .map_err(SyncAccountVaultError::from)?; + let chain_tip = result.chain_tip(); + let (last_included_block, updates) = result.inner; let updates = updates .into_iter() @@ -345,11 +350,13 @@ impl rpc_server::Rpc for StoreApi { )? .into_inclusive_range::(&chain_tip)?; - let (storage_maps_page, chain_tip) = self + let result = self .state .sync_account_storage_maps(account_id, block_range) .await .map_err(SyncAccountStorageMapsError::from)?; + let chain_tip = result.chain_tip(); + let storage_maps_page = result.inner; let updates = storage_maps_page .values @@ -423,11 +430,13 @@ impl rpc_server::Rpc for StoreApi { // Validate account IDs count check::(account_ids.len())?; - let (last_block_included, transaction_records_db) = self + let result = self .state .sync_transactions(account_ids, block_range.clone()) .await .map_err(SyncTransactionsError::from)?; + let chain_tip = result.chain_tip(); + let (last_block_included, transaction_records_db) = result.inner; // Convert database TransactionRecords directly to proto TransactionRecords. // All data needed for the proto TransactionHeader is stored in the transactions table. diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index e9522f469..823186e04 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -102,6 +102,33 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } +// SCOPED RESULT +// ================================================================================================ + +/// A query result scoped to a specific chain tip. +/// +/// Wraps an inner value `T` with the [`BlockNumber`] of the snapshot that was used to produce it. +/// This ensures callers always know which block the data corresponds to. +#[derive(Debug)] +pub struct Scoped { + /// The chain tip at the time the query was executed. + chain_tip: BlockNumber, + /// The query result. + pub inner: T, +} + +impl Scoped { + /// Creates a new scoped result. + pub fn new(chain_tip: BlockNumber, inner: T) -> Self { + Self { chain_tip, inner } + } + + /// Returns the chain tip at the time the query was executed. + pub fn chain_tip(&self) -> BlockNumber { + self.chain_tip + } +} + // IN-MEMORY STATE // ================================================================================================ @@ -477,8 +504,6 @@ impl State { return Ok(None); } - // Scope the DB query to the snapshot's block number to ensure consistency between - // the block header (from SQLite) and the blockchain peaks (from the snapshot). let block_header: BlockHeader = self .db .select_block_header_by_block_num(Some(snapshot.block_num)) @@ -520,12 +545,15 @@ impl State { return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); } + let snapshot = self.snapshot(); + let latest_block_num = snapshot.block_num; + // First we grab note inclusion proofs for the known notes. These proofs only // prove that the note was included in a given block. We then also need to prove that // each of those blocks is included in the chain. let note_proofs = self .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) + .select_note_inclusion_proofs(unauthenticated_note_commitments, latest_block_num) .await .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; @@ -538,9 +566,6 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - let snapshot = self.snapshot(); - let latest_block_num = snapshot.block_num; - let highest_block_num = *blocks.last().expect("we should have checked for empty block references"); if highest_block_num > latest_block_num { @@ -613,10 +638,13 @@ impl State { unauthenticated_note_commitments: BTreeSet, reference_blocks: BTreeSet, ) -> Result { + let snapshot = self.snapshot(); + let latest_block_number = snapshot.block_num; + // Get the note inclusion proofs from the DB. let unauthenticated_note_proofs = self .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) + .select_note_inclusion_proofs(unauthenticated_note_commitments, snapshot.block_num) .await .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; @@ -628,8 +656,8 @@ impl State { let mut blocks = reference_blocks; blocks.extend(note_proof_reference_blocks); - let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; + let (account_witnesses, nullifier_witnesses, partial_mmr) = + Self::get_block_inputs_witnesses(&snapshot, &mut blocks, &account_ids, &nullifiers)?; // Fetch the block headers for all blocks in the partial MMR plus the latest one which will // be used as the previous block header of the block being built. @@ -669,21 +697,18 @@ impl State { /// number is removed from `blocks` and returned separately. #[expect(clippy::type_complexity)] fn get_block_inputs_witnesses( - &self, + snapshot: &Arc, blocks: &mut BTreeSet, account_ids: &[AccountId], nullifiers: &[Nullifier], ) -> Result< ( - BlockNumber, BTreeMap, BTreeMap, PartialMmr, ), GetBlockInputsError, > { - // Take a snapshot and extract everything we need from it. - let snapshot = self.snapshot(); let latest_block_number = snapshot.block_num; // If `blocks` is empty, use the latest block number which will never trigger the error. @@ -719,7 +744,7 @@ impl State { .map(|nullifier| (nullifier, snapshot.nullifier_tree.open(&nullifier))) .collect(); - Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) + Ok((account_witnesses, nullifier_witnesses, partial_mmr)) } /// Returns data needed by the block producer to verify transactions validity and the @@ -730,7 +755,7 @@ impl State { account_id: AccountId, nullifiers: &[Nullifier], unauthenticated_note_commitments: Vec, - ) -> Result<(TransactionInputs, BlockNumber), DatabaseError> { + ) -> Result, DatabaseError> { info!(target: COMPONENT, account_id = %account_id.to_string(), nullifiers = %format_array(nullifiers)); // Take a snapshot and extract everything we need, then drop it so readers of newer @@ -748,12 +773,12 @@ impl State { // Non-unique account Id prefixes for new accounts are not allowed. if let Some(false) = new_account_id_prefix_is_unique { - return Ok(( + return Ok(Scoped::new( + block_height, TransactionInputs { new_account_id_prefix_is_unique, ..Default::default() }, - block_height, )); } @@ -773,14 +798,14 @@ impl State { .select_existing_note_commitments(unauthenticated_note_commitments, block_height) .await?; - Ok(( + Ok(Scoped::new( + block_height, TransactionInputs { account_commitment, nullifiers, found_unauthenticated_notes, new_account_id_prefix_is_unique, }, - block_height, )) } @@ -905,6 +930,10 @@ impl State { return Err(GetAccountError::AccountNotPublic(account_id)); } + if block_num > snapshot.block_num { + return Err(GetAccountError::UnknownBlock(block_num)); + } + // Query account header and storage header together in a single DB call let (account_header, storage_header) = db .select_account_header_with_storage_header_at_block(account_id, block_num) diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 13c4e58ac..8c2b44771 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -7,7 +7,7 @@ use miden_protocol::block::{BlockHeader, BlockNumber}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrProof}; use tracing::instrument; -use super::State; +use super::{Scoped, State}; use crate::COMPONENT; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo}; @@ -34,8 +34,16 @@ impl State { &self, account_ids: Vec, block_range: RangeInclusive, - ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.select_transactions_records(account_ids, block_range).await + ) -> Result)>, DatabaseError> { + let snapshot = self.snapshot(); + let chain_tip = snapshot.block_num; + let block_to = *block_range.end(); + if block_to > chain_tip { + return Err(DatabaseError::UnknownBlock(block_to)); + } + let (last_block_included, transactions) = + self.db.select_transactions_records(account_ids, block_range).await?; + Ok(Scoped::new(chain_tip, (last_block_included, transactions))) } /// Returns the chain MMR delta and the `block_to` block header for the specified block range. @@ -97,12 +105,13 @@ impl State { /// /// Also returns the last block number checked. If this equals `block_range.end()`, the /// sync is complete. + #[expect(clippy::type_complexity)] #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn sync_notes( &self, note_tags: Vec, block_range: RangeInclusive, - ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber, BlockNumber), NoteSyncError> { + ) -> Result, BlockNumber)>, NoteSyncError> { // Ensure the requested block range is within the chain's current tip. let snapshot = self.snapshot(); let chain_tip = snapshot.block_num; @@ -146,7 +155,7 @@ impl State { let last_block_checked = results.last().map_or(block_end, |(update, _)| update.block_header.block_num()); - Ok((results, last_block_checked, chain_tip)) + Ok(Scoped::new(chain_tip, (results, last_block_checked))) } /// Returns nullifiers matching the given prefixes within the block range. @@ -158,7 +167,7 @@ impl State { prefix_len: u32, nullifier_prefixes: Vec, block_range: RangeInclusive, - ) -> Result<(Vec, BlockNumber, BlockNumber), DatabaseError> { + ) -> Result, BlockNumber)>, DatabaseError> { // Ensure the db query is scoped by the snapshot's chain tip. let chain_tip = self.snapshot().block_num; if block_range.end() > &chain_tip { @@ -170,7 +179,7 @@ impl State { .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await?; - Ok((nullifiers, block_num, chain_tip)) + Ok(Scoped::new(chain_tip, (nullifiers, block_num))) } // ACCOUNT STATE SYNCHRONIZATION @@ -182,7 +191,7 @@ impl State { &self, account_id: AccountId, block_range: RangeInclusive, - ) -> Result<(BlockNumber, Vec, BlockNumber), DatabaseError> { + ) -> Result)>, DatabaseError> { // Ensure the db query is scoped by the snapshot's chain tip. let chain_tip = self.snapshot().block_num; if block_range.end() > &chain_tip { @@ -190,7 +199,7 @@ impl State { } let (last_included_block, vault_updates) = self.db.get_account_vault_sync(account_id, block_range).await?; - Ok((last_included_block, vault_updates, chain_tip)) + Ok(Scoped::new(chain_tip, (last_included_block, vault_updates))) } /// Returns storage map values for syncing within a block range including the chain tip. @@ -198,7 +207,7 @@ impl State { &self, account_id: AccountId, block_range: RangeInclusive, - ) -> Result<(StorageMapValuesPage, BlockNumber), DatabaseError> { + ) -> Result, DatabaseError> { // Ensure the db query is scoped by the snapshot's chain tip. let chain_tip = self.snapshot().block_num; if block_range.end() > &chain_tip { @@ -206,6 +215,6 @@ impl State { } let storage_map_values = self.db.select_storage_map_sync_values(account_id, block_range, None).await?; - Ok((storage_map_values, chain_tip)) + Ok(Scoped::new(chain_tip, storage_map_values)) } } From 954400847280ad4fb6ed141db64bec781d1e42c2 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 15:04:16 +1200 Subject: [PATCH 39/48] Fix ntx-builder store fns and add Scoped::into_inner() --- crates/store/src/server/block_producer.rs | 2 +- crates/store/src/server/ntx_builder.rs | 19 +++++++------------ crates/store/src/server/rpc_api.rs | 10 +++++----- crates/store/src/state/mod.rs | 18 +++++++++++++++--- crates/store/src/state/sync_state.rs | 14 ++++++-------- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index 95ae51ec7..93cde678b 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -191,7 +191,7 @@ impl block_producer_server::BlockProducer for StoreApi { .inspect_err(|err| tracing::Span::current().set_error(err)) .map_err(|err| tonic::Status::internal(err.as_report()))?; let block_height = result.chain_tip(); - let tx_inputs = result.inner; + let tx_inputs = result.into_inner(); Ok(Response::new(proto::store::TransactionInputs { account_state: Some(proto::store::transaction_inputs::AccountTransactionInputRecord { diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 71dec0595..ffc888088 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -103,8 +103,6 @@ impl ntx_builder_server::NtxBuilder for StoreApi { request.account_id, )?; - let state = self.state.clone(); - let size = NonZero::try_from(request.page_size as usize).map_err(|err: TryFromIntError| { invalid_argument(err.as_report_context("invalid page_size")) @@ -112,7 +110,8 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let page = Page { token: request.page_token, size }; // TODO: no need to get the whole NoteRecord here, a NetworkNote wrapper should be created // instead - let (notes, next_page) = state + let (notes, next_page) = self + .state .get_unconsumed_network_notes_for_account(account_id, block_num, page) .await .map_err(internal_error)?; @@ -146,22 +145,18 @@ impl ntx_builder_server::NtxBuilder for StoreApi { ) -> Result, Status> { let request = request.into_inner(); - let mut chain_tip = self.state.chain_tip(Finality::Committed); + let chain_tip = self.state.chain_tip(Finality::Committed); let block_range = read_block_range::(Some(request), "GetNetworkAccountIds")? .into_inclusive_range::(&chain_tip)?; - let (account_ids, mut last_block_included) = + let result = self.state.get_all_network_accounts(block_range).await.map_err(internal_error)?; + let chain_tip = result.chain_tip(); + let (account_ids, last_block_included) = result.into_inner(); let account_ids = Vec::from_iter(account_ids.into_iter().map(Into::into)); - if last_block_included > chain_tip { - last_block_included = chain_tip; - } - - chain_tip = self.state.chain_tip(Finality::Committed); - Ok(Response::new(proto::store::NetworkAccountIdList { account_ids, pagination_info: Some(proto::rpc::PaginationInfo { @@ -321,7 +316,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { key: Some(map_key.into()), proof: Some(proof.into()), }), - block_num: self.state.chain_tip(Finality::Committed).as_u32(), + block_num: block_num.as_u32(), })) } } diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index bb0f35c3c..a4d371636 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -105,7 +105,7 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncNullifiersError::from)?; let chain_tip = result.chain_tip(); - let (nullifiers, block_num) = result.inner; + let (nullifiers, block_num) = result.into_inner(); let nullifiers = nullifiers .into_iter() @@ -141,7 +141,7 @@ impl rpc_server::Rpc for StoreApi { let result = self.state.sync_notes(request.note_tags, block_range).await?; let chain_tip = result.chain_tip(); - let (results, last_block_checked) = result.inner; + let (results, last_block_checked) = result.into_inner(); let blocks = results .into_iter() @@ -302,7 +302,7 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncAccountVaultError::from)?; let chain_tip = result.chain_tip(); - let (last_included_block, updates) = result.inner; + let (last_included_block, updates) = result.into_inner(); let updates = updates .into_iter() @@ -356,7 +356,7 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncAccountStorageMapsError::from)?; let chain_tip = result.chain_tip(); - let storage_maps_page = result.inner; + let storage_maps_page = result.into_inner(); let updates = storage_maps_page .values @@ -436,7 +436,7 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncTransactionsError::from)?; let chain_tip = result.chain_tip(); - let (last_block_included, transaction_records_db) = result.inner; + let (last_block_included, transaction_records_db) = result.into_inner(); // Convert database TransactionRecords directly to proto TransactionRecords. // All data needed for the proto TransactionHeader is stored in the transactions table. diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 823186e04..e104ff348 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -114,7 +114,7 @@ pub struct Scoped { /// The chain tip at the time the query was executed. chain_tip: BlockNumber, /// The query result. - pub inner: T, + inner: T, } impl Scoped { @@ -127,6 +127,11 @@ impl Scoped { pub fn chain_tip(&self) -> BlockNumber { self.chain_tip } + + /// Consumes the scoped type and returns the inner value. + pub fn into_inner(self) -> T { + self.inner + } } // IN-MEMORY STATE @@ -827,8 +832,15 @@ impl State { pub async fn get_all_network_accounts( &self, block_range: RangeInclusive, - ) -> Result<(Vec, BlockNumber), DatabaseError> { - self.db.select_all_network_account_ids(block_range).await + ) -> Result, BlockNumber)>, DatabaseError> { + let snapshot = self.snapshot(); + let chain_tip = snapshot.block_num; + let block_to = block_range.end(); + if block_to > &chain_tip { + return Err(DatabaseError::UnknownBlock(*block_to)); + } + let result = self.db.select_all_network_account_ids(block_range).await?; + Ok(Scoped::new(chain_tip, result)) } /// Returns an account witness and optionally account details at a specific block. diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 8c2b44771..2ff7df6d6 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -41,9 +41,8 @@ impl State { if block_to > chain_tip { return Err(DatabaseError::UnknownBlock(block_to)); } - let (last_block_included, transactions) = - self.db.select_transactions_records(account_ids, block_range).await?; - Ok(Scoped::new(chain_tip, (last_block_included, transactions))) + let result = self.db.select_transactions_records(account_ids, block_range).await?; + Ok(Scoped::new(chain_tip, result)) } /// Returns the chain MMR delta and the `block_to` block header for the specified block range. @@ -174,12 +173,12 @@ impl State { return Err(DatabaseError::UnknownBlock(*block_range.end())); } - let (nullifiers, block_num) = self + let result = self .db .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await?; - Ok(Scoped::new(chain_tip, (nullifiers, block_num))) + Ok(Scoped::new(chain_tip, result)) } // ACCOUNT STATE SYNCHRONIZATION @@ -197,9 +196,8 @@ impl State { if block_range.end() > &chain_tip { return Err(DatabaseError::UnknownBlock(*block_range.end())); } - let (last_included_block, vault_updates) = - self.db.get_account_vault_sync(account_id, block_range).await?; - Ok(Scoped::new(chain_tip, (last_included_block, vault_updates))) + let result = self.db.get_account_vault_sync(account_id, block_range).await?; + Ok(Scoped::new(chain_tip, result)) } /// Returns storage map values for syncing within a block range including the chain tip. From 5fd49541d154f54eb1b321d02d841ce4b559fa9d Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 14 Apr 2026 21:14:19 +1200 Subject: [PATCH 40/48] Update docstring --- crates/store/src/state/writer.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index c3c010523..20bf82ec3 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -53,19 +53,17 @@ pub(crate) async fn writer_loop( /// /// ## Consistency model /// -/// This function is the sole writer to all state. The writer owns the writable trees directly -/// (no locks or interior mutability). It applies mutations, then creates a new `InMemoryState` -/// with snapshot-backed read-only copies and atomically swaps the pointer. +/// This function is the sole writer to all state. The writer owns the writable trees directly. +/// +/// Because SQLite/files are committed **before** the in-memory swap, there is a window where the +/// DB is ahead of the in-memory state. Reader methods that combine in-memory and SQLite data +/// must scope their DB queries by the snapshot's `block_num` to maintain consistency (see the +/// doc comment on [`State`] for the full rules). /// /// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an /// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either /// the old or new state, never a partial update. Readers holding an `Arc` to the old state are /// completely unaffected by the swap. -/// -/// ## Performance -/// -/// No deep clone of tree data occurs in `RocksDB` mode. The snapshot-backed trees read directly -/// from `RocksDB` snapshots. Readers pay only an atomic refcount bump per `snapshot()` call. #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] async fn apply_block_inner( state: &State, From 40495bebc7363781d232eb36c08df8e4d8c29ef3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Apr 2026 12:27:34 +1200 Subject: [PATCH 41/48] Allow max block in get_all_network_accounts --- crates/store/src/state/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index e104ff348..8f12f320e 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -835,10 +835,10 @@ impl State { ) -> Result, BlockNumber)>, DatabaseError> { let snapshot = self.snapshot(); let chain_tip = snapshot.block_num; - let block_to = block_range.end(); - if block_to > &chain_tip { - return Err(DatabaseError::UnknownBlock(*block_to)); - } + // Clamp the upper bound to the chain tip so callers can use BlockNumber::MAX to mean + // "up to the latest block". + let clamped_end = std::cmp::min(*block_range.end(), chain_tip); + let block_range = *block_range.start()..=clamped_end; let result = self.db.select_all_network_account_ids(block_range).await?; Ok(Scoped::new(chain_tip, result)) } From 9c1c261c42e481296b753ead7817b542fbb579f7 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Apr 2026 12:28:00 +1200 Subject: [PATCH 42/48] Update deps --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b20cb5f4..441964a52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" dependencies = [ "blake3", "cc", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" dependencies = [ "quote", "syn 2.0.117", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-clone#bcb1daae09b0d821739e89c06d31f50cdc092ae7" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" dependencies = [ "p3-field", "p3-goldilocks", diff --git a/Cargo.toml b/Cargo.toml index 9ac2bada0..97a984d6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,8 +144,8 @@ should_panic_without_expect = "allow" # We don't care about the specific panic files.extend-exclude = ["*.svg"] # Ignore SVG files. [patch.crates-io] -miden-crypto = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } -miden-serde-utils = { branch = "sergerad-clone", git = "https://github.com/0xmiden/crypto" } +miden-crypto = { branch = "sergerad-largesmt-reader-trait", git = "https://github.com/0xmiden/crypto" } +miden-serde-utils = { branch = "sergerad-largesmt-reader-trait", git = "https://github.com/0xmiden/crypto" } miden-agglayer = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } miden-block-prover = { branch = "sergerad-clone", git = "https://github.com/0xmiden/protocol" } From 99925ffa51480c132d68c10a2200b1a92240c1bd Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Apr 2026 13:52:31 +1200 Subject: [PATCH 43/48] Clamp block num for get_unconsumed_network_notes_for_account --- crates/store/src/state/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 8f12f320e..394dd54ca 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -1083,9 +1083,7 @@ impl State { page: Page, ) -> Result<(Vec, Page), DatabaseError> { let snapshot = self.snapshot(); - if block_num > snapshot.block_num { - return Err(DatabaseError::UnknownBlock(block_num)); - } + let block_num = std::cmp::min(block_num, snapshot.block_num); self.db.select_unconsumed_network_notes(account_id, block_num, page).await } From 92926fe7c0bf844fd254e41ee8fa4843d1d5885d Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Apr 2026 14:43:13 +1200 Subject: [PATCH 44/48] Add BlockWriter --- crates/store/src/server/api.rs | 3 + crates/store/src/server/block_producer.rs | 2 +- crates/store/src/server/mod.rs | 7 +- crates/store/src/state/mod.rs | 105 ++-- crates/store/src/state/writer.rs | 571 ++++++++++++---------- 5 files changed, 355 insertions(+), 333 deletions(-) diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index d73cad9c1..4a0a7a623 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -17,6 +17,7 @@ use tracing::{info, instrument}; use crate::COMPONENT; use crate::errors::GetBlockInputsError; use crate::state::State; +use crate::state::writer::WriteHandle; // STORE API // ================================================================================================ @@ -24,6 +25,8 @@ use crate::state::State; #[derive(Clone)] pub struct StoreApi { pub(super) state: Arc, + /// Handle for submitting blocks to the writer loop. + pub(super) write_handle: WriteHandle, /// Sender used to notify the proof scheduler of the latest committed block number. pub(super) chain_tip_sender: watch::Sender, } diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs index 93cde678b..e7ea9eac1 100644 --- a/crates/store/src/server/block_producer.rs +++ b/crates/store/src/server/block_producer.rs @@ -93,7 +93,7 @@ impl block_producer_server::BlockProducer for StoreApi { .map_err(|err| Status::new(tonic::Code::Internal, err.as_report()))?; // Apply the block. - self.state + self.write_handle .apply_block(signed_block, Some(proving_inputs)) .await .inspect(|_| { diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 88b43083d..a9f702bac 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -97,7 +97,7 @@ impl Store { // Load initial state. let (termination_ask, mut termination_signal) = tokio::sync::mpsc::channel::(1); - let (state, tx_proven_tip) = + let (state, write_handle, tx_proven_tip) = State::load(&self.data_directory, self.storage_options, termination_ask) .await .context("failed to load state")?; @@ -113,6 +113,7 @@ impl Store { // Spawn gRPC Servers. let mut join_set = Self::spawn_grpc_servers( state, + write_handle, chain_tip_sender, self.grpc_options, self.rpc_listener, @@ -176,6 +177,7 @@ impl Store { /// Spawns the gRPC servers and the DB maintenance background task. fn spawn_grpc_servers( state: Arc, + write_handle: crate::state::writer::WriteHandle, chain_tip_sender: watch::Sender, grpc_options: GrpcOptionsInternal, rpc_listener: TcpListener, @@ -184,15 +186,18 @@ impl Store { ) -> anyhow::Result>> { let rpc_service = store::rpc_server::RpcServer::new(api::StoreApi { state: Arc::clone(&state), + write_handle: write_handle.clone(), chain_tip_sender: chain_tip_sender.clone(), }); let ntx_builder_service = store::ntx_builder_server::NtxBuilderServer::new(api::StoreApi { state: Arc::clone(&state), + write_handle: write_handle.clone(), chain_tip_sender: chain_tip_sender.clone(), }); let block_producer_service = store::block_producer_server::BlockProducerServer::new(api::StoreApi { state: Arc::clone(&state), + write_handle, chain_tip_sender, }); let reflection_service = tonic_reflection::server::Builder::configure() diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 394dd54ca..2d6a78836 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -15,7 +15,6 @@ use std::path::Path; use std::sync::Arc; use arc_swap::ArcSwap; -use miden_node_proto::BlockProofRequest; use miden_node_proto::domain::account::{ AccountDetailRequest, AccountDetails, @@ -37,12 +36,12 @@ use miden_protocol::account::{AccountId, StorageMapKey, StorageMapWitness, Stora use miden_protocol::asset::{AssetVaultKey, AssetWitness}; use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, SignedBlock}; +use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; use miden_protocol::crypto::merkle::mmr::{MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtProof}; use miden_protocol::note::{NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use tracing::{info, instrument}; use crate::account_state_forest::{AccountStateForest, WitnessError}; @@ -180,9 +179,10 @@ pub(crate) struct InMemoryState { /// /// A single writer task (serialized by a channel) mutates all three data sets. The writer owns /// writable copies of the in-memory trees directly (passed as owned values to -/// [`writer::writer_loop`]) and creates snapshot-backed read-only copies for [`InMemoryState`] -/// after each block. The writer commits to SQLite and the block store *before* swapping the -/// in-memory pointer, so there is a window where the DB/files are ahead of the in-memory state. +/// [`writer::BlockWriter::run`]) and creates snapshot-backed read-only copies for +/// [`InMemoryState`] after each block. The writer commits to SQLite and the block store *before* +/// swapping the in-memory pointer, so there is a window where the DB/files are ahead of the +/// in-memory state. /// /// ## Consistency rules for reader methods /// @@ -204,28 +204,14 @@ pub struct State { /// The block store which stores full block contents for all blocks. pub(super) block_store: Arc, - /// Handle to the `RocksDB` database used for account tree storage. - /// Used by the writer to create snapshot storage instances for `InMemoryState`. - #[cfg(feature = "rocksdb")] - pub(super) account_db: std::sync::Arc, - - /// Handle to the `RocksDB` database used for nullifier tree storage. - /// Used by the writer to create snapshot storage instances for `InMemoryState`. - #[cfg(feature = "rocksdb")] - pub(super) nullifier_db: std::sync::Arc, - /// All in-memory state held atomically behind an `ArcSwap`. /// /// Readers call `snapshot()` which returns `Arc` via a wait-free atomic /// refcount bump — no data cloning. The writer builds a new `InMemoryState` with /// snapshot-backed trees after each block and atomically swaps via `ArcSwap::store()`. - pub(super) in_memory: ArcSwap, - - /// Channel to the single writer task. - writer_tx: mpsc::Sender, - - /// Request termination of the process due to a fatal internal state error. - pub(super) termination_ask: tokio::sync::mpsc::Sender, + /// + /// Wrapped in `Arc` so the writer context can share the same `ArcSwap` instance. + pub(super) in_memory: Arc>, /// The latest proven-in-sequence block number, updated by the proof scheduler. proven_tip: ProvenTipReader, @@ -237,15 +223,15 @@ impl State { /// Loads the state from the data directory. /// - /// The returned `Arc` is ready to use. The writer task is spawned internally and - /// holds a clone of the `Arc`. Dropping all external clones and closing the writer channel - /// will terminate the writer task. + /// Returns `(Arc, WriteHandle, ProvenTipWriter)`. The `WriteHandle` is the only way + /// to submit blocks to the writer loop. The writer task is spawned internally; dropping the + /// `WriteHandle` closes the channel and terminates the writer task. #[instrument(target = COMPONENT, skip_all)] pub async fn load( data_path: &Path, storage_options: StorageOptions, termination_ask: tokio::sync::mpsc::Sender, - ) -> Result<(Arc, ProvenTipWriter), StateInitializationError> { + ) -> Result<(Arc, writer::WriteHandle, ProvenTipWriter), StateInitializationError> { let data_directory = DataDirectory::load(data_path.to_path_buf()) .map_err(StateInitializationError::DataDirectoryLoadError)?; @@ -348,37 +334,37 @@ impl State { // Create the writer channel. Buffer size of 1: only one block can be in flight. let (writer_tx, writer_rx) = mpsc::channel(1); - let in_memory = ArcSwap::from_pointee(InMemoryState { + let in_memory = Arc::new(ArcSwap::from_pointee(InMemoryState { block_num: latest_block_num, nullifier_tree: snapshot_nullifier_tree, account_tree: snapshot_account_tree, blockchain, forest, - }); + })); - let state = Arc::new(Self { - db, - block_store, + // Build the writer context. + let writer_ctx = writer::BlockWriter { + rx: writer_rx, + account_tree: account_tree_with_history, + nullifier_tree, + db: Arc::clone(&db), + block_store: Arc::clone(&block_store), + in_memory: Arc::clone(&in_memory), + termination_ask: termination_ask.clone(), #[cfg(feature = "rocksdb")] - account_db, + account_db: std::sync::Arc::clone(&account_db), #[cfg(feature = "rocksdb")] - nullifier_db, - in_memory, - writer_tx, - termination_ask, - proven_tip, - }); - - // Spawn the single writer task with owned writable trees. - let writer_state = Arc::clone(&state); - tokio::spawn(writer::writer_loop( - writer_rx, - writer_state, - nullifier_tree, - account_tree_with_history, - )); + nullifier_db: std::sync::Arc::clone(&nullifier_db), + }; + + let state = Arc::new(Self { db, block_store, in_memory, proven_tip }); - Ok((state, proven_tip_writer)) + // Spawn the single writer task. + tokio::spawn(writer_ctx.run()); + + let write_handle = writer::WriteHandle::new(writer_tx); + + Ok((state, write_handle, proven_tip_writer)) } /// Returns the database. @@ -391,27 +377,6 @@ impl State { Arc::clone(&self.block_store) } - // BLOCK APPLICATION - // -------------------------------------------------------------------------------------------- - - /// Apply changes of a new block to the DB and in-memory data structures. - /// - /// This sends the block to the single writer task via a channel and awaits the result. - /// The writer task handles all validation, DB writes, and in-memory mutations. - #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] - pub async fn apply_block( - &self, - signed_block: SignedBlock, - proving_inputs: Option, - ) -> Result<(), ApplyBlockError> { - let (result_tx, result_rx) = oneshot::channel(); - self.writer_tx - .send(writer::WriteRequest { signed_block, proving_inputs, result_tx }) - .await - .map_err(|e| ApplyBlockError::WriterTaskSendFailed(Box::new(e)))?; - result_rx.await? - } - // STATE ACCESSORS // -------------------------------------------------------------------------------------------- diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 20bf82ec3..289321995 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use miden_large_smt_backend_rocksdb::RocksDbStorage; +use arc_swap::ArcSwap; use miden_node_proto::BlockProofRequest; use miden_node_utils::ErrorReport; use miden_protocol::account::delta::AccountUpdateDetails; @@ -15,12 +15,83 @@ use tokio::sync::{mpsc, oneshot}; use tracing::{info, instrument}; use crate::accounts::AccountTreeWithHistory; -use crate::db::NoteRecord; +use crate::blocks::BlockStore; +use crate::db::{Db, NoteRecord}; use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::InMemoryState; use crate::state::loader::{SnapshotTreeStorage, TreeStorage}; -use crate::state::{InMemoryState, State}; use crate::{COMPONENT, HistoricalError}; +// WRITE HANDLE +// ================================================================================================ + +/// Handle for submitting blocks to the writer loop. +/// +/// This is intentionally separated from [`super::State`] to avoid a circular reference: the writer +/// loop must not hold a reference back to the sender's owner. +#[derive(Clone)] +pub struct WriteHandle { + tx: mpsc::Sender, +} + +impl WriteHandle { + pub(crate) fn new(tx: mpsc::Sender) -> Self { + Self { tx } + } + + /// Sends a block to the writer loop and awaits the result. + pub async fn apply_block( + &self, + signed_block: SignedBlock, + proving_inputs: Option, + ) -> Result<(), ApplyBlockError> { + let (result_tx, result_rx) = oneshot::channel(); + self.tx + .send(WriteRequest { signed_block, proving_inputs, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(Box::new(e)))?; + result_rx.await? + } +} + +// BLOCK WRITER +// ================================================================================================ + +/// Single writer task that serializes all block mutations. +/// +/// Owns the channel receiver and writable trees, and holds shared references to the database, +/// block store, and in-memory state. Deliberately does not reference `State` to avoid circular +/// references — the channel sender lives in [`WriteHandle`], which is independent of `State`. +pub(crate) struct BlockWriter { + /// Channel receiver for incoming block write requests. + pub rx: mpsc::Receiver, + + /// Writable account tree with historical overlays, owned exclusively by the writer. + pub account_tree: AccountTreeWithHistory, + /// Writable nullifier tree, owned exclusively by the writer. + pub nullifier_tree: NullifierTree>, + + /// Shared database for persisting blocks, accounts, notes, and nullifiers. + pub db: Arc, + /// Shared block store for persisting raw block data. + pub block_store: Arc, + /// Shared in-memory state. The writer publishes new snapshots via `ArcSwap::store()`. + pub in_memory: Arc>, + + /// Channel to request process termination on fatal internal state errors. + pub termination_ask: mpsc::Sender, + + /// Handle to the `RocksDB` database used to create snapshot-backed account trees. + #[cfg(feature = "rocksdb")] + pub account_db: std::sync::Arc, + /// Handle to the `RocksDB` database used to create snapshot-backed nullifier trees. + #[cfg(feature = "rocksdb")] + pub nullifier_db: std::sync::Arc, +} + +// WRITE REQUEST +// ================================================================================================ + /// A request to apply a new block, sent through the writer channel. pub struct WriteRequest { pub signed_block: SignedBlock, @@ -28,229 +99,257 @@ pub struct WriteRequest { pub result_tx: oneshot::Sender>, } -/// Runs the single writer loop. Receives blocks through the channel and applies them -/// sequentially. The writer owns the writable trees — no locks or interior mutability needed. -pub(crate) async fn writer_loop( - mut rx: mpsc::Receiver, - state: Arc, - mut nullifier_tree: NullifierTree>, - mut account_tree: AccountTreeWithHistory, -) { - while let Some(req) = rx.recv().await { - let result = Box::pin(apply_block_inner( - &state, - &mut nullifier_tree, - &mut account_tree, - req.signed_block, - req.proving_inputs, - )) - .await; - let _ = req.result_tx.send(result); +impl BlockWriter { + /// Runs the single writer loop. Receives blocks through the channel and applies them + /// sequentially. + pub(crate) async fn run(mut self) { + while let Some(req) = self.rx.recv().await { + let result = Box::pin(self.write_block(req.signed_block, req.proving_inputs)).await; + let _ = req.result_tx.send(result); + } } -} -/// Apply changes of a new block to the DB and in-memory data structures. -/// -/// ## Consistency model -/// -/// This function is the sole writer to all state. The writer owns the writable trees directly. -/// -/// Because SQLite/files are committed **before** the in-memory swap, there is a window where the -/// DB is ahead of the in-memory state. Reader methods that combine in-memory and SQLite data -/// must scope their DB queries by the snapshot's `block_num` to maintain consistency (see the -/// doc comment on [`State`] for the full rules). -/// -/// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only an -/// atomic refcount increment with no data cloning. The atomic swap guarantees readers see either -/// the old or new state, never a partial update. Readers holding an `Arc` to the old state are -/// completely unaffected by the swap. -#[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] -async fn apply_block_inner( - state: &State, - nullifier_tree: &mut NullifierTree>, - account_tree: &mut AccountTreeWithHistory, - signed_block: SignedBlock, - proving_inputs: Option, -) -> Result<(), ApplyBlockError> { - let header = signed_block.header(); - let body = signed_block.body(); - let block_num = header.block_num(); - let block_commitment = header.commitment(); - - validate_block_header(state, header, body).await?; - - // Load the current in-memory state snapshot for validation (wait-free). - let snapshot = state.in_memory.load_full(); - - // Compute mutations required for updating account and nullifier trees. - let (nullifier_tree_update, account_tree_update) = - compute_tree_mutations(state, &snapshot, header, body, nullifier_tree, account_tree)?; - - let notes = build_note_records(header, body)?; - - // Extract public account deltas before block is moved into the DB task. - let account_deltas = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Delta(delta) => Some(delta.clone()), - AccountUpdateDetails::Private => None, - }, - )); - - // Apply mutations to the writable trees (writes to RocksDB). - nullifier_tree - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - // Build new read-only snapshot-backed trees for the new in-memory state. - let snapshot_nullifier_tree = build_snapshot_nullifier_tree(state, nullifier_tree); - let snapshot_account_tree = build_snapshot_account_tree(state, account_tree); - - let mut new_blockchain = snapshot.blockchain.clone(); - new_blockchain.push(block_commitment); - - let mut new_forest = snapshot.forest.clone(); - new_forest.apply_block_updates(block_num, account_deltas)?; - - let new_state = InMemoryState { - block_num, - nullifier_tree: snapshot_nullifier_tree, - account_tree: snapshot_account_tree, - blockchain: new_blockchain, - forest: new_forest, - }; - - // We have completed all in-memory mutations on the new clone of in-memory state. Now commit to - // storage before swapping the Arc. - - // Save the block to the block store. - let signed_block_bytes = signed_block.to_bytes(); - state.block_store.save_block(block_num, &signed_block_bytes).await?; - - // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while - // the DB commits. We ensure consistency by scoping all RPC queries that hit DB data by the - // block number that is Arc swapped at the end of this function. - state - .db - .apply_block(signed_block, notes, proving_inputs) - .await - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // Atomically publish the new state. Readers that call snapshot() after this point - // will see the updated state. Readers holding the old Arc continue unaffected. - state.in_memory.store(Arc::new(new_state)); - - info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); - - Ok(()) -} + /// Apply changes of a new block to the DB, file, and in-memory data structures. + /// + /// ## Consistency model + /// + /// This function is the sole writer to all state. The writer owns the writable trees directly. + /// + /// Because SQLite/files are committed **before** the in-memory swap, there is a window where + /// the DB is ahead of the in-memory state. Reader methods that combine in-memory and SQLite + /// data must scope their DB queries by the snapshot's `block_num` to maintain consistency + /// (see the doc comment on [`super::State`] for the full rules). + /// + /// Readers never block: they obtain an `Arc` via `ArcSwap::load_full()`, which performs only + /// an atomic refcount increment with no data cloning. The atomic swap guarantees readers see + /// either the old or new state, never a partial update. Readers holding an `Arc` to the old + /// state are completely unaffected by the swap. + #[instrument(target = COMPONENT, skip_all, err, fields(block.number = signed_block.header().block_num().as_u32()))] + async fn write_block( + &mut self, + signed_block: SignedBlock, + proving_inputs: Option, + ) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + let block_num = header.block_num(); + let block_commitment = header.commitment(); + + self.validate_block_header(header, body).await?; + + // Load the current in-memory state snapshot for validation (wait-free). + let snapshot = self.in_memory.load_full(); + + // Compute mutations required for updating account and nullifier trees. + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(&snapshot, header, body)?; + + let notes = build_note_records(header, body)?; + + // Extract public account deltas before block is moved into the DB task. + let account_deltas = + Vec::from_iter(body.updated_accounts().iter().filter_map( + |update| match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + }, + )); -/// Validates that the block header is consistent with the body and follows the previous block. -async fn validate_block_header( - state: &State, - header: &BlockHeader, - body: &BlockBody, -) -> Result<(), ApplyBlockError> { - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); + // Apply mutations to the writable trees (writes to RocksDB). + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + self.account_tree + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + // Build new read-only snapshot-backed trees for the new in-memory state. + let snapshot_nullifier_tree = self.build_snapshot_nullifier_tree(); + let snapshot_account_tree = self.build_snapshot_account_tree(); + + let mut new_blockchain = snapshot.blockchain.clone(); + new_blockchain.push(block_commitment); + + let mut new_forest = snapshot.forest.clone(); + new_forest.apply_block_updates(block_num, account_deltas)?; + + let new_state = InMemoryState { + block_num, + nullifier_tree: snapshot_nullifier_tree, + account_tree: snapshot_account_tree, + blockchain: new_blockchain, + forest: new_forest, + }; + + // We have completed all in-memory mutations on the new clone of in-memory state. Now + // commit to storage before swapping the Arc. + + // Save the block to the block store. + let signed_block_bytes = signed_block.to_bytes(); + self.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to DB. Readers continue to see the old in-memory state (via their Arc) while + // the DB commits. We ensure consistency by scoping all RPC queries that hit DB data by + // the block number that is Arc swapped at the end of this function. + self.db + .apply_block(signed_block, notes, proving_inputs) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // Atomically publish the new state. Readers that call snapshot() after this point + // will see the updated state. Readers holding the old Arc continue unaffected. + self.in_memory.store(Arc::new(new_state)); + + info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); + + Ok(()) } - let block_num = header.block_num(); - let prev_block = state - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, + /// Validates that the block header is consistent with the body and follows the previous block. + async fn validate_block_header( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(), ApplyBlockError> { + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - Ok(()) -} + let block_num = header.block_num(); + let prev_block = self + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } -/// Compute mutations for the nullifier tree and account tree. -fn compute_tree_mutations( - state: &State, - snapshot: &Arc, - header: &BlockHeader, - body: &BlockBody, - nullifier_tree: &mut NullifierTree>, - account_tree: &mut AccountTreeWithHistory, -) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { - // Nullifiers can be produced only once. - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + Ok(()) } - // new_block.chain_root must be equal to the chain MMR root prior to the update. - let peaks = snapshot.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } + /// Compute mutations for the nullifier tree and account tree. + fn compute_tree_mutations( + &mut self, + snapshot: &Arc, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + // Nullifiers can be produced only once. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // new_block.chain_root must be equal to the chain MMR root prior to the update. + let peaks = snapshot.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // Compute update for nullifier tree. + let nullifier_tree_update = self + .nullifier_tree + .compute_mutations( + body.created_nullifiers() + .iter() + .map(|nullifier| (*nullifier, header.block_num())), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + let _ = self.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidNullifierRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // Compute update for account tree from the writable tree (always in sync with DB). + let account_tree_update = self + .account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + let _ = self.termination_ask.try_send(ApplyBlockError::InvalidBlockError( + InvalidBlockError::NewBlockInvalidAccountRoot, + )); + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } - // Compute update for nullifier tree. - let nullifier_tree_update = nullifier_tree - .compute_mutations( - body.created_nullifiers() - .iter() - .map(|nullifier| (*nullifier, header.block_num())), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidNullifierRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + Ok((nullifier_tree_update, account_tree_update)) } - // Compute update for account tree from the writable tree (always in sync with DB). - let account_tree_update = account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - let _ = state.termination_ask.try_send(ApplyBlockError::InvalidBlockError( - InvalidBlockError::NewBlockInvalidAccountRoot, - )); - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + /// Builds a snapshot-backed nullifier tree for the new in-memory state. + fn build_snapshot_nullifier_tree(&self) -> NullifierTree> { + #[cfg(feature = "rocksdb")] + { + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&self.nullifier_db), + ); + let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + NullifierTree::new_unchecked(snapshot_smt) + } + #[cfg(not(feature = "rocksdb"))] + { + self.nullifier_tree.clone() + } } - Ok((nullifier_tree_update, account_tree_update)) + /// Builds a snapshot-backed account tree for the new in-memory state. + fn build_snapshot_account_tree(&self) -> AccountTreeWithHistory { + #[cfg(feature = "rocksdb")] + { + let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( + std::sync::Arc::clone(&self.account_db), + ); + let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) + .expect("Unreachable: snapshot reads from data just written by apply_mutations"); + let snapshot_tree = + miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); + + AccountTreeWithHistory::from_parts( + snapshot_tree, + self.account_tree.block_number_latest(), + self.account_tree.overlays().clone(), + ) + } + #[cfg(not(feature = "rocksdb"))] + { + self.account_tree.clone() + } + } } /// Builds the note tree, validates its root against the header, and collects note records. @@ -291,53 +390,3 @@ fn build_note_records( .collect::, InvalidBlockError>>() .map_err(Into::into) } - -/// Builds a snapshot-backed nullifier tree for the new in-memory state. -fn build_snapshot_nullifier_tree( - state: &State, - nullifier_tree: &NullifierTree>, -) -> NullifierTree> { - #[cfg(feature = "rocksdb")] - { - let _ = nullifier_tree; - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&state.nullifier_db), - ); - let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - NullifierTree::new_unchecked(snapshot_smt) - } - #[cfg(not(feature = "rocksdb"))] - { - let _ = state; - nullifier_tree.clone() - } -} - -/// Builds a snapshot-backed account tree for the new in-memory state. -fn build_snapshot_account_tree( - state: &State, - account_tree: &AccountTreeWithHistory, -) -> AccountTreeWithHistory { - #[cfg(feature = "rocksdb")] - { - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&state.account_db), - ); - let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - let snapshot_tree = - miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); - - AccountTreeWithHistory::from_parts( - snapshot_tree, - account_tree.block_number_latest(), - account_tree.overlays().clone(), - ) - } - #[cfg(not(feature = "rocksdb"))] - { - let _ = state; - account_tree.clone() - } -} From 8f351e0e83281c61ec2af0dbfb82ea3c31989bed Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 16 Apr 2026 09:10:09 +1200 Subject: [PATCH 45/48] Update deps --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 441964a52..aa8379d35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -188,7 +188,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1653,7 +1653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2845,7 +2845,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "alloy-sol-types", "fs-err", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" dependencies = [ "blake3", "cc", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" dependencies = [ "quote", "syn 2.0.117", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "bech32", "fs-err", @@ -3566,7 +3566,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "proc-macro2", "quote", @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#22c1b78f47be84f42a189651c589fe02c8cec8ab" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" dependencies = [ "p3-field", "p3-goldilocks", @@ -3661,7 +3661,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "fs-err", "miden-assembly", @@ -3678,7 +3678,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3700,7 +3700,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "miden-processor", "miden-protocol", @@ -3713,7 +3713,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#172df00d27c13f82274b6db16540e82060b0eb08" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" dependencies = [ "miden-protocol", "miden-tx", @@ -3927,7 +3927,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5342,7 +5342,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5413,7 +5413,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5820,7 +5820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6035,7 +6035,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6044,7 +6044,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6063,7 +6063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7073,7 +7073,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From cc0b4ee6667f2b55d3321a978e2030749efcebe6 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 16 Apr 2026 09:21:21 +1200 Subject: [PATCH 46/48] Update deps --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa8379d35..29b37cff0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -188,7 +188,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1653,7 +1653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2845,7 +2845,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "alloy-sol-types", "fs-err", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" dependencies = [ "blake3", "cc", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" dependencies = [ "quote", "syn 2.0.117", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "bech32", "fs-err", @@ -3566,7 +3566,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "proc-macro2", "quote", @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#83ce9f383747927b53ca9a132a7a3492e9aa85f5" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" dependencies = [ "p3-field", "p3-goldilocks", @@ -3661,7 +3661,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "fs-err", "miden-assembly", @@ -3678,7 +3678,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3700,7 +3700,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "miden-processor", "miden-protocol", @@ -3713,7 +3713,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#99feef0004f9489508216b8cadc67050bffe9c2a" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" dependencies = [ "miden-protocol", "miden-tx", @@ -3927,7 +3927,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5342,7 +5342,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5413,7 +5413,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5820,7 +5820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6035,7 +6035,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6044,7 +6044,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6063,7 +6063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7073,7 +7073,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 02689973e4f5a297aa71f946a7556f3d779dbc91 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 16 Apr 2026 20:28:51 +1200 Subject: [PATCH 47/48] Use reader() and remove unnecessary db handling --- Cargo.lock | 46 +++++++++---------- .../large-smt-backend-rocksdb/src/rocksdb.rs | 9 +++- crates/store/src/accounts/mod.rs | 19 ++++++++ crates/store/src/state/mod.rs | 4 -- crates/store/src/state/writer.rs | 28 +---------- 5 files changed, 52 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29b37cff0..64d683efe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -188,7 +188,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1653,7 +1653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2845,7 +2845,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miden-agglayer" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "alloy-sol-types", "fs-err", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "miden-block-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "miden-protocol", "thiserror 2.0.18", @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "miden-crypto" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4441d9c2ab43ae3e9c8dcf1544d3ad4bb2001407" dependencies = [ "blake3", "cc", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "miden-crypto-derive" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4441d9c2ab43ae3e9c8dcf1544d3ad4bb2001407" dependencies = [ "quote", "syn 2.0.117", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "miden-field" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4441d9c2ab43ae3e9c8dcf1544d3ad4bb2001407" dependencies = [ "miden-serde-utils", "num-bigint", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "bech32", "fs-err", @@ -3566,7 +3566,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "proc-macro2", "quote", @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "miden-serde-utils" version = "0.23.0" -source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4661f8507e756653b89ff09cbf15e772185e06d6" +source = "git+https://github.com/0xmiden/crypto?branch=sergerad-largesmt-reader-trait#4441d9c2ab43ae3e9c8dcf1544d3ad4bb2001407" dependencies = [ "p3-field", "p3-goldilocks", @@ -3661,7 +3661,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "fs-err", "miden-assembly", @@ -3678,7 +3678,7 @@ dependencies = [ [[package]] name = "miden-testing" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3700,7 +3700,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "miden-processor", "miden-protocol", @@ -3713,7 +3713,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.15.0" -source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#25b84a04b1c92c1ebbff9f57bf23769dbcd6d52c" +source = "git+https://github.com/0xmiden/protocol?branch=sergerad-clone#b82601603cc329f42d9da782e980e5afeaba9a48" dependencies = [ "miden-protocol", "miden-tx", @@ -3927,7 +3927,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5342,7 +5342,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5413,7 +5413,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5820,7 +5820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6035,7 +6035,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6044,7 +6044,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6063,7 +6063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7073,7 +7073,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs index 22a6f931b..7ded1e643 100644 --- a/crates/large-smt-backend-rocksdb/src/rocksdb.rs +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -43,7 +43,7 @@ use crate::helpers::{ read_subtree_batch, remove_from_leaf, }; -use crate::{EMPTY_WORD, Word}; +use crate::{EMPTY_WORD, RocksDbSnapshotStorage, Word}; pub(crate) const IN_MEMORY_DEPTH: u8 = 24; @@ -494,6 +494,8 @@ impl SmtStorageReader for RocksDbStorage { } impl SmtStorage for RocksDbStorage { + type Reader = RocksDbSnapshotStorage; + /// Inserts a key-value pair into the SMT leaf at the specified logical `index`. /// /// This operation involves: @@ -922,6 +924,11 @@ impl SmtStorage for RocksDbStorage { Ok(()) } + + /// Returns the read-only snapshot storage. + fn reader(&self) -> Self::Reader { + self.snapshot_storage() + } } /// Syncs the RocksDB database to disk before dropping the storage. diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 3a2d2be60..f43d1c8cd 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -408,3 +408,22 @@ impl AccountTreeWithHistory { Ok(()) } } + +impl AccountTreeWithHistory +where + S: SmtStorage, +{ + /// Returns a read-only `AccountTreeWithHistory` backed by a reader view of this tree's + /// storage. + /// + /// The returned tree shares the same block number and historical overlays as `self`, and its + /// latest `AccountTree` is produced by [`AccountTree::reader`]. The returned tree's storage + /// type is `S::Reader: SmtStorageReader`, so it cannot be used for mutations. + pub fn reader(&self) -> AccountTreeWithHistory { + AccountTreeWithHistory { + block_number: self.block_number, + latest: self.latest.reader(), + overlays: self.overlays.clone(), + } + } +} diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 2d6a78836..83eccecc9 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -351,10 +351,6 @@ impl State { block_store: Arc::clone(&block_store), in_memory: Arc::clone(&in_memory), termination_ask: termination_ask.clone(), - #[cfg(feature = "rocksdb")] - account_db: std::sync::Arc::clone(&account_db), - #[cfg(feature = "rocksdb")] - nullifier_db: std::sync::Arc::clone(&nullifier_db), }; let state = Arc::new(Self { db, block_store, in_memory, proven_tip }); diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 289321995..36d3dfafb 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -80,13 +80,6 @@ pub(crate) struct BlockWriter { /// Channel to request process termination on fatal internal state errors. pub termination_ask: mpsc::Sender, - - /// Handle to the `RocksDB` database used to create snapshot-backed account trees. - #[cfg(feature = "rocksdb")] - pub account_db: std::sync::Arc, - /// Handle to the `RocksDB` database used to create snapshot-backed nullifier trees. - #[cfg(feature = "rocksdb")] - pub nullifier_db: std::sync::Arc, } // WRITE REQUEST @@ -314,12 +307,7 @@ impl BlockWriter { fn build_snapshot_nullifier_tree(&self) -> NullifierTree> { #[cfg(feature = "rocksdb")] { - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&self.nullifier_db), - ); - let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - NullifierTree::new_unchecked(snapshot_smt) + self.nullifier_tree.reader() } #[cfg(not(feature = "rocksdb"))] { @@ -331,19 +319,7 @@ impl BlockWriter { fn build_snapshot_account_tree(&self) -> AccountTreeWithHistory { #[cfg(feature = "rocksdb")] { - let snapshot_storage = miden_large_smt_backend_rocksdb::RocksDbSnapshotStorage::new( - std::sync::Arc::clone(&self.account_db), - ); - let snapshot_smt = crate::state::loader::load_smt(snapshot_storage) - .expect("Unreachable: snapshot reads from data just written by apply_mutations"); - let snapshot_tree = - miden_protocol::block::account_tree::AccountTree::new_unchecked(snapshot_smt); - - AccountTreeWithHistory::from_parts( - snapshot_tree, - self.account_tree.block_number_latest(), - self.account_tree.overlays().clone(), - ) + self.account_tree.reader() } #[cfg(not(feature = "rocksdb"))] { From 45c5bc005d86b80272378651677a8cd5a58f77ff Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 17 Apr 2026 13:33:31 +1200 Subject: [PATCH 48/48] block_in_place in write_block --- crates/store/src/state/writer.rs | 58 +++++++++++++++++--------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 36d3dfafb..4786406e7 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -133,33 +133,37 @@ impl BlockWriter { // Load the current in-memory state snapshot for validation (wait-free). let snapshot = self.in_memory.load_full(); - // Compute mutations required for updating account and nullifier trees. - let (nullifier_tree_update, account_tree_update) = - self.compute_tree_mutations(&snapshot, header, body)?; - - let notes = build_note_records(header, body)?; - - // Extract public account deltas before block is moved into the DB task. - let account_deltas = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Delta(delta) => Some(delta.clone()), - AccountUpdateDetails::Private => None, - }, - )); - - // Apply mutations to the writable trees (writes to RocksDB). - self.nullifier_tree - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - self.account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: mutations were computed from the current tree state"); - - // Build new read-only snapshot-backed trees for the new in-memory state. - let snapshot_nullifier_tree = self.build_snapshot_nullifier_tree(); - let snapshot_account_tree = self.build_snapshot_account_tree(); + // Tree mutation, RocksDB writes, and snapshot construction are all CPU- and I/O-bound + // synchronous workloads. Run them inside block_in_place so tokio can evacuate other tasks + // from this thread for the duration. + let (snapshot_nullifier_tree, snapshot_account_tree, notes, account_deltas) = + tokio::task::block_in_place(|| -> Result<_, ApplyBlockError> { + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(&snapshot, header, body)?; + + let notes = build_note_records(header, body)?; + + let account_deltas = + Vec::from_iter(body.updated_accounts().iter().filter_map(|update| { + match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + } + })); + + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + self.account_tree + .apply_mutations(account_tree_update) + .expect("Unreachable: mutations were computed from the current tree state"); + + let snapshot_nullifier_tree = self.build_snapshot_nullifier_tree(); + let snapshot_account_tree = self.build_snapshot_account_tree(); + + Ok((snapshot_nullifier_tree, snapshot_account_tree, notes, account_deltas)) + })?; let mut new_blockchain = snapshot.blockchain.clone(); new_blockchain.push(block_commitment);