From eccb24c5017129d534097a33b6d0ad3bb2c19820 Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:09:48 -0400 Subject: [PATCH 1/4] working dynamic dust filter for tweaks --- rust-toolchain.toml | 2 +- src/electrum/server.rs | 9 +++-- src/new_index/query.rs | 7 ++++ src/new_index/schema.rs | 80 +++++++++++++++++++++++++++++++++-------- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 64aead883..0be45db79 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.75.0" +channel = "1.88.0" components = [ "cargo", "clippy", diff --git a/src/electrum/server.rs b/src/electrum/server.rs index e518f42ea..04982e7ea 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -314,6 +314,11 @@ impl Connection { // let _historical_mode = // bool_from_value_or(params.get(2), "historical", false).unwrap_or(false); + // Parse optional dust_limit parameter + let dust_limit: Option = params.get(1) + .map(|v| v.as_u64()) + .flatten(); + let sp_begin_height = self.query.sp_begin_height(); // let last_header_entry = self.query.chain().best_header(); @@ -323,7 +328,7 @@ impl Connection { height }; - let tweaks = self.query.block_tweaks(scan_height); + let tweaks = self.query.block_tweaks_with_dust_limit(scan_height, dust_limit)?; Ok(json!(tweaks)) } @@ -607,7 +612,7 @@ impl Connection { let result = match method { "blockchain.block.header" => self.blockchain_block_header(¶ms), "blockchain.block.headers" => self.blockchain_block_headers(¶ms), - "blockchain.block.tweaks" => self.blockchain_block_tweaks(params), + "blockchain.block.tweaks" => self.blockchain_block_tweaks(¶ms), "blockchain.tweaks.subscribe" => self.tweaks_subscribe(params), // "blockchain.tweaks.register" => self.tweaks_subscribe(params), // "blockchain.tweaks.erase" => self.tweaks_subscribe(params), diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 175f8ed9b..10afdf20a 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -117,6 +117,13 @@ impl Query { .get_block_tweaks(&self.chain.hash_by_height(height as usize).unwrap()) } + pub fn block_tweaks_with_dust_limit(&self, height: u32, dust_limit: Option) -> Result> { + let block_hash = self.chain + .hash_by_height(height as usize) + .chain_err(|| format!("block not found at height {}", height))?; + Ok(self.chain.get_block_tweaks_with_dust_limit(&block_hash, dust_limit)) + } + pub fn tweaks_iter_scan_reverse(&self, height: u32) -> ReverseScanIterator { self.chain.tweaks_iter_scan_reverse(height) } diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index f8d29a6ee..c1e1fb194 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -581,7 +581,7 @@ impl Indexer { let amount = (txo.value as Amount).to_sat(); #[allow(deprecated)] if txo.script_pubkey.is_v1_p2tr() - && amount >= self.iconfig.sp_min_dust.unwrap_or(1_000) as u64 + && amount >= self.iconfig.sp_min_dust.unwrap_or(0) as u64 { output_pubkeys.push(VoutData { vout: txo_index, @@ -634,19 +634,39 @@ impl Indexer { let pubkeys_ref: Vec<_> = pubkeys.iter().collect(); if !pubkeys_ref.is_empty() { if let Some(tweak) = calculate_tweak_data(&pubkeys_ref, &outpoints).ok() { - // persist tweak index: - // K{blockhash}{txid} → {tweak}{serialized-vout-data} + let tweak_hex = tweak.serialize().to_lower_hex_string(); + + // persist detailed tweak index: + // K{blockheight}{txid} → {tweak}{serialized-vout-data} rows.push( TweakTxRow::new( blockheight, txid.clone(), &TweakData { - tweak: tweak.serialize().to_lower_hex_string(), + tweak: tweak_hex.clone(), vout_data: output_pubkeys.clone(), }, ) .into_row(), ); + + // Calculate max output amount for efficient dust filtering + // let max_output_amount = output_pubkeys.iter() + // .map(|vout| vout.amount) + // .max() + // .unwrap_or(0); + // // persist summary tweak index for efficient dust filtering: + // // S{blockheight}{txid} → {tweak}{max_output_amount} + // rows.push( + // TweakSummaryRow::new( + // blockheight, + // txid.clone(), + // tweak_hex, + // max_output_amount, + // ) + // .into_row(), + // ); + tweaks.push(tweak.serialize().to_vec()); } } @@ -1137,19 +1157,51 @@ impl ChainQuery { } pub fn get_block_tweaks(&self, hash: &BlockHash) -> Vec { + self.get_block_tweaks_with_dust_limit(hash, None) + } + + pub fn get_block_tweaks_with_dust_limit(&self, hash: &BlockHash, min_dust: Option) -> Vec { let _timer = self.start_timer("get_block_tweaks"); - let tweaks: Vec> = self - .store - .tweak_db - .get(&BlockRow::tweaks_key(full_hash(&hash[..]))) - .map(|val| bincode::deserialize_little(&val).expect("failed to parse block tweaks")) - .unwrap(); + // If no dust limit specified, use the fast path (existing behavior) + if min_dust.is_none() || min_dust == Some(0) { + let tweaks: Vec> = self + .store + .tweak_db + .get(&BlockRow::tweaks_key(full_hash(&hash[..]))) + .map(|val| bincode::deserialize_little(&val).expect("failed to parse block tweaks")) + .unwrap_or_default(); + + return tweaks + .into_iter() + .map(|tweak| tweak.to_lower_hex_string()) + .collect(); + } - tweaks - .into_iter() - .map(|tweak| tweak.to_lower_hex_string()) - .collect() + // Dust filtering path: use lightweight summary rows for efficient filtering + let block_height = self.height_by_hash(hash); + if block_height.is_none() { + return vec![]; + } + let block_height = block_height.unwrap() as u32; + + let min_dust_value = min_dust.unwrap(); + let mut filtered_tweaks = Vec::new(); + + // Use prefix scan to only read TweakTxRow entries for this specific block height + let prefix = TweakTxRow::prefix_blockheight(block_height); // [b'K', height_bytes] + let tweak_iter = self.store.tweak_db.iter_scan(&prefix) // Only scan K{height}* entries + .map(TweakTxRow::from_row) + .take_while(|row| row.key.blockheight == block_height); + + for tweak_row in tweak_iter { + let tweak_data = tweak_row.get_tweak_data(); + if tweak_data.vout_data.iter().any(|vout| vout.amount >= min_dust_value) { + filtered_tweaks.push(tweak_data.tweak); + } + } + + filtered_tweaks } pub fn hash_by_height(&self, height: usize) -> Option { From 782c7ab21c3c5e71c8ce7c4a87fa8300254361ae Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Tue, 12 Aug 2025 19:58:26 -0400 Subject: [PATCH 2/4] working dynamic filter_spent for tweaks --- src/electrum/server.rs | 5 ++- src/new_index/query.rs | 5 +-- src/new_index/schema.rs | 67 +++++++++++++++++++++++++++-------------- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 04982e7ea..98ce5d779 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -319,6 +319,9 @@ impl Connection { .map(|v| v.as_u64()) .flatten(); + // Parse optional exclude_spent parameter + let filter_spent: bool = bool_from_value_or(params.get(2), "filter_spent", false)?; + let sp_begin_height = self.query.sp_begin_height(); // let last_header_entry = self.query.chain().best_header(); @@ -328,7 +331,7 @@ impl Connection { height }; - let tweaks = self.query.block_tweaks_with_dust_limit(scan_height, dust_limit)?; + let tweaks = self.query.block_tweaks_with_filters(scan_height, dust_limit, filter_spent)?; Ok(json!(tweaks)) } diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 10afdf20a..7721adf89 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -117,11 +117,12 @@ impl Query { .get_block_tweaks(&self.chain.hash_by_height(height as usize).unwrap()) } - pub fn block_tweaks_with_dust_limit(&self, height: u32, dust_limit: Option) -> Result> { + pub fn block_tweaks_with_filters(&self, height: u32, dust_limit: Option, filter_spent: bool) -> Result> { let block_hash = self.chain .hash_by_height(height as usize) .chain_err(|| format!("block not found at height {}", height))?; - Ok(self.chain.get_block_tweaks_with_dust_limit(&block_hash, dust_limit)) + // Note: Parameter name was changed to filter_spent in schema.rs + Ok(self.chain.get_block_tweaks_with_filters(&block_hash, dust_limit, filter_spent)) } pub fn tweaks_iter_scan_reverse(&self, height: u32) -> ReverseScanIterator { diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index c1e1fb194..eb4fea6cf 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -1157,53 +1157,74 @@ impl ChainQuery { } pub fn get_block_tweaks(&self, hash: &BlockHash) -> Vec { - self.get_block_tweaks_with_dust_limit(hash, None) + self.get_block_tweaks_with_filters(hash, None, false) } - pub fn get_block_tweaks_with_dust_limit(&self, hash: &BlockHash, min_dust: Option) -> Vec { + pub fn get_block_tweaks_with_filters(&self, hash: &BlockHash, min_dust: Option, filter_spent: bool) -> Vec { let _timer = self.start_timer("get_block_tweaks"); - // If no dust limit specified, use the fast path (existing behavior) - if min_dust.is_none() || min_dust == Some(0) { - let tweaks: Vec> = self - .store - .tweak_db - .get(&BlockRow::tweaks_key(full_hash(&hash[..]))) - .map(|val| bincode::deserialize_little(&val).expect("failed to parse block tweaks")) - .unwrap_or_default(); - - return tweaks - .into_iter() - .map(|tweak| tweak.to_lower_hex_string()) - .collect(); + // If no filtering needed, use the fast path (existing behavior) + if (min_dust.is_none() || min_dust == Some(0)) && !filter_spent { + return self.get_block_tweaks_fast_path(hash); } - // Dust filtering path: use lightweight summary rows for efficient filtering + // Filtering path: scan TweakTxRow entries for this block let block_height = self.height_by_hash(hash); if block_height.is_none() { return vec![]; } let block_height = block_height.unwrap() as u32; - let min_dust_value = min_dust.unwrap(); let mut filtered_tweaks = Vec::new(); - - // Use prefix scan to only read TweakTxRow entries for this specific block height - let prefix = TweakTxRow::prefix_blockheight(block_height); // [b'K', height_bytes] - let tweak_iter = self.store.tweak_db.iter_scan(&prefix) // Only scan K{height}* entries + let prefix = TweakTxRow::prefix_blockheight(block_height); + let tweak_iter = self.store.tweak_db.iter_scan(&prefix) .map(TweakTxRow::from_row) .take_while(|row| row.key.blockheight == block_height); for tweak_row in tweak_iter { let tweak_data = tweak_row.get_tweak_data(); - if tweak_data.vout_data.iter().any(|vout| vout.amount >= min_dust_value) { - filtered_tweaks.push(tweak_data.tweak); + + // Apply dust filter if specified + if let Some(min_dust_value) = min_dust { + if !tweak_data.vout_data.iter().any(|vout| vout.amount >= min_dust_value) { + continue; // Skip this tweak - no outputs above dust limit + } + } + + // Apply spent filter if specified + if filter_spent { + let has_unspent_output = tweak_data.vout_data.iter().any(|vout| { + let outpoint = OutPoint { + txid: tweak_row.key.txid, + vout: vout.vout as u32, + }; + // On-demand spend lookup - if lookup_spend returns None, the output is unspent + self.lookup_spend(&outpoint).is_none() + }); + + if !has_unspent_output { + continue; // Skip this tweak - all outputs are spent + } } + + filtered_tweaks.push(tweak_data.tweak); } filtered_tweaks } + // Fast path for backward compatibility + fn get_block_tweaks_fast_path(&self, hash: &BlockHash) -> Vec { + let tweaks: Vec> = self + .store + .tweak_db + .get(&BlockRow::tweaks_key(full_hash(&hash[..]))) + .map(|val| bincode::deserialize_little(&val).expect("failed to parse block tweaks")) + .unwrap_or_default(); + + tweaks.into_iter().map(|tweak| tweak.to_lower_hex_string()).collect() + } + pub fn hash_by_height(&self, height: usize) -> Option { self.store .indexed_headers From e59f09908ab5606d3f76f3ad55515f3c22731ac2 Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:26:39 -0400 Subject: [PATCH 3/4] cleanup / remove unnecessary changes --- src/electrum/server.rs | 2 +- src/new_index/schema.rs | 21 +-------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 98ce5d779..8c6f89a76 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -615,7 +615,7 @@ impl Connection { let result = match method { "blockchain.block.header" => self.blockchain_block_header(¶ms), "blockchain.block.headers" => self.blockchain_block_headers(¶ms), - "blockchain.block.tweaks" => self.blockchain_block_tweaks(¶ms), + "blockchain.block.tweaks" => self.blockchain_block_tweaks(params), "blockchain.tweaks.subscribe" => self.tweaks_subscribe(params), // "blockchain.tweaks.register" => self.tweaks_subscribe(params), // "blockchain.tweaks.erase" => self.tweaks_subscribe(params), diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index eb4fea6cf..0b62b0f12 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -634,8 +634,6 @@ impl Indexer { let pubkeys_ref: Vec<_> = pubkeys.iter().collect(); if !pubkeys_ref.is_empty() { if let Some(tweak) = calculate_tweak_data(&pubkeys_ref, &outpoints).ok() { - let tweak_hex = tweak.serialize().to_lower_hex_string(); - // persist detailed tweak index: // K{blockheight}{txid} → {tweak}{serialized-vout-data} rows.push( @@ -643,30 +641,13 @@ impl Indexer { blockheight, txid.clone(), &TweakData { - tweak: tweak_hex.clone(), + tweak: tweak.serialize().to_lower_hex_string(), vout_data: output_pubkeys.clone(), }, ) .into_row(), ); - // Calculate max output amount for efficient dust filtering - // let max_output_amount = output_pubkeys.iter() - // .map(|vout| vout.amount) - // .max() - // .unwrap_or(0); - // // persist summary tweak index for efficient dust filtering: - // // S{blockheight}{txid} → {tweak}{max_output_amount} - // rows.push( - // TweakSummaryRow::new( - // blockheight, - // txid.clone(), - // tweak_hex, - // max_output_amount, - // ) - // .into_row(), - // ); - tweaks.push(tweak.serialize().to_vec()); } } From e177f56842faf93776f1575a8ad88c809b435f92 Mon Sep 17 00:00:00 2001 From: macgyver13 <4712150+macgyver13@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:01:30 -0400 Subject: [PATCH 4/4] another minor cleanup --- src/new_index/schema.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 0b62b0f12..5f2e8ee61 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -1201,9 +1201,12 @@ impl ChainQuery { .tweak_db .get(&BlockRow::tweaks_key(full_hash(&hash[..]))) .map(|val| bincode::deserialize_little(&val).expect("failed to parse block tweaks")) - .unwrap_or_default(); + .unwrap(); - tweaks.into_iter().map(|tweak| tweak.to_lower_hex_string()).collect() + tweaks + .into_iter() + .map(|tweak| tweak.to_lower_hex_string()) + .collect() } pub fn hash_by_height(&self, height: usize) -> Option {