diff --git a/.gitignore b/.gitignore index 8b08ddc6..03d035b2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ Cargo.lock .idea/ docs/ +.claude/*.local.* +CLAUDE.local.md diff --git a/Cargo.toml b/Cargo.toml index 4e267647..4fb31b7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ signet-node-tests = { version = "0.17.0", path = "crates/node-tests" } signet-node-types = { version = "0.17.0", path = "crates/node-types" } signet-rpc = { version = "0.17.0", path = "crates/rpc" } -init4-bin-base = { version = "0.19.0", features = ["alloy"] } +init4-bin-base = { version = "0.19.1", features = ["alloy"] } signet-bundle = "0.16.0" signet-constants = "0.16.0" @@ -56,12 +56,12 @@ signet-tx-cache = "0.16.0" signet-types = "0.16.0" signet-zenith = "0.16.0" signet-journal = "0.16.0" -signet-storage = "0.6.5" -signet-cold = "0.6.5" -signet-hot = "0.6.5" -signet-hot-mdbx = "0.6.5" -signet-cold-mdbx = "0.6.5" -signet-storage-types = "0.6.5" +signet-storage = "0.7" +signet-cold = "0.7" +signet-hot = "0.7" +signet-hot-mdbx = "0.7" +signet-cold-mdbx = "0.7" +signet-storage-types = "0.7" # ajj ajj = "0.7.0" @@ -116,5 +116,3 @@ url = "2.5.4" # Test Utils tempfile = "3.17.0" - -# init4-bin-base = { path = "../shared" } diff --git a/crates/node-config/Cargo.toml b/crates/node-config/Cargo.toml index 3dc100f7..9a1a34cb 100644 --- a/crates/node-config/Cargo.toml +++ b/crates/node-config/Cargo.toml @@ -28,5 +28,5 @@ signet-genesis.workspace = true [features] test_utils = [] -postgres = ["signet-storage/postgres"] -sqlite = ["signet-storage/sqlite"] +postgres = ["signet-storage/postgres", "init4-bin-base/cold-sql"] +sqlite = ["signet-storage/sqlite", "init4-bin-base/cold-sql"] diff --git a/crates/node-config/src/storage.rs b/crates/node-config/src/storage.rs index f26bee66..6270b565 100644 --- a/crates/node-config/src/storage.rs +++ b/crates/node-config/src/storage.rs @@ -3,6 +3,8 @@ use init4_bin_base::utils::from_env::FromEnv; use signet_storage::SqlConnector; use signet_storage::{DatabaseEnv, MdbxConnector, UnifiedStorage, builder::StorageBuilder}; use std::borrow::Cow; +#[cfg(any(feature = "postgres", feature = "sqlite"))] +use std::time::Duration; use tokio_util::sync::CancellationToken; /// Configuration for signet unified storage. @@ -19,6 +21,13 @@ use tokio_util::sync::CancellationToken; /// /// Exactly one of `SIGNET_COLD_PATH` or `SIGNET_COLD_SQL_URL` must be set. /// +/// When using SQL cold storage, connection pool tuning is configured via +/// [`SqlConnector`]'s own environment variables (e.g. +/// `SIGNET_COLD_SQL_MAX_CONNECTIONS`). See the `cold-sql` feature of +/// `init4-bin-base` for the full list. +/// +/// [`SqlConnector`]: signet_storage::SqlConnector +/// /// # Example /// /// ```rust,no_run @@ -30,7 +39,7 @@ use tokio_util::sync::CancellationToken; /// # Ok(()) /// # } /// ``` -#[derive(Debug, Clone, serde::Deserialize, FromEnv)] +#[derive(Debug, Clone, FromEnv)] pub struct StorageConfig { /// Path to the hot MDBX database. #[from_env(var = "SIGNET_HOT_PATH", desc = "Path to hot MDBX storage", infallible)] @@ -40,21 +49,85 @@ pub struct StorageConfig { #[from_env(var = "SIGNET_COLD_PATH", desc = "Path to cold MDBX storage", infallible)] cold_path: Cow<'static, str>, - /// SQL connection URL for cold storage (requires `postgres` or `sqlite` - /// feature). - #[from_env( - var = "SIGNET_COLD_SQL_URL", - desc = "SQL connection URL for cold storage", - infallible - )] - #[serde(default)] - cold_sql_url: Cow<'static, str>, + /// Pre-configured SQL connector with pool settings. Populated + /// automatically from environment variables via [`FromEnv`], or from + /// the pool-tuning fields when deserializing via serde. + #[cfg(any(feature = "postgres", feature = "sqlite"))] + cold_sql: Option, +} + +impl<'de> serde::Deserialize<'de> for StorageConfig { + fn deserialize>(deserializer: D) -> Result { + /// Flat helper that mirrors the env-var layout so config files use the + /// same field names operators already know. + #[derive(serde::Deserialize)] + struct Helper { + hot_path: Cow<'static, str>, + cold_path: Cow<'static, str>, + #[serde(default)] + cold_sql_url: Option, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + #[serde(default)] + cold_sql_max_connections: Option, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + #[serde(default)] + cold_sql_min_connections: Option, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + #[serde(default)] + cold_sql_acquire_timeout_secs: Option, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + #[serde(default)] + cold_sql_idle_timeout_secs: Option, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + #[serde(default)] + cold_sql_max_lifetime_secs: Option, + } + + let helper = Helper::deserialize(deserializer)?; + + #[cfg(not(any(feature = "postgres", feature = "sqlite")))] + if helper.cold_sql_url.as_ref().is_some_and(|url| !url.is_empty()) { + return Err(serde::de::Error::custom( + "cold_sql_url requires the 'postgres' or 'sqlite' feature", + )); + } + + // Defaults must match bin-base's `FromEnv for SqlConnector` so that + // env-var and config-file paths produce identical pool behavior. + #[cfg(any(feature = "postgres", feature = "sqlite"))] + let cold_sql = helper.cold_sql_url.filter(|url| !url.is_empty()).map(|url| { + SqlConnector::new(url) + .with_max_connections(helper.cold_sql_max_connections.unwrap_or(100)) + .with_min_connections(helper.cold_sql_min_connections.unwrap_or(5)) + .with_acquire_timeout(Duration::from_secs( + helper.cold_sql_acquire_timeout_secs.unwrap_or(5), + )) + .with_idle_timeout(Some(Duration::from_secs( + helper.cold_sql_idle_timeout_secs.unwrap_or(600), + ))) + .with_max_lifetime(Some(Duration::from_secs( + helper.cold_sql_max_lifetime_secs.unwrap_or(1800), + ))) + }); + + Ok(StorageConfig { + hot_path: helper.hot_path, + cold_path: helper.cold_path, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + cold_sql, + }) + } } impl StorageConfig { /// Create a new storage configuration with MDBX cold backend. pub const fn new(hot_path: Cow<'static, str>, cold_path: Cow<'static, str>) -> Self { - Self { hot_path, cold_path, cold_sql_url: Cow::Borrowed("") } + Self { + hot_path, + cold_path, + #[cfg(any(feature = "postgres", feature = "sqlite"))] + cold_sql: None, + } } /// Get the hot storage path. @@ -67,9 +140,18 @@ impl StorageConfig { &self.cold_path } - /// Get the cold SQL connection URL. + /// Get the cold SQL connection URL. Returns an empty string when SQL + /// cold storage is not configured. + #[cfg_attr( + not(any(feature = "postgres", feature = "sqlite")), + expect(clippy::missing_const_for_fn, reason = "not const when SQL features are enabled") + )] pub fn cold_sql_url(&self) -> &str { - &self.cold_sql_url + #[cfg(any(feature = "postgres", feature = "sqlite"))] + if let Some(connector) = &self.cold_sql { + return connector.url(); + } + "" } /// Build unified storage from this configuration. @@ -77,14 +159,18 @@ impl StorageConfig { /// Creates connectors from the configured paths, spawns the cold storage /// background task, and returns a [`UnifiedStorage`] ready for use. /// - /// Exactly one of `cold_path` or `cold_sql_url` must be non-empty. + /// Exactly one of `cold_path` or `cold_sql` must be configured. pub async fn build_storage( &self, cancel: CancellationToken, ) -> eyre::Result> { let hot = MdbxConnector::new(self.hot_path.as_ref()); let has_mdbx = !self.cold_path.is_empty(); - let has_sql = !self.cold_sql_url.is_empty(); + + #[cfg(any(feature = "postgres", feature = "sqlite"))] + let has_sql = self.cold_sql.is_some(); + #[cfg(not(any(feature = "postgres", feature = "sqlite")))] + let has_sql = std::env::var("SIGNET_COLD_SQL_URL").is_ok_and(|v| !v.is_empty()); match (has_mdbx, has_sql) { (true, false) => Ok(StorageBuilder::new() @@ -94,12 +180,16 @@ impl StorageConfig { .build() .await?), #[cfg(any(feature = "postgres", feature = "sqlite"))] - (false, true) => Ok(StorageBuilder::new() - .hot(hot) - .cold(SqlConnector::new(self.cold_sql_url.as_ref())) - .cancel_token(cancel) - .build() - .await?), + (false, true) => { + let connector = + self.cold_sql.clone().expect("cold_sql must be Some when has_sql is true"); + Ok(StorageBuilder::new() + .hot(hot) + .cold(connector) + .cancel_token(cancel) + .build() + .await?) + } #[cfg(not(any(feature = "postgres", feature = "sqlite")))] (false, true) => { eyre::bail!("SIGNET_COLD_SQL_URL requires the 'postgres' or 'sqlite' feature")