[4/9] feat: upgrade opendal + add Vercel Artifacts cache backend

Cargo.toml

@@ -31,6 +31,7 @@ strip = true
 anyhow = { version = "1.0", features = ["backtrace"] }
 ar = "0.9"
 async-trait = "0.1"
+aws-lc-sys = { version = "0.39.1", features = ["prebuilt-nasm"] }
 backon = { version = "1", default-features = false, features = [
   "std-blocking-sleep",
 ] }
@@ -69,7 +70,9 @@ memmap2 = "0.9.4"
 mime = "0.3"
 number_prefix = "0.4"
 object = "0.37"
-opendal = { version = "0.55.0", optional = true, default-features = false }
+opendal = { version = "0.55.0", optional = true, default-features = false, features = [
+  "layers-logging",
+] }
 openssl = { version = "0.10.75", optional = true }
 rand = "0.8.4"
 regex = "1.10.3"
@@ -167,6 +170,7 @@ all = [
   "webdav",
   "oss",
   "cos",
+  "vercel_artifacts",
 ]
 azure = ["opendal/services-azblob", "reqsign", "reqwest"]
 cos = ["opendal/services-cos", "reqsign", "reqwest"]
@@ -178,6 +182,7 @@ native-zlib = []
 oss = ["opendal/services-oss", "reqsign", "reqwest"]
 redis = ["url", "opendal/services-redis"]
 s3 = ["opendal/services-s3", "reqsign", "reqwest"]
+vercel_artifacts = ["opendal/services-vercel-artifacts", "reqwest"]
 webdav = ["opendal/services-webdav", "reqwest"]
 # Enable features that will build a vendored version of openssl and
 # statically linked with it, instead of linking against the system-wide openssl
@@ -233,3 +238,6 @@ ptr_as_ptr = "warn"
 ref_option = "warn"
 semicolon_if_nothing_returned = "warn"
 unnecessary_semicolon = "warn"
+
+[patch.crates-io]
+opendal = { git = "https://github.com/mmastrac/opendal.git", rev = "723ba7f7359e", package = "opendal" }

docs/Configuration.md

@@ -310,3 +310,10 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`.
 * `SCCACHE_COS_KEY_PREFIX`
 * `TENCENTCLOUD_SECRET_ID`
 * `TENCENTCLOUD_SECRET_KEY`
+
+#### Vercel Artifacts
+
+* `SCCACHE_VERCEL_ARTIFACTS_TOKEN` Vercel access token for the artifacts API (required)
+* `SCCACHE_VERCEL_ARTIFACTS_ENDPOINT` API endpoint (default: `https://api.vercel.com`)
+* `SCCACHE_VERCEL_ARTIFACTS_TEAM_ID` Vercel team ID, appended as `teamId` query parameter
+* `SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG` Vercel team slug, appended as `slug` query parameter

src/cache/cache.rs

@@ -40,9 +40,12 @@ use crate::cache::s3::S3Cache;
     feature = "s3",
     feature = "webdav",
     feature = "oss",
-    feature = "cos"
+    feature = "cos",
+    feature = "vercel_artifacts"
 ))]
 use crate::cache::utils::normalize_key;
+#[cfg(feature = "vercel_artifacts")]
+use crate::cache::vercel_artifacts::VercelArtifactsCache;
 #[cfg(feature = "webdav")]
 use crate::cache::webdav::WebdavCache;
 use crate::compiler::PreprocessorCacheEntry;
@@ -170,6 +173,10 @@ pub trait Storage: Send + Sync {
 pub struct RemoteStorage {
     operator: opendal::Operator,
     basedirs: Vec<Vec<u8>>,
+    /// Optional transform applied to every key (including health-check paths)
+    /// before it is sent to the operator.  Used by backends like Vercel Artifacts
+    /// that only accept alphanumeric artifact IDs.
+    key_transform: Option<fn(&str) -> String>,
 }
 
 #[cfg(any(
@@ -185,7 +192,31 @@ pub struct RemoteStorage {
 ))]
 impl RemoteStorage {
     pub fn new(operator: opendal::Operator, basedirs: Vec<Vec<u8>>) -> Self {
-        Self { operator, basedirs }
+        Self {
+            operator,
+            basedirs,
+            key_transform: None,
+        }
+    }
+
+    pub fn new_with_key_transform(
+        operator: opendal::Operator,
+        basedirs: Vec<Vec<u8>>,
+        key_transform: fn(&str) -> String,
+    ) -> Self {
+        Self {
+            operator,
+            basedirs,
+            key_transform: Some(key_transform),
+        }
+    }
+
+    fn key_path(&self, key: &str) -> String {
+        let normalized = normalize_key(key);
+        match self.key_transform {
+            Some(transform) => transform(&normalized),
+            None => normalized,
+        }
     }
 }
 
@@ -204,7 +235,7 @@ impl RemoteStorage {
 #[async_trait]
 impl Storage for RemoteStorage {
     async fn get(&self, key: &str) -> Result<Cache> {
-        match self.operator.read(&normalize_key(key)).await {
+        match self.operator.read(&self.key_path(key)).await {
             Ok(res) => {
                 let hit = CacheRead::from(io::Cursor::new(res.to_bytes()))?;
                 Ok(Cache::Hit(hit))
@@ -227,10 +258,10 @@ impl Storage for RemoteStorage {
     async fn check(&self) -> Result<CacheMode> {
         use opendal::ErrorKind;
 
-        let path = ".sccache_check";
+        let path = self.key_path(".sccache_check");
 
         // Read is required, return error directly if we can't read .
-        match self.operator.read(path).await {
+        match self.operator.read(&path).await {
             Ok(_) => (),
             // Read not exist file with not found is ok.
             Err(err) if err.kind() == ErrorKind::NotFound => (),
@@ -249,7 +280,7 @@ impl Storage for RemoteStorage {
             Err(err) => bail!("cache storage failed to read: {:?}", err),
         }
 
-        let can_write = match self.operator.write(path, "Hello, World!").await {
+        let can_write = match self.operator.write(&path, "Hello, World!").await {
             Ok(_) => true,
             Err(err) if err.kind() == ErrorKind::AlreadyExists => true,
             // Tolerate all other write errors because we can do read at least.
@@ -306,7 +337,7 @@ impl Storage for RemoteStorage {
     /// For backfill we need the raw bytes to write directly to another cache level.
     async fn get_raw(&self, key: &str) -> Result<Option<Bytes>> {
         trace!("opendal::Operator::get_raw({})", key);
-        match self.operator.read(&normalize_key(key)).await {
+        match self.operator.read(&self.key_path(key)).await {
             Ok(res) => {
                 let data = res.to_bytes();
                 trace!(
@@ -337,7 +368,7 @@ impl Storage for RemoteStorage {
         trace!("opendal::Operator::put_raw({}, {} bytes)", key, data.len());
         let start = std::time::Instant::now();
 
-        self.operator.write(&normalize_key(key), data).await?;
+        self.operator.write(&self.key_path(key), data).await?;
 
         Ok(start.elapsed())
     }
@@ -542,6 +573,24 @@ pub fn build_single_cache(
             let storage = RemoteStorage::new(operator, basedirs.to_vec());
             Ok(Arc::new(storage))
         }
+        #[cfg(feature = "vercel_artifacts")]
+        CacheType::VercelArtifacts(c) => {
+            debug!("Init vercel artifacts cache");
+
+            let operator = VercelArtifactsCache::build(
+                &c.access_token,
+                c.endpoint.as_deref(),
+                c.team_id.as_deref(),
+                c.team_slug.as_deref(),
+            )
+            .map_err(|err| anyhow!("create vercel artifacts cache failed: {err:?}"))?;
+            let storage = RemoteStorage::new_with_key_transform(
+                operator,
+                basedirs.to_vec(),
+                crate::cache::vercel_artifacts::sanitize_key,
+            );
+            Ok(Arc::new(storage))
+        }
         #[allow(unreachable_patterns)]
         _ => {
             bail!("Cache type not supported with current feature configuration")

src/cache/memcached.rs

@@ -21,6 +21,21 @@ use opendal::services::Memcached;
 
 use crate::errors::*;
 
+/// Resolve hostname in a memcached endpoint URL to an IP address.
+/// The new opendal memcached service uses SocketAddr::parse internally which
+/// doesn't support DNS hostnames, only IP:port. This works around that by
+/// resolving tcp://hostname:port to tcp://ip:port.
+fn resolve_memcached_endpoint(url: &str) -> String {
+    if let Some(rest) = url.strip_prefix("tcp://") {
+        if let Ok(addrs) = std::net::ToSocketAddrs::to_socket_addrs(&rest) {
+            if let Some(addr) = addrs.into_iter().next() {
+                return format!("tcp://{}", addr);
+            }
+        }
+    }
+    url.to_string()
+}
+
 #[derive(Clone)]
 pub struct MemcachedCache;
 
@@ -32,7 +47,10 @@ impl MemcachedCache {
         key_prefix: &str,
         expiration: u32,
     ) -> Result<Operator> {
-        let mut builder = Memcached::default().endpoint(url);
+        // The new opendal memcached service uses SocketAddr::parse which doesn't
+        // support hostnames. Resolve hostname to IP if the endpoint uses tcp://.
+        let url = resolve_memcached_endpoint(url);
+        let mut builder = Memcached::default().endpoint(&url);
 
         if let Some(username) = username {
             builder = builder.username(username);

src/cache/mod.rs

@@ -37,6 +37,8 @@ pub mod redis;
 #[cfg(feature = "s3")]
 pub mod s3;
 pub(crate) mod utils;
+#[cfg(feature = "vercel_artifacts")]
+pub mod vercel_artifacts;
 #[cfg(feature = "webdav")]
 pub mod webdav;
 
@@ -47,7 +49,8 @@ pub mod webdav;
     feature = "s3",
     feature = "webdav",
     feature = "oss",
-    feature = "cos"
+    feature = "cos",
+    feature = "vercel_artifacts"
 ))]
 pub(crate) mod http_client;
 

src/cache/multilevel.rs

@@ -427,7 +427,8 @@ impl MultiLevelStorage {
                     feature = "s3",
                     feature = "webdav",
                     feature = "oss",
-                    feature = "cos"
+                    feature = "cos",
+                    feature = "vercel_artifacts"
                 ))]
                 {
                     let cache_type = match level_name.to_lowercase().as_str() {
@@ -453,6 +454,12 @@ impl MultiLevelStorage {
                         "oss" => config.cache_configs.oss.clone().map(CacheType::OSS),
                         #[cfg(feature = "cos")]
                         "cos" => config.cache_configs.cos.clone().map(CacheType::COS),
+                        #[cfg(feature = "vercel_artifacts")]
+                        "vercel_artifacts" => config
+                            .cache_configs
+                            .vercel_artifacts
+                            .clone()
+                            .map(CacheType::VercelArtifacts),
                         _ => {
                             return Err(anyhow!("Unknown cache level: '{}'", level_name));
                         }

src/cache/vercel_artifacts.rs

@@ -0,0 +1,65 @@
+use opendal::Operator;
+use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::services::VercelArtifacts;
+
+use crate::errors::*;
+
+use super::http_client::set_user_agent;
+
+/// Sanitize a cache key so it matches the Vercel Artifacts API's hash regex
+/// (`/^[a-fA-F0-9]+$/`).  Only hex characters [0-9a-f] are passed through;
+/// every other byte is replaced with its two-character uppercase hex encoding
+/// (e.g. `/` → `2F`, `.` → `2E`, `k` → `6B`).
+///
+/// This keeps already-valid lowercase hex hash keys (the common case) untouched
+/// while safely encoding the `/` separators from `normalize_key` and any other
+/// non-hex characters.
+pub fn sanitize_key(key: &str) -> String {
+    let mut out = String::with_capacity(key.len());
+    for b in key.bytes() {
+        if b.is_ascii_hexdigit() {
+            out.push(b as char);
+        } else {
+            out.push(
+                char::from_digit((b >> 4) as u32, 16)
+                    .unwrap()
+                    .to_ascii_uppercase(),
+            );
+            out.push(
+                char::from_digit((b & 0xf) as u32, 16)
+                    .unwrap()
+                    .to_ascii_uppercase(),
+            );
+        }
+    }
+    out
+}
+
+/// A cache that stores entries in Vercel Artifacts.
+pub struct VercelArtifactsCache;
+
+impl VercelArtifactsCache {
+    pub fn build(
+        access_token: &str,
+        endpoint: Option<&str>,
+        team_id: Option<&str>,
+        team_slug: Option<&str>,
+    ) -> Result<Operator> {
+        let mut builder = VercelArtifacts::default().access_token(access_token);
+        if let Some(endpoint) = endpoint {
+            builder = builder.endpoint(endpoint);
+        }
+        if let Some(team_id) = team_id {
+            builder = builder.team_id(team_id);
+        }
+        if let Some(team_slug) = team_slug {
+            builder = builder.team_slug(team_slug);
+        }
+
+        let op = Operator::new(builder)?
+            .layer(HttpClientLayer::new(set_user_agent()))
+            .layer(LoggingLayer::default())
+            .finish();
+        Ok(op)
+    }
+}