From 2049b882927efa822220cbf2b2f35852aa59cf4c Mon Sep 17 00:00:00 2001 From: Matt Mastracci Date: Thu, 2 Apr 2026 10:21:34 -0600 Subject: [PATCH 1/2] feat: strip basedirs from Rust hash key for cross-machine cache hits SCCACHE_BASEDIRS now normalizes cwd, CARGO_MANIFEST_DIR, CARGO_WORKSPACE_DIR, and dep-info env var values in the Rust compiler's hash key computation. This enables cache hits when the same crate is compiled from different absolute paths on different machines (e.g., CI runners with different checkout dirs). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/compiler/rust.rs | 235 +++++++++++++++++++++++++++++++++++++-- src/test/mock_storage.rs | 21 ++++ 2 files changed, 246 insertions(+), 10 deletions(-) diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 2fdbcedb0e..e92523a2bd 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -62,6 +62,52 @@ use std::time; use crate::errors::*; +/// CARGO_* environment variables known to contain absolute paths that should +/// have basedir prefixes stripped for cross-machine cache portability. +/// See: https://doc.rust-lang.org/cargo/reference/environment-variables.html +const CARGO_PATH_ENV_VARS: &[&str] = &[ + "CARGO_MANIFEST_DIR", + "CARGO_MANIFEST_PATH", + "CARGO_TARGET_TMPDIR", + "CARGO_WORKSPACE_DIR", +]; + +/// Prefixes of CARGO_* environment variables that contain absolute paths. +/// Variables matching these prefixes have their values basedir-stripped. +const CARGO_PATH_ENV_PREFIXES: &[&str] = &["CARGO_BIN_EXE_"]; + +/// Returns true if a CARGO_* env var is known to contain an absolute path. +fn is_cargo_path_var(var: &str) -> bool { + CARGO_PATH_ENV_VARS.contains(&var) + || CARGO_PATH_ENV_PREFIXES.iter().any(|&p| var.starts_with(p)) +} + +/// Strip a basedir prefix from a byte slice, returning the relative portion. +/// Basedirs are pre-normalized with trailing `/` (see config.rs), so the +/// result is a clean relative path. On Windows, the value is normalized +/// (lowercase + forward slashes) before comparison since basedirs are stored +/// normalized. +fn strip_basedir_prefix<'a>(value: &'a [u8], basedirs: &[Vec]) -> Cow<'a, [u8]> { + if basedirs.is_empty() { + return Cow::Borrowed(value); + } + + #[cfg(target_os = "windows")] + let normalized = crate::util::normalize_win_path(value); + #[cfg(not(target_os = "windows"))] + let normalized = value; + + for basedir in basedirs { + if normalized.starts_with(basedir) { + #[cfg(target_os = "windows")] + return Cow::Owned(normalized[basedir.len()..].to_vec()); + #[cfg(not(target_os = "windows"))] + return Cow::Borrowed(&value[basedir.len()..]); + } + } + Cow::Borrowed(value) +} + #[cfg(feature = "dist-client")] const RLIB_PREFIX: &str = "lib"; #[cfg(feature = "dist-client")] @@ -1343,10 +1389,11 @@ where _may_dist: bool, pool: &tokio::runtime::Handle, _rewrite_includes_only: bool, - _storage: Arc, + storage: Arc, _cache_control: CacheControl, ) -> Result> { trace!("[{}]: generate_hash_key", self.parsed_args.crate_name); + let basedirs = storage.basedirs(); // TODO: this doesn't produce correct arguments if they should be concatenated - should use iter_os_strings let os_string_arguments: Vec<(OsString, Option)> = self .parsed_args @@ -1493,7 +1540,12 @@ where a }) }; - args.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from arguments before hashing. Arguments like + // --remap-path-prefix=/abs/path=..., -Clinker=/abs/path, etc. contain + // absolute paths that differ across machines. See mozilla/sccache#2652. + let args_bytes = args.as_encoded_bytes(); + crate::util::strip_basedirs(args_bytes, basedirs) + .hash(&mut HashToDigest { digest: &mut m }); // 4. The digest of all source files (this includes src file from cmdline). // 5. The digest of all files listed on the commandline (self.externs). // 6. The digest of all static libraries listed on the commandline (self.staticlibs). @@ -1513,7 +1565,10 @@ where for (var, val) in env_deps.iter() { var.hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - val.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from dep-info env var values (e.g. OUT_DIR) + // to enable cross-machine cache hits. + let val_bytes = val.as_encoded_bytes(); + strip_basedir_prefix(val_bytes, basedirs).hash(&mut HashToDigest { digest: &mut m }); } let mut env_vars: Vec<_> = env_vars .iter() @@ -1544,10 +1599,21 @@ where var.hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - val.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from path-containing CARGO_* vars + // to enable cross-machine cache hits. + let var_str = var.to_string_lossy(); + if is_cargo_path_var(&var_str) { + let val_bytes = val.as_encoded_bytes(); + strip_basedir_prefix(val_bytes, basedirs) + .hash(&mut HashToDigest { digest: &mut m }); + } else { + val.hash(&mut HashToDigest { digest: &mut m }); + } } // 9. The cwd of the compile. This will wind up in the rlib. - cwd.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefix for cross-machine cache portability. + let cwd_bytes = cwd.as_os_str().as_encoded_bytes(); + strip_basedir_prefix(cwd_bytes, basedirs).hash(&mut HashToDigest { digest: &mut m }); // 10. The version of the compiler. self.version.hash(&mut HashToDigest { digest: &mut m }); // 11. SCCACHE_RUST_CRATE_TYPE_ALLOW_HASH, if set and we have unsupported @@ -3541,8 +3607,12 @@ proc_macro false m.update(CACHE_VERSION); // sysroot shlibs digests. m.update(FAKE_DIGEST.as_bytes()); - // Arguments, with cfgs sorted at the end. - OsStr::new("ab--cfgabc--cfgxyz").hash(&mut HashToDigest { digest: &mut m }); + // Arguments, with cfgs sorted at the end (hashed as bytes via strip_basedirs). + // With empty basedirs, strip_basedirs returns Cow::Borrowed of the original bytes. + let args_str = OsStr::new("ab--cfgabc--cfgxyz"); + args_str + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); // bar.rs (source file, from dep-info) m.update(empty_digest.as_bytes()); // foo.rs (source file, from dep-info) @@ -3552,14 +3622,21 @@ proc_macro false // libbaz.a (static library, from staticlibs), containing a single // file, baz.o, consisting of 1024 bytes of zeroes. m.update(libbaz_a_digest.as_bytes()); - // Env vars + // Env vars (dep-info env vars hashed as bytes via strip_basedir_prefix) OsStr::new("CARGO_BLAH").hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - OsStr::new("abc").hash(&mut HashToDigest { digest: &mut m }); + OsStr::new("abc") + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); OsStr::new("CARGO_PKG_NAME").hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); OsStr::new("foo").hash(&mut HashToDigest { digest: &mut m }); - f.tempdir.path().hash(&mut HashToDigest { digest: &mut m }); + // cwd (hashed as bytes via strip_basedir_prefix) + f.tempdir + .path() + .as_os_str() + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); TEST_RUSTC_VERSION.hash(&mut HashToDigest { digest: &mut m }); let digest = m.finish(); assert_eq!(res.key, digest); @@ -3893,6 +3970,144 @@ proc_macro false ); } + fn hash_key_with_basedirs( + f: &TestFixture, + args: &[&'static str], + env_vars: &[(OsString, OsString)], + pre_func: F, + basedirs: Vec>, + ) -> String + where + F: Fn(&Path) -> Result<()>, + { + let oargs = args.iter().map(OsString::from).collect::>(); + let parsed_args = match parse_arguments(&oargs, f.tempdir.path()) { + CompilerArguments::Ok(parsed_args) => parsed_args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + { + let src = &"foo.rs"; + f.touch(src).expect("Failed to create foo.rs"); + } + for e in parsed_args.externs.iter() { + f.touch(e.to_str().unwrap()) + .expect("Failed to create extern"); + } + pre_func(f.tempdir.path()).expect("Failed to execute pre_func"); + let mut hasher = Box::new(RustHasher { + executable: "rustc".into(), + host: "x86-64-unknown-unknown-unknown".to_owned(), + version: TEST_RUSTC_VERSION.to_string(), + sysroot: f.tempdir.path().join("sysroot"), + compiler_shlibs_digests: vec![], + #[cfg(feature = "dist-client")] + rlib_dep_reader: None, + parsed_args, + }); + + let creator = new_creator(); + let runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); + + mock_dep_info(&creator, &["foo.rs"]); + mock_file_names(&creator, &["foo.rlib"]); + hasher + .generate_hash_key( + &creator, + f.tempdir.path().to_owned(), + env_vars.to_owned(), + false, + &pool, + false, + Arc::new(MockStorage::with_basedirs(None, false, basedirs)), + CacheControl::Default, + ) + .wait() + .unwrap() + .key + } + + #[test] + fn test_basedirs_strips_cwd_and_cargo_manifest_dir() { + let f = TestFixture::new(); + let cwd = f.tempdir.path().to_string_lossy().into_owned(); + + let args = &[ + "--emit", + "link", + "foo.rs", + "--out-dir", + "out", + "--crate-name", + "foo", + "--crate-type", + "lib", + ]; + + let manifest_dir = format!("{}/some/pkg", cwd); + let env_vars = vec![ + ( + OsString::from("CARGO_MANIFEST_DIR"), + OsString::from(&manifest_dir), + ), + (OsString::from("CARGO_PKG_NAME"), OsString::from("foo")), + ]; + + // Hash without basedirs + let key_without = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![]); + + // Hash with basedirs that strip the cwd prefix. + // Basedirs are normalized at config time (forward slashes, lowercase on Windows, + // trailing slash) — replicate that here. + let basedir = cwd.into_bytes(); + #[cfg(target_os = "windows")] + let basedir = crate::util::normalize_win_path(&basedir); + let mut basedir = basedir; + basedir.push(b'/'); + let key_with = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir]); + + // The keys should differ because basedirs changes the hash + assert_ne!(key_without, key_with, "basedirs should change the hash key"); + + // Two different "machines" with different cwds but same basedirs should + // produce the same hash. We simulate this by noting that the basedir- + // stripped hash is deterministic regardless of cwd. + // (We can't easily create two different tempdirs with the same content + // in this test framework, but we verify the stripping changes the hash.) + } + + #[test] + fn test_basedirs_deterministic() { + // Running the same compilation with the same basedirs twice should + // produce the same hash, and it should differ from no-basedirs. + let f = TestFixture::new(); + let cwd = f.tempdir.path().to_string_lossy().into_owned(); + + let args = &[ + "--emit", + "link", + "foo.rs", + "--out-dir", + "out", + "--crate-name", + "foo", + "--crate-type", + "lib", + ]; + let env_vars = vec![(OsString::from("CARGO_PKG_NAME"), OsString::from("foo"))]; + + let basedir = cwd.into_bytes(); + #[cfg(target_os = "windows")] + let basedir = crate::util::normalize_win_path(&basedir); + let mut basedir = basedir; + basedir.push(b'/'); + + let key1 = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir.clone()]); + let key2 = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir]); + + assert_eq!(key1, key2, "Same basedir should produce deterministic hash"); + } + #[test] fn test_parse_unstable_profile_flag() { let h = parses!( diff --git a/src/test/mock_storage.rs b/src/test/mock_storage.rs index d4260e79e2..e2bd75f114 100644 --- a/src/test/mock_storage.rs +++ b/src/test/mock_storage.rs @@ -28,6 +28,7 @@ pub struct MockStorage { tx: mpsc::UnboundedSender>, delay: Option, preprocessor_cache_mode: bool, + basedirs: Vec>, } impl MockStorage { @@ -39,6 +40,23 @@ impl MockStorage { rx: Arc::new(Mutex::new(rx)), delay, preprocessor_cache_mode, + basedirs: vec![], + } + } + + /// Create a new `MockStorage` with basedirs configured. + pub(crate) fn with_basedirs( + delay: Option, + preprocessor_cache_mode: bool, + basedirs: Vec>, + ) -> MockStorage { + let (tx, rx) = mpsc::unbounded(); + Self { + tx, + rx: Arc::new(Mutex::new(rx)), + delay, + preprocessor_cache_mode, + basedirs, } } @@ -75,6 +93,9 @@ impl Storage for MockStorage { async fn max_size(&self) -> Result> { Ok(None) } + fn basedirs(&self) -> &[Vec] { + &self.basedirs + } fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { PreprocessorCacheModeConfig { use_preprocessor_cache_mode: self.preprocessor_cache_mode, From 4838c6d53c00b0ada03f279008de373925097639 Mon Sep 17 00:00:00 2001 From: Matt Mastracci Date: Mon, 30 Mar 2026 13:35:47 -0600 Subject: [PATCH 2/2] feat: upgrade opendal + add Vercel Artifacts cache backend Upgrades opendal to mmastrac/opendal vercel_opts branch via [patch.crates-io], which tracks apache/opendal main plus additional Vercel Artifacts builder methods (endpoint, team_id, team_slug) from apache/opendal#7334. Adds layers-logging feature to retain LoggingLayer after upstream split. Adds aws-lc-sys with prebuilt-nasm and bindgen features to avoid nasm/bindgen build requirements in CI cross-compilation. Adds Vercel Artifacts cache backend configured via: - SCCACHE_VERCEL_ARTIFACTS_TOKEN (required) - SCCACHE_VERCEL_ARTIFACTS_ENDPOINT (optional) - SCCACHE_VERCEL_ARTIFACTS_TEAM_ID (optional) - SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG (optional) Upstream: https://github.com/apache/opendal/pull/7334 Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 611 ++++++++++++++++++++++++++++------ Cargo.toml | 10 +- docs/Configuration.md | 7 + src/cache/cache.rs | 65 +++- src/cache/memcached.rs | 20 +- src/cache/mod.rs | 5 +- src/cache/multilevel.rs | 9 +- src/cache/vercel_artifacts.rs | 65 ++++ src/config.rs | 43 ++- tests/harness/mod.rs | 1 + 10 files changed, 729 insertions(+), 107 deletions(-) create mode 100644 src/cache/vercel_artifacts.rs diff --git a/Cargo.lock b/Cargo.lock index 1126c5f004..430a8e8e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,9 +120,18 @@ checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69" [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" [[package]] name = "arrayref" @@ -157,6 +166,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -180,6 +200,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backon" version = "1.6.0" @@ -187,8 +230,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", - "gloo-timers", - "tokio", ] [[package]] @@ -230,17 +271,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bb8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d8b8e1a22743d9241575c6ba822cf9c8fef34771c86ab7e477a4fbfd254e5" -dependencies = [ - "futures-util", - "parking_lot", - "tokio", -] - [[package]] name = "bincode" version = "1.3.3" @@ -343,10 +373,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.16" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -440,6 +471,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "codspeed" version = "4.2.0" @@ -530,6 +570,15 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "condtype" version = "1.3.0" @@ -798,6 +847,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.9.0" @@ -852,6 +907,37 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastpool" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "505402589aaeb2f89357bf8dfb259046c693a3c9a68b874a0ca8c0fb99e0fb4c" +dependencies = [ + "mea", + "scopeguard", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -870,6 +956,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "flate2" version = "1.1.5" @@ -940,6 +1032,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.30" @@ -1087,18 +1185,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "gzp" version = "2.0.1" @@ -1501,24 +1587,26 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", - "serde", - "windows-sys 0.59.0", + "serde_core", + "wasm-bindgen", + "windows-sys 0.61.1", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -1574,6 +1662,23 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.11", + "js-sys", + "pem", + "serde", + "serde_json", + "signature", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1646,9 +1751,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" @@ -1666,6 +1771,15 @@ dependencies = [ "digest", ] +[[package]] +name = "mea" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.7.5" @@ -1901,35 +2015,231 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opendal" version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "opendal-core", + "opendal-layer-logging", + "opendal-service-azblob", + "opendal-service-cos", + "opendal-service-gcs", + "opendal-service-ghac", + "opendal-service-memcached", + "opendal-service-oss", + "opendal-service-redis", + "opendal-service-s3", + "opendal-service-vercel-artifacts", + "opendal-service-webdav", +] + +[[package]] +name = "opendal-core" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" dependencies = [ "anyhow", - "backon", "base64 0.22.1", - "bb8", "bytes", - "crc32c", "futures", - "getrandom 0.2.11", - "ghac", "http", "http-body", "jiff", "log", "md-5", + "mea", "percent-encoding", - "prost", "quick-xml 0.38.4", - "redis", - "reqsign 0.16.5", + "reqsign-core 3.0.0", "reqwest", "serde", "serde_json", - "sha2", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "base64 0.22.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", + "sha2", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "http", + "opendal-core", +] + +[[package]] +name = "opendal-service-cos" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "reqsign-tencent-cos", + "serde", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "async-trait", + "bytes", + "http", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-ghac" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "ghac", + "http", + "log", + "opendal-core", + "opendal-service-azblob", + "prost", + "reqsign-azure-storage", + "reqsign-core 3.0.0", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "opendal-service-memcached" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "fastpool", + "opendal-core", + "serde", + "tokio", + "url", +] + +[[package]] +name = "opendal-service-oss" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aliyun-oss", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", +] + +[[package]] +name = "opendal-service-redis" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "fastpool", + "http", + "opendal-core", + "redis", + "serde", + "tokio", +] + +[[package]] +name = "opendal-service-s3" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "http", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", + "url", +] + +[[package]] +name = "opendal-service-vercel-artifacts" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-webdav" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "anyhow", + "bytes", + "http", + "log", + "mea", + "opendal-core", + "quick-xml 0.38.4", + "serde", ] [[package]] @@ -2002,6 +2312,12 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2257,9 +2573,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -2267,9 +2583,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" dependencies = [ "memchr", "serde", @@ -2413,18 +2729,19 @@ dependencies = [ [[package]] name = "redis" -version = "0.32.7" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" +checksum = "d76e41a79ae5cbb41257d84cf4cf0db0bb5a95b11bf05c62c351de4fe748620d" dependencies = [ "arc-swap", + "arcstr", + "async-lock", "backon", "bytes", "cfg-if 1.0.0", "combine", "crc16", "futures-channel", - "futures-sink", "futures-util", "itoa", "log", @@ -2441,6 +2758,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "url", + "xxhash-rust", ] [[package]] @@ -2500,46 +2818,74 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqsign" -version = "0.16.5" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a80170eaab619a5dfa6428b0596c0cb85734bfa36b717a764a16abc3456a7b" +dependencies = [ + "reqsign-command-execute-tokio", + "reqsign-core 2.0.0", + "reqsign-file-read-tokio 2.0.0", + "reqsign-http-send-reqwest", +] + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" dependencies = [ "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", "form_urlencoded", - "getrandom 0.2.11", - "hex", - "hmac", - "home", "http", - "jsonwebtoken", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", - "rsa", + "reqsign-core 3.0.0", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +dependencies = [ + "anyhow", + "bytes", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "quick-xml 0.39.2", + "reqsign-core 3.0.0", "rust-ini", "serde", "serde_json", + "serde_urlencoded", "sha1", - "sha2", - "tokio", ] [[package]] -name = "reqsign" -version = "0.18.0" +name = "reqsign-azure-storage" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9a80170eaab619a5dfa6428b0596c0cb85734bfa36b717a764a16abc3456a7b" +checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" dependencies = [ - "reqsign-command-execute-tokio", - "reqsign-core", - "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "http", + "jsonwebtoken 10.3.0", + "log", + "pem", + "percent-encoding", + "reqsign-core 3.0.0", + "rsa", + "serde", + "serde_json", + "sha1", ] [[package]] @@ -2549,7 +2895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d74ecba4eca9afdd6bebf86d71e442dd4acab3fcec4461f3b96b33cf6a16b5" dependencies = [ "async-trait", - "reqsign-core", + "reqsign-core 2.0.0", "tokio", ] @@ -2575,6 +2921,28 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "reqsign-core" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "futures", + "hex", + "hmac", + "http", + "jiff", + "log", + "percent-encoding", + "sha1", + "sha2", + "windows-sys 0.61.1", +] + [[package]] name = "reqsign-file-read-tokio" version = "2.0.0" @@ -2583,7 +2951,38 @@ checksum = "262eb485bb6e8213b13ef10e86ef8613539fb03daa2123b57d96675f784b15b6" dependencies = [ "anyhow", "async-trait", - "reqsign-core", + "reqsign-core 2.0.0", + "tokio", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +dependencies = [ + "anyhow", + "reqsign-core 3.0.0", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +dependencies = [ + "form_urlencoded", + "http", + "jsonwebtoken 10.3.0", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core 3.0.0", + "rsa", + "serde", + "serde_json", + "sha2", "tokio", ] @@ -2598,10 +2997,25 @@ dependencies = [ "bytes", "http", "http-body-util", - "reqsign-core", + "reqsign-core 2.0.0", "reqwest", ] +[[package]] +name = "reqsign-tencent-cos" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +dependencies = [ + "anyhow", + "http", + "log", + "percent-encoding", + "reqsign-core 3.0.0", + "serde", + "serde_json", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -2661,7 +3075,7 @@ dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.11", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2826,7 +3240,7 @@ checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -2891,6 +3305,7 @@ dependencies = [ "ar", "assert_cmd", "async-trait", + "aws-lc-sys", "backon", "base64 0.21.7", "bincode", @@ -2916,7 +3331,7 @@ dependencies = [ "hyper-util", "itertools", "jobserver", - "jsonwebtoken", + "jsonwebtoken 9.2.0", "libc", "libmount", "linked-hash-map", @@ -2932,7 +3347,7 @@ dependencies = [ "predicates", "rand 0.8.5", "regex", - "reqsign 0.18.0", + "reqsign", "reqwest", "rouille", "semver", @@ -3223,12 +3638,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -3659,7 +4071,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.0", @@ -3912,6 +4323,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -4459,6 +4876,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yoke" version = "0.8.1" @@ -4505,9 +4928,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 8005fb8956..9e1a1f52c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ strip = true anyhow = { version = "1.0", features = ["backtrace"] } ar = "0.9" async-trait = "0.1" +aws-lc-sys = { version = "0.39.1", features = ["prebuilt-nasm"] } backon = { version = "1", default-features = false, features = [ "std-blocking-sleep", ] } @@ -69,7 +70,9 @@ memmap2 = "0.9.4" mime = "0.3" number_prefix = "0.4" object = "0.37" -opendal = { version = "0.55.0", optional = true, default-features = false } +opendal = { version = "0.55.0", optional = true, default-features = false, features = [ + "layers-logging", +] } openssl = { version = "0.10.75", optional = true } rand = "0.8.4" regex = "1.10.3" @@ -167,6 +170,7 @@ all = [ "webdav", "oss", "cos", + "vercel_artifacts", ] azure = ["opendal/services-azblob", "reqsign", "reqwest"] cos = ["opendal/services-cos", "reqsign", "reqwest"] @@ -178,6 +182,7 @@ native-zlib = [] oss = ["opendal/services-oss", "reqsign", "reqwest"] redis = ["url", "opendal/services-redis"] s3 = ["opendal/services-s3", "reqsign", "reqwest"] +vercel_artifacts = ["opendal/services-vercel-artifacts", "reqwest"] webdav = ["opendal/services-webdav", "reqwest"] # Enable features that will build a vendored version of openssl and # statically linked with it, instead of linking against the system-wide openssl @@ -233,3 +238,6 @@ ptr_as_ptr = "warn" ref_option = "warn" semicolon_if_nothing_returned = "warn" unnecessary_semicolon = "warn" + +[patch.crates-io] +opendal = { git = "https://github.com/mmastrac/opendal.git", rev = "723ba7f7359e", package = "opendal" } diff --git a/docs/Configuration.md b/docs/Configuration.md index 5a65173ef1..29fdfcb00a 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -310,3 +310,10 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`. * `SCCACHE_COS_KEY_PREFIX` * `TENCENTCLOUD_SECRET_ID` * `TENCENTCLOUD_SECRET_KEY` + +#### Vercel Artifacts + +* `SCCACHE_VERCEL_ARTIFACTS_TOKEN` Vercel access token for the artifacts API (required) +* `SCCACHE_VERCEL_ARTIFACTS_ENDPOINT` API endpoint (default: `https://api.vercel.com`) +* `SCCACHE_VERCEL_ARTIFACTS_TEAM_ID` Vercel team ID, appended as `teamId` query parameter +* `SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG` Vercel team slug, appended as `slug` query parameter diff --git a/src/cache/cache.rs b/src/cache/cache.rs index f0b79eca63..f684424378 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -40,9 +40,12 @@ use crate::cache::s3::S3Cache; feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] use crate::cache::utils::normalize_key; +#[cfg(feature = "vercel_artifacts")] +use crate::cache::vercel_artifacts::VercelArtifactsCache; #[cfg(feature = "webdav")] use crate::cache::webdav::WebdavCache; use crate::compiler::PreprocessorCacheEntry; @@ -170,6 +173,10 @@ pub trait Storage: Send + Sync { pub struct RemoteStorage { operator: opendal::Operator, basedirs: Vec>, + /// Optional transform applied to every key (including health-check paths) + /// before it is sent to the operator. Used by backends like Vercel Artifacts + /// that only accept alphanumeric artifact IDs. + key_transform: Option String>, } #[cfg(any( @@ -185,7 +192,31 @@ pub struct RemoteStorage { ))] impl RemoteStorage { pub fn new(operator: opendal::Operator, basedirs: Vec>) -> Self { - Self { operator, basedirs } + Self { + operator, + basedirs, + key_transform: None, + } + } + + pub fn new_with_key_transform( + operator: opendal::Operator, + basedirs: Vec>, + key_transform: fn(&str) -> String, + ) -> Self { + Self { + operator, + basedirs, + key_transform: Some(key_transform), + } + } + + fn key_path(&self, key: &str) -> String { + let normalized = normalize_key(key); + match self.key_transform { + Some(transform) => transform(&normalized), + None => normalized, + } } } @@ -204,7 +235,7 @@ impl RemoteStorage { #[async_trait] impl Storage for RemoteStorage { async fn get(&self, key: &str) -> Result { - match self.operator.read(&normalize_key(key)).await { + match self.operator.read(&self.key_path(key)).await { Ok(res) => { let hit = CacheRead::from(io::Cursor::new(res.to_bytes()))?; Ok(Cache::Hit(hit)) @@ -227,10 +258,10 @@ impl Storage for RemoteStorage { async fn check(&self) -> Result { use opendal::ErrorKind; - let path = ".sccache_check"; + let path = self.key_path(".sccache_check"); // Read is required, return error directly if we can't read . - match self.operator.read(path).await { + match self.operator.read(&path).await { Ok(_) => (), // Read not exist file with not found is ok. Err(err) if err.kind() == ErrorKind::NotFound => (), @@ -249,7 +280,7 @@ impl Storage for RemoteStorage { Err(err) => bail!("cache storage failed to read: {:?}", err), } - let can_write = match self.operator.write(path, "Hello, World!").await { + let can_write = match self.operator.write(&path, "Hello, World!").await { Ok(_) => true, Err(err) if err.kind() == ErrorKind::AlreadyExists => true, // Tolerate all other write errors because we can do read at least. @@ -306,7 +337,7 @@ impl Storage for RemoteStorage { /// For backfill we need the raw bytes to write directly to another cache level. async fn get_raw(&self, key: &str) -> Result> { trace!("opendal::Operator::get_raw({})", key); - match self.operator.read(&normalize_key(key)).await { + match self.operator.read(&self.key_path(key)).await { Ok(res) => { let data = res.to_bytes(); trace!( @@ -337,7 +368,7 @@ impl Storage for RemoteStorage { trace!("opendal::Operator::put_raw({}, {} bytes)", key, data.len()); let start = std::time::Instant::now(); - self.operator.write(&normalize_key(key), data).await?; + self.operator.write(&self.key_path(key), data).await?; Ok(start.elapsed()) } @@ -542,6 +573,24 @@ pub fn build_single_cache( let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) } + #[cfg(feature = "vercel_artifacts")] + CacheType::VercelArtifacts(c) => { + debug!("Init vercel artifacts cache"); + + let operator = VercelArtifactsCache::build( + &c.access_token, + c.endpoint.as_deref(), + c.team_id.as_deref(), + c.team_slug.as_deref(), + ) + .map_err(|err| anyhow!("create vercel artifacts cache failed: {err:?}"))?; + let storage = RemoteStorage::new_with_key_transform( + operator, + basedirs.to_vec(), + crate::cache::vercel_artifacts::sanitize_key, + ); + Ok(Arc::new(storage)) + } #[allow(unreachable_patterns)] _ => { bail!("Cache type not supported with current feature configuration") diff --git a/src/cache/memcached.rs b/src/cache/memcached.rs index 35f65a8f79..37427e19c1 100644 --- a/src/cache/memcached.rs +++ b/src/cache/memcached.rs @@ -21,6 +21,21 @@ use opendal::services::Memcached; use crate::errors::*; +/// Resolve hostname in a memcached endpoint URL to an IP address. +/// The new opendal memcached service uses SocketAddr::parse internally which +/// doesn't support DNS hostnames, only IP:port. This works around that by +/// resolving tcp://hostname:port to tcp://ip:port. +fn resolve_memcached_endpoint(url: &str) -> String { + if let Some(rest) = url.strip_prefix("tcp://") { + if let Ok(addrs) = std::net::ToSocketAddrs::to_socket_addrs(&rest) { + if let Some(addr) = addrs.into_iter().next() { + return format!("tcp://{}", addr); + } + } + } + url.to_string() +} + #[derive(Clone)] pub struct MemcachedCache; @@ -32,7 +47,10 @@ impl MemcachedCache { key_prefix: &str, expiration: u32, ) -> Result { - let mut builder = Memcached::default().endpoint(url); + // The new opendal memcached service uses SocketAddr::parse which doesn't + // support hostnames. Resolve hostname to IP if the endpoint uses tcp://. + let url = resolve_memcached_endpoint(url); + let mut builder = Memcached::default().endpoint(&url); if let Some(username) = username { builder = builder.username(username); diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 23647d1fed..ed0fee44bf 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -37,6 +37,8 @@ pub mod redis; #[cfg(feature = "s3")] pub mod s3; pub(crate) mod utils; +#[cfg(feature = "vercel_artifacts")] +pub mod vercel_artifacts; #[cfg(feature = "webdav")] pub mod webdav; @@ -47,7 +49,8 @@ pub mod webdav; feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] pub(crate) mod http_client; diff --git a/src/cache/multilevel.rs b/src/cache/multilevel.rs index f9296a4ab1..8284f4643f 100644 --- a/src/cache/multilevel.rs +++ b/src/cache/multilevel.rs @@ -427,7 +427,8 @@ impl MultiLevelStorage { feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] { let cache_type = match level_name.to_lowercase().as_str() { @@ -453,6 +454,12 @@ impl MultiLevelStorage { "oss" => config.cache_configs.oss.clone().map(CacheType::OSS), #[cfg(feature = "cos")] "cos" => config.cache_configs.cos.clone().map(CacheType::COS), + #[cfg(feature = "vercel_artifacts")] + "vercel_artifacts" => config + .cache_configs + .vercel_artifacts + .clone() + .map(CacheType::VercelArtifacts), _ => { return Err(anyhow!("Unknown cache level: '{}'", level_name)); } diff --git a/src/cache/vercel_artifacts.rs b/src/cache/vercel_artifacts.rs new file mode 100644 index 0000000000..0a9ba46359 --- /dev/null +++ b/src/cache/vercel_artifacts.rs @@ -0,0 +1,65 @@ +use opendal::Operator; +use opendal::layers::{HttpClientLayer, LoggingLayer}; +use opendal::services::VercelArtifacts; + +use crate::errors::*; + +use super::http_client::set_user_agent; + +/// Sanitize a cache key so it matches the Vercel Artifacts API's hash regex +/// (`/^[a-fA-F0-9]+$/`). Only hex characters [0-9a-f] are passed through; +/// every other byte is replaced with its two-character uppercase hex encoding +/// (e.g. `/` → `2F`, `.` → `2E`, `k` → `6B`). +/// +/// This keeps already-valid lowercase hex hash keys (the common case) untouched +/// while safely encoding the `/` separators from `normalize_key` and any other +/// non-hex characters. +pub fn sanitize_key(key: &str) -> String { + let mut out = String::with_capacity(key.len()); + for b in key.bytes() { + if b.is_ascii_hexdigit() { + out.push(b as char); + } else { + out.push( + char::from_digit((b >> 4) as u32, 16) + .unwrap() + .to_ascii_uppercase(), + ); + out.push( + char::from_digit((b & 0xf) as u32, 16) + .unwrap() + .to_ascii_uppercase(), + ); + } + } + out +} + +/// A cache that stores entries in Vercel Artifacts. +pub struct VercelArtifactsCache; + +impl VercelArtifactsCache { + pub fn build( + access_token: &str, + endpoint: Option<&str>, + team_id: Option<&str>, + team_slug: Option<&str>, + ) -> Result { + let mut builder = VercelArtifacts::default().access_token(access_token); + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } + if let Some(team_id) = team_id { + builder = builder.team_id(team_id); + } + if let Some(team_slug) = team_slug { + builder = builder.team_slug(team_slug); + } + + let op = Operator::new(builder)? + .layer(HttpClientLayer::new(set_user_agent())) + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} diff --git a/src/config.rs b/src/config.rs index ecbe98b8d8..92f46635d0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -349,6 +349,15 @@ pub struct GHACacheConfig { pub version: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VercelArtifactsCacheConfig { + pub access_token: String, + pub endpoint: Option, + pub team_id: Option, + pub team_slug: Option, +} + /// Memcached's default value of expiration is 10800s (3 hours), which is too /// short for use case of sccache. /// @@ -484,6 +493,7 @@ pub enum CacheType { Webdav(WebdavCacheConfig), OSS(OSSCacheConfig), COS(COSCacheConfig), + VercelArtifacts(VercelArtifactsCacheConfig), } #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -499,6 +509,7 @@ pub struct CacheConfigs { pub webdav: Option, pub oss: Option, pub cos: Option, + pub vercel_artifacts: Option, /// Multi-level cache configuration pub multilevel: Option, } @@ -518,6 +529,7 @@ impl CacheConfigs { webdav, oss, cos, + vercel_artifacts, multilevel: _, } = self; @@ -530,7 +542,8 @@ impl CacheConfigs { .or_else(|| azure.map(CacheType::Azure)) .or_else(|| webdav.map(CacheType::Webdav)) .or_else(|| oss.map(CacheType::OSS)) - .or_else(|| cos.map(CacheType::COS)); + .or_else(|| cos.map(CacheType::COS)) + .or_else(|| vercel_artifacts.map(CacheType::VercelArtifacts)); let fallback = disk.unwrap_or_default(); @@ -576,6 +589,13 @@ impl CacheConfigs { "oss" => self.oss.clone().map(CacheType::OSS).ok_or_else(|| { anyhow!("OSS cache not configured but specified in levels") })?, + "vercel_artifacts" => self + .vercel_artifacts + .clone() + .map(CacheType::VercelArtifacts) + .ok_or_else(|| { + anyhow!("Vercel Artifacts cache not configured but specified in levels") + })?, "disk" => { // Disk cache is handled separately in MultiLevelStorage::from_config // Mark it by continuing - it will be added to the storage list there @@ -606,6 +626,7 @@ impl CacheConfigs { webdav, oss, cos, + vercel_artifacts, multilevel, } = other; @@ -639,6 +660,9 @@ impl CacheConfigs { if cos.is_some() { self.cos = cos; } + if vercel_artifacts.is_some() { + self.vercel_artifacts = vercel_artifacts; + } if multilevel.is_some() { self.multilevel = multilevel; @@ -1092,6 +1116,21 @@ fn config_from_env() -> Result { None }; + // ======= Vercel Artifacts ======= + let vercel_artifacts = if let Ok(access_token) = env::var("SCCACHE_VERCEL_ARTIFACTS_TOKEN") { + let endpoint = env::var("SCCACHE_VERCEL_ARTIFACTS_ENDPOINT").ok(); + let team_id = env::var("SCCACHE_VERCEL_ARTIFACTS_TEAM_ID").ok(); + let team_slug = env::var("SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG").ok(); + Some(VercelArtifactsCacheConfig { + access_token, + endpoint, + team_id, + team_slug, + }) + } else { + None + }; + // ======= Local ======= let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from); let disk_sz = env::var("SCCACHE_CACHE_SIZE") @@ -1162,6 +1201,7 @@ fn config_from_env() -> Result { webdav, oss, cos, + vercel_artifacts, multilevel, }; @@ -2362,6 +2402,7 @@ key_prefix = "cosprefix" endpoint: Some("cos.na-siliconvalley.myqcloud.com".to_owned()), key_prefix: "cosprefix".into(), }), + vercel_artifacts: None, multilevel: None, }, dist: DistConfig { diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 255a8c1021..1440e5b912 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -181,6 +181,7 @@ pub fn sccache_client_cfg( webdav: None, oss: None, cos: None, + vercel_artifacts: None, multilevel: None, }, dist: sccache::config::DistConfig {