diff --git a/Cargo.lock b/Cargo.lock index 1126c5f004..430a8e8e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,9 +120,18 @@ checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69" [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" [[package]] name = "arrayref" @@ -157,6 +166,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -180,6 +200,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backon" version = "1.6.0" @@ -187,8 +230,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", - "gloo-timers", - "tokio", ] [[package]] @@ -230,17 +271,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bb8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d8b8e1a22743d9241575c6ba822cf9c8fef34771c86ab7e477a4fbfd254e5" -dependencies = [ - "futures-util", - "parking_lot", - "tokio", -] - [[package]] name = "bincode" version = "1.3.3" @@ -343,10 +373,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.16" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -440,6 +471,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "codspeed" version = "4.2.0" @@ -530,6 +570,15 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "condtype" version = "1.3.0" @@ -798,6 +847,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.9.0" @@ -852,6 +907,37 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastpool" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "505402589aaeb2f89357bf8dfb259046c693a3c9a68b874a0ca8c0fb99e0fb4c" +dependencies = [ + "mea", + "scopeguard", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -870,6 +956,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "flate2" version = "1.1.5" @@ -940,6 +1032,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.30" @@ -1087,18 +1185,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "gzp" version = "2.0.1" @@ -1501,24 +1587,26 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", - "serde", - "windows-sys 0.59.0", + "serde_core", + "wasm-bindgen", + "windows-sys 0.61.1", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -1574,6 +1662,23 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.11", + "js-sys", + "pem", + "serde", + "serde_json", + "signature", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1646,9 +1751,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" @@ -1666,6 +1771,15 @@ dependencies = [ "digest", ] +[[package]] +name = "mea" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.7.5" @@ -1901,35 +2015,231 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opendal" version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "opendal-core", + "opendal-layer-logging", + "opendal-service-azblob", + "opendal-service-cos", + "opendal-service-gcs", + "opendal-service-ghac", + "opendal-service-memcached", + "opendal-service-oss", + "opendal-service-redis", + "opendal-service-s3", + "opendal-service-vercel-artifacts", + "opendal-service-webdav", +] + +[[package]] +name = "opendal-core" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" dependencies = [ "anyhow", - "backon", "base64 0.22.1", - "bb8", "bytes", - "crc32c", "futures", - "getrandom 0.2.11", - "ghac", "http", "http-body", "jiff", "log", "md-5", + "mea", "percent-encoding", - "prost", "quick-xml 0.38.4", - "redis", - "reqsign 0.16.5", + "reqsign-core 3.0.0", "reqwest", "serde", "serde_json", - "sha2", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "base64 0.22.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", + "sha2", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "http", + "opendal-core", +] + +[[package]] +name = "opendal-service-cos" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "reqsign-tencent-cos", + "serde", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "async-trait", + "bytes", + "http", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-ghac" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "ghac", + "http", + "log", + "opendal-core", + "opendal-service-azblob", + "prost", + "reqsign-azure-storage", + "reqsign-core 3.0.0", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "opendal-service-memcached" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "fastpool", + "opendal-core", + "serde", + "tokio", + "url", +] + +[[package]] +name = "opendal-service-oss" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aliyun-oss", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", +] + +[[package]] +name = "opendal-service-redis" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "fastpool", + "http", + "opendal-core", + "redis", + "serde", + "tokio", +] + +[[package]] +name = "opendal-service-s3" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "http", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core 3.0.0", + "reqsign-file-read-tokio 3.0.0", + "serde", + "url", +] + +[[package]] +name = "opendal-service-vercel-artifacts" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-webdav" +version = "0.55.0" +source = "git+https://github.com/mmastrac/opendal.git?rev=723ba7f7359e#723ba7f7359ede8ed52755f8bc1d4823e7070c8c" +dependencies = [ + "anyhow", + "bytes", + "http", + "log", + "mea", + "opendal-core", + "quick-xml 0.38.4", + "serde", ] [[package]] @@ -2002,6 +2312,12 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2257,9 +2573,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -2267,9 +2583,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" dependencies = [ "memchr", "serde", @@ -2413,18 +2729,19 @@ dependencies = [ [[package]] name = "redis" -version = "0.32.7" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" +checksum = "d76e41a79ae5cbb41257d84cf4cf0db0bb5a95b11bf05c62c351de4fe748620d" dependencies = [ "arc-swap", + "arcstr", + "async-lock", "backon", "bytes", "cfg-if 1.0.0", "combine", "crc16", "futures-channel", - "futures-sink", "futures-util", "itoa", "log", @@ -2441,6 +2758,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "url", + "xxhash-rust", ] [[package]] @@ -2500,46 +2818,74 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqsign" -version = "0.16.5" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a80170eaab619a5dfa6428b0596c0cb85734bfa36b717a764a16abc3456a7b" +dependencies = [ + "reqsign-command-execute-tokio", + "reqsign-core 2.0.0", + "reqsign-file-read-tokio 2.0.0", + "reqsign-http-send-reqwest", +] + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" dependencies = [ "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", "form_urlencoded", - "getrandom 0.2.11", - "hex", - "hmac", - "home", "http", - "jsonwebtoken", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", - "rsa", + "reqsign-core 3.0.0", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +dependencies = [ + "anyhow", + "bytes", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "quick-xml 0.39.2", + "reqsign-core 3.0.0", "rust-ini", "serde", "serde_json", + "serde_urlencoded", "sha1", - "sha2", - "tokio", ] [[package]] -name = "reqsign" -version = "0.18.0" +name = "reqsign-azure-storage" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9a80170eaab619a5dfa6428b0596c0cb85734bfa36b717a764a16abc3456a7b" +checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" dependencies = [ - "reqsign-command-execute-tokio", - "reqsign-core", - "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "http", + "jsonwebtoken 10.3.0", + "log", + "pem", + "percent-encoding", + "reqsign-core 3.0.0", + "rsa", + "serde", + "serde_json", + "sha1", ] [[package]] @@ -2549,7 +2895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d74ecba4eca9afdd6bebf86d71e442dd4acab3fcec4461f3b96b33cf6a16b5" dependencies = [ "async-trait", - "reqsign-core", + "reqsign-core 2.0.0", "tokio", ] @@ -2575,6 +2921,28 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "reqsign-core" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "futures", + "hex", + "hmac", + "http", + "jiff", + "log", + "percent-encoding", + "sha1", + "sha2", + "windows-sys 0.61.1", +] + [[package]] name = "reqsign-file-read-tokio" version = "2.0.0" @@ -2583,7 +2951,38 @@ checksum = "262eb485bb6e8213b13ef10e86ef8613539fb03daa2123b57d96675f784b15b6" dependencies = [ "anyhow", "async-trait", - "reqsign-core", + "reqsign-core 2.0.0", + "tokio", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +dependencies = [ + "anyhow", + "reqsign-core 3.0.0", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +dependencies = [ + "form_urlencoded", + "http", + "jsonwebtoken 10.3.0", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core 3.0.0", + "rsa", + "serde", + "serde_json", + "sha2", "tokio", ] @@ -2598,10 +2997,25 @@ dependencies = [ "bytes", "http", "http-body-util", - "reqsign-core", + "reqsign-core 2.0.0", "reqwest", ] +[[package]] +name = "reqsign-tencent-cos" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +dependencies = [ + "anyhow", + "http", + "log", + "percent-encoding", + "reqsign-core 3.0.0", + "serde", + "serde_json", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -2661,7 +3075,7 @@ dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.11", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2826,7 +3240,7 @@ checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -2891,6 +3305,7 @@ dependencies = [ "ar", "assert_cmd", "async-trait", + "aws-lc-sys", "backon", "base64 0.21.7", "bincode", @@ -2916,7 +3331,7 @@ dependencies = [ "hyper-util", "itertools", "jobserver", - "jsonwebtoken", + "jsonwebtoken 9.2.0", "libc", "libmount", "linked-hash-map", @@ -2932,7 +3347,7 @@ dependencies = [ "predicates", "rand 0.8.5", "regex", - "reqsign 0.18.0", + "reqsign", "reqwest", "rouille", "semver", @@ -3223,12 +3638,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -3659,7 +4071,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.0", @@ -3912,6 +4323,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -4459,6 +4876,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yoke" version = "0.8.1" @@ -4505,9 +4928,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index 8005fb8956..9e1a1f52c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ strip = true anyhow = { version = "1.0", features = ["backtrace"] } ar = "0.9" async-trait = "0.1" +aws-lc-sys = { version = "0.39.1", features = ["prebuilt-nasm"] } backon = { version = "1", default-features = false, features = [ "std-blocking-sleep", ] } @@ -69,7 +70,9 @@ memmap2 = "0.9.4" mime = "0.3" number_prefix = "0.4" object = "0.37" -opendal = { version = "0.55.0", optional = true, default-features = false } +opendal = { version = "0.55.0", optional = true, default-features = false, features = [ + "layers-logging", +] } openssl = { version = "0.10.75", optional = true } rand = "0.8.4" regex = "1.10.3" @@ -167,6 +170,7 @@ all = [ "webdav", "oss", "cos", + "vercel_artifacts", ] azure = ["opendal/services-azblob", "reqsign", "reqwest"] cos = ["opendal/services-cos", "reqsign", "reqwest"] @@ -178,6 +182,7 @@ native-zlib = [] oss = ["opendal/services-oss", "reqsign", "reqwest"] redis = ["url", "opendal/services-redis"] s3 = ["opendal/services-s3", "reqsign", "reqwest"] +vercel_artifacts = ["opendal/services-vercel-artifacts", "reqwest"] webdav = ["opendal/services-webdav", "reqwest"] # Enable features that will build a vendored version of openssl and # statically linked with it, instead of linking against the system-wide openssl @@ -233,3 +238,6 @@ ptr_as_ptr = "warn" ref_option = "warn" semicolon_if_nothing_returned = "warn" unnecessary_semicolon = "warn" + +[patch.crates-io] +opendal = { git = "https://github.com/mmastrac/opendal.git", rev = "723ba7f7359e", package = "opendal" } diff --git a/docs/Configuration.md b/docs/Configuration.md index 5a65173ef1..29fdfcb00a 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -310,3 +310,10 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`. * `SCCACHE_COS_KEY_PREFIX` * `TENCENTCLOUD_SECRET_ID` * `TENCENTCLOUD_SECRET_KEY` + +#### Vercel Artifacts + +* `SCCACHE_VERCEL_ARTIFACTS_TOKEN` Vercel access token for the artifacts API (required) +* `SCCACHE_VERCEL_ARTIFACTS_ENDPOINT` API endpoint (default: `https://api.vercel.com`) +* `SCCACHE_VERCEL_ARTIFACTS_TEAM_ID` Vercel team ID, appended as `teamId` query parameter +* `SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG` Vercel team slug, appended as `slug` query parameter diff --git a/src/cache/cache.rs b/src/cache/cache.rs index f0b79eca63..f684424378 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -40,9 +40,12 @@ use crate::cache::s3::S3Cache; feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] use crate::cache::utils::normalize_key; +#[cfg(feature = "vercel_artifacts")] +use crate::cache::vercel_artifacts::VercelArtifactsCache; #[cfg(feature = "webdav")] use crate::cache::webdav::WebdavCache; use crate::compiler::PreprocessorCacheEntry; @@ -170,6 +173,10 @@ pub trait Storage: Send + Sync { pub struct RemoteStorage { operator: opendal::Operator, basedirs: Vec>, + /// Optional transform applied to every key (including health-check paths) + /// before it is sent to the operator. Used by backends like Vercel Artifacts + /// that only accept alphanumeric artifact IDs. + key_transform: Option String>, } #[cfg(any( @@ -185,7 +192,31 @@ pub struct RemoteStorage { ))] impl RemoteStorage { pub fn new(operator: opendal::Operator, basedirs: Vec>) -> Self { - Self { operator, basedirs } + Self { + operator, + basedirs, + key_transform: None, + } + } + + pub fn new_with_key_transform( + operator: opendal::Operator, + basedirs: Vec>, + key_transform: fn(&str) -> String, + ) -> Self { + Self { + operator, + basedirs, + key_transform: Some(key_transform), + } + } + + fn key_path(&self, key: &str) -> String { + let normalized = normalize_key(key); + match self.key_transform { + Some(transform) => transform(&normalized), + None => normalized, + } } } @@ -204,7 +235,7 @@ impl RemoteStorage { #[async_trait] impl Storage for RemoteStorage { async fn get(&self, key: &str) -> Result { - match self.operator.read(&normalize_key(key)).await { + match self.operator.read(&self.key_path(key)).await { Ok(res) => { let hit = CacheRead::from(io::Cursor::new(res.to_bytes()))?; Ok(Cache::Hit(hit)) @@ -227,10 +258,10 @@ impl Storage for RemoteStorage { async fn check(&self) -> Result { use opendal::ErrorKind; - let path = ".sccache_check"; + let path = self.key_path(".sccache_check"); // Read is required, return error directly if we can't read . - match self.operator.read(path).await { + match self.operator.read(&path).await { Ok(_) => (), // Read not exist file with not found is ok. Err(err) if err.kind() == ErrorKind::NotFound => (), @@ -249,7 +280,7 @@ impl Storage for RemoteStorage { Err(err) => bail!("cache storage failed to read: {:?}", err), } - let can_write = match self.operator.write(path, "Hello, World!").await { + let can_write = match self.operator.write(&path, "Hello, World!").await { Ok(_) => true, Err(err) if err.kind() == ErrorKind::AlreadyExists => true, // Tolerate all other write errors because we can do read at least. @@ -306,7 +337,7 @@ impl Storage for RemoteStorage { /// For backfill we need the raw bytes to write directly to another cache level. async fn get_raw(&self, key: &str) -> Result> { trace!("opendal::Operator::get_raw({})", key); - match self.operator.read(&normalize_key(key)).await { + match self.operator.read(&self.key_path(key)).await { Ok(res) => { let data = res.to_bytes(); trace!( @@ -337,7 +368,7 @@ impl Storage for RemoteStorage { trace!("opendal::Operator::put_raw({}, {} bytes)", key, data.len()); let start = std::time::Instant::now(); - self.operator.write(&normalize_key(key), data).await?; + self.operator.write(&self.key_path(key), data).await?; Ok(start.elapsed()) } @@ -542,6 +573,24 @@ pub fn build_single_cache( let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) } + #[cfg(feature = "vercel_artifacts")] + CacheType::VercelArtifacts(c) => { + debug!("Init vercel artifacts cache"); + + let operator = VercelArtifactsCache::build( + &c.access_token, + c.endpoint.as_deref(), + c.team_id.as_deref(), + c.team_slug.as_deref(), + ) + .map_err(|err| anyhow!("create vercel artifacts cache failed: {err:?}"))?; + let storage = RemoteStorage::new_with_key_transform( + operator, + basedirs.to_vec(), + crate::cache::vercel_artifacts::sanitize_key, + ); + Ok(Arc::new(storage)) + } #[allow(unreachable_patterns)] _ => { bail!("Cache type not supported with current feature configuration") diff --git a/src/cache/memcached.rs b/src/cache/memcached.rs index 35f65a8f79..37427e19c1 100644 --- a/src/cache/memcached.rs +++ b/src/cache/memcached.rs @@ -21,6 +21,21 @@ use opendal::services::Memcached; use crate::errors::*; +/// Resolve hostname in a memcached endpoint URL to an IP address. +/// The new opendal memcached service uses SocketAddr::parse internally which +/// doesn't support DNS hostnames, only IP:port. This works around that by +/// resolving tcp://hostname:port to tcp://ip:port. +fn resolve_memcached_endpoint(url: &str) -> String { + if let Some(rest) = url.strip_prefix("tcp://") { + if let Ok(addrs) = std::net::ToSocketAddrs::to_socket_addrs(&rest) { + if let Some(addr) = addrs.into_iter().next() { + return format!("tcp://{}", addr); + } + } + } + url.to_string() +} + #[derive(Clone)] pub struct MemcachedCache; @@ -32,7 +47,10 @@ impl MemcachedCache { key_prefix: &str, expiration: u32, ) -> Result { - let mut builder = Memcached::default().endpoint(url); + // The new opendal memcached service uses SocketAddr::parse which doesn't + // support hostnames. Resolve hostname to IP if the endpoint uses tcp://. + let url = resolve_memcached_endpoint(url); + let mut builder = Memcached::default().endpoint(&url); if let Some(username) = username { builder = builder.username(username); diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 23647d1fed..ed0fee44bf 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -37,6 +37,8 @@ pub mod redis; #[cfg(feature = "s3")] pub mod s3; pub(crate) mod utils; +#[cfg(feature = "vercel_artifacts")] +pub mod vercel_artifacts; #[cfg(feature = "webdav")] pub mod webdav; @@ -47,7 +49,8 @@ pub mod webdav; feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] pub(crate) mod http_client; diff --git a/src/cache/multilevel.rs b/src/cache/multilevel.rs index f9296a4ab1..8284f4643f 100644 --- a/src/cache/multilevel.rs +++ b/src/cache/multilevel.rs @@ -427,7 +427,8 @@ impl MultiLevelStorage { feature = "s3", feature = "webdav", feature = "oss", - feature = "cos" + feature = "cos", + feature = "vercel_artifacts" ))] { let cache_type = match level_name.to_lowercase().as_str() { @@ -453,6 +454,12 @@ impl MultiLevelStorage { "oss" => config.cache_configs.oss.clone().map(CacheType::OSS), #[cfg(feature = "cos")] "cos" => config.cache_configs.cos.clone().map(CacheType::COS), + #[cfg(feature = "vercel_artifacts")] + "vercel_artifacts" => config + .cache_configs + .vercel_artifacts + .clone() + .map(CacheType::VercelArtifacts), _ => { return Err(anyhow!("Unknown cache level: '{}'", level_name)); } diff --git a/src/cache/vercel_artifacts.rs b/src/cache/vercel_artifacts.rs new file mode 100644 index 0000000000..0a9ba46359 --- /dev/null +++ b/src/cache/vercel_artifacts.rs @@ -0,0 +1,65 @@ +use opendal::Operator; +use opendal::layers::{HttpClientLayer, LoggingLayer}; +use opendal::services::VercelArtifacts; + +use crate::errors::*; + +use super::http_client::set_user_agent; + +/// Sanitize a cache key so it matches the Vercel Artifacts API's hash regex +/// (`/^[a-fA-F0-9]+$/`). Only hex characters [0-9a-f] are passed through; +/// every other byte is replaced with its two-character uppercase hex encoding +/// (e.g. `/` → `2F`, `.` → `2E`, `k` → `6B`). +/// +/// This keeps already-valid lowercase hex hash keys (the common case) untouched +/// while safely encoding the `/` separators from `normalize_key` and any other +/// non-hex characters. +pub fn sanitize_key(key: &str) -> String { + let mut out = String::with_capacity(key.len()); + for b in key.bytes() { + if b.is_ascii_hexdigit() { + out.push(b as char); + } else { + out.push( + char::from_digit((b >> 4) as u32, 16) + .unwrap() + .to_ascii_uppercase(), + ); + out.push( + char::from_digit((b & 0xf) as u32, 16) + .unwrap() + .to_ascii_uppercase(), + ); + } + } + out +} + +/// A cache that stores entries in Vercel Artifacts. +pub struct VercelArtifactsCache; + +impl VercelArtifactsCache { + pub fn build( + access_token: &str, + endpoint: Option<&str>, + team_id: Option<&str>, + team_slug: Option<&str>, + ) -> Result { + let mut builder = VercelArtifacts::default().access_token(access_token); + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } + if let Some(team_id) = team_id { + builder = builder.team_id(team_id); + } + if let Some(team_slug) = team_slug { + builder = builder.team_slug(team_slug); + } + + let op = Operator::new(builder)? + .layer(HttpClientLayer::new(set_user_agent())) + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 2fdbcedb0e..e92523a2bd 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -62,6 +62,52 @@ use std::time; use crate::errors::*; +/// CARGO_* environment variables known to contain absolute paths that should +/// have basedir prefixes stripped for cross-machine cache portability. +/// See: https://doc.rust-lang.org/cargo/reference/environment-variables.html +const CARGO_PATH_ENV_VARS: &[&str] = &[ + "CARGO_MANIFEST_DIR", + "CARGO_MANIFEST_PATH", + "CARGO_TARGET_TMPDIR", + "CARGO_WORKSPACE_DIR", +]; + +/// Prefixes of CARGO_* environment variables that contain absolute paths. +/// Variables matching these prefixes have their values basedir-stripped. +const CARGO_PATH_ENV_PREFIXES: &[&str] = &["CARGO_BIN_EXE_"]; + +/// Returns true if a CARGO_* env var is known to contain an absolute path. +fn is_cargo_path_var(var: &str) -> bool { + CARGO_PATH_ENV_VARS.contains(&var) + || CARGO_PATH_ENV_PREFIXES.iter().any(|&p| var.starts_with(p)) +} + +/// Strip a basedir prefix from a byte slice, returning the relative portion. +/// Basedirs are pre-normalized with trailing `/` (see config.rs), so the +/// result is a clean relative path. On Windows, the value is normalized +/// (lowercase + forward slashes) before comparison since basedirs are stored +/// normalized. +fn strip_basedir_prefix<'a>(value: &'a [u8], basedirs: &[Vec]) -> Cow<'a, [u8]> { + if basedirs.is_empty() { + return Cow::Borrowed(value); + } + + #[cfg(target_os = "windows")] + let normalized = crate::util::normalize_win_path(value); + #[cfg(not(target_os = "windows"))] + let normalized = value; + + for basedir in basedirs { + if normalized.starts_with(basedir) { + #[cfg(target_os = "windows")] + return Cow::Owned(normalized[basedir.len()..].to_vec()); + #[cfg(not(target_os = "windows"))] + return Cow::Borrowed(&value[basedir.len()..]); + } + } + Cow::Borrowed(value) +} + #[cfg(feature = "dist-client")] const RLIB_PREFIX: &str = "lib"; #[cfg(feature = "dist-client")] @@ -1343,10 +1389,11 @@ where _may_dist: bool, pool: &tokio::runtime::Handle, _rewrite_includes_only: bool, - _storage: Arc, + storage: Arc, _cache_control: CacheControl, ) -> Result> { trace!("[{}]: generate_hash_key", self.parsed_args.crate_name); + let basedirs = storage.basedirs(); // TODO: this doesn't produce correct arguments if they should be concatenated - should use iter_os_strings let os_string_arguments: Vec<(OsString, Option)> = self .parsed_args @@ -1493,7 +1540,12 @@ where a }) }; - args.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from arguments before hashing. Arguments like + // --remap-path-prefix=/abs/path=..., -Clinker=/abs/path, etc. contain + // absolute paths that differ across machines. See mozilla/sccache#2652. + let args_bytes = args.as_encoded_bytes(); + crate::util::strip_basedirs(args_bytes, basedirs) + .hash(&mut HashToDigest { digest: &mut m }); // 4. The digest of all source files (this includes src file from cmdline). // 5. The digest of all files listed on the commandline (self.externs). // 6. The digest of all static libraries listed on the commandline (self.staticlibs). @@ -1513,7 +1565,10 @@ where for (var, val) in env_deps.iter() { var.hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - val.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from dep-info env var values (e.g. OUT_DIR) + // to enable cross-machine cache hits. + let val_bytes = val.as_encoded_bytes(); + strip_basedir_prefix(val_bytes, basedirs).hash(&mut HashToDigest { digest: &mut m }); } let mut env_vars: Vec<_> = env_vars .iter() @@ -1544,10 +1599,21 @@ where var.hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - val.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefixes from path-containing CARGO_* vars + // to enable cross-machine cache hits. + let var_str = var.to_string_lossy(); + if is_cargo_path_var(&var_str) { + let val_bytes = val.as_encoded_bytes(); + strip_basedir_prefix(val_bytes, basedirs) + .hash(&mut HashToDigest { digest: &mut m }); + } else { + val.hash(&mut HashToDigest { digest: &mut m }); + } } // 9. The cwd of the compile. This will wind up in the rlib. - cwd.hash(&mut HashToDigest { digest: &mut m }); + // Strip basedir prefix for cross-machine cache portability. + let cwd_bytes = cwd.as_os_str().as_encoded_bytes(); + strip_basedir_prefix(cwd_bytes, basedirs).hash(&mut HashToDigest { digest: &mut m }); // 10. The version of the compiler. self.version.hash(&mut HashToDigest { digest: &mut m }); // 11. SCCACHE_RUST_CRATE_TYPE_ALLOW_HASH, if set and we have unsupported @@ -3541,8 +3607,12 @@ proc_macro false m.update(CACHE_VERSION); // sysroot shlibs digests. m.update(FAKE_DIGEST.as_bytes()); - // Arguments, with cfgs sorted at the end. - OsStr::new("ab--cfgabc--cfgxyz").hash(&mut HashToDigest { digest: &mut m }); + // Arguments, with cfgs sorted at the end (hashed as bytes via strip_basedirs). + // With empty basedirs, strip_basedirs returns Cow::Borrowed of the original bytes. + let args_str = OsStr::new("ab--cfgabc--cfgxyz"); + args_str + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); // bar.rs (source file, from dep-info) m.update(empty_digest.as_bytes()); // foo.rs (source file, from dep-info) @@ -3552,14 +3622,21 @@ proc_macro false // libbaz.a (static library, from staticlibs), containing a single // file, baz.o, consisting of 1024 bytes of zeroes. m.update(libbaz_a_digest.as_bytes()); - // Env vars + // Env vars (dep-info env vars hashed as bytes via strip_basedir_prefix) OsStr::new("CARGO_BLAH").hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); - OsStr::new("abc").hash(&mut HashToDigest { digest: &mut m }); + OsStr::new("abc") + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); OsStr::new("CARGO_PKG_NAME").hash(&mut HashToDigest { digest: &mut m }); m.update(b"="); OsStr::new("foo").hash(&mut HashToDigest { digest: &mut m }); - f.tempdir.path().hash(&mut HashToDigest { digest: &mut m }); + // cwd (hashed as bytes via strip_basedir_prefix) + f.tempdir + .path() + .as_os_str() + .as_encoded_bytes() + .hash(&mut HashToDigest { digest: &mut m }); TEST_RUSTC_VERSION.hash(&mut HashToDigest { digest: &mut m }); let digest = m.finish(); assert_eq!(res.key, digest); @@ -3893,6 +3970,144 @@ proc_macro false ); } + fn hash_key_with_basedirs( + f: &TestFixture, + args: &[&'static str], + env_vars: &[(OsString, OsString)], + pre_func: F, + basedirs: Vec>, + ) -> String + where + F: Fn(&Path) -> Result<()>, + { + let oargs = args.iter().map(OsString::from).collect::>(); + let parsed_args = match parse_arguments(&oargs, f.tempdir.path()) { + CompilerArguments::Ok(parsed_args) => parsed_args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + { + let src = &"foo.rs"; + f.touch(src).expect("Failed to create foo.rs"); + } + for e in parsed_args.externs.iter() { + f.touch(e.to_str().unwrap()) + .expect("Failed to create extern"); + } + pre_func(f.tempdir.path()).expect("Failed to execute pre_func"); + let mut hasher = Box::new(RustHasher { + executable: "rustc".into(), + host: "x86-64-unknown-unknown-unknown".to_owned(), + version: TEST_RUSTC_VERSION.to_string(), + sysroot: f.tempdir.path().join("sysroot"), + compiler_shlibs_digests: vec![], + #[cfg(feature = "dist-client")] + rlib_dep_reader: None, + parsed_args, + }); + + let creator = new_creator(); + let runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); + + mock_dep_info(&creator, &["foo.rs"]); + mock_file_names(&creator, &["foo.rlib"]); + hasher + .generate_hash_key( + &creator, + f.tempdir.path().to_owned(), + env_vars.to_owned(), + false, + &pool, + false, + Arc::new(MockStorage::with_basedirs(None, false, basedirs)), + CacheControl::Default, + ) + .wait() + .unwrap() + .key + } + + #[test] + fn test_basedirs_strips_cwd_and_cargo_manifest_dir() { + let f = TestFixture::new(); + let cwd = f.tempdir.path().to_string_lossy().into_owned(); + + let args = &[ + "--emit", + "link", + "foo.rs", + "--out-dir", + "out", + "--crate-name", + "foo", + "--crate-type", + "lib", + ]; + + let manifest_dir = format!("{}/some/pkg", cwd); + let env_vars = vec![ + ( + OsString::from("CARGO_MANIFEST_DIR"), + OsString::from(&manifest_dir), + ), + (OsString::from("CARGO_PKG_NAME"), OsString::from("foo")), + ]; + + // Hash without basedirs + let key_without = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![]); + + // Hash with basedirs that strip the cwd prefix. + // Basedirs are normalized at config time (forward slashes, lowercase on Windows, + // trailing slash) — replicate that here. + let basedir = cwd.into_bytes(); + #[cfg(target_os = "windows")] + let basedir = crate::util::normalize_win_path(&basedir); + let mut basedir = basedir; + basedir.push(b'/'); + let key_with = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir]); + + // The keys should differ because basedirs changes the hash + assert_ne!(key_without, key_with, "basedirs should change the hash key"); + + // Two different "machines" with different cwds but same basedirs should + // produce the same hash. We simulate this by noting that the basedir- + // stripped hash is deterministic regardless of cwd. + // (We can't easily create two different tempdirs with the same content + // in this test framework, but we verify the stripping changes the hash.) + } + + #[test] + fn test_basedirs_deterministic() { + // Running the same compilation with the same basedirs twice should + // produce the same hash, and it should differ from no-basedirs. + let f = TestFixture::new(); + let cwd = f.tempdir.path().to_string_lossy().into_owned(); + + let args = &[ + "--emit", + "link", + "foo.rs", + "--out-dir", + "out", + "--crate-name", + "foo", + "--crate-type", + "lib", + ]; + let env_vars = vec![(OsString::from("CARGO_PKG_NAME"), OsString::from("foo"))]; + + let basedir = cwd.into_bytes(); + #[cfg(target_os = "windows")] + let basedir = crate::util::normalize_win_path(&basedir); + let mut basedir = basedir; + basedir.push(b'/'); + + let key1 = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir.clone()]); + let key2 = hash_key_with_basedirs(&f, args, &env_vars, nothing, vec![basedir]); + + assert_eq!(key1, key2, "Same basedir should produce deterministic hash"); + } + #[test] fn test_parse_unstable_profile_flag() { let h = parses!( diff --git a/src/config.rs b/src/config.rs index ecbe98b8d8..92f46635d0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -349,6 +349,15 @@ pub struct GHACacheConfig { pub version: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VercelArtifactsCacheConfig { + pub access_token: String, + pub endpoint: Option, + pub team_id: Option, + pub team_slug: Option, +} + /// Memcached's default value of expiration is 10800s (3 hours), which is too /// short for use case of sccache. /// @@ -484,6 +493,7 @@ pub enum CacheType { Webdav(WebdavCacheConfig), OSS(OSSCacheConfig), COS(COSCacheConfig), + VercelArtifacts(VercelArtifactsCacheConfig), } #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -499,6 +509,7 @@ pub struct CacheConfigs { pub webdav: Option, pub oss: Option, pub cos: Option, + pub vercel_artifacts: Option, /// Multi-level cache configuration pub multilevel: Option, } @@ -518,6 +529,7 @@ impl CacheConfigs { webdav, oss, cos, + vercel_artifacts, multilevel: _, } = self; @@ -530,7 +542,8 @@ impl CacheConfigs { .or_else(|| azure.map(CacheType::Azure)) .or_else(|| webdav.map(CacheType::Webdav)) .or_else(|| oss.map(CacheType::OSS)) - .or_else(|| cos.map(CacheType::COS)); + .or_else(|| cos.map(CacheType::COS)) + .or_else(|| vercel_artifacts.map(CacheType::VercelArtifacts)); let fallback = disk.unwrap_or_default(); @@ -576,6 +589,13 @@ impl CacheConfigs { "oss" => self.oss.clone().map(CacheType::OSS).ok_or_else(|| { anyhow!("OSS cache not configured but specified in levels") })?, + "vercel_artifacts" => self + .vercel_artifacts + .clone() + .map(CacheType::VercelArtifacts) + .ok_or_else(|| { + anyhow!("Vercel Artifacts cache not configured but specified in levels") + })?, "disk" => { // Disk cache is handled separately in MultiLevelStorage::from_config // Mark it by continuing - it will be added to the storage list there @@ -606,6 +626,7 @@ impl CacheConfigs { webdav, oss, cos, + vercel_artifacts, multilevel, } = other; @@ -639,6 +660,9 @@ impl CacheConfigs { if cos.is_some() { self.cos = cos; } + if vercel_artifacts.is_some() { + self.vercel_artifacts = vercel_artifacts; + } if multilevel.is_some() { self.multilevel = multilevel; @@ -1092,6 +1116,21 @@ fn config_from_env() -> Result { None }; + // ======= Vercel Artifacts ======= + let vercel_artifacts = if let Ok(access_token) = env::var("SCCACHE_VERCEL_ARTIFACTS_TOKEN") { + let endpoint = env::var("SCCACHE_VERCEL_ARTIFACTS_ENDPOINT").ok(); + let team_id = env::var("SCCACHE_VERCEL_ARTIFACTS_TEAM_ID").ok(); + let team_slug = env::var("SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG").ok(); + Some(VercelArtifactsCacheConfig { + access_token, + endpoint, + team_id, + team_slug, + }) + } else { + None + }; + // ======= Local ======= let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from); let disk_sz = env::var("SCCACHE_CACHE_SIZE") @@ -1162,6 +1201,7 @@ fn config_from_env() -> Result { webdav, oss, cos, + vercel_artifacts, multilevel, }; @@ -2362,6 +2402,7 @@ key_prefix = "cosprefix" endpoint: Some("cos.na-siliconvalley.myqcloud.com".to_owned()), key_prefix: "cosprefix".into(), }), + vercel_artifacts: None, multilevel: None, }, dist: DistConfig { diff --git a/src/test/mock_storage.rs b/src/test/mock_storage.rs index d4260e79e2..e2bd75f114 100644 --- a/src/test/mock_storage.rs +++ b/src/test/mock_storage.rs @@ -28,6 +28,7 @@ pub struct MockStorage { tx: mpsc::UnboundedSender>, delay: Option, preprocessor_cache_mode: bool, + basedirs: Vec>, } impl MockStorage { @@ -39,6 +40,23 @@ impl MockStorage { rx: Arc::new(Mutex::new(rx)), delay, preprocessor_cache_mode, + basedirs: vec![], + } + } + + /// Create a new `MockStorage` with basedirs configured. + pub(crate) fn with_basedirs( + delay: Option, + preprocessor_cache_mode: bool, + basedirs: Vec>, + ) -> MockStorage { + let (tx, rx) = mpsc::unbounded(); + Self { + tx, + rx: Arc::new(Mutex::new(rx)), + delay, + preprocessor_cache_mode, + basedirs, } } @@ -75,6 +93,9 @@ impl Storage for MockStorage { async fn max_size(&self) -> Result> { Ok(None) } + fn basedirs(&self) -> &[Vec] { + &self.basedirs + } fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { PreprocessorCacheModeConfig { use_preprocessor_cache_mode: self.preprocessor_cache_mode, diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 255a8c1021..1440e5b912 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -181,6 +181,7 @@ pub fn sccache_client_cfg( webdav: None, oss: None, cos: None, + vercel_artifacts: None, multilevel: None, }, dist: sccache::config::DistConfig {