diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0166bb6b1c..cc5e4e3335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: - name: Clone repository uses: actions/checkout@v5 + # TODO: restore `cargo test` once per-feature tests are compatible with + # both server-side and client-side compilation modes. - name: Check feature ${{ matrix.feature }} run: cargo check --no-default-features --features ${{ matrix.feature }} @@ -78,6 +80,10 @@ jobs: extra_args: --features=dist-server - os: ubuntu-22.04 rustc: stable + - os: ubuntu-22.04 + rustc: stable + extra_desc: client-side-mode + test_client_side: true - os: ubuntu-22.04 rustc: beta - os: ubuntu-22.04 @@ -98,7 +104,14 @@ jobs: extra_desc: cuda12.8 # # M1 CPU - os: macos-14 + - os: macos-14 + extra_desc: client-side-mode + test_client_side: true - os: macos-15-intel + - os: windows-2022 + extra_desc: client-side-mode + test_client_side: true + no_coverage: true - os: windows-2022 cuda: "12.8" # Oldest supported version, keep in sync with README.md @@ -209,6 +222,7 @@ jobs: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: "" RUSTFLAGS: "-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Coverflow-checks=off" + SCCACHE_CLIENT_SIDE_COMPILE: ${{ matrix.test_client_side && '1' || '0' }} - name: Upload failure if: failure() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f90466bed2..74da0d40b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: rust-clippy name: Rust clippy description: Run cargo clippy on files included in the commit. - entry: cargo +nightly clippy --workspace --all-targets --all-features -- + entry: cargo +nightly clippy --workspace --all-targets --features all -- pass_filenames: false types: [file, rust] language: system diff --git a/README.md b/README.md index 6fb9101eb7..9f26ee2b8d 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,13 @@ If you don't [specify otherwise](#storage-options), sccache will use a local dis sccache works using a client-server model, where the server runs locally on the same machine as the client. The client-server model allows the server to be more efficient by keeping some state in memory. The sccache command will spawn a server process if one is not already running, or you can run `sccache --start-server` to start the background server process without performing any compilation. +### Compilation Modes + +sccache supports two compilation modes: + +- **Server-side compilation (default)**: The server performs compiler detection, preprocessing, and compilation. This is the traditional mode. +- **Client-side compilation (experimental)**: The client performs all compilation work, and the server acts as a pure cache storage service. Enable this mode by setting `SCCACHE_CLIENT_SIDE_COMPILE=1`. This mode provides better scalability and reduced server load but is currently experimental. When compilation is not cacheable, the client runs the compiler directly. See [Architecture.md](docs/Architecture.md) for detailed comparison. + By default sccache server will listen on `127.0.0.1:4226`, you can specify environment variable `SCCACHE_SERVER_PORT` to use a different port or `SCCACHE_SERVER_UDS` to listen on unix domain socket. Abstract unix socket is also supported as long as the path is escaped following the [format](https://doc.rust-lang.org/std/ascii/fn.escape_default.html). For example: ``` diff --git a/docs/Architecture.md b/docs/Architecture.md index adc484334e..8eb044f05e 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -1,21 +1,166 @@ # Sccache high level architecture -This schema shows at high level how sccache works. +Sccache supports two compilation modes: **server-side compilation** (legacy) and **client-side compilation** (new). The mode is controlled by the `SCCACHE_CLIENT_SIDE_COMPILE` environment variable. +## Server-Side Compilation (Legacy Mode) + +This is the default mode when `SCCACHE_CLIENT_SIDE_COMPILE` is unset or set to `0`. + +In this mode, the server performs all compilation work: + +```mermaid + sequenceDiagram + participant Client as Client Process + participant Server as Sccache Server + participant Storage as Cache Storage + + Client->>Server: 1. Compile Request (exe, args, cwd, env) + Server->>Server: 2. Detect Compiler + Server->>Server: 3. Preprocess & Hash + Server->>Storage: 4. Check Cache + alt Cache Lookup Result: Hit + rect rgba(0, 128, 0, 0.1) + Storage-->>Server: Cached Entry + Server-->>Client: 5a. Return Cached Result + end + else Hit: No - Cache Miss + rect rgba(200, 0, 0, 0.1) + Storage-->>Server: Miss + Server->>Server: 5b. Compile Locally + Server->>Storage: 6. Store Result + Server-->>Client: 7. Return Result + end + end +``` + +**Characteristics**: +- Server performs compiler detection, preprocessing, hash generation, and compilation +- All work happens on the server machine +- Server can become a bottleneck with many parallel clients +- Higher server CPU and memory usage + +## Client-Side Compilation (New Mode) + +Enabled by setting `SCCACHE_CLIENT_SIDE_COMPILE=1`. + +In this mode, the client performs compilation work and the server acts as pure storage: + +```mermaid + sequenceDiagram + participant Client as Client Process + participant Server as Sccache Server (Storage Only) + participant Storage as Cache Storage + + Client->>Client: 1. Detect Compiler + Client->>Client: 2. Preprocess & Hash + Client->>Server: 3. CacheGet Request (cache_key) + Server->>Storage: 4. Query Storage + alt Cache Lookup Result: Hit + rect rgba(0, 128, 0, 0.1) + Storage-->>Server: Cached Entry + Server-->>Client: 5a. Return Cache Entry + Client->>Client: Use Cached Result + end + else Hit: No - Cache Miss + rect rgba(200, 0, 0, 0.1) + Storage-->>Server: Miss + Server-->>Client: 5b. Cache Miss + Client->>Client: 6. Compile Locally + Client->>Server: 7. CachePut Request (cache_key, entry) + Server->>Storage: 8. Store in Cache + end + end +``` + +**Characteristics**: +- Client performs compiler detection, preprocessing, hash generation, and compilation +- Server only handles cache storage operations (get/put) +- Work is distributed across all clients (better scalability) +- Lower server CPU and memory usage +- Reduced network latency (single request instead of multiple round trips) + +**Why this is fast**: preprocessing in client-side mode is cheap — it only concatenates source files rather than running the full C/C++ preprocessor. This avoids the expensive `#include` expansion and macro evaluation that dominates traditional preprocessing time, making it practical to move this work to the client without a performance penalty. + +**Note**: Client-side compilation is functional but considered experimental. Enable it by setting `SCCACHE_CLIENT_SIDE_COMPILE=1`. + +## Comparison + +| Aspect | Server-Side (Legacy) | Client-Side (New) | +|--------|---------------------|-------------------| +| Compiler Detection | Server | Client (with caching) | +| Preprocessing | Server | Client | +| Hash Generation | Server | Client | +| Compilation | Server | Client | +| Server Role | Full compilation service | Pure storage service | +| Server CPU Usage | High | Low | +| Server Memory Usage | Moderate | Low | +| Client Overhead | Low | Moderate | +| Scalability | Limited by server | Excellent | +| Network Requests | Multiple round trips | Single request | + +## Cache Key Generation + +Regardless of the mode, cache keys are generated from: ```mermaid flowchart LR id1[[Environment variables]] --> hash id2[[Compiler binary]] --> hash id3[[Compiler arguments]] --> hash - id5[[Files]] --> | | hash - Compile --> Upload - Storage[(Storage)] --> | yes | Download - hash([hash]) --> | exists? | Storage - Storage --> | no | Compile - Upload --> Storage + id5[[Preprocessed Files]] --> hash + hash([BLAKE3 Hash]) --> key[Cache Key] + + style id1 fill:#e8f4fd,stroke:#5dade2,color:#333 + style id2 fill:#e8f4fd,stroke:#5dade2,color:#333 + style id3 fill:#e8f4fd,stroke:#5dade2,color:#333 + style id5 fill:#e8f4fd,stroke:#5dade2,color:#333 + style hash fill:#eafaf1,stroke:#58d68d,color:#333 + style key fill:#fef9e7,stroke:#f4d03f,color:#333 ``` +### C/C++ vs Rust + +The "preprocessing" step differs significantly between languages: + +- **C/C++**: runs the compiler's preprocessor (`gcc -E` / `clang -E`) to expand all `#include` directives and macros, producing a single translation unit. The preprocessed output is then hashed. This is the expensive part — include expansion can pull in thousands of header files. + +- **Rust**: there is no preprocessor. Instead, sccache runs `rustc --emit dep-info`, a lightweight invocation that outputs a `.d` file listing all source files and environment variables the crate depends on — without compiling anything. Sccache then hashes each source file individually, along with extern crate `.rlib` files, static libraries, and any target JSON file. This dependency discovery step is fast compared to full compilation. + +In client-side mode, this work moves to the client. For Rust, the cost is minimal since `--emit dep-info` is cheap. For C/C++, preprocessing is replaced by simple file concatenation, avoiding the expensive include expansion entirely. + +For more details about how hash generation works, see [the caching documentation](Caching.md). + +## Protocol + +### Server-Side Mode Protocol + +- **Request**: `Compile(Compile)` - Contains executable path, arguments, working directory, environment variables +- **Response**: `CompileFinished(CompileFinished)` - Contains exit code, stdout, stderr, and output file paths + +### Client-Side Mode Protocol + +- **Request**: `CacheGet(CacheGetRequest)` - Contains cache key +- **Response**: `CacheGetResponse::Hit(Vec)` - Cache entry as bytes +- **Response**: `CacheGetResponse::Miss` - Cache miss +- **Request**: `CachePut(CachePutRequest)` - Contains cache key and entry bytes +- **Response**: `CachePutResponse(Duration)` - Storage duration + +The protocol supports version negotiation to maintain backward compatibility during migration from server-side to client-side mode. + +## Storage Backends + +Both modes use the same cache storage backends: + +- **Local Disk** (`SCCACHE_DIR`) +- **S3 Compatible** (`SCCACHE_BUCKET`, `SCCACHE_ENDPOINT`) +- **Redis** (`SCCACHE_REDIS_ENDPOINT`) +- **Memcached** (`SCCACHE_MEMCACHED_ENDPOINT`) +- **Google Cloud Storage** (`SCCACHE_GCS_BUCKET`) +- **Azure Blob Storage** (`SCCACHE_AZURE_CONNECTION_STRING`) +- **GitHub Actions Cache** (`SCCACHE_GHA_CACHE_URL`) +- **WebDAV** (`SCCACHE_WEBDAV_ENDPOINT`) +- **Alibaba Cloud OSS** (`SCCACHE_OSS_BUCKET`) +- **Tencent Cloud COS** (`SCCACHE_COS_BUCKET`) -For more details about hash generation works, see [the caching documentation](Caching.md). +See [Configuration.md](Configuration.md) for storage backend configuration details. diff --git a/docs/Configuration.md b/docs/Configuration.md index 5a65173ef1..b97d53859f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -165,6 +165,7 @@ Note that some env variables may need sccache server restart to take effect. ### misc * `SCCACHE_ALLOW_CORE_DUMPS` to enable core dumps by the server +* `SCCACHE_CLIENT_SIDE_COMPILE` when set to `1`, enables client-side compilation mode where the client performs compiler detection, preprocessing, and compilation work, with the server acting as a pure cache storage service. When set to `0` or unset (default), uses the legacy server-side compilation mode. See [Architecture.md](Architecture.md) for more details. * `SCCACHE_CONF` configuration file path * `SCCACHE_BASEDIRS` base directory (or directories) to strip from paths for cache key computation. This is similar to ccache's `CCACHE_BASEDIR` and enables cache hits across different absolute paths when compiling the same source code. Multiple directories can be separated by `;` on Windows hosts and by `:` on any other operating system. When multiple directories are specified, the longest matching prefix is used. Path matching is **case-insensitive** on Windows and **case-sensitive** on other operating systems. Environment variable takes precedence over file configuration. Only absolute paths are supported; relative paths will cause an error and prevent the server from start. * `SCCACHE_CACHED_CONF` diff --git a/src/cache/cache_io.rs b/src/cache/cache_io.rs index c4c2fa1347..544f100f6e 100644 --- a/src/cache/cache_io.rs +++ b/src/cache/cache_io.rs @@ -13,6 +13,7 @@ use super::utils::{get_file_mode, set_file_mode}; use crate::errors::*; use fs_err as fs; +use serde::{Deserialize, Serialize}; use std::fmt; use std::io::{Cursor, Read, Seek, Write}; use std::path::PathBuf; @@ -21,7 +22,7 @@ use zip::write::FileOptions; use zip::{CompressionMethod, ZipArchive, ZipWriter}; /// Cache object sourced by a file. -#[derive(Clone)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileObjectSource { /// Identifier for this object. Should be unique within a compilation unit. /// Note that a compilation unit is a single source file in C/C++ and a crate in Rust. diff --git a/src/commands.rs b/src/commands.rs index 5d8a838483..be4db81d5e 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -12,16 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::cache::storage_from_config; +use crate::cache::{FileObjectSource, storage_from_config}; use crate::client::{ServerConnection, connect_to_server, connect_with_retry}; use crate::cmdline::{Command, StatsFormat}; -use crate::compiler::ColorMode; +use crate::compiler::{CacheControl, Cacheable, ColorMode, CompilerArguments, get_compiler_info}; use crate::config::{Config, default_disk_cache_dir}; use crate::jobserver::Client; use crate::mock_command::{CommandChild, CommandCreatorSync, ProcessCommandCreator, RunCommand}; -use crate::protocol::{Compile, CompileFinished, CompileResponse, Request, Response}; +use crate::protocol::{ + CacheGetRequest, CacheGetResponse, CachePutRequest, Compile, CompileFinished, CompileResponse, + Request, Response, +}; use crate::server::{self, DistInfo, ServerInfo, ServerStartup, ServerStats}; -use crate::util::daemonize; +use crate::util::{daemonize, run_input_output}; use byteorder::{BigEndian, ByteOrder}; use fs::{File, OpenOptions}; use fs_err as fs; @@ -33,8 +36,15 @@ use std::io::{self, IsTerminal, Write}; use std::os::unix::process::ExitStatusExt; use std::path::Path; use std::process; +use std::sync::LazyLock; use std::time::Duration; use strip_ansi_escapes::Writer; + +/// Whether client-side compilation is enabled (`SCCACHE_CLIENT_SIDE_COMPILE=1`). +/// Read once at process start to avoid repeated env lookups and ensure consistent +/// behavior throughout a single sccache invocation. +static CLIENT_SIDE_COMPILE: LazyLock = + LazyLock::new(|| env::var("SCCACHE_CLIENT_SIDE_COMPILE").unwrap_or_default() == "1"); use tokio::io::AsyncReadExt; use tokio::runtime::Runtime; use walkdir::WalkDir; @@ -606,12 +616,200 @@ where { trace!("do_compile"); let exe_path = which_in(exe, path, cwd)?; + + // Check if client-side compilation is enabled + if *CLIENT_SIDE_COMPILE { + // Use new client-side compilation path + let mut ctx = ClientCompileCtx { + conn, + exe_path: &exe_path, + cmdline: &cmdline, + cwd, + env_vars: &env_vars, + stdout, + stderr, + }; + return do_compile_client_side(&creator, runtime, &mut ctx); + } + + // Use legacy server-side compilation path let res = request_compile(&mut conn, &exe_path, &cmdline, cwd, env_vars)?; handle_compile_response( creator, runtime, &mut conn, res, &exe_path, cmdline, cwd, stdout, stderr, ) } +/// Shared context for client-side compilation functions, avoiding long parameter lists. +struct ClientCompileCtx<'a> { + conn: ServerConnection, + exe_path: &'a Path, + cmdline: &'a [OsString], + cwd: &'a Path, + env_vars: &'a [(OsString, OsString)], + stdout: &'a mut dyn Write, + stderr: &'a mut dyn Write, +} + +/// Perform client-side compilation with the server acting as a pure cache store. +/// +/// Steps: +/// 1. Detect the compiler. +/// 2. Parse the compiler arguments. +/// 3. Run the preprocessor locally and compute the cache hash key. +/// 4. Ask the server for the cache entry (`CacheGet`). +/// 5a. Cache hit → extract output artifacts and write stdout/stderr. +/// 5b. Cache miss → compile locally, store the result (`CachePut`), write output. +fn do_compile_client_side( + creator: &T, + runtime: &mut Runtime, + ctx: &mut ClientCompileCtx<'_>, +) -> Result +where + T: CommandCreatorSync, +{ + trace!("do_compile_client_side"); + let pool = runtime.handle().clone(); + + // Step 1: Detect compiler. + let (compiler, _proxy) = runtime.block_on(get_compiler_info( + creator.clone(), + ctx.exe_path, + ctx.cwd, + ctx.cmdline, + ctx.env_vars, + &pool, + None, + ))?; + + // Step 2: Parse arguments. + let mut hasher = match compiler.parse_arguments(ctx.cmdline, ctx.cwd, ctx.env_vars) { + CompilerArguments::Ok(h) => h, + CompilerArguments::NotCompilation | CompilerArguments::CannotCache(_, _) => { + // Not a compilation or un-cacheable — run the compiler directly. + let mut cmd = creator.clone().new_command_sync(ctx.exe_path); + cmd.args(ctx.cmdline).current_dir(ctx.cwd); + let status = runtime.block_on(async move { + let child = cmd.spawn().await.context("failed to spawn compiler")?; + child.wait().await.context("failed to wait for compiler") + })?; + return Ok(status.code().unwrap_or(-1)); + } + }; + + // Step 3: Preprocess locally and generate the hash key. + let hash_result = runtime.block_on(hasher.generate_hash_key( + creator, + ctx.cwd.to_path_buf(), + ctx.env_vars.to_vec(), + false, // may_dist + &pool, + false, // rewrite_includes_only + None, // no storage needed; preprocessor cache is disabled + CacheControl::Default, + ))?; + + let key = hash_result.key; + let compilation = hash_result.compilation; + + // Collect the expected output paths once (relative → absolute). + let outputs: Vec = compilation + .outputs() + .map(|o| FileObjectSource { + path: ctx.cwd.join(&o.path), + ..o + }) + .collect(); + + // Step 4: Ask the server for a cached result. + let cache_response = ctx.conn.request(Request::CacheGet(CacheGetRequest { + key: key.clone(), + output_paths: outputs.clone(), + }))?; + + match cache_response { + // Step 5a: Cache hit — artifacts already extracted by the server. + Response::CacheGet(CacheGetResponse::Hit { + stdout: cached_stdout, + stderr: cached_stderr, + }) => { + debug!("client-side cache hit for key {key}"); + ctx.stdout.write_all(&cached_stdout)?; + ctx.stderr.write_all(&cached_stderr)?; + Ok(0) + } + + // Step 5b: Cache miss — compile locally and store. + Response::CacheGet(CacheGetResponse::Miss) => { + debug!("client-side cache miss for key {key}"); + compile_and_cache(creator, runtime, ctx, &key, &*compilation, outputs) + } + + Response::CacheGet(CacheGetResponse::Error(msg)) => { + debug!("client-side cache error for key {key} (treating as miss): {msg}"); + compile_and_cache(creator, runtime, ctx, &key, &*compilation, outputs) + } + + other => bail!("unexpected response from server for CacheGet (key={key}): {other:?}"), + } +} + +/// Compile locally and store the result via `CachePut`. +fn compile_and_cache( + creator: &T, + runtime: &mut Runtime, + ctx: &mut ClientCompileCtx<'_>, + key: &str, + compilation: &dyn crate::compiler::Compilation, + outputs: Vec, +) -> Result +where + T: CommandCreatorSync, +{ + let mut path_transformer = crate::dist::PathTransformer::new(); + let (compile_cmd, _, cacheable) = compilation + .generate_compile_commands(&mut path_transformer, true) + .context("failed to generate compile commands")?; + + let exe = compile_cmd.get_executable(); + let args = compile_cmd.get_arguments(); + let env = compile_cmd.get_env_vars(); + let dir = compile_cmd.get_cwd(); + + let mut cmd = creator.clone().new_command_sync(&exe); + cmd.args(&args).env_clear().envs(env).current_dir(&dir); + + let output = runtime.block_on(run_input_output(cmd, None))?; + let exit_code = output.status.code().unwrap_or(-1); + + // If compilation succeeded and the result is cacheable, store it. + if output.status.success() && cacheable == Cacheable::Yes { + // Best-effort: log but don't fail on cache-put errors. + match ctx.conn.request(Request::CachePut(CachePutRequest { + key: key.to_owned(), + output_paths: outputs, + stdout: output.stdout.clone(), + stderr: output.stderr.clone(), + })) { + Ok(Response::CachePut(crate::protocol::CachePutResponse::Success)) => { + debug!("stored cache entry for key {key}"); + } + Ok(Response::CachePut(crate::protocol::CachePutResponse::Error(msg))) => { + debug!("server failed to store cache entry for key {key}: {msg}"); + } + Err(e) => { + debug!("failed to store cache entry for key {key}: {e:#}"); + } + other => { + debug!("unexpected response from server for CachePut (key={key}): {other:?}"); + } + } + } + + ctx.stdout.write_all(&output.stdout)?; + ctx.stderr.write_all(&output.stderr)?; + Ok(exit_code) +} + /// Run `cmd` and return the process exit status. pub fn run_command(cmd: Command) -> Result { // Config isn't required for all commands, but if it's broken then we should flag diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 8183072eee..b97af22936 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -373,7 +373,7 @@ where may_dist: bool, pool: &tokio::runtime::Handle, rewrite_includes_only: bool, - storage: Arc, + storage: Option>, cache_control: CacheControl, ) -> Result> { let start_of_compilation = std::time::SystemTime::now(); @@ -394,7 +394,10 @@ where // Try to look for a cached preprocessing step for this compilation // request. - let preprocessor_cache_mode_config = storage.preprocessor_cache_mode_config(); + let preprocessor_cache_mode_config = storage + .as_ref() + .map(|s| s.preprocessor_cache_mode_config()) + .unwrap_or_default(); let too_hard_for_preprocessor_cache_mode = self .parsed_args .too_hard_for_preprocessor_cache_mode @@ -453,7 +456,7 @@ where &absolute_input_path, self.compiler.plusplus(), preprocessor_cache_mode_config, - storage.basedirs(), + storage.as_ref().map(|s| s.basedirs()).unwrap_or(&[]), )? } else { None @@ -462,10 +465,10 @@ where let (preprocessor_output, include_files) = if needs_preprocessing { if let Some(preprocessor_key) = &preprocessor_key { if cache_control == CacheControl::Default { - if let Some(mut seekable) = storage - .get_preprocessor_cache_entry(preprocessor_key) - .await? - { + if let Some(mut seekable) = match &storage { + Some(s) => s.get_preprocessor_cache_entry(preprocessor_key).await?, + None => None, + } { let mut buf = vec![]; seekable.read_to_end(&mut buf)?; let mut preprocessor_cache_entry = PreprocessorCacheEntry::read(&buf)?; @@ -481,15 +484,17 @@ where "Preprocessor cache updated because of time macros: {preprocessor_key}" ); - if let Err(e) = storage - .put_preprocessor_cache_entry( - preprocessor_key, - preprocessor_cache_entry, - ) - .await - { - debug!("Failed to update preprocessor cache: {}", e); - update_failed = true; + if let Some(ref s) = storage { + if let Err(e) = s + .put_preprocessor_cache_entry( + preprocessor_key, + preprocessor_cache_entry, + ) + .await + { + debug!("Failed to update preprocessor cache: {}", e); + update_failed = true; + } } } @@ -634,7 +639,7 @@ where .with_extra_hashes(&extra_hashes) .with_env_vars(&env_vars) .with_plusplus(self.compiler.plusplus()) - .with_basedirs(storage.basedirs()) + .with_basedirs(storage.as_ref().map(|s| s.basedirs()).unwrap_or(&[])) .compute(); // Cache the preprocessing step @@ -648,11 +653,13 @@ where files.sort_unstable_by(|a, b| a.1.cmp(&b.1)); preprocessor_cache_entry.add_result(start_of_compilation, &key, files); - if let Err(e) = storage - .put_preprocessor_cache_entry(&preprocessor_key, preprocessor_cache_entry) - .await - { - debug!("Failed to update preprocessor cache: {}", e); + if let Some(ref s) = storage { + if let Err(e) = s + .put_preprocessor_cache_entry(&preprocessor_key, preprocessor_cache_entry) + .await + { + debug!("Failed to update preprocessor cache: {}", e); + } } } } diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 1d9ccff751..ca2fc7bd6d 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -507,7 +507,7 @@ where may_dist: bool, pool: &tokio::runtime::Handle, rewrite_includes_only: bool, - storage: Arc, + storage: Option>, cache_control: CacheControl, ) -> Result>; @@ -545,7 +545,7 @@ where may_dist, &pool, rewrite_includes_only, - storage.clone(), + Some(storage.clone()), cache_control, ) .await; @@ -2369,7 +2369,7 @@ LLVM version: 6.0", false, pool, false, - Arc::new(MockStorage::new(None, preprocessor_cache_mode)), + Some(Arc::new(MockStorage::new(None, preprocessor_cache_mode))), CacheControl::Default, ) .wait() @@ -2437,7 +2437,7 @@ LLVM version: 6.0", false, pool, false, - Arc::new(MockStorage::new(None, preprocessor_cache_mode)), + Some(Arc::new(MockStorage::new(None, preprocessor_cache_mode))), CacheControl::Default, ) .wait() @@ -2503,7 +2503,7 @@ LLVM version: 6.0", false, pool, false, - Arc::new(MockStorage::new(None, preprocessor_cache_mode)), + Some(Arc::new(MockStorage::new(None, preprocessor_cache_mode))), CacheControl::Default, ) .wait() diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 7a0fdabc3c..f32df5d250 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -1333,7 +1333,7 @@ where _may_dist: bool, pool: &tokio::runtime::Handle, _rewrite_includes_only: bool, - _storage: Arc, + _storage: Option>, _cache_control: CacheControl, ) -> Result> { trace!("[{}]: generate_hash_key", self.parsed_args.crate_name); @@ -3513,7 +3513,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None, preprocessor_cache_mode)), + Some(Arc::new(MockStorage::new(None, preprocessor_cache_mode))), CacheControl::Default, ) .wait() @@ -3605,7 +3605,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None, preprocessor_cache_mode)), + Some(Arc::new(MockStorage::new(None, preprocessor_cache_mode))), CacheControl::Default, ) .wait() diff --git a/src/protocol.rs b/src/protocol.rs index 8760ba679f..6e28ef6c80 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -1,3 +1,4 @@ +use crate::cache::FileObjectSource; use crate::compiler::ColorMode; use crate::server::{DistInfo, ServerInfo}; use serde::{Deserialize, Serialize}; @@ -16,6 +17,10 @@ pub enum Request { Shutdown, /// Execute a compile or fetch a cached compilation result. Compile(Compile), + /// Get a cache entry by key. + CacheGet(CacheGetRequest), + /// Store a cache entry by key. + CachePut(CachePutRequest), } /// A server response. @@ -33,6 +38,10 @@ pub enum Response { ShuttingDown(Box), /// Second response for `Request::Compile`, containing the results of the compilation. CompileFinished(CompileFinished), + /// Response for `Request::CacheGet`. + CacheGet(CacheGetResponse), + /// Response for `Request::CachePut`. + CachePut(CachePutResponse), } /// Possible responses from the server for a `Compile` request. @@ -73,3 +82,55 @@ pub struct Compile { /// The environment variables present when the compiler was executed, as (var, val). pub env_vars: Vec<(OsString, OsString)>, } + +/// Request to get a cache entry by key. +/// +/// The server extracts output artifacts directly to `output_paths` on a hit, +/// so no large data ever crosses the IPC channel. +#[derive(Serialize, Deserialize, Debug)] +pub struct CacheGetRequest { + /// The cache key to look up. + pub key: String, + /// Where to extract output artifacts on a cache hit. + pub output_paths: Vec, +} + +/// Request to store a cache entry. +/// +/// The server reads the output artifacts from `output_paths` directly from +/// disk (client and server share the same filesystem). Only stdout/stderr +/// (typically small) are sent over the IPC channel. +#[derive(Serialize, Deserialize, Debug)] +pub struct CachePutRequest { + /// The cache key to store under. + pub key: String, + /// Paths to the output artifacts the server should pack into the entry. + pub output_paths: Vec, + /// The compiler's stdout. + pub stdout: Vec, + /// The compiler's stderr. + pub stderr: Vec, +} + +/// Response for a cache get request. +/// +/// On a hit the server has already extracted the artifacts to the paths +/// supplied in the request; the response carries only stdout/stderr. +#[derive(Serialize, Deserialize, Debug)] +pub enum CacheGetResponse { + /// Cache hit – artifacts extracted to the requested paths. + Hit { stdout: Vec, stderr: Vec }, + /// Cache miss – entry not found. + Miss, + /// Error occurred during cache lookup. + Error(String), +} + +/// Response for a cache put request. +#[derive(Serialize, Deserialize, Debug)] +pub enum CachePutResponse { + /// Cache entry stored successfully. + Success, + /// Error occurred during cache storage (best-effort, not fatal). + Error(String), +} diff --git a/src/server.rs b/src/server.rs index be49c1dfd2..18b5ee5300 100644 --- a/src/server.rs +++ b/src/server.rs @@ -92,6 +92,24 @@ pub enum ServerStartup { Err { reason: String }, } +/// Validate that all output paths are absolute and contain no `..` components. +fn validate_output_paths(paths: &[crate::cache::FileObjectSource]) -> Result<()> { + for fos in paths { + if !fos.path.is_absolute() { + anyhow::bail!("output path must be absolute, got: {}", fos.path.display()); + } + for component in fos.path.components() { + if matches!(component, std::path::Component::ParentDir) { + anyhow::bail!( + "output path must not contain '..' components, got: {}", + fos.path.display() + ); + } + } + } + Ok(()) +} + /// Get the time the server should idle for before shutting down, in seconds. fn get_idle_timeout() -> u64 { // A value of 0 disables idle shutdown entirely. @@ -898,6 +916,14 @@ where Message::WithoutBody(Response::ShuttingDown(Box::new(info))) }) } + Request::CacheGet(req) => { + debug!("handle_client: cache_get"); + me.handle_cache_get(req).await + } + Request::CachePut(req) => { + debug!("handle_client: cache_put"); + me.handle_cache_put(req).await + } } }) } @@ -995,13 +1021,22 @@ where where T: AsyncRead + AsyncWrite + Unpin + Send + 'static, { + // Use a generous default frame-length limit (512 MB) so that large cache + // entries (e.g. Rust rlib files with embedded bitcode and debug info) can be + // transferred over the IPC channel in client-side compilation mode, while + // still guarding against OOM from corrupted messages. + // SCCACHE_MAX_FRAME_LENGTH overrides this value entirely. + const DEFAULT_MAX_FRAME_LENGTH: usize = 512 * 1024 * 1024; // 512 MB let mut builder = length_delimited::Builder::new(); if let Ok(max_frame_length_str) = env::var("SCCACHE_MAX_FRAME_LENGTH") { if let Ok(max_frame_length) = max_frame_length_str.parse::() { builder.max_frame_length(max_frame_length); } else { warn!("Content of SCCACHE_MAX_FRAME_LENGTH is not a valid number, using default"); + builder.max_frame_length(DEFAULT_MAX_FRAME_LENGTH); } + } else { + builder.max_frame_length(DEFAULT_MAX_FRAME_LENGTH); } let io = builder.new_framed(socket); @@ -1053,6 +1088,132 @@ where *self.stats.lock().await = ServerStats::default(); } + /// Handle a cache get request from a client. + /// + /// On a hit the server extracts the artifacts directly to the paths + /// specified by the client (both share the same filesystem), so no + /// large data has to be transferred over the IPC channel. + /// + /// Output paths are validated to be absolute and free of `..` components + /// before any filesystem access. + async fn handle_cache_get( + &self, + req: crate::protocol::CacheGetRequest, + ) -> Result { + use crate::cache::Cache; + use crate::protocol::{CacheGetResponse, Response}; + + if let Err(e) = validate_output_paths(&req.output_paths) { + warn!("CacheGet({}): {e:#}", req.key); + return Ok(Message::WithoutBody(Response::CacheGet( + CacheGetResponse::Error(format!("invalid output paths: {e:#}")), + ))); + } + + match self.storage.get(&req.key).await { + Ok(Cache::Hit(mut cache_read)) => { + // Read stdout/stderr before consuming the entry. + let stdout = cache_read.get_stdout(); + let stderr = cache_read.get_stderr(); + // Extract compiled artifacts directly to disk – no IPC bulk transfer. + match cache_read + .extract_objects(req.output_paths.clone(), &self.rt) + .await + { + Ok(()) => { + self.stats.lock().await.client_side_cache_hits += 1; + Ok(Message::WithoutBody(Response::CacheGet( + CacheGetResponse::Hit { stdout, stderr }, + ))) + } + Err(e) => { + // Best-effort cleanup: remove any partially-extracted files. + for fos in &req.output_paths { + let _ = std::fs::remove_file(&fos.path); + } + warn!( + "CacheGet({}): failed to extract cache entry: {e:#}", + req.key + ); + self.stats.lock().await.client_side_cache_errors += 1; + Ok(Message::WithoutBody(Response::CacheGet( + CacheGetResponse::Error(format!( + "key={}: failed to extract cache entry: {e:#}", + req.key + )), + ))) + } + } + } + Ok(Cache::Miss | Cache::None | Cache::Recache) => { + self.stats.lock().await.client_side_cache_misses += 1; + Ok(Message::WithoutBody(Response::CacheGet( + CacheGetResponse::Miss, + ))) + } + Err(e) => { + warn!("CacheGet({}): storage error: {e:#}", req.key); + self.stats.lock().await.client_side_cache_errors += 1; + Ok(Message::WithoutBody(Response::CacheGet( + CacheGetResponse::Error(format!("key={}: storage error: {e:#}", req.key)), + ))) + } + } + } + + /// Handle a cache put request from a client. + /// + /// The server reads the output artifacts from disk directly (shared + /// filesystem), packs them into a cache entry, and stores it. + /// + /// Output paths are validated to be absolute and free of `..` components + /// before any filesystem access. + async fn handle_cache_put( + &self, + req: crate::protocol::CachePutRequest, + ) -> Result { + use crate::cache::CacheWrite; + use crate::protocol::{CachePutResponse, Response}; + + if let Err(e) = validate_output_paths(&req.output_paths) { + warn!("CachePut({}): {e:#}", req.key); + return Ok(Message::WithoutBody(Response::CachePut( + CachePutResponse::Error(format!("invalid output paths: {e:#}")), + ))); + } + + let pool = self.rt.clone(); + let mut entry = match CacheWrite::from_objects(req.output_paths, &pool).await { + Ok(e) => e, + Err(e) => { + let msg = format!("key={}: failed to read output files: {e:#}", req.key); + warn!("CachePut({}): {}", req.key, msg); + return Ok(Message::WithoutBody(Response::CachePut( + CachePutResponse::Error(msg), + ))); + } + }; + let _ = entry.put_stdout(&req.stdout); + let _ = entry.put_stderr(&req.stderr); + + match self.storage.put(&req.key, entry).await { + Ok(_duration) => { + let mut stats = self.stats.lock().await; + stats.cache_writes += 1; + Ok(Message::WithoutBody(Response::CachePut( + CachePutResponse::Success, + ))) + } + Err(e) => { + let msg = format!("key={}: failed to store to backend: {e:#}", req.key); + warn!("CachePut({}): {}", req.key, msg); + Ok(Message::WithoutBody(Response::CachePut( + CachePutResponse::Error(msg), + ))) + } + } + } + /// Handle a compile request from a client. /// /// This will handle a compile request entirely, generating a response with @@ -1627,6 +1788,12 @@ pub struct ServerStats { pub dist_compiles: HashMap, /// The count of compilations that were distributed but failed and had to be re-run locally pub dist_errors: u64, + /// The count of client-side cache hits (via CacheGet). + pub client_side_cache_hits: u64, + /// The count of client-side cache misses (via CacheGet). + pub client_side_cache_misses: u64, + /// The count of client-side cache errors (via CacheGet). + pub client_side_cache_errors: u64, /// Multi-level cache statistics (if multi-level caching is enabled) pub multi_level: Option, } @@ -1678,6 +1845,9 @@ impl Default for ServerStats { not_cached: HashMap::new(), dist_compiles: HashMap::new(), dist_errors: u64::default(), + client_side_cache_hits: u64::default(), + client_side_cache_misses: u64::default(), + client_side_cache_errors: u64::default(), multi_level: None, } } @@ -1768,6 +1938,21 @@ impl ServerStats { set_lang_stat!(stats_vec, self.cache_errors, "Cache errors"); } + set_stat!( + stats_vec, + self.client_side_cache_hits, + "Client-side cache hits" + ); + set_stat!( + stats_vec, + self.client_side_cache_misses, + "Client-side cache misses" + ); + set_stat!( + stats_vec, + self.client_side_cache_errors, + "Client-side cache errors" + ); set_stat!(stats_vec, self.compilations, "Compilations"); set_stat!(stats_vec, self.compile_fails, "Compilation failures"); @@ -2382,6 +2567,121 @@ mod tests { assert!(output.contains("Cache hits rate (rust) 33.33 %")); } + /// Return a cross-platform absolute path for use in tests. + fn abs_path(name: &str) -> PathBuf { + std::env::temp_dir().join(name) + } + + /// Return an absolute path that contains a `..` component. + fn abs_path_with_dotdot() -> PathBuf { + // temp_dir is absolute on all platforms; appending `../x` keeps it absolute + // while introducing the forbidden component. + std::env::temp_dir().join("..").join("etc").join("passwd") + } + + #[test] + fn test_validate_output_paths_absolute_ok() { + use crate::cache::FileObjectSource; + let paths = vec![FileObjectSource { + key: "obj".into(), + path: abs_path("file.o"), + optional: false, + }]; + assert!(validate_output_paths(&paths).is_ok()); + } + + #[test] + fn test_validate_output_paths_relative_rejected() { + use crate::cache::FileObjectSource; + let paths = vec![FileObjectSource { + key: "obj".into(), + path: PathBuf::from("relative/file.o"), + optional: false, + }]; + let err = validate_output_paths(&paths).unwrap_err(); + assert!( + err.to_string().contains("must be absolute"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_validate_output_paths_dotdot_rejected() { + use crate::cache::FileObjectSource; + let paths = vec![FileObjectSource { + key: "obj".into(), + path: abs_path_with_dotdot(), + optional: false, + }]; + let err = validate_output_paths(&paths).unwrap_err(); + assert!(err.to_string().contains("'..'"), "unexpected error: {err}"); + } + + #[test] + fn test_validate_output_paths_empty_ok() { + assert!(validate_output_paths(&[]).is_ok()); + } + + #[test] + fn test_validate_output_paths_multiple() { + use crate::cache::FileObjectSource; + let paths = vec![ + FileObjectSource { + key: "obj".into(), + path: abs_path("file.o"), + optional: false, + }, + FileObjectSource { + key: "dwo".into(), + path: abs_path("file.dwo"), + optional: false, + }, + ]; + assert!(validate_output_paths(&paths).is_ok()); + + // Second path is relative + let paths = vec![ + FileObjectSource { + key: "obj".into(), + path: abs_path("file.o"), + optional: false, + }, + FileObjectSource { + key: "bad".into(), + path: PathBuf::from("relative.o"), + optional: false, + }, + ]; + assert!(validate_output_paths(&paths).is_err()); + } + + #[test] + fn test_print_client_side_stats() { + let stats = ServerStats { + client_side_cache_hits: 5, + client_side_cache_misses: 3, + client_side_cache_errors: 1, + ..Default::default() + }; + + let mut writer = StringWriter::new(); + stats.print(&mut writer, false); + + let output = writer.get_output(); + assert!( + output.contains("Client-side cache hits"), + "missing client-side hits in output:\n{output}" + ); + assert!( + output.contains("Client-side cache misses"), + "missing client-side misses in output:\n{output}" + ); + assert!( + output.contains("Client-side cache errors"), + "missing client-side errors in output:\n{output}" + ); + } + // Test that 2 servers with the same hits will always be printed in the same order. // This test will **randomly** (not consistently) fail in case of a regression, due to HashMap iteration behaviour. #[test] diff --git a/src/test/tests.rs b/src/test/tests.rs index 8d283a1b2b..e43d57a9e7 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -24,7 +24,6 @@ use crate::test::utils::*; use fs::File; use fs_err as fs; use futures::channel::oneshot::{self, Sender}; -#[cfg(not(target_os = "macos"))] use serial_test::serial; use std::io::{Cursor, Write}; #[cfg(not(target_os = "macos"))] @@ -173,124 +172,342 @@ fn test_server_stats() { } #[test] +#[serial] fn test_server_unsupported_compiler() { + // This test exercises server-side compilation; disable client-side mode. + // Use temp_env + #[serial] to avoid races and ensure the var is restored. + temp_env::with_var("SCCACHE_CLIENT_SIDE_COMPILE", Some("0"), || { + let f = TestFixture::new(); + let (addr, sender, server_creator, child) = run_server_thread(f.tempdir.path(), None); + // Connect to the server. + let conn = connect_to_server(&addr).unwrap(); + { + let mut c = server_creator.lock().unwrap(); + // fail rust driver check + c.next_command_spawns(Ok(MockChild::new(exit_status(1), "hello", "error"))); + // The server will check the compiler, so pretend to be an unsupported + // compiler. + c.next_command_spawns(Ok(MockChild::new(exit_status(0), "hello", "error"))); + } + // Ask the server to compile something. + //TODO: MockCommand should validate these! + let exe = &f.bins[0]; + let cmdline = vec!["-c".into(), "file.c".into(), "-o".into(), "file.o".into()]; + let cwd = f.tempdir.path(); + // This creator shouldn't create any processes. It will assert if + // it tries to. + let client_creator = new_creator(); + let mut stdout = Cursor::new(Vec::new()); + let mut stderr = Cursor::new(Vec::new()); + let path = Some(f.paths); + let mut runtime = Runtime::new().unwrap(); + let res = do_compile( + client_creator, + &mut runtime, + conn, + exe, + cmdline, + cwd, + path, + vec![], + &mut stdout, + &mut stderr, + ); + match res { + Ok(_) => panic!("do_compile should have failed!"), + Err(e) => assert_eq!("Compiler not supported: \"error\"", e.to_string()), + } + // Make sure we ran the mock processes. + assert_eq!(0, server_creator.lock().unwrap().children.len()); + // Shut down the server. + sender.send(ServerMessage::Shutdown).ok().unwrap(); + // Ensure that it shuts down. + child.join().unwrap(); + }); +} + +#[test] +#[serial] +fn test_server_compile() { + // This test exercises server-side compilation; disable client-side mode. + // Use temp_env + #[serial] to avoid races and ensure the var is restored. + temp_env::with_var("SCCACHE_CLIENT_SIDE_COMPILE", Some("0"), || { + let _ = env_logger::try_init(); + let f = TestFixture::new(); + let gcc = f.mk_bin("gcc").unwrap(); + let (addr, sender, server_creator, child) = run_server_thread(f.tempdir.path(), None); + // Connect to the server. + const PREPROCESSOR_STDOUT: &[u8] = b"preprocessor stdout"; + const PREPROCESSOR_STDERR: &[u8] = b"preprocessor stderr"; + const STDOUT: &[u8] = b"some stdout"; + const STDERR: &[u8] = b"some stderr"; + let conn = connect_to_server(&addr).unwrap(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("file.c"), "whatever").unwrap(); + { + let mut c = server_creator.lock().unwrap(); + // The server will check the compiler. Pretend it's GCC. + c.next_command_spawns(Ok(MockChild::new(exit_status(0), "compiler_id=gcc", ""))); + // Preprocessor invocation. + c.next_command_spawns(Ok(MockChild::new( + exit_status(0), + PREPROCESSOR_STDOUT, + PREPROCESSOR_STDERR, + ))); + // Compiler invocation. + //TODO: wire up a way to get data written to stdin. + let obj = f.tempdir.path().join("file.o"); + c.next_command_calls(move |_| { + // Pretend to compile something. + let mut f = File::create(&obj)?; + f.write_all(b"file contents")?; + Ok(MockChild::new(exit_status(0), STDOUT, STDERR)) + }); + } + // Ask the server to compile something. + //TODO: MockCommand should validate these! + let exe = &gcc; + let cmdline = vec!["-c".into(), "file.c".into(), "-o".into(), "file.o".into()]; + let cwd = f.tempdir.path(); + // This creator shouldn't create any processes. It will assert if + // it tries to. + let client_creator = new_creator(); + let mut stdout = Cursor::new(Vec::new()); + let mut stderr = Cursor::new(Vec::new()); + let path = Some(f.paths); + let mut runtime = Runtime::new().unwrap(); + assert_eq!( + 0, + do_compile( + client_creator, + &mut runtime, + conn, + exe, + cmdline, + cwd, + path, + vec![], + &mut stdout, + &mut stderr + ) + .unwrap() + ); + // Make sure we ran the mock processes. + assert_eq!(0, server_creator.lock().unwrap().children.len()); + assert_eq!(STDOUT, stdout.into_inner().as_slice()); + assert_eq!(STDERR, stderr.into_inner().as_slice()); + // Shut down the server. + sender.send(ServerMessage::Shutdown).ok().unwrap(); + // Ensure that it shuts down. + child.join().unwrap(); + }); +} + +/// Test the CacheGet → Miss → CachePut → CacheGet → Hit round-trip at the +/// protocol level. This exercises `handle_cache_get` and `handle_cache_put` +/// in `server.rs` without going through `do_compile_client_side`. +#[test] +#[serial] +fn test_cache_get_put_round_trip() { let f = TestFixture::new(); - let (addr, sender, server_creator, child) = run_server_thread(f.tempdir.path(), None); - // Connect to the server. - let conn = connect_to_server(&addr).unwrap(); - { - let mut c = server_creator.lock().unwrap(); - // fail rust driver check - c.next_command_spawns(Ok(MockChild::new(exit_status(1), "hello", "error"))); - // The server will check the compiler, so pretend to be an unsupported - // compiler. - c.next_command_spawns(Ok(MockChild::new(exit_status(0), "hello", "error"))); + let (addr, sender, _server_creator, child) = run_server_thread(f.tempdir.path(), None); + + // --- CacheGet on a key that doesn't exist → Miss ------------------------- + let mut conn = connect_to_server(&addr).unwrap(); + let output_path = f.tempdir.path().join("output.o"); + let output_paths = vec![crate::cache::FileObjectSource { + key: "obj".into(), + path: output_path.clone(), + optional: false, + }]; + let resp = conn + .request(crate::protocol::Request::CacheGet( + crate::protocol::CacheGetRequest { + key: "test-key-1".into(), + output_paths: output_paths.clone(), + }, + )) + .unwrap(); + assert!( + matches!( + resp, + crate::protocol::Response::CacheGet(crate::protocol::CacheGetResponse::Miss) + ), + "expected CacheGet Miss, got: {resp:?}" + ); + + // --- CachePut: write an artifact file and store it ----------------------- + fs::write(&output_path, b"object file contents").unwrap(); + let resp = conn + .request(crate::protocol::Request::CachePut( + crate::protocol::CachePutRequest { + key: "test-key-1".into(), + output_paths: output_paths.clone(), + stdout: b"compile stdout".to_vec(), + stderr: b"compile stderr".to_vec(), + }, + )) + .unwrap(); + assert!( + matches!( + resp, + crate::protocol::Response::CachePut(crate::protocol::CachePutResponse::Success) + ), + "expected CachePut Success, got: {resp:?}" + ); + + // Remove the artifact so we can verify CacheGet recreates it. + fs::remove_file(&output_path).unwrap(); + assert!(!output_path.exists()); + + // --- CacheGet on the same key → Hit, artifacts extracted ------------------ + let resp = conn + .request(crate::protocol::Request::CacheGet( + crate::protocol::CacheGetRequest { + key: "test-key-1".into(), + output_paths: output_paths.clone(), + }, + )) + .unwrap(); + match resp { + crate::protocol::Response::CacheGet(crate::protocol::CacheGetResponse::Hit { + stdout, + stderr, + }) => { + assert_eq!(stdout, b"compile stdout"); + assert_eq!(stderr, b"compile stderr"); + } + other => panic!("expected CacheGet Hit, got: {other:?}"), } - // Ask the server to compile something. - //TODO: MockCommand should validate these! - let exe = &f.bins[0]; - let cmdline = vec!["-c".into(), "file.c".into(), "-o".into(), "file.o".into()]; - let cwd = f.tempdir.path(); - // This creator shouldn't create any processes. It will assert if - // it tries to. - let client_creator = new_creator(); - let mut stdout = Cursor::new(Vec::new()); - let mut stderr = Cursor::new(Vec::new()); - let path = Some(f.paths); - let mut runtime = Runtime::new().unwrap(); - let res = do_compile( - client_creator, - &mut runtime, - conn, - exe, - cmdline, - cwd, - path, - vec![], - &mut stdout, - &mut stderr, + // The artifact should have been extracted back to disk. + assert!(output_path.exists(), "artifact was not extracted to disk"); + assert_eq!(fs::read(&output_path).unwrap(), b"object file contents"); + + // Drop the connection so the server can shut down cleanly. + drop(conn); + + // --- Check stats reflect the operations ---------------------------------- + // request_stats consumes the connection, so it will be dropped. + let stats_conn = connect_to_server(&addr).unwrap(); + let info = request_stats(stats_conn).unwrap(); + assert_eq!( + info.stats.client_side_cache_misses, 1, + "expected 1 client-side cache miss" ); - match res { - Ok(_) => panic!("do_compile should have failed!"), - Err(e) => assert_eq!("Compiler not supported: \"error\"", e.to_string()), + assert_eq!( + info.stats.client_side_cache_hits, 1, + "expected 1 client-side cache hit" + ); + assert_eq!(info.stats.cache_writes, 1, "expected 1 cache write"); + + // Shut down. + sender.send(ServerMessage::Shutdown).ok().unwrap(); + child.join().unwrap(); +} + +/// CacheGet with a relative path should return an error, not extract anything. +#[test] +#[serial] +fn test_cache_get_rejects_relative_path() { + let f = TestFixture::new(); + let (addr, sender, _server_creator, child) = run_server_thread(f.tempdir.path(), None); + + let mut conn = connect_to_server(&addr).unwrap(); + let resp = conn + .request(crate::protocol::Request::CacheGet( + crate::protocol::CacheGetRequest { + key: "test-key-bad".into(), + output_paths: vec![crate::cache::FileObjectSource { + key: "obj".into(), + path: "relative/file.o".into(), + optional: false, + }], + }, + )) + .unwrap(); + match resp { + crate::protocol::Response::CacheGet(crate::protocol::CacheGetResponse::Error(msg)) => { + assert!( + msg.contains("must be absolute"), + "unexpected error message: {msg}" + ); + } + other => panic!("expected CacheGet Error for relative path, got: {other:?}"), } - // Make sure we ran the mock processes. - assert_eq!(0, server_creator.lock().unwrap().children.len()); - // Shut down the server. + + drop(conn); sender.send(ServerMessage::Shutdown).ok().unwrap(); - // Ensure that it shuts down. child.join().unwrap(); } +/// CachePut with a path containing '..' should return an error. #[test] -fn test_server_compile() { - let _ = env_logger::try_init(); +#[serial] +fn test_cache_put_rejects_dotdot_path() { let f = TestFixture::new(); - let gcc = f.mk_bin("gcc").unwrap(); - let (addr, sender, server_creator, child) = run_server_thread(f.tempdir.path(), None); - // Connect to the server. - const PREPROCESSOR_STDOUT: &[u8] = b"preprocessor stdout"; - const PREPROCESSOR_STDERR: &[u8] = b"preprocessor stderr"; - const STDOUT: &[u8] = b"some stdout"; - const STDERR: &[u8] = b"some stderr"; - let conn = connect_to_server(&addr).unwrap(); - // Write a dummy input file so the preprocessor cache mode can work - std::fs::write(f.tempdir.path().join("file.c"), "whatever").unwrap(); - { - let mut c = server_creator.lock().unwrap(); - // The server will check the compiler. Pretend it's GCC. - c.next_command_spawns(Ok(MockChild::new(exit_status(0), "compiler_id=gcc", ""))); - // Preprocessor invocation. - c.next_command_spawns(Ok(MockChild::new( - exit_status(0), - PREPROCESSOR_STDOUT, - PREPROCESSOR_STDERR, - ))); - // Compiler invocation. - //TODO: wire up a way to get data written to stdin. - let obj = f.tempdir.path().join("file.o"); - c.next_command_calls(move |_| { - // Pretend to compile something. - let mut f = File::create(&obj)?; - f.write_all(b"file contents")?; - Ok(MockChild::new(exit_status(0), STDOUT, STDERR)) - }); + let (addr, sender, _server_creator, child) = run_server_thread(f.tempdir.path(), None); + + let mut conn = connect_to_server(&addr).unwrap(); + let resp = conn + .request(crate::protocol::Request::CachePut( + crate::protocol::CachePutRequest { + key: "test-key-bad".into(), + output_paths: vec![crate::cache::FileObjectSource { + key: "obj".into(), + path: std::env::temp_dir().join("..").join("etc").join("passwd"), + optional: false, + }], + stdout: vec![], + stderr: vec![], + }, + )) + .unwrap(); + match resp { + crate::protocol::Response::CachePut(crate::protocol::CachePutResponse::Error(msg)) => { + assert!(msg.contains("'..'"), "unexpected error message: {msg}"); + } + other => panic!("expected CachePut Error for dotdot path, got: {other:?}"), } - // Ask the server to compile something. - //TODO: MockCommand should validate these! - let exe = &gcc; - let cmdline = vec!["-c".into(), "file.c".into(), "-o".into(), "file.o".into()]; - let cwd = f.tempdir.path(); - // This creator shouldn't create any processes. It will assert if - // it tries to. - let client_creator = new_creator(); - let mut stdout = Cursor::new(Vec::new()); - let mut stderr = Cursor::new(Vec::new()); - let path = Some(f.paths); - let mut runtime = Runtime::new().unwrap(); - assert_eq!( - 0, - do_compile( - client_creator, - &mut runtime, - conn, - exe, - cmdline, - cwd, - path, - vec![], - &mut stdout, - &mut stderr - ) - .unwrap() - ); - // Make sure we ran the mock processes. - assert_eq!(0, server_creator.lock().unwrap().children.len()); - assert_eq!(STDOUT, stdout.into_inner().as_slice()); - assert_eq!(STDERR, stderr.into_inner().as_slice()); - // Shut down the server. + + drop(conn); + sender.send(ServerMessage::Shutdown).ok().unwrap(); + child.join().unwrap(); +} + +/// CachePut when the output file doesn't exist should return an error, not panic. +#[test] +#[serial] +fn test_cache_put_missing_file() { + let f = TestFixture::new(); + let (addr, sender, _server_creator, child) = run_server_thread(f.tempdir.path(), None); + + let mut conn = connect_to_server(&addr).unwrap(); + let resp = conn + .request(crate::protocol::Request::CachePut( + crate::protocol::CachePutRequest { + key: "test-key-missing".into(), + output_paths: vec![crate::cache::FileObjectSource { + key: "obj".into(), + path: f.tempdir.path().join("nonexistent.o"), + optional: false, + }], + stdout: vec![], + stderr: vec![], + }, + )) + .unwrap(); + match resp { + crate::protocol::Response::CachePut(crate::protocol::CachePutResponse::Error(msg)) => { + assert!( + msg.contains("failed to read output files"), + "unexpected error message: {msg}" + ); + } + other => panic!("expected CachePut Error for missing file, got: {other:?}"), + } + + drop(conn); sender.send(ServerMessage::Shutdown).ok().unwrap(); - // Ensure that it shuts down. child.join().unwrap(); } diff --git a/tests/cache_hit_rate.rs b/tests/cache_hit_rate.rs index dfda90abed..b229d6ce83 100644 --- a/tests/cache_hit_rate.rs +++ b/tests/cache_hit_rate.rs @@ -1,5 +1,6 @@ pub mod helpers; +use std::ffi::OsString; use std::process::Command; use anyhow::Result; @@ -8,10 +9,16 @@ use helpers::{CARGO, CRATE_DIR, SccacheTest, cargo_clean}; use predicates::{boolean::PredicateBooleanExt, str::PredicateStrExt}; use serial_test::serial; +/// These tests check server-side "Cache hits rate" stats, so client-side +/// compilation must be disabled to ensure compilations go through the server. +fn server_side_envs() -> Vec<(&'static str, OsString)> { + vec![("SCCACHE_CLIENT_SIDE_COMPILE", OsString::from("0"))] +} + #[test] #[serial] fn test_cache_hit_rate() -> Result<()> { - let test_info = SccacheTest::new(None)?; + let test_info = SccacheTest::new(Some(&server_side_envs()))?; Command::new(CARGO.as_os_str()) .args(["build", "--color=never"]) @@ -66,7 +73,7 @@ fn test_cache_hit_rate() -> Result<()> { #[test] #[serial] fn test_adv_cache_hit_rate() -> Result<()> { - let test_info = SccacheTest::new(None)?; + let test_info = SccacheTest::new(Some(&server_side_envs()))?; Command::new(CARGO.as_os_str()) .args(["build", "--color=never"]) diff --git a/tests/sccache_cargo.rs b/tests/sccache_cargo.rs index 6734a33f5b..c62eb57121 100644 --- a/tests/sccache_cargo.rs +++ b/tests/sccache_cargo.rs @@ -13,34 +13,41 @@ use fs_err as fs; use helpers::{SCCACHE_BIN, SccacheTest}; use predicates::prelude::*; use serial_test::serial; +use std::ffi::OsString; use std::path::Path; use std::process::Command; #[macro_use] extern crate log; +/// These tests check server-side stats (cache_hits/cache_misses), so client-side +/// compilation must be disabled to ensure compilations go through the server. +fn server_side_envs() -> Vec<(&'static str, OsString)> { + vec![("SCCACHE_CLIENT_SIDE_COMPILE", OsString::from("0"))] +} + #[test] #[serial] fn test_rust_cargo_check() -> Result<()> { - test_rust_cargo_cmd("check", SccacheTest::new(None)?) + test_rust_cargo_cmd("check", SccacheTest::new(Some(&server_side_envs()))?) } #[test] #[serial] fn test_rust_cargo_check_readonly() -> Result<()> { - test_rust_cargo_cmd_readonly("check", SccacheTest::new(None)?) + test_rust_cargo_cmd_readonly("check", SccacheTest::new(Some(&server_side_envs()))?) } #[test] #[serial] fn test_rust_cargo_build() -> Result<()> { - test_rust_cargo_cmd("build", SccacheTest::new(None)?) + test_rust_cargo_cmd("build", SccacheTest::new(Some(&server_side_envs()))?) } #[test] #[serial] fn test_rust_cargo_build_readonly() -> Result<()> { - test_rust_cargo_cmd_readonly("build", SccacheTest::new(None)?) + test_rust_cargo_cmd_readonly("build", SccacheTest::new(Some(&server_side_envs()))?) } #[test] @@ -87,67 +94,43 @@ fn test_run_log() -> Result<()> { #[test] #[serial] fn test_rust_cargo_run_with_env_dep_parsing() -> Result<()> { - test_rust_cargo_env_dep(SccacheTest::new(None)?) + test_rust_cargo_env_dep(SccacheTest::new(Some(&server_side_envs()))?) } #[cfg(feature = "unstable")] #[test] #[serial] fn test_rust_cargo_check_nightly() -> Result<()> { - use std::ffi::OsString; - - test_rust_cargo_cmd( - "check", - SccacheTest::new(Some(&[( - "RUSTFLAGS", - OsString::from("-Cprofile-generate=."), - )]))?, - ) + let mut envs = server_side_envs(); + envs.push(("RUSTFLAGS", OsString::from("-Cprofile-generate=."))); + test_rust_cargo_cmd("check", SccacheTest::new(Some(&envs))?) } #[cfg(feature = "unstable")] #[test] #[serial] fn test_rust_cargo_check_nightly_readonly() -> Result<()> { - use std::ffi::OsString; - - test_rust_cargo_cmd_readonly( - "check", - SccacheTest::new(Some(&[( - "RUSTFLAGS", - OsString::from("-Cprofile-generate=."), - )]))?, - ) + let mut envs = server_side_envs(); + envs.push(("RUSTFLAGS", OsString::from("-Cprofile-generate=."))); + test_rust_cargo_cmd_readonly("check", SccacheTest::new(Some(&envs))?) } #[cfg(feature = "unstable")] #[test] #[serial] fn test_rust_cargo_build_nightly() -> Result<()> { - use std::ffi::OsString; - - test_rust_cargo_cmd( - "build", - SccacheTest::new(Some(&[( - "RUSTFLAGS", - OsString::from("-Cprofile-generate=."), - )]))?, - ) + let mut envs = server_side_envs(); + envs.push(("RUSTFLAGS", OsString::from("-Cprofile-generate=."))); + test_rust_cargo_cmd("build", SccacheTest::new(Some(&envs))?) } #[cfg(feature = "unstable")] #[test] #[serial] fn test_rust_cargo_build_nightly_readonly() -> Result<()> { - use std::ffi::OsString; - - test_rust_cargo_cmd_readonly( - "build", - SccacheTest::new(Some(&[( - "RUSTFLAGS", - OsString::from("-Cprofile-generate=."), - )]))?, - ) + let mut envs = server_side_envs(); + envs.push(("RUSTFLAGS", OsString::from("-Cprofile-generate=."))); + test_rust_cargo_cmd_readonly("build", SccacheTest::new(Some(&envs))?) } /// Test that building a simple Rust crate with cargo using sccache results in a cache hit @@ -350,7 +333,7 @@ fn test_rust_cargo_env_dep(test_info: SccacheTest) -> Result<()> { #[test] #[serial] fn test_rust_cargo_cmd_readonly_preemtive_block() -> Result<()> { - let test_info = SccacheTest::new(None)?; + let test_info = SccacheTest::new(Some(&server_side_envs()))?; // `cargo clean` first, just to be sure there's no leftover build objects. cargo_clean(&test_info)?;