diff --git a/.config/rail.toml b/.config/rail.toml new file mode 100644 index 0000000000..14114221da --- /dev/null +++ b/.config/rail.toml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cargo-rail change detection configuration +# See: https://github.com/loadingalias/cargo-rail + +[change-detection] +# Changes to these paths trigger a full workspace rebuild/retest +infrastructure = [ + "Cargo.lock", + "rust-toolchain.toml", + ".cargo/**", + ".github/**", +] diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 0daa91b5da..742d489f9d 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -59,6 +59,50 @@ runs: esac shell: bash + # DAG-based test scoping: use cargo-rail to compute affected crates from the + # workspace dependency graph + git diff, avoiding the full test suite when only + # a subset of crates changed. + # Safety: cargo check/clippy run on the full workspace separately, catching all + # compilation errors. This only scopes test BUILD and EXECUTION. + - name: Fetch base branch for DAG analysis + if: startsWith(inputs.task, 'test-') + run: git fetch origin master --depth=1 2>/dev/null || true + shell: bash + + - name: Install cargo-rail + if: startsWith(inputs.task, 'test-') + uses: taiki-e/install-action@v2 + with: + tool: cargo-rail + + - name: Compute affected crates (cargo-rail) + if: startsWith(inputs.task, 'test-') + run: | + TOTAL_CRATES=$(cargo metadata --format-version 1 --no-deps 2>/dev/null | jq '.workspace_members | length' 2>/dev/null || echo "?") + echo "$TOTAL_CRATES" > /tmp/total-crates.txt + + PLAN_JSON=$(cargo rail plan --since origin/master -f json 2>/tmp/affected-stderr.txt || echo "") + + if [[ -n "$PLAN_JSON" ]]; then + MODE=$(echo "$PLAN_JSON" | jq -r '.scope.mode') + if [[ "$MODE" == "crates" ]]; then + CRATES=$(echo "$PLAN_JSON" | jq -r '.scope.crates[]') + CRATE_COUNT=$(echo "$CRATES" | wc -l) + # Build -p flags for cargo build/test + echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/packages.txt + # Build nextest filter expression for cargo nextest run + echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd ' | ' > /tmp/nextest-filter.txt + echo "::notice::DAG analysis: ${CRATE_COUNT} affected crates (of ${TOTAL_CRATES} total)" + else + echo "::notice::Full workspace affected (${TOTAL_CRATES} crates)" + fi + else + STDERR=$(cat /tmp/affected-stderr.txt 2>/dev/null || echo "") + echo "::warning::Could not compute affected crates, running full test suite. ${STDERR}" + rm -f /tmp/nextest-filter.txt /tmp/packages.txt + fi + shell: bash + # Individual lint tasks for parallel execution - name: Cargo check if: inputs.task == 'check' @@ -117,16 +161,45 @@ runs: echo "::notice::Running test partition ${PARTITION_INDEX}/2" fi + # Read DAG-based affected crate filter (computed in earlier step) + NEXTEST_FILTER="" + PACKAGE_FLAGS="" + TOTAL_CRATES="?" + if [[ -f /tmp/nextest-filter.txt ]]; then + NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt) + fi + if [[ -f /tmp/packages.txt ]]; then + PACKAGE_FLAGS=$(cat /tmp/packages.txt) + fi + if [[ -f /tmp/total-crates.txt ]]; then + TOTAL_CRATES=$(cat /tmp/total-crates.txt) + fi + + if [[ -n "$PACKAGE_FLAGS" ]]; then + CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l) + echo "::notice::DAG-scoped build: ${CRATE_COUNT} crates (cargo check/clippy cover full workspace separately)" + else + echo "::notice::Full workspace build (no DAG filter available)" + fi + source <(cargo llvm-cov show-env --export-prefix) bins_start=$(date +%s) - cargo build --locked + if [[ -n "$PACKAGE_FLAGS" ]]; then + cargo build --locked $PACKAGE_FLAGS + else + cargo build --locked + fi bins_end=$(date +%s) bins_duration=$((bins_end - bins_start)) echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" compile_start=$(date +%s) - cargo test --locked --no-run + if [[ -n "$PACKAGE_FLAGS" ]]; then + cargo test --locked --no-run $PACKAGE_FLAGS + else + cargo test --locked --no-run + fi compile_end=$(date +%s) compile_duration=$((compile_end - compile_start)) echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" @@ -144,12 +217,20 @@ runs: test_start=$(date +%s) if command -v cargo-nextest &> /dev/null; then - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG + if [[ -n "$NEXTEST_FILTER" ]]; then + cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG -E "$NEXTEST_FILTER" + else + cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG + fi else if [[ -n "$PARTITION_FLAG" ]]; then echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)" fi - cargo test --locked --no-fail-fast + if [[ -n "$PACKAGE_FLAGS" ]]; then + cargo test --locked --no-fail-fast $PACKAGE_FLAGS + else + cargo test --locked --no-fail-fast + fi fi test_end=$(date +%s) test_duration=$((test_end - test_start)) @@ -159,6 +240,12 @@ runs: total_duration=$((build_duration + test_duration)) echo "" echo "=========================================" + if [[ -n "$PACKAGE_FLAGS" ]]; then + CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l) + echo "DAG scope: ${CRATE_COUNT}/${TOTAL_CRATES} crates" + else + echo "DAG scope: full workspace (${TOTAL_CRATES} crates)" + fi echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" echo "Tests execute: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" diff --git a/.github/config/components.yml b/.github/config/components.yml index 7585a91b9b..a898f30dbb 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -51,9 +51,17 @@ components: paths: - "core/common/**" + # Leaf crate: zero-copy I/O buffer, depended on by binary_protocol and cluster + rust-iobuf: + depends_on: + - "rust-workspace" + paths: + - "core/iobuf/**" + rust-binary-protocol: depends_on: - "rust-workspace" # Protocol is affected by workspace changes + - "rust-iobuf" # binary_protocol depends on iobuf - "ci-infrastructure" # CI changes trigger full regression paths: - "core/binary_protocol/**" @@ -67,10 +75,13 @@ components: - "rust-cluster" paths: - "core/server/**" + - "core/server-ng/**" rust-cluster: depends_on: - "rust-workspace" + - "rust-iobuf" # cluster crates depend on iobuf + - "rust-binary-protocol" # cluster crates depend on binary_protocol paths: - "core/clock/**" - "core/consensus/**" @@ -79,18 +90,28 @@ components: - "core/metadata/**" - "core/message_bus/**" - "core/partitions/**" + + # Standalone simulation tool, does NOT affect server binary or foreign SDKs. + # Split from rust-cluster to avoid triggering SDK tests on simulator-only changes. + rust-simulator: + depends_on: + - "rust-workspace" + - "rust-cluster" + paths: - "core/simulator/**" # Main Rust workspace testing rust: depends_on: - "rust-workspace" + - "rust-iobuf" - "rust-configs" - "rust-sdk" - "rust-common" - "rust-binary-protocol" - "rust-server" - "rust-cluster" + - "rust-simulator" - "rust-tools" - "rust-cli" - "rust-bench"