feat(ci): dependency-DAG-based test scoping for Rust CI

.config/rail.toml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cargo-rail change detection configuration
+# See: https://github.com/loadingalias/cargo-rail
+
+[change-detection]
+# Changes to these paths trigger a full workspace rebuild/retest
+infrastructure = [
+    "Cargo.lock",
+    "rust-toolchain.toml",
+    ".cargo/**",
+    ".github/**",
+]

.github/actions/rust/pre-merge/action.yml

@@ -59,6 +59,50 @@ runs:
         esac
       shell: bash
 
+    # DAG-based test scoping: use cargo-rail to compute affected crates from the
+    # workspace dependency graph + git diff, avoiding the full test suite when only
+    # a subset of crates changed.
+    # Safety: cargo check/clippy run on the full workspace separately, catching all
+    # compilation errors. This only scopes test BUILD and EXECUTION.
+    - name: Fetch base branch for DAG analysis
+      if: startsWith(inputs.task, 'test-')
+      run: git fetch origin master --depth=1 2>/dev/null || true
+      shell: bash
+
+    - name: Install cargo-rail
+      if: startsWith(inputs.task, 'test-')
+      uses: taiki-e/install-action@v2
+      with:
+        tool: cargo-rail
+
+    - name: Compute affected crates (cargo-rail)
+      if: startsWith(inputs.task, 'test-')
+      run: |
+        TOTAL_CRATES=$(cargo metadata --format-version 1 --no-deps 2>/dev/null | jq '.workspace_members | length' 2>/dev/null || echo "?")
+        echo "$TOTAL_CRATES" > /tmp/total-crates.txt
+
+        PLAN_JSON=$(cargo rail plan --since origin/master -f json 2>/tmp/affected-stderr.txt || echo "")
+
+        if [[ -n "$PLAN_JSON" ]]; then
+          MODE=$(echo "$PLAN_JSON" | jq -r '.scope.mode')
+          if [[ "$MODE" == "crates" ]]; then
+            CRATES=$(echo "$PLAN_JSON" | jq -r '.scope.crates[]')
+            CRATE_COUNT=$(echo "$CRATES" | wc -l)
+            # Build -p flags for cargo build/test
+            echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/packages.txt
+            # Build nextest filter expression for cargo nextest run
+            echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd ' | ' > /tmp/nextest-filter.txt
+            echo "::notice::DAG analysis: ${CRATE_COUNT} affected crates (of ${TOTAL_CRATES} total)"
+          else
+            echo "::notice::Full workspace affected (${TOTAL_CRATES} crates)"
+          fi
+        else
+          STDERR=$(cat /tmp/affected-stderr.txt 2>/dev/null || echo "")
+          echo "::warning::Could not compute affected crates, running full test suite. ${STDERR}"
+          rm -f /tmp/nextest-filter.txt /tmp/packages.txt
+        fi
+      shell: bash
+
     # Individual lint tasks for parallel execution
     - name: Cargo check
       if: inputs.task == 'check'
@@ -117,16 +161,45 @@ runs:
           echo "::notice::Running test partition ${PARTITION_INDEX}/2"
         fi
 
+        # Read DAG-based affected crate filter (computed in earlier step)
+        NEXTEST_FILTER=""
+        PACKAGE_FLAGS=""
+        TOTAL_CRATES="?"
+        if [[ -f /tmp/nextest-filter.txt ]]; then
+          NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt)
+        fi
+        if [[ -f /tmp/packages.txt ]]; then
+          PACKAGE_FLAGS=$(cat /tmp/packages.txt)
+        fi
+        if [[ -f /tmp/total-crates.txt ]]; then
+          TOTAL_CRATES=$(cat /tmp/total-crates.txt)
+        fi
+
+        if [[ -n "$PACKAGE_FLAGS" ]]; then
+          CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l)
+          echo "::notice::DAG-scoped build: ${CRATE_COUNT} crates (cargo check/clippy cover full workspace separately)"
+        else
+          echo "::notice::Full workspace build (no DAG filter available)"
+        fi
+
         source <(cargo llvm-cov show-env --export-prefix)
 
         bins_start=$(date +%s)
-        cargo build --locked
+        if [[ -n "$PACKAGE_FLAGS" ]]; then
+          cargo build --locked $PACKAGE_FLAGS
+        else
+          cargo build --locked
+        fi
         bins_end=$(date +%s)
         bins_duration=$((bins_end - bins_start))
         echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
 
         compile_start=$(date +%s)
-        cargo test --locked --no-run
+        if [[ -n "$PACKAGE_FLAGS" ]]; then
+          cargo test --locked --no-run $PACKAGE_FLAGS
+        else
+          cargo test --locked --no-run
+        fi
         compile_end=$(date +%s)
         compile_duration=$((compile_end - compile_start))
         echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
@@ -144,12 +217,20 @@ runs:
 
         test_start=$(date +%s)
         if command -v cargo-nextest &> /dev/null; then
-          cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG
+          if [[ -n "$NEXTEST_FILTER" ]]; then
+            cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG -E "$NEXTEST_FILTER"
+          else
+            cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG
+          fi
         else
           if [[ -n "$PARTITION_FLAG" ]]; then
             echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)"
           fi
-          cargo test --locked --no-fail-fast
+          if [[ -n "$PACKAGE_FLAGS" ]]; then
+            cargo test --locked --no-fail-fast $PACKAGE_FLAGS
+          else
+            cargo test --locked --no-fail-fast
+          fi
         fi
         test_end=$(date +%s)
         test_duration=$((test_end - test_start))
@@ -159,6 +240,12 @@ runs:
         total_duration=$((build_duration + test_duration))
         echo ""
         echo "========================================="
+        if [[ -n "$PACKAGE_FLAGS" ]]; then
+          CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l)
+          echo "DAG scope:                     ${CRATE_COUNT}/${TOTAL_CRATES} crates"
+        else
+          echo "DAG scope:                     full workspace (${TOTAL_CRATES} crates)"
+        fi
         echo "All targets build:             ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
         echo "Tests compile:                 ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
         echo "Tests execute:                 ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"

.github/config/components.yml

@@ -51,9 +51,17 @@ components:
     paths:
       - "core/common/**"
 
+  # Leaf crate: zero-copy I/O buffer, depended on by binary_protocol and cluster
+  rust-iobuf:
+    depends_on:
+      - "rust-workspace"
+    paths:
+      - "core/iobuf/**"
+
   rust-binary-protocol:
     depends_on:
       - "rust-workspace" # Protocol is affected by workspace changes
+      - "rust-iobuf" # binary_protocol depends on iobuf
       - "ci-infrastructure" # CI changes trigger full regression
     paths:
       - "core/binary_protocol/**"
@@ -67,10 +75,13 @@ components:
       - "rust-cluster"
     paths:
       - "core/server/**"
+      - "core/server-ng/**"
 
   rust-cluster:
     depends_on:
       - "rust-workspace"
+      - "rust-iobuf" # cluster crates depend on iobuf
+      - "rust-binary-protocol" # cluster crates depend on binary_protocol
     paths:
       - "core/clock/**"
       - "core/consensus/**"
@@ -79,18 +90,28 @@ components:
       - "core/metadata/**"
       - "core/message_bus/**"
       - "core/partitions/**"
+
+  # Standalone simulation tool, does NOT affect server binary or foreign SDKs.
+  # Split from rust-cluster to avoid triggering SDK tests on simulator-only changes.
+  rust-simulator:
+    depends_on:
+      - "rust-workspace"
+      - "rust-cluster"
+    paths:
       - "core/simulator/**"
 
   # Main Rust workspace testing
   rust:
     depends_on:
       - "rust-workspace"
+      - "rust-iobuf"
       - "rust-configs"
       - "rust-sdk"
       - "rust-common"
       - "rust-binary-protocol"
       - "rust-server"
       - "rust-cluster"
+      - "rust-simulator"
       - "rust-tools"
       - "rust-cli"
       - "rust-bench"