From c67eb7e4957d2428fcc374f429d0e64cf1e85e31 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 1/6] rust_test: shard by stable test name hash Sort libtest names before execution and assign shards by a stable FNV-1a hash of each test name. This keeps existing tests in the same shard when unrelated tests are added or libtest list order changes. Co-authored-by: Codex --- rust/private/rust.bzl | 2 +- rust/private/test_sharding_wrapper.bat | 36 ++++++--- rust/private/test_sharding_wrapper.sh | 33 +++++++-- .../unit/test_sharding/fake_libtest_binary.sh | 67 +++++++++++++++++ test/unit/test_sharding/test_sharding.bzl | 19 +++++ ...st_sharding_wrapper_hashes_sorted_names.sh | 73 +++++++++++++++++++ 6 files changed, 209 insertions(+), 21 deletions(-) create mode 100755 test/unit/test_sharding/fake_libtest_binary.sh create mode 100755 test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 0dc2d993e7..ace9953a68 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -968,7 +968,7 @@ _RUST_TEST_ATTRS = { When enabled, tests are executed via a wrapper script that: 1. Enumerates tests using libtest's --list flag - 2. Partitions tests across shards based on TEST_SHARD_INDEX/TEST_TOTAL_SHARDS + 2. Sorts tests by name and partitions them across shards by stable name hash 3. Runs only the tests assigned to the current shard This attribute only has an effect when use_libtest_harness is True. diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index 5c90681c81..6835790e7a 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -14,7 +14,8 @@ @REM Wrapper script for rust_test that enables Bazel test sharding support. @REM This script intercepts test execution, enumerates tests using libtest's -@REM --list flag, partitions them by shard index, and runs only the relevant subset. +@REM --list flag, partitions them by stable test-name hash, and runs only the +@REM relevant subset. @ECHO OFF SETLOCAL EnableDelayedExpansion @@ -86,27 +87,38 @@ IF NOT "%TEST_SHARD_STATUS_FILE%"=="" ( @REM Create a temporary file for test list SET TEMP_LIST=%TEMP%\rust_test_list_%RANDOM%.txt +SET TEMP_SHARD_LIST=%TEMP%\rust_test_shard_%RANDOM%.txt @REM Enumerate all tests using libtest's --list flag !TEST_BINARY_PATH! --list --format terse 2>NUL > "!TEMP_LIST!" -@REM Count tests and filter for this shard -SET INDEX=0 +@REM Sort tests by ordinal name and filter this shard by stable FNV-1a hash so +@REM adding or removing one test does not move unrelated tests between shards. +powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ + "$ErrorActionPreference = 'Stop';" ^ + "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ + "[Array]::Sort($tests, [StringComparer]::Ordinal);" ^ + "$totalShards = [uint32]$env:TEST_TOTAL_SHARDS; $shardIndex = [uint32]$env:TEST_SHARD_INDEX;" ^ + "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32]((([uint64]($hash -bxor $byte)) * 16777619) -band 0xffffffff) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ + > "!TEMP_SHARD_LIST!" +IF ERRORLEVEL 1 ( + DEL "!TEMP_LIST!" 2>NUL + DEL "!TEMP_SHARD_LIST!" 2>NUL + EXIT /B 1 +) + SET SHARD_TESTS= -FOR /F "tokens=1 delims=:" %%T IN ('TYPE "!TEMP_LIST!" ^| FINDSTR /E ": test"') DO ( - SET /A MOD=!INDEX! %% %TEST_TOTAL_SHARDS% - IF !MOD! EQU %TEST_SHARD_INDEX% ( - IF "!SHARD_TESTS!"=="" ( - SET SHARD_TESTS=%%T - ) ELSE ( - SET SHARD_TESTS=!SHARD_TESTS! %%T - ) +FOR /F "usebackq delims=" %%T IN ("!TEMP_SHARD_LIST!") DO ( + IF "!SHARD_TESTS!"=="" ( + SET SHARD_TESTS=%%T + ) ELSE ( + SET SHARD_TESTS=!SHARD_TESTS! %%T ) - SET /A INDEX=!INDEX! + 1 ) DEL "!TEMP_LIST!" 2>NUL +DEL "!TEMP_SHARD_LIST!" 2>NUL @REM If no tests for this shard, exit successfully IF "!SHARD_TESTS!"=="" ( diff --git a/rust/private/test_sharding_wrapper.sh b/rust/private/test_sharding_wrapper.sh index e05970ba0a..4f65bcf3dd 100644 --- a/rust/private/test_sharding_wrapper.sh +++ b/rust/private/test_sharding_wrapper.sh @@ -15,12 +15,30 @@ # Wrapper script for rust_test that enables Bazel test sharding support. # This script intercepts test execution, enumerates tests using libtest's -# --list flag, partitions them by shard index, and runs only the relevant subset. +# --list flag, partitions them by stable test-name hash, and runs only the +# relevant subset. set -euo pipefail TEST_BINARY="{{TEST_BINARY}}" +test_shard_index() { + local test_name="$1" + local hash=2166136261 + local byte + local char + local i + local LC_ALL=C + + for ((i = 0; i < ${#test_name}; i++)); do + char="${test_name:i:1}" + printf -v byte "%d" "'$char" + hash=$(( ((hash ^ byte) * 16777619) & 0xffffffff )) + done + + echo $(( hash % TEST_TOTAL_SHARDS )) +} + # If sharding is not enabled, run test binary directly if [[ -z "${TEST_TOTAL_SHARDS:-}" ]]; then exec "./${TEST_BINARY}" "$@" @@ -31,24 +49,23 @@ if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then touch "${TEST_SHARD_STATUS_FILE}" fi -# Enumerate all tests using libtest's --list flag +# Enumerate all tests using libtest's --list flag. Sort the list so execution +# order does not depend on libtest's output order. # Output format: "test_name: test" - we need to strip the ": test" suffix -test_list=$("./${TEST_BINARY}" --list --format terse 2>/dev/null | grep ': test$' | sed 's/: test$//' || true) +test_list=$("./${TEST_BINARY}" --list --format terse 2>/dev/null | grep ': test$' | sed 's/: test$//' | LC_ALL=C sort || true) # If no tests found, exit successfully if [[ -z "$test_list" ]]; then exit 0 fi -# Filter tests for this shard -# test_index % TEST_TOTAL_SHARDS == TEST_SHARD_INDEX +# Filter tests for this shard. Use a stable name hash instead of list position +# so adding or removing one test does not move unrelated tests between shards. shard_tests=() -index=0 while IFS= read -r test_name; do - if (( index % TEST_TOTAL_SHARDS == TEST_SHARD_INDEX )); then + if (( $(test_shard_index "$test_name") == TEST_SHARD_INDEX )); then shard_tests+=("$test_name") fi - ((index++)) || true done <<< "$test_list" # If no tests for this shard, exit successfully diff --git a/test/unit/test_sharding/fake_libtest_binary.sh b/test/unit/test_sharding/fake_libtest_binary.sh new file mode 100755 index 0000000000..fef029be5c --- /dev/null +++ b/test/unit/test_sharding/fake_libtest_binary.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +emit_base_tests() { + cat <<'EOF' +delta::test_delta: test +alpha::test_alpha: test +foxtrot::test_foxtrot: test +bravo::test_bravo: test +echo::test_echo: test +charlie::test_charlie: test +helper::bench: bench +EOF +} + +emit_reversed_base_tests() { + cat <<'EOF' +helper::bench: bench +charlie::test_charlie: test +echo::test_echo: test +bravo::test_bravo: test +foxtrot::test_foxtrot: test +alpha::test_alpha: test +delta::test_delta: test +EOF +} + +emit_tests_with_added_test() { + cat <<'EOF' +delta::test_delta: test +alpha::test_alpha: test +foxtrot::test_foxtrot: test +aardvark::test_added: test +bravo::test_bravo: test +echo::test_echo: test +charlie::test_charlie: test +helper::bench: bench +EOF +} + +if [[ "${1:-}" == "--list" ]]; then + case "${TEST_LIST_VARIANT:-base}:${TEST_LIST_ORDER:-normal}" in + base:normal) + emit_base_tests + ;; + base:reversed) + emit_reversed_base_tests + ;; + with_added:normal) + emit_tests_with_added_test + ;; + *) + echo "unknown test list variant: ${TEST_LIST_VARIANT:-base}:${TEST_LIST_ORDER:-normal}" >&2 + exit 1 + ;; + esac + exit 0 +fi + +: "${TEST_SHARD_OUTPUT:?}" + +for test_name in "$@"; do + if [[ "$test_name" != "--exact" ]]; then + printf '%s\n' "$test_name" >> "$TEST_SHARD_OUTPUT" + fi +done diff --git a/test/unit/test_sharding/test_sharding.bzl b/test/unit/test_sharding/test_sharding.bzl index aea76f170e..14005f7160 100644 --- a/test/unit/test_sharding/test_sharding.bzl +++ b/test/unit/test_sharding/test_sharding.bzl @@ -1,6 +1,7 @@ """Tests for rust_test sharding support.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//rust:defs.bzl", "rust_test") def _sharding_enabled_test(ctx): @@ -68,6 +69,23 @@ def _test_sharding_targets(): shard_count = 3, ) + sh_test( + name = "test_sharding_wrapper_hashes_sorted_names", + srcs = ["test_sharding_wrapper_hashes_sorted_names.sh"], + args = [ + "$(location //rust/private:test_sharding_wrapper.sh)", + "$(location :fake_libtest_binary.sh)", + ], + data = [ + ":fake_libtest_binary.sh", + "//rust/private:test_sharding_wrapper.sh", + ], + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + ) + def test_sharding_test_suite(name): _test_sharding_targets() @@ -76,6 +94,7 @@ def test_sharding_test_suite(name): tests = [ ":sharding_enabled_test", ":sharding_disabled_test", + ":test_sharding_wrapper_hashes_sorted_names", ":sharded_integration_test", ], ) diff --git a/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh new file mode 100755 index 0000000000..2be08f8e15 --- /dev/null +++ b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euo pipefail + +wrapper_template=$1 +fake_binary_src=$2 + +workdir="${TEST_TMPDIR:-$(mktemp -d)}" +fake_binary="$workdir/fake_libtest_binary" +wrapper="$workdir/wrapper.sh" + +cp "$fake_binary_src" "$fake_binary" +chmod +x "$fake_binary" + +sed 's|{{TEST_BINARY}}|fake_libtest_binary|g' "$wrapper_template" > "$wrapper" +chmod +x "$wrapper" + +collect_mapping() { + local variant=$1 + local order=$2 + local output=$3 + local unsorted_output="${output}.unsorted" + local shard + + : > "$unsorted_output" + for shard in 0 1 2; do + local shard_output="$workdir/${variant}_${order}_${shard}.txt" + : > "$shard_output" + + ( + cd "$workdir" + TEST_LIST_VARIANT="$variant" \ + TEST_LIST_ORDER="$order" \ + TEST_SHARD_INDEX="$shard" \ + TEST_SHARD_OUTPUT="$shard_output" \ + TEST_TOTAL_SHARDS=3 \ + ./wrapper.sh + ) + + while IFS= read -r test_name; do + printf '%s %s\n' "$test_name" "$shard" >> "$unsorted_output" + done < "$shard_output" + done + + LC_ALL=C sort "$unsorted_output" > "$output" +} + +assert_same_mapping() { + local expected=$1 + local actual=$2 + local message=$3 + + if ! diff -u "$expected" "$actual"; then + echo "$message" >&2 + exit 1 + fi +} + +base_normal="$workdir/base_normal.txt" +base_reversed="$workdir/base_reversed.txt" +with_added="$workdir/with_added.txt" +with_added_existing_tests="$workdir/with_added_existing_tests.txt" + +collect_mapping base normal "$base_normal" +collect_mapping base reversed "$base_reversed" +collect_mapping with_added normal "$with_added" + +assert_same_mapping "$base_normal" "$base_reversed" \ + "test shard assignment changed when libtest list order changed" + +sed '/^aardvark::test_added /d' "$with_added" > "$with_added_existing_tests" +assert_same_mapping "$base_normal" "$with_added_existing_tests" \ + "existing test shard assignment changed after adding a new test" From f2c4f47c5db21b00a814650ed03d5a1a24d7a6c0 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 2/6] docs: explain test sharding hash constants Document that the sharding wrapper uses FNV-1a and identify the offset basis and prime constants in both Unix and Windows wrappers. Co-authored-by: Codex --- rust/private/test_sharding_wrapper.bat | 2 ++ rust/private/test_sharding_wrapper.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index 6835790e7a..dcac2baaf4 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -94,6 +94,8 @@ SET TEMP_SHARD_LIST=%TEMP%\rust_test_shard_%RANDOM%.txt @REM Sort tests by ordinal name and filter this shard by stable FNV-1a hash so @REM adding or removing one test does not move unrelated tests between shards. +@REM In the PowerShell fragment below, 2166136261 is the 32-bit FNV offset basis +@REM and 16777619 is the FNV prime. powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ "$ErrorActionPreference = 'Stop';" ^ "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ diff --git a/rust/private/test_sharding_wrapper.sh b/rust/private/test_sharding_wrapper.sh index 4f65bcf3dd..c91012dd19 100644 --- a/rust/private/test_sharding_wrapper.sh +++ b/rust/private/test_sharding_wrapper.sh @@ -24,6 +24,9 @@ TEST_BINARY="{{TEST_BINARY}}" test_shard_index() { local test_name="$1" + # FNV-1a 32-bit hash. The initial value is the FNV offset basis, and + # 16777619 is the FNV prime. This gives a stable, cheap string hash without + # depending on platform-specific tools being present in the test sandbox. local hash=2166136261 local byte local char From 5f972598adf221ce82e63436aae39ce82f41d34f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 3/6] rust_test: support explicit shard env vars Allow generated shard targets to drive the sharding wrapper with RULES_RUST_TEST_TOTAL_SHARDS and RULES_RUST_TEST_SHARD_INDEX without conflicting with Bazel reserved TEST_* variables. Co-authored-by: Codex --- rust/private/rust.bzl | 4 +++- rust/private/test_sharding_wrapper.bat | 24 ++++++++++++++++--- rust/private/test_sharding_wrapper.sh | 18 ++++++++++---- ...st_sharding_wrapper_hashes_sorted_names.sh | 4 ++-- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index ace9953a68..929dc56656 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -969,7 +969,9 @@ _RUST_TEST_ATTRS = { When enabled, tests are executed via a wrapper script that: 1. Enumerates tests using libtest's --list flag 2. Sorts tests by name and partitions them across shards by stable name hash - 3. Runs only the tests assigned to the current shard + 3. Uses either Bazel's native TEST_TOTAL_SHARDS/TEST_SHARD_INDEX env + or explicit RULES_RUST_TEST_TOTAL_SHARDS/RULES_RUST_TEST_SHARD_INDEX env + 4. Runs only the tests assigned to the current shard This attribute only has an effect when use_libtest_harness is True. diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index dcac2baaf4..3e0acbb549 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -74,14 +74,32 @@ IF !FOUND_BINARY! EQU 0 ( EXIT /B 1 ) +@REM Native Bazel test sharding sets TEST_TOTAL_SHARDS/TEST_SHARD_INDEX. +@REM Explicit shard test targets can set RULES_RUST_TEST_TOTAL_SHARDS/ +@REM RULES_RUST_TEST_SHARD_INDEX instead because Bazel may reserve TEST_* +@REM variables for its own test runner env. +SET TOTAL_SHARDS=%RULES_RUST_TEST_TOTAL_SHARDS% +IF "%TOTAL_SHARDS%"=="" SET TOTAL_SHARDS=%TEST_TOTAL_SHARDS% +SET SHARD_INDEX=%RULES_RUST_TEST_SHARD_INDEX% +IF "%SHARD_INDEX%"=="" SET SHARD_INDEX=%TEST_SHARD_INDEX% + @REM If sharding is not enabled, run test binary directly -IF "%TEST_TOTAL_SHARDS%"=="" ( +IF "%TOTAL_SHARDS%"=="" ( + !TEST_BINARY_PATH! %* + EXIT /B !ERRORLEVEL! +) +IF "%TOTAL_SHARDS%"=="0" ( !TEST_BINARY_PATH! %* EXIT /B !ERRORLEVEL! ) +IF "%SHARD_INDEX%"=="" ( + ECHO ERROR: TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled + EXIT /B 1 +) + @REM Touch status file to advertise sharding support to Bazel -IF NOT "%TEST_SHARD_STATUS_FILE%"=="" ( +IF NOT "%TEST_SHARD_STATUS_FILE%"=="" IF NOT "%TEST_TOTAL_SHARDS%"=="" IF NOT "%TEST_TOTAL_SHARDS%"=="0" ( TYPE NUL > "%TEST_SHARD_STATUS_FILE%" ) @@ -100,7 +118,7 @@ powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ "$ErrorActionPreference = 'Stop';" ^ "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ "[Array]::Sort($tests, [StringComparer]::Ordinal);" ^ - "$totalShards = [uint32]$env:TEST_TOTAL_SHARDS; $shardIndex = [uint32]$env:TEST_SHARD_INDEX;" ^ + "$totalShards = [uint32]$env:TOTAL_SHARDS; $shardIndex = [uint32]$env:SHARD_INDEX;" ^ "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32]((([uint64]($hash -bxor $byte)) * 16777619) -band 0xffffffff) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ > "!TEMP_SHARD_LIST!" IF ERRORLEVEL 1 ( diff --git a/rust/private/test_sharding_wrapper.sh b/rust/private/test_sharding_wrapper.sh index c91012dd19..b1f0fb55d7 100644 --- a/rust/private/test_sharding_wrapper.sh +++ b/rust/private/test_sharding_wrapper.sh @@ -21,6 +21,11 @@ set -euo pipefail TEST_BINARY="{{TEST_BINARY}}" +# Native Bazel test sharding sets TEST_TOTAL_SHARDS/TEST_SHARD_INDEX. Explicit +# shard test targets can set RULES_RUST_TEST_TOTAL_SHARDS/RULES_RUST_TEST_SHARD_INDEX +# instead because Bazel may reserve TEST_* variables for its own test runner env. +TOTAL_SHARDS="${RULES_RUST_TEST_TOTAL_SHARDS:-${TEST_TOTAL_SHARDS:-}}" +SHARD_INDEX="${RULES_RUST_TEST_SHARD_INDEX:-${TEST_SHARD_INDEX:-}}" test_shard_index() { local test_name="$1" @@ -39,16 +44,21 @@ test_shard_index() { hash=$(( ((hash ^ byte) * 16777619) & 0xffffffff )) done - echo $(( hash % TEST_TOTAL_SHARDS )) + echo $(( hash % TOTAL_SHARDS )) } # If sharding is not enabled, run test binary directly -if [[ -z "${TEST_TOTAL_SHARDS:-}" ]]; then +if [[ -z "${TOTAL_SHARDS}" || "${TOTAL_SHARDS}" == "0" ]]; then exec "./${TEST_BINARY}" "$@" fi +if [[ -z "${SHARD_INDEX}" ]]; then + echo "TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled" >&2 + exit 1 +fi + # Touch status file to advertise sharding support to Bazel -if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" && "${TEST_TOTAL_SHARDS:-0}" != "0" ]]; then touch "${TEST_SHARD_STATUS_FILE}" fi @@ -66,7 +76,7 @@ fi # so adding or removing one test does not move unrelated tests between shards. shard_tests=() while IFS= read -r test_name; do - if (( $(test_shard_index "$test_name") == TEST_SHARD_INDEX )); then + if (( $(test_shard_index "$test_name") == SHARD_INDEX )); then shard_tests+=("$test_name") fi done <<< "$test_list" diff --git a/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh index 2be08f8e15..626f33da44 100755 --- a/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh +++ b/test/unit/test_sharding/test_sharding_wrapper_hashes_sorted_names.sh @@ -31,9 +31,9 @@ collect_mapping() { cd "$workdir" TEST_LIST_VARIANT="$variant" \ TEST_LIST_ORDER="$order" \ - TEST_SHARD_INDEX="$shard" \ TEST_SHARD_OUTPUT="$shard_output" \ - TEST_TOTAL_SHARDS=3 \ + RULES_RUST_TEST_SHARD_INDEX="$shard" \ + RULES_RUST_TEST_TOTAL_SHARDS=3 \ ./wrapper.sh ) From 23267f3249e10eeec4238cb4cd025b3a9ed70daa Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 4/6] rust_test: resolve sharded test binaries via manifest When a downstream test rule wraps a rust_test sharding wrapper on Windows, the wrapper may execute from another test's runfiles tree. Add a manifest lookup fallback so the real test binary can still be resolved through the active Bazel runfiles manifest. Co-authored-by: Codex --- rust/private/test_sharding_wrapper.bat | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index 3e0acbb549..15c481f157 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -69,6 +69,35 @@ IF !FOUND_BINARY! EQU 0 IF DEFINED RUNFILES_DIR ( ) ) +@REM Try 4: manifest-based runfile lookup. This covers nested launchers that +@REM execute the sharding wrapper from another test's runfiles tree. +IF !FOUND_BINARY! EQU 0 ( + SET "MANIFEST=!RUNFILES_MANIFEST_FILE!" + IF NOT DEFINED MANIFEST IF EXIST "%~f0.runfiles_manifest" SET "MANIFEST=%~f0.runfiles_manifest" + IF NOT DEFINED MANIFEST IF EXIST "%~dpn0.runfiles_manifest" SET "MANIFEST=%~dpn0.runfiles_manifest" + IF NOT DEFINED MANIFEST IF EXIST "%~f0.exe.runfiles_manifest" SET "MANIFEST=%~f0.exe.runfiles_manifest" + + IF DEFINED MANIFEST IF EXIST "!MANIFEST!" ( + SET "TEST_BINARY_MANIFEST_PATH=!TEST_BINARY_RAW!" + SET "TEST_BINARY_MANIFEST_PATH=!TEST_BINARY_MANIFEST_PATH:\=/!" + IF DEFINED TEST_WORKSPACE SET "TEST_BINARY_MANIFEST_WORKSPACE_PATH=!TEST_WORKSPACE!/!TEST_BINARY_MANIFEST_PATH!" + FOR /F "usebackq tokens=1,* delims= " %%A IN ("!MANIFEST!") DO ( + IF "%%A"=="!TEST_BINARY_MANIFEST_PATH!" ( + SET "TEST_BINARY_PATH=%%B" + SET FOUND_BINARY=1 + GOTO :FOUND_TEST_BINARY + ) + IF DEFINED TEST_BINARY_MANIFEST_WORKSPACE_PATH IF "%%A"=="!TEST_BINARY_MANIFEST_WORKSPACE_PATH!" ( + SET "TEST_BINARY_PATH=%%B" + SET FOUND_BINARY=1 + GOTO :FOUND_TEST_BINARY + ) + ) + ) +) + +:FOUND_TEST_BINARY + IF !FOUND_BINARY! EQU 0 ( ECHO ERROR: Could not find test binary at any expected location EXIT /B 1 From 7fe032086269a06f6692a1f332bdac1fd0a9182e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 5/6] rust_test: keep Windows shard hash within u32 Windows PowerShell can interpret 0xffffffff as -1, which means the FNV multiply result was not narrowed before casting back to UInt32. Use explicit UInt64 decimal constants for the FNV prime and UInt32 mask so the sharding wrapper stays within the expected 32-bit range. Co-authored-by: Codex --- rust/private/test_sharding_wrapper.bat | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index 15c481f157..21dd7675ac 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -141,14 +141,16 @@ SET TEMP_SHARD_LIST=%TEMP%\rust_test_shard_%RANDOM%.txt @REM Sort tests by ordinal name and filter this shard by stable FNV-1a hash so @REM adding or removing one test does not move unrelated tests between shards. -@REM In the PowerShell fragment below, 2166136261 is the 32-bit FNV offset basis -@REM and 16777619 is the FNV prime. +@REM In the PowerShell fragment below, 2166136261 is the 32-bit FNV offset basis, +@REM 16777619 is the FNV prime, and 4294967295 is the UInt32 mask. Use decimal +@REM constants because Windows PowerShell can interpret 0xffffffff as -1. powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ "$ErrorActionPreference = 'Stop';" ^ "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ "[Array]::Sort($tests, [StringComparer]::Ordinal);" ^ "$totalShards = [uint32]$env:TOTAL_SHARDS; $shardIndex = [uint32]$env:SHARD_INDEX;" ^ - "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32]((([uint64]($hash -bxor $byte)) * 16777619) -band 0xffffffff) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ + "$fnvPrime = [uint64]16777619; $u32Mask = [uint64]4294967295;" ^ + "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32](([uint64]($hash -bxor $byte) * $fnvPrime) -band $u32Mask) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ > "!TEMP_SHARD_LIST!" IF ERRORLEVEL 1 ( DEL "!TEMP_LIST!" 2>NUL From da81c4ffcd4ab35a7ba22c86f8640099a35b7614 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 08:25:11 -0700 Subject: [PATCH 6/6] rust_test: isolate Windows shard temp files Use Bazel's per-test TEST_TMPDIR when available and create a unique temporary directory for each Windows sharding wrapper invocation. This avoids shared %TEMP% filename collisions when many test shards run concurrently and one shard deletes another shard's libtest list file. Co-authored-by: Codex --- rust/private/rust.bzl | 18 +++++++++--------- rust/private/test_sharding_wrapper.bat | 25 ++++++++++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 929dc56656..dfc65fabbf 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -951,15 +951,6 @@ _RUST_TEST_ATTRS = { "env_inherit": attr.string_list( doc = "Specifies additional environment variables to inherit from the external environment when the test is executed by bazel test.", ), - "use_libtest_harness": attr.bool( - mandatory = False, - default = True, - doc = dedent("""\ - Whether to use `libtest`. For targets using this flag, individual tests can be run by using the - [--test_arg](https://docs.bazel.build/versions/4.0.0/command-line-reference.html#flag--test_arg) flag. - E.g. `bazel test //src:rust_test --test_arg=foo::test::test_fn`. - """), - ), "experimental_enable_sharding": attr.bool( mandatory = False, default = False, @@ -978,6 +969,15 @@ _RUST_TEST_ATTRS = { This is experimental and may change in future releases. """), ), + "use_libtest_harness": attr.bool( + mandatory = False, + default = True, + doc = dedent("""\ + Whether to use `libtest`. For targets using this flag, individual tests can be run by using the + [--test_arg](https://docs.bazel.build/versions/4.0.0/command-line-reference.html#flag--test_arg) flag. + E.g. `bazel test //src:rust_test --test_arg=foo::test::test_fn`. + """), + ), "_test_sharding_wrapper_unix": attr.label( default = Label("//rust/private:test_sharding_wrapper.sh"), allow_single_file = True, diff --git a/rust/private/test_sharding_wrapper.bat b/rust/private/test_sharding_wrapper.bat index 21dd7675ac..e90a803f1c 100644 --- a/rust/private/test_sharding_wrapper.bat +++ b/rust/private/test_sharding_wrapper.bat @@ -132,12 +132,25 @@ IF NOT "%TEST_SHARD_STATUS_FILE%"=="" IF NOT "%TEST_TOTAL_SHARDS%"=="" IF NOT "% TYPE NUL > "%TEST_SHARD_STATUS_FILE%" ) -@REM Create a temporary file for test list -SET TEMP_LIST=%TEMP%\rust_test_list_%RANDOM%.txt -SET TEMP_SHARD_LIST=%TEMP%\rust_test_shard_%RANDOM%.txt +@REM Create per-wrapper temporary files. Prefer Bazel's per-test temp directory; +@REM when falling back to the shared temp directory, avoid %RANDOM%-only file +@REM names that can collide across concurrently running Windows test shards. +SET "TEMP_ROOT=%TEST_TMPDIR%" +IF NOT DEFINED TEMP_ROOT SET "TEMP_ROOT=%TEMP%" +IF NOT DEFINED TEMP_ROOT SET "TEMP_ROOT=." +:CREATE_TEMP_DIR +SET "TEMP_DIR=!TEMP_ROOT!\rust_test_sharding_!RANDOM!_!RANDOM!_!RANDOM!" +MKDIR "!TEMP_DIR!" 2>NUL +IF ERRORLEVEL 1 GOTO :CREATE_TEMP_DIR +SET "TEMP_LIST=!TEMP_DIR!\list.txt" +SET "TEMP_SHARD_LIST=!TEMP_DIR!\shard.txt" @REM Enumerate all tests using libtest's --list flag !TEST_BINARY_PATH! --list --format terse 2>NUL > "!TEMP_LIST!" +IF ERRORLEVEL 1 ( + RMDIR /S /Q "!TEMP_DIR!" 2>NUL + EXIT /B 1 +) @REM Sort tests by ordinal name and filter this shard by stable FNV-1a hash so @REM adding or removing one test does not move unrelated tests between shards. @@ -153,8 +166,7 @@ powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32](([uint64]($hash -bxor $byte) * $fnvPrime) -band $u32Mask) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ > "!TEMP_SHARD_LIST!" IF ERRORLEVEL 1 ( - DEL "!TEMP_LIST!" 2>NUL - DEL "!TEMP_SHARD_LIST!" 2>NUL + RMDIR /S /Q "!TEMP_DIR!" 2>NUL EXIT /B 1 ) @@ -168,8 +180,7 @@ FOR /F "usebackq delims=" %%T IN ("!TEMP_SHARD_LIST!") DO ( ) ) -DEL "!TEMP_LIST!" 2>NUL -DEL "!TEMP_SHARD_LIST!" 2>NUL +RMDIR /S /Q "!TEMP_DIR!" 2>NUL @REM If no tests for this shard, exit successfully IF "!SHARD_TESTS!"=="" (