Draft: hybrid_w4a16 quant kernel perf regression tests

.github/workflows/build-rocm-wheels.yml

@@ -15,12 +15,12 @@ on:
         required: false
       rocm_arch:
         description: 'ROCm architecture (e.g., gfx1151, gfx1150;gfx1151)'
-        default: 'gfx1103;gfx1150;gfx1151'
+        default: 'gfx1151'  # TODO: restore to 'gfx1103;gfx1150;gfx1151' before merging
         required: false
 
 env:
   PYTORCH_INDEX_URL: ${{ github.event.inputs.pytorch_index || 'https://rocm.nightlies.amd.com/v2-staging/gfx1151' }}
-  PYTORCH_ROCM_ARCH: ${{ github.event.inputs.rocm_arch || 'gfx1103;gfx1150;gfx1151' }}
+  PYTORCH_ROCM_ARCH: ${{ github.event.inputs.rocm_arch || 'gfx1151' }}  # TODO: restore to 'gfx1103;gfx1150;gfx1151' before merging
   CI_IMAGE: ghcr.io/rocm/vllm/gfx11-ci:latest
 
 jobs:
@@ -186,6 +186,22 @@ jobs:
           else
             echo "amd-smi not found"
           fi
+          echo "=== GPU clocks, power profile, and temperature ==="
+          for card in /sys/class/drm/card[0-9]/device /sys/class/drm/card[0-9][0-9]/device; do
+            [ -d "$card" ] || continue
+            echo "--- ${card} ---"
+            for f in "$card"/hwmon/hwmon*/temp*_input; do
+              [ -f "$f" ] || continue
+              echo "${f#"$card"/}: $(( $(cat "$f") / 1000 )) °C"
+            done
+            f=power_dpm_force_performance_level
+            [ -f "$card/$f" ] && echo "$f: $(cat "$card/$f")"
+            for f in pp_dpm_mclk pp_dpm_sclk pp_power_profile_mode; do
+              [ -f "$card/$f" ] || continue
+              echo "$f:"
+              head -20 "$card/$f" | sed 's/^/  /'
+            done
+          done
 
       - name: Install wheel and test dependencies
         run: |
@@ -233,7 +249,8 @@ jobs:
             tests/kernels/quantization/test_hybrid_w4a16_triton.py \
             tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py \
             tests/kernels/quantization/test_rocm_skinny_gemms.py \
-            tests/quantization/test_hip_w4a16_kernel.py
+            tests/quantization/test_hip_w4a16_kernel.py \
+            tests/kernels/quantization/test_hybrid_w4a16_perf.py
 
   upload-wheel:
     runs-on: ubuntu-latest

.gitignore

@@ -245,3 +245,6 @@ vllm/grpc/vllm_engine_pb2.pyi
 
 # Ignore generated cpu headers 
 csrc/cpu/cpu_attn_dispatch_generated.h
+
+# Measured performance baselines (never committed)
+tests/kernels/quantization/measured/

pyproject.toml

@@ -109,6 +109,7 @@ markers = [
     "split: run this test as part of a split",
     "distributed: run this test only in distributed GPU tests",
     "optional: optional tests that are automatically skipped, include --optional to run them",
+    "benchmark: performance regression tests",
 ]
 
 [tool.ty.src]

tests/conftest.py

@@ -1439,6 +1439,18 @@ def pytest_addoption(parser):
         default=1000 + secrets.randbelow(9000),
         help="random seed for tests that opt in",
     )
+    parser.addoption(
+        "--measure-baselines",
+        action="store_true",
+        default=False,
+        help="record performance measurements instead of asserting",
+    )
+    parser.addoption(
+        "--intermittent",
+        action="store_true",
+        default=False,
+        help="include intermittent (noisy) benchmark cases",
+    )
 
 
 def pytest_report_header(config):

tests/kernels/quantization/conftest.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""conftest for quantization kernel tests.
+
+Adds session-finish hook that writes measured results to the ``measured/``
+directory when ``--measure-baselines`` is set.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import shutil
+
+import pytest
+
+_HERE = pathlib.Path(__file__).resolve().parent
+_MEASURED_DIR = _HERE / "measured"
+
+# Attribute name on config for the measured-results dict.
+_ATTR = "_hybrid_w4a16_measured_results"
+_TEMP_ATTR = "_hybrid_w4a16_temp_log"
+
+
+def get_measured_results(config: pytest.Config) -> dict[str, list[dict]]:
+    """Return (creating if needed) the session-scoped measurement dict."""
+    d = getattr(config, _ATTR, None)
+    if d is None:
+        d = {}
+        setattr(config, _ATTR, d)
+    return d
+
+
+def get_temp_log(config: pytest.Config) -> list[tuple[float, str, float]]:
+    """Return (creating if needed) the session-scoped temperature log."""
+    log = getattr(config, _TEMP_ATTR, None)
+    if log is None:
+        log = []
+        setattr(config, _TEMP_ATTR, log)
+    return log
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    if config.getoption("--measure-baselines", default=False):
+        # Wipe previous measured/ so aborted runs don't leave stale data.
+        if _MEASURED_DIR.exists():
+            shutil.rmtree(_MEASURED_DIR)
+        _MEASURED_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
+    # Write temperature log if path was set
+    import os
+
+    temp_log_path = os.environ.get("TEMP_LOG_PATH", "")
+    if temp_log_path:
+        log = get_temp_log(session.config)
+        if log:
+            t0 = log[0][0]
+            with open(temp_log_path, "w") as f:
+                f.write("elapsed_s,label,temp_C\n")
+                for ts, label, temp in log:
+                    f.write(f"{ts - t0:.2f},{label},{temp:.1f}\n")
+
+    if not session.config.getoption("--measure-baselines", default=False):
+        return
+
+    results = get_measured_results(session.config)
+    if not results:
+        return
+
+    _MEASURED_DIR.mkdir(parents=True, exist_ok=True)
+
+    for gpu, shapes in results.items():
+        out_path = _MEASURED_DIR / f"hybrid_w4a16_{gpu}.json"
+        data = {"gpu": gpu, "shapes": shapes}
+        out_path.write_text(json.dumps(data, indent=2) + "\n")