bench: share one source_sha() across the harnesses (#1387)

benchmarks/harness/head_prefetch_ab.py

@@ -50,14 +50,14 @@
 
 import argparse
 import json
-import re
 import statistics
-import subprocess
 import sys
 import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+from source_provenance import source_sha
+
 
 def _harness() -> Any:
     """Import the sibling probe and labeller, whichever directory this ran from."""
@@ -122,58 +122,8 @@ def arm_name(arm: Optional[int]) -> str:
 
 
 def _source_sha() -> str:
-    """The commit of the tree that ``soup_cli`` got imported from, or ``unknown``.
-
-    I resolve this from ``soup_cli.__file__``, not from this driver's own file:
-    the driver and the ``soup_cli`` it measures can come from different
-    checkouts (a ``PYTHONPATH`` override, an editable install elsewhere), and I
-    want the tree that was actually measured. ``-dirty`` covers an uncommitted
-    change on top of that commit, scoped to ``src`` so an untracked output file
-    left at the repo root (this driver's own ``--out``, say) does not turn a
-    clean tree dirty. Written the way ``variant2_gate.py``'s ``_source_sha`` is.
-    """
-    import soup_cli
-
-    package_file = getattr(soup_cli, "__file__", None)
-    if package_file is None:
-        return "unknown"
-    try:
-        package_dir = Path(package_file).resolve().parent
-        root = subprocess.run(
-            ["git", "rev-parse", "--show-toplevel"],
-            cwd=package_dir,
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout.strip()
-        if not root:
-            return "unknown"
-        commit = (
-            subprocess.run(
-                ["git", "rev-parse", "HEAD"],
-                cwd=Path(root),
-                check=True,
-                capture_output=True,
-                text=True,
-                timeout=5,
-            )
-            .stdout.strip()
-            .lower()
-        )
-        dirty = subprocess.run(
-            ["git", "status", "--porcelain", "--untracked-files=normal", "--", "src"],
-            cwd=Path(root),
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout
-    except (OSError, subprocess.SubprocessError):
-        return "unknown"
-    if not re.fullmatch(r"[0-9a-f]{40}", commit):
-        return "unknown"
-    return f"{commit}-dirty" if dirty else commit
+    """Shared with the other harnesses; see ``source_provenance``."""
+    return source_sha(dirty_pathspec="src")
 
 
 def group_of(name: str) -> str:

benchmarks/harness/issue361_nf4_throughput.py

@@ -50,14 +50,15 @@
 
 import argparse
 import json
-import re
 import statistics
 import subprocess
 import sys
 import threading
 import time
 from pathlib import Path
 
+from source_provenance import source_sha
+
 DEFAULT_SEQ = 512
 DEFAULT_BATCH = 1
 DEFAULT_WARMUP = 10
@@ -99,58 +100,8 @@ def _soup_version() -> str:
 
 
 def _source_sha() -> str:
-    """The tree the row was produced from, or ``unknown``.
-
-    ``soup-cli`` reports a release version that does not move between commits,
-    so without this a row cannot name its own tree — and this harness measures a
-    path that three post-#361-base commits changed (#989 above all). Resolved
-    from the ``soup_cli`` package's own location (not this harness file's), so a
-    shadow install elsewhere on ``sys.path`` is not misreported as the checkout
-    that produced the row, and marked ``-dirty`` when that tree has uncommitted
-    changes (#1085's ``variant2_gate._source_sha`` first added both checks).
-    """
-    import soup_cli
-
-    package_file = getattr(soup_cli, "__file__", None)
-    if package_file is None:
-        return "unknown"
-    try:
-        package_dir = Path(package_file).resolve().parent
-        root = subprocess.run(
-            ["git", "rev-parse", "--show-toplevel"],
-            cwd=package_dir,
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout.strip()
-        if not root:
-            return "unknown"
-        commit = (
-            subprocess.run(
-                ["git", "rev-parse", "HEAD"],
-                cwd=Path(root),
-                check=True,
-                capture_output=True,
-                text=True,
-                timeout=5,
-            )
-            .stdout.strip()
-            .lower()
-        )
-        dirty = subprocess.run(
-            ["git", "status", "--porcelain", "--untracked-files=normal"],
-            cwd=Path(root),
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout
-    except (OSError, subprocess.SubprocessError):
-        return "unknown"
-    if not re.fullmatch(r"[0-9a-f]{40}", commit):
-        return "unknown"
-    return f"{commit}-dirty" if dirty else commit
+    """Shared with the other harnesses; see ``source_provenance``."""
+    return source_sha()
 
 
 def _versions() -> dict:
@@ -302,7 +253,6 @@ def __init__(self, interval_s: float = 0.5):
 
     def _poll(self) -> None:
         import shutil
-        import subprocess
 
         tool = shutil.which("nvidia-smi")
         if tool is None:

benchmarks/harness/source_provenance.py

@@ -0,0 +1,59 @@
+"""Which tree a benchmark row measured (#1387).
+
+``soup-cli`` reports a release version that does not move between commits, so a
+harness records the commit of the ``soup_cli`` it imported. Resolved from the
+package's own location, not the harness file's: the two can come from different
+checkouts (a ``PYTHONPATH`` override, an editable install elsewhere).
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+
+def _git(args: list[str], cwd: Path) -> str:
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=5,
+    ).stdout
+
+
+def source_sha(*, dirty_pathspec: Optional[str] = None) -> str:
+    """The commit of the tree ``soup_cli`` was imported from, or ``unknown``.
+
+    ``-dirty`` marks uncommitted changes on top of that commit; ``dirty_pathspec``
+    limits which ones count (``"src"`` keeps a harness's own output file at the
+    repo root from marking a clean tree dirty).
+
+    A ``soup-cli`` wheel installed into a gitignored ``.venv/`` inside the checkout
+    resolves to the checkout too, but the code measured is the wheel's, so
+    ``soup_cli/__init__.py`` must be a file the checkout tracks.
+    """
+    import soup_cli
+
+    package_file = getattr(soup_cli, "__file__", None)
+    if package_file is None:
+        return "unknown"
+    try:
+        package_path = Path(package_file).resolve()
+        root = _git(["rev-parse", "--show-toplevel"], package_path.parent).strip()
+        if not root:
+            return "unknown"
+        _git(["ls-files", "--error-unmatch", str(package_path)], Path(root))
+        commit = _git(["rev-parse", "HEAD"], Path(root)).strip().lower()
+        status = ["status", "--porcelain", "--untracked-files=normal"]
+        if dirty_pathspec is not None:
+            status += ["--", dirty_pathspec]
+        dirty = _git(status, Path(root))
+    except (OSError, subprocess.SubprocessError):
+        return "unknown"
+    if not re.fullmatch(r"[0-9a-f]{40}", commit):
+        return "unknown"
+    return f"{commit}-dirty" if dirty else commit

benchmarks/harness/variant2_gate.py

@@ -47,7 +47,6 @@
 import json
 import re
 import statistics
-import subprocess
 import sys
 import time
 from dataclasses import asdict, dataclass
@@ -58,6 +57,7 @@
 from typing import Any, Callable
 
 import bitexact as shared
+from source_provenance import source_sha
 
 DEFAULT_BATCH = 1
 DEFAULT_BUFFERS = 2
@@ -211,48 +211,8 @@ def _rewired_modules(model: Any) -> int:
 
 
 def _source_sha() -> str:
-    import soup_cli
-
-    package_file = getattr(soup_cli, "__file__", None)
-    if package_file is None:
-        return "unknown"
-    try:
-        package_dir = Path(package_file).resolve().parent
-        root = subprocess.run(
-            ["git", "rev-parse", "--show-toplevel"],
-            cwd=package_dir,
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout.strip()
-        if not root:
-            return "unknown"
-        commit = (
-            subprocess.run(
-                ["git", "rev-parse", "HEAD"],
-                cwd=Path(root),
-                check=True,
-                capture_output=True,
-                text=True,
-                timeout=5,
-            )
-            .stdout.strip()
-            .lower()
-        )
-        dirty = subprocess.run(
-            ["git", "status", "--porcelain", "--untracked-files=normal"],
-            cwd=Path(root),
-            check=True,
-            capture_output=True,
-            text=True,
-            timeout=5,
-        ).stdout
-    except (OSError, subprocess.SubprocessError):
-        return "unknown"
-    if not re.fullmatch(r"[0-9a-f]{40}", commit):
-        return "unknown"
-    return f"{commit}-dirty" if dirty else commit
+    """Shared with the other harnesses; see ``source_provenance``."""
+    return source_sha()
 
 
 def _versions() -> dict[str, str]:

tests/test_issue1174_head_prefetch_ab_provenance.py

@@ -11,6 +11,7 @@
 
 import importlib.util
 import json
+import subprocess
 import sys
 from pathlib import Path
 from types import SimpleNamespace
@@ -62,11 +63,12 @@ def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
         return SimpleNamespace(stdout=status)
 
     monkeypatch.chdir(tmp_path)
-    monkeypatch.setattr(_driver.subprocess, "run", fake_run)
+    monkeypatch.setattr(subprocess, "run", fake_run)
 
     assert _driver._source_sha() == expected_sha + ("-dirty" if status else "")
     assert calls == [
         (["git", "rev-parse", "--show-toplevel"], package_dir),
+        (["git", "ls-files", "--error-unmatch", str(Path(package_file).resolve())], tmp_path),
         (["git", "rev-parse", "HEAD"], tmp_path),
         (["git", "status", "--porcelain", "--untracked-files=normal", "--", "src"], tmp_path),
     ]
@@ -76,15 +78,15 @@ def test_source_sha_is_unknown_without_a_git_toplevel(monkeypatch: pytest.Monkey
     def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
         return SimpleNamespace(stdout="\n")
 
-    monkeypatch.setattr(_driver.subprocess, "run", fake_run)
+    monkeypatch.setattr(subprocess, "run", fake_run)
     assert _driver._source_sha() == "unknown"
 
 
 def test_source_sha_is_unknown_when_git_is_missing(monkeypatch: pytest.MonkeyPatch) -> None:
     def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
         raise FileNotFoundError("no git")
 
-    monkeypatch.setattr(_driver.subprocess, "run", fake_run)
+    monkeypatch.setattr(subprocess, "run", fake_run)
     assert _driver._source_sha() == "unknown"
 
 
@@ -93,9 +95,9 @@ def test_source_sha_is_unknown_when_git_exits_non_zero(monkeypatch: pytest.Monke
     --show-toplevel`` exit 128; with ``check=True`` that is a ``CalledProcessError``."""
 
     def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
-        raise _driver.subprocess.CalledProcessError(128, command)
+        raise subprocess.CalledProcessError(128, command)
 
-    monkeypatch.setattr(_driver.subprocess, "run", fake_run)
+    monkeypatch.setattr(subprocess, "run", fake_run)
     assert _driver._source_sha() == "unknown"
 
 
@@ -109,7 +111,7 @@ def fake_run(command: list[str], *, cwd: Path, **kwargs: object) -> Any:
             return SimpleNamespace(stdout="not-a-commit\n")
         return SimpleNamespace(stdout="")
 
-    monkeypatch.setattr(_driver.subprocess, "run", fake_run)
+    monkeypatch.setattr(subprocess, "run", fake_run)
     assert _driver._source_sha() == "unknown"