bench: find llvm-config-19 like the Makefile; bench-save refuses when an expected backend has no rows

1 files changed, 24 insertions, 0 deletions

diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index 656e877..3816cc3 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -24,6 +24,8 @@ when both sides have a checks count and it rose. when anything looks slower the
 measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
 so one noisy run does not fail the check.
 """
+import shutil
+import platform
 import argparse
 import os
 import statistics
@@ -172,6 +174,26 @@ def pct(now, base):
     return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
 
 
+def expected_backends():
+    """the backends bench.sh should report on this machine, found the same way it finds them"""
+    want = ["zy"]
+    if platform.system() == "Linux" and platform.machine() == "x86_64":
+        want.append("x86-64-jit")
+    if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"):
+        want.append("llvm")
+    if INTERP:
+        want.append("interp")
+    return want
+
+
+def require_rows(rows):
+    """bench-save refuses a table that is missing a backend this machine has"""
+    have = {r["program"].split(":")[0] for r in rows}
+    missing = [b for b in expected_backends() if b not in have]
+    if missing:
+        sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing))
+
+
 def main():
     ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
     ap.add_argument("mode", choices=["run", "check", "selftest"])
@@ -184,6 +206,8 @@ def main():
         return selftest()
     rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy)
     if a.mode == "run":
+        if os.environ.get("BENCH_SAVE") == "1":
+            require_rows(rows)
         write_table(Path(a.out), rows)
         print(Path(a.out).read_text(), end="")
         return 0