bench: find llvm-config-19 like the Makefile; bench-save refuses when an expected backend has no rows

3 files changed, 28 insertions, 2 deletions

diff --git a/tests/bench/__pycache__/suite.cpython-313.pyc b/tests/bench/__pycache__/suite.cpython-313.pyc
new file mode 100644
index 0000000..21783a7
--- /dev/null
+++ b/tests/bench/__pycache__/suite.cpython-313.pyc
diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
index 0437b0b..e532f93 100644
--- a/tests/bench/bench.sh
+++ b/tests/bench/bench.sh
@@ -14,8 +14,10 @@ fi
 if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then
 	$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); }
 fi
-if command -v llvm-config >/dev/null; then
-	$CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
+# the llvm: LLVM_CONFIG when given, else llvm-config, else llvm-config-19 (as in the Makefile)
+LC=${LLVM_CONFIG:-$(command -v llvm-config || command -v llvm-config-19)}
+if [ -n "$LC" ]; then
+	$CC -O2 -w -DJTYPE=int -DZYLLVM $($LC --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $($LC --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
 fi
 if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
 	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); }
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index 656e877..3816cc3 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -24,6 +24,8 @@ when both sides have a checks count and it rose. when anything looks slower the
 measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
 so one noisy run does not fail the check.
 """
+import shutil
+import platform
 import argparse
 import os
 import statistics
@@ -172,6 +174,26 @@ def pct(now, base):
     return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
 
 
+def expected_backends():
+    """the backends bench.sh should report on this machine, found the same way it finds them"""
+    want = ["zy"]
+    if platform.system() == "Linux" and platform.machine() == "x86_64":
+        want.append("x86-64-jit")
+    if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"):
+        want.append("llvm")
+    if INTERP:
+        want.append("interp")
+    return want
+
+
+def require_rows(rows):
+    """bench-save refuses a table that is missing a backend this machine has"""
+    have = {r["program"].split(":")[0] for r in rows}
+    missing = [b for b in expected_backends() if b not in have]
+    if missing:
+        sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing))
+
+
 def main():
     ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
     ap.add_argument("mode", choices=["run", "check", "selftest"])
@@ -184,6 +206,8 @@ def main():
         return selftest()
     rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy)
     if a.mode == "run":
+        if os.environ.get("BENCH_SAVE") == "1":
+            require_rows(rows)
         write_table(Path(a.out), rows)
         print(Path(a.out).read_text(), end="")
         return 0