bench: a nonzero zy exit (1 compile error, 2 trap) or a wrong checksum fails the row with the stderr tail, in suite.py and bench.sh; selftest cases for exit 1, exit 2 and a wrong checksum. arrays rows scaled to ~3 ms a repetition (matmul 40x40, gather 98304 words); the baseline's arrays rows are stale until a re-save

1 files changed, 54 insertions, 9 deletions

diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index fadb5b3..a30f40f 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -54,6 +54,12 @@ SWEEPS = [
     ("zy:arrays.matmul", "tests/bench/arrays.zc", "m"),
     ("zy:arrays.gather", "tests/bench/arrays.zc", "g"),
 ]
+# what each row prints, when the suite knows it (the checksum doesn't depend on the
+# repetition count). a row that prints something else fails the bench
+EXPECT = {
+    "zy:arrays.matmul": "0",
+    "zy:arrays.gather": "4831789056",
+}
 # where each row's checks are counted: the source and the function (bench.sh rows are
 # backend:kernel, all from tests/programs/bench.zc)
 CHECK_FNS = {
@@ -89,29 +95,43 @@ def bench_sh():
     return rows
 
 
-def run_once(zy, src, letter, reps):
+def zy_failure(what, code, out, err, expect=None):
+    """None when a zy run is good, else the message: a nonzero exit (1 is a compile error,
+    a trap will be 2) or output other than the expected checksum"""
+    tail = lambda b: "\n".join(b.decode(errors="replace").rstrip().splitlines()[-8:]) or "(empty)"
+    if code != 0:
+        kind = {1: "compile error", 2: "runtime trap"}.get(code, "failed")
+        return "bench: %s: zy exited %d (%s); stderr tail:\n%s" % (what, code, kind, tail(err))
+    if expect is not None and out.decode(errors="replace").strip() != expect:
+        return "bench: %s: printed %r, expected %r; stderr tail:\n%s" % (what, out.decode(errors="replace").strip()[-200:], expect, tail(err))
+    return None
+
+
+def run_once(zy, src, letter, reps, what="", expect=None):
     data = src + b"`" + letter.encode() + str(reps).encode()
     t = time.perf_counter()
     r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     dt = time.perf_counter() - t
-    if r.returncode != 0:
-        sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+    bad = zy_failure("%s (%s, kernel %s, %d reps)" % (what or letter, zy, letter, reps), r.returncode, r.stdout, r.stderr, expect)
+    if bad:
+        sys.exit(bad)
     return dt
 
 
-def slope_ms(zy, path, letter):
+def slope_ms(zy, path, letter, name=""):
     src = (ROOT / path).read_bytes()
+    go = lambda reps: run_once(zy, src, letter, reps, name or path, EXPECT.get(name))
     n = 4
     while True:
-        one = min(run_once(zy, src, letter, 1) for _ in range(2))
-        many = run_once(zy, src, letter, n)
+        one = min(go(1) for _ in range(2))
+        many = go(n)
         if many - one >= MIN_S or n >= 1 << 16:
             break
         n *= 4
     slopes = []
     for _ in range(TRIALS):
-        one = run_once(zy, src, letter, 1)
-        many = run_once(zy, src, letter, n)
+        one = go(1)
+        many = go(n)
         slopes.append((many - one) / (n - 1) * 1000)
     return statistics.median(slopes)
 
@@ -181,6 +201,9 @@ def region_guard(zy):
     src = (ROOT / "tests/programs/o3.zc").read_bytes()
     env = dict(os.environ, ZYTRACE="1")
     r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    bad = zy_failure("tests/programs/o3.zc (%s)" % zy, r.returncode, r.stdout, r.stderr)
+    if bad:
+        sys.exit(bad)
     err = r.stderr.decode(errors="replace").splitlines()
     done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l]
     stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l]
@@ -210,7 +233,7 @@ def run_all(zy):
         print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
         rows.append({"program": name, "plain": plain, "o3": o3, "checks": checks_of(zy, name), "peak": "-", "gc": "-"})
     for name, path, letter in SWEEPS:
-        ms = slope_ms(zy, path, letter)
+        ms = slope_ms(zy, path, letter, name)
         print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
         rows.append({"program": name, "plain": ms, "o3": None, "checks": checks_of(zy, name), "peak": "-", "gc": "-"})
     o3_guard(rows)
@@ -335,6 +358,28 @@ def selftest():
         case("checks fell 3 -> 0", two("0"), True),
         case("checks '-' now (no --explain-checks)", two("-"), False),
     ]
+    # zy exits: a fake zy that exits with the given code must fail the row, and a wrong
+    # checksum must too
+    def zcase(name, code, out, want):
+        fake = t / "fakezy"
+        fake.write_text("#!/bin/sh\ncat >/dev/null\necho %s\necho 'line 3: oops' >&2\nexit %d\n" % (out, code))
+        fake.chmod(0o755)
+        try:
+            run_once(str(fake), b"", "m", 1, "a:row", "42")
+            failed, msg = False, ""
+        except SystemExit as e:
+            failed, msg = True, str(e.code)
+        ok = failed == want and (not want or "a:row" in msg)
+        print("selftest: %-36s %s, want %s: %s" % (name, "failed" if failed else "ok", "fail" if want else "ok", "pass" if ok else "WRONG"))
+        if not ok or failed and os.environ.get("V") == "1":
+            print(msg)
+        return ok
+    good += [
+        zcase("zy exit 0, right checksum", 0, "42", False),
+        zcase("zy exit 1 (compile error)", 1, "", True),
+        zcase("zy exit 2 (runtime trap)", 2, "42", True),
+        zcase("zy exit 0, wrong checksum", 0, "41", True),
+    ]
     print("selftest: %s" % ("ok" if all(good) else "FAIL"))
     return 0 if all(good) else 1