bench: a nonzero zy exit (1 compile error, 2 trap) or a wrong checksum fails the row with the stderr tail, in suite.py and bench.sh; selftest cases for exit 1, exit 2 and a wrong checksum. arrays rows scaled to ~3 ms a repetition (matmul 40x40, gather 98304 words); the baseline's arrays rows are stale until a re-save

3 files changed, 73 insertions, 21 deletions

diff --git a/tests/bench/arrays.zc b/tests/bench/arrays.zc
index f83cebe..bbe0f01 100644
--- a/tests/bench/arrays.zc
+++ b/tests/bench/arrays.zc
@@ -4,19 +4,19 @@ include rt;
 // two array kernels where the bounds checks are the point, one whose hot loop the
 // compiler proves safe and one that keeps its check on purpose. zy --explain-checks
 // shows which checks stay (suite.py puts the count in the checks column).
-//   m  matmul: c = a * b, 24x24, flat row-major. each row of a and column of b is
+//   m  matmul: c = a * b, 40x40, flat row-major. each row of a and column of b is
 //      copied into a local int[N], and the inner k loop runs k < N over those only,
 //      so its checks are proved. the copies index the flat matrices with i * N + k,
 //      which keeps their checks, but they run N^2 times, not N^3
-//   g  gather: s += data[idx[i]] over 4096 words. idx[i] is proved by i < len(idx);
+//   g  gather: s += data[idx[i]] over 98304 words. idx[i] is proved by i < len(idx);
 //      data[idx[i]] is indexed by a loaded value, so its check stays in the hot loop
 // input: a letter (m, g, or z for neither) and how many times to run it. prints a checksum.
-const int N = 24;
-const int G = 4096;
+const int N = 40;
+const int G = 98304;
 
 int matmul(fixed int[?] a, fixed int[?] b, fixed int[?] c) {
-	int[24] ra;
-	int[24] cb;
+	int[40] ra;
+	int[40] cb;
 	int s = 0;
 	for (int i = 0; i < N; i++) {
 		for (int k = 0; k < N; k++) ra[k] = a[i * N + k];
@@ -44,7 +44,7 @@ fn main() {
 	for (int i = 0; i < N * N; i++) { a[i] = i % 7 - 3; b[i] = i % 5 - 2; }
 	fixed int[?] data = rt.new(fixed int[?], G);
 	fixed int[?] idx = rt.new(fixed int[?], G);
-	// 7919 is odd, so i * 7919 mod 4096 visits every slot once, in a scattered order
+	// 7919 is prime and not 2 or 3, so i * 7919 mod 98304 (2^15 * 3) visits every slot once, in a scattered order
 	for (int i = 0; i < G; i++) { data[i] = i; idx[i] = i * 7919 % G; }
 	while (reps > 0) {
 		if (which == 'm') r = matmul(a, b, c);
diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
index e532f93..4032aa0 100644
--- a/tests/bench/bench.sh
+++ b/tests/bench/bench.sh
@@ -24,15 +24,22 @@ if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/
 fi
 # seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run,
 # best of three each, so starting up and compiling cancel out
-t(){ local s e; s=$(date +%s.%N); (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null; e=$(date +%s.%N); echo "$e - $s" | bc; }
-best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3); b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
+# a nonzero zy exit (1 compile error, 2 trap) fails the whole bench, with the stderr tail
+t(){
+	local s e c err=$(mktemp); s=$(date +%s.%N)
+	(cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null 2>$err; c=$?
+	e=$(date +%s.%N)
+	if [ $c != 0 ]; then echo "bench.sh: tests/programs/bench.zc kernel $2 ($3 reps) on $1: zy exited $c; stderr tail:" >&2; tail -n 8 $err >&2; rm -f $err; return 1; fi
+	rm -f $err; echo "$e - $s" | bc
+}
+best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3) || return 1; b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
 # grows the run count until the difference is at least a second, so the noise
 # of starting up is small next to it
 per(){
 	local one n=4 many
-	one=$(best "$1" $2 1)
+	one=$(best "$1" $2 1) || return 1
 	while :; do
-		many=$(best "$1" $2 $n)
+		many=$(best "$1" $2 $n) || return 1
 		if [ "$(echo "$many - $one > 1" | bc)" = 1 ] || [ $n -ge 4096 ]; then break; fi
 		n=$((n * 4))
 	done
@@ -42,7 +49,7 @@ printf "%-14s %-8s %10s %10s %8s\n" backend kernel "plain ms" "o3 ms" speedup
 for i in "${!vms[@]}"; do
 	for k in "collatz a b" "sieve c d" "digits e f" "fib g h"; do
 		set -- $k
-		p=$(per "${vms[$i]}" $2); o=$(per "${vms[$i]}" $3)
+		p=$(per "${vms[$i]}" $2) || exit 1; o=$(per "${vms[$i]}" $3) || exit 1
 		printf "%-14s %-8s %10.2f %10.2f %7.1fx\n" "${names[$i]}" $1 $(echo "$p * 1000" | bc) $(echo "$o * 1000" | bc) $(echo "scale=2; $p / $o" | bc)
 	done
 done
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index fadb5b3..a30f40f 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -54,6 +54,12 @@ SWEEPS = [
     ("zy:arrays.matmul", "tests/bench/arrays.zc", "m"),
     ("zy:arrays.gather", "tests/bench/arrays.zc", "g"),
 ]
+# what each row prints, when the suite knows it (the checksum doesn't depend on the
+# repetition count). a row that prints something else fails the bench
+EXPECT = {
+    "zy:arrays.matmul": "0",
+    "zy:arrays.gather": "4831789056",
+}
 # where each row's checks are counted: the source and the function (bench.sh rows are
 # backend:kernel, all from tests/programs/bench.zc)
 CHECK_FNS = {
@@ -89,29 +95,43 @@ def bench_sh():
     return rows
 
 
-def run_once(zy, src, letter, reps):
+def zy_failure(what, code, out, err, expect=None):
+    """None when a zy run is good, else the message: a nonzero exit (1 is a compile error,
+    a trap will be 2) or output other than the expected checksum"""
+    tail = lambda b: "\n".join(b.decode(errors="replace").rstrip().splitlines()[-8:]) or "(empty)"
+    if code != 0:
+        kind = {1: "compile error", 2: "runtime trap"}.get(code, "failed")
+        return "bench: %s: zy exited %d (%s); stderr tail:\n%s" % (what, code, kind, tail(err))
+    if expect is not None and out.decode(errors="replace").strip() != expect:
+        return "bench: %s: printed %r, expected %r; stderr tail:\n%s" % (what, out.decode(errors="replace").strip()[-200:], expect, tail(err))
+    return None
+
+
+def run_once(zy, src, letter, reps, what="", expect=None):
     data = src + b"`" + letter.encode() + str(reps).encode()
     t = time.perf_counter()
     r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     dt = time.perf_counter() - t
-    if r.returncode != 0:
-        sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+    bad = zy_failure("%s (%s, kernel %s, %d reps)" % (what or letter, zy, letter, reps), r.returncode, r.stdout, r.stderr, expect)
+    if bad:
+        sys.exit(bad)
     return dt
 
 
-def slope_ms(zy, path, letter):
+def slope_ms(zy, path, letter, name=""):
     src = (ROOT / path).read_bytes()
+    go = lambda reps: run_once(zy, src, letter, reps, name or path, EXPECT.get(name))
     n = 4
     while True:
-        one = min(run_once(zy, src, letter, 1) for _ in range(2))
-        many = run_once(zy, src, letter, n)
+        one = min(go(1) for _ in range(2))
+        many = go(n)
         if many - one >= MIN_S or n >= 1 << 16:
             break
         n *= 4
     slopes = []
     for _ in range(TRIALS):
-        one = run_once(zy, src, letter, 1)
-        many = run_once(zy, src, letter, n)
+        one = go(1)
+        many = go(n)
         slopes.append((many - one) / (n - 1) * 1000)
     return statistics.median(slopes)
 
@@ -181,6 +201,9 @@ def region_guard(zy):
     src = (ROOT / "tests/programs/o3.zc").read_bytes()
     env = dict(os.environ, ZYTRACE="1")
     r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    bad = zy_failure("tests/programs/o3.zc (%s)" % zy, r.returncode, r.stdout, r.stderr)
+    if bad:
+        sys.exit(bad)
     err = r.stderr.decode(errors="replace").splitlines()
     done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l]
     stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l]
@@ -210,7 +233,7 @@ def run_all(zy):
         print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
         rows.append({"program": name, "plain": plain, "o3": o3, "checks": checks_of(zy, name), "peak": "-", "gc": "-"})
     for name, path, letter in SWEEPS:
-        ms = slope_ms(zy, path, letter)
+        ms = slope_ms(zy, path, letter, name)
         print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
         rows.append({"program": name, "plain": ms, "o3": None, "checks": checks_of(zy, name), "peak": "-", "gc": "-"})
     o3_guard(rows)
@@ -335,6 +358,28 @@ def selftest():
         case("checks fell 3 -> 0", two("0"), True),
         case("checks '-' now (no --explain-checks)", two("-"), False),
     ]
+    # zy exits: a fake zy that exits with the given code must fail the row, and a wrong
+    # checksum must too
+    def zcase(name, code, out, want):
+        fake = t / "fakezy"
+        fake.write_text("#!/bin/sh\ncat >/dev/null\necho %s\necho 'line 3: oops' >&2\nexit %d\n" % (out, code))
+        fake.chmod(0o755)
+        try:
+            run_once(str(fake), b"", "m", 1, "a:row", "42")
+            failed, msg = False, ""
+        except SystemExit as e:
+            failed, msg = True, str(e.code)
+        ok = failed == want and (not want or "a:row" in msg)
+        print("selftest: %-36s %s, want %s: %s" % (name, "failed" if failed else "ok", "fail" if want else "ok", "pass" if ok else "WRONG"))
+        if not ok or failed and os.environ.get("V") == "1":
+            print(msg)
+        return ok
+    good += [
+        zcase("zy exit 0, right checksum", 0, "42", False),
+        zcase("zy exit 1 (compile error)", 1, "", True),
+        zcase("zy exit 2 (runtime trap)", 2, "42", True),
+        zcase("zy exit 0, wrong checksum", 0, "41", True),
+    ]
     print("selftest: %s" % ("ok" if all(good) else "FAIL"))
     return 0 if all(good) else 1