bench: no-region guard, selftest and --rows, interp rows behind INTERP=1, fail only over 10% and 0.1 ms

3 files changed, 90 insertions, 13 deletions

diff --git a/Makefile b/Makefile
index 6deb3a9..6569bd5 100644
--- a/Makefile
+++ b/Makefile
@@ -213,18 +213,21 @@ xref: $(BIN)/zclsp$(EXE)
 
 # benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per
 # program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check
-# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline.
+# (or make bench CHECK=1) fails when a column is over 10% and over 0.1 ms slower than the
+# baseline. interp rows only with INTERP=1. bench-selftest checks the verdicts, no timing.
 bench: $(BIN)/zy$(EXE)
 ifeq ($(CHECK),1)
-	python3 tests/bench/suite.py check
+	INTERP=$(INTERP) python3 tests/bench/suite.py check
 else
-	python3 tests/bench/suite.py run
+	INTERP=$(INTERP) python3 tests/bench/suite.py run
 endif
 bench-save: bench
 	cp $(B)/bench.txt tests/bench.baseline
 	@echo "bench: wrote tests/bench.baseline"
 bench-check: $(BIN)/zy$(EXE)
-	python3 tests/bench/suite.py check
+	INTERP=$(INTERP) python3 tests/bench/suite.py check
+bench-selftest:
+	INTERP=$(INTERP) python3 tests/bench/suite.py selftest
 
 # JOBS sets how many sections run at once: make test JOBS=4
 test:
@@ -237,4 +240,4 @@ check:
 clean:
 	rm -rf $(B)
 
-.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean
+.PHONY: bench bench-save bench-selftest bench-check all lsp genome js lspimage vsix web run parity xref test check clean
diff --git a/ONBOARD.md b/ONBOARD.md
index f60063a..2ccb987 100644
--- a/ONBOARD.md
+++ b/ONBOARD.md
@@ -214,7 +214,15 @@ and gc mark columns for runtime counters that do not exist yet; they show `-`. a
 run takes about ten minutes.
 `make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check`
 (or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a
-row is more than 10% slower, or a row keeps more checks once both sides have a count.
+row is more than 10% slower and also more than 0.1 ms slower (so sub-ms cells don't
+flake), or a row keeps more checks once both sides have a count. rows missing from a run
+don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times
+the interpreter today, so this doesn't make the run shorter yet). before timing, the suite
+runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and
+it refuses when jit o3 collatz isn't under a fifth of interp o3; that is how a baseline of
+interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on
+doctored tables in a second, and `suite.py check --rows FILE` compares a saved table
+without measuring.
 when something looks slower the whole run is repeated once and each cell keeps its best
 figure. the baseline belongs to the machine it
 was saved on: save a new one after changing machines, and expect noise on a busy host.
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index bb81e46..f7d5c75 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -7,7 +7,10 @@ usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
 it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc
 (collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the
 256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded,
-timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the
+timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before
+timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and
+jit o3 collatz must be under a fifth of interp o3 (when both rows exist), or it refuses.
+selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the
 median of TRIALS slopes. bench.sh itself is run as it is (it needs bc).
 
 one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the
@@ -15,7 +18,8 @@ number of bounds checks the compiler keeps (from --explain-checks, which does no
 yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'.
 
 check compares with the baseline (tests/bench.baseline): each of the plain and o3
-columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails
+columns fails on its own when it is more than TOLERANCE (10%) slower and also more than
+MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails
 when both sides have a checks count and it rose. when anything looks slower the whole
 measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
 so one noisy run does not fail the check.
@@ -32,6 +36,8 @@ ROOT = Path(__file__).resolve().parent.parent.parent
 TRIALS = int(os.environ.get("TRIALS", "5"))
 MIN_S = float(os.environ.get("MIN_S", "0.4"))
 TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
+MIN_MS = float(os.environ.get("MIN_MS", "0.1"))
+INTERP = os.environ.get("INTERP", "") == "1"
 RETRIES = int(os.environ.get("RETRIES", "1"))
 
 # the rows this script times itself: name, source, kernel letter (bench.sh has the rest)
@@ -55,6 +61,8 @@ def bench_sh():
         if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"):
             backend = "-".join(w[:-4])
             rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2])))
+    if not INTERP:
+        rows = [r for r in rows if not r[0].startswith("interp:")]
     if not rows:
         sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:])
     return rows
@@ -115,7 +123,31 @@ def read_table(path):
     return rows
 
 
+def region_guard(zy):
+    """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc"""
+    src = (ROOT / "tests/programs/o3.zc").read_bytes()
+    env = dict(os.environ, ZYTRACE="1")
+    r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    err = r.stderr.decode(errors="replace").splitlines()
+    done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l]
+    stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l]
+    print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True)
+    if not done:
+        sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); "
+                 "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy)
+    return len(done), len(stay)
+
+
+def o3_guard(rows):
+    """jit o3 collatz must be far below interp o3 collatz when both rows are there"""
+    d = {r["program"]: r for r in rows}
+    j, i = d.get("x86-64-jit:collatz"), d.get("interp:collatz")
+    if j and i and j["o3"] is not None and i["o3"] is not None and j["o3"] * 5 > i["o3"]:
+        sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not far below interp o3 %.2f ms; o3 regions are not compiled" % (j["o3"], i["o3"]))
+
+
 def run_all(zy):
+    region_guard(zy)
     rows = []
     for name, plain, o3 in bench_sh():
         print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
@@ -124,11 +156,12 @@ def run_all(zy):
         ms = slope_ms(zy, path, letter)
         print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
         rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"})
+    o3_guard(rows)
     return rows
 
 
 def slower(now, base):
-    return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE)
+    return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS
 
 
 def pct(now, base):
@@ -137,12 +170,15 @@ def pct(now, base):
 
 def main():
     ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
-    ap.add_argument("mode", choices=["run", "check"])
+    ap.add_argument("mode", choices=["run", "check", "selftest"])
+    ap.add_argument("--rows", help="check: compare this saved table instead of measuring")
     ap.add_argument("--zy", default=str(ROOT / "build/bin/zy"))
     ap.add_argument("--out", default=str(ROOT / "build/bench.txt"))
     ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline"))
     a = ap.parse_args()
-    rows = run_all(a.zy)
+    if a.mode == "selftest":
+        return selftest()
+    rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy)
     if a.mode == "run":
         write_table(Path(a.out), rows)
         print(Path(a.out).read_text(), end="")
@@ -152,7 +188,7 @@ def main():
         sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
     base = read_table(base_path)
     tries = 0
-    while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
+    while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
         tries += 1
         print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True)
         again = {r["program"]: r for r in run_all(a.zy)}
@@ -183,9 +219,39 @@ def main():
         if name not in names:
             print("%-26s %9s %9s  gone" % (name, num(base[name]["plain"]), "-"))
     write_table(Path(a.out), rows)
-    print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100)))
+    print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS))
     return 1 if bad else 0
 
 
+def selftest():
+    """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts"""
+    import tempfile
+    t = Path(tempfile.mkdtemp())
+    base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in
+            [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]]
+    write_table(t / "base", base)
+    def case(name, now, want):
+        write_table(t / "now", now)
+        r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")],
+                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+        ok = (r.returncode != 0) == want
+        print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG"))
+        if not ok:
+            print(r.stdout)
+        return ok
+    row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"}
+    good = [
+        case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False),
+        case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True),
+        case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True),
+        case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True),
+        case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False),
+        case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False),
+        case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False),
+    ]
+    print("selftest: %s" % ("ok" if all(good) else "FAIL"))
+    return 0 if all(good) else 1
+
+
 if __name__ == "__main__":
     sys.exit(main())