1 files changed, 285 insertions, 0 deletions
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
new file mode 100644
index 0000000..3816cc3
--- /dev/null
+++ b/tests/bench/suite.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`.
+
+usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
+       tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE]
+
+it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc
+(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the
+256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded,
+timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before
+timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and
+jit o3 collatz must be under half of jit plain collatz, or it refuses.
+selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the
+median of TRIALS slopes. bench.sh itself is run as it is (it needs bc).
+
+one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the
+number of bounds checks the compiler keeps (from --explain-checks, which does not exist
+yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'.
+
+check compares with the baseline (tests/bench.baseline): each of the plain and o3
+columns fails on its own when it is more than TOLERANCE (10%) slower and also more than
+MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails
+when both sides have a checks count and it rose. when anything looks slower the whole
+measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
+so one noisy run does not fail the check.
+"""
+import shutil
+import platform
+import argparse
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+MIN_S = float(os.environ.get("MIN_S", "0.4"))
+TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
+MIN_MS = float(os.environ.get("MIN_MS", "0.1"))
+INTERP = os.environ.get("INTERP", "") == "1"
+RETRIES = int(os.environ.get("RETRIES", "1"))
+
+# the rows this script times itself: name, source, kernel letter (bench.sh has the rest)
+SWEEPS = [
+    ("zy:sweep.checked", "tests/bench/sweep.zc", "c"),
+    ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"),
+]
+COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"]
+KERNELS = ("collatz", "sieve", "digits", "fib")
+
+
+def bench_sh():
+    """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)"""
+    r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if r.returncode != 0:
+        sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:])
+    rows = []
+    for line in r.stdout.splitlines():
+        w = line.split()
+        # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end
+        if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"):
+            backend = "-".join(w[:-4])
+            rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2])))
+    if not INTERP:
+        rows = [r for r in rows if not r[0].startswith("interp:")]
+    if not rows:
+        sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:])
+    return rows
+
+
+def run_once(zy, src, letter, reps):
+    data = src + b"`" + letter.encode() + str(reps).encode()
+    t = time.perf_counter()
+    r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    dt = time.perf_counter() - t
+    if r.returncode != 0:
+        sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+    return dt
+
+
+def slope_ms(zy, path, letter):
+    src = (ROOT / path).read_bytes()
+    n = 4
+    while True:
+        one = min(run_once(zy, src, letter, 1) for _ in range(2))
+        many = run_once(zy, src, letter, n)
+        if many - one >= MIN_S or n >= 1 << 16:
+            break
+        n *= 4
+    slopes = []
+    for _ in range(TRIALS):
+        one = run_once(zy, src, letter, 1)
+        many = run_once(zy, src, letter, n)
+        slopes.append((many - one) / (n - 1) * 1000)
+    return statistics.median(slopes)
+
+
+def fmt_row(cells):
+    return "%-26s %10s %10s %8s %16s %11s" % tuple(cells)
+
+
+def num(x):
+    return "-" if x is None else "%.3f" % x
+
+
+def write_table(path, rows):
+    lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available",
+             fmt_row(COLUMNS)]
+    for r in rows:
+        lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]]))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n")
+
+
+def read_table(path):
+    rows = {}
+    f = lambda x: None if x == "-" else float(x)
+    for line in path.read_text().splitlines():
+        if not line.strip() or line.startswith("#") or line.split()[0] == "program":
+            continue
+        w = line.split()
+        rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]}
+    return rows
+
+
+def region_guard(zy):
+    """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc"""
+    src = (ROOT / "tests/programs/o3.zc").read_bytes()
+    env = dict(os.environ, ZYTRACE="1")
+    r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    err = r.stderr.decode(errors="replace").splitlines()
+    done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l]
+    stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l]
+    print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True)
+    if not done:
+        sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); "
+                 "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy)
+    return len(done), len(stay)
+
+
+def o3_guard(rows):
+    """jit o3 collatz must be under half of jit plain collatz (needs no interp rows)"""
+    d = {r["program"]: r for r in rows}
+    j = d.get("x86-64-jit:collatz")
+    if j is None:
+        return
+    if j["o3"] is None or j["plain"] is None:
+        sys.exit("bench: REFUSING: x86-64-jit:collatz has no plain or o3 figure")
+    if j["o3"] * 2 >= j["plain"]:
+        sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not under half of jit plain %.2f ms; o3 regions are not compiled" % (j["o3"], j["plain"]))
+
+
+def run_all(zy):
+    region_guard(zy)
+    rows = []
+    for name, plain, o3 in bench_sh():
+        print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
+        rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"})
+    for name, path, letter in SWEEPS:
+        ms = slope_ms(zy, path, letter)
+        print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
+        rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"})
+    o3_guard(rows)
+    return rows
+
+
+def slower(now, base):
+    return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS
+
+
+def pct(now, base):
+    return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
+
+
+def expected_backends():
+    """the backends bench.sh should report on this machine, found the same way it finds them"""
+    want = ["zy"]
+    if platform.system() == "Linux" and platform.machine() == "x86_64":
+        want.append("x86-64-jit")
+    if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"):
+        want.append("llvm")
+    if INTERP:
+        want.append("interp")
+    return want
+
+
+def require_rows(rows):
+    """bench-save refuses a table that is missing a backend this machine has"""
+    have = {r["program"].split(":")[0] for r in rows}
+    missing = [b for b in expected_backends() if b not in have]
+    if missing:
+        sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing))
+
+
+def main():
+    ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
+    ap.add_argument("mode", choices=["run", "check", "selftest"])
+    ap.add_argument("--rows", help="check: compare this saved table instead of measuring")
+    ap.add_argument("--zy", default=str(ROOT / "build/bin/zy"))
+    ap.add_argument("--out", default=str(ROOT / "build/bench.txt"))
+    ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline"))
+    a = ap.parse_args()
+    if a.mode == "selftest":
+        return selftest()
+    rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy)
+    if a.mode == "run":
+        if os.environ.get("BENCH_SAVE") == "1":
+            require_rows(rows)
+        write_table(Path(a.out), rows)
+        print(Path(a.out).read_text(), end="")
+        return 0
+    base_path = Path(a.baseline)
+    if not base_path.exists():
+        sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
+    base = read_table(base_path)
+    tries = 0
+    while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
+        tries += 1
+        print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True)
+        again = {r["program"]: r for r in run_all(a.zy)}
+        for r in rows:
+            g = again.get(r["program"])
+            if g:
+                for c in ("plain", "o3"):
+                    if r[c] is not None and g[c] is not None:
+                        r[c] = min(r[c], g[c])
+    bad = 0
+    print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict"))
+    names = set()
+    for r in rows:
+        names.add(r["program"])
+        b = base.get(r["program"])
+        if not b:
+            print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"]))
+            continue
+        why = [c for c in ("plain", "o3") if slower(r[c], b[c])]
+        verdict = ", ".join(c + " SLOWER" for c in why)
+        if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]):
+            verdict = (verdict + ", " if verdict else "") + "MORE CHECKS"
+        if verdict:
+            bad += 1
+        print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]),
+              num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok"))
+    for name in base:
+        if name not in names:
+            print("%-26s %9s %9s  gone" % (name, num(base[name]["plain"]), "-"))
+    write_table(Path(a.out), rows)
+    print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS))
+    return 1 if bad else 0
+
+
+def selftest():
+    """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts"""
+    import tempfile
+    t = Path(tempfile.mkdtemp())
+    base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in
+            [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]]
+    write_table(t / "base", base)
+    def case(name, now, want):
+        write_table(t / "now", now)
+        r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")],
+                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+        ok = (r.returncode != 0) == want
+        print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG"))
+        if not ok:
+            print(r.stdout)
+        return ok
+    row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"}
+    good = [
+        case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False),
+        case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True),
+        case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True),
+        case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True),
+        case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False),
+        case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False),
+        case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False),
+    ]
+    print("selftest: %s" % ("ok" if all(good) else "FAIL"))
+    return 0 if all(good) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())