wip: bench wraps bench.sh

1 files changed, 87 insertions, 61 deletions

diff --git a/tests/bench/suite.py b/tests/bench/suite.py
index a5a61ed..bb81e46 100644
--- a/tests/bench/suite.py
+++ b/tests/bench/suite.py
@@ -4,22 +4,21 @@
 usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
        tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE]
 
-each program is a kernel of tests/programs/bench.zc (the kernels bench.sh times, plain
-and o3) or of tests/bench/sweep.zc (a 256-word pointer sweep, checked and unbounded),
-run on one core (build/bin/zy by default, the host's jit where there is one). a kernel
-is selected by a letter and a repetition count on stdin, after the source. the time per
-repetition is the slope between 1 and N repetitions, so starting the vm and compiling
-the program cancel out; N grows until the N run takes at least MIN_S seconds. each
-cell is the median of TRIALS slopes.
-
-columns: ms per repetition; checks, the number of bounds checks the compiler keeps
-(from --explain-checks, which does not exist yet: '-'); peak heap words and gc mark
-time, which the runtime does not expose: '-'.
-
-check compares with the baseline (tests/bench.baseline): it fails when a program is
-more than TOLERANCE (10%) slower, or when both sides have a checks count and it rose.
-a program that looks slower is measured again, up to RETRIES more times, and its best
-median is kept, so one noisy trial does not fail the check.
+it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc
+(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the
+256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded,
+timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the
+median of TRIALS slopes. bench.sh itself is run as it is (it needs bc).
+
+one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the
+number of bounds checks the compiler keeps (from --explain-checks, which does not exist
+yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'.
+
+check compares with the baseline (tests/bench.baseline): each of the plain and o3
+columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails
+when both sides have a checks count and it rose. when anything looks slower the whole
+measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
+so one noisy run does not fail the check.
 """
 import argparse
 import os
@@ -33,22 +32,32 @@ ROOT = Path(__file__).resolve().parent.parent.parent
 TRIALS = int(os.environ.get("TRIALS", "5"))
 MIN_S = float(os.environ.get("MIN_S", "0.4"))
 TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
-RETRIES = int(os.environ.get("RETRIES", "2"))
-
-# name, source, kernel letter
-PROGRAMS = [
-    ("collatz", "tests/programs/bench.zc", "a"),
-    ("collatz.o3", "tests/programs/bench.zc", "b"),
-    ("sieve", "tests/programs/bench.zc", "c"),
-    ("sieve.o3", "tests/programs/bench.zc", "d"),
-    ("digits", "tests/programs/bench.zc", "e"),
-    ("digits.o3", "tests/programs/bench.zc", "f"),
-    ("fib", "tests/programs/bench.zc", "g"),
-    ("fib.o3", "tests/programs/bench.zc", "h"),
-    ("sweep.checked", "tests/bench/sweep.zc", "c"),
-    ("sweep.unbounded", "tests/bench/sweep.zc", "u"),
+RETRIES = int(os.environ.get("RETRIES", "1"))
+
+# the rows this script times itself: name, source, kernel letter (bench.sh has the rest)
+SWEEPS = [
+    ("zy:sweep.checked", "tests/bench/sweep.zc", "c"),
+    ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"),
 ]
-COLUMNS = ["program", "ms", "checks", "peak_heap_words", "gc_mark_ms"]
+COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"]
+KERNELS = ("collatz", "sieve", "digits", "fib")
+
+
+def bench_sh():
+    """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)"""
+    r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if r.returncode != 0:
+        sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:])
+    rows = []
+    for line in r.stdout.splitlines():
+        w = line.split()
+        # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end
+        if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"):
+            backend = "-".join(w[:-4])
+            rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2])))
+    if not rows:
+        sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:])
+    return rows
 
 
 def run_once(zy, src, letter, reps):
@@ -78,45 +87,54 @@ def slope_ms(zy, path, letter):
     return statistics.median(slopes)
 
 
-def measure(zy, name):
-    for p, path, letter in PROGRAMS:
-        if p == name:
-            return slope_ms(zy, path, letter)
-    raise KeyError(name)
+def fmt_row(cells):
+    return "%-26s %10s %10s %8s %16s %11s" % tuple(cells)
 
 
-def fmt_row(cells):
-    return "%-16s %10s %8s %16s %11s" % tuple(cells)
+def num(x):
+    return "-" if x is None else "%.3f" % x
 
 
 def write_table(path, rows):
-    lines = ["# tinyvm bench: ms per repetition (median of %d slopes); '-' where a figure is not available" % TRIALS,
+    lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available",
              fmt_row(COLUMNS)]
     for r in rows:
-        lines.append(fmt_row([r["program"], "%.3f" % r["ms"], r["checks"], r["peak"], r["gc"]]))
+        lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]]))
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text("\n".join(lines) + "\n")
 
 
 def read_table(path):
     rows = {}
+    f = lambda x: None if x == "-" else float(x)
     for line in path.read_text().splitlines():
         if not line.strip() or line.startswith("#") or line.split()[0] == "program":
             continue
         w = line.split()
-        rows[w[0]] = {"program": w[0], "ms": float(w[1]), "checks": w[2], "peak": w[3], "gc": w[4]}
+        rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]}
     return rows
 
 
 def run_all(zy):
     rows = []
-    for name, _, _ in PROGRAMS:
-        ms = measure(zy, name)
-        print("bench: %-16s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
-        rows.append({"program": name, "ms": ms, "checks": "-", "peak": "-", "gc": "-"})
+    for name, plain, o3 in bench_sh():
+        print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
+        rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"})
+    for name, path, letter in SWEEPS:
+        ms = slope_ms(zy, path, letter)
+        print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
+        rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"})
     return rows
 
 
+def slower(now, base):
+    return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE)
+
+
+def pct(now, base):
+    return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
+
+
 def main():
     ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
     ap.add_argument("mode", choices=["run", "check"])
@@ -133,31 +151,39 @@ def main():
     if not base_path.exists():
         sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
     base = read_table(base_path)
+    tries = 0
+    while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
+        tries += 1
+        print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True)
+        again = {r["program"]: r for r in run_all(a.zy)}
+        for r in rows:
+            g = again.get(r["program"])
+            if g:
+                for c in ("plain", "o3"):
+                    if r[c] is not None and g[c] is not None:
+                        r[c] = min(r[c], g[c])
     bad = 0
-    print("%-16s %10s %10s %8s %9s %9s  %s" % ("program", "base ms", "now ms", "change", "base chk", "now chk", "verdict"))
+    print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict"))
+    names = set()
     for r in rows:
+        names.add(r["program"])
         b = base.get(r["program"])
         if not b:
-            print("%-16s %10s %10.3f %8s %9s %9s  new" % (r["program"], "-", r["ms"], "-", "-", r["checks"]))
+            print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"]))
             continue
-        tries = 0
-        while r["ms"] > b["ms"] * (1 + TOLERANCE) and tries < RETRIES:
-            r["ms"] = min(r["ms"], measure(a.zy, r["program"]))
-            tries += 1
-        change = (r["ms"] - b["ms"]) / b["ms"] * 100 if b["ms"] > 0 else 0.0
-        verdict = "ok"
-        if r["ms"] > b["ms"] * (1 + TOLERANCE):
-            verdict = "SLOWER"
+        why = [c for c in ("plain", "o3") if slower(r[c], b[c])]
+        verdict = ", ".join(c + " SLOWER" for c in why)
         if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]):
-            verdict = "MORE CHECKS" if verdict == "ok" else verdict + ", MORE CHECKS"
-        if verdict != "ok":
+            verdict = (verdict + ", " if verdict else "") + "MORE CHECKS"
+        if verdict:
             bad += 1
-        print("%-16s %10.3f %10.3f %+7.1f%% %9s %9s  %s" % (r["program"], b["ms"], r["ms"], change, b["checks"], r["checks"], verdict))
+        print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s  %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]),
+              num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok"))
     for name in base:
-        if name not in {r["program"] for r in rows}:
-            print("%-16s %10.3f %10s %8s %9s %9s  gone" % (name, base[name]["ms"], "-", "-", base[name]["checks"], "-"))
+        if name not in names:
+            print("%-26s %9s %9s  gone" % (name, num(base[name]["plain"]), "-"))
     write_table(Path(a.out), rows)
-    print("bench-check: %s (%d of %d programs over the limit; tolerance %d%%)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100)))
+    print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100)))
     return 1 if bad else 0