wip: bench suite

2 files changed, 188 insertions, 0 deletions

diff --git a/tests/bench/suite.py b/tests/bench/suite.py
new file mode 100644
index 0000000..a5a61ed
--- /dev/null
+++ b/tests/bench/suite.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`.
+
+usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
+       tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE]
+
+each program is a kernel of tests/programs/bench.zc (the kernels bench.sh times, plain
+and o3) or of tests/bench/sweep.zc (a 256-word pointer sweep, checked and unbounded),
+run on one core (build/bin/zy by default, the host's jit where there is one). a kernel
+is selected by a letter and a repetition count on stdin, after the source. the time per
+repetition is the slope between 1 and N repetitions, so starting the vm and compiling
+the program cancel out; N grows until the N run takes at least MIN_S seconds. each
+cell is the median of TRIALS slopes.
+
+columns: ms per repetition; checks, the number of bounds checks the compiler keeps
+(from --explain-checks, which does not exist yet: '-'); peak heap words and gc mark
+time, which the runtime does not expose: '-'.
+
+check compares with the baseline (tests/bench.baseline): it fails when a program is
+more than TOLERANCE (10%) slower, or when both sides have a checks count and it rose.
+a program that looks slower is measured again, up to RETRIES more times, and its best
+median is kept, so one noisy trial does not fail the check.
+"""
+import argparse
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+MIN_S = float(os.environ.get("MIN_S", "0.4"))
+TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
+RETRIES = int(os.environ.get("RETRIES", "2"))
+
+# name, source, kernel letter
+PROGRAMS = [
+    ("collatz", "tests/programs/bench.zc", "a"),
+    ("collatz.o3", "tests/programs/bench.zc", "b"),
+    ("sieve", "tests/programs/bench.zc", "c"),
+    ("sieve.o3", "tests/programs/bench.zc", "d"),
+    ("digits", "tests/programs/bench.zc", "e"),
+    ("digits.o3", "tests/programs/bench.zc", "f"),
+    ("fib", "tests/programs/bench.zc", "g"),
+    ("fib.o3", "tests/programs/bench.zc", "h"),
+    ("sweep.checked", "tests/bench/sweep.zc", "c"),
+    ("sweep.unbounded", "tests/bench/sweep.zc", "u"),
+]
+COLUMNS = ["program", "ms", "checks", "peak_heap_words", "gc_mark_ms"]
+
+
+def run_once(zy, src, letter, reps):
+    data = src + b"`" + letter.encode() + str(reps).encode()
+    t = time.perf_counter()
+    r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    dt = time.perf_counter() - t
+    if r.returncode != 0:
+        sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+    return dt
+
+
+def slope_ms(zy, path, letter):
+    src = (ROOT / path).read_bytes()
+    n = 4
+    while True:
+        one = min(run_once(zy, src, letter, 1) for _ in range(2))
+        many = run_once(zy, src, letter, n)
+        if many - one >= MIN_S or n >= 1 << 16:
+            break
+        n *= 4
+    slopes = []
+    for _ in range(TRIALS):
+        one = run_once(zy, src, letter, 1)
+        many = run_once(zy, src, letter, n)
+        slopes.append((many - one) / (n - 1) * 1000)
+    return statistics.median(slopes)
+
+
+def measure(zy, name):
+    for p, path, letter in PROGRAMS:
+        if p == name:
+            return slope_ms(zy, path, letter)
+    raise KeyError(name)
+
+
+def fmt_row(cells):
+    return "%-16s %10s %8s %16s %11s" % tuple(cells)
+
+
+def write_table(path, rows):
+    lines = ["# tinyvm bench: ms per repetition (median of %d slopes); '-' where a figure is not available" % TRIALS,
+             fmt_row(COLUMNS)]
+    for r in rows:
+        lines.append(fmt_row([r["program"], "%.3f" % r["ms"], r["checks"], r["peak"], r["gc"]]))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n")
+
+
+def read_table(path):
+    rows = {}
+    for line in path.read_text().splitlines():
+        if not line.strip() or line.startswith("#") or line.split()[0] == "program":
+            continue
+        w = line.split()
+        rows[w[0]] = {"program": w[0], "ms": float(w[1]), "checks": w[2], "peak": w[3], "gc": w[4]}
+    return rows
+
+
+def run_all(zy):
+    rows = []
+    for name, _, _ in PROGRAMS:
+        ms = measure(zy, name)
+        print("bench: %-16s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
+        rows.append({"program": name, "ms": ms, "checks": "-", "peak": "-", "gc": "-"})
+    return rows
+
+
+def main():
+    ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
+    ap.add_argument("mode", choices=["run", "check"])
+    ap.add_argument("--zy", default=str(ROOT / "build/bin/zy"))
+    ap.add_argument("--out", default=str(ROOT / "build/bench.txt"))
+    ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline"))
+    a = ap.parse_args()
+    rows = run_all(a.zy)
+    if a.mode == "run":
+        write_table(Path(a.out), rows)
+        print(Path(a.out).read_text(), end="")
+        return 0
+    base_path = Path(a.baseline)
+    if not base_path.exists():
+        sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
+    base = read_table(base_path)
+    bad = 0
+    print("%-16s %10s %10s %8s %9s %9s  %s" % ("program", "base ms", "now ms", "change", "base chk", "now chk", "verdict"))
+    for r in rows:
+        b = base.get(r["program"])
+        if not b:
+            print("%-16s %10s %10.3f %8s %9s %9s  new" % (r["program"], "-", r["ms"], "-", "-", r["checks"]))
+            continue
+        tries = 0
+        while r["ms"] > b["ms"] * (1 + TOLERANCE) and tries < RETRIES:
+            r["ms"] = min(r["ms"], measure(a.zy, r["program"]))
+            tries += 1
+        change = (r["ms"] - b["ms"]) / b["ms"] * 100 if b["ms"] > 0 else 0.0
+        verdict = "ok"
+        if r["ms"] > b["ms"] * (1 + TOLERANCE):
+            verdict = "SLOWER"
+        if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]):
+            verdict = "MORE CHECKS" if verdict == "ok" else verdict + ", MORE CHECKS"
+        if verdict != "ok":
+            bad += 1
+        print("%-16s %10.3f %10.3f %+7.1f%% %9s %9s  %s" % (r["program"], b["ms"], r["ms"], change, b["checks"], r["checks"], verdict))
+    for name in base:
+        if name not in {r["program"] for r in rows}:
+            print("%-16s %10.3f %10s %8s %9s %9s  gone" % (name, base[name]["ms"], "-", "-", base[name]["checks"], "-"))
+    write_table(Path(a.out), rows)
+    print("bench-check: %s (%d of %d programs over the limit; tolerance %d%%)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100)))
+    return 1 if bad else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/bench/sweep.zc b/tests/bench/sweep.zc
new file mode 100644
index 0000000..9bc7070
--- /dev/null
+++ b/tests/bench/sweep.zc
@@ -0,0 +1,23 @@
+namespace;
+include rt;
+
+// a 256-word sweep through a heap pointer, with bounds checks (the default) and without
+// (unbounded): each pass adds every word of the block 1000 times. the index is variable
+// and the block is reached through a pointer, so the checked version keeps its checks.
+// input: a letter (c checked, u unbounded, z none) and how many times to run it.
+int sweep(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+unbounded int sweepu(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+fn main() {
+	int* p = rt.new(256);
+	for (int i = 0; i < 256; i++) p[i] = i;
+	let which = rt.get_c(), reps = 0, ch = rt.get_c(), r = 0;
+	while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = rt.get_c(); }
+	while (reps > 0) {
+		if (which == 'c') r = sweep(p, 256);
+		if (which == 'u') r = sweepu(p, 256);
+		reps--;
+	}
+	rt.put_n(r); rt.put_c(10);
+	rt.free(p);
+	return 0;
+}