aboutsummaryrefslogtreecommitdiff
path: root/tests/bench
diff options
context:
space:
mode:
Diffstat (limited to 'tests/bench')
-rw-r--r--tests/bench/suite.py191
-rw-r--r--tests/bench/sweep.zc23
2 files changed, 214 insertions, 0 deletions
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
new file mode 100644
index 0000000..bb81e46
--- /dev/null
+++ b/tests/bench/suite.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`.
+
+usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
+ tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE]
+
+it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc
+(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the
+256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded,
+timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the
+median of TRIALS slopes. bench.sh itself is run as it is (it needs bc).
+
+one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the
+number of bounds checks the compiler keeps (from --explain-checks, which does not exist
+yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'.
+
+check compares with the baseline (tests/bench.baseline): each of the plain and o3
+columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails
+when both sides have a checks count and it rose. when anything looks slower the whole
+measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
+so one noisy run does not fail the check.
+"""
+import argparse
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+MIN_S = float(os.environ.get("MIN_S", "0.4"))
+TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
+RETRIES = int(os.environ.get("RETRIES", "1"))
+
+# the rows this script times itself: name, source, kernel letter (bench.sh has the rest)
+SWEEPS = [
+ ("zy:sweep.checked", "tests/bench/sweep.zc", "c"),
+ ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"),
+]
+COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"]
+KERNELS = ("collatz", "sieve", "digits", "fib")
+
+
+def bench_sh():
+ """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)"""
+ r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ if r.returncode != 0:
+ sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:])
+ rows = []
+ for line in r.stdout.splitlines():
+ w = line.split()
+ # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end
+ if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"):
+ backend = "-".join(w[:-4])
+ rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2])))
+ if not rows:
+ sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:])
+ return rows
+
+
+def run_once(zy, src, letter, reps):
+ data = src + b"`" + letter.encode() + str(reps).encode()
+ t = time.perf_counter()
+ r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ dt = time.perf_counter() - t
+ if r.returncode != 0:
+ sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+ return dt
+
+
+def slope_ms(zy, path, letter):
+ src = (ROOT / path).read_bytes()
+ n = 4
+ while True:
+ one = min(run_once(zy, src, letter, 1) for _ in range(2))
+ many = run_once(zy, src, letter, n)
+ if many - one >= MIN_S or n >= 1 << 16:
+ break
+ n *= 4
+ slopes = []
+ for _ in range(TRIALS):
+ one = run_once(zy, src, letter, 1)
+ many = run_once(zy, src, letter, n)
+ slopes.append((many - one) / (n - 1) * 1000)
+ return statistics.median(slopes)
+
+
+def fmt_row(cells):
+ return "%-26s %10s %10s %8s %16s %11s" % tuple(cells)
+
+
+def num(x):
+ return "-" if x is None else "%.3f" % x
+
+
+def write_table(path, rows):
+ lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available",
+ fmt_row(COLUMNS)]
+ for r in rows:
+ lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]]))
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("\n".join(lines) + "\n")
+
+
+def read_table(path):
+ rows = {}
+ f = lambda x: None if x == "-" else float(x)
+ for line in path.read_text().splitlines():
+ if not line.strip() or line.startswith("#") or line.split()[0] == "program":
+ continue
+ w = line.split()
+ rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]}
+ return rows
+
+
+def run_all(zy):
+ rows = []
+ for name, plain, o3 in bench_sh():
+ print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
+ rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"})
+ for name, path, letter in SWEEPS:
+ ms = slope_ms(zy, path, letter)
+ print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
+ rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"})
+ return rows
+
+
+def slower(now, base):
+ return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE)
+
+
+def pct(now, base):
+ return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
+
+
+def main():
+ ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
+ ap.add_argument("mode", choices=["run", "check"])
+ ap.add_argument("--zy", default=str(ROOT / "build/bin/zy"))
+ ap.add_argument("--out", default=str(ROOT / "build/bench.txt"))
+ ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline"))
+ a = ap.parse_args()
+ rows = run_all(a.zy)
+ if a.mode == "run":
+ write_table(Path(a.out), rows)
+ print(Path(a.out).read_text(), end="")
+ return 0
+ base_path = Path(a.baseline)
+ if not base_path.exists():
+ sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
+ base = read_table(base_path)
+ tries = 0
+ while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
+ tries += 1
+ print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True)
+ again = {r["program"]: r for r in run_all(a.zy)}
+ for r in rows:
+ g = again.get(r["program"])
+ if g:
+ for c in ("plain", "o3"):
+ if r[c] is not None and g[c] is not None:
+ r[c] = min(r[c], g[c])
+ bad = 0
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict"))
+ names = set()
+ for r in rows:
+ names.add(r["program"])
+ b = base.get(r["program"])
+ if not b:
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"]))
+ continue
+ why = [c for c in ("plain", "o3") if slower(r[c], b[c])]
+ verdict = ", ".join(c + " SLOWER" for c in why)
+ if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]):
+ verdict = (verdict + ", " if verdict else "") + "MORE CHECKS"
+ if verdict:
+ bad += 1
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]),
+ num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok"))
+ for name in base:
+ if name not in names:
+ print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-"))
+ write_table(Path(a.out), rows)
+ print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100)))
+ return 1 if bad else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/bench/sweep.zc b/tests/bench/sweep.zc
new file mode 100644
index 0000000..9bc7070
--- /dev/null
+++ b/tests/bench/sweep.zc
@@ -0,0 +1,23 @@
+namespace;
+include rt;
+
+// a 256-word sweep through a heap pointer, with bounds checks (the default) and without
+// (unbounded): each pass adds every word of the block 1000 times. the index is variable
+// and the block is reached through a pointer, so the checked version keeps its checks.
+// input: a letter (c checked, u unbounded, z none) and how many times to run it.
+int sweep(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+unbounded int sweepu(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+fn main() {
+ int* p = rt.new(256);
+ for (int i = 0; i < 256; i++) p[i] = i;
+ let which = rt.get_c(), reps = 0, ch = rt.get_c(), r = 0;
+ while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = rt.get_c(); }
+ while (reps > 0) {
+ if (which == 'c') r = sweep(p, 256);
+ if (which == 'u') r = sweepu(p, 256);
+ reps--;
+ }
+ rt.put_n(r); rt.put_c(10);
+ rt.free(p);
+ return 0;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-27 13:54:35 +0000