diff options
| -rw-r--r-- | tests/bench/suite.py | 78 |
1 files changed, 68 insertions, 10 deletions
diff --git a/tests/bench/suite.py b/tests/bench/suite.py index 3816cc3..fadb5b3 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -6,27 +6,32 @@ usage: tests/bench/suite.py run [--zy PATH] [--out FILE] it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc (collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the -256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, -timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before +256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, and +the two array kernels of tests/bench/arrays.zc (a matmul whose hot loop is proved, a +gather that keeps its check), timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and jit o3 collatz must be under half of jit plain collatz, or it refuses. selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the -number of bounds checks the compiler keeps (from --explain-checks, which does not exist -yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'. +number of bounds checks the compiler keeps in the row's plain function (the lines of +`zy --explain-checks` with that fn; o3 and unbounded code is never listed, so it counts +0), or '-' when zy has no --explain-checks; peak heap words and gc mark time, which the +runtime does not expose: '-'. check compares with the baseline (tests/bench.baseline): each of the plain and o3 columns fails on its own when it is more than TOLERANCE (10%) slower and also more than MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails -when both sides have a checks count and it rose. when anything looks slower the whole +when both sides have a checks count and it changed (more is a lost proof; fewer means the +row no longer measures what it did, so re-save the baseline on purpose). when anything looks slower the whole measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, so one noisy run does not fail the check. """ import shutil import platform import argparse +import json import os import statistics import subprocess @@ -46,7 +51,21 @@ RETRIES = int(os.environ.get("RETRIES", "1")) SWEEPS = [ ("zy:sweep.checked", "tests/bench/sweep.zc", "c"), ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"), + ("zy:arrays.matmul", "tests/bench/arrays.zc", "m"), + ("zy:arrays.gather", "tests/bench/arrays.zc", "g"), ] +# where each row's checks are counted: the source and the function (bench.sh rows are +# backend:kernel, all from tests/programs/bench.zc) +CHECK_FNS = { + "collatz": ("tests/programs/bench.zc", "csum"), + "sieve": ("tests/programs/bench.zc", "sieve"), + "digits": ("tests/programs/bench.zc", "digits"), + "fib": ("tests/programs/bench.zc", "fib"), + "sweep.checked": ("tests/bench/sweep.zc", "sweep"), + "sweep.unbounded": ("tests/bench/sweep.zc", "sweepu"), + "arrays.matmul": ("tests/bench/arrays.zc", "matmul"), + "arrays.gather": ("tests/bench/arrays.zc", "gather"), +} COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"] KERNELS = ("collatz", "sieve", "digits", "fib") @@ -97,6 +116,38 @@ def slope_ms(zy, path, letter): return statistics.median(slopes) +_explained = {} + + +def explain(zy, path): + """the kept-check counts per function of path from zy --explain-checks, or None when + this zy has no --explain-checks (or its output is not format v1)""" + if (zy, path) not in _explained: + counts = None + try: + r = subprocess.run([zy, "--explain-checks", path], input=(ROOT / path).read_bytes(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60) + lines = r.stdout.decode(errors="replace").splitlines() + if r.returncode == 0 and lines and json.loads(lines[0]).get("version") == 1: + counts = {} + for l in lines[1:]: + fn = json.loads(l).get("fn") + counts[fn] = counts.get(fn, 0) + 1 + except (OSError, ValueError, subprocess.TimeoutExpired): + counts = None + _explained[(zy, path)] = counts + return _explained[(zy, path)] + + +def checks_of(zy, program): + """the checks column of a row: its function's kept checks, or '-'""" + where = CHECK_FNS.get(program.split(":", 1)[-1]) + if not where: + return "-" + counts = explain(zy, where[0]) + return "-" if counts is None else str(counts.get(where[1], 0)) + + def fmt_row(cells): return "%-26s %10s %10s %8s %16s %11s" % tuple(cells) @@ -157,11 +208,11 @@ def run_all(zy): rows = [] for name, plain, o3 in bench_sh(): print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) - rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"}) + rows.append({"program": name, "plain": plain, "o3": o3, "checks": checks_of(zy, name), "peak": "-", "gc": "-"}) for name, path, letter in SWEEPS: ms = slope_ms(zy, path, letter) print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) - rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"}) + rows.append({"program": name, "plain": ms, "o3": None, "checks": checks_of(zy, name), "peak": "-", "gc": "-"}) o3_guard(rows) return rows @@ -237,8 +288,9 @@ def main(): continue why = [c for c in ("plain", "o3") if slower(r[c], b[c])] verdict = ", ".join(c + " SLOWER" for c in why) - if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]): - verdict = (verdict + ", " if verdict else "") + "MORE CHECKS" + if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) != int(b["checks"]): + more = int(r["checks"]) > int(b["checks"]) + verdict = (verdict + ", " if verdict else "") + ("MORE CHECKS" if more else "FEWER CHECKS") if verdict: bad += 1 print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]), @@ -257,6 +309,7 @@ def selftest(): t = Path(tempfile.mkdtemp()) base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]] + base.append({"program": "a:chk", "plain": 1.0, "o3": None, "checks": "3", "peak": "-", "gc": "-"}) write_table(t / "base", base) def case(name, now, want): write_table(t / "now", now) @@ -267,7 +320,8 @@ def selftest(): if not ok: print(r.stdout) return ok - row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} + row = lambda n, p, o, k="-": {"program": n, "plain": p, "o3": o, "checks": k, "peak": "-", "gc": "-"} + two = lambda k: [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4), row("a:chk", 1.0, None, k)] good = [ case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False), case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True), @@ -276,6 +330,10 @@ def selftest(): case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False), case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False), case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False), + case("checks the same", two("3"), False), + case("checks rose 3 -> 4", two("4"), True), + case("checks fell 3 -> 0", two("0"), True), + case("checks '-' now (no --explain-checks)", two("-"), False), ] print("selftest: %s" % ("ok" if all(good) else "FAIL")) return 0 if all(good) else 1 |