diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 24 | ||||
| -rw-r--r-- | ONBOARD.md | 26 | ||||
| -rw-r--r-- | tests/bench.baseline | 12 | ||||
| -rw-r--r-- | tests/bench/bench.sh | 14 | ||||
| -rw-r--r-- | tests/bench/suite.py | 285 | ||||
| -rw-r--r-- | tests/bench/sweep.zc | 23 |
7 files changed, 380 insertions, 5 deletions
@@ -5,3 +5,4 @@ *.o *.obj *.exe +__pycache__/ @@ -12,6 +12,9 @@ # make xref who writes and who reads each global of the compiler # (src/tools/xref.sh): build/xref/report.txt and report.json; # make xref XREF_ARGS=--check compares with tests/xref.baseline +# make bench time the benchmark programs -> build/bench.txt +# make bench-save ... and keep the result as tests/bench.baseline +# make bench-check compare with tests/bench.baseline (or make bench CHECK=1) # make genome build/core/genome.bin from src/, with the fixpoint check # make LLVM=1 on x86_64, build zy with the llvm backend instead of the x86-64 jit # make NOLLVM=1 on other hosts, leave out the llvm backend (in by default when a @@ -208,6 +211,25 @@ parity: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/zy$(EXE) xref: $(BIN)/zclsp$(EXE) bash src/tools/xref.sh $(XREF_ARGS) +# benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per +# program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check +# (or make bench CHECK=1) fails when a column is over 10% and over 0.1 ms slower than the +# baseline. interp rows only with INTERP=1. bench-selftest checks the verdicts, no timing. +bench: $(BIN)/zy$(EXE) +ifeq ($(CHECK),1) + INTERP=$(INTERP) python3 tests/bench/suite.py check +else + INTERP=$(INTERP) python3 tests/bench/suite.py run +endif +bench-save: + BENCH_SAVE=1 $(MAKE) bench + cp $(B)/bench.txt tests/bench.baseline + @echo "bench: wrote tests/bench.baseline" +bench-check: $(BIN)/zy$(EXE) + INTERP=$(INTERP) python3 tests/bench/suite.py check +bench-selftest: + INTERP=$(INTERP) python3 tests/bench/suite.py selftest + # JOBS sets how many sections run at once: make test JOBS=4 test: bash tests/test.sh -j $(or $(JOBS),1) @@ -219,4 +241,4 @@ check: clean: rm -rf $(B) -.PHONY: all lsp genome js lspimage vsix web run parity xref test check clean +.PHONY: bench bench-save bench-selftest bench-check all lsp genome js lspimage vsix web run parity xref test check clean @@ -201,6 +201,32 @@ language server is interpreted and slow; `ZC_TEST_TIMEOUT_SCALE` stretches the drivers' deadlines (the suite sets it). `LLVM_CONFIG` selects an llvm with headers for the llvm section. +### benchmarks + +`make bench` runs `tests/bench/bench.sh` (it needs `bc`), which times the kernels of +`tests/programs/bench.zc` (collatz, sieve, digits, fib), plain and o3, on every backend +it can build here (interpreter, the host's jit, llvm, arm64 under qemu), and adds the +pointer sweep of `tests/bench/sweep.zc` (checked and unbounded) on `build/bin/zy`. the +table goes to `build/bench.txt`, one row per backend and kernel, ms per repetition in a +plain and an o3 column; start-up and compile time cancel out (slopes between one +repetition and many). the checks column waits for `--explain-checks` and the peak heap +and gc mark columns for runtime counters that do not exist yet; they show `-`. a full +run takes about ten minutes. +`make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check` +(or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a +row is more than 10% slower and also more than 0.1 ms slower (so sub-ms cells don't +flake), or a row keeps more checks once both sides have a count. rows missing from a run +don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times +the interpreter today, so this doesn't make the run shorter yet). before timing, the suite +runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and +it refuses when jit o3 collatz isn't under half of jit plain collatz; that is how a baseline of +interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on +doctored tables in a second, and `suite.py check --rows FILE` compares a saved table +without measuring. +when something looks slower the whole run is repeated once and each cell keeps its best +figure. the baseline belongs to the machine it +was saved on: save a new one after changing machines, and expect noise on a busy host. + ## 9. history and open work `docs/history/README.md` is the scrapbook of earlier stages and `TODO.md` the diff --git a/tests/bench.baseline b/tests/bench.baseline new file mode 100644 index 0000000..1ff5561 --- /dev/null +++ b/tests/bench.baseline @@ -0,0 +1,12 @@ +# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available +program plain_ms o3_ms checks peak_heap_words gc_mark_ms +x86-64-jit:collatz 36.240 11.720 - - - +x86-64-jit:sieve 4.910 0.440 - - - +x86-64-jit:digits 16.900 3.380 - - - +x86-64-jit:fib 0.830 0.950 - - - +llvm:collatz 60.050 3.680 - - - +llvm:sieve 14.640 0.220 - - - +llvm:digits 29.240 1.630 - - - +llvm:fib 3.040 0.170 - - - +zy:sweep.checked 3.937 - - - - +zy:sweep.unbounded 2.800 - - - - diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 0a9acef..e532f93 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -4,14 +4,20 @@ cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 make -s build/core/genome.bin || exit 1 -$CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy -vms=("$T/zy"); names=("interp") +# the interpreter core only with INTERP=1 (it is slow, and the rows are rarely wanted) +vms=(); names=() +if [ "$INTERP" = 1 ]; then + $CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy || exit 1 + vms+=("$T/zy"); names+=("interp") +fi # the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions compiled; ZYO3INTERP=1 runs them on the region interpreter) if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } fi -if command -v llvm-config >/dev/null; then - $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } +# the llvm: LLVM_CONFIG when given, else llvm-config, else llvm-config-19 (as in the Makefile) +LC=${LLVM_CONFIG:-$(command -v llvm-config || command -v llvm-config-19)} +if [ -n "$LC" ]; then + $CC -O2 -w -DJTYPE=int -DZYLLVM $($LC --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $($LC --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); } diff --git a/tests/bench/suite.py b/tests/bench/suite.py new file mode 100644 index 0000000..3816cc3 --- /dev/null +++ b/tests/bench/suite.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`. + +usage: tests/bench/suite.py run [--zy PATH] [--out FILE] + tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE] + +it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc +(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the +256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, +timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before +timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and +jit o3 collatz must be under half of jit plain collatz, or it refuses. +selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the +median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). + +one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the +number of bounds checks the compiler keeps (from --explain-checks, which does not exist +yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'. + +check compares with the baseline (tests/bench.baseline): each of the plain and o3 +columns fails on its own when it is more than TOLERANCE (10%) slower and also more than +MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails +when both sides have a checks count and it rose. when anything looks slower the whole +measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, +so one noisy run does not fail the check. +""" +import shutil +import platform +import argparse +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +TRIALS = int(os.environ.get("TRIALS", "5")) +MIN_S = float(os.environ.get("MIN_S", "0.4")) +TOLERANCE = float(os.environ.get("TOLERANCE", "0.10")) +MIN_MS = float(os.environ.get("MIN_MS", "0.1")) +INTERP = os.environ.get("INTERP", "") == "1" +RETRIES = int(os.environ.get("RETRIES", "1")) + +# the rows this script times itself: name, source, kernel letter (bench.sh has the rest) +SWEEPS = [ + ("zy:sweep.checked", "tests/bench/sweep.zc", "c"), + ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"), +] +COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"] +KERNELS = ("collatz", "sieve", "digits", "fib") + + +def bench_sh(): + """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)""" + r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:]) + rows = [] + for line in r.stdout.splitlines(): + w = line.split() + # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end + if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"): + backend = "-".join(w[:-4]) + rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2]))) + if not INTERP: + rows = [r for r in rows if not r[0].startswith("interp:")] + if not rows: + sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:]) + return rows + + +def run_once(zy, src, letter, reps): + data = src + b"`" + letter.encode() + str(reps).encode() + t = time.perf_counter() + r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + dt = time.perf_counter() - t + if r.returncode != 0: + sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300])) + return dt + + +def slope_ms(zy, path, letter): + src = (ROOT / path).read_bytes() + n = 4 + while True: + one = min(run_once(zy, src, letter, 1) for _ in range(2)) + many = run_once(zy, src, letter, n) + if many - one >= MIN_S or n >= 1 << 16: + break + n *= 4 + slopes = [] + for _ in range(TRIALS): + one = run_once(zy, src, letter, 1) + many = run_once(zy, src, letter, n) + slopes.append((many - one) / (n - 1) * 1000) + return statistics.median(slopes) + + +def fmt_row(cells): + return "%-26s %10s %10s %8s %16s %11s" % tuple(cells) + + +def num(x): + return "-" if x is None else "%.3f" % x + + +def write_table(path, rows): + lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available", + fmt_row(COLUMNS)] + for r in rows: + lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]])) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n") + + +def read_table(path): + rows = {} + f = lambda x: None if x == "-" else float(x) + for line in path.read_text().splitlines(): + if not line.strip() or line.startswith("#") or line.split()[0] == "program": + continue + w = line.split() + rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]} + return rows + + +def region_guard(zy): + """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc""" + src = (ROOT / "tests/programs/o3.zc").read_bytes() + env = dict(os.environ, ZYTRACE="1") + r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + err = r.stderr.decode(errors="replace").splitlines() + done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l] + stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l] + print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True) + if not done: + sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); " + "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy) + return len(done), len(stay) + + +def o3_guard(rows): + """jit o3 collatz must be under half of jit plain collatz (needs no interp rows)""" + d = {r["program"]: r for r in rows} + j = d.get("x86-64-jit:collatz") + if j is None: + return + if j["o3"] is None or j["plain"] is None: + sys.exit("bench: REFUSING: x86-64-jit:collatz has no plain or o3 figure") + if j["o3"] * 2 >= j["plain"]: + sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not under half of jit plain %.2f ms; o3 regions are not compiled" % (j["o3"], j["plain"])) + + +def run_all(zy): + region_guard(zy) + rows = [] + for name, plain, o3 in bench_sh(): + print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) + rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"}) + for name, path, letter in SWEEPS: + ms = slope_ms(zy, path, letter) + print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) + rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"}) + o3_guard(rows) + return rows + + +def slower(now, base): + return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS + + +def pct(now, base): + return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100) + + +def expected_backends(): + """the backends bench.sh should report on this machine, found the same way it finds them""" + want = ["zy"] + if platform.system() == "Linux" and platform.machine() == "x86_64": + want.append("x86-64-jit") + if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"): + want.append("llvm") + if INTERP: + want.append("interp") + return want + + +def require_rows(rows): + """bench-save refuses a table that is missing a backend this machine has""" + have = {r["program"].split(":")[0] for r in rows} + missing = [b for b in expected_backends() if b not in have] + if missing: + sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing)) + + +def main(): + ap = argparse.ArgumentParser(description="tinyvm benchmark suite") + ap.add_argument("mode", choices=["run", "check", "selftest"]) + ap.add_argument("--rows", help="check: compare this saved table instead of measuring") + ap.add_argument("--zy", default=str(ROOT / "build/bin/zy")) + ap.add_argument("--out", default=str(ROOT / "build/bench.txt")) + ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline")) + a = ap.parse_args() + if a.mode == "selftest": + return selftest() + rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy) + if a.mode == "run": + if os.environ.get("BENCH_SAVE") == "1": + require_rows(rows) + write_table(Path(a.out), rows) + print(Path(a.out).read_text(), end="") + return 0 + base_path = Path(a.baseline) + if not base_path.exists(): + sys.exit("bench: no baseline at %s (run make bench-save)" % base_path) + base = read_table(base_path) + tries = 0 + while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): + tries += 1 + print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True) + again = {r["program"]: r for r in run_all(a.zy)} + for r in rows: + g = again.get(r["program"]) + if g: + for c in ("plain", "o3"): + if r[c] is not None and g[c] is not None: + r[c] = min(r[c], g[c]) + bad = 0 + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict")) + names = set() + for r in rows: + names.add(r["program"]) + b = base.get(r["program"]) + if not b: + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"])) + continue + why = [c for c in ("plain", "o3") if slower(r[c], b[c])] + verdict = ", ".join(c + " SLOWER" for c in why) + if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]): + verdict = (verdict + ", " if verdict else "") + "MORE CHECKS" + if verdict: + bad += 1 + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]), + num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok")) + for name in base: + if name not in names: + print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-")) + write_table(Path(a.out), rows) + print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS)) + return 1 if bad else 0 + + +def selftest(): + """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts""" + import tempfile + t = Path(tempfile.mkdtemp()) + base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in + [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]] + write_table(t / "base", base) + def case(name, now, want): + write_table(t / "now", now) + r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + ok = (r.returncode != 0) == want + print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG")) + if not ok: + print(r.stdout) + return ok + row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} + good = [ + case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True), + case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True), + case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True), + case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False), + case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False), + ] + print("selftest: %s" % ("ok" if all(good) else "FAIL")) + return 0 if all(good) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/bench/sweep.zc b/tests/bench/sweep.zc new file mode 100644 index 0000000..9bc7070 --- /dev/null +++ b/tests/bench/sweep.zc @@ -0,0 +1,23 @@ +namespace; +include rt; + +// a 256-word sweep through a heap pointer, with bounds checks (the default) and without +// (unbounded): each pass adds every word of the block 1000 times. the index is variable +// and the block is reached through a pointer, so the checked version keeps its checks. +// input: a letter (c checked, u unbounded, z none) and how many times to run it. +int sweep(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; } +unbounded int sweepu(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; } +fn main() { + int* p = rt.new(256); + for (int i = 0; i < 256; i++) p[i] = i; + let which = rt.get_c(), reps = 0, ch = rt.get_c(), r = 0; + while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = rt.get_c(); } + while (reps > 0) { + if (which == 'c') r = sweep(p, 256); + if (which == 'u') r = sweepu(p, 256); + reps--; + } + rt.put_n(r); rt.put_c(10); + rt.free(p); + return 0; +} |