From 58b86428cee46268002d23fd985a1813f641115c Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 16:53:22 +0330 Subject: wip: bench suite --- Makefile | 20 ++++++- tests/bench/suite.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++ tests/bench/sweep.zc | 23 +++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tests/bench/suite.py create mode 100644 tests/bench/sweep.zc diff --git a/Makefile b/Makefile index b3b7d56..6deb3a9 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ # make xref who writes and who reads each global of the compiler # (src/tools/xref.sh): build/xref/report.txt and report.json; # make xref XREF_ARGS=--check compares with tests/xref.baseline +# make bench time the benchmark programs -> build/bench.txt +# make bench-save ... and keep the result as tests/bench.baseline +# make bench-check compare with tests/bench.baseline (or make bench CHECK=1) # make genome build/core/genome.bin from src/, with the fixpoint check # make LLVM=1 on x86_64, build zy with the llvm backend instead of the x86-64 jit # make NOLLVM=1 on other hosts, leave out the llvm backend (in by default when a @@ -208,6 +211,21 @@ parity: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/zy$(EXE) xref: $(BIN)/zclsp$(EXE) bash src/tools/xref.sh $(XREF_ARGS) +# benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per +# program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check +# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline. +bench: $(BIN)/zy$(EXE) +ifeq ($(CHECK),1) + python3 tests/bench/suite.py check +else + python3 tests/bench/suite.py run +endif +bench-save: bench + cp $(B)/bench.txt tests/bench.baseline + @echo "bench: wrote tests/bench.baseline" +bench-check: $(BIN)/zy$(EXE) + python3 tests/bench/suite.py check + # JOBS sets how many sections run at once: make test JOBS=4 test: bash tests/test.sh -j $(or $(JOBS),1) @@ -219,4 +237,4 @@ check: clean: rm -rf $(B) -.PHONY: all lsp genome js lspimage vsix web run parity xref test check clean +.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean diff --git a/tests/bench/suite.py b/tests/bench/suite.py new file mode 100644 index 0000000..a5a61ed --- /dev/null +++ b/tests/bench/suite.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`. + +usage: tests/bench/suite.py run [--zy PATH] [--out FILE] + tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE] + +each program is a kernel of tests/programs/bench.zc (the kernels bench.sh times, plain +and o3) or of tests/bench/sweep.zc (a 256-word pointer sweep, checked and unbounded), +run on one core (build/bin/zy by default, the host's jit where there is one). a kernel +is selected by a letter and a repetition count on stdin, after the source. the time per +repetition is the slope between 1 and N repetitions, so starting the vm and compiling +the program cancel out; N grows until the N run takes at least MIN_S seconds. each +cell is the median of TRIALS slopes. + +columns: ms per repetition; checks, the number of bounds checks the compiler keeps +(from --explain-checks, which does not exist yet: '-'); peak heap words and gc mark +time, which the runtime does not expose: '-'. + +check compares with the baseline (tests/bench.baseline): it fails when a program is +more than TOLERANCE (10%) slower, or when both sides have a checks count and it rose. +a program that looks slower is measured again, up to RETRIES more times, and its best +median is kept, so one noisy trial does not fail the check. +""" +import argparse +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +TRIALS = int(os.environ.get("TRIALS", "5")) +MIN_S = float(os.environ.get("MIN_S", "0.4")) +TOLERANCE = float(os.environ.get("TOLERANCE", "0.10")) +RETRIES = int(os.environ.get("RETRIES", "2")) + +# name, source, kernel letter +PROGRAMS = [ + ("collatz", "tests/programs/bench.zc", "a"), + ("collatz.o3", "tests/programs/bench.zc", "b"), + ("sieve", "tests/programs/bench.zc", "c"), + ("sieve.o3", "tests/programs/bench.zc", "d"), + ("digits", "tests/programs/bench.zc", "e"), + ("digits.o3", "tests/programs/bench.zc", "f"), + ("fib", "tests/programs/bench.zc", "g"), + ("fib.o3", "tests/programs/bench.zc", "h"), + ("sweep.checked", "tests/bench/sweep.zc", "c"), + ("sweep.unbounded", "tests/bench/sweep.zc", "u"), +] +COLUMNS = ["program", "ms", "checks", "peak_heap_words", "gc_mark_ms"] + + +def run_once(zy, src, letter, reps): + data = src + b"`" + letter.encode() + str(reps).encode() + t = time.perf_counter() + r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + dt = time.perf_counter() - t + if r.returncode != 0: + sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300])) + return dt + + +def slope_ms(zy, path, letter): + src = (ROOT / path).read_bytes() + n = 4 + while True: + one = min(run_once(zy, src, letter, 1) for _ in range(2)) + many = run_once(zy, src, letter, n) + if many - one >= MIN_S or n >= 1 << 16: + break + n *= 4 + slopes = [] + for _ in range(TRIALS): + one = run_once(zy, src, letter, 1) + many = run_once(zy, src, letter, n) + slopes.append((many - one) / (n - 1) * 1000) + return statistics.median(slopes) + + +def measure(zy, name): + for p, path, letter in PROGRAMS: + if p == name: + return slope_ms(zy, path, letter) + raise KeyError(name) + + +def fmt_row(cells): + return "%-16s %10s %8s %16s %11s" % tuple(cells) + + +def write_table(path, rows): + lines = ["# tinyvm bench: ms per repetition (median of %d slopes); '-' where a figure is not available" % TRIALS, + fmt_row(COLUMNS)] + for r in rows: + lines.append(fmt_row([r["program"], "%.3f" % r["ms"], r["checks"], r["peak"], r["gc"]])) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n") + + +def read_table(path): + rows = {} + for line in path.read_text().splitlines(): + if not line.strip() or line.startswith("#") or line.split()[0] == "program": + continue + w = line.split() + rows[w[0]] = {"program": w[0], "ms": float(w[1]), "checks": w[2], "peak": w[3], "gc": w[4]} + return rows + + +def run_all(zy): + rows = [] + for name, _, _ in PROGRAMS: + ms = measure(zy, name) + print("bench: %-16s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) + rows.append({"program": name, "ms": ms, "checks": "-", "peak": "-", "gc": "-"}) + return rows + + +def main(): + ap = argparse.ArgumentParser(description="tinyvm benchmark suite") + ap.add_argument("mode", choices=["run", "check"]) + ap.add_argument("--zy", default=str(ROOT / "build/bin/zy")) + ap.add_argument("--out", default=str(ROOT / "build/bench.txt")) + ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline")) + a = ap.parse_args() + rows = run_all(a.zy) + if a.mode == "run": + write_table(Path(a.out), rows) + print(Path(a.out).read_text(), end="") + return 0 + base_path = Path(a.baseline) + if not base_path.exists(): + sys.exit("bench: no baseline at %s (run make bench-save)" % base_path) + base = read_table(base_path) + bad = 0 + print("%-16s %10s %10s %8s %9s %9s %s" % ("program", "base ms", "now ms", "change", "base chk", "now chk", "verdict")) + for r in rows: + b = base.get(r["program"]) + if not b: + print("%-16s %10s %10.3f %8s %9s %9s new" % (r["program"], "-", r["ms"], "-", "-", r["checks"])) + continue + tries = 0 + while r["ms"] > b["ms"] * (1 + TOLERANCE) and tries < RETRIES: + r["ms"] = min(r["ms"], measure(a.zy, r["program"])) + tries += 1 + change = (r["ms"] - b["ms"]) / b["ms"] * 100 if b["ms"] > 0 else 0.0 + verdict = "ok" + if r["ms"] > b["ms"] * (1 + TOLERANCE): + verdict = "SLOWER" + if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]): + verdict = "MORE CHECKS" if verdict == "ok" else verdict + ", MORE CHECKS" + if verdict != "ok": + bad += 1 + print("%-16s %10.3f %10.3f %+7.1f%% %9s %9s %s" % (r["program"], b["ms"], r["ms"], change, b["checks"], r["checks"], verdict)) + for name in base: + if name not in {r["program"] for r in rows}: + print("%-16s %10.3f %10s %8s %9s %9s gone" % (name, base[name]["ms"], "-", "-", base[name]["checks"], "-")) + write_table(Path(a.out), rows) + print("bench-check: %s (%d of %d programs over the limit; tolerance %d%%)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100))) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/bench/sweep.zc b/tests/bench/sweep.zc new file mode 100644 index 0000000..9bc7070 --- /dev/null +++ b/tests/bench/sweep.zc @@ -0,0 +1,23 @@ +namespace; +include rt; + +// a 256-word sweep through a heap pointer, with bounds checks (the default) and without +// (unbounded): each pass adds every word of the block 1000 times. the index is variable +// and the block is reached through a pointer, so the checked version keeps its checks. +// input: a letter (c checked, u unbounded, z none) and how many times to run it. +int sweep(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; } +unbounded int sweepu(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; } +fn main() { + int* p = rt.new(256); + for (int i = 0; i < 256; i++) p[i] = i; + let which = rt.get_c(), reps = 0, ch = rt.get_c(), r = 0; + while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = rt.get_c(); } + while (reps > 0) { + if (which == 'c') r = sweep(p, 256); + if (which == 'u') r = sweepu(p, 256); + reps--; + } + rt.put_n(r); rt.put_c(10); + rt.free(p); + return 0; +} -- cgit v1.2.3 From 89eb56be4546f0588c2736883d3c6a7438771b38 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:12:04 +0330 Subject: wip: bench wraps bench.sh --- ONBOARD.md | 15 ++++++ tests/bench.baseline | 12 +++++ tests/bench/suite.py | 148 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 114 insertions(+), 61 deletions(-) create mode 100644 tests/bench.baseline diff --git a/ONBOARD.md b/ONBOARD.md index 35d5e35..7a7d46d 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -201,6 +201,21 @@ language server is interpreted and slow; `ZC_TEST_TIMEOUT_SCALE` stretches the drivers' deadlines (the suite sets it). `LLVM_CONFIG` selects an llvm with headers for the llvm section. +### benchmarks + +`make bench` times the benchmark programs on `build/bin/zy` (the host's jit where +there is one) and writes `build/bench.txt`: the kernels of `tests/programs/bench.zc` +(collatz, sieve, digits, fib, each plain and o3) and the pointer sweep of +`tests/bench/sweep.zc` (checked and unbounded). each figure is ms per repetition, the +median of five slopes between one repetition and many, so start-up and compile time +cancel out. the checks column waits for `--explain-checks` and the peak heap and gc +mark columns for runtime counters that do not exist yet; they show `-`. +`make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check` +(or `make bench CHECK=1`) measures again and fails when a program is more than 10% +slower, or keeps more checks once both sides have a count. a program that looks slower +is measured up to twice more before it counts. the baseline belongs to the machine it +was saved on: save a new one after changing machines, and expect noise on a busy host. + ## 9. history and open work `docs/history/README.md` is the scrapbook of earlier stages and `TODO.md` the diff --git a/tests/bench.baseline b/tests/bench.baseline new file mode 100644 index 0000000..b3ac5d3 --- /dev/null +++ b/tests/bench.baseline @@ -0,0 +1,12 @@ +# tinyvm bench: ms per repetition (median of 5 slopes); '-' where a figure is not available +program ms checks peak_heap_words gc_mark_ms +collatz 32.547 - - - +collatz.o3 255.490 - - - +sieve 4.411 - - - +sieve.o3 8.302 - - - +digits 15.497 - - - +digits.o3 81.287 - - - +fib 0.822 - - - +fib.o3 3.267 - - - +sweep.checked 3.371 - - - +sweep.unbounded 2.382 - - - diff --git a/tests/bench/suite.py b/tests/bench/suite.py index a5a61ed..bb81e46 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -4,22 +4,21 @@ usage: tests/bench/suite.py run [--zy PATH] [--out FILE] tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE] -each program is a kernel of tests/programs/bench.zc (the kernels bench.sh times, plain -and o3) or of tests/bench/sweep.zc (a 256-word pointer sweep, checked and unbounded), -run on one core (build/bin/zy by default, the host's jit where there is one). a kernel -is selected by a letter and a repetition count on stdin, after the source. the time per -repetition is the slope between 1 and N repetitions, so starting the vm and compiling -the program cancel out; N grows until the N run takes at least MIN_S seconds. each -cell is the median of TRIALS slopes. - -columns: ms per repetition; checks, the number of bounds checks the compiler keeps -(from --explain-checks, which does not exist yet: '-'); peak heap words and gc mark -time, which the runtime does not expose: '-'. - -check compares with the baseline (tests/bench.baseline): it fails when a program is -more than TOLERANCE (10%) slower, or when both sides have a checks count and it rose. -a program that looks slower is measured again, up to RETRIES more times, and its best -median is kept, so one noisy trial does not fail the check. +it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc +(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the +256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, +timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the +median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). + +one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the +number of bounds checks the compiler keeps (from --explain-checks, which does not exist +yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'. + +check compares with the baseline (tests/bench.baseline): each of the plain and o3 +columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails +when both sides have a checks count and it rose. when anything looks slower the whole +measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, +so one noisy run does not fail the check. """ import argparse import os @@ -33,22 +32,32 @@ ROOT = Path(__file__).resolve().parent.parent.parent TRIALS = int(os.environ.get("TRIALS", "5")) MIN_S = float(os.environ.get("MIN_S", "0.4")) TOLERANCE = float(os.environ.get("TOLERANCE", "0.10")) -RETRIES = int(os.environ.get("RETRIES", "2")) - -# name, source, kernel letter -PROGRAMS = [ - ("collatz", "tests/programs/bench.zc", "a"), - ("collatz.o3", "tests/programs/bench.zc", "b"), - ("sieve", "tests/programs/bench.zc", "c"), - ("sieve.o3", "tests/programs/bench.zc", "d"), - ("digits", "tests/programs/bench.zc", "e"), - ("digits.o3", "tests/programs/bench.zc", "f"), - ("fib", "tests/programs/bench.zc", "g"), - ("fib.o3", "tests/programs/bench.zc", "h"), - ("sweep.checked", "tests/bench/sweep.zc", "c"), - ("sweep.unbounded", "tests/bench/sweep.zc", "u"), +RETRIES = int(os.environ.get("RETRIES", "1")) + +# the rows this script times itself: name, source, kernel letter (bench.sh has the rest) +SWEEPS = [ + ("zy:sweep.checked", "tests/bench/sweep.zc", "c"), + ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"), ] -COLUMNS = ["program", "ms", "checks", "peak_heap_words", "gc_mark_ms"] +COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"] +KERNELS = ("collatz", "sieve", "digits", "fib") + + +def bench_sh(): + """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)""" + r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:]) + rows = [] + for line in r.stdout.splitlines(): + w = line.split() + # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end + if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"): + backend = "-".join(w[:-4]) + rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2]))) + if not rows: + sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:]) + return rows def run_once(zy, src, letter, reps): @@ -78,45 +87,54 @@ def slope_ms(zy, path, letter): return statistics.median(slopes) -def measure(zy, name): - for p, path, letter in PROGRAMS: - if p == name: - return slope_ms(zy, path, letter) - raise KeyError(name) +def fmt_row(cells): + return "%-26s %10s %10s %8s %16s %11s" % tuple(cells) -def fmt_row(cells): - return "%-16s %10s %8s %16s %11s" % tuple(cells) +def num(x): + return "-" if x is None else "%.3f" % x def write_table(path, rows): - lines = ["# tinyvm bench: ms per repetition (median of %d slopes); '-' where a figure is not available" % TRIALS, + lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available", fmt_row(COLUMNS)] for r in rows: - lines.append(fmt_row([r["program"], "%.3f" % r["ms"], r["checks"], r["peak"], r["gc"]])) + lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]])) path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(lines) + "\n") def read_table(path): rows = {} + f = lambda x: None if x == "-" else float(x) for line in path.read_text().splitlines(): if not line.strip() or line.startswith("#") or line.split()[0] == "program": continue w = line.split() - rows[w[0]] = {"program": w[0], "ms": float(w[1]), "checks": w[2], "peak": w[3], "gc": w[4]} + rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]} return rows def run_all(zy): rows = [] - for name, _, _ in PROGRAMS: - ms = measure(zy, name) - print("bench: %-16s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) - rows.append({"program": name, "ms": ms, "checks": "-", "peak": "-", "gc": "-"}) + for name, plain, o3 in bench_sh(): + print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) + rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"}) + for name, path, letter in SWEEPS: + ms = slope_ms(zy, path, letter) + print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) + rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"}) return rows +def slower(now, base): + return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) + + +def pct(now, base): + return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100) + + def main(): ap = argparse.ArgumentParser(description="tinyvm benchmark suite") ap.add_argument("mode", choices=["run", "check"]) @@ -133,31 +151,39 @@ def main(): if not base_path.exists(): sys.exit("bench: no baseline at %s (run make bench-save)" % base_path) base = read_table(base_path) + tries = 0 + while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): + tries += 1 + print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True) + again = {r["program"]: r for r in run_all(a.zy)} + for r in rows: + g = again.get(r["program"]) + if g: + for c in ("plain", "o3"): + if r[c] is not None and g[c] is not None: + r[c] = min(r[c], g[c]) bad = 0 - print("%-16s %10s %10s %8s %9s %9s %s" % ("program", "base ms", "now ms", "change", "base chk", "now chk", "verdict")) + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict")) + names = set() for r in rows: + names.add(r["program"]) b = base.get(r["program"]) if not b: - print("%-16s %10s %10.3f %8s %9s %9s new" % (r["program"], "-", r["ms"], "-", "-", r["checks"])) + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"])) continue - tries = 0 - while r["ms"] > b["ms"] * (1 + TOLERANCE) and tries < RETRIES: - r["ms"] = min(r["ms"], measure(a.zy, r["program"])) - tries += 1 - change = (r["ms"] - b["ms"]) / b["ms"] * 100 if b["ms"] > 0 else 0.0 - verdict = "ok" - if r["ms"] > b["ms"] * (1 + TOLERANCE): - verdict = "SLOWER" + why = [c for c in ("plain", "o3") if slower(r[c], b[c])] + verdict = ", ".join(c + " SLOWER" for c in why) if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]): - verdict = "MORE CHECKS" if verdict == "ok" else verdict + ", MORE CHECKS" - if verdict != "ok": + verdict = (verdict + ", " if verdict else "") + "MORE CHECKS" + if verdict: bad += 1 - print("%-16s %10.3f %10.3f %+7.1f%% %9s %9s %s" % (r["program"], b["ms"], r["ms"], change, b["checks"], r["checks"], verdict)) + print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]), + num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok")) for name in base: - if name not in {r["program"] for r in rows}: - print("%-16s %10.3f %10s %8s %9s %9s gone" % (name, base[name]["ms"], "-", "-", base[name]["checks"], "-")) + if name not in names: + print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-")) write_table(Path(a.out), rows) - print("bench-check: %s (%d of %d programs over the limit; tolerance %d%%)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100))) + print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100))) return 1 if bad else 0 -- cgit v1.2.3 From bd5aa93e5c989d01e7dae6db6bc58f57fcf6592f Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:51:21 +0330 Subject: make bench, bench-save, bench-check: wrap tests/bench/bench.sh plus a pointer sweep; baseline after jit step 2 --- ONBOARD.md | 23 +++++++++++++---------- tests/bench.baseline | 28 ++++++++++++++++------------ 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/ONBOARD.md b/ONBOARD.md index 7a7d46d..f60063a 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -203,17 +203,20 @@ headers for the llvm section. ### benchmarks -`make bench` times the benchmark programs on `build/bin/zy` (the host's jit where -there is one) and writes `build/bench.txt`: the kernels of `tests/programs/bench.zc` -(collatz, sieve, digits, fib, each plain and o3) and the pointer sweep of -`tests/bench/sweep.zc` (checked and unbounded). each figure is ms per repetition, the -median of five slopes between one repetition and many, so start-up and compile time -cancel out. the checks column waits for `--explain-checks` and the peak heap and gc -mark columns for runtime counters that do not exist yet; they show `-`. +`make bench` runs `tests/bench/bench.sh` (it needs `bc`), which times the kernels of +`tests/programs/bench.zc` (collatz, sieve, digits, fib), plain and o3, on every backend +it can build here (interpreter, the host's jit, llvm, arm64 under qemu), and adds the +pointer sweep of `tests/bench/sweep.zc` (checked and unbounded) on `build/bin/zy`. the +table goes to `build/bench.txt`, one row per backend and kernel, ms per repetition in a +plain and an o3 column; start-up and compile time cancel out (slopes between one +repetition and many). the checks column waits for `--explain-checks` and the peak heap +and gc mark columns for runtime counters that do not exist yet; they show `-`. a full +run takes about ten minutes. `make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check` -(or `make bench CHECK=1`) measures again and fails when a program is more than 10% -slower, or keeps more checks once both sides have a count. a program that looks slower -is measured up to twice more before it counts. the baseline belongs to the machine it +(or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a +row is more than 10% slower, or a row keeps more checks once both sides have a count. +when something looks slower the whole run is repeated once and each cell keeps its best +figure. the baseline belongs to the machine it was saved on: save a new one after changing machines, and expect noise on a busy host. ## 9. history and open work diff --git a/tests/bench.baseline b/tests/bench.baseline index b3ac5d3..18fba1c 100644 --- a/tests/bench.baseline +++ b/tests/bench.baseline @@ -1,12 +1,16 @@ -# tinyvm bench: ms per repetition (median of 5 slopes); '-' where a figure is not available -program ms checks peak_heap_words gc_mark_ms -collatz 32.547 - - - -collatz.o3 255.490 - - - -sieve 4.411 - - - -sieve.o3 8.302 - - - -digits 15.497 - - - -digits.o3 81.287 - - - -fib 0.822 - - - -fib.o3 3.267 - - - -sweep.checked 3.371 - - - -sweep.unbounded 2.382 - - - +# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available +program plain_ms o3_ms checks peak_heap_words gc_mark_ms +interp:collatz 594.240 259.680 - - - +interp:sieve 82.310 8.400 - - - +interp:digits 328.180 84.460 - - - +interp:fib 10.200 2.820 - - - +x86-64-jit:collatz 31.410 10.750 - - - +x86-64-jit:sieve 4.320 0.410 - - - +x86-64-jit:digits 14.750 3.070 - - - +x86-64-jit:fib 0.780 0.860 - - - +llvm:collatz 54.080 3.420 - - - +llvm:sieve 13.560 0.190 - - - +llvm:digits 26.590 1.600 - - - +llvm:fib 2.970 0.150 - - - +zy:sweep.checked 3.535 - - - - +zy:sweep.unbounded 2.625 - - - - -- cgit v1.2.3 From b3273f04b876228d7b84a9d5a0fe053eb95cdb95 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:54:07 +0330 Subject: bench: no-region guard, selftest and --rows, interp rows behind INTERP=1, fail only over 10% and 0.1 ms --- Makefile | 13 +++++---- ONBOARD.md | 10 ++++++- tests/bench/suite.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 90 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 6deb3a9..6569bd5 100644 --- a/Makefile +++ b/Makefile @@ -213,18 +213,21 @@ xref: $(BIN)/zclsp$(EXE) # benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per # program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check -# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline. +# (or make bench CHECK=1) fails when a column is over 10% and over 0.1 ms slower than the +# baseline. interp rows only with INTERP=1. bench-selftest checks the verdicts, no timing. bench: $(BIN)/zy$(EXE) ifeq ($(CHECK),1) - python3 tests/bench/suite.py check + INTERP=$(INTERP) python3 tests/bench/suite.py check else - python3 tests/bench/suite.py run + INTERP=$(INTERP) python3 tests/bench/suite.py run endif bench-save: bench cp $(B)/bench.txt tests/bench.baseline @echo "bench: wrote tests/bench.baseline" bench-check: $(BIN)/zy$(EXE) - python3 tests/bench/suite.py check + INTERP=$(INTERP) python3 tests/bench/suite.py check +bench-selftest: + INTERP=$(INTERP) python3 tests/bench/suite.py selftest # JOBS sets how many sections run at once: make test JOBS=4 test: @@ -237,4 +240,4 @@ check: clean: rm -rf $(B) -.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean +.PHONY: bench bench-save bench-selftest bench-check all lsp genome js lspimage vsix web run parity xref test check clean diff --git a/ONBOARD.md b/ONBOARD.md index f60063a..2ccb987 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -214,7 +214,15 @@ and gc mark columns for runtime counters that do not exist yet; they show `-`. a run takes about ten minutes. `make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check` (or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a -row is more than 10% slower, or a row keeps more checks once both sides have a count. +row is more than 10% slower and also more than 0.1 ms slower (so sub-ms cells don't +flake), or a row keeps more checks once both sides have a count. rows missing from a run +don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times +the interpreter today, so this doesn't make the run shorter yet). before timing, the suite +runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and +it refuses when jit o3 collatz isn't under a fifth of interp o3; that is how a baseline of +interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on +doctored tables in a second, and `suite.py check --rows FILE` compares a saved table +without measuring. when something looks slower the whole run is repeated once and each cell keeps its best figure. the baseline belongs to the machine it was saved on: save a new one after changing machines, and expect noise on a busy host. diff --git a/tests/bench/suite.py b/tests/bench/suite.py index bb81e46..f7d5c75 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -7,7 +7,10 @@ usage: tests/bench/suite.py run [--zy PATH] [--out FILE] it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc (collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the 256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, -timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the +timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before +timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and +jit o3 collatz must be under a fifth of interp o3 (when both rows exist), or it refuses. +selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the @@ -15,7 +18,8 @@ number of bounds checks the compiler keeps (from --explain-checks, which does no yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'. check compares with the baseline (tests/bench.baseline): each of the plain and o3 -columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails +columns fails on its own when it is more than TOLERANCE (10%) slower and also more than +MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails when both sides have a checks count and it rose. when anything looks slower the whole measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, so one noisy run does not fail the check. @@ -32,6 +36,8 @@ ROOT = Path(__file__).resolve().parent.parent.parent TRIALS = int(os.environ.get("TRIALS", "5")) MIN_S = float(os.environ.get("MIN_S", "0.4")) TOLERANCE = float(os.environ.get("TOLERANCE", "0.10")) +MIN_MS = float(os.environ.get("MIN_MS", "0.1")) +INTERP = os.environ.get("INTERP", "") == "1" RETRIES = int(os.environ.get("RETRIES", "1")) # the rows this script times itself: name, source, kernel letter (bench.sh has the rest) @@ -55,6 +61,8 @@ def bench_sh(): if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"): backend = "-".join(w[:-4]) rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2]))) + if not INTERP: + rows = [r for r in rows if not r[0].startswith("interp:")] if not rows: sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:]) return rows @@ -115,7 +123,31 @@ def read_table(path): return rows +def region_guard(zy): + """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc""" + src = (ROOT / "tests/programs/o3.zc").read_bytes() + env = dict(os.environ, ZYTRACE="1") + r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + err = r.stderr.decode(errors="replace").splitlines() + done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l] + stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l] + print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True) + if not done: + sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); " + "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy) + return len(done), len(stay) + + +def o3_guard(rows): + """jit o3 collatz must be far below interp o3 collatz when both rows are there""" + d = {r["program"]: r for r in rows} + j, i = d.get("x86-64-jit:collatz"), d.get("interp:collatz") + if j and i and j["o3"] is not None and i["o3"] is not None and j["o3"] * 5 > i["o3"]: + sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not far below interp o3 %.2f ms; o3 regions are not compiled" % (j["o3"], i["o3"])) + + def run_all(zy): + region_guard(zy) rows = [] for name, plain, o3 in bench_sh(): print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) @@ -124,11 +156,12 @@ def run_all(zy): ms = slope_ms(zy, path, letter) print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"}) + o3_guard(rows) return rows def slower(now, base): - return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) + return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS def pct(now, base): @@ -137,12 +170,15 @@ def pct(now, base): def main(): ap = argparse.ArgumentParser(description="tinyvm benchmark suite") - ap.add_argument("mode", choices=["run", "check"]) + ap.add_argument("mode", choices=["run", "check", "selftest"]) + ap.add_argument("--rows", help="check: compare this saved table instead of measuring") ap.add_argument("--zy", default=str(ROOT / "build/bin/zy")) ap.add_argument("--out", default=str(ROOT / "build/bench.txt")) ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline")) a = ap.parse_args() - rows = run_all(a.zy) + if a.mode == "selftest": + return selftest() + rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy) if a.mode == "run": write_table(Path(a.out), rows) print(Path(a.out).read_text(), end="") @@ -152,7 +188,7 @@ def main(): sys.exit("bench: no baseline at %s (run make bench-save)" % base_path) base = read_table(base_path) tries = 0 - while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): + while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): tries += 1 print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True) again = {r["program"]: r for r in run_all(a.zy)} @@ -183,9 +219,39 @@ def main(): if name not in names: print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-")) write_table(Path(a.out), rows) - print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100))) + print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS)) return 1 if bad else 0 +def selftest(): + """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts""" + import tempfile + t = Path(tempfile.mkdtemp()) + base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in + [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]] + write_table(t / "base", base) + def case(name, now, want): + write_table(t / "now", now) + r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + ok = (r.returncode != 0) == want + print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG")) + if not ok: + print(r.stdout) + return ok + row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} + good = [ + case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True), + case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True), + case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True), + case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False), + case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False), + ] + print("selftest: %s" % ("ok" if all(good) else "FAIL")) + return 0 if all(good) else 1 + + if __name__ == "__main__": sys.exit(main()) -- cgit v1.2.3 From 5cfd43a291b0a3b292f0e26f8732578a7c57ed46 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:55:09 +0330 Subject: bench.sh: interp core built and timed only with INTERP=1 --- tests/bench/bench.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 0a9acef..0437b0b 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -4,8 +4,12 @@ cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 make -s build/core/genome.bin || exit 1 -$CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy -vms=("$T/zy"); names=("interp") +# the interpreter core only with INTERP=1 (it is slow, and the rows are rarely wanted) +vms=(); names=() +if [ "$INTERP" = 1 ]; then + $CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy || exit 1 + vms+=("$T/zy"); names+=("interp") +fi # the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions compiled; ZYO3INTERP=1 runs them on the region interpreter) if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } -- cgit v1.2.3 From f134199a5b6fc458216bf2187bb0a6b963f0ba9f Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:56:24 +0330 Subject: bench: o3 guard is jit o3 collatz under half of jit plain (no interp rows needed) --- ONBOARD.md | 2 +- tests/bench/suite.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ONBOARD.md b/ONBOARD.md index 2ccb987..0f1c95c 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -219,7 +219,7 @@ flake), or a row keeps more checks once both sides have a count. rows missing fr don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times the interpreter today, so this doesn't make the run shorter yet). before timing, the suite runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and -it refuses when jit o3 collatz isn't under a fifth of interp o3; that is how a baseline of +it refuses when jit o3 collatz isn't under half of jit plain collatz; that is how a baseline of interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on doctored tables in a second, and `suite.py check --rows FILE` compares a saved table without measuring. diff --git a/tests/bench/suite.py b/tests/bench/suite.py index f7d5c75..656e877 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -9,7 +9,7 @@ it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.z 256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and -jit o3 collatz must be under a fifth of interp o3 (when both rows exist), or it refuses. +jit o3 collatz must be under half of jit plain collatz, or it refuses. selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). @@ -139,11 +139,15 @@ def region_guard(zy): def o3_guard(rows): - """jit o3 collatz must be far below interp o3 collatz when both rows are there""" + """jit o3 collatz must be under half of jit plain collatz (needs no interp rows)""" d = {r["program"]: r for r in rows} - j, i = d.get("x86-64-jit:collatz"), d.get("interp:collatz") - if j and i and j["o3"] is not None and i["o3"] is not None and j["o3"] * 5 > i["o3"]: - sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not far below interp o3 %.2f ms; o3 regions are not compiled" % (j["o3"], i["o3"])) + j = d.get("x86-64-jit:collatz") + if j is None: + return + if j["o3"] is None or j["plain"] is None: + sys.exit("bench: REFUSING: x86-64-jit:collatz has no plain or o3 figure") + if j["o3"] * 2 >= j["plain"]: + sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not under half of jit plain %.2f ms; o3 regions are not compiled" % (j["o3"], j["plain"])) def run_all(zy): -- cgit v1.2.3 From 44d9724e3cfb6d25c19ed0a45cb33b95c38003e5 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 18:21:07 +0330 Subject: bench: find llvm-config-19 like the Makefile; bench-save refuses when an expected backend has no rows --- Makefile | 3 ++- tests/bench/__pycache__/suite.cpython-313.pyc | Bin 0 -> 23177 bytes tests/bench/bench.sh | 6 ++++-- tests/bench/suite.py | 24 ++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/bench/__pycache__/suite.cpython-313.pyc diff --git a/Makefile b/Makefile index 6569bd5..94d9b49 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,8 @@ ifeq ($(CHECK),1) else INTERP=$(INTERP) python3 tests/bench/suite.py run endif -bench-save: bench +bench-save: + BENCH_SAVE=1 $(MAKE) bench cp $(B)/bench.txt tests/bench.baseline @echo "bench: wrote tests/bench.baseline" bench-check: $(BIN)/zy$(EXE) diff --git a/tests/bench/__pycache__/suite.cpython-313.pyc b/tests/bench/__pycache__/suite.cpython-313.pyc new file mode 100644 index 0000000..21783a7 Binary files /dev/null and b/tests/bench/__pycache__/suite.cpython-313.pyc differ diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 0437b0b..e532f93 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -14,8 +14,10 @@ fi if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } fi -if command -v llvm-config >/dev/null; then - $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } +# the llvm: LLVM_CONFIG when given, else llvm-config, else llvm-config-19 (as in the Makefile) +LC=${LLVM_CONFIG:-$(command -v llvm-config || command -v llvm-config-19)} +if [ -n "$LC" ]; then + $CC -O2 -w -DJTYPE=int -DZYLLVM $($LC --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $($LC --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); } diff --git a/tests/bench/suite.py b/tests/bench/suite.py index 656e877..3816cc3 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -24,6 +24,8 @@ when both sides have a checks count and it rose. when anything looks slower the measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, so one noisy run does not fail the check. """ +import shutil +import platform import argparse import os import statistics @@ -172,6 +174,26 @@ def pct(now, base): return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100) +def expected_backends(): + """the backends bench.sh should report on this machine, found the same way it finds them""" + want = ["zy"] + if platform.system() == "Linux" and platform.machine() == "x86_64": + want.append("x86-64-jit") + if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"): + want.append("llvm") + if INTERP: + want.append("interp") + return want + + +def require_rows(rows): + """bench-save refuses a table that is missing a backend this machine has""" + have = {r["program"].split(":")[0] for r in rows} + missing = [b for b in expected_backends() if b not in have] + if missing: + sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing)) + + def main(): ap = argparse.ArgumentParser(description="tinyvm benchmark suite") ap.add_argument("mode", choices=["run", "check", "selftest"]) @@ -184,6 +206,8 @@ def main(): return selftest() rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy) if a.mode == "run": + if os.environ.get("BENCH_SAVE") == "1": + require_rows(rows) write_table(Path(a.out), rows) print(Path(a.out).read_text(), end="") return 0 -- cgit v1.2.3 From a868055b1e5ca899c8e537061b5bdd7095bdd930 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 18:33:52 +0330 Subject: bench: baseline at 89f35bd (jit, llvm, sweep; no interp) --- tests/bench.baseline | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/bench.baseline b/tests/bench.baseline index 18fba1c..d1730db 100644 --- a/tests/bench.baseline +++ b/tests/bench.baseline @@ -1,16 +1,12 @@ # tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available program plain_ms o3_ms checks peak_heap_words gc_mark_ms -interp:collatz 594.240 259.680 - - - -interp:sieve 82.310 8.400 - - - -interp:digits 328.180 84.460 - - - -interp:fib 10.200 2.820 - - - -x86-64-jit:collatz 31.410 10.750 - - - -x86-64-jit:sieve 4.320 0.410 - - - -x86-64-jit:digits 14.750 3.070 - - - -x86-64-jit:fib 0.780 0.860 - - - -llvm:collatz 54.080 3.420 - - - -llvm:sieve 13.560 0.190 - - - -llvm:digits 26.590 1.600 - - - -llvm:fib 2.970 0.150 - - - -zy:sweep.checked 3.535 - - - - -zy:sweep.unbounded 2.625 - - - - +x86-64-jit:collatz 34.350 10.950 - - - +x86-64-jit:sieve 4.470 0.440 - - - +x86-64-jit:digits 16.120 3.220 - - - +x86-64-jit:fib 0.810 0.940 - - - +llvm:collatz 58.870 3.750 - - - +llvm:sieve 14.440 0.210 - - - +llvm:digits 29.810 1.740 - - - +llvm:fib 3.150 0.170 - - - +zy:sweep.checked 3.782 - - - - +zy:sweep.unbounded 2.876 - - - - -- cgit v1.2.3 From 3832b3aff413bffa91c9bf6e80ebec91ee693ca0 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 19:20:55 +0330 Subject: bench: idle re-save of baseline at 89f35bd --- tests/bench.baseline | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/bench.baseline b/tests/bench.baseline index d1730db..1ff5561 100644 --- a/tests/bench.baseline +++ b/tests/bench.baseline @@ -1,12 +1,12 @@ # tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available program plain_ms o3_ms checks peak_heap_words gc_mark_ms -x86-64-jit:collatz 34.350 10.950 - - - -x86-64-jit:sieve 4.470 0.440 - - - -x86-64-jit:digits 16.120 3.220 - - - -x86-64-jit:fib 0.810 0.940 - - - -llvm:collatz 58.870 3.750 - - - -llvm:sieve 14.440 0.210 - - - -llvm:digits 29.810 1.740 - - - -llvm:fib 3.150 0.170 - - - -zy:sweep.checked 3.782 - - - - -zy:sweep.unbounded 2.876 - - - - +x86-64-jit:collatz 36.240 11.720 - - - +x86-64-jit:sieve 4.910 0.440 - - - +x86-64-jit:digits 16.900 3.380 - - - +x86-64-jit:fib 0.830 0.950 - - - +llvm:collatz 60.050 3.680 - - - +llvm:sieve 14.640 0.220 - - - +llvm:digits 29.240 1.630 - - - +llvm:fib 3.040 0.170 - - - +zy:sweep.checked 3.937 - - - - +zy:sweep.unbounded 2.800 - - - - -- cgit v1.2.3 From 28c88e210a691ace1e59692ec4e243f6ee28accb Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 19:34:16 +0330 Subject: bench: drop committed pycache, ignore __pycache__ --- .gitignore | 1 + tests/bench/__pycache__/suite.cpython-313.pyc | Bin 23177 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 tests/bench/__pycache__/suite.cpython-313.pyc diff --git a/.gitignore b/.gitignore index 904cd31..b2d7604 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.o *.obj *.exe +__pycache__/ diff --git a/tests/bench/__pycache__/suite.cpython-313.pyc b/tests/bench/__pycache__/suite.cpython-313.pyc deleted file mode 100644 index 21783a7..0000000 Binary files a/tests/bench/__pycache__/suite.cpython-313.pyc and /dev/null differ -- cgit v1.2.3