From 58b86428cee46268002d23fd985a1813f641115c Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 16:53:22 +0330 Subject: wip: bench suite --- Makefile | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index b3b7d56..6deb3a9 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ # make xref who writes and who reads each global of the compiler # (src/tools/xref.sh): build/xref/report.txt and report.json; # make xref XREF_ARGS=--check compares with tests/xref.baseline +# make bench time the benchmark programs -> build/bench.txt +# make bench-save ... and keep the result as tests/bench.baseline +# make bench-check compare with tests/bench.baseline (or make bench CHECK=1) # make genome build/core/genome.bin from src/, with the fixpoint check # make LLVM=1 on x86_64, build zy with the llvm backend instead of the x86-64 jit # make NOLLVM=1 on other hosts, leave out the llvm backend (in by default when a @@ -208,6 +211,21 @@ parity: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/zy$(EXE) xref: $(BIN)/zclsp$(EXE) bash src/tools/xref.sh $(XREF_ARGS) +# benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per +# program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check +# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline. +bench: $(BIN)/zy$(EXE) +ifeq ($(CHECK),1) + python3 tests/bench/suite.py check +else + python3 tests/bench/suite.py run +endif +bench-save: bench + cp $(B)/bench.txt tests/bench.baseline + @echo "bench: wrote tests/bench.baseline" +bench-check: $(BIN)/zy$(EXE) + python3 tests/bench/suite.py check + # JOBS sets how many sections run at once: make test JOBS=4 test: bash tests/test.sh -j $(or $(JOBS),1) @@ -219,4 +237,4 @@ check: clean: rm -rf $(B) -.PHONY: all lsp genome js lspimage vsix web run parity xref test check clean +.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean -- cgit v1.2.3 From b3273f04b876228d7b84a9d5a0fe053eb95cdb95 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 17:54:07 +0330 Subject: bench: no-region guard, selftest and --rows, interp rows behind INTERP=1, fail only over 10% and 0.1 ms --- Makefile | 13 +++++---- ONBOARD.md | 10 ++++++- tests/bench/suite.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 90 insertions(+), 13 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 6deb3a9..6569bd5 100644 --- a/Makefile +++ b/Makefile @@ -213,18 +213,21 @@ xref: $(BIN)/zclsp$(EXE) # benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per # program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check -# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline. +# (or make bench CHECK=1) fails when a column is over 10% and over 0.1 ms slower than the +# baseline. interp rows only with INTERP=1. bench-selftest checks the verdicts, no timing. bench: $(BIN)/zy$(EXE) ifeq ($(CHECK),1) - python3 tests/bench/suite.py check + INTERP=$(INTERP) python3 tests/bench/suite.py check else - python3 tests/bench/suite.py run + INTERP=$(INTERP) python3 tests/bench/suite.py run endif bench-save: bench cp $(B)/bench.txt tests/bench.baseline @echo "bench: wrote tests/bench.baseline" bench-check: $(BIN)/zy$(EXE) - python3 tests/bench/suite.py check + INTERP=$(INTERP) python3 tests/bench/suite.py check +bench-selftest: + INTERP=$(INTERP) python3 tests/bench/suite.py selftest # JOBS sets how many sections run at once: make test JOBS=4 test: @@ -237,4 +240,4 @@ check: clean: rm -rf $(B) -.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean +.PHONY: bench bench-save bench-selftest bench-check all lsp genome js lspimage vsix web run parity xref test check clean diff --git a/ONBOARD.md b/ONBOARD.md index f60063a..2ccb987 100644 --- a/ONBOARD.md +++ b/ONBOARD.md @@ -214,7 +214,15 @@ and gc mark columns for runtime counters that do not exist yet; they show `-`. a run takes about ten minutes. `make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check` (or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a -row is more than 10% slower, or a row keeps more checks once both sides have a count. +row is more than 10% slower and also more than 0.1 ms slower (so sub-ms cells don't +flake), or a row keeps more checks once both sides have a count. rows missing from a run +don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times +the interpreter today, so this doesn't make the run shorter yet). before timing, the suite +runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and +it refuses when jit o3 collatz isn't under a fifth of interp o3; that is how a baseline of +interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on +doctored tables in a second, and `suite.py check --rows FILE` compares a saved table +without measuring. when something looks slower the whole run is repeated once and each cell keeps its best figure. the baseline belongs to the machine it was saved on: save a new one after changing machines, and expect noise on a busy host. diff --git a/tests/bench/suite.py b/tests/bench/suite.py index bb81e46..f7d5c75 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -7,7 +7,10 @@ usage: tests/bench/suite.py run [--zy PATH] [--out FILE] it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc (collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the 256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded, -timed here on build/bin/zy the same way: the slope between 1 and N repetitions, the +timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before +timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and +jit o3 collatz must be under a fifth of interp o3 (when both rows exist), or it refuses. +selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the median of TRIALS slopes. bench.sh itself is run as it is (it needs bc). one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the @@ -15,7 +18,8 @@ number of bounds checks the compiler keeps (from --explain-checks, which does no yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'. check compares with the baseline (tests/bench.baseline): each of the plain and o3 -columns fails on its own when it is more than TOLERANCE (10%) slower, and a row fails +columns fails on its own when it is more than TOLERANCE (10%) slower and also more than +MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails when both sides have a checks count and it rose. when anything looks slower the whole measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, so one noisy run does not fail the check. @@ -32,6 +36,8 @@ ROOT = Path(__file__).resolve().parent.parent.parent TRIALS = int(os.environ.get("TRIALS", "5")) MIN_S = float(os.environ.get("MIN_S", "0.4")) TOLERANCE = float(os.environ.get("TOLERANCE", "0.10")) +MIN_MS = float(os.environ.get("MIN_MS", "0.1")) +INTERP = os.environ.get("INTERP", "") == "1" RETRIES = int(os.environ.get("RETRIES", "1")) # the rows this script times itself: name, source, kernel letter (bench.sh has the rest) @@ -55,6 +61,8 @@ def bench_sh(): if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"): backend = "-".join(w[:-4]) rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2]))) + if not INTERP: + rows = [r for r in rows if not r[0].startswith("interp:")] if not rows: sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:]) return rows @@ -115,7 +123,31 @@ def read_table(path): return rows +def region_guard(zy): + """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc""" + src = (ROOT / "tests/programs/o3.zc").read_bytes() + env = dict(os.environ, ZYTRACE="1") + r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + err = r.stderr.decode(errors="replace").splitlines() + done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l] + stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l] + print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True) + if not done: + sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); " + "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy) + return len(done), len(stay) + + +def o3_guard(rows): + """jit o3 collatz must be far below interp o3 collatz when both rows are there""" + d = {r["program"]: r for r in rows} + j, i = d.get("x86-64-jit:collatz"), d.get("interp:collatz") + if j and i and j["o3"] is not None and i["o3"] is not None and j["o3"] * 5 > i["o3"]: + sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not far below interp o3 %.2f ms; o3 regions are not compiled" % (j["o3"], i["o3"])) + + def run_all(zy): + region_guard(zy) rows = [] for name, plain, o3 in bench_sh(): print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) @@ -124,11 +156,12 @@ def run_all(zy): ms = slope_ms(zy, path, letter) print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"}) + o3_guard(rows) return rows def slower(now, base): - return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) + return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS def pct(now, base): @@ -137,12 +170,15 @@ def pct(now, base): def main(): ap = argparse.ArgumentParser(description="tinyvm benchmark suite") - ap.add_argument("mode", choices=["run", "check"]) + ap.add_argument("mode", choices=["run", "check", "selftest"]) + ap.add_argument("--rows", help="check: compare this saved table instead of measuring") ap.add_argument("--zy", default=str(ROOT / "build/bin/zy")) ap.add_argument("--out", default=str(ROOT / "build/bench.txt")) ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline")) a = ap.parse_args() - rows = run_all(a.zy) + if a.mode == "selftest": + return selftest() + rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy) if a.mode == "run": write_table(Path(a.out), rows) print(Path(a.out).read_text(), end="") @@ -152,7 +188,7 @@ def main(): sys.exit("bench: no baseline at %s (run make bench-save)" % base_path) base = read_table(base_path) tries = 0 - while tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): + while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")): tries += 1 print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True) again = {r["program"]: r for r in run_all(a.zy)} @@ -183,9 +219,39 @@ def main(): if name not in names: print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-")) write_table(Path(a.out), rows) - print("bench-check: %s (%d of %d rows over the limit; tolerance %d%% per column)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100))) + print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS)) return 1 if bad else 0 +def selftest(): + """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts""" + import tempfile + t = Path(tempfile.mkdtemp()) + base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in + [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]] + write_table(t / "base", base) + def case(name, now, want): + write_table(t / "now", now) + r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + ok = (r.returncode != 0) == want + print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG")) + if not ok: + print(r.stdout) + return ok + row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} + good = [ + case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True), + case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True), + case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True), + case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False), + case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False), + case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False), + ] + print("selftest: %s" % ("ok" if all(good) else "FAIL")) + return 0 if all(good) else 1 + + if __name__ == "__main__": sys.exit(main()) -- cgit v1.2.3 From 44d9724e3cfb6d25c19ed0a45cb33b95c38003e5 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 18:21:07 +0330 Subject: bench: find llvm-config-19 like the Makefile; bench-save refuses when an expected backend has no rows --- Makefile | 3 ++- tests/bench/__pycache__/suite.cpython-313.pyc | Bin 0 -> 23177 bytes tests/bench/bench.sh | 6 ++++-- tests/bench/suite.py | 24 ++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/bench/__pycache__/suite.cpython-313.pyc (limited to 'Makefile') diff --git a/Makefile b/Makefile index 6569bd5..94d9b49 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,8 @@ ifeq ($(CHECK),1) else INTERP=$(INTERP) python3 tests/bench/suite.py run endif -bench-save: bench +bench-save: + BENCH_SAVE=1 $(MAKE) bench cp $(B)/bench.txt tests/bench.baseline @echo "bench: wrote tests/bench.baseline" bench-check: $(BIN)/zy$(EXE) diff --git a/tests/bench/__pycache__/suite.cpython-313.pyc b/tests/bench/__pycache__/suite.cpython-313.pyc new file mode 100644 index 0000000..21783a7 Binary files /dev/null and b/tests/bench/__pycache__/suite.cpython-313.pyc differ diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 0437b0b..e532f93 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -14,8 +14,10 @@ fi if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } fi -if command -v llvm-config >/dev/null; then - $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } +# the llvm: LLVM_CONFIG when given, else llvm-config, else llvm-config-19 (as in the Makefile) +LC=${LLVM_CONFIG:-$(command -v llvm-config || command -v llvm-config-19)} +if [ -n "$LC" ]; then + $CC -O2 -w -DJTYPE=int -DZYLLVM $($LC --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $($LC --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); } diff --git a/tests/bench/suite.py b/tests/bench/suite.py index 656e877..3816cc3 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -24,6 +24,8 @@ when both sides have a checks count and it rose. when anything looks slower the measurement is repeated, up to RETRIES more times, and each cell keeps its best figure, so one noisy run does not fail the check. """ +import shutil +import platform import argparse import os import statistics @@ -172,6 +174,26 @@ def pct(now, base): return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100) +def expected_backends(): + """the backends bench.sh should report on this machine, found the same way it finds them""" + want = ["zy"] + if platform.system() == "Linux" and platform.machine() == "x86_64": + want.append("x86-64-jit") + if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"): + want.append("llvm") + if INTERP: + want.append("interp") + return want + + +def require_rows(rows): + """bench-save refuses a table that is missing a backend this machine has""" + have = {r["program"].split(":")[0] for r in rows} + missing = [b for b in expected_backends() if b not in have] + if missing: + sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing)) + + def main(): ap = argparse.ArgumentParser(description="tinyvm benchmark suite") ap.add_argument("mode", choices=["run", "check", "selftest"]) @@ -184,6 +206,8 @@ def main(): return selftest() rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy) if a.mode == "run": + if os.environ.get("BENCH_SAVE") == "1": + require_rows(rows) write_table(Path(a.out), rows) print(Path(a.out).read_text(), end="") return 0 -- cgit v1.2.3