a

12 files changed, 801 insertions, 0 deletions

diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
new file mode 100644
index 0000000..ab6b07d
--- /dev/null
+++ b/tests/bench/bench.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# o3 against plain code: each kernel in tests/programs/bench.zc on each backend, with
+# the time to start up and compile (the 'z' run) taken off. needs bc.
+cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT
+gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zy
+vms=("$T/zy"); names=("interp")
+if command -v llvm-config >/dev/null; then
+	gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); }
+fi
+# seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run,
+# best of three each, so starting up and compiling cancel out
+t(){ local s e; s=$(date +%s.%N); (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null; e=$(date +%s.%N); echo "$e - $s" | bc; }
+best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3); b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
+# grows the run count until the difference is at least a second, so the noise
+# of starting up is small next to it
+per(){
+	local one n=4 many
+	one=$(best "$1" $2 1)
+	while :; do
+		many=$(best "$1" $2 $n)
+		if [ "$(echo "$many - $one > 1" | bc)" = 1 ] || [ $n -ge 4096 ]; then break; fi
+		n=$((n * 4))
+	done
+	echo "scale=6; ($many - $one) / ($n - 1)" | bc
+}
+printf "%-14s %-8s %10s %10s %8s\n" backend kernel "plain ms" "o3 ms" speedup
+for i in "${!vms[@]}"; do
+	for k in "collatz a b" "sieve c d" "digits e f" "fib g h"; do
+		set -- $k
+		p=$(per "${vms[$i]}" $2); o=$(per "${vms[$i]}" $3)
+		printf "%-14s %-8s %10.2f %10.2f %7.1fx\n" "${names[$i]}" $1 $(echo "$p * 1000" | bc) $(echo "$o * 1000" | bc) $(echo "scale=2; $p / $o" | bc)
+	done
+done
diff --git a/tests/bench/bench_bytes.sh b/tests/bench/bench_bytes.sh
new file mode 100644
index 0000000..3287a91
--- /dev/null
+++ b/tests/bench/bench_bytes.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# byte-load loop, timed on a run-only image so the resident compiler is not
+# in the process. the slope between 1 repetition and N drops startup.
+# prints: backend  ms/rep  checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_bytes.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+	gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+		&& { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+		&& { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+# seconds for $2 repetitions on vm $1
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_bytes.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+	one=$(once "${vms[$i]}" 1)
+	many=$(once "${vms[$i]}" 40)
+	ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+	sum=$(tr -d '\n' < /tmp/bench_bytes.out)
+	printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/bench_float.sh b/tests/bench/bench_float.sh
new file mode 100644
index 0000000..794e976
--- /dev/null
+++ b/tests/bench/bench_float.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# plain float loop against the same loop lifted by o3. the program is
+# tests/programs/fbench.zc: 'a' is plain, 'b' is o3, and the digits are repetitions.
+# the slope between 1 repetition and N drops startup and compile time, so
+# this does not bootstrap the compiler. each kernel is poly() of 20000,
+# and (int) of one call is 59997.
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zy" || exit 1
+vms=("$T/zy"); names=("interp")
+gcc -O2 -w src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -o "$T/zym" && { vms+=("$T/zym"); names+=("micro"); }
+if command -v llvm-config >/dev/null; then
+	gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyl" $(llvm-config --ldflags --libs) \
+		&& { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyj" \
+		&& { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+src=$(cat tests/programs/fbench.zc)
+# milliseconds per call of kernel $2 (a or b) on vm $1
+per() {
+	python3 - "$1" "$2" "$src" << 'PY'
+import sys, time, subprocess
+vm, which, src = sys.argv[1], sys.argv[2], sys.argv[3].encode()
+def once(arg):
+    t0 = time.perf_counter()
+    p = subprocess.run(vm.split(), input=src + b"`" + arg.encode(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    return time.perf_counter() - t0, p.returncode, p.stdout
+def best(arg):
+    b, out = 1e9, b""
+    for _ in range(3):
+        d, r, o = once(arg)
+        if r == 0 and d < b:
+            b, out = d, o
+    return b, out
+one, _ = best(which + "1")
+n = 4
+while True:
+    many, out = best("%s%d" % (which, n))
+    if many - one > 0.8 or n >= 1024:
+        break
+    n *= 4
+print("%.3f" % ((many - one) / (n - 1) * 1000))
+print(out.decode().strip(), file=sys.stderr)
+PY
+}
+printf "%-10s %12s %12s %8s\n" backend "plain ms" "o3 ms" speedup
+for i in "${!vms[@]}"; do
+	p=$(per "${vms[$i]}" a)
+	o=$(per "${vms[$i]}" b)
+	printf "%-10s %12s %12s %8s\n" "${names[$i]}" "$p" "$o" "$(python3 -c "print('%.2fx' % ($p / $o) if $o else 'n/a')")"
+done
diff --git a/tests/bench/bench_share.sh b/tests/bench/bench_share.sh
new file mode 100644
index 0000000..b449e09
--- /dev/null
+++ b/tests/bench/bench_share.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# walk a shared cons spine from two parents, timed on a run-only image so the
+# resident compiler is not in the process. the slope between 1 repetition
+# and N drops startup. prints: backend  ms/rep  checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_share.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+	gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+		&& { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+		&& { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_share.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+	one=$(once "${vms[$i]}" 1)
+	many=$(once "${vms[$i]}" 40)
+	ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+	sum=$(tr -d '\n' < /tmp/bench_share.out)
+	printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/bench_strings.sh b/tests/bench/bench_strings.sh
new file mode 100644
index 0000000..c353342
--- /dev/null
+++ b/tests/bench/bench_strings.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# byte-string scan, concat and slice, timed on a run-only image so the
+# resident compiler is not in the process. the slope between 1 repetition
+# and N drops startup. prints: backend  ms/rep  checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_strings.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+	gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+		&& { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+	aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+		&& { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_strings.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+	one=$(once "${vms[$i]}" 1)
+	many=$(once "${vms[$i]}" 40)
+	ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+	sum=$(tr -d '\n' < /tmp/bench_strings.out)
+	printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/run.py b/tests/bench/run.py
new file mode 100755
index 0000000..69cb943
--- /dev/null
+++ b/tests/bench/run.py
@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+# Time tests/bench/sieve and tests/bench/trees on C and on zc.
+# The number is the slope between 1 repetition and N: that drops process
+# startup and, for zc, the time spent compiling the benchmark. It does not
+# bootstrap the compiler. Each cell is the median of TRIALS slopes, with the
+# min and the max as the spread.
+#
+#   python3 tests/bench/run.py            # everything
+#   python3 tests/bench/run.py check      # checksums only
+#   python3 tests/bench/run.py sieve      # one program
+#   TRIALS=5 GAP=0.8 python3 tests/bench/run.py
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BENCH = Path(__file__).resolve().parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+GAP = float(os.environ.get("GAP", "0.8"))
+NMAX = int(os.environ.get("NMAX", "200000"))
+SIEVE_N = b"9592"   # primes below 100000
+TREE_N = b"8191"    # nodes in a full tree of depth 12
+
+
+def log(msg):
+    print(msg, file=sys.stderr, flush=True)
+
+
+def run(cmd, stdin=None, env=None, cwd=None):
+    e = os.environ.copy()
+    if env:
+        e.update(env)
+    t0 = time.perf_counter()
+    p = subprocess.run(cmd, input=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=e, cwd=cwd)
+    return time.perf_counter() - t0, p.returncode, p.stdout, p.stderr
+
+
+def pin(cmd):
+    if shutil.which("taskset"):
+        return ["taskset", "-c", "0"] + list(cmd)
+    return list(cmd)
+
+
+def need(cmd, cwd=None):
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+    if p.returncode != 0:
+        sys.stderr.write(p.stderr.decode(errors="replace"))
+        raise SystemExit(f"command failed: {' '.join(cmd)}")
+    return p
+
+
+def machine_lines():
+    lines = []
+    u = subprocess.check_output(["uname", "-srm"], text=True).strip()
+    lines.append(u)
+    model = ""
+    if Path("/proc/cpuinfo").exists():
+        for line in Path("/proc/cpuinfo").read_text().splitlines():
+            if line.startswith("model name"):
+                model = line.split(":", 1)[1].strip()
+                break
+    if model:
+        lines.append(model)
+    lines.append("gcc " + subprocess.check_output(["gcc", "-dumpversion"], text=True).strip())
+    if shutil.which("clang"):
+        lines.append(subprocess.check_output(["clang", "--version"], text=True).splitlines()[0])
+    if shutil.which("qemu-aarch64"):
+        lines.append(subprocess.check_output(["qemu-aarch64", "--version"], text=True).splitlines()[0])
+    if shutil.which("llvm-config"):
+        lines.append("llvm " + subprocess.check_output(["llvm-config", "--version"], text=True).strip())
+    lines.append("pinned to cpu 0 with taskset" if shutil.which("taskset") else "not pinned")
+    return lines
+
+
+def build(td):
+    td = Path(td)
+    log("building vms")
+    zy = td / "zy"
+    need(["gcc", "-O3", "-w", "-DJTYPE=int", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zy)], cwd=ROOT)
+    vms = {"interp": [str(zy)]}
+    if shutil.which("llvm-config"):
+        cflags = subprocess.check_output(["llvm-config", "--cflags"], text=True).split()
+        ld = subprocess.check_output(["llvm-config", "--ldflags", "--libs"], text=True).split()
+        zyl = td / "zyl"
+        need(["gcc", "-O3", "-w", "-DJTYPE=int", "-DZYLLVM", *cflags, "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyl), *ld], cwd=ROOT)
+        vms["llvm"] = [str(zyl)]
+    if shutil.which("aarch64-linux-gnu-gcc") and shutil.which("qemu-aarch64"):
+        zyj = td / "zyj"
+        need(["aarch64-linux-gnu-gcc", "-O3", "-w", "-DJTYPE=int", "-static", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyj)], cwd=ROOT)
+        vms["arm64"] = ["qemu-aarch64", str(zyj)]
+    log("building C")
+    bins = {}
+    for comp, tag in (("gcc", "gcc"), ("clang", "clang")):
+        if comp == "clang" and not shutil.which("clang"):
+            continue
+        for lto in (False, True):
+            name = tag + ("-lto" if lto else "")
+            flags = [comp, "-O3", "-march=native"] + (["-flto"] if lto else [])
+            sw = td / f"sieve-{name}"
+            tw = td / f"trees-{name}"
+            need(flags + ["-o", str(sw), str(BENCH / "sieve.c")])
+            need(flags + ["-o", str(tw), str(BENCH / "trees.c")])
+            bins[f"sieve-{name}"] = sw
+            bins[f"trees-{name}"] = tw
+    # PGO: train and use the same output path so the .gcda name matches.
+    if shutil.which("gcc"):
+        for src, key, arg in (("sieve.c", "sieve-gcc-pgo", ["word", "80"]), ("trees.c", "trees-gcc-pgo", ["malloc", "40"])):
+            out = td / key
+            prof = td / f"prof-{key}"
+            prof.mkdir()
+            base = ["gcc", "-O3", "-march=native", "-flto", "-o", str(out), str(BENCH / src)]
+            need(base + [f"-fprofile-generate={prof}"])
+            need([str(out), *arg])
+            p = subprocess.run(base + [f"-fprofile-use={prof}", "-fprofile-correction"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if p.returncode != 0 or b"profile count data file not found" in p.stderr:
+                log(f"PGO skipped for {key}: {p.stderr.decode(errors='replace')[:200]}")
+            else:
+                bins[key] = out
+                log(f"PGO applied for {key}")
+    return vms, bins
+
+
+def slope(invoke, expected, trials=TRIALS):
+    def once(n):
+        cmd, stdin, env = invoke(n)
+        dt, rc, out, err = run(pin(cmd), stdin, env)
+        if rc != 0 or out.strip() != expected:
+            raise SystemExit(
+                f"bad run n={n} rc={rc}\ncmd={cmd}\nout={out[:200]!r}\nerr={err[:400]!r}"
+            )
+        return dt
+
+    dt1 = once(1)
+    n = 16
+    per = 0.0
+    while n <= NMAX:
+        dtn = once(n)
+        per = (dtn - dt1) / (n - 1)
+        gap = dtn - dt1
+        if per > 0 and gap >= GAP:
+            break
+        if per > 0 and gap >= 0.05:
+            need_n = int(GAP / per) + 1
+            need_n = min(NMAX, max(need_n, n))
+            if need_n != n:
+                dtn = once(need_n)
+                per = (dtn - dt1) / (need_n - 1)
+                n = need_n
+                # a noisy calibrating run can undershoot the gap. grow until
+                # the kernel time actually dominates, or we hit the cap.
+                while n < NMAX and per > 0 and (per * (n - 1)) < GAP:
+                    n = min(NMAX, max(n + 1, int(GAP / per) + 1))
+                    dtn = once(n)
+                    per = (dtn - dt1) / (n - 1)
+            break
+        nxt = n * 4
+        if nxt > NMAX:
+            n = NMAX
+            break
+        n = nxt
+    samples = []
+    for i in range(trials):
+        a = once(1)
+        b = once(n)
+        ms = (b - a) / (n - 1) * 1000.0
+        samples.append(ms)
+        log(f"    trial {i + 1}/{trials}: {ms:.4f} ms  (n={n})")
+    samples.sort()
+    mid = samples[len(samples) // 2]
+    return {"median": mid, "min": samples[0], "max": samples[-1], "n": n, "samples": samples}
+
+
+def zc_call(vm, src, which, env=None):
+    def invoke(n):
+        return vm, src + b"`" + f"{which}{n}".encode(), env
+    return invoke
+
+
+def c_call(binary, mode):
+    def invoke(n):
+        return [str(binary), mode, str(n)], None, None
+    return invoke
+
+
+def show(title, rows):
+    print()
+    print(title)
+    print(f"{'path':<52} {'median ms':>10} {'min':>10} {'max':>10} {'reps':>8}")
+    for name, r in rows:
+        print(f"{name:<52} {r['median']:10.4f} {r['min']:10.4f} {r['max']:10.4f} {r['n']:8d}")
+    print("samples:")
+    for name, r in rows:
+        nums = " ".join(f"{s:.4f}" for s in r["samples"])
+        print(f"  {name}: {nums}")
+    sys.stdout.flush()
+
+
+def main():
+    which = sys.argv[1] if len(sys.argv) > 1 else "all"
+    os.chdir(ROOT)
+    for line in machine_lines():
+        print(line)
+    print(f"trials {TRIALS}, gap {GAP}s, slope of 1 vs N, median of the trials")
+    print("arm64 is qemu-user on this host, not native arm64 hardware")
+    sys.stdout.flush()
+    td = tempfile.mkdtemp(prefix="zcbench-")
+    log(f"work dir {td}")
+    vms, bins = build(td)
+    sieve_src = (BENCH / "sieve.zc").read_bytes()
+    tree_src = (BENCH / "trees.zc").read_bytes()
+
+    log("checksums")
+    checks = [
+        (pin(vms["interp"]), sieve_src + b"`a1", SIEVE_N, None),
+        (pin(vms["interp"]), sieve_src + b"`b1", SIEVE_N, None),
+        (pin(vms["interp"]), sieve_src + b"`c1", SIEVE_N, None),
+        (pin(vms["interp"]), tree_src + b"`a2", TREE_N, None),
+        (pin(vms["interp"]), tree_src + b"`b1", TREE_N, None),
+        (pin(vms["interp"]), tree_src + b"`c1", TREE_N, None),
+        (pin([str(bins["sieve-gcc"]), "word", "1"]), None, SIEVE_N, None),
+        (pin([str(bins["sieve-gcc"]), "byte", "1"]), None, SIEVE_N, None),
+        (pin([str(bins["trees-gcc"]), "malloc", "1"]), None, TREE_N, None),
+        (pin([str(bins["trees-gcc"]), "arena", "1"]), None, TREE_N, None),
+    ]
+    if "llvm" in vms:
+        checks.append((pin(vms["llvm"]), sieve_src + b"`c1", SIEVE_N, {"ZYHOT": "1"}))
+    if "arm64" in vms:
+        checks.append((pin(vms["arm64"]), sieve_src + b"`c1", SIEVE_N, None))
+    for cmd, stdin, expect, env in checks:
+        dt, rc, out, err = run(cmd, stdin, env)
+        if rc != 0 or out.strip() != expect:
+            raise SystemExit(f"checksum failed {cmd} rc={rc} out={out!r} err={err[:300]!r}")
+        log(f"  ok {expect.decode()} in {dt:.2f}s")
+    if which == "check":
+        print("checksums ok")
+        return
+
+    sieve_rows = []
+    tree_rows = []
+
+    def add(rows, name, invoke, expect):
+        log(f"timing {name}")
+        rows.append((name, slope(invoke, expect)))
+        r = rows[-1][1]
+        log(f"  median {r['median']:.4f} ms  spread {r['min']:.4f}..{r['max']:.4f}")
+
+    if which in ("all", "sieve"):
+        for key, label, mode in (
+            ("sieve-gcc", "gcc -O3 -march=native, word flags", "word"),
+            ("sieve-gcc-lto", "gcc -O3 -march=native -flto, word flags", "word"),
+            ("sieve-gcc-pgo", "gcc -O3 -march=native -flto PGO, word flags", "word"),
+            ("sieve-clang", "clang -O3 -march=native, word flags", "word"),
+            ("sieve-clang-lto", "clang -O3 -march=native -flto, word flags", "word"),
+            ("sieve-gcc-lto", "gcc -O3 -march=native -flto, byte flags", "byte"),
+        ):
+            if key not in bins:
+                continue
+            add(sieve_rows, label, c_call(bins[key], mode), SIEVE_N)
+        add(sieve_rows, "zc interp, plain, bounds-checked []int", zc_call(vms["interp"], sieve_src, "a"), SIEVE_N)
+        add(sieve_rows, "zc interp, o3, bounds-checked []int", zc_call(vms["interp"], sieve_src, "b"), SIEVE_N)
+        add(sieve_rows, "zc interp, o3, raw *int", zc_call(vms["interp"], sieve_src, "c"), SIEVE_N)
+        if "llvm" in vms:
+            hot = {"ZYHOT": "1"}
+            add(sieve_rows, "zc llvm O3, ZYHOT=1, o3 raw *int", zc_call(vms["llvm"], sieve_src, "c", hot), SIEVE_N)
+            add(sieve_rows, "zc llvm O3, ZYHOT=1, o3 bounds-checked", zc_call(vms["llvm"], sieve_src, "b", hot), SIEVE_N)
+        if "arm64" in vms:
+            add(sieve_rows, "zc arm64 jit, qemu-user, o3 raw *int", zc_call(vms["arm64"], sieve_src, "c"), SIEVE_N)
+        show("sieve: primes below 100000, checksum 9592", sieve_rows)
+
+    if which in ("all", "trees"):
+        for key, label, mode in (
+            ("trees-gcc", "gcc -O3 -march=native, malloc", "malloc"),
+            ("trees-gcc-lto", "gcc -O3 -march=native -flto, malloc", "malloc"),
+            ("trees-gcc-pgo", "gcc -O3 -march=native -flto PGO, malloc", "malloc"),
+            ("trees-clang", "clang -O3 -march=native, malloc", "malloc"),
+            ("trees-clang-lto", "clang -O3 -march=native -flto, malloc", "malloc"),
+            ("trees-gcc-lto", "gcc -O3 -march=native -flto, arena", "arena"),
+            ("trees-clang-lto", "clang -O3 -march=native -flto, arena", "arena"),
+        ):
+            if key not in bins:
+                continue
+            add(tree_rows, label, c_call(bins[key], mode), TREE_N)
+        add(tree_rows, "zc interp, plain new/free", zc_call(vms["interp"], tree_src, "a"), TREE_N)
+        add(tree_rows, "zc interp, o3 new/free", zc_call(vms["interp"], tree_src, "b"), TREE_N)
+        add(tree_rows, "zc interp, o3 arena", zc_call(vms["interp"], tree_src, "c"), TREE_N)
+        if "llvm" in vms:
+            hot = {"ZYHOT": "1"}
+            add(tree_rows, "zc llvm O3, ZYHOT=1, plain new/free", zc_call(vms["llvm"], tree_src, "a", hot), TREE_N)
+            add(tree_rows, "zc llvm O3, ZYHOT=1, o3 new/free", zc_call(vms["llvm"], tree_src, "b", hot), TREE_N)
+            add(tree_rows, "zc llvm O3, ZYHOT=1, o3 arena", zc_call(vms["llvm"], tree_src, "c", hot), TREE_N)
+        if "arm64" in vms:
+            add(tree_rows, "zc arm64 jit, qemu-user, plain new/free", zc_call(vms["arm64"], tree_src, "a"), TREE_N)
+            add(tree_rows, "zc arm64 jit, qemu-user, o3 arena", zc_call(vms["arm64"], tree_src, "c"), TREE_N)
+        show("trees: depth 12, checksum 8191", tree_rows)
+
+    print(f"\nwork dir left at {td}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/bench/run.sh b/tests/bench/run.sh
new file mode 100755
index 0000000..214d0a2
--- /dev/null
+++ b/tests/bench/run.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+# build the VMs and the C binaries, check the checksums, then time them.
+# see tests/bench/run.py for the slope, the trials, and the kernel letters.
+cd "$(dirname "$0")/../.."
+exec python3 tests/bench/run.py "$@"
diff --git a/tests/bench/selfmod.zc b/tests/bench/selfmod.zc
new file mode 100644
index 0000000..7e16fdb
--- /dev/null
+++ b/tests/bench/selfmod.zc
@@ -0,0 +1,19 @@
+// self-modifying code in a loop: each pass rewrites a literal inside g, so the
+// translation of g is dropped and made again, while a pile of other code stays
+// translated. prints the sum so the engines can be compared.
+int litw(int v) { return v + v + 1; }
+int g() { return 1000; }
+int work(int n) { int i = 0, s = 0; while (i < n) { s = s + i % 7; i++; } return s; }
+int a1() { return work(50); } int a2() { return work(60); } int a3() { return work(70); } int a4() { return a1() + a2(); }
+int main() {
+	word p = g; int i = 0, tot = 0;
+	while (*p != litw(1000)) p++;
+	a4(); a3();
+	while (i < 20000) {
+		*p = litw(i);
+		tot = tot + g() + a3();
+		i++;
+	}
+	put_n(tot); put_c(10);
+	return 0;
+}
diff --git a/tests/bench/sieve.c b/tests/bench/sieve.c
new file mode 100644
index 0000000..6ac503e
--- /dev/null
+++ b/tests/bench/sieve.c
@@ -0,0 +1,58 @@
+/* primes below N. the same loops as tests/bench/sieve.zc.
+   word: one int64 flag per number (what zc's fast path can do).
+   byte: one byte per number, the representation a C sieve would actually use.
+   the count is 9592 either way.
+   usage: sieve word|byte REPS */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+enum { N = 100000, EXPECT = 9592 };
+
+static int sieve_word(int64_t *restrict f)
+{
+	int c = 0;
+	for (int i = 2; i < N; i++) f[i] = 1;
+	for (int i = 2; (long long)i * i < N; i++)
+		if (f[i])
+			for (int j = i * i; j < N; j += i) f[j] = 0;
+	for (int i = 2; i < N; i++) c += (int)f[i];
+	return c;
+}
+
+static int sieve_byte(unsigned char *restrict f)
+{
+	int c = 0;
+	for (int i = 2; i < N; i++) f[i] = 1;
+	for (int i = 2; (long long)i * i < N; i++)
+		if (f[i])
+			for (int j = i * i; j < N; j += i) f[j] = 0;
+	for (int i = 2; i < N; i++) c += f[i];
+	return c;
+}
+
+int main(int argc, char **argv)
+{
+	int reps, r = 0, k, byte;
+	int64_t *w = 0;
+	unsigned char *b = 0;
+	if (argc != 3) return 2;
+	byte = argv[1][0] == 'b';
+	reps = atoi(argv[2]);
+	if (byte) b = malloc(N);
+	else w = malloc((size_t)N * sizeof *w);
+	if (!w && !b) return 3;
+	for (k = 0; k < reps; k++) {
+		r = byte ? sieve_byte(b) : sieve_word(w);
+		/* the stores have to happen: the next pass must see a clobbered buffer */
+		__asm__ __volatile__("" : "+r"(r) : "r"(byte ? (void *)b : (void *)w) : "memory");
+	}
+	if (r != EXPECT) {
+		fprintf(stderr, "bad %d\n", r);
+		return 1;
+	}
+	printf("%d\n", r);
+	free(w);
+	free(b);
+	return 0;
+}
diff --git a/tests/bench/sieve.zc b/tests/bench/sieve.zc
new file mode 100644
index 0000000..1fb579b
--- /dev/null
+++ b/tests/bench/sieve.zc
@@ -0,0 +1,51 @@
+// primes below N. the same loops as tests/bench/sieve.c: one word per flag,
+// marks start at i*i. the count is 9592.
+//   a  fixed []int, bounds-checked, plain bytecode
+//   b  the same function lifted by o3
+//   c  o3 through a raw *int, no per-index check
+const int N = 100000;
+
+int sieve(fixed []int f) {
+	int n = len(f), c = 0;
+	for (int i = 2; i < n; i++) f[i] = 1;
+	for (int i = 2; i * i < n; i++)
+		if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+	for (int i = 2; i < n; i++) c += f[i];
+	return c;
+}
+o3 int sieve3(fixed []int f) {
+	int n = len(f), c = 0;
+	for (int i = 2; i < n; i++) f[i] = 1;
+	for (int i = 2; i * i < n; i++)
+		if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+	for (int i = 2; i < n; i++) c += f[i];
+	return c;
+}
+o3 int sieve_raw(int* f) {
+	int n = N, c = 0;
+	for (int i = 2; i < n; i++) f[i] = 1;
+	for (int i = 2; i * i < n; i++)
+		if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+	for (int i = 2; i < n; i++) c += f[i];
+	return c;
+}
+int main() {
+	int which = get_c(), reps = 0, ch = get_c(), r = 0;
+	while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = get_c(); }
+	if (which == 'c') {
+		int* raw = new(int, N);
+		while (reps > 0) { r = sieve_raw(raw); reps--; }
+		put_n(r); put_c(10);
+		free(d*raw);
+		return 0;
+	}
+	fixed []int f = new(fixed []int, N);
+	while (reps > 0) {
+		if (which == 'b') r = sieve3(f);
+		else r = sieve(f);
+		reps--;
+	}
+	put_n(r); put_c(10);
+	free(d*f);
+	return 0;
+}
diff --git a/tests/bench/trees.c b/tests/bench/trees.c
new file mode 100644
index 0000000..df46447
--- /dev/null
+++ b/tests/bench/trees.c
@@ -0,0 +1,85 @@
+/* a full binary tree of depth D. checksum is the node count, 8191.
+   malloc: one malloc per node, then a recursive free. same shape as new().
+   arena: one buffer, a bump pointer, no allocator inside the loop.
+   usage: trees malloc|arena REPS */
+#include <stdio.h>
+#include <stdlib.h>
+
+enum { D = 12, EXPECT = 8191 };
+
+typedef struct node {
+	struct node *left, *right;
+} node;
+
+static node *make(int d)
+{
+	node *n = (node *)malloc(sizeof *n);
+	if (d > 0) {
+		n->left = make(d - 1);
+		n->right = make(d - 1);
+	} else
+		n->left = n->right = 0;
+	return n;
+}
+
+static int check(const node *n)
+{
+	if (!n->left) return 1;
+	return 1 + check(n->left) + check(n->right);
+}
+
+static void destroy(node *n)
+{
+	if (!n) return;
+	destroy(n->left);
+	destroy(n->right);
+	free(n);
+}
+
+static node *bump;
+
+static node *make_at(int d)
+{
+	node *n = bump++;
+	if (d > 0) {
+		n->left = make_at(d - 1);
+		n->right = make_at(d - 1);
+	} else
+		n->left = n->right = 0;
+	return n;
+}
+
+int main(int argc, char **argv)
+{
+	int reps, r = 0, k, arena, nodes;
+	node *buf = 0;
+	if (argc != 3) return 2;
+	arena = argv[1][0] == 'a';
+	reps = atoi(argv[2]);
+	nodes = (1 << (D + 1)) - 1;
+	if (arena) {
+		buf = malloc((size_t)nodes * sizeof *buf);
+		if (!buf) return 3;
+	}
+	for (k = 0; k < reps; k++) {
+		if (arena) {
+			node *root;
+			bump = buf;
+			root = make_at(D);
+			r = check(root);
+			__asm__ __volatile__("" : "+r"(r) : "r"(root) : "memory");
+		} else {
+			node *root = make(D);
+			r = check(root);
+			__asm__ __volatile__("" : "+r"(r) : "r"(root) : "memory");
+			destroy(root);
+		}
+	}
+	if (r != EXPECT) {
+		fprintf(stderr, "bad %d\n", r);
+		return 1;
+	}
+	printf("%d\n", r);
+	free(buf);
+	return 0;
+}
diff --git a/tests/bench/trees.zc b/tests/bench/trees.zc
new file mode 100644
index 0000000..c8f46ef
--- /dev/null
+++ b/tests/bench/trees.zc
@@ -0,0 +1,92 @@
+// a full binary tree of depth D. checksum is the node count, 8191.
+// each node is two pointers. new() rounds that plus the 5-word header up
+// to 8 words (a 4-word block cannot hold the header). free of the owned
+// root recycles the tree onto the size-class lists.
+//   a  plain new / own / free
+//   b  the same, marked o3 (calls and new still leave the region)
+//   c  one array, used as an arena of pairs. no allocator in the loop.
+// the same shapes are tests/bench/trees.c.
+const int D = 12;
+const int WORDS = 16382; // two words for each of the 8191 nodes
+
+struct node { node* left, right; };
+int* heap;
+int used;
+
+node*.u make(int depth) {
+	node* n = new(node);
+	if (depth > 0) {
+		n.left = make(depth - 1);
+		n.right = make(depth - 1);
+	} else {
+		n.left = 0;
+		n.right = 0;
+	}
+	return u*n;
+}
+o3 node*.u make3(int depth) {
+	node* n = new(node);
+	if (depth > 0) {
+		n.left = make3(depth - 1);
+		n.right = make3(depth - 1);
+	} else {
+		n.left = 0;
+		n.right = 0;
+	}
+	return u*n;
+}
+int check(node* n) {
+	if (!n.left) return 1;
+	return 1 + check(n.left) + check(n.right);
+}
+o3 int check3(node* n) {
+	if (!n.left) return 1;
+	return 1 + check3(n.left) + check3(n.right);
+}
+
+// a node id is its word index plus one, so 0 can mean "no child"
+o3 int build(int depth) {
+	int i = used;
+	used = used + 2;
+	if (depth > 0) {
+		heap[i] = build(depth - 1);
+		heap[i + 1] = build(depth - 1);
+	} else {
+		heap[i] = 0;
+		heap[i + 1] = 0;
+	}
+	return i + 1;
+}
+o3 int ck(int id) {
+	int i, l;
+	if (id == 0) return 0;
+	i = id - 1;
+	l = heap[i];
+	if (l == 0) return 1;
+	return 1 + ck(l) + ck(heap[i + 1]);
+}
+
+int main() {
+	int which = get_c(), reps = 0, ch = get_c(), r = 0;
+	while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = get_c(); }
+	if (which == 'c') {
+		heap = new(int, WORDS);
+		while (reps > 0) { used = 0; r = ck(build(D)); reps--; }
+		put_n(r); put_c(10);
+		return 0;
+	}
+	while (reps > 0) {
+		if (which == 'b') {
+			node* root = make3(D);
+			r = check3(root);
+			free(d*root);
+		} else {
+			node* root = make(D);
+			r = check(root);
+			free(d*root);
+		}
+		reps--;
+	}
+	put_n(r); put_c(10);
+	return 0;
+}