aboutsummaryrefslogtreecommitdiff
path: root/tests/bench
diff options
context:
space:
mode:
Diffstat (limited to 'tests/bench')
-rw-r--r--tests/bench/bench.sh36
-rw-r--r--tests/bench/bench_bytes.sh33
-rw-r--r--tests/bench/bench_float.sh54
-rw-r--r--tests/bench/bench_share.sh32
-rw-r--r--tests/bench/bench_strings.sh32
-rwxr-xr-xtests/bench/run.py304
-rwxr-xr-xtests/bench/run.sh5
-rw-r--r--tests/bench/selfmod.zc19
-rw-r--r--tests/bench/sieve.c58
-rw-r--r--tests/bench/sieve.zc51
-rw-r--r--tests/bench/trees.c85
-rw-r--r--tests/bench/trees.zc92
12 files changed, 801 insertions, 0 deletions
diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
new file mode 100644
index 0000000..ab6b07d
--- /dev/null
+++ b/tests/bench/bench.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# o3 against plain code: each kernel in tests/programs/bench.zc on each backend, with
+# the time to start up and compile (the 'z' run) taken off. needs bc.
+cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT
+gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zy
+vms=("$T/zy"); names=("interp")
+if command -v llvm-config >/dev/null; then
+ gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+ aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); }
+fi
+# seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run,
+# best of three each, so starting up and compiling cancel out
+t(){ local s e; s=$(date +%s.%N); (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null; e=$(date +%s.%N); echo "$e - $s" | bc; }
+best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3); b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
+# grows the run count until the difference is at least a second, so the noise
+# of starting up is small next to it
+per(){
+ local one n=4 many
+ one=$(best "$1" $2 1)
+ while :; do
+ many=$(best "$1" $2 $n)
+ if [ "$(echo "$many - $one > 1" | bc)" = 1 ] || [ $n -ge 4096 ]; then break; fi
+ n=$((n * 4))
+ done
+ echo "scale=6; ($many - $one) / ($n - 1)" | bc
+}
+printf "%-14s %-8s %10s %10s %8s\n" backend kernel "plain ms" "o3 ms" speedup
+for i in "${!vms[@]}"; do
+ for k in "collatz a b" "sieve c d" "digits e f" "fib g h"; do
+ set -- $k
+ p=$(per "${vms[$i]}" $2); o=$(per "${vms[$i]}" $3)
+ printf "%-14s %-8s %10.2f %10.2f %7.1fx\n" "${names[$i]}" $1 $(echo "$p * 1000" | bc) $(echo "$o * 1000" | bc) $(echo "scale=2; $p / $o" | bc)
+ done
+done
diff --git a/tests/bench/bench_bytes.sh b/tests/bench/bench_bytes.sh
new file mode 100644
index 0000000..3287a91
--- /dev/null
+++ b/tests/bench/bench_bytes.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# byte-load loop, timed on a run-only image so the resident compiler is not
+# in the process. the slope between 1 repetition and N drops startup.
+# prints: backend ms/rep checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_bytes.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+ gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+ && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+ aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+ && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+# seconds for $2 repetitions on vm $1
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_bytes.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+ one=$(once "${vms[$i]}" 1)
+ many=$(once "${vms[$i]}" 40)
+ ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+ sum=$(tr -d '\n' < /tmp/bench_bytes.out)
+ printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/bench_float.sh b/tests/bench/bench_float.sh
new file mode 100644
index 0000000..794e976
--- /dev/null
+++ b/tests/bench/bench_float.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# plain float loop against the same loop lifted by o3. the program is
+# tests/programs/fbench.zc: 'a' is plain, 'b' is o3, and the digits are repetitions.
+# the slope between 1 repetition and N drops startup and compile time, so
+# this does not bootstrap the compiler. each kernel is poly() of 20000,
+# and (int) of one call is 59997.
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zy" || exit 1
+vms=("$T/zy"); names=("interp")
+gcc -O2 -w src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -o "$T/zym" && { vms+=("$T/zym"); names+=("micro"); }
+if command -v llvm-config >/dev/null; then
+ gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyl" $(llvm-config --ldflags --libs) \
+ && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+ aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyj" \
+ && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+src=$(cat tests/programs/fbench.zc)
+# milliseconds per call of kernel $2 (a or b) on vm $1
+per() {
+ python3 - "$1" "$2" "$src" << 'PY'
+import sys, time, subprocess
+vm, which, src = sys.argv[1], sys.argv[2], sys.argv[3].encode()
+def once(arg):
+ t0 = time.perf_counter()
+ p = subprocess.run(vm.split(), input=src + b"`" + arg.encode(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+ return time.perf_counter() - t0, p.returncode, p.stdout
+def best(arg):
+ b, out = 1e9, b""
+ for _ in range(3):
+ d, r, o = once(arg)
+ if r == 0 and d < b:
+ b, out = d, o
+ return b, out
+one, _ = best(which + "1")
+n = 4
+while True:
+ many, out = best("%s%d" % (which, n))
+ if many - one > 0.8 or n >= 1024:
+ break
+ n *= 4
+print("%.3f" % ((many - one) / (n - 1) * 1000))
+print(out.decode().strip(), file=sys.stderr)
+PY
+}
+printf "%-10s %12s %12s %8s\n" backend "plain ms" "o3 ms" speedup
+for i in "${!vms[@]}"; do
+ p=$(per "${vms[$i]}" a)
+ o=$(per "${vms[$i]}" b)
+ printf "%-10s %12s %12s %8s\n" "${names[$i]}" "$p" "$o" "$(python3 -c "print('%.2fx' % ($p / $o) if $o else 'n/a')")"
+done
diff --git a/tests/bench/bench_share.sh b/tests/bench/bench_share.sh
new file mode 100644
index 0000000..b449e09
--- /dev/null
+++ b/tests/bench/bench_share.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# walk a shared cons spine from two parents, timed on a run-only image so the
+# resident compiler is not in the process. the slope between 1 repetition
+# and N drops startup. prints: backend ms/rep checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_share.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+ gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+ && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+ aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+ && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_share.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+ one=$(once "${vms[$i]}" 1)
+ many=$(once "${vms[$i]}" 40)
+ ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+ sum=$(tr -d '\n' < /tmp/bench_share.out)
+ printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/bench_strings.sh b/tests/bench/bench_strings.sh
new file mode 100644
index 0000000..c353342
--- /dev/null
+++ b/tests/bench/bench_strings.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# byte-string scan, concat and slice, timed on a run-only image so the
+# resident compiler is not in the process. the slope between 1 repetition
+# and N drops startup. prints: backend ms/rep checksum
+cd "$(dirname "$0")/../.."
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1
+"$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1
+cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/"
+gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1
+cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_strings.zc | "$T/zc0" > "$T/run/genome.h" || exit 1
+gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1
+gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1
+vms=("$T/zy" "$T/zym"); names=("interp" "micro")
+if command -v llvm-config >/dev/null; then
+ gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \
+ && { vms+=("$T/zyl"); names+=("llvm"); }
+fi
+if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
+ aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyj" \
+ && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); }
+fi
+once(){ local s e; s=$(date +%s.%N); printf '%s' "$2" | $1 >/tmp/bench_strings.out; e=$(date +%s.%N); python3 -c "print($e - $s)"; }
+printf "%-10s %12s %s\n" backend "ms/rep" checksum
+for i in "${!vms[@]}"; do
+ one=$(once "${vms[$i]}" 1)
+ many=$(once "${vms[$i]}" 40)
+ ms=$(python3 -c "print(round(($many - $one) / 39 * 1000, 3))")
+ sum=$(tr -d '\n' < /tmp/bench_strings.out)
+ printf "%-10s %12s %s\n" "${names[$i]}" "$ms" "$sum"
+done
diff --git a/tests/bench/run.py b/tests/bench/run.py
new file mode 100755
index 0000000..69cb943
--- /dev/null
+++ b/tests/bench/run.py
@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+# Time tests/bench/sieve and tests/bench/trees on C and on zc.
+# The number is the slope between 1 repetition and N: that drops process
+# startup and, for zc, the time spent compiling the benchmark. It does not
+# bootstrap the compiler. Each cell is the median of TRIALS slopes, with the
+# min and the max as the spread.
+#
+# python3 tests/bench/run.py # everything
+# python3 tests/bench/run.py check # checksums only
+# python3 tests/bench/run.py sieve # one program
+# TRIALS=5 GAP=0.8 python3 tests/bench/run.py
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BENCH = Path(__file__).resolve().parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+GAP = float(os.environ.get("GAP", "0.8"))
+NMAX = int(os.environ.get("NMAX", "200000"))
+SIEVE_N = b"9592" # primes below 100000
+TREE_N = b"8191" # nodes in a full tree of depth 12
+
+
+def log(msg):
+ print(msg, file=sys.stderr, flush=True)
+
+
+def run(cmd, stdin=None, env=None, cwd=None):
+ e = os.environ.copy()
+ if env:
+ e.update(env)
+ t0 = time.perf_counter()
+ p = subprocess.run(cmd, input=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=e, cwd=cwd)
+ return time.perf_counter() - t0, p.returncode, p.stdout, p.stderr
+
+
+def pin(cmd):
+ if shutil.which("taskset"):
+ return ["taskset", "-c", "0"] + list(cmd)
+ return list(cmd)
+
+
+def need(cmd, cwd=None):
+ p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+ if p.returncode != 0:
+ sys.stderr.write(p.stderr.decode(errors="replace"))
+ raise SystemExit(f"command failed: {' '.join(cmd)}")
+ return p
+
+
+def machine_lines():
+ lines = []
+ u = subprocess.check_output(["uname", "-srm"], text=True).strip()
+ lines.append(u)
+ model = ""
+ if Path("/proc/cpuinfo").exists():
+ for line in Path("/proc/cpuinfo").read_text().splitlines():
+ if line.startswith("model name"):
+ model = line.split(":", 1)[1].strip()
+ break
+ if model:
+ lines.append(model)
+ lines.append("gcc " + subprocess.check_output(["gcc", "-dumpversion"], text=True).strip())
+ if shutil.which("clang"):
+ lines.append(subprocess.check_output(["clang", "--version"], text=True).splitlines()[0])
+ if shutil.which("qemu-aarch64"):
+ lines.append(subprocess.check_output(["qemu-aarch64", "--version"], text=True).splitlines()[0])
+ if shutil.which("llvm-config"):
+ lines.append("llvm " + subprocess.check_output(["llvm-config", "--version"], text=True).strip())
+ lines.append("pinned to cpu 0 with taskset" if shutil.which("taskset") else "not pinned")
+ return lines
+
+
+def build(td):
+ td = Path(td)
+ log("building vms")
+ zy = td / "zy"
+ need(["gcc", "-O3", "-w", "-DJTYPE=int", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zy)], cwd=ROOT)
+ vms = {"interp": [str(zy)]}
+ if shutil.which("llvm-config"):
+ cflags = subprocess.check_output(["llvm-config", "--cflags"], text=True).split()
+ ld = subprocess.check_output(["llvm-config", "--ldflags", "--libs"], text=True).split()
+ zyl = td / "zyl"
+ need(["gcc", "-O3", "-w", "-DJTYPE=int", "-DZYLLVM", *cflags, "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyl), *ld], cwd=ROOT)
+ vms["llvm"] = [str(zyl)]
+ if shutil.which("aarch64-linux-gnu-gcc") and shutil.which("qemu-aarch64"):
+ zyj = td / "zyj"
+ need(["aarch64-linux-gnu-gcc", "-O3", "-w", "-DJTYPE=int", "-static", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyj)], cwd=ROOT)
+ vms["arm64"] = ["qemu-aarch64", str(zyj)]
+ log("building C")
+ bins = {}
+ for comp, tag in (("gcc", "gcc"), ("clang", "clang")):
+ if comp == "clang" and not shutil.which("clang"):
+ continue
+ for lto in (False, True):
+ name = tag + ("-lto" if lto else "")
+ flags = [comp, "-O3", "-march=native"] + (["-flto"] if lto else [])
+ sw = td / f"sieve-{name}"
+ tw = td / f"trees-{name}"
+ need(flags + ["-o", str(sw), str(BENCH / "sieve.c")])
+ need(flags + ["-o", str(tw), str(BENCH / "trees.c")])
+ bins[f"sieve-{name}"] = sw
+ bins[f"trees-{name}"] = tw
+ # PGO: train and use the same output path so the .gcda name matches.
+ if shutil.which("gcc"):
+ for src, key, arg in (("sieve.c", "sieve-gcc-pgo", ["word", "80"]), ("trees.c", "trees-gcc-pgo", ["malloc", "40"])):
+ out = td / key
+ prof = td / f"prof-{key}"
+ prof.mkdir()
+ base = ["gcc", "-O3", "-march=native", "-flto", "-o", str(out), str(BENCH / src)]
+ need(base + [f"-fprofile-generate={prof}"])
+ need([str(out), *arg])
+ p = subprocess.run(base + [f"-fprofile-use={prof}", "-fprofile-correction"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if p.returncode != 0 or b"profile count data file not found" in p.stderr:
+ log(f"PGO skipped for {key}: {p.stderr.decode(errors='replace')[:200]}")
+ else:
+ bins[key] = out
+ log(f"PGO applied for {key}")
+ return vms, bins
+
+
+def slope(invoke, expected, trials=TRIALS):
+ def once(n):
+ cmd, stdin, env = invoke(n)
+ dt, rc, out, err = run(pin(cmd), stdin, env)
+ if rc != 0 or out.strip() != expected:
+ raise SystemExit(
+ f"bad run n={n} rc={rc}\ncmd={cmd}\nout={out[:200]!r}\nerr={err[:400]!r}"
+ )
+ return dt
+
+ dt1 = once(1)
+ n = 16
+ per = 0.0
+ while n <= NMAX:
+ dtn = once(n)
+ per = (dtn - dt1) / (n - 1)
+ gap = dtn - dt1
+ if per > 0 and gap >= GAP:
+ break
+ if per > 0 and gap >= 0.05:
+ need_n = int(GAP / per) + 1
+ need_n = min(NMAX, max(need_n, n))
+ if need_n != n:
+ dtn = once(need_n)
+ per = (dtn - dt1) / (need_n - 1)
+ n = need_n
+ # a noisy calibrating run can undershoot the gap. grow until
+ # the kernel time actually dominates, or we hit the cap.
+ while n < NMAX and per > 0 and (per * (n - 1)) < GAP:
+ n = min(NMAX, max(n + 1, int(GAP / per) + 1))
+ dtn = once(n)
+ per = (dtn - dt1) / (n - 1)
+ break
+ nxt = n * 4
+ if nxt > NMAX:
+ n = NMAX
+ break
+ n = nxt
+ samples = []
+ for i in range(trials):
+ a = once(1)
+ b = once(n)
+ ms = (b - a) / (n - 1) * 1000.0
+ samples.append(ms)
+ log(f" trial {i + 1}/{trials}: {ms:.4f} ms (n={n})")
+ samples.sort()
+ mid = samples[len(samples) // 2]
+ return {"median": mid, "min": samples[0], "max": samples[-1], "n": n, "samples": samples}
+
+
+def zc_call(vm, src, which, env=None):
+ def invoke(n):
+ return vm, src + b"`" + f"{which}{n}".encode(), env
+ return invoke
+
+
+def c_call(binary, mode):
+ def invoke(n):
+ return [str(binary), mode, str(n)], None, None
+ return invoke
+
+
+def show(title, rows):
+ print()
+ print(title)
+ print(f"{'path':<52} {'median ms':>10} {'min':>10} {'max':>10} {'reps':>8}")
+ for name, r in rows:
+ print(f"{name:<52} {r['median']:10.4f} {r['min']:10.4f} {r['max']:10.4f} {r['n']:8d}")
+ print("samples:")
+ for name, r in rows:
+ nums = " ".join(f"{s:.4f}" for s in r["samples"])
+ print(f" {name}: {nums}")
+ sys.stdout.flush()
+
+
+def main():
+ which = sys.argv[1] if len(sys.argv) > 1 else "all"
+ os.chdir(ROOT)
+ for line in machine_lines():
+ print(line)
+ print(f"trials {TRIALS}, gap {GAP}s, slope of 1 vs N, median of the trials")
+ print("arm64 is qemu-user on this host, not native arm64 hardware")
+ sys.stdout.flush()
+ td = tempfile.mkdtemp(prefix="zcbench-")
+ log(f"work dir {td}")
+ vms, bins = build(td)
+ sieve_src = (BENCH / "sieve.zc").read_bytes()
+ tree_src = (BENCH / "trees.zc").read_bytes()
+
+ log("checksums")
+ checks = [
+ (pin(vms["interp"]), sieve_src + b"`a1", SIEVE_N, None),
+ (pin(vms["interp"]), sieve_src + b"`b1", SIEVE_N, None),
+ (pin(vms["interp"]), sieve_src + b"`c1", SIEVE_N, None),
+ (pin(vms["interp"]), tree_src + b"`a2", TREE_N, None),
+ (pin(vms["interp"]), tree_src + b"`b1", TREE_N, None),
+ (pin(vms["interp"]), tree_src + b"`c1", TREE_N, None),
+ (pin([str(bins["sieve-gcc"]), "word", "1"]), None, SIEVE_N, None),
+ (pin([str(bins["sieve-gcc"]), "byte", "1"]), None, SIEVE_N, None),
+ (pin([str(bins["trees-gcc"]), "malloc", "1"]), None, TREE_N, None),
+ (pin([str(bins["trees-gcc"]), "arena", "1"]), None, TREE_N, None),
+ ]
+ if "llvm" in vms:
+ checks.append((pin(vms["llvm"]), sieve_src + b"`c1", SIEVE_N, {"ZYHOT": "1"}))
+ if "arm64" in vms:
+ checks.append((pin(vms["arm64"]), sieve_src + b"`c1", SIEVE_N, None))
+ for cmd, stdin, expect, env in checks:
+ dt, rc, out, err = run(cmd, stdin, env)
+ if rc != 0 or out.strip() != expect:
+ raise SystemExit(f"checksum failed {cmd} rc={rc} out={out!r} err={err[:300]!r}")
+ log(f" ok {expect.decode()} in {dt:.2f}s")
+ if which == "check":
+ print("checksums ok")
+ return
+
+ sieve_rows = []
+ tree_rows = []
+
+ def add(rows, name, invoke, expect):
+ log(f"timing {name}")
+ rows.append((name, slope(invoke, expect)))
+ r = rows[-1][1]
+ log(f" median {r['median']:.4f} ms spread {r['min']:.4f}..{r['max']:.4f}")
+
+ if which in ("all", "sieve"):
+ for key, label, mode in (
+ ("sieve-gcc", "gcc -O3 -march=native, word flags", "word"),
+ ("sieve-gcc-lto", "gcc -O3 -march=native -flto, word flags", "word"),
+ ("sieve-gcc-pgo", "gcc -O3 -march=native -flto PGO, word flags", "word"),
+ ("sieve-clang", "clang -O3 -march=native, word flags", "word"),
+ ("sieve-clang-lto", "clang -O3 -march=native -flto, word flags", "word"),
+ ("sieve-gcc-lto", "gcc -O3 -march=native -flto, byte flags", "byte"),
+ ):
+ if key not in bins:
+ continue
+ add(sieve_rows, label, c_call(bins[key], mode), SIEVE_N)
+ add(sieve_rows, "zc interp, plain, bounds-checked []int", zc_call(vms["interp"], sieve_src, "a"), SIEVE_N)
+ add(sieve_rows, "zc interp, o3, bounds-checked []int", zc_call(vms["interp"], sieve_src, "b"), SIEVE_N)
+ add(sieve_rows, "zc interp, o3, raw *int", zc_call(vms["interp"], sieve_src, "c"), SIEVE_N)
+ if "llvm" in vms:
+ hot = {"ZYHOT": "1"}
+ add(sieve_rows, "zc llvm O3, ZYHOT=1, o3 raw *int", zc_call(vms["llvm"], sieve_src, "c", hot), SIEVE_N)
+ add(sieve_rows, "zc llvm O3, ZYHOT=1, o3 bounds-checked", zc_call(vms["llvm"], sieve_src, "b", hot), SIEVE_N)
+ if "arm64" in vms:
+ add(sieve_rows, "zc arm64 jit, qemu-user, o3 raw *int", zc_call(vms["arm64"], sieve_src, "c"), SIEVE_N)
+ show("sieve: primes below 100000, checksum 9592", sieve_rows)
+
+ if which in ("all", "trees"):
+ for key, label, mode in (
+ ("trees-gcc", "gcc -O3 -march=native, malloc", "malloc"),
+ ("trees-gcc-lto", "gcc -O3 -march=native -flto, malloc", "malloc"),
+ ("trees-gcc-pgo", "gcc -O3 -march=native -flto PGO, malloc", "malloc"),
+ ("trees-clang", "clang -O3 -march=native, malloc", "malloc"),
+ ("trees-clang-lto", "clang -O3 -march=native -flto, malloc", "malloc"),
+ ("trees-gcc-lto", "gcc -O3 -march=native -flto, arena", "arena"),
+ ("trees-clang-lto", "clang -O3 -march=native -flto, arena", "arena"),
+ ):
+ if key not in bins:
+ continue
+ add(tree_rows, label, c_call(bins[key], mode), TREE_N)
+ add(tree_rows, "zc interp, plain new/free", zc_call(vms["interp"], tree_src, "a"), TREE_N)
+ add(tree_rows, "zc interp, o3 new/free", zc_call(vms["interp"], tree_src, "b"), TREE_N)
+ add(tree_rows, "zc interp, o3 arena", zc_call(vms["interp"], tree_src, "c"), TREE_N)
+ if "llvm" in vms:
+ hot = {"ZYHOT": "1"}
+ add(tree_rows, "zc llvm O3, ZYHOT=1, plain new/free", zc_call(vms["llvm"], tree_src, "a", hot), TREE_N)
+ add(tree_rows, "zc llvm O3, ZYHOT=1, o3 new/free", zc_call(vms["llvm"], tree_src, "b", hot), TREE_N)
+ add(tree_rows, "zc llvm O3, ZYHOT=1, o3 arena", zc_call(vms["llvm"], tree_src, "c", hot), TREE_N)
+ if "arm64" in vms:
+ add(tree_rows, "zc arm64 jit, qemu-user, plain new/free", zc_call(vms["arm64"], tree_src, "a"), TREE_N)
+ add(tree_rows, "zc arm64 jit, qemu-user, o3 arena", zc_call(vms["arm64"], tree_src, "c"), TREE_N)
+ show("trees: depth 12, checksum 8191", tree_rows)
+
+ print(f"\nwork dir left at {td}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/bench/run.sh b/tests/bench/run.sh
new file mode 100755
index 0000000..214d0a2
--- /dev/null
+++ b/tests/bench/run.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+# build the VMs and the C binaries, check the checksums, then time them.
+# see tests/bench/run.py for the slope, the trials, and the kernel letters.
+cd "$(dirname "$0")/../.."
+exec python3 tests/bench/run.py "$@"
diff --git a/tests/bench/selfmod.zc b/tests/bench/selfmod.zc
new file mode 100644
index 0000000..7e16fdb
--- /dev/null
+++ b/tests/bench/selfmod.zc
@@ -0,0 +1,19 @@
+// self-modifying code in a loop: each pass rewrites a literal inside g, so the
+// translation of g is dropped and made again, while a pile of other code stays
+// translated. prints the sum so the engines can be compared.
+int litw(int v) { return v + v + 1; }
+int g() { return 1000; }
+int work(int n) { int i = 0, s = 0; while (i < n) { s = s + i % 7; i++; } return s; }
+int a1() { return work(50); } int a2() { return work(60); } int a3() { return work(70); } int a4() { return a1() + a2(); }
+int main() {
+ word p = g; int i = 0, tot = 0;
+ while (*p != litw(1000)) p++;
+ a4(); a3();
+ while (i < 20000) {
+ *p = litw(i);
+ tot = tot + g() + a3();
+ i++;
+ }
+ put_n(tot); put_c(10);
+ return 0;
+}
diff --git a/tests/bench/sieve.c b/tests/bench/sieve.c
new file mode 100644
index 0000000..6ac503e
--- /dev/null
+++ b/tests/bench/sieve.c
@@ -0,0 +1,58 @@
+/* primes below N. the same loops as tests/bench/sieve.zc.
+ word: one int64 flag per number (what zc's fast path can do).
+ byte: one byte per number, the representation a C sieve would actually use.
+ the count is 9592 either way.
+ usage: sieve word|byte REPS */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+enum { N = 100000, EXPECT = 9592 };
+
+static int sieve_word(int64_t *restrict f)
+{
+ int c = 0;
+ for (int i = 2; i < N; i++) f[i] = 1;
+ for (int i = 2; (long long)i * i < N; i++)
+ if (f[i])
+ for (int j = i * i; j < N; j += i) f[j] = 0;
+ for (int i = 2; i < N; i++) c += (int)f[i];
+ return c;
+}
+
+static int sieve_byte(unsigned char *restrict f)
+{
+ int c = 0;
+ for (int i = 2; i < N; i++) f[i] = 1;
+ for (int i = 2; (long long)i * i < N; i++)
+ if (f[i])
+ for (int j = i * i; j < N; j += i) f[j] = 0;
+ for (int i = 2; i < N; i++) c += f[i];
+ return c;
+}
+
+int main(int argc, char **argv)
+{
+ int reps, r = 0, k, byte;
+ int64_t *w = 0;
+ unsigned char *b = 0;
+ if (argc != 3) return 2;
+ byte = argv[1][0] == 'b';
+ reps = atoi(argv[2]);
+ if (byte) b = malloc(N);
+ else w = malloc((size_t)N * sizeof *w);
+ if (!w && !b) return 3;
+ for (k = 0; k < reps; k++) {
+ r = byte ? sieve_byte(b) : sieve_word(w);
+ /* the stores have to happen: the next pass must see a clobbered buffer */
+ __asm__ __volatile__("" : "+r"(r) : "r"(byte ? (void *)b : (void *)w) : "memory");
+ }
+ if (r != EXPECT) {
+ fprintf(stderr, "bad %d\n", r);
+ return 1;
+ }
+ printf("%d\n", r);
+ free(w);
+ free(b);
+ return 0;
+}
diff --git a/tests/bench/sieve.zc b/tests/bench/sieve.zc
new file mode 100644
index 0000000..1fb579b
--- /dev/null
+++ b/tests/bench/sieve.zc
@@ -0,0 +1,51 @@
+// primes below N. the same loops as tests/bench/sieve.c: one word per flag,
+// marks start at i*i. the count is 9592.
+// a fixed []int, bounds-checked, plain bytecode
+// b the same function lifted by o3
+// c o3 through a raw *int, no per-index check
+const int N = 100000;
+
+int sieve(fixed []int f) {
+ int n = len(f), c = 0;
+ for (int i = 2; i < n; i++) f[i] = 1;
+ for (int i = 2; i * i < n; i++)
+ if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+ for (int i = 2; i < n; i++) c += f[i];
+ return c;
+}
+o3 int sieve3(fixed []int f) {
+ int n = len(f), c = 0;
+ for (int i = 2; i < n; i++) f[i] = 1;
+ for (int i = 2; i * i < n; i++)
+ if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+ for (int i = 2; i < n; i++) c += f[i];
+ return c;
+}
+o3 int sieve_raw(int* f) {
+ int n = N, c = 0;
+ for (int i = 2; i < n; i++) f[i] = 1;
+ for (int i = 2; i * i < n; i++)
+ if (f[i]) for (int j = i * i; j < n; j += i) f[j] = 0;
+ for (int i = 2; i < n; i++) c += f[i];
+ return c;
+}
+int main() {
+ int which = get_c(), reps = 0, ch = get_c(), r = 0;
+ while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = get_c(); }
+ if (which == 'c') {
+ int* raw = new(int, N);
+ while (reps > 0) { r = sieve_raw(raw); reps--; }
+ put_n(r); put_c(10);
+ free(d*raw);
+ return 0;
+ }
+ fixed []int f = new(fixed []int, N);
+ while (reps > 0) {
+ if (which == 'b') r = sieve3(f);
+ else r = sieve(f);
+ reps--;
+ }
+ put_n(r); put_c(10);
+ free(d*f);
+ return 0;
+}
diff --git a/tests/bench/trees.c b/tests/bench/trees.c
new file mode 100644
index 0000000..df46447
--- /dev/null
+++ b/tests/bench/trees.c
@@ -0,0 +1,85 @@
+/* a full binary tree of depth D. checksum is the node count, 8191.
+ malloc: one malloc per node, then a recursive free. same shape as new().
+ arena: one buffer, a bump pointer, no allocator inside the loop.
+ usage: trees malloc|arena REPS */
+#include <stdio.h>
+#include <stdlib.h>
+
+enum { D = 12, EXPECT = 8191 };
+
+typedef struct node {
+ struct node *left, *right;
+} node;
+
+static node *make(int d)
+{
+ node *n = (node *)malloc(sizeof *n);
+ if (d > 0) {
+ n->left = make(d - 1);
+ n->right = make(d - 1);
+ } else
+ n->left = n->right = 0;
+ return n;
+}
+
+static int check(const node *n)
+{
+ if (!n->left) return 1;
+ return 1 + check(n->left) + check(n->right);
+}
+
+static void destroy(node *n)
+{
+ if (!n) return;
+ destroy(n->left);
+ destroy(n->right);
+ free(n);
+}
+
+static node *bump;
+
+static node *make_at(int d)
+{
+ node *n = bump++;
+ if (d > 0) {
+ n->left = make_at(d - 1);
+ n->right = make_at(d - 1);
+ } else
+ n->left = n->right = 0;
+ return n;
+}
+
+int main(int argc, char **argv)
+{
+ int reps, r = 0, k, arena, nodes;
+ node *buf = 0;
+ if (argc != 3) return 2;
+ arena = argv[1][0] == 'a';
+ reps = atoi(argv[2]);
+ nodes = (1 << (D + 1)) - 1;
+ if (arena) {
+ buf = malloc((size_t)nodes * sizeof *buf);
+ if (!buf) return 3;
+ }
+ for (k = 0; k < reps; k++) {
+ if (arena) {
+ node *root;
+ bump = buf;
+ root = make_at(D);
+ r = check(root);
+ __asm__ __volatile__("" : "+r"(r) : "r"(root) : "memory");
+ } else {
+ node *root = make(D);
+ r = check(root);
+ __asm__ __volatile__("" : "+r"(r) : "r"(root) : "memory");
+ destroy(root);
+ }
+ }
+ if (r != EXPECT) {
+ fprintf(stderr, "bad %d\n", r);
+ return 1;
+ }
+ printf("%d\n", r);
+ free(buf);
+ return 0;
+}
diff --git a/tests/bench/trees.zc b/tests/bench/trees.zc
new file mode 100644
index 0000000..c8f46ef
--- /dev/null
+++ b/tests/bench/trees.zc
@@ -0,0 +1,92 @@
+// a full binary tree of depth D. checksum is the node count, 8191.
+// each node is two pointers. new() rounds that plus the 5-word header up
+// to 8 words (a 4-word block cannot hold the header). free of the owned
+// root recycles the tree onto the size-class lists.
+// a plain new / own / free
+// b the same, marked o3 (calls and new still leave the region)
+// c one array, used as an arena of pairs. no allocator in the loop.
+// the same shapes are tests/bench/trees.c.
+const int D = 12;
+const int WORDS = 16382; // two words for each of the 8191 nodes
+
+struct node { node* left, right; };
+int* heap;
+int used;
+
+node*.u make(int depth) {
+ node* n = new(node);
+ if (depth > 0) {
+ n.left = make(depth - 1);
+ n.right = make(depth - 1);
+ } else {
+ n.left = 0;
+ n.right = 0;
+ }
+ return u*n;
+}
+o3 node*.u make3(int depth) {
+ node* n = new(node);
+ if (depth > 0) {
+ n.left = make3(depth - 1);
+ n.right = make3(depth - 1);
+ } else {
+ n.left = 0;
+ n.right = 0;
+ }
+ return u*n;
+}
+int check(node* n) {
+ if (!n.left) return 1;
+ return 1 + check(n.left) + check(n.right);
+}
+o3 int check3(node* n) {
+ if (!n.left) return 1;
+ return 1 + check3(n.left) + check3(n.right);
+}
+
+// a node id is its word index plus one, so 0 can mean "no child"
+o3 int build(int depth) {
+ int i = used;
+ used = used + 2;
+ if (depth > 0) {
+ heap[i] = build(depth - 1);
+ heap[i + 1] = build(depth - 1);
+ } else {
+ heap[i] = 0;
+ heap[i + 1] = 0;
+ }
+ return i + 1;
+}
+o3 int ck(int id) {
+ int i, l;
+ if (id == 0) return 0;
+ i = id - 1;
+ l = heap[i];
+ if (l == 0) return 1;
+ return 1 + ck(l) + ck(heap[i + 1]);
+}
+
+int main() {
+ int which = get_c(), reps = 0, ch = get_c(), r = 0;
+ while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = get_c(); }
+ if (which == 'c') {
+ heap = new(int, WORDS);
+ while (reps > 0) { used = 0; r = ck(build(D)); reps--; }
+ put_n(r); put_c(10);
+ return 0;
+ }
+ while (reps > 0) {
+ if (which == 'b') {
+ node* root = make3(D);
+ r = check3(root);
+ free(d*root);
+ } else {
+ node* root = make(D);
+ r = check(root);
+ free(d*root);
+ }
+ reps--;
+ }
+ put_n(r); put_c(10);
+ return 0;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-27 13:43:12 +0000