diff options
Diffstat (limited to 'tests/bench')
| -rw-r--r-- | tests/bench/arrays.zc | 14 | ||||
| -rw-r--r-- | tests/bench/bench.sh | 17 | ||||
| -rw-r--r-- | tests/bench/suite.py | 63 |
3 files changed, 73 insertions, 21 deletions
diff --git a/tests/bench/arrays.zc b/tests/bench/arrays.zc index f83cebe..bbe0f01 100644 --- a/tests/bench/arrays.zc +++ b/tests/bench/arrays.zc @@ -4,19 +4,19 @@ include rt; // two array kernels where the bounds checks are the point, one whose hot loop the // compiler proves safe and one that keeps its check on purpose. zy --explain-checks // shows which checks stay (suite.py puts the count in the checks column). -// m matmul: c = a * b, 24x24, flat row-major. each row of a and column of b is +// m matmul: c = a * b, 40x40, flat row-major. each row of a and column of b is // copied into a local int[N], and the inner k loop runs k < N over those only, // so its checks are proved. the copies index the flat matrices with i * N + k, // which keeps their checks, but they run N^2 times, not N^3 -// g gather: s += data[idx[i]] over 4096 words. idx[i] is proved by i < len(idx); +// g gather: s += data[idx[i]] over 98304 words. idx[i] is proved by i < len(idx); // data[idx[i]] is indexed by a loaded value, so its check stays in the hot loop // input: a letter (m, g, or z for neither) and how many times to run it. prints a checksum. -const int N = 24; -const int G = 4096; +const int N = 40; +const int G = 98304; int matmul(fixed int[?] a, fixed int[?] b, fixed int[?] c) { - int[24] ra; - int[24] cb; + int[40] ra; + int[40] cb; int s = 0; for (int i = 0; i < N; i++) { for (int k = 0; k < N; k++) ra[k] = a[i * N + k]; @@ -44,7 +44,7 @@ fn main() { for (int i = 0; i < N * N; i++) { a[i] = i % 7 - 3; b[i] = i % 5 - 2; } fixed int[?] data = rt.new(fixed int[?], G); fixed int[?] idx = rt.new(fixed int[?], G); - // 7919 is odd, so i * 7919 mod 4096 visits every slot once, in a scattered order + // 7919 is prime and not 2 or 3, so i * 7919 mod 98304 (2^15 * 3) visits every slot once, in a scattered order for (int i = 0; i < G; i++) { data[i] = i; idx[i] = i * 7919 % G; } while (reps > 0) { if (which == 'm') r = matmul(a, b, c); diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index e532f93..4032aa0 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -24,15 +24,22 @@ if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/ fi # seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run, # best of three each, so starting up and compiling cancel out -t(){ local s e; s=$(date +%s.%N); (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null; e=$(date +%s.%N); echo "$e - $s" | bc; } -best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3); b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; } +# a nonzero zy exit (1 compile error, 2 trap) fails the whole bench, with the stderr tail +t(){ + local s e c err=$(mktemp); s=$(date +%s.%N) + (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null 2>$err; c=$? + e=$(date +%s.%N) + if [ $c != 0 ]; then echo "bench.sh: tests/programs/bench.zc kernel $2 ($3 reps) on $1: zy exited $c; stderr tail:" >&2; tail -n 8 $err >&2; rm -f $err; return 1; fi + rm -f $err; echo "$e - $s" | bc +} +best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3) || return 1; b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; } # grows the run count until the difference is at least a second, so the noise # of starting up is small next to it per(){ local one n=4 many - one=$(best "$1" $2 1) + one=$(best "$1" $2 1) || return 1 while :; do - many=$(best "$1" $2 $n) + many=$(best "$1" $2 $n) || return 1 if [ "$(echo "$many - $one > 1" | bc)" = 1 ] || [ $n -ge 4096 ]; then break; fi n=$((n * 4)) done @@ -42,7 +49,7 @@ printf "%-14s %-8s %10s %10s %8s\n" backend kernel "plain ms" "o3 ms" speedup for i in "${!vms[@]}"; do for k in "collatz a b" "sieve c d" "digits e f" "fib g h"; do set -- $k - p=$(per "${vms[$i]}" $2); o=$(per "${vms[$i]}" $3) + p=$(per "${vms[$i]}" $2) || exit 1; o=$(per "${vms[$i]}" $3) || exit 1 printf "%-14s %-8s %10.2f %10.2f %7.1fx\n" "${names[$i]}" $1 $(echo "$p * 1000" | bc) $(echo "$o * 1000" | bc) $(echo "scale=2; $p / $o" | bc) done done diff --git a/tests/bench/suite.py b/tests/bench/suite.py index fadb5b3..a30f40f 100644 --- a/tests/bench/suite.py +++ b/tests/bench/suite.py @@ -54,6 +54,12 @@ SWEEPS = [ ("zy:arrays.matmul", "tests/bench/arrays.zc", "m"), ("zy:arrays.gather", "tests/bench/arrays.zc", "g"), ] +# what each row prints, when the suite knows it (the checksum doesn't depend on the +# repetition count). a row that prints something else fails the bench +EXPECT = { + "zy:arrays.matmul": "0", + "zy:arrays.gather": "4831789056", +} # where each row's checks are counted: the source and the function (bench.sh rows are # backend:kernel, all from tests/programs/bench.zc) CHECK_FNS = { @@ -89,29 +95,43 @@ def bench_sh(): return rows -def run_once(zy, src, letter, reps): +def zy_failure(what, code, out, err, expect=None): + """None when a zy run is good, else the message: a nonzero exit (1 is a compile error, + a trap will be 2) or output other than the expected checksum""" + tail = lambda b: "\n".join(b.decode(errors="replace").rstrip().splitlines()[-8:]) or "(empty)" + if code != 0: + kind = {1: "compile error", 2: "runtime trap"}.get(code, "failed") + return "bench: %s: zy exited %d (%s); stderr tail:\n%s" % (what, code, kind, tail(err)) + if expect is not None and out.decode(errors="replace").strip() != expect: + return "bench: %s: printed %r, expected %r; stderr tail:\n%s" % (what, out.decode(errors="replace").strip()[-200:], expect, tail(err)) + return None + + +def run_once(zy, src, letter, reps, what="", expect=None): data = src + b"`" + letter.encode() + str(reps).encode() t = time.perf_counter() r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE) dt = time.perf_counter() - t - if r.returncode != 0: - sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300])) + bad = zy_failure("%s (%s, kernel %s, %d reps)" % (what or letter, zy, letter, reps), r.returncode, r.stdout, r.stderr, expect) + if bad: + sys.exit(bad) return dt -def slope_ms(zy, path, letter): +def slope_ms(zy, path, letter, name=""): src = (ROOT / path).read_bytes() + go = lambda reps: run_once(zy, src, letter, reps, name or path, EXPECT.get(name)) n = 4 while True: - one = min(run_once(zy, src, letter, 1) for _ in range(2)) - many = run_once(zy, src, letter, n) + one = min(go(1) for _ in range(2)) + many = go(n) if many - one >= MIN_S or n >= 1 << 16: break n *= 4 slopes = [] for _ in range(TRIALS): - one = run_once(zy, src, letter, 1) - many = run_once(zy, src, letter, n) + one = go(1) + many = go(n) slopes.append((many - one) / (n - 1) * 1000) return statistics.median(slopes) @@ -181,6 +201,9 @@ def region_guard(zy): src = (ROOT / "tests/programs/o3.zc").read_bytes() env = dict(os.environ, ZYTRACE="1") r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + bad = zy_failure("tests/programs/o3.zc (%s)" % zy, r.returncode, r.stdout, r.stderr) + if bad: + sys.exit(bad) err = r.stderr.decode(errors="replace").splitlines() done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l] stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l] @@ -210,7 +233,7 @@ def run_all(zy): print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True) rows.append({"program": name, "plain": plain, "o3": o3, "checks": checks_of(zy, name), "peak": "-", "gc": "-"}) for name, path, letter in SWEEPS: - ms = slope_ms(zy, path, letter) + ms = slope_ms(zy, path, letter, name) print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True) rows.append({"program": name, "plain": ms, "o3": None, "checks": checks_of(zy, name), "peak": "-", "gc": "-"}) o3_guard(rows) @@ -335,6 +358,28 @@ def selftest(): case("checks fell 3 -> 0", two("0"), True), case("checks '-' now (no --explain-checks)", two("-"), False), ] + # zy exits: a fake zy that exits with the given code must fail the row, and a wrong + # checksum must too + def zcase(name, code, out, want): + fake = t / "fakezy" + fake.write_text("#!/bin/sh\ncat >/dev/null\necho %s\necho 'line 3: oops' >&2\nexit %d\n" % (out, code)) + fake.chmod(0o755) + try: + run_once(str(fake), b"", "m", 1, "a:row", "42") + failed, msg = False, "" + except SystemExit as e: + failed, msg = True, str(e.code) + ok = failed == want and (not want or "a:row" in msg) + print("selftest: %-36s %s, want %s: %s" % (name, "failed" if failed else "ok", "fail" if want else "ok", "pass" if ok else "WRONG")) + if not ok or failed and os.environ.get("V") == "1": + print(msg) + return ok + good += [ + zcase("zy exit 0, right checksum", 0, "42", False), + zcase("zy exit 1 (compile error)", 1, "", True), + zcase("zy exit 2 (runtime trap)", 2, "42", True), + zcase("zy exit 0, wrong checksum", 0, "41", True), + ] print("selftest: %s" % ("ok" if all(good) else "FAIL")) return 0 if all(good) else 1 |