bench: a nonzero zy exit (1 compile error, 2 trap) or a wrong checksum fails the row with the stderr tail, in suite.py and bench.sh; selftest cases for exit 1, exit 2 and a wrong checksum. arrays rows scaled to ~3 ms a repetition (matmul 40x40, gather 98304 words); the baseline's arrays rows are stale until a re-save

1 files changed, 12 insertions, 5 deletions

diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
index e532f93..4032aa0 100644
--- a/tests/bench/bench.sh
+++ b/tests/bench/bench.sh
@@ -24,15 +24,22 @@ if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/
 fi
 # seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run,
 # best of three each, so starting up and compiling cancel out
-t(){ local s e; s=$(date +%s.%N); (cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null; e=$(date +%s.%N); echo "$e - $s" | bc; }
-best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3); b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
+# a nonzero zy exit (1 compile error, 2 trap) fails the whole bench, with the stderr tail
+t(){
+	local s e c err=$(mktemp); s=$(date +%s.%N)
+	(cat tests/programs/bench.zc; printf '`%s%s' $2 $3) | $1 >/dev/null 2>$err; c=$?
+	e=$(date +%s.%N)
+	if [ $c != 0 ]; then echo "bench.sh: tests/programs/bench.zc kernel $2 ($3 reps) on $1: zy exited $c; stderr tail:" >&2; tail -n 8 $err >&2; rm -f $err; return 1; fi
+	rm -f $err; echo "$e - $s" | bc
+}
+best(){ local b=999 x i; for i in 1 2 3; do x=$(t "$1" $2 $3) || return 1; b=$(echo "if ($x < $b) $x else $b" | bc); done; echo $b; }
 # grows the run count until the difference is at least a second, so the noise
 # of starting up is small next to it
 per(){
 	local one n=4 many
-	one=$(best "$1" $2 1)
+	one=$(best "$1" $2 1) || return 1
 	while :; do
-		many=$(best "$1" $2 $n)
+		many=$(best "$1" $2 $n) || return 1
 		if [ "$(echo "$many - $one > 1" | bc)" = 1 ] || [ $n -ge 4096 ]; then break; fi
 		n=$((n * 4))
 	done
@@ -42,7 +49,7 @@ printf "%-14s %-8s %10s %10s %8s\n" backend kernel "plain ms" "o3 ms" speedup
 for i in "${!vms[@]}"; do
 	for k in "collatz a b" "sieve c d" "digits e f" "fib g h"; do
 		set -- $k
-		p=$(per "${vms[$i]}" $2); o=$(per "${vms[$i]}" $3)
+		p=$(per "${vms[$i]}" $2) || exit 1; o=$(per "${vms[$i]}" $3) || exit 1
 		printf "%-14s %-8s %10.2f %10.2f %7.1fx\n" "${names[$i]}" $1 $(echo "$p * 1000" | bc) $(echo "$o * 1000" | bc) $(echo "scale=2; $p / $o" | bc)
 	done
 done