conformance: port the previous agent's suite, fix six expectations, add sections 5 to 8 and appendix b, known.txt, zc0 skips, wire into test.sh

1 files changed, 13 insertions, 7 deletions

diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh
index 433b37d..c87b142 100755
--- a/tests/conformance/run.sh
+++ b/tests/conformance/run.sh
@@ -15,7 +15,7 @@
 # tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run.
 set -u
 usage() {
-    printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
+    printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
         'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \
         'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \
         'prints ok, FAIL, known or skip per program and implementation, then a summary per section and a total;' \
@@ -23,18 +23,22 @@ usage() {
         'options:' \
         '  --zym        also run every program on build/bin/zym' \
         '  --zc0        also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \
+        '  --zy BINARY  run this core instead of build/bin/zy (tests/test.sh passes its own)' \
+        '  -w DIR       keep work files under DIR instead of build/conformance' \
         '  -q           print only FAIL lines and the summary' \
         '  -t SECONDS   time limit for each run (default 20)' \
         '  --help, -h   show this help and exit' \
         'work files are under build/conformance/ and build/parity/.'
 }
-zym=0; zc0=0; quiet=0; limit=20; args=()
+zym=0; zc0=0; quiet=0; limit=20; args=(); zy=build/bin/zy; W=build/conformance
 while [ $# -gt 0 ]; do
     case "$1" in
         --help|-h) usage; exit 0 ;;
         --zym) zym=1; shift ;;
         --zc0) zc0=1; shift ;;
         -q) quiet=1; shift ;;
+        --zy) [ $# -ge 2 ] || { echo "run.sh: --zy needs a binary" >&2; exit 2; }; zy=$2; shift 2 ;;
+        -w) [ $# -ge 2 ] || { echo "run.sh: -w needs a folder" >&2; exit 2; }; W=$2; shift 2 ;;
         -t) [ $# -ge 2 ] || { echo "run.sh: -t needs a number of seconds" >&2; exit 2; }
             case "$2" in ''|*[!0-9]*|0) echo "run.sh: -t needs a positive number of seconds" >&2; exit 2 ;; esac
             limit=$2; shift 2 ;;
@@ -54,9 +58,11 @@ for a in "${args[@]}"; do
     elif [ -f "$a" ]; then files+=("$a")
     else echo "run.sh: no such file or folder: $a" >&2; exit 2; fi
 done
-[ -x build/bin/zy ] || { echo "run.sh: build/bin/zy is missing; run make first" >&2; exit 2; }
+case $zy in /*) ;; *) zy=$PWD/$zy ;; esac
+case $W in /*) ;; *) W=$PWD/$W ;; esac
+[ -x "$zy" ] || { echo "run.sh: $zy is missing; run make first" >&2; exit 2; }
 [ $zym = 0 ] || [ -x build/bin/zym ] || { echo "run.sh: build/bin/zym is missing; run make first" >&2; exit 2; }
-W=build/conformance; mkdir -p "$W"
+mkdir -p "$W"
 known=$conf/known.txt
 
 declare -A cnt
@@ -109,15 +115,15 @@ done
 run_one() { # file name impl binary
     local f=$1 name=$2 impl=$3 bin=$4 d out rc
     d=$W/$(echo "${name%.zc}" | tr '/' '_'); mkdir -p "$d"; out=$d/out.$impl
-    ( cd "$d" && timeout -k 1 "$limit" "$root/$bin" < "$root/$f" > "$root/$out" 2>&1 ); rc=$?
+    ( cd "$d" && timeout -k 1 "$limit" "$bin" < "$root/$f" > "$out" 2>&1 ); rc=$?
     if [ $rc = 124 ]; then emit "$name" "$impl" FAIL "timed out after ${limit}s"; return; fi
     if check "$f" "$out"; then emit "$name" "$impl" ok ""; else emit "$name" "$impl" FAIL "$detail"; fi
 }
 for f in "${files[@]}"; do
     case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
     head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue
-    run_one "$f" "$name" zy build/bin/zy
-    [ $zym = 0 ] || run_one "$f" "$name" zym build/bin/zym
+    run_one "$f" "$name" zy "$zy"
+    [ $zym = 0 ] || run_one "$f" "$name" zym "$root/build/bin/zym"
 done
 
 # zc0 through parity.sh: one call for every program that is meant for it