diff options
Diffstat (limited to 'tests/conformance/run.sh')
| -rwxr-xr-x | tests/conformance/run.sh | 180 |
1 files changed, 180 insertions, 0 deletions
diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh new file mode 100755 index 0000000..433b37d --- /dev/null +++ b/tests/conformance/run.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# runs the language conformance programs (tests/conformance/<section>/<name>.zc) against docs/SPEC.md. +# the first line of each program is its expectation, derived from the spec text alone: +# // expect: TEXT the program must compile and print exactly TEXT, newlines written as / +# // expect error: TEXT the compiler must reject it, with TEXT as a substring of its output +# directive lines (a line `// zc0: ...` anywhere in the file, by convention the last one): +# // zc0: skip outside zc0's subset (appendix a): not run on zc0. a program without +# this line that zc0 rejects is a failure of zc0 +# // zc0: error zc0 must reject the program too (error programs are otherwise +# not run on zc0, which has no checks) +# a program may read stdin after a source-ending backtick line, as everywhere else. +# every program runs on build/bin/zy; --zym adds build/bin/zym, --zc0 adds zc0 (through the +# run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are +# understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to +# tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run. +set -u +usage() { + printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \ + 'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \ + 'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \ + 'prints ok, FAIL, known or skip per program and implementation, then a summary per section and a total;' \ + 'exit status 1 when anything fails that is not listed in tests/conformance/known.txt.' \ + 'options:' \ + ' --zym also run every program on build/bin/zym' \ + ' --zc0 also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \ + ' -q print only FAIL lines and the summary' \ + ' -t SECONDS time limit for each run (default 20)' \ + ' --help, -h show this help and exit' \ + 'work files are under build/conformance/ and build/parity/.' +} +zym=0; zc0=0; quiet=0; limit=20; args=() +while [ $# -gt 0 ]; do + case "$1" in + --help|-h) usage; exit 0 ;; + --zym) zym=1; shift ;; + --zc0) zc0=1; shift ;; + -q) quiet=1; shift ;; + -t) [ $# -ge 2 ] || { echo "run.sh: -t needs a number of seconds" >&2; exit 2; } + case "$2" in ''|*[!0-9]*|0) echo "run.sh: -t needs a positive number of seconds" >&2; exit 2 ;; esac + limit=$2; shift 2 ;; + --) shift; args+=("$@"); break ;; + -*) echo "run.sh: unknown option $1; use --help for usage" >&2; exit 2 ;; + *) args+=("$1"); shift ;; + esac +done +cd "$(dirname "$0")/../.." || exit 2 +root=$PWD +conf=tests/conformance +[ ${#args[@]} -gt 0 ] || args=("$conf") +files=() +for a in "${args[@]}"; do + if [ -d "$a" ]; then + while IFS= read -r f; do files+=("$f"); done < <(find "$a" -name '*.zc' | sort) + elif [ -f "$a" ]; then files+=("$a") + else echo "run.sh: no such file or folder: $a" >&2; exit 2; fi +done +[ -x build/bin/zy ] || { echo "run.sh: build/bin/zy is missing; run make first" >&2; exit 2; } +[ $zym = 0 ] || [ -x build/bin/zym ] || { echo "run.sh: build/bin/zym is missing; run make first" >&2; exit 2; } +W=build/conformance; mkdir -p "$W" +known=$conf/known.txt + +declare -A cnt +secs=() +fails=0 +bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); } + +# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run +emit() { # name impl status detail + local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why + if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then + st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]" + fi + bump "$sec:$impl:$st" + case $st in + FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;; + known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;; + ok) [ $quiet = 1 ] || echo "ok $impl $name" ;; + skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;; + esac +} + +kind_of() { if head -n1 "$1" | grep -q '^// expect error: '; then echo err; else echo out; fi; } +text_of() { head -n1 "$1" | sed -e 's#^// expect error: ##' -e 's#^// expect: ##'; } + +# check FILE OUTPUTFILE: sets detail and returns 1 on a mismatch +detail="" +check() { + local f=$1 out=$2 kind text got + kind=$(kind_of "$f"); text=$(text_of "$f") + if [ "$kind" = out ]; then + got=$(tr '\n' '/' < "$out" | tr -c '[:print:]' '?') + [ "$got" = "$text" ] && return 0 + detail="expected [$text] got [${got:0:300}]"; return 1 + fi + grep -qF -- "$text" "$out" && return 0 + detail="expected an error containing [$text] got [$(head -c 300 "$out" | tr '\n' '/')]"; return 1 +} + +for f in "${files[@]}"; do + case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac + sec=${name%%/*} + case " ${secs[*]:-} " in *" $sec "*) ;; *) secs+=("$sec") ;; esac + if ! head -n1 "$f" | grep -q '^// expect\( error\)\?: '; then + emit "$name" zy FAIL "no expectation on the first line"; continue + fi +done + +# zy and zym +run_one() { # file name impl binary + local f=$1 name=$2 impl=$3 bin=$4 d out rc + d=$W/$(echo "${name%.zc}" | tr '/' '_'); mkdir -p "$d"; out=$d/out.$impl + ( cd "$d" && timeout -k 1 "$limit" "$root/$bin" < "$root/$f" > "$root/$out" 2>&1 ); rc=$? + if [ $rc = 124 ]; then emit "$name" "$impl" FAIL "timed out after ${limit}s"; return; fi + if check "$f" "$out"; then emit "$name" "$impl" ok ""; else emit "$name" "$impl" FAIL "$detail"; fi +} +for f in "${files[@]}"; do + case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac + head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue + run_one "$f" "$name" zy build/bin/zy + [ $zym = 0 ] || run_one "$f" "$name" zym build/bin/zym +done + +# zc0 through parity.sh: one call for every program that is meant for it +if [ $zc0 = 1 ]; then + elig=(); errs=() + for f in "${files[@]}"; do + case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac + head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue + if grep -q '^// zc0: skip' "$f"; then emit "$name" zc0 skip "marked // zc0: skip"; continue; fi + if [ "$(kind_of "$f")" = err ]; then + if grep -q '^// zc0: error' "$f"; then errs+=("$f"); else emit "$name" zc0 skip "error program, zc0 has no checks"; fi + continue + fi + elig+=("$f") + done + if [ ${#elig[@]} -gt 0 ] || [ ${#errs[@]} -gt 0 ]; then + bash src/tools/parity.sh -t "$limit" "${elig[@]}" > "$W/parity.log" 2>&1 + pre=build/parity/prefix.zc + for f in "${elig[@]}"; do + name=${f#"$conf"/}; d=build/parity/$(echo "${f%.zc}" | tr '/' '_') + if ! cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then + emit "$name" zc0 FAIL "zc0 rejects a program inside its subset: $(head -n1 "$W/zc0.err" | cut -c1-120)"; continue + fi + if grep -q "^broken: $f" "$W/parity.log"; then emit "$name" zc0 FAIL "$(grep "^broken: $f" "$W/parity.log")"; continue; fi + if [ ! -f "$d/out0" ]; then emit "$name" zc0 FAIL "no output (see $W/parity.log)"; continue; fi + if check "$f" "$d/out0"; then emit "$name" zc0 ok ""; else emit "$name" zc0 FAIL "$detail"; fi + done + for f in "${errs[@]}"; do + name=${f#"$conf"/} + if cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then + emit "$name" zc0 FAIL "zc0 accepted a program the spec says is an error" + else emit "$name" zc0 ok ""; fi + done + fi +fi + +echo "summary (programs, ok, FAIL, known, skip per implementation):" +impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0" +tot=0 +declare -A itot +for s in "${secs[@]}"; do + n=0; line="" + for f in "${files[@]}"; do + case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac + [ "${name%%/*}" = "$s" ] && n=$((n+1)) + done + tot=$((tot+n)) + for i in $impls; do + o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; sk=${cnt["$s:$i:skip"]:-0} + line="$line $i $o/$b/$k/$sk" + itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b )) + itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk )) + done + printf ' %-24s %3d programs%s\n' "$s" "$n" "$line" +done +line="" +for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:skip"]:-0}"; done +printf ' %-24s %3d programs%s\n' total "$tot" "$line" +echo "(per implementation: ok/FAIL/known/skip)" +[ $fails -eq 0 ] |