diff options
| -rwxr-xr-x | src/tools/parity.sh | 13 | ||||
| -rwxr-xr-x | tests/conformance/run.sh | 25 | ||||
| -rw-r--r-- | tests/parity.known | 3 |
3 files changed, 26 insertions, 15 deletions
diff --git a/src/tools/parity.sh b/src/tools/parity.sh index 2b9b870..2d2086d 100755 --- a/src/tools/parity.sh +++ b/src/tools/parity.sh @@ -21,7 +21,8 @@ # # tests/parity.known lists the programs that differ on purpose (zc0 has no # ownership analysis, for one). they are still compared and shown as known, and -# do not fail the run; one that agrees again is reported as stale. +# do not fail the run; one that agrees again is reported as stale and fails the run, +# so its line is removed once the difference is fixed. set -e usage() { printf '%s\n' 'usage: src/tools/parity.sh [-v] [-t SECONDS] [FILE.zc...]' \ @@ -30,7 +31,8 @@ usage() { 'a program zc0 rejects is skipped and counted. both runs see the same stdin:' \ 'the text after a backtick line in the source, else nothing.' \ 'prints one line per mismatch and a summary; exit status 1 when anything differs' \ - '(programs listed in tests/parity.known are shown but do not count).' \ + '(programs listed in tests/parity.known are shown but do not count) or when a listed' \ + 'program agrees again (stale: remove its line from tests/parity.known).' \ 'options:' \ ' -v also list the skipped programs and why' \ ' -t SECONDS time limit for each run (default 30); a program that hits it on both sides is skipped' \ @@ -98,7 +100,7 @@ mkdir -p "$W/root" # (bareio's note) would otherwise lead the next write back into the repository find "$W/root" -mindepth 1 -maxdepth 1 -type l -delete for f in *; do [ "$f" = build ] || ln -sfn "$root/$f" "$W/root/$f"; done -compared=0; differ=0; known=0; skipped=0; timedout=0; broken=0 +compared=0; differ=0; known=0; stale=0; skipped=0; timedout=0; broken=0 skips="" for p in "${progs[@]}"; do name=${p%.zc}; name=${name//\//_} @@ -144,7 +146,7 @@ for p in "${progs[@]}"; do msg="$p: zc0 [$(short "$d/out0")], self-hosted [$(short "$d/outy")]" fi if [ -z "$msg" ]; then - isknown "$p" && echo "stale: $p agrees now; remove it from $KNOWN" + if isknown "$p"; then stale=$((stale+1)); echo "stale: $p agrees now; remove it from $KNOWN"; fi elif isknown "$p"; then known=$((known+1)) echo "known: $msg ($(reason "$p"))" @@ -161,5 +163,6 @@ extra="" [ $timedout -gt 0 ] && extra="$extra, $timedout skipped (timeout)" [ $known -gt 0 ] && extra="$extra, $known known ($KNOWN)" [ $broken -gt 0 ] && extra="$extra, $broken did not build" +[ $stale -gt 0 ] && extra="$extra, $stale stale" echo "parity: $compared compared, $differ differ, $skipped skipped (outside zc0)$extra" -[ $differ -eq 0 ] && [ $broken -eq 0 ] +[ $differ -eq 0 ] && [ $broken -eq 0 ] && [ $stale -eq 0 ] diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh index c87b142..c351d1f 100755 --- a/tests/conformance/run.sh +++ b/tests/conformance/run.sh @@ -13,13 +13,16 @@ # run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are # understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to # tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run. +# a listed program that passes on that implementation prints as `stale` and fails the run, so +# the line is removed once the failure is fixed. set -u usage() { printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \ 'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \ 'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \ - 'prints ok, FAIL, known or skip per program and implementation, then a summary per section and a total;' \ - 'exit status 1 when anything fails that is not listed in tests/conformance/known.txt.' \ + 'prints ok, FAIL, known, stale or skip per program and implementation, then a summary per section and a total;' \ + 'exit status 1 when anything fails that is not listed in tests/conformance/known.txt, or when a listed' \ + 'program passes (stale: remove its line from known.txt).' \ 'options:' \ ' --zym also run every program on build/bin/zym' \ ' --zc0 also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \ @@ -70,16 +73,20 @@ secs=() fails=0 bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); } -# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run +# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run; +# an ok listed there is stale (the failure is gone but its line is not) and fails the run emit() { # name impl status detail local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]" + elif [ "$st" = ok ] && [ -f "$known" ] && grep -q -m1 "^$name $impl " "$known"; then + st=stale; detail="passes now; remove its line from $known" fi bump "$sec:$impl:$st" case $st in FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;; known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;; + stale) fails=$((fails+1)); echo "stale $impl $name: $detail" ;; ok) [ $quiet = 1 ] || echo "ok $impl $name" ;; skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;; esac @@ -160,7 +167,7 @@ if [ $zc0 = 1 ]; then fi fi -echo "summary (programs, ok, FAIL, known, skip per implementation):" +echo "summary (programs, ok, FAIL, known, stale, skip per implementation):" impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0" tot=0 declare -A itot @@ -172,15 +179,15 @@ for s in "${secs[@]}"; do done tot=$((tot+n)) for i in $impls; do - o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; sk=${cnt["$s:$i:skip"]:-0} - line="$line $i $o/$b/$k/$sk" + o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; st=${cnt["$s:$i:stale"]:-0}; sk=${cnt["$s:$i:skip"]:-0} + line="$line $i $o/$b/$k/$st/$sk" itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b )) - itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk )) + itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:stale"]=$(( ${itot["$i:stale"]:-0} + st )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk )) done printf ' %-24s %3d programs%s\n' "$s" "$n" "$line" done line="" -for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:skip"]:-0}"; done +for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:stale"]:-0}/${itot["$i:skip"]:-0}"; done printf ' %-24s %3d programs%s\n' total "$tot" "$line" -echo "(per implementation: ok/FAIL/known/skip)" +echo "(per implementation: ok/FAIL/known/stale/skip)" [ $fails -eq 0 ] diff --git a/tests/parity.known b/tests/parity.known index 5ddbdf2..37874f9 100644 --- a/tests/parity.known +++ b/tests/parity.known @@ -1,6 +1,7 @@ # programs where zc0 and the self-hosted compiler print different things on purpose. # src/tools/parity.sh still runs them and shows the difference as known, and it does -# not fail the run; when one starts to agree it says so, and its line can go. +# not fail the run; when one starts to agree it is reported as stale and fails the +# run, so its line goes. # one program per line: the path, then # and the reason. # # zc0 reads types and ownership tags and throws them away. it has no ownership |