merge work/xref: stale known-line check for conformance and parity

2 files changed, 18 insertions, 10 deletions

diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh
index c87b142..c351d1f 100755
--- a/tests/conformance/run.sh
+++ b/tests/conformance/run.sh
@@ -13,13 +13,16 @@
 # run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are
 # understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to
 # tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run.
+# a listed program that passes on that implementation prints as `stale` and fails the run, so
+# the line is removed once the failure is fixed.
 set -u
 usage() {
     printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
         'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \
         'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \
-        'prints ok, FAIL, known or skip per program and implementation, then a summary per section and a total;' \
-        'exit status 1 when anything fails that is not listed in tests/conformance/known.txt.' \
+        'prints ok, FAIL, known, stale or skip per program and implementation, then a summary per section and a total;' \
+        'exit status 1 when anything fails that is not listed in tests/conformance/known.txt, or when a listed' \
+        'program passes (stale: remove its line from known.txt).' \
         'options:' \
         '  --zym        also run every program on build/bin/zym' \
         '  --zc0        also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \
@@ -70,16 +73,20 @@ secs=()
 fails=0
 bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); }
 
-# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run
+# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run;
+# an ok listed there is stale (the failure is gone but its line is not) and fails the run
 emit() { # name impl status detail
     local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why
     if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then
         st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]"
+    elif [ "$st" = ok ] && [ -f "$known" ] && grep -q -m1 "^$name $impl " "$known"; then
+        st=stale; detail="passes now; remove its line from $known"
     fi
     bump "$sec:$impl:$st"
     case $st in
         FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;;
         known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;;
+        stale) fails=$((fails+1)); echo "stale $impl $name: $detail" ;;
         ok) [ $quiet = 1 ] || echo "ok   $impl $name" ;;
         skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;;
     esac
@@ -160,7 +167,7 @@ if [ $zc0 = 1 ]; then
     fi
 fi
 
-echo "summary (programs, ok, FAIL, known, skip per implementation):"
+echo "summary (programs, ok, FAIL, known, stale, skip per implementation):"
 impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0"
 tot=0
 declare -A itot
@@ -172,15 +179,15 @@ for s in "${secs[@]}"; do
     done
     tot=$((tot+n))
     for i in $impls; do
-        o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
-        line="$line  $i $o/$b/$k/$sk"
+        o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; st=${cnt["$s:$i:stale"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
+        line="$line  $i $o/$b/$k/$st/$sk"
         itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b ))
-        itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
+        itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:stale"]=$(( ${itot["$i:stale"]:-0} + st )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
     done
     printf '  %-24s %3d programs%s\n' "$s" "$n" "$line"
 done
 line=""
-for i in $impls; do line="$line  $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:skip"]:-0}"; done
+for i in $impls; do line="$line  $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:stale"]:-0}/${itot["$i:skip"]:-0}"; done
 printf '  %-24s %3d programs%s\n' total "$tot" "$line"
-echo "(per implementation: ok/FAIL/known/skip)"
+echo "(per implementation: ok/FAIL/known/stale/skip)"
 [ $fails -eq 0 ]
diff --git a/tests/parity.known b/tests/parity.known
index 5ddbdf2..37874f9 100644
--- a/tests/parity.known
+++ b/tests/parity.known
@@ -1,6 +1,7 @@
 # programs where zc0 and the self-hosted compiler print different things on purpose.
 # src/tools/parity.sh still runs them and shows the difference as known, and it does
-# not fail the run; when one starts to agree it says so, and its line can go.
+# not fail the run; when one starts to agree it is reported as stale and fails the
+# run, so its line goes.
 # one program per line: the path, then # and the reason.
 #
 # zc0 reads types and ownership tags and throws them away. it has no ownership