conformance and parity: a known entry that passes again is stale and fails the run

1 files changed, 8 insertions, 5 deletions

diff --git a/src/tools/parity.sh b/src/tools/parity.sh
index 2b9b870..2d2086d 100755
--- a/src/tools/parity.sh
+++ b/src/tools/parity.sh
@@ -21,7 +21,8 @@
 #
 # tests/parity.known lists the programs that differ on purpose (zc0 has no
 # ownership analysis, for one). they are still compared and shown as known, and
-# do not fail the run; one that agrees again is reported as stale.
+# do not fail the run; one that agrees again is reported as stale and fails the run,
+# so its line is removed once the difference is fixed.
 set -e
 usage() {
     printf '%s\n' 'usage: src/tools/parity.sh [-v] [-t SECONDS] [FILE.zc...]' \
@@ -30,7 +31,8 @@ usage() {
         'a program zc0 rejects is skipped and counted. both runs see the same stdin:' \
         'the text after a backtick line in the source, else nothing.' \
         'prints one line per mismatch and a summary; exit status 1 when anything differs' \
-        '(programs listed in tests/parity.known are shown but do not count).' \
+        '(programs listed in tests/parity.known are shown but do not count) or when a listed' \
+        'program agrees again (stale: remove its line from tests/parity.known).' \
         'options:' \
         '  -v          also list the skipped programs and why' \
         '  -t SECONDS  time limit for each run (default 30); a program that hits it on both sides is skipped' \
@@ -98,7 +100,7 @@ mkdir -p "$W/root"
 # (bareio's note) would otherwise lead the next write back into the repository
 find "$W/root" -mindepth 1 -maxdepth 1 -type l -delete
 for f in *; do [ "$f" = build ] || ln -sfn "$root/$f" "$W/root/$f"; done
-compared=0; differ=0; known=0; skipped=0; timedout=0; broken=0
+compared=0; differ=0; known=0; stale=0; skipped=0; timedout=0; broken=0
 skips=""
 for p in "${progs[@]}"; do
     name=${p%.zc}; name=${name//\//_}
@@ -144,7 +146,7 @@ for p in "${progs[@]}"; do
         msg="$p: zc0 [$(short "$d/out0")], self-hosted [$(short "$d/outy")]"
     fi
     if [ -z "$msg" ]; then
-        isknown "$p" && echo "stale: $p agrees now; remove it from $KNOWN"
+        if isknown "$p"; then stale=$((stale+1)); echo "stale: $p agrees now; remove it from $KNOWN"; fi
     elif isknown "$p"; then
         known=$((known+1))
         echo "known: $msg ($(reason "$p"))"
@@ -161,5 +163,6 @@ extra=""
 [ $timedout -gt 0 ] && extra="$extra, $timedout skipped (timeout)"
 [ $known -gt 0 ] && extra="$extra, $known known ($KNOWN)"
 [ $broken -gt 0 ] && extra="$extra, $broken did not build"
+[ $stale -gt 0 ] && extra="$extra, $stale stale"
 echo "parity: $compared compared, $differ differ, $skipped skipped (outside zc0)$extra"
-[ $differ -eq 0 ] && [ $broken -eq 0 ]
+[ $differ -eq 0 ] && [ $broken -eq 0 ] && [ $stale -eq 0 ]