diff options
Diffstat (limited to 'src/tools')
| -rwxr-xr-x | src/tools/parity.sh | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/src/tools/parity.sh b/src/tools/parity.sh new file mode 100755 index 0000000..2739a85 --- /dev/null +++ b/src/tools/parity.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# compiles the same programs with both compilers and compares what they print. +# zc0 (stage 0, in c) compiles runtime + program into an image that runs on a +# microcore with the image embedded; the self-hosted compiler compiles the +# same source inside the vm (build/bin/zy < prog.zc). a program zc0 rejects is +# outside its subset and is skipped. for every other program the two outputs +# must be equal. the bootstrap's fixpoint cannot see a language rule both +# compilers got wrong the same way, but this can see one they disagree on. +# +# the image is built the way src/tools/zcc builds a run-only program, on the +# default map so that it behaves like the resident compiler's programs do: +# rt.zc, core.zc, src/runtime/run.zc, a `namespace;` (the program starts in the +# global namespace, as under zy) and the program go through build/bin/zc0, +# build/bin/mkgenome checks the image, and a microcore embeds it (c23 #embed). +# the core is compiled per program (about a second); it is kept under +# build/parity/ and reused until the program, the compilers or the runtime change. +# +# stdin: zy reads the program from stdin and the program then reads what follows +# a backtick that starts a line (see examples/learn/24-backtick.zc); the image +# gets that same text, or nothing when the source has no such line. +# +# tests/parity.known lists the programs that differ on purpose (zc0 has no +# ownership analysis, for one). they are still compared and shown as known, and +# do not fail the run; one that agrees again is reported as stale. +set -e +usage() { + printf '%s\n' 'usage: src/tools/parity.sh [-v] [-t SECONDS] [FILE.zc...]' \ + 'compiles each program with zc0 and with the self-hosted compiler and compares stdout.' \ + 'without FILE the corpus is tests/programs/*.zc, examples/*.zc and examples/learn/*.zc.' \ + 'a program zc0 rejects is skipped and counted. both runs see the same stdin:' \ + 'the text after a backtick line in the source, else nothing.' \ + 'prints one line per mismatch and a summary; exit status 1 when anything differs' \ + '(programs listed in tests/parity.known are shown but do not count).' \ + 'options:' \ + ' -v also list the skipped programs and why' \ + ' -t SECONDS time limit for each run (default 30); a program that hits it on both sides is skipped' \ + ' --help, -h show this help and exit' \ + 'work files are under build/parity/. environment: CC selects the c compiler (one with #embed).' +} +verbose=0; limit=30 +while [ $# -gt 0 ]; do + case "$1" in + --help|-h) usage; exit 0 ;; + -v) verbose=1; shift ;; + -t) [ $# -ge 2 ] || { echo "parity: -t needs a number of seconds" >&2; exit 2; } + case "$2" in ''|*[!0-9]*|0) echo "parity: -t needs a positive number of seconds" >&2; exit 2 ;; esac + limit=$2; shift 2 ;; + --) shift; break ;; + -*) echo "parity: unknown option $1; use --help for usage" >&2; exit 2 ;; + *) break ;; + esac +done +cd "$(dirname "$0")/../.." +for f in "$@"; do [ -f "$f" ] || { echo "parity: no such file: $f" >&2; exit 2; }; done +if [ $# -gt 0 ]; then progs=("$@"); else progs=(tests/programs/*.zc examples/*.zc examples/learn/*.zc); fi + +make -s build/bin/zc0 build/bin/mkgenome build/bin/zy >&2 +CC=${CC:-$(sh src/tools/findcc.sh)} +root=$PWD +ZC0=build/bin/zc0; MKG=build/bin/mkgenome; ZY=build/bin/zy +W=build/parity +KNOWN=tests/parity.known +mkdir -p "$W" +# what every image starts with, and how many lines that is (zc0 counts from the top) +SRCS="src/runtime/rt.zc src/core/core.zc src/runtime/run.zc" +{ cat $SRCS; echo 'namespace;'; } > "$W/prefix.new" +if cmp -s "$W/prefix.new" "$W/prefix.zc"; then rm "$W/prefix.new"; else mv "$W/prefix.new" "$W/prefix.zc"; fi +nprefix=$(wc -l < "$W/prefix.zc") +# what the compiled image depends on besides the program +DEPS="$ZC0 $MKG $W/prefix.zc src/micro/micro.c src/micro/port_linux.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h build/core/map/memmap.h" + +# one line of a program's output: newlines as /, control bytes as ?, cut to 60 columns +short() { + local s + s=$(head -c 400 "$1" | tr '\n' '/' | tr -c '[:print:]' '?') + if [ ${#s} -gt 60 ]; then s="${s:0:60}..."; fi + printf '%s' "$s" +} +stale() { # $1 = built file: missing, or older than a dependency or the program $2? + local d + [ -x "$1" ] || return 0 + for d in $DEPS "$2"; do [ "$d" -nt "$1" ] && return 0; done + return 1 +} +isknown() { [ -f "$KNOWN" ] && grep -q "^$1[ #]" "$KNOWN"; } +reason() { grep "^$1[ #]" "$KNOWN" | head -n1 | sed 's/^[^#]*# *//'; } +# zc0 reports lines of the whole input; the program's own line is what helps +rebase() { + local msg=$1 re='^zc0: line ([0-9]+): (.*)$' + if [[ $msg =~ $re ]]; then msg="zc0: line $(( BASH_REMATCH[1] - nprefix )): ${BASH_REMATCH[2]}"; fi + printf '%s' "$msg" +} + +mkdir -p "$W/root" +for f in *; do [ "$f" = build ] || ln -sfn "$root/$f" "$W/root/$f"; done +compared=0; differ=0; known=0; skipped=0; timedout=0; broken=0 +skips="" +for p in "${progs[@]}"; do + name=${p%.zc}; name=${name//\//_} + d=$W/$name + mkdir -p "$d" + if ! cat "$W/prefix.zc" "$p" | "$ZC0" > "$d/img.bin" 2> "$d/zc0.err"; then + skipped=$((skipped+1)) + skips="$skips$p: $(rebase "$(head -n1 "$d/zc0.err")")"$'\n' + continue + fi + if stale "$d/run" "$p"; then + if ! "$MKG" < "$d/img.bin" > "$d/genome.bin" 2> "$d/build.err"; then + echo "broken: $p: zc0 accepts it but mkgenome rejects the image: $(head -n1 "$d/build.err")" + broken=$((broken+1)); continue + fi + if ! $CC -O2 -w -fwrapv -pthread -DZY_THREADS src/micro/micro.c src/micro/port_linux.c \ + -Isrc/core -Ibuild/core/map -Isrc/micro --embed-dir="$d" -o "$d/run" 2> "$d/build.err"; then + echo "broken: $p: the c compiler failed on the zc0 image: $(head -n1 "$d/build.err")" + broken=$((broken+1)); continue + fi + fi + # the image already holds the program. both sides then read the same stdin + sed -n '/^`/,$p' "$p" | tail -c +2 > "$d/stdin" + # from a directory that shows the repository through links, so that a program that + # reads a file by its path finds it, and one that writes a file leaves it under build/ + find "$W/root" -mindepth 1 -maxdepth 1 ! -type l -exec rm -rf {} + + case $p in /*) pabs=$p ;; *) pabs=$root/$p ;; esac + rc0=0; rcy=0 + ( cd "$W/root" && timeout -k 1 "$limit" "$root/$d/run" < "$root/$d/stdin" > "$root/$d/out0" 2> "$root/$d/err0" ) || rc0=$? + ( cd "$W/root" && timeout -k 1 "$limit" "$root/$ZY" < "$pabs" > "$root/$d/outy" 2> "$root/$d/erry" ) || rcy=$? + if [ $rc0 = 124 ] && [ $rcy = 124 ]; then + timedout=$((timedout+1)) + skips="$skips$p: no result within ${limit}s on either side"$'\n' + continue + fi + compared=$((compared+1)) + msg="" + if [ $rc0 = 124 ] || [ $rcy = 124 ]; then + [ $rc0 = 124 ] && a="(timed out after ${limit}s)" || a="[$(short "$d/out0")]" + [ $rcy = 124 ] && b="(timed out after ${limit}s)" || b="[$(short "$d/outy")]" + msg="$p: zc0 $a, self-hosted $b" + elif ! cmp -s "$d/out0" "$d/outy"; then + msg="$p: zc0 [$(short "$d/out0")], self-hosted [$(short "$d/outy")]" + fi + if [ -z "$msg" ]; then + isknown "$p" && echo "stale: $p agrees now; remove it from $KNOWN" + elif isknown "$p"; then + known=$((known+1)) + echo "known: $msg ($(reason "$p"))" + else + differ=$((differ+1)) + echo "differ: $msg" + fi +done +if [ $verbose = 1 ] && [ -n "$skips" ]; then + echo "skipped:" + printf '%s' "$skips" | sed 's/^/ /' +fi +extra="" +[ $timedout -gt 0 ] && extra="$extra, $timedout skipped (timeout)" +[ $known -gt 0 ] && extra="$extra, $known known ($KNOWN)" +[ $broken -gt 0 ] && extra="$extra, $broken did not build" +echo "parity: $compared compared, $differ differ, $skipped skipped (outside zc0)$extra" +[ $differ -eq 0 ] && [ $broken -eq 0 ] |