1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
#!/bin/bash
# runs the language conformance programs (tests/conformance/<section>/<name>.zc) against docs/SPEC.md.
# the first line of each program is its expectation, derived from the spec text alone:
# // expect: TEXT the program must compile and print exactly TEXT, newlines written as /
# // expect error: TEXT the compiler must reject it, with TEXT as a substring of its output
# directive lines (a line `// zc0: ...` anywhere in the file, by convention the last one):
# // zc0: skip outside zc0's subset (appendix a): not run on zc0. a program without
# this line that zc0 rejects is a failure of zc0
# // zc0: error zc0 must reject the program too (error programs are otherwise
# not run on zc0, which has no checks)
# a program may read stdin after a source-ending backtick line, as everywhere else.
# every program runs on build/bin/zy; --zym adds build/bin/zym, --zc0 adds zc0 (through the
# run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are
# understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to
# tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run.
# a listed program that passes on that implementation prints as `stale` and fails the run, so
# the line is removed once the failure is fixed.
set -u
usage() {
printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \
'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \
'prints ok, FAIL, known, stale or skip per program and implementation, then a summary per section and a total;' \
'exit status 1 when anything fails that is not listed in tests/conformance/known.txt, or when a listed' \
'program passes (stale: remove its line from known.txt).' \
'options:' \
' --zym also run every program on build/bin/zym' \
' --zc0 also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \
' --zy BINARY run this core instead of build/bin/zy (tests/test.sh passes its own)' \
' -w DIR keep work files under DIR instead of build/conformance' \
' -q print only FAIL lines and the summary' \
' -t SECONDS time limit for each run (default 20)' \
' --help, -h show this help and exit' \
'work files are under build/conformance/ and build/parity/.'
}
zym=0; zc0=0; quiet=0; limit=20; args=(); zy=build/bin/zy; W=build/conformance
while [ $# -gt 0 ]; do
case "$1" in
--help|-h) usage; exit 0 ;;
--zym) zym=1; shift ;;
--zc0) zc0=1; shift ;;
-q) quiet=1; shift ;;
--zy) [ $# -ge 2 ] || { echo "run.sh: --zy needs a binary" >&2; exit 2; }; zy=$2; shift 2 ;;
-w) [ $# -ge 2 ] || { echo "run.sh: -w needs a folder" >&2; exit 2; }; W=$2; shift 2 ;;
-t) [ $# -ge 2 ] || { echo "run.sh: -t needs a number of seconds" >&2; exit 2; }
case "$2" in ''|*[!0-9]*|0) echo "run.sh: -t needs a positive number of seconds" >&2; exit 2 ;; esac
limit=$2; shift 2 ;;
--) shift; args+=("$@"); break ;;
-*) echo "run.sh: unknown option $1; use --help for usage" >&2; exit 2 ;;
*) args+=("$1"); shift ;;
esac
done
cd "$(dirname "$0")/../.." || exit 2
root=$PWD
conf=tests/conformance
[ ${#args[@]} -gt 0 ] || args=("$conf")
files=()
for a in "${args[@]}"; do
if [ -d "$a" ]; then
while IFS= read -r f; do files+=("$f"); done < <(find "$a" -name '*.zc' | sort)
elif [ -f "$a" ]; then files+=("$a")
else echo "run.sh: no such file or folder: $a" >&2; exit 2; fi
done
case $zy in /*) ;; *) zy=$PWD/$zy ;; esac
case $W in /*) ;; *) W=$PWD/$W ;; esac
[ -x "$zy" ] || { echo "run.sh: $zy is missing; run make first" >&2; exit 2; }
[ $zym = 0 ] || [ -x build/bin/zym ] || { echo "run.sh: build/bin/zym is missing; run make first" >&2; exit 2; }
mkdir -p "$W"
known=$conf/known.txt
declare -A cnt
secs=()
fails=0
bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); }
# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run;
# an ok listed there is stale (the failure is gone but its line is not) and fails the run
emit() { # name impl status detail
local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why
if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then
st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]"
elif [ "$st" = ok ] && [ -f "$known" ] && grep -q -m1 "^$name $impl " "$known"; then
st=stale; detail="passes now; remove its line from $known"
fi
bump "$sec:$impl:$st"
case $st in
FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;;
known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;;
stale) fails=$((fails+1)); echo "stale $impl $name: $detail" ;;
ok) [ $quiet = 1 ] || echo "ok $impl $name" ;;
skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;;
esac
}
kind_of() { if head -n1 "$1" | grep -q '^// expect error: '; then echo err; else echo out; fi; }
text_of() { head -n1 "$1" | sed -e 's#^// expect error: ##' -e 's#^// expect: ##'; }
# check FILE OUTPUTFILE: sets detail and returns 1 on a mismatch
detail=""
check() {
local f=$1 out=$2 kind text got
kind=$(kind_of "$f"); text=$(text_of "$f")
if [ "$kind" = out ]; then
got=$(tr '\n' '/' < "$out" | tr -c '[:print:]' '?')
[ "$got" = "$text" ] && return 0
detail="expected [$text] got [${got:0:300}]"; return 1
fi
grep -qF -- "$text" "$out" && return 0
detail="expected an error containing [$text] got [$(head -c 300 "$out" | tr '\n' '/')]"; return 1
}
for f in "${files[@]}"; do
case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
sec=${name%%/*}
case " ${secs[*]:-} " in *" $sec "*) ;; *) secs+=("$sec") ;; esac
if ! head -n1 "$f" | grep -q '^// expect\( error\)\?: '; then
emit "$name" zy FAIL "no expectation on the first line"; continue
fi
done
# zy and zym
run_one() { # file name impl binary
local f=$1 name=$2 impl=$3 bin=$4 d out rc
d=$W/$(echo "${name%.zc}" | tr '/' '_'); mkdir -p "$d"; out=$d/out.$impl
( cd "$d" && timeout -k 1 "$limit" "$bin" < "$root/$f" > "$out" 2>&1 ); rc=$?
if [ $rc = 124 ]; then emit "$name" "$impl" FAIL "timed out after ${limit}s"; return; fi
if check "$f" "$out"; then emit "$name" "$impl" ok ""; else emit "$name" "$impl" FAIL "$detail"; fi
}
for f in "${files[@]}"; do
case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue
run_one "$f" "$name" zy "$zy"
[ $zym = 0 ] || run_one "$f" "$name" zym "$root/build/bin/zym"
done
# zc0 through parity.sh: one call for every program that is meant for it
if [ $zc0 = 1 ]; then
elig=(); errs=()
for f in "${files[@]}"; do
case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue
if grep -q '^// zc0: skip' "$f"; then emit "$name" zc0 skip "marked // zc0: skip"; continue; fi
if [ "$(kind_of "$f")" = err ]; then
if grep -q '^// zc0: error' "$f"; then errs+=("$f"); else emit "$name" zc0 skip "error program, zc0 has no checks"; fi
continue
fi
elig+=("$f")
done
if [ ${#elig[@]} -gt 0 ] || [ ${#errs[@]} -gt 0 ]; then
bash src/tools/parity.sh -t "$limit" "${elig[@]}" > "$W/parity.log" 2>&1
pre=build/parity/prefix.zc
for f in "${elig[@]}"; do
name=${f#"$conf"/}; d=build/parity/$(echo "${f%.zc}" | tr '/' '_')
if ! cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then
emit "$name" zc0 FAIL "zc0 rejects a program inside its subset: $(head -n1 "$W/zc0.err" | cut -c1-120)"; continue
fi
if grep -q "^broken: $f" "$W/parity.log"; then emit "$name" zc0 FAIL "$(grep "^broken: $f" "$W/parity.log")"; continue; fi
if [ ! -f "$d/out0" ]; then emit "$name" zc0 FAIL "no output (see $W/parity.log)"; continue; fi
if check "$f" "$d/out0"; then emit "$name" zc0 ok ""; else emit "$name" zc0 FAIL "$detail"; fi
done
for f in "${errs[@]}"; do
name=${f#"$conf"/}
if cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then
emit "$name" zc0 FAIL "zc0 accepted a program the spec says is an error"
else emit "$name" zc0 ok ""; fi
done
fi
fi
echo "summary (programs, ok, FAIL, known, stale, skip per implementation):"
impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0"
tot=0
declare -A itot
for s in "${secs[@]}"; do
n=0; line=""
for f in "${files[@]}"; do
case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
[ "${name%%/*}" = "$s" ] && n=$((n+1))
done
tot=$((tot+n))
for i in $impls; do
o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; st=${cnt["$s:$i:stale"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
line="$line $i $o/$b/$k/$st/$sk"
itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b ))
itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:stale"]=$(( ${itot["$i:stale"]:-0} + st )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
done
printf ' %-24s %3d programs%s\n' "$s" "$n" "$line"
done
line=""
for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:stale"]:-0}/${itot["$i:skip"]:-0}"; done
printf ' %-24s %3d programs%s\n' total "$tot" "$line"
echo "(per implementation: ok/FAIL/known/stale/skip)"
[ $fails -eq 0 ]
|
