#!/bin/bash
# runs the language conformance programs (tests/conformance/<section>/<name>.zc) against docs/SPEC.md.
# the first line of each program is its expectation, derived from the spec text alone:
#   // expect: TEXT          the program must compile and print exactly TEXT, newlines written as /
#   // expect error: TEXT    the compiler must reject it, with TEXT as a substring of its output
# directive lines (a line `// zc0: ...` anywhere in the file, by convention the last one):
#   // zc0: skip             outside zc0's subset (appendix a): not run on zc0. a program without
#                            this line that zc0 rejects is a failure of zc0
#   // zc0: error            zc0 must reject the program too (error programs are otherwise
#                            not run on zc0, which has no checks)
# a program may read stdin after a source-ending backtick line, as everywhere else.
# every program runs on build/bin/zy; --zym adds build/bin/zym, --zc0 adds zc0 (through the
# run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are
# understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to
# tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run.
# a listed program that passes on that implementation prints as `stale` and fails the run, so
# the line is removed once the failure is fixed.
set -u
usage() {
    printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
        'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \
        'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \
        'prints ok, FAIL, known, stale or skip per program and implementation, then a summary per section and a total;' \
        'exit status 1 when anything fails that is not listed in tests/conformance/known.txt, or when a listed' \
        'program passes (stale: remove its line from known.txt).' \
        'options:' \
        '  --zym        also run every program on build/bin/zym' \
        '  --zc0        also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \
        '  --zy BINARY  run this core instead of build/bin/zy (tests/test.sh passes its own)' \
        '  -w DIR       keep work files under DIR instead of build/conformance' \
        '  -q           print only FAIL lines and the summary' \
        '  -t SECONDS   time limit for each run (default 20)' \
        '  --help, -h   show this help and exit' \
        'work files are under build/conformance/ and build/parity/.'
}
zym=0; zc0=0; quiet=0; limit=20; args=(); zy=build/bin/zy; W=build/conformance
while [ $# -gt 0 ]; do
    case "$1" in
        --help|-h) usage; exit 0 ;;
        --zym) zym=1; shift ;;
        --zc0) zc0=1; shift ;;
        -q) quiet=1; shift ;;
        --zy) [ $# -ge 2 ] || { echo "run.sh: --zy needs a binary" >&2; exit 2; }; zy=$2; shift 2 ;;
        -w) [ $# -ge 2 ] || { echo "run.sh: -w needs a folder" >&2; exit 2; }; W=$2; shift 2 ;;
        -t) [ $# -ge 2 ] || { echo "run.sh: -t needs a number of seconds" >&2; exit 2; }
            case "$2" in ''|*[!0-9]*|0) echo "run.sh: -t needs a positive number of seconds" >&2; exit 2 ;; esac
            limit=$2; shift 2 ;;
        --) shift; args+=("$@"); break ;;
        -*) echo "run.sh: unknown option $1; use --help for usage" >&2; exit 2 ;;
        *) args+=("$1"); shift ;;
    esac
done
cd "$(dirname "$0")/../.." || exit 2
root=$PWD
conf=tests/conformance
[ ${#args[@]} -gt 0 ] || args=("$conf")
files=()
for a in "${args[@]}"; do
    if [ -d "$a" ]; then
        while IFS= read -r f; do files+=("$f"); done < <(find "$a" -name '*.zc' | sort)
    elif [ -f "$a" ]; then files+=("$a")
    else echo "run.sh: no such file or folder: $a" >&2; exit 2; fi
done
case $zy in /*) ;; *) zy=$PWD/$zy ;; esac
case $W in /*) ;; *) W=$PWD/$W ;; esac
[ -x "$zy" ] || { echo "run.sh: $zy is missing; run make first" >&2; exit 2; }
[ $zym = 0 ] || [ -x build/bin/zym ] || { echo "run.sh: build/bin/zym is missing; run make first" >&2; exit 2; }
mkdir -p "$W"
known=$conf/known.txt

declare -A cnt
secs=()
fails=0
bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); }

# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run;
# an ok listed there is stale (the failure is gone but its line is not) and fails the run
emit() { # name impl status detail
    local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why
    if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then
        st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]"
    elif [ "$st" = ok ] && [ -f "$known" ] && grep -q -m1 "^$name $impl " "$known"; then
        st=stale; detail="passes now; remove its line from $known"
    fi
    bump "$sec:$impl:$st"
    case $st in
        FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;;
        known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;;
        stale) fails=$((fails+1)); echo "stale $impl $name: $detail" ;;
        ok) [ $quiet = 1 ] || echo "ok   $impl $name" ;;
        skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;;
    esac
}

kind_of() { if head -n1 "$1" | grep -q '^// expect error: '; then echo err; else echo out; fi; }
text_of() { head -n1 "$1" | sed -e 's#^// expect error: ##' -e 's#^// expect: ##'; }

# check FILE OUTPUTFILE: sets detail and returns 1 on a mismatch
detail=""
check() {
    local f=$1 out=$2 kind text got
    kind=$(kind_of "$f"); text=$(text_of "$f")
    if [ "$kind" = out ]; then
        got=$(tr '\n' '/' < "$out" | tr -c '[:print:]' '?')
        [ "$got" = "$text" ] && return 0
        detail="expected [$text] got [${got:0:300}]"; return 1
    fi
    grep -qF -- "$text" "$out" && return 0
    detail="expected an error containing [$text] got [$(head -c 300 "$out" | tr '\n' '/')]"; return 1
}

for f in "${files[@]}"; do
    case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
    sec=${name%%/*}
    case " ${secs[*]:-} " in *" $sec "*) ;; *) secs+=("$sec") ;; esac
    if ! head -n1 "$f" | grep -q '^// expect\( error\)\?: '; then
        emit "$name" zy FAIL "no expectation on the first line"; continue
    fi
done

# zy and zym
run_one() { # file name impl binary
    local f=$1 name=$2 impl=$3 bin=$4 d out rc
    d=$W/$(echo "${name%.zc}" | tr '/' '_'); mkdir -p "$d"; out=$d/out.$impl
    ( cd "$d" && timeout -k 1 "$limit" "$bin" < "$root/$f" > "$out" 2>&1 ); rc=$?
    if [ $rc = 124 ]; then emit "$name" "$impl" FAIL "timed out after ${limit}s"; return; fi
    if check "$f" "$out"; then emit "$name" "$impl" ok ""; else emit "$name" "$impl" FAIL "$detail"; fi
}
for f in "${files[@]}"; do
    case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
    head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue
    run_one "$f" "$name" zy "$zy"
    [ $zym = 0 ] || run_one "$f" "$name" zym "$root/build/bin/zym"
done

# zc0 through parity.sh: one call for every program that is meant for it
if [ $zc0 = 1 ]; then
    elig=(); errs=()
    for f in "${files[@]}"; do
        case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
        head -n1 "$f" | grep -q '^// expect\( error\)\?: ' || continue
        if grep -q '^// zc0: skip' "$f"; then emit "$name" zc0 skip "marked // zc0: skip"; continue; fi
        if [ "$(kind_of "$f")" = err ]; then
            if grep -q '^// zc0: error' "$f"; then errs+=("$f"); else emit "$name" zc0 skip "error program, zc0 has no checks"; fi
            continue
        fi
        elig+=("$f")
    done
    if [ ${#elig[@]} -gt 0 ] || [ ${#errs[@]} -gt 0 ]; then
        bash src/tools/parity.sh -t "$limit" "${elig[@]}" > "$W/parity.log" 2>&1
        pre=build/parity/prefix.zc
        for f in "${elig[@]}"; do
            name=${f#"$conf"/}; d=build/parity/$(echo "${f%.zc}" | tr '/' '_')
            if ! cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then
                emit "$name" zc0 FAIL "zc0 rejects a program inside its subset: $(head -n1 "$W/zc0.err" | cut -c1-120)"; continue
            fi
            if grep -q "^broken: $f" "$W/parity.log"; then emit "$name" zc0 FAIL "$(grep "^broken: $f" "$W/parity.log")"; continue; fi
            if [ ! -f "$d/out0" ]; then emit "$name" zc0 FAIL "no output (see $W/parity.log)"; continue; fi
            if check "$f" "$d/out0"; then emit "$name" zc0 ok ""; else emit "$name" zc0 FAIL "$detail"; fi
        done
        for f in "${errs[@]}"; do
            name=${f#"$conf"/}
            if cat "$pre" "$f" | build/bin/zc0 > /dev/null 2> "$W/zc0.err"; then
                emit "$name" zc0 FAIL "zc0 accepted a program the spec says is an error"
            else emit "$name" zc0 ok ""; fi
        done
    fi
fi

echo "summary (programs, ok, FAIL, known, stale, skip per implementation):"
impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0"
tot=0
declare -A itot
for s in "${secs[@]}"; do
    n=0; line=""
    for f in "${files[@]}"; do
        case $f in "$conf"/*) name=${f#"$conf"/} ;; *) name=$f ;; esac
        [ "${name%%/*}" = "$s" ] && n=$((n+1))
    done
    tot=$((tot+n))
    for i in $impls; do
        o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; st=${cnt["$s:$i:stale"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
        line="$line  $i $o/$b/$k/$st/$sk"
        itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b ))
        itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:stale"]=$(( ${itot["$i:stale"]:-0} + st )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
    done
    printf '  %-24s %3d programs%s\n' "$s" "$n" "$line"
done
line=""
for i in $impls; do line="$line  $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:stale"]:-0}/${itot["$i:skip"]:-0}"; done
printf '  %-24s %3d programs%s\n' total "$tot" "$line"
echo "(per implementation: ok/FAIL/known/stale/skip)"
[ $fails -eq 0 ]