aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
-rw-r--r--Makefile24
-rw-r--r--ONBOARD.md26
-rw-r--r--TODO.md25
-rw-r--r--examples/actors_demo.zc92
-rwxr-xr-xsrc/tools/parity.sh13
-rw-r--r--tests/bench.baseline12
-rw-r--r--tests/bench/bench.sh14
-rw-r--r--tests/bench/suite.py285
-rw-r--r--tests/bench/sweep.zc23
-rwxr-xr-xtests/conformance/run.sh25
-rw-r--r--tests/parity.known3
12 files changed, 517 insertions, 26 deletions
diff --git a/.gitignore b/.gitignore
index 904cd31..b2d7604 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@
*.o
*.obj
*.exe
+__pycache__/
diff --git a/Makefile b/Makefile
index b3b7d56..94d9b49 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,9 @@
# make xref who writes and who reads each global of the compiler
# (src/tools/xref.sh): build/xref/report.txt and report.json;
# make xref XREF_ARGS=--check compares with tests/xref.baseline
+# make bench time the benchmark programs -> build/bench.txt
+# make bench-save ... and keep the result as tests/bench.baseline
+# make bench-check compare with tests/bench.baseline (or make bench CHECK=1)
# make genome build/core/genome.bin from src/, with the fixpoint check
# make LLVM=1 on x86_64, build zy with the llvm backend instead of the x86-64 jit
# make NOLLVM=1 on other hosts, leave out the llvm backend (in by default when a
@@ -208,6 +211,25 @@ parity: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/zy$(EXE)
xref: $(BIN)/zclsp$(EXE)
bash src/tools/xref.sh $(XREF_ARGS)
+# benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per
+# program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check
+# (or make bench CHECK=1) fails when a column is over 10% and over 0.1 ms slower than the
+# baseline. interp rows only with INTERP=1. bench-selftest checks the verdicts, no timing.
+bench: $(BIN)/zy$(EXE)
+ifeq ($(CHECK),1)
+ INTERP=$(INTERP) python3 tests/bench/suite.py check
+else
+ INTERP=$(INTERP) python3 tests/bench/suite.py run
+endif
+bench-save:
+ BENCH_SAVE=1 $(MAKE) bench
+ cp $(B)/bench.txt tests/bench.baseline
+ @echo "bench: wrote tests/bench.baseline"
+bench-check: $(BIN)/zy$(EXE)
+ INTERP=$(INTERP) python3 tests/bench/suite.py check
+bench-selftest:
+ INTERP=$(INTERP) python3 tests/bench/suite.py selftest
+
# JOBS sets how many sections run at once: make test JOBS=4
test:
bash tests/test.sh -j $(or $(JOBS),1)
@@ -219,4 +241,4 @@ check:
clean:
rm -rf $(B)
-.PHONY: all lsp genome js lspimage vsix web run parity xref test check clean
+.PHONY: bench bench-save bench-selftest bench-check all lsp genome js lspimage vsix web run parity xref test check clean
diff --git a/ONBOARD.md b/ONBOARD.md
index 35d5e35..0f1c95c 100644
--- a/ONBOARD.md
+++ b/ONBOARD.md
@@ -201,6 +201,32 @@ language server is interpreted and slow; `ZC_TEST_TIMEOUT_SCALE` stretches the
drivers' deadlines (the suite sets it). `LLVM_CONFIG` selects an llvm with
headers for the llvm section.
+### benchmarks
+
+`make bench` runs `tests/bench/bench.sh` (it needs `bc`), which times the kernels of
+`tests/programs/bench.zc` (collatz, sieve, digits, fib), plain and o3, on every backend
+it can build here (interpreter, the host's jit, llvm, arm64 under qemu), and adds the
+pointer sweep of `tests/bench/sweep.zc` (checked and unbounded) on `build/bin/zy`. the
+table goes to `build/bench.txt`, one row per backend and kernel, ms per repetition in a
+plain and an o3 column; start-up and compile time cancel out (slopes between one
+repetition and many). the checks column waits for `--explain-checks` and the peak heap
+and gc mark columns for runtime counters that do not exist yet; they show `-`. a full
+run takes about ten minutes.
+`make bench-save` keeps the result as `tests/bench.baseline`, and `make bench-check`
+(or `make bench CHECK=1`) measures again and fails when the plain or the o3 column of a
+row is more than 10% slower and also more than 0.1 ms slower (so sub-ms cells don't
+flake), or a row keeps more checks once both sides have a count. rows missing from a run
+don't fail. interp rows are left out of the table unless `INTERP=1` (bench.sh still times
+the interpreter today, so this doesn't make the run shorter yet). before timing, the suite
+runs `ZYTRACE=1` on `tests/programs/o3.zc` and refuses when zy compiles no o3 region, and
+it refuses when jit o3 collatz isn't under half of jit plain collatz; that is how a baseline of
+interpreter numbers got saved once. `make bench-selftest` checks the pass/fail verdicts on
+doctored tables in a second, and `suite.py check --rows FILE` compares a saved table
+without measuring.
+when something looks slower the whole run is repeated once and each cell keeps its best
+figure. the baseline belongs to the machine it
+was saved on: save a new one after changing machines, and expect noise on a busy host.
+
## 9. history and open work
`docs/history/README.md` is the scrapbook of earlier stages and `TODO.md` the
diff --git a/TODO.md b/TODO.md
index 426555b..37d17ef 100644
--- a/TODO.md
+++ b/TODO.md
@@ -507,8 +507,7 @@ brief someone without re-reading the code.
- [x] remaining zc0 gaps, one commit each on work/xref: string literals have no length limit (the token text grows; `1-source/lit_string_over_4095_bytes.zc`); `.u` on a typed parameter is rejected with the compiler's message, "a parameter tag is .m or .d" (`5-declarations/fn_param_tag_u_rejected.zc`); `x as T` reads a whole type, so `x as int*` and `x as char**` work (`3-expressions/cast_as_pointer_type.zc`); `embed` of an array type inside a function puts the elements in the code and the local holds their address (`tests/programs/embed_local_array.zc`, a parity program); integer literals as the compiler reads them, so `010` is 10, `0x` is 0 and values wrap modulo 2^64 (the eleven class b literal entries left `known.txt`). `docs/history/43-stdlib/notes.md` has a forward note that zc0 no longer reserves u/m/d. the genome is byte-identical to main's.
- [x] verify both compilers against `docs/SPEC.md`: `tests/conformance/<section>/` (sections 1 to 8 plus `9-appendix-b`, 657 programs), runner `tests/conformance/run.sh` (`--zc0`, `--zym`, `--zy`, `-w`), understood failures in `tests/conformance/known.txt` (class a self-hosted bug, b zc0, c spec issue), wired into the interp section of `tests/test.sh`. zy 626 ok, 31 known; zc0 221 ok, 38 known, 398 skipped (outside appendix a). appendix b.2 items 1 to 16 confirmed except b.2.8, which is documented behavior (5.3.2, 3.14.2; the spec wins). top-level `o3 { }` is rejected (4.10.2, class c, left alone). the seven class a bugs are fixed, one commit each: a bad literal in a function's first statement (capture_scan left panic set), `u*d` with visible `u`/`d`, the bstr flag surviving a cast, the 24-parameter limit, `return e;` in a void function (now dropped, yields 0), `memdebug scrub;`, and `case member():` on a member with fields. the conformance run is in `tests/test.sh` but not in `-q`. open: `rt.memreport()` reports buddy capacity (`rt.new(2)` is 3 words), whether that is the spec's W; whether top-level `unchecked;` is in the zc0 subset.
-- [ ] fix the implementation bugs listed in `docs/SPEC.md` appendix b, once confirmed: enum members can't reference earlier members; a plain function redefinition silently shadows the first; a prototype after the definition hides it; `?:` with one int and one float branch gives garbage; `bool` not normalised; float global initializers rejected; `include std;` inside a function body; syntax-error recovery hiding later errors; unterminated `/*` and string literals; `x as int * y`; prototypes needing parameter names; `len(char*)` with typecheck off; duplicate `case` labels; local `const` in constant expressions; constant expressions without precedence (`1+2*3` is 9). (`memdebug scrub;` is fixed; a global initializer naming another global giving its address is documented behavior, 5.3.2.) fix the prose items the appendix lists too.
-- [ ] (partly done, see "agent work at the end of the session") a cross-reference report: for every global of the compiler, the functions that write it and the ones that only read it. build it from the language server's references (`textDocument/references` on `build/bin/zclsp`, so scoping is exact) and classify each reference as a write when an assignment operator, `++` or `--` follows it. output under `build/xref/`; a committed baseline and a `--check` mode that fails when a new writer of a global appears, so it can run in ci.
+- [ ] fix the implementation bugs still open in `docs/SPEC.md` appendix b: unterminated string literals are only reported as a later syntax error (b.2.11 fixed the `/*` half); duplicate `case` labels are not diagnosed (b.2.15); a local `const` cannot be used in a constant expression (b.2.16, b.1.7); constant expressions have no precedence, so `1+2*3` is 9 (b.1.3); syntax-error recovery hides errors in the skipped statements (b.2.10, by design; decide whether to keep it). fixed and merged from work/spec: enum members naming earlier members (b.2.2), function redefinition (b.2.3), prototype after the definition (b.2.4), `?:` with int and float branches (b.2.5), `bool` normalisation (b.2.6), float global initializers (b.2.7, integer constants too via 46825f2), `include std;` in a function body (b.2.9), unterminated `/*` (b.2.11), `x as int * y` settled as a syntax error (b.2.12, doc only), unnamed prototype parameters (b.2.13), `len(char*)` with typecheck off (b.2.14). fix the prose items appendix b.1 lists too (verify which are still open).
- [ ] optional, low priority: plain enums and `?:` in zc0 (both are small and produce ordinary code), so parity covers more programs. no structs in zc0 (decided).
- [ ] group the compiler's parallel per-local arrays (`lname`, `ltype`, `lstate`, `lhome`, `lown`, `lslot`, `ltag`, `lpos`, `lnsz`, `lleaf`, ...) without structs: declare each family together in one place, with one comment block stating each array's meaning and invariant (for example when `lhome` is meaningful), route snapshot and restore of the family through one pair of functions, and use the cross-reference report to confirm nothing else writes them. then symbols and types the same way. fixpoint and analyzer behavior unchanged.
- [ ] continue removing `unchecked` from the runtime. a reverted trial showed: `copy_w`/`fill_w` pass the analyzer when their gate stores go through `syscall(SYS_COPY, ...)`, `dgrow` with `own(w, it)` instead of `w[-2] = it`, `dpush` with `own` before the store, `dnew` as is; but the checked versions changed behavior (7 program checks failed, `tests/programs/share.zc` hung with a checked `cons`), so checked code inserts cleanup these functions can't afford. each needs its contract taught to the analyzer. `free`, `recycle`, `recycle_leaf`, `gc` are the allocator and collector the analyzer's own cleanup calls: document them as the irreducible escapes. `src/lib/actor.zc` has about 20 more.
@@ -520,22 +519,36 @@ brief someone without re-reading the code.
- [x] (done on work/line-tables, see "agent work at the end of the session") runtime source positions. architecture: at each statement start (`stmt()`) record (code address `here`, line); per function a permanent data block [entry, end, name, file, count, pairs...] linked into a runtime list (`rt.line_blocks`), or appended to run-only images by `build_image` like the symbol blob. `lines on;`/`lines off;` like `docs on;`; on for user programs, off for the genome unless the growth is small. a new syscall op returns the return-stack pointer (the return stack is in vm memory, 0x1A4100..0x1B40FF, growing up); implement it in `src/core/core.c`, `src/micro/micro.c`, `src/js/tinyvm.ts` and the wasm port. `rt.trace()` walks the return stack, printing ` at file:line in function` for words inside a known function range. the runtime's fatal reports (`oob`, bad heap pointer, double free, out of memory, frame overflow) print the trace after their unchanged first line. check mode skips all of it.
- [ ] borrow returns tied to an argument. inference already works for a callee defined before the call: `char* skip(char* s) { ...; return s + i; }` makes `let r = skip(t); rt.free(t); r[0]` an error. the gap: when the call comes before the callee's definition, nothing is reported and the program reads freed memory. extend the deferred effect requirements (`effect_require`, `effect_finish` at unit end) so a forward-called function whose return turns out to alias a parameter produces the same error. for bodyless declarations, externs and function types, add an unobtrusive spelling, for example a `borrows s` clause after the parameter list (`char* skip(char* s) borrows s;`), checked against the body when one exists.
- [ ] variadic arguments and `std.print`. design: a pack is passed as one hidden argument, a caller-built read-only growable-array handle (`fixed T[?]` view: [len, cap, items, element size] in the caller's frame, so `len(a)` and `a[i]` work unchanged and nothing is allocated). `...int a` is a typed pack; several packs per function when their element types differ (`void f(...int a, ...char b)`): each extra argument goes to the pack whose element type matches its static type, an ambiguous match is an error, packs come after the fixed parameters. `...any args` packs each value with a kind code (int, float, char, string, bstr, pointer, ...) and its type record: `any` is a small built-in record { kind, value, type }. `...*any args` makes the call comma-free: after the fixed arguments, the parser reads unary expressions until `)`, commas optional, so `std.print("x = " name ", age " age "\n");` works and binary expressions must be parenthesized. `std.print(...*any args)` prints each argument by its kind (`put_n`, the float printer, `put_s`, `put_bs`, a character for `char`, hex for pointers).
-- [ ] (inlining and o3 regions done on work/jit2) x86-64 jit phase 2: compile o3 regions to x86-64 (port `rcompile` and the region register allocation; regions now run on the region interpreter and are 2 to 4 times slower than their plain versions on this jit) and inline enhancements (lower the optimized micro-ops to x86 registers as the arm64 path does, instead of calling `xrun`). validate with `tests/drivers/jit_x64_test.sh` and `tests/bench/bench.sh`, which now has an x86-64 jit column. add the driver to `tests/test.sh` on x86-64 linux.
- done so far: enhancements are inlined (`x64_spec_enh`, `x64_enh`, `x64_uop` in core.c; seven value registers rsi, rdi, r8..r11, rbp; rax, rcx, rdx scratch; floats, byte loads and stores, native calls, popcnt and f2s/s2f still go through `xrun`; `ZYNOENH=1` calls everything as before). bench.sh plain kernels on the x86-64 jit, ms before/after: collatz 49.1/32.1, sieve 4.4/4.1, digits 41.9/14.7, fib 0.83/0.77; o3 unchanged (regions still interpreted). a 256-word pointer sweep with bounds checks 291 to 72 ms. `tests/programs/enh_ops.zc` covers every lowered op on edge values; jit_x64_test runs in test.sh's jit section (not in the quick path). o3 regions now compile (`rjit_x64`, `x64_ralloc`, `remit_x64` in core.c): vars stay in their frame slots, values in six registers, calls go through x_disp; `ZYO3INTERP=1` keeps the region interpreter. bench.sh o3 kernels on the x86-64 jit, ms interpreted/compiled: collatz 289/10.4, sieve 8.7/0.40, digits 89/3.0, fib 3.0/0.92 (fib is still slower than its plain version: calls and returns go through the dispatch stub). still interpreted: regions with more than six live values (one in tests/programs/o3.zc) or native calls. later: vars in registers across a loop, direct o3 calls, floats in xmm registers, spilling instead of giving up. next: the loop-entry flag.
- [x] `memdebug`'s `obj_base` used to search only 64 words back for a block header, so `buf[255]` on a 256-word block reported "bad heap pointer". it now rounds the pointer down to each size class's alignment (largest first) and takes the first header whose check word, size and class agree and whose payload holds the pointer, so interiors at any depth work, a deep pointer into a freed block reports use after free, and header words and free space still report "bad heap pointer". regression: `tests/programs/memdebug_interior.zc` (in `tests/test.sh` beside the other memdebug checks). note that a stray pointer that lands inside another live block is accepted, since memdebug cannot know which block it came from.
- [ ] (partly done, see "agent work at the end of the session") `defer` follow-ups (small): accept `defer if (...) ...;` and other single statements besides an expression; decide whether deferred code may name an owner declared after the `defer` (now rejected by the lexical rule; one line in `defer_one`).
-- [ ] build the llvm core by default when `llvm-config` is available (the makefile falls back to the interpreter otherwise), so x86 hosts get a native tier and the language server stops being interpreted; keep `ZYNOLLVM=1` and `-DZYINTERP` as the opt-outs.
- [ ] stack-effect checking: every function and `asm` block declares or is inferred to have a fixed data-stack effect, and the compiler reports a block whose paths leave the stack unbalanced. the first step of a proving scaffold.
- [ ] declared side effects at module boundaries (globals touched, syscalls or not), checked against the effect masks the compiler already computes.
- [ ] `requires`/`ensures` contracts on functions, compiled to run-time checks under a switch, with the cases the value facts settle (constants, simple ranges) checked statically.
- [ ] (partly done, see "agent work at the end of the session") record and replay: a core option that logs every syscall gate result to a file and replays a run from the log exactly, for reproducing failures and adding tracing after the fact.
-- [ ] a debug adapter for vscode (breakpoints, stepping, locals) on the interpreter, built on the line tables.
+- [ ] a debug adapter for vscode (breakpoints, stepping, locals) on the interpreter, built on the line tables. goes to whoever finishes the bounds hints.
- [ ] reduce the cost of word-per-character strings: measure, then decide between making `bstr` the default string, unifying the two apis behind one name, or both.
- [ ] continue fuzzing the compiler, analyzer, runtime, standard library, lsp, build tools and generated artifacts, including analyzer properties, memory-limit edges, compiler image mutation and long-running lsp memory use; feed generated programs to `make parity` for differential fuzzing between zc0 and the self-hosted compiler; fix discovered bugs.
- [ ] keep improving the borrow checker: fewer annotations in ordinary code, clearer messages naming both the owner and the conflicting use, and new fixtures for each rule that changes.
+- [ ] x86-64 jit, phase 2 follow-ups (phase 2 is in the done list). we are not chasing llvm parity on the jit (owner's decision).
+ - region 474..692 in `tests/programs/o3.zc` falls back to the region interpreter; find out why and compile it.
+ - fib is slightly slower under o3 than plain (0.86 against 0.78 ms), likely region entry and exit on every recursive call; related to direct o3-to-o3 calls.
+ - a compiled o3 region that stores into its own code finishes the current activation on the old code; the change takes effect on the next entry. either document it or bail to the region interpreter when the hook drops the running region.
+- [ ] (in progress on work/exactsize) exact-size bounds checks: n is stored in p[-4] bits 32..62 and the check is 2 loads.
+- [ ] (in progress on work/checks) loop-entry check hoisting, dropping checks proven safe by `len()` or a known size, and `--explain-checks` v1 json: header `{"version":1}`, then one line per kept check with file, line, col, end_line, end_col, fn, kind, loop_depth, reason, fix and an optional edit.
+- [ ] (in progress on work/bench) `make bench`, `bench-check` and `bench-selftest`: baseline from ec49c6d, jit and llvm rows, interp rows only with `INTERP=1`. follow-up: fill the checks column from `--explain-checks` and re-save the baseline; the rule that fails when the check count rises is already in bench-check.
+- [ ] lsp and vscode bounds hints: diagnostics for kept checks with an all / loops / off setting, quick fixes, and a hot-checks command; the vscode extension supports all of it. later: `o3-candidate` hints for hot loops outside o3.
+- [ ] hoisting v2: use the u/m/d ownership tags to prove a loop can't free a pointer (v1 refuses to hoist when the loop body has any free, release or realloc, or any call that might free).
+- [ ] gc roots change: its own branch, memdebug scrub, full suite, bench comparison, and the owner reviews it before merge.
+- [ ] shrink the 500 genome words added by b.2.13.
+- [ ] file-level `o3;` and `unchecked;` directives that apply to every function after them, like `memdebug scrub;`. they replace the deferred top-level `o3 {}` and `unchecked {}` blocks.
- [ ] run the full suite once everything above is done (`make test JOBS=4`; `make check` is the quick one), fix what it finds, and only then tick this item.
## done in the sorted list
+- [x] a cross-reference report: for every global of the compiler, the functions that write it and the ones that only read it. build it from the language server's references (`textDocument/references` on `build/bin/zclsp`, so scoping is exact) and classify each reference as a write when an assignment operator, `++` or `--` follows it. output under `build/xref/`; a committed baseline and a `--check` mode that fails when a new writer of a global appears, so it can run in ci. done: `--check` runs in `tests/test.sh` through `xref_selftest.sh` against `tests/xref.baseline`.
+- [x] x86-64 jit phase 2: compile o3 regions to x86-64 (port `rcompile` and the region register allocation; regions now run on the region interpreter and are 2 to 4 times slower than their plain versions on this jit) and inline enhancements (lower the optimized micro-ops to x86 registers as the arm64 path does, instead of calling `xrun`). validate with `tests/drivers/jit_x64_test.sh` and `tests/bench/bench.sh`, which now has an x86-64 jit column. add the driver to `tests/test.sh` on x86-64 linux. done: inline enhancements merged as 1f4aceb, compiled o3 regions merged as ec49c6d.
+ done so far: enhancements are inlined (`x64_spec_enh`, `x64_enh`, `x64_uop` in core.c; seven value registers rsi, rdi, r8..r11, rbp; rax, rcx, rdx scratch; floats, byte loads and stores, native calls, popcnt and f2s/s2f still go through `xrun`; `ZYNOENH=1` calls everything as before). bench.sh plain kernels on the x86-64 jit, ms before/after: collatz 49.1/32.1, sieve 4.4/4.1, digits 41.9/14.7, fib 0.83/0.77; o3 unchanged (regions still interpreted). a 256-word pointer sweep with bounds checks 291 to 72 ms. `tests/programs/enh_ops.zc` covers every lowered op on edge values; jit_x64_test runs in test.sh's jit section (not in the quick path). o3 regions now compile (`rjit_x64`, `x64_ralloc`, `remit_x64` in core.c): vars stay in their frame slots, values in six registers, calls go through x_disp; `ZYO3INTERP=1` keeps the region interpreter. bench.sh o3 kernels on the x86-64 jit, ms interpreted/compiled: collatz 289/10.4, sieve 8.7/0.40, digits 89/3.0, fib 3.0/0.92 (fib is still slower than its plain version: calls and returns go through the dispatch stub). still interpreted: regions with more than six live values (one in tests/programs/o3.zc) or native calls. later: vars in registers across a loop, direct o3 calls, floats in xmm registers, spilling instead of giving up. next: the loop-entry flag.
+- [x] llvm by default (decided): llvm is turned on automatically on non-x86-64 hosts; on x86-64 the jit stays the default and llvm is opt-in with `LLVM=1`.
+- [x] stale known-line check: conformance and parity fail when a known entry passes again (merged in c1072ab).
- [x] `defer expr;` and `defer { ... }`: tokens grabbed at the `defer`, recompiled inline at every exit (block end through `block_close`, `return` after its value, `break`/`continue` for entries deeper than the loop), newest first; only defers reached so far register. names are lexical (a local declared after the `defer` is hidden in the replay). a deferred `rt.free(p)` is `p`'s transfer to the analyzer. each entry replays in the analyzer mode it was written in; closures get their own base through `ctx_push`. `return`, `break` and `continue` that would leave a deferred statement are rejected. a failed entry reports once. limit 256 pending per function (`DFR_MAX`). spec 4.14, learn section 34, `tests/programs/defer.zc`, seven reject fixtures.
- [x] four small fixes: hover on standard-library members (`hover_embedded_sym` sends std/icon symbols to the library lookup; `bs_hover` renders a member hit), `capture.end`'s terminator (the buffer has `cap + 1` words; latent, since the allocator's power-of-two slack hid it; `tests/programs/capture_long.zc`), bare sibling static calls inside a struct's methods (`mstruct` set for every non-generic struct; the struct's own constant or static beats a top-level name, a local beats both; `tests/programs/struct_static_call.zc`, spec 5.5.5), and `void main()` in run-only images (`run.zc` declares an untyped `fn main();`; spec 5.2.5).
- [x] a hand-written x86-64 jit, phase 1 (`ZYJIT_X64` in `src/core/core.c`, default on x86-64 linux; `-DZYINTERP` and `make build/bin/zyi` give the interpreter). r12 memory, r13 data stack, r14 return stack, r15 table, rbx state; stack slots addressed with a virtual offset so r13 moves only at calls and exits; shares the table, coverage map, store hook and invalidation with the arm64 jit. enhancements are called through `xrun`, o3 regions run on the region interpreter (`ZYNOO3=1` runs their vm code through the jit instead). 179 of 179 programs agree with the interpreter (`tests/drivers/jit_x64_test.sh`), the self-compile is byte-identical and about 7x faster (1.5 s against 11 s), and the rejection, learn, icon, actor, lsp and hint suites pass on it.
diff --git a/examples/actors_demo.zc b/examples/actors_demo.zc
new file mode 100644
index 0000000..656e131
--- /dev/null
+++ b/examples/actors_demo.zc
@@ -0,0 +1,92 @@
+namespace;
+include rt, actor;
+typecheck on;
+
+// actors demo: a ring of isolated worker processes passes a token around,
+// then a fan-out of workers crunches collatz ranges in parallel.
+// run: build/bin/zy < examples/actors_demo.zc
+
+const N = 6, LAPS = 3, W = 4;
+
+void color(int c) { rt.put_c(27); rt.put_s("[1;"); rt.put_n(30 + c); rt.put_c('m'); return; }
+void plain() { rt.put_c(27); rt.put_s("[0m"); return; }
+void bar(int n, int c) { color(c); for (int i = 0; i < n; i++) rt.put_s("#"); plain(); return; }
+
+// one stage of the ring: mixes the token and counts its own hops.
+// `seen` lives in the worker, so the caller's copy stays 0.
+actor Hop {
+ int seen;
+ int handle(int tok) {
+ this.seen += 1;
+ return (tok * 31 + this.seen * 7) % 9973;
+ }
+}
+
+// a cruncher: longest collatz chain that starts in [lo, lo + 20000)
+actor Crunch {
+ int handle(int lo) {
+ int best = 0;
+ for (int s = lo; s < lo + 20000; s++) {
+ int n = s, k = 0;
+ while (n != 1) { if (n % 2) n = 3 * n + 1; else n = n / 2; k++; }
+ if (k > best) best = k;
+ }
+ return best;
+ }
+}
+
+fn main() {
+ color(5); rt.put_s("== tinyvm actors: isolated workers, typed channels ==\n"); plain();
+
+ // the ring: the receive callback of stage i forwards to stage i + 1
+ Hop*[6] ring;
+ for (int i = 0; i < N; i++) { ring[i] = rt.new(Hop); ring[i].spawn(); }
+ Hop** rp = &ring[0]; // closures capture a pointer, not the array
+ let hops = 0, last = 0, done = 0;
+ for (int i = 0; i < N; i++) {
+ let me = i;
+ ring[i].recv(fn(int v) {
+ hops++; last = v;
+ color(1 + me % 6); rt.put_s(" stage "); rt.put_n(me); plain();
+ rt.put_s(" -> token "); rt.put_n(v); rt.put_c(' ');
+ bar(v % 40 + 1, 1 + me % 6); rt.put_c(10);
+ if (hops < N * LAPS) {
+ if (me == N - 1) { color(3); rt.put_s(" -- lap "); rt.put_n(hops / N); rt.put_s(" done --\n"); plain(); }
+ rp[(me + 1) % N].send(v);
+ } else done = 1;
+ return 0;
+ });
+ }
+ color(2); rt.put_s("ring of "); rt.put_n(N); rt.put_s(" actors, "); rt.put_n(LAPS); rt.put_s(" laps, seed 42\n"); plain();
+ ring[0].send(42);
+ while (!done) for (int i = 0; i < N; i++) ring[i].poll();
+ for (int i = 0; i < N; i++) ring[i].close();
+
+ // fan-out: W workers in parallel, results arrive as they finish
+ color(2); rt.put_s("\nfan-out: "); rt.put_n(W); rt.put_s(" crunchers, collatz over 80000 starts\n"); plain();
+ Crunch*[4] pool;
+ let answers = 0, best = 0;
+ for (int i = 0; i < W; i++) {
+ pool[i] = rt.new(Crunch); pool[i].spawn();
+ let me = i;
+ pool[i].recv(fn(int k) {
+ answers++; if (k > best) best = k;
+ color(4); rt.put_s(" worker "); rt.put_n(me); plain();
+ rt.put_s(" ["); rt.put_n(me * 20000 + 1); rt.put_s(".."); rt.put_n(me * 20000 + 20000); rt.put_s("] longest chain ");
+ color(6); rt.put_n(k); plain(); rt.put_c(' '); bar(k / 8, 4 + me % 3); rt.put_c(10);
+ return 0;
+ });
+ pool[i].send(i * 20000 + 1);
+ }
+ for (int i = 0; i < W; i++) pool[i].await();
+ for (int i = 0; i < W; i++) pool[i].close();
+
+ color(5); rt.put_s("\n== summary ==\n"); plain();
+ rt.put_s(" ring hops: "); color(3); rt.put_n(hops); plain(); rt.put_c(10);
+ rt.put_s(" final token: "); color(3); rt.put_n(last); plain(); rt.put_c(10);
+ rt.put_s(" worker replies: "); color(3); rt.put_n(answers); plain(); rt.put_c(10);
+ rt.put_s(" longest chain: "); color(3); rt.put_n(best); plain(); rt.put_c(10);
+ rt.put_s(" caller's ring[0].seen: "); color(3); rt.put_n(ring[0].seen); plain();
+ rt.put_s(" (state stayed in the worker)\n");
+ return 0;
+}
diff --git a/src/tools/parity.sh b/src/tools/parity.sh
index 2b9b870..2d2086d 100755
--- a/src/tools/parity.sh
+++ b/src/tools/parity.sh
@@ -21,7 +21,8 @@
#
# tests/parity.known lists the programs that differ on purpose (zc0 has no
# ownership analysis, for one). they are still compared and shown as known, and
-# do not fail the run; one that agrees again is reported as stale.
+# do not fail the run; one that agrees again is reported as stale and fails the run,
+# so its line is removed once the difference is fixed.
set -e
usage() {
printf '%s\n' 'usage: src/tools/parity.sh [-v] [-t SECONDS] [FILE.zc...]' \
@@ -30,7 +31,8 @@ usage() {
'a program zc0 rejects is skipped and counted. both runs see the same stdin:' \
'the text after a backtick line in the source, else nothing.' \
'prints one line per mismatch and a summary; exit status 1 when anything differs' \
- '(programs listed in tests/parity.known are shown but do not count).' \
+ '(programs listed in tests/parity.known are shown but do not count) or when a listed' \
+ 'program agrees again (stale: remove its line from tests/parity.known).' \
'options:' \
' -v also list the skipped programs and why' \
' -t SECONDS time limit for each run (default 30); a program that hits it on both sides is skipped' \
@@ -98,7 +100,7 @@ mkdir -p "$W/root"
# (bareio's note) would otherwise lead the next write back into the repository
find "$W/root" -mindepth 1 -maxdepth 1 -type l -delete
for f in *; do [ "$f" = build ] || ln -sfn "$root/$f" "$W/root/$f"; done
-compared=0; differ=0; known=0; skipped=0; timedout=0; broken=0
+compared=0; differ=0; known=0; stale=0; skipped=0; timedout=0; broken=0
skips=""
for p in "${progs[@]}"; do
name=${p%.zc}; name=${name//\//_}
@@ -144,7 +146,7 @@ for p in "${progs[@]}"; do
msg="$p: zc0 [$(short "$d/out0")], self-hosted [$(short "$d/outy")]"
fi
if [ -z "$msg" ]; then
- isknown "$p" && echo "stale: $p agrees now; remove it from $KNOWN"
+ if isknown "$p"; then stale=$((stale+1)); echo "stale: $p agrees now; remove it from $KNOWN"; fi
elif isknown "$p"; then
known=$((known+1))
echo "known: $msg ($(reason "$p"))"
@@ -161,5 +163,6 @@ extra=""
[ $timedout -gt 0 ] && extra="$extra, $timedout skipped (timeout)"
[ $known -gt 0 ] && extra="$extra, $known known ($KNOWN)"
[ $broken -gt 0 ] && extra="$extra, $broken did not build"
+[ $stale -gt 0 ] && extra="$extra, $stale stale"
echo "parity: $compared compared, $differ differ, $skipped skipped (outside zc0)$extra"
-[ $differ -eq 0 ] && [ $broken -eq 0 ]
+[ $differ -eq 0 ] && [ $broken -eq 0 ] && [ $stale -eq 0 ]
diff --git a/tests/bench.baseline b/tests/bench.baseline
new file mode 100644
index 0000000..1ff5561
--- /dev/null
+++ b/tests/bench.baseline
@@ -0,0 +1,12 @@
+# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available
+program plain_ms o3_ms checks peak_heap_words gc_mark_ms
+x86-64-jit:collatz 36.240 11.720 - - -
+x86-64-jit:sieve 4.910 0.440 - - -
+x86-64-jit:digits 16.900 3.380 - - -
+x86-64-jit:fib 0.830 0.950 - - -
+llvm:collatz 60.050 3.680 - - -
+llvm:sieve 14.640 0.220 - - -
+llvm:digits 29.240 1.630 - - -
+llvm:fib 3.040 0.170 - - -
+zy:sweep.checked 3.937 - - - -
+zy:sweep.unbounded 2.800 - - - -
diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
index 0a9acef..e532f93 100644
--- a/tests/bench/bench.sh
+++ b/tests/bench/bench.sh
@@ -4,14 +4,20 @@
cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT
CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1
make -s build/core/genome.bin || exit 1
-$CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy
-vms=("$T/zy"); names=("interp")
+# the interpreter core only with INTERP=1 (it is slow, and the rows are rarely wanted)
+vms=(); names=()
+if [ "$INTERP" = 1 ]; then
+ $CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy || exit 1
+ vms+=("$T/zy"); names+=("interp")
+fi
# the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions compiled; ZYO3INTERP=1 runs them on the region interpreter)
if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then
$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); }
fi
-if command -v llvm-config >/dev/null; then
- $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
+# the llvm: LLVM_CONFIG when given, else llvm-config, else llvm-config-19 (as in the Makefile)
+LC=${LLVM_CONFIG:-$(command -v llvm-config || command -v llvm-config-19)}
+if [ -n "$LC" ]; then
+ $CC -O2 -w -DJTYPE=int -DZYLLVM $($LC --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $($LC --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
fi
if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then
aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); }
diff --git a/tests/bench/suite.py b/tests/bench/suite.py
new file mode 100644
index 0000000..3816cc3
--- /dev/null
+++ b/tests/bench/suite.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""the benchmark suite behind `make bench`, `make bench-save` and `make bench-check`.
+
+usage: tests/bench/suite.py run [--zy PATH] [--out FILE]
+ tests/bench/suite.py check [--zy PATH] [--baseline FILE] [--out FILE]
+
+it wraps tests/bench/bench.sh, which times the kernels of tests/programs/bench.zc
+(collatz, sieve, digits, fib) plain and o3 on each backend it can build, and adds the
+256-word pointer sweep of tests/bench/sweep.zc, checked (plain column) and unbounded,
+timed here on build/bin/zy the same way. interp rows are dropped unless INTERP=1. before
+timing, ZYTRACE=1 on tests/programs/o3.zc must show at least one compiled o3 region, and
+jit o3 collatz must be under half of jit plain collatz, or it refuses.
+selftest checks the verdict logic on doctored tables in a second, without timing: the slope between 1 and N repetitions, the
+median of TRIALS slopes. bench.sh itself is run as it is (it needs bc).
+
+one row per backend and kernel. columns: plain ms and o3 ms per repetition; checks, the
+number of bounds checks the compiler keeps (from --explain-checks, which does not exist
+yet: '-'); peak heap words and gc mark time, which the runtime does not expose: '-'.
+
+check compares with the baseline (tests/bench.baseline): each of the plain and o3
+columns fails on its own when it is more than TOLERANCE (10%) slower and also more than
+MIN_MS (0.1 ms) slower, so sub-ms cells don't flake; rows missing from a run don't fail, and a row fails
+when both sides have a checks count and it rose. when anything looks slower the whole
+measurement is repeated, up to RETRIES more times, and each cell keeps its best figure,
+so one noisy run does not fail the check.
+"""
+import shutil
+import platform
+import argparse
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+TRIALS = int(os.environ.get("TRIALS", "5"))
+MIN_S = float(os.environ.get("MIN_S", "0.4"))
+TOLERANCE = float(os.environ.get("TOLERANCE", "0.10"))
+MIN_MS = float(os.environ.get("MIN_MS", "0.1"))
+INTERP = os.environ.get("INTERP", "") == "1"
+RETRIES = int(os.environ.get("RETRIES", "1"))
+
+# the rows this script times itself: name, source, kernel letter (bench.sh has the rest)
+SWEEPS = [
+ ("zy:sweep.checked", "tests/bench/sweep.zc", "c"),
+ ("zy:sweep.unbounded", "tests/bench/sweep.zc", "u"),
+]
+COLUMNS = ["program", "plain_ms", "o3_ms", "checks", "peak_heap_words", "gc_mark_ms"]
+KERNELS = ("collatz", "sieve", "digits", "fib")
+
+
+def bench_sh():
+ """runs tests/bench/bench.sh and returns its rows: (backend:kernel, plain ms, o3 ms)"""
+ r = subprocess.run(["bash", str(ROOT / "tests/bench/bench.sh")], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ if r.returncode != 0:
+ sys.exit("bench: tests/bench/bench.sh failed:\n" + r.stdout[-2000:] + r.stderr[-2000:])
+ rows = []
+ for line in r.stdout.splitlines():
+ w = line.split()
+ # backend names have spaces ("x86-64 jit"): the kernel is the fourth field from the end
+ if len(w) >= 5 and w[-4] in KERNELS and w[-1].endswith("x"):
+ backend = "-".join(w[:-4])
+ rows.append((backend + ":" + w[-4], float(w[-3]), float(w[-2])))
+ if not INTERP:
+ rows = [r for r in rows if not r[0].startswith("interp:")]
+ if not rows:
+ sys.exit("bench: no rows in the output of tests/bench/bench.sh:\n" + r.stdout[-2000:])
+ return rows
+
+
+def run_once(zy, src, letter, reps):
+ data = src + b"`" + letter.encode() + str(reps).encode()
+ t = time.perf_counter()
+ r = subprocess.run([zy], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ dt = time.perf_counter() - t
+ if r.returncode != 0:
+ sys.exit("bench: %s %s failed: %s" % (zy, letter, r.stderr.decode(errors="replace")[:300]))
+ return dt
+
+
+def slope_ms(zy, path, letter):
+ src = (ROOT / path).read_bytes()
+ n = 4
+ while True:
+ one = min(run_once(zy, src, letter, 1) for _ in range(2))
+ many = run_once(zy, src, letter, n)
+ if many - one >= MIN_S or n >= 1 << 16:
+ break
+ n *= 4
+ slopes = []
+ for _ in range(TRIALS):
+ one = run_once(zy, src, letter, 1)
+ many = run_once(zy, src, letter, n)
+ slopes.append((many - one) / (n - 1) * 1000)
+ return statistics.median(slopes)
+
+
+def fmt_row(cells):
+ return "%-26s %10s %10s %8s %16s %11s" % tuple(cells)
+
+
+def num(x):
+ return "-" if x is None else "%.3f" % x
+
+
+def write_table(path, rows):
+ lines = ["# tinyvm bench: ms per repetition, from tests/bench/bench.sh and tests/bench/suite.py; '-' where a figure is not available",
+ fmt_row(COLUMNS)]
+ for r in rows:
+ lines.append(fmt_row([r["program"], num(r["plain"]), num(r["o3"]), r["checks"], r["peak"], r["gc"]]))
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("\n".join(lines) + "\n")
+
+
+def read_table(path):
+ rows = {}
+ f = lambda x: None if x == "-" else float(x)
+ for line in path.read_text().splitlines():
+ if not line.strip() or line.startswith("#") or line.split()[0] == "program":
+ continue
+ w = line.split()
+ rows[w[0]] = {"program": w[0], "plain": f(w[1]), "o3": f(w[2]), "checks": w[3], "peak": w[4], "gc": w[5]}
+ return rows
+
+
+def region_guard(zy):
+ """refuses to go on when the jit compiles none of the o3 regions of tests/programs/o3.zc"""
+ src = (ROOT / "tests/programs/o3.zc").read_bytes()
+ env = dict(os.environ, ZYTRACE="1")
+ r = subprocess.run([zy], input=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ err = r.stderr.decode(errors="replace").splitlines()
+ done = [l for l in err if l.startswith("o3: ") and " bytes of " in l or " compiled by llvm" in l]
+ stay = [l for l in err if l.startswith("o3: ") and "stays on the region interpreter" in l]
+ print("bench: o3 regions in o3.zc: %d compiled, %d on the region interpreter" % (len(done), len(stay)), file=sys.stderr, flush=True)
+ if not done:
+ sys.exit("bench: REFUSING: %s compiled no o3 regions (ZYTRACE=1 on tests/programs/o3.zc); "
+ "the numbers would be the region interpreter's. rebuild zy with the jit and unset ZYO3INTERP" % zy)
+ return len(done), len(stay)
+
+
+def o3_guard(rows):
+ """jit o3 collatz must be under half of jit plain collatz (needs no interp rows)"""
+ d = {r["program"]: r for r in rows}
+ j = d.get("x86-64-jit:collatz")
+ if j is None:
+ return
+ if j["o3"] is None or j["plain"] is None:
+ sys.exit("bench: REFUSING: x86-64-jit:collatz has no plain or o3 figure")
+ if j["o3"] * 2 >= j["plain"]:
+ sys.exit("bench: REFUSING: jit o3 collatz %.2f ms is not under half of jit plain %.2f ms; o3 regions are not compiled" % (j["o3"], j["plain"]))
+
+
+def run_all(zy):
+ region_guard(zy)
+ rows = []
+ for name, plain, o3 in bench_sh():
+ print("bench: %-26s %10.3f %10.3f ms" % (name, plain, o3), file=sys.stderr, flush=True)
+ rows.append({"program": name, "plain": plain, "o3": o3, "checks": "-", "peak": "-", "gc": "-"})
+ for name, path, letter in SWEEPS:
+ ms = slope_ms(zy, path, letter)
+ print("bench: %-26s %10.3f ms" % (name, ms), file=sys.stderr, flush=True)
+ rows.append({"program": name, "plain": ms, "o3": None, "checks": "-", "peak": "-", "gc": "-"})
+ o3_guard(rows)
+ return rows
+
+
+def slower(now, base):
+ return now is not None and base is not None and base > 0 and now > base * (1 + TOLERANCE) and now - base > MIN_MS
+
+
+def pct(now, base):
+ return "-" if now is None or base is None or base <= 0 else "%+.1f%%" % ((now - base) / base * 100)
+
+
+def expected_backends():
+ """the backends bench.sh should report on this machine, found the same way it finds them"""
+ want = ["zy"]
+ if platform.system() == "Linux" and platform.machine() == "x86_64":
+ want.append("x86-64-jit")
+ if os.environ.get("LLVM_CONFIG") or shutil.which("llvm-config") or shutil.which("llvm-config-19"):
+ want.append("llvm")
+ if INTERP:
+ want.append("interp")
+ return want
+
+
+def require_rows(rows):
+ """bench-save refuses a table that is missing a backend this machine has"""
+ have = {r["program"].split(":")[0] for r in rows}
+ missing = [b for b in expected_backends() if b not in have]
+ if missing:
+ sys.exit("bench: REFUSING to save: no rows for %s (a backend failed to build?)" % ", ".join(missing))
+
+
+def main():
+ ap = argparse.ArgumentParser(description="tinyvm benchmark suite")
+ ap.add_argument("mode", choices=["run", "check", "selftest"])
+ ap.add_argument("--rows", help="check: compare this saved table instead of measuring")
+ ap.add_argument("--zy", default=str(ROOT / "build/bin/zy"))
+ ap.add_argument("--out", default=str(ROOT / "build/bench.txt"))
+ ap.add_argument("--baseline", default=str(ROOT / "tests/bench.baseline"))
+ a = ap.parse_args()
+ if a.mode == "selftest":
+ return selftest()
+ rows = list(read_table(Path(a.rows)).values()) if a.rows else run_all(a.zy)
+ if a.mode == "run":
+ if os.environ.get("BENCH_SAVE") == "1":
+ require_rows(rows)
+ write_table(Path(a.out), rows)
+ print(Path(a.out).read_text(), end="")
+ return 0
+ base_path = Path(a.baseline)
+ if not base_path.exists():
+ sys.exit("bench: no baseline at %s (run make bench-save)" % base_path)
+ base = read_table(base_path)
+ tries = 0
+ while not a.rows and tries < RETRIES and any(slower(r[c], base[r["program"]][c]) for r in rows if r["program"] in base for c in ("plain", "o3")):
+ tries += 1
+ print("bench: something looks slower; measuring again (%d of %d)" % (tries, RETRIES), file=sys.stderr, flush=True)
+ again = {r["program"]: r for r in run_all(a.zy)}
+ for r in rows:
+ g = again.get(r["program"])
+ if g:
+ for c in ("plain", "o3"):
+ if r[c] is not None and g[c] is not None:
+ r[c] = min(r[c], g[c])
+ bad = 0
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % ("program", "base pl", "now pl", "change", "base o3", "now o3", "change", "b chk", "n chk", "verdict"))
+ names = set()
+ for r in rows:
+ names.add(r["program"])
+ b = base.get(r["program"])
+ if not b:
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s new" % (r["program"], "-", num(r["plain"]), "-", "-", num(r["o3"]), "-", "-", r["checks"]))
+ continue
+ why = [c for c in ("plain", "o3") if slower(r[c], b[c])]
+ verdict = ", ".join(c + " SLOWER" for c in why)
+ if b["checks"] != "-" and r["checks"] != "-" and int(r["checks"]) > int(b["checks"]):
+ verdict = (verdict + ", " if verdict else "") + "MORE CHECKS"
+ if verdict:
+ bad += 1
+ print("%-26s %9s %9s %8s %9s %9s %8s %6s %6s %s" % (r["program"], num(b["plain"]), num(r["plain"]), pct(r["plain"], b["plain"]),
+ num(b["o3"]), num(r["o3"]), pct(r["o3"], b["o3"]), b["checks"], r["checks"], verdict or "ok"))
+ for name in base:
+ if name not in names:
+ print("%-26s %9s %9s gone" % (name, num(base[name]["plain"]), "-"))
+ write_table(Path(a.out), rows)
+ print("bench-check: %s (%d of %d rows over the limit; a column fails when over %d%% and over %g ms slower)" % ("FAIL" if bad else "ok", bad, len(rows), round(TOLERANCE * 100), MIN_MS))
+ return 1 if bad else 0
+
+
+def selftest():
+ """no timing: runs check on a doctored table against a doctored baseline and expects the verdicts"""
+ import tempfile
+ t = Path(tempfile.mkdtemp())
+ base = [{"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"} for n, p, o in
+ [("a:big", 10.0, 2.0), ("a:tiny", 0.5, 0.4), ("a:gone", 1.0, 1.0)]]
+ write_table(t / "base", base)
+ def case(name, now, want):
+ write_table(t / "now", now)
+ r = subprocess.run([sys.executable, __file__, "check", "--rows", str(t / "now"), "--baseline", str(t / "base"), "--out", str(t / "out")],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ ok = (r.returncode != 0) == want
+ print("selftest: %-36s exit %d, want %s: %s" % (name, r.returncode, "fail" if want else "ok", "pass" if ok else "WRONG"))
+ if not ok:
+ print(r.stdout)
+ return ok
+ row = lambda n, p, o: {"program": n, "plain": p, "o3": o, "checks": "-", "peak": "-", "gc": "-"}
+ good = [
+ case("same numbers", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.4)], False),
+ case("big plain 20% slower", [row("a:big", 12.0, 2.0), row("a:tiny", 0.5, 0.4)], True),
+ case("big o3 +0.3 ms (15%)", [row("a:big", 10.0, 2.3), row("a:tiny", 0.5, 0.4)], True),
+ case("tiny 30% but +0.12 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.62, 0.4)], True),
+ case("tiny 20% but only +0.08 ms", [row("a:big", 10.0, 2.0), row("a:tiny", 0.5, 0.48)], False),
+ case("big +0.5 ms but only 5%", [row("a:big", 10.5, 2.0), row("a:tiny", 0.5, 0.4)], False),
+ case("rows missing (no interp)", [row("a:big", 10.0, 2.0)], False),
+ ]
+ print("selftest: %s" % ("ok" if all(good) else "FAIL"))
+ return 0 if all(good) else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/bench/sweep.zc b/tests/bench/sweep.zc
new file mode 100644
index 0000000..9bc7070
--- /dev/null
+++ b/tests/bench/sweep.zc
@@ -0,0 +1,23 @@
+namespace;
+include rt;
+
+// a 256-word sweep through a heap pointer, with bounds checks (the default) and without
+// (unbounded): each pass adds every word of the block 1000 times. the index is variable
+// and the block is reached through a pointer, so the checked version keeps its checks.
+// input: a letter (c checked, u unbounded, z none) and how many times to run it.
+int sweep(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+unbounded int sweepu(int* p, int n) { int s = 0; for (int k = 0; k < 1000; k++) for (int i = 0; i < n; i++) s = s + p[i]; return s; }
+fn main() {
+ int* p = rt.new(256);
+ for (int i = 0; i < 256; i++) p[i] = i;
+ let which = rt.get_c(), reps = 0, ch = rt.get_c(), r = 0;
+ while (ch >= '0' && ch <= '9') { reps = reps * 10 + ch - '0'; ch = rt.get_c(); }
+ while (reps > 0) {
+ if (which == 'c') r = sweep(p, 256);
+ if (which == 'u') r = sweepu(p, 256);
+ reps--;
+ }
+ rt.put_n(r); rt.put_c(10);
+ rt.free(p);
+ return 0;
+}
diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh
index c87b142..c351d1f 100755
--- a/tests/conformance/run.sh
+++ b/tests/conformance/run.sh
@@ -13,13 +13,16 @@
# run-only images of src/tools/parity.sh). tests/conformance/known.txt lists failures that are
# understood: one line per failure, `PATH IMPL CLASS reason` with PATH relative to
# tests/conformance and IMPL one of zy, zym, zc0. they print as `known` and do not fail the run.
+# a listed program that passes on that implementation prints as `stale` and fails the run, so
+# the line is removed once the failure is fixed.
set -u
usage() {
printf '%s\n' 'usage: tests/conformance/run.sh [--zym] [--zc0] [--zy BINARY] [-w DIR] [-q] [-t SECONDS] [FILE.zc | DIR ...]' \
'runs the conformance programs under tests/conformance (or the given files and folders) on build/bin/zy.' \
'each program starts with `// expect: TEXT` (output, newlines as /) or `// expect error: TEXT`.' \
- 'prints ok, FAIL, known or skip per program and implementation, then a summary per section and a total;' \
- 'exit status 1 when anything fails that is not listed in tests/conformance/known.txt.' \
+ 'prints ok, FAIL, known, stale or skip per program and implementation, then a summary per section and a total;' \
+ 'exit status 1 when anything fails that is not listed in tests/conformance/known.txt, or when a listed' \
+ 'program passes (stale: remove its line from known.txt).' \
'options:' \
' --zym also run every program on build/bin/zym' \
' --zc0 also run the programs zc0 accepts through zc0 (build/bin/zc0 and a microcore per program)' \
@@ -70,16 +73,20 @@ secs=()
fails=0
bump() { cnt["$1"]=$(( ${cnt["$1"]:-0} + 1 )); }
-# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run
+# status: ok FAIL skip. a FAIL listed in known.txt is shown as known and does not fail the run;
+# an ok listed there is stale (the failure is gone but its line is not) and fails the run
emit() { # name impl status detail
local name=$1 impl=$2 st=$3 detail=$4 sec=${1%%/*} why
if [ "$st" = FAIL ] && [ -f "$known" ] && why=$(grep -m1 "^$name $impl " "$known"); then
st=known; detail="$detail [$(echo "$why" | cut -d' ' -f3-)]"
+ elif [ "$st" = ok ] && [ -f "$known" ] && grep -q -m1 "^$name $impl " "$known"; then
+ st=stale; detail="passes now; remove its line from $known"
fi
bump "$sec:$impl:$st"
case $st in
FAIL) fails=$((fails+1)); echo "FAIL $impl $name: $detail" ;;
known) [ $quiet = 1 ] || echo "known $impl $name: $detail" ;;
+ stale) fails=$((fails+1)); echo "stale $impl $name: $detail" ;;
ok) [ $quiet = 1 ] || echo "ok $impl $name" ;;
skip) [ $quiet = 1 ] || echo "skip $impl $name: $detail" ;;
esac
@@ -160,7 +167,7 @@ if [ $zc0 = 1 ]; then
fi
fi
-echo "summary (programs, ok, FAIL, known, skip per implementation):"
+echo "summary (programs, ok, FAIL, known, stale, skip per implementation):"
impls="zy"; [ $zym = 1 ] && impls="$impls zym"; [ $zc0 = 1 ] && impls="$impls zc0"
tot=0
declare -A itot
@@ -172,15 +179,15 @@ for s in "${secs[@]}"; do
done
tot=$((tot+n))
for i in $impls; do
- o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
- line="$line $i $o/$b/$k/$sk"
+ o=${cnt["$s:$i:ok"]:-0}; b=${cnt["$s:$i:FAIL"]:-0}; k=${cnt["$s:$i:known"]:-0}; st=${cnt["$s:$i:stale"]:-0}; sk=${cnt["$s:$i:skip"]:-0}
+ line="$line $i $o/$b/$k/$st/$sk"
itot["$i:ok"]=$(( ${itot["$i:ok"]:-0} + o )); itot["$i:FAIL"]=$(( ${itot["$i:FAIL"]:-0} + b ))
- itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
+ itot["$i:known"]=$(( ${itot["$i:known"]:-0} + k )); itot["$i:stale"]=$(( ${itot["$i:stale"]:-0} + st )); itot["$i:skip"]=$(( ${itot["$i:skip"]:-0} + sk ))
done
printf ' %-24s %3d programs%s\n' "$s" "$n" "$line"
done
line=""
-for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:skip"]:-0}"; done
+for i in $impls; do line="$line $i ${itot["$i:ok"]:-0}/${itot["$i:FAIL"]:-0}/${itot["$i:known"]:-0}/${itot["$i:stale"]:-0}/${itot["$i:skip"]:-0}"; done
printf ' %-24s %3d programs%s\n' total "$tot" "$line"
-echo "(per implementation: ok/FAIL/known/skip)"
+echo "(per implementation: ok/FAIL/known/stale/skip)"
[ $fails -eq 0 ]
diff --git a/tests/parity.known b/tests/parity.known
index 5ddbdf2..37874f9 100644
--- a/tests/parity.known
+++ b/tests/parity.known
@@ -1,6 +1,7 @@
# programs where zc0 and the self-hosted compiler print different things on purpose.
# src/tools/parity.sh still runs them and shows the difference as known, and it does
-# not fail the run; when one starts to agree it says so, and its line can go.
+# not fail the run; when one starts to agree it is reported as stale and fails the
+# run, so its line goes.
# one program per line: the path, then # and the reason.
#
# zc0 reads types and ownership tags and throws them away. it has no ownership