aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar zinc <zinc@downsta.ir> 2026-10-14 10:34:44 +0000
committerGravatar zinc <zinc@downsta.ir> 2026-10-14 10:34:44 +0000
commit1d3c45e270341c820de221acb6f21f193e78a8c0 (patch)
tree48aeaad7ba13ad2705ea985a531f3a70a45e2d3d
parenttodo: record the vision, priorities and settled feature decisions; add the it... (diff)
parentx86-64 jit: zyi target, ZYNOO3, bench and test timeouts, docs; drop a stray file (diff)
Merge the x86-64 jit, phase 1; todo: tick it, add phase 2
-rw-r--r--Makefile6
-rw-r--r--README.md12
-rw-r--r--TODO.md22
-rw-r--r--src/core/core.c620
-rw-r--r--tests/bench/bench.sh6
-rw-r--r--tests/drivers/jit_x64_test.sh34
-rwxr-xr-xtests/test.sh7
7 files changed, 668 insertions, 39 deletions
diff --git a/Makefile b/Makefile
index 6195b78..75eb058 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,7 @@
# make parity compile the test programs with zc0 and with the self-hosted
# compiler and compare what they print (src/tools/parity.sh)
# make genome build/core/genome.bin from src/, with the fixpoint check
+# make build/bin/zyi the interpreter alone (zy has a jit on arm64 and on x86-64 linux)
# make lsp build/bin/zclsp, the language server
# make web build/web/: the page, its script and zc.wasm
# make js build/js/: the javascript vm, its layout and genome
@@ -91,9 +92,12 @@ $(B)/core/genome.bin: $(ZC_DEPS) $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) src/core
cp $(B)/s1/genome.bin $@
@echo "fixpoint: $$(( $$(wc -c < $@) / 8 )) words, $@ updated"
-# the interpreter: the vm core with the compiler genome inside
+# the vm core with the compiler genome inside: a jit on arm64 and x86-64 linux, else the interpreter
$(BIN)/zy$(EXE): src/core/core.c $(IO) $(B)/core/genome.bin $(CORE_H) | $(BIN)
$(CC) $(CFLAGS) -DJTYPE=int src/core/core.c $(IO) $(COREINC) -o $@
+# the interpreter alone, even on a host with a jit: make build/bin/zyi, to compare against zy
+$(BIN)/zyi$(EXE): src/core/core.c $(IO) $(B)/core/genome.bin $(CORE_H) | $(BIN)
+ $(CC) $(CFLAGS) -DZYINTERP -DJTYPE=int src/core/core.c $(IO) $(COREINC) -o $@
# the portable microcore
$(BIN)/zym$(EXE): src/micro/micro.c src/micro/micro_port.h $(PORT) $(B)/core/genome.bin $(CORE_H) | $(BIN)
$(CC) $(CFLAGS) $(MICRO_THREADS) src/micro/micro.c $(PORT) $(COREINC) -Isrc/micro -o $@
diff --git a/README.md b/README.md
index ed16c42..4537188 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,12 @@
# tinyvm zygote
-a small stack cpu began as a C89 code golf entry and became a self-hosting runtime. a native core runs vm code with an arm64 jit, through llvm on supported builds, or in the interpreter. the genome contains a compiler for zc, a C-like language with forth influences. the compiler resides in vm memory alongside the program; code can call its functions, modify its tables, add syntax, and compile new code at runtime. heap objects carry an ownership tag (u, m or d) checked by a static analyzer, with deterministic frees at scope ends and a mark-sweep collector underneath. types are a compiler primitive, with structs, generics, reflection and an optional typechecker. programs can install up to 512 custom opcodes (virtual enhancements), which the jits inline as native code. code marked `o3` bypasses the stack machine: the core compiles the region as a unit and keeps its variables in registers.
+a small stack cpu began as a C89 code golf entry and became a self-hosting runtime. a native core runs vm code with an arm64 or x86-64 jit, through llvm on supported builds, or in the interpreter. the genome contains a compiler for zc, a C-like language with forth influences. the compiler resides in vm memory alongside the program; code can call its functions, modify its tables, add syntax, and compile new code at runtime. heap objects carry an ownership tag (u, m or d) checked by a static analyzer, with deterministic frees at scope ends and a mark-sweep collector underneath. types are a compiler primitive, with structs, generics, reflection and an optional typechecker. programs can install up to 512 custom opcodes (virtual enhancements), which the jits inline as native code. code marked `o3` bypasses the stack machine: the core compiles the region as a unit and keeps its variables in registers.
this folder has the latest code. `docs/SPEC.md` is the language reference, and `docs/LEARN.md` is a tour of the language. `docs/history/` is the scrapbook: notes for every stage, plus frozen code from the stages that still have it, archived as `docs/history/code-snapshots.tar.gz`.
## files
-`src/core/core.c` is the native core. it contains the arm64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; the compiler is the files of `src/compiler/`, named in order by `src/compiler/parts` (the lexer, types, expressions, statements, analyzer, declarations, check mode and image building each have a file; `base.zc` begins with the overview). all are written in zc. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by the compiler. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` checks that image against the memory map before it becomes `build/core/genome.bin`. an image is a file of little-endian 64-bit words, and every core takes its genome in at compile time with c23 `#embed` (`src/core/genome.h`). `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept:
+`src/core/core.c` is the native core. it contains the arm64 jit, the x86-64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; the compiler is the files of `src/compiler/`, named in order by `src/compiler/parts` (the lexer, types, expressions, statements, analyzer, declarations, check mode and image building each have a file; `base.zc` begins with the overview). all are written in zc. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by the compiler. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` checks that image against the memory map before it becomes `build/core/genome.bin`. an image is a file of little-endian 64-bit words, and every core takes its genome in at compile time with c23 `#embed` (`src/core/genome.h`). `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept:
- `mandel.zc`: the mandelbrot set in 16.16 fixed point, with the fixed-point multiply wired in as a custom opcode.
- `bf.zc`: a brainfuck compiler. it turns brainfuck into zc source, has the resident compiler compile it, and runs it (native code on the jit). it takes a program after a backtick, or runs hello world.
@@ -72,7 +72,7 @@ windows needs clang or gcc and powershell for the tests. the batch file is the s
### native backends
-on arm64, `src/core/core.c` uses its jit by default. other hosts use the interpreter. `-DZYINTERP` forces interpretation; `-DZYLLVM` enables the llvm backend when its headers and libraries are available. direct builds from the root are:
+on arm64, `src/core/core.c` uses its arm64 jit by default, and on x86-64 linux its x86-64 jit. other hosts use the interpreter. `-DZYINTERP` forces interpretation (`make build/bin/zyi` builds that core beside `zy`); `-DZYLLVM` enables the llvm backend when its headers and libraries are available. direct builds from the root are:
```sh
make genome
@@ -434,7 +434,7 @@ going into o3 is free: o3 code reads normal objects the same way and ignores the
## the llvm backend
-a tiered jit for hosts the arm64 backend doesn't cover. the interpreter counts how often each block start gets reached, and a hot block is translated to llvm ir using the same block rules as the arm64 jit: it ends at call, jump, ret or halt, keeps going past a jz, and literal targets are fused. the ir goes through the O2 pipeline and gets compiled by orc's lljit. stores into compiled words drop the blocks covering them, like on arm64. it pays off for long-running code: a loop-and-recursion benchmark runs about 3.3x faster than the interpreter. short bursty work like the self-compile is slower, since compile time dominates there. `ZYHOT=1` compiles every block, which is a good stress test, and the test suite runs that way too.
+a tiered jit for hosts the native jits don't cover, or where it is asked for explicitly (`-DZYLLVM` replaces the x86-64 jit). the interpreter counts how often each block start gets reached, and a hot block is translated to llvm ir using the same block rules as the arm64 jit: it ends at call, jump, ret or halt, keeps going past a jz, and literal targets are fused. the ir goes through the O2 pipeline and gets compiled by orc's lljit. stores into compiled words drop the blocks covering them, like on arm64. it pays off for long-running code: a loop-and-recursion benchmark runs about 3.3x faster than the interpreter. short bursty work like the self-compile is slower, since compile time dominates there. `ZYHOT=1` compiles every block, which is a good stress test, and the test suite runs that way too.
## the heap
@@ -444,7 +444,7 @@ the heap is a buddy allocator over [HEAP_LO, top), where top starts at HEAP_HI a
## c abi
-zc can call native c functions and hand zc functions to native code. it is built on the syscall gate (ops 14 to 17), so it needs a host with `dlsym`: the interpreter, the llvm backend and the arm64 jit on linux and macos (the jit has to be a dynamic binary, since a static one has no `dlsym`). the microcore, the javascript vm, the web build and the windows scripts answer -1, and a native call there gives -1 and nothing else.
+zc can call native c functions and hand zc functions to native code. it is built on the syscall gate (ops 14 to 17), so it needs a host with `dlsym`: the interpreter, the llvm backend and the arm64 and x86-64 jits on linux and macos (the jit has to be a dynamic binary, since a static one has no `dlsym`). the microcore, the javascript vm, the web build and the windows scripts answer -1, and a native call there gives -1 and nothing else.
```
include rt;
@@ -569,6 +569,8 @@ calls push the arguments left to right and the callee's prologue pops them into
registers: x19 holds the memory base, x20 the data stack pointer, x21 the return stack pointer, x22 the translation table and x23 the state block (memory, s, r, table, coverage map, Q, G, &J, store hook, install hook, enhancement fallback). o3 regions also use x26..x28, which the entry stub saves.
+the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are not compiled to x86-64: a region entry is never chained to, so control reaches it through the c loop, which runs the region on the region interpreter exactly as the interpreter build does, and a store from either side into the other's code drops what it should (the regions mark their words in the same coverage map). the region interpreter is slower than the jit's own code for the same words (the `bench.sh` o3 kernels take 2 to 4 times longer than their plain versions on this jit, where they are 3 to 8 times faster on the interpreter), so `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does, until a region compiler for x86-64 exists.
+
compilation is lazy. the first time control reaches an address, that block is translated from live memory. a block ends at call, jump, ret or halt. a `z` doesn't end it: the not-taken side keeps compiling, which makes superblocks. a literal followed by `c`, `j` or `z` is fused into a constant target. every exit goes through a dispatch stub that checks both stack pointers against their limits (kept in x24 and x25), bounds-checks the target, loads `tab[t]`, and branches straight to it when it's already compiled. otherwise it returns to c, which compiles the block and re-enters. enhancements are inlined: inputs load from the data stack into x0..x15, the optimized micro-ops run in registers with x16 and x17 as scratch, and the outputs go back. an install calls into c and invalidates translations and o3 regions that depend on its slot. invalidated code is rebuilt on its next entry. when the loop reaches an o3 entry point, it compiles the region (or reuses it) instead of translating a block.
self-modification is handled with a counted coverage map: one byte per word, the number of live translations that cover it, sticky at 255. a store checks that byte inline: an uncovered target, the usual case, gets a plain store. a covered target calls the hook, which looks the word up in a page index (64-word pages, each with a list of the blocks that touch it) and drops only the translations containing that word, decrementing the coverage of just their words. the jit code then returns to c at the next instruction, so even patches to the very next word take effect. writing back the same value is a no-op. installing an enhancement does not flush: every translation records which enhancement slots it inlined, and an install drops only those (and the o3 regions that depend on the slot). if the 16 mb code buffer or the block index fills, everything is flushed. on llvm each block has its own resource tracker, so dropped code is freed once no compiled code is running.
diff --git a/TODO.md b/TODO.md
index 05995e6..d7fa925 100644
--- a/TODO.md
+++ b/TODO.md
@@ -163,8 +163,11 @@ and pushed there.
`ZC_TEST_TIMEOUT_SCALE=4 node tests/drivers/lsp_test.js build/bin/zclsp`,
`ZC_TEST_TIMEOUT_SCALE=4 node tests/drivers/hint_fixes_test.js
build/bin/zclsp`, `make parity` (zc0 against the self-hosted compiler; by
- design differences live in `tests/parity.known`). on x86 the language server
- is interpreted and slow; `ZC_TEST_TIMEOUT_SCALE` stretches the deadlines.
+ design differences live in `tests/parity.known`). on x86-64 linux `zy` and
+ the language server now run on the x86-64 jit (o3 regions still interpreted);
+ `ZC_TEST_TIMEOUT_SCALE` stretches the deadlines on slower hosts.
+ `bash tests/drivers/jit_x64_test.sh build/bin/zyi build/bin/zy` compares the
+ jit with the interpreter (`make build/bin/zyi`) on every program.
- work was done by delegating each item to a separate agent in its own git
worktree (outside the repo, under `/workspace/tinyvm-wt/`) and merging its branch.
every brief started with "read ONBOARD.md first" plus the sections relevant
@@ -196,8 +199,8 @@ the reasoning behind the open items survives the session.
friction: the toolchain's own sources must stay in zc0's subset; word-per-
character strings cost eight times the memory of `bstr` and split every api
in two (`put_s` and `put_bs`); the compiler's state is global parallel
- arrays; on x86 the language server is interpreted (about 18 s to check the
- compiler).
+ arrays; on x86 the language server was interpreted (about 18 s to check the
+ compiler) until the x86-64 jit landed.
- priorities, in order: things that turn wrong behavior into a located,
reproducible failure come first, because everything else (fuzzing, a proving
scaffold, outside users) depends on them: runtime line tables, bounds
@@ -236,7 +239,8 @@ the reasoning behind the open items survives the session.
outside the self-compiling loop (a prover inside the genome can approve its
own mistakes, as the fixpoint approved the precedence bug); prefer
translation validation of each compile over proving the compiler whole.
-- native code: the hand-written jit exists only for arm64; the llvm backend
+- native code: the hand-written jit exists for arm64 and (phase 1, no o3
+ regions or inlined enhancements yet) x86-64 linux; the llvm backend
(`-DZYLLVM`) is the portable native tier but the default build leaves it
out. cheap win: build the llvm core by default whenever `llvm-config` is
found. the x86-64 port of the hand-written jit is worth it for compile
@@ -283,8 +287,9 @@ these were running as agents in worktrees. if their branches were not merged
before the session ended, the work is lost and each item below must be started
again from its description in the open list. a zip of the repository with every
worktree is uploaded to the owner after each update.
-- spec verification (`tests/conformance/`), see "verify both compilers".
-- x86-64 jit phase 1, see "a hand-written x86-64 jit".
+- spec verification: the first agent was stopped by a session interruption
+ and its partial `tests/conformance/` was not merged; start it again from
+ "verify both compilers".
- runtime line tables and backtraces, see "runtime source positions".
- bounds checking with `unbounded`, see "bounds checking".
- the cross-reference report, see "a cross-reference report".
@@ -320,11 +325,11 @@ brief someone without re-reading the code.
- [ ] test blocks: `test "name" { ... }` with `expect(cond)`, compiled only by `zy --test file.zc`, which runs each block and prints one line per test with the position of the first failed `expect`; skipped at parse time otherwise. build it as a library through `run.syntax` and `run.statement_hook`, like `src/lib/actor.zc`.
- [ ] (in flight) bounds checking on by default, `unbounded` to turn it off (function prefix, block, top-level `unbounded;`, mirroring `unchecked`), implied by `o3`. architecture: indexing is compiled in the postfix `[` handling of the expressions part; growable arrays already check (`dat`, `range_jz`, the `outside` enhancement, `proven()` removing checks a loop test already did). fixed arrays `T[n]` with a variable index get the same inline check against the constant `n`. pointers get a runtime call (for example `rt.bchk(p, i)`) that checks only when `p` is the start of a live heap block (header check word `p ^ MAGIC`, `paylen`), skipping constant index 0 and sizes the analyzer knows (`lnsz`). reports go through `oob`. the runtime, core, compiler parts, server, `run.zc` and `build.zc` start with `unbounded;`, which zc0 accepts and ignores (genome bytes unchanged). keep std, icon and actor checked unless a function touches raw memory on purpose. measure interpreter time before and after.
- [ ] after the fuzzing work, add more general functionality to the standard library.
-- [ ] (in flight) a hand-written x86-64 jit, ported from the arm64 one in `src/core/core.c` (the `#ifdef ZYJIT` section; `ZYJIT` is only defined for `__aarch64__`). phase 1: a separate macro (for example `ZYJIT_X64`), shared architecture-neutral parts (translation table `tab`, coverage `cov`, invalidation `be_drop`, block boundaries, dispatcher) and an x86 emitter. registers: r12 memory base, r13 data stack, r14 return stack, r15 translation table, rbx state block (the arm64 x19..x23); system v calls to c helpers. semantics identical to the interpreter (wrapping, 0/1 comparisons, literal encoding, stack and `MW` checks, retranslation after stores into code). o3 regions take the portable path in phase 1. validate every `tests/programs`, `examples` and `examples/learn` program on interpreter and jit with identical output (a driver under `tests/drivers/`), the rejection suites, icon and actor fixtures; report timings for a compute example and for compiling the compiler. make it the default x86-64 linux core only when everything matches; `-DZYINTERP` still forces the interpreter. phase 2: o3 regions and inlined enhancements. note the llvm backend (`-DZYLLVM`) is the other native path; the default build could also enable it whenever `llvm-config` is found.
- [ ] struct destructors: a struct may declare `fn drop()`, called wherever the compiler frees an owned value of that type, before the memory is released. architecture: the compiler frees owned locals in `freevar` (analyzer part), called from `exit_vars`, reassignment and explicit `rt.free`/`rt.recycle` calls (`s_free`, `s_recycle` handling in the call code); when the local's static type is a pointer to a struct with a `Name.drop` method, emit a call to it first. objects freed by the collector or inside unchecked code do not run `drop`: document that. first users: file descriptors in `std.Host`, processes in `actor.zc`.
- [ ] (in flight) runtime source positions. architecture: at each statement start (`stmt()`) record (code address `here`, line); per function a permanent data block [entry, end, name, file, count, pairs...] linked into a runtime list (`rt.line_blocks`), or appended to run-only images by `build_image` like the symbol blob. `lines on;`/`lines off;` like `docs on;`; on for user programs, off for the genome unless the growth is small. a new syscall op returns the return-stack pointer (the return stack is in vm memory, 0x1A4100..0x1B40FF, growing up); implement it in `src/core/core.c`, `src/micro/micro.c`, `src/js/tinyvm.ts` and the wasm port. `rt.trace()` walks the return stack, printing ` at file:line in function` for words inside a known function range. the runtime's fatal reports (`oob`, bad heap pointer, double free, out of memory, frame overflow) print the trace after their unchanged first line. check mode skips all of it.
- [ ] borrow returns tied to an argument. inference already works for a callee defined before the call: `char* skip(char* s) { ...; return s + i; }` makes `let r = skip(t); rt.free(t); r[0]` an error. the gap: when the call comes before the callee's definition, nothing is reported and the program reads freed memory. extend the deferred effect requirements (`effect_require`, `effect_finish` at unit end) so a forward-called function whose return turns out to alias a parameter produces the same error. for bodyless declarations, externs and function types, add an unobtrusive spelling, for example a `borrows s` clause after the parameter list (`char* skip(char* s) borrows s;`), checked against the body when one exists.
- [ ] variadic arguments and `std.print`. design: a pack is passed as one hidden argument, a caller-built read-only growable-array handle (`fixed T[?]` view: [len, cap, items, element size] in the caller's frame, so `len(a)` and `a[i]` work unchanged and nothing is allocated). `...int a` is a typed pack; several packs per function when their element types differ (`void f(...int a, ...char b)`): each extra argument goes to the pack whose element type matches its static type, an ambiguous match is an error, packs come after the fixed parameters. `...any args` packs each value with a kind code (int, float, char, string, bstr, pointer, ...) and its type record: `any` is a small built-in record { kind, value, type }. `...*any args` makes the call comma-free: after the fixed arguments, the parser reads unary expressions until `)`, commas optional, so `std.print("x = " name ", age " age "\n");` works and binary expressions must be parenthesized. `std.print(...*any args)` prints each argument by its kind (`put_n`, the float printer, `put_s`, `put_bs`, a character for `char`, hex for pointers).
+- [ ] x86-64 jit phase 2: compile o3 regions to x86-64 (port `rcompile` and the region register allocation; regions now run on the region interpreter and are 2 to 4 times slower than their plain versions on this jit) and inline enhancements (lower the optimized micro-ops to x86 registers as the arm64 path does, instead of calling `xrun`). validate with `tests/drivers/jit_x64_test.sh` and `tests/bench/bench.sh`, which now has an x86-64 jit column. add the driver to `tests/test.sh` on x86-64 linux.
- [ ] build the llvm core by default when `llvm-config` is available (the makefile falls back to the interpreter otherwise), so x86 hosts get a native tier and the language server stops being interpreted; keep `ZYNOLLVM=1` and `-DZYINTERP` as the opt-outs.
- [ ] stack-effect checking: every function and `asm` block declares or is inferred to have a fixed data-stack effect, and the compiler reports a block whose paths leave the stack unbalanced. the first step of a proving scaffold.
- [ ] declared side effects at module boundaries (globals touched, syscalls or not), checked against the effect masks the compiler already computes.
@@ -337,6 +342,7 @@ brief someone without re-reading the code.
- [ ] run the full suite once everything above is done (`make test JOBS=4`; `make check` is the quick one), fix what it finds, and only then tick this item.
## done in the sorted list
+- [x] a hand-written x86-64 jit, phase 1 (`ZYJIT_X64` in `src/core/core.c`, default on x86-64 linux; `-DZYINTERP` and `make build/bin/zyi` give the interpreter). r12 memory, r13 data stack, r14 return stack, r15 table, rbx state; stack slots addressed with a virtual offset so r13 moves only at calls and exits; shares the table, coverage map, store hook and invalidation with the arm64 jit. enhancements are called through `xrun`, o3 regions run on the region interpreter (`ZYNOO3=1` runs their vm code through the jit instead). 179 of 179 programs agree with the interpreter (`tests/drivers/jit_x64_test.sh`), the self-compile is byte-identical and about 7x faster (1.5 s against 11 s), and the rejection, learn, icon, actor, lsp and hint suites pass on it.
- [x] remove temporary markdown files (`TASKLIST.md`, `docs/design/handoff.md` deleted; `BOOTSTRAP-OWNERSHIP.md` moved to `docs/history/`).
- [x] verify every command in `ONBOARD.md` from a clean checkout and fix the ones that drifted.
diff --git a/src/core/core.c b/src/core/core.c
index dc50644..b571981 100644
--- a/src/core/core.c
+++ b/src/core/core.c
@@ -1,10 +1,12 @@
/* zygote core: puts the genome (the zc compiler and its runtime, both vm code)
- into memory and runs it. on arm64, every block of vm code is translated to
- arm64 the first time it is reached, straight from live memory. stores that
- land on translated words throw those translations away, so code that
- rewrites itself, including the resident compiler, just gets retranslated
- on the next visit. on other hosts (or with -DZYINTERP) a plain interpreter
- with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */
+ into memory and runs it. on arm64 and on x86-64 linux, every block of vm code
+ is translated to native code the first time it is reached, straight from live
+ memory. stores that land on translated words throw those translations away,
+ so code that rewrites itself, including the resident compiler, just gets
+ retranslated on the next visit. the arm64 jit also compiles o3 regions; on
+ x86-64 they run on the region interpreter, and enhancements are called, not
+ inlined. on other hosts (or with -DZYINTERP) a plain interpreter with the same
+ semantics runs instead. ZYTRACE=1 logs translations on stderr. */
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
@@ -53,6 +55,15 @@
#include <libkern/OSCacheControl.h>
#endif
#endif
+/* the x86-64 jit: linux hosts only, and not with the interpreter forced or llvm asked for.
+ o3 regions run on the region interpreter there. ZYNATIVE is either jit. */
+#if defined(__x86_64__) && defined(__linux__) && !defined(ZYINTERP) && !defined(ZYLLVM) && !defined(ZYJIT)
+#define ZYJIT_X64 1
+#include <sys/mman.h>
+#endif
+#if defined(ZYJIT) || defined(ZYJIT_X64)
+#define ZYNATIVE 1
+#endif
#if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK)
#define ZYBIGSTACK 1
#include <sys/mman.h>
@@ -3286,12 +3297,24 @@ static Rg *rat(int p)
return R;
}
-#ifdef ZYJIT
+/* ---- the native jits' shared state and rules ----
+ both jits (arm64 and x86-64) keep the translation table, the coverage map, the
+ state block and the store hook here, so a store into translated code, a flush
+ and a dropped translation mean the same thing on either. the emitters and the
+ instruction selection are each jit's own. */
+#ifdef ZYNATIVE
static VM_TLS unsigned char *cov; /* how many live translations cover the word (255 sticks) */
static VM_TLS void **tab; /* vm address -> translation */
static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); }
-static VM_TLS unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
+#ifdef ZYJIT
+typedef unsigned JW; /* code is a string of 32-bit instruction words */
+#else
+typedef unsigned char JW; /* code is a string of bytes */
+#define rcov cov /* a lifted o3 region marks its words in the same map */
+#endif
+static VM_TLS JW *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
static VM_TLS long S[15]; /* memory, stacks, hooks, limits, and the target saved on overflow */
+#ifdef ZYJIT
static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc)
{
o3_invoke(ps, pr, target, retpc);
@@ -3403,6 +3426,158 @@ static void build_stubs(void)
E(0xd61f0120); /* br x9 */
x_first = cp;
}
+#else
+/* ---- the x86-64 emitter ----
+ registers: r12 holds the memory base, r13 the data stack pointer, r14 the
+ return stack pointer, r15 the translation table and rbx the state block S
+ (the arm64 jit's x19..x23). rax, rcx, rdx, rsi, rdi and r8..r11 are scratch.
+ the callee-saved ones keep their meaning across calls into c, and rsp stays
+ 16-byte aligned inside translated code so those calls are plain system v
+ calls. code is built in a byte buffer: a few encoders for the forms the
+ translator uses (modrm with base, index and displacement, an immediate,
+ or a register), then one function per vm operation. */
+enum { r_ax, r_cx, r_dx, r_bx, r_sp, r_bp, r_si, r_di, r_8, r_9, r_10, r_11, r_12, r_13, r_14, r_15 };
+enum { a_add = 0, a_or = 1, a_and = 4, a_sub = 5, a_xor = 6, a_cmp = 7 }; /* alu opcode groups */
+enum { c_b = 2, c_ae = 3, c_e = 4, c_ne = 5, c_be = 6, c_a = 7, c_np = 11, c_l = 12, c_ge = 13, c_le = 14, c_g = 15 };
+#define rMB r_12
+#define rSP r_13
+#define rRP r_14
+#define rTAB r_15
+#define rST r_bx
+
+static void xb(int b) { *cp++ = (unsigned char)b; }
+static void xd(unsigned v) { memcpy(cp, &v, 4); cp += 4; }
+static void xq(uint64_t v) { memcpy(cp, &v, 8); cp += 8; }
+static void rex(int w, int r, int x, int b)
+{
+ int v = 0x40 | (w ? 8 : 0) | ((r & 8) ? 4 : 0) | ((x & 8) ? 2 : 0) | ((b & 8) ? 1 : 0);
+ if (v != 0x40) xb(v);
+}
+/* modrm for [base + index * scale + disp]; index < 0 means none */
+static void modrm_mem(int reg, int base, int index, int scale, int disp)
+{
+ int mod, sib = index >= 0 || (base & 7) == 4;
+ if (disp == 0 && (base & 7) != 5) mod = 0;
+ else if (disp >= -128 && disp <= 127) mod = 1;
+ else mod = 2;
+ xb(mod << 6 | (reg & 7) << 3 | (sib ? 4 : (base & 7)));
+ if (sib) xb((scale == 8 ? 3 : scale == 4 ? 2 : scale == 2 ? 1 : 0) << 6 | ((index >= 0 ? index : 4) & 7) << 3 | (base & 7));
+ if (mod == 1) xb(disp); else if (mod == 2) xd((unsigned)disp);
+}
+/* pre: a mandatory prefix byte or 0; two: the 0f escape */
+static void mem_op(int pre, int w, int two, int opc, int reg, int base, int index, int scale, int disp)
+{
+ if (pre) xb(pre);
+ rex(w, reg, index < 0 ? 0 : index, base);
+ if (two) xb(0x0f);
+ xb(opc);
+ modrm_mem(reg, base, index, scale, disp);
+}
+static void reg_op(int pre, int w, int two, int opc, int reg, int rm)
+{
+ if (pre) xb(pre);
+ rex(w, reg, 0, rm);
+ if (two) xb(0x0f);
+ xb(opc);
+ xb(0xc0 | (reg & 7) << 3 | (rm & 7));
+}
+static void mov_rr(int d, int s) { reg_op(0, 1, 0, 0x89, s, d); }
+static void mov_rm(int r, int b, int disp) { mem_op(0, 1, 0, 0x8b, r, b, -1, 1, disp); }
+static void mov_mr(int b, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, -1, 1, disp); }
+static void mov_rx(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8b, r, b, x, sc, disp); }
+static void mov_xr(int b, int x, int sc, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, x, sc, disp); }
+static void mov_r32m(int r, int b, int disp) { mem_op(0, 0, 0, 0x8b, r, b, -1, 1, disp); } /* zero extends */
+static void mov_mi(int b, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, -1, 1, disp); xd((unsigned)imm); }
+static void mov_r32i(int r, unsigned imm) { rex(0, 0, 0, r); xb(0xb8 | (r & 7)); xd(imm); }
+static void mov_ri(int r, Cell v)
+{
+ UCell u = (UCell)v;
+ if (u <= 0xffffffffu) mov_r32i(r, (unsigned)u);
+ else if (v >= -2147483647 - 1 && v < 0) { reg_op(0, 1, 0, 0xc7, 0, r); xd((unsigned)(int)v); }
+ else { rex(1, 0, 0, r); xb(0xb8 | (r & 7)); xq(u); }
+}
+static void lea(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8d, r, b, x, sc, disp); }
+static void xor32(int r) { reg_op(0, 0, 0, 0x31, r, r); }
+static void alu_rr(int op, int d, int s) { reg_op(0, 1, 0, op * 8 + 1, s, d); }
+static void alu_rm(int op, int d, int b, int disp) { mem_op(0, 1, 0, op * 8 + 3, d, b, -1, 1, disp); }
+static void alu_ri(int op, int d, int imm)
+{
+ if (imm >= -128 && imm <= 127) { reg_op(0, 1, 0, 0x83, op, d); xb(imm); }
+ else { reg_op(0, 1, 0, 0x81, op, d); xd((unsigned)imm); }
+}
+static void alu_mi(int op, int b, int disp, int imm)
+{
+ if (imm >= -128 && imm <= 127) { mem_op(0, 1, 0, 0x83, op, b, -1, 1, disp); xb(imm); }
+ else { mem_op(0, 1, 0, 0x81, op, b, -1, 1, disp); xd((unsigned)imm); }
+}
+static void test_rr(int a, int b) { reg_op(0, 1, 0, 0x85, b, a); }
+static void test32(int a, int b) { reg_op(0, 0, 0, 0x85, b, a); }
+static void imul_rm(int d, int b, int disp) { mem_op(0, 1, 1, 0xaf, d, b, -1, 1, disp); }
+static void imul_rmi(int d, int b, int disp, int imm) { mem_op(0, 1, 0, 0x69, d, b, -1, 1, disp); xd((unsigned)imm); }
+static void shift_cl(int op, int r) { reg_op(0, 1, 0, 0xd3, op, r); } /* 4 shl, 5 shr, 7 sar */
+static void shift_ri(int op, int r, int n) { reg_op(0, 1, 0, 0xc1, op, r); xb(n); }
+static void shift_mi(int op, int b, int disp, int n) { mem_op(0, 1, 0, 0xc1, op, b, -1, 1, disp); xb(n); }
+static void setcc(int cc, int r) { reg_op(0, 0, 1, 0x90 + cc, 0, r); } /* al, cl or dl only */
+static void movsx_rr8(int d, int s) { reg_op(0, 0, 1, 0xbe, d, s); } /* movsx r32, r8 (al..dl) */
+static void movsx64_rr8(int d, int s) { reg_op(0, 1, 1, 0xbe, d, s); }
+static void movsxd_rr(int d, int s) { reg_op(0, 1, 0, 0x63, d, s); }
+static void push_r(int r) { rex(0, 0, 0, r); xb(0x50 | (r & 7)); }
+static void pop_r(int r) { rex(0, 0, 0, r); xb(0x58 | (r & 7)); }
+static void call_m(int b, int disp) { mem_op(0, 0, 0, 0xff, 2, b, -1, 1, disp); }
+static void call_r(int r) { reg_op(0, 0, 0, 0xff, 2, r); }
+static void jmp_r(int r) { reg_op(0, 0, 0, 0xff, 4, r); }
+static void call_abs(void *f) { mov_ri(r_ax, (Cell)(uintptr_t)f); call_r(r_ax); }
+/* branches. j8/j32 return the place to patch; the *_to forms go to a known address */
+static JW *j32(int cc) { JW *at; xb(0x0f); xb(0x80 + cc); at = cp; xd(0); return at; }
+static JW *jm32(void) { JW *at; xb(0xe9); at = cp; xd(0); return at; }
+static JW *j8(int cc) { JW *at; xb(0x70 + cc); at = cp; xb(0); return at; }
+static void fix32(JW *at, JW *to) { int d = (int)(to - (at + 4)); memcpy(at, &d, 4); }
+static void fix8(JW *at, JW *to) { *at = (JW)(to - (at + 1)); }
+static void jcc_to(int cc, JW *to) { JW *at = j32(cc); fix32(at, to); }
+static void jmp_to(JW *to) { JW *at = jm32(); fix32(at, to); }
+/* sse2 on xmm0 and xmm1 only */
+static void sd_m(int opc, int x, int b, int disp) { mem_op(0xf2, 0, 1, opc, x, b, -1, 1, disp); } /* movsd 10/11, addsd 58, mulsd 59, subsd 5c, divsd 5e */
+
+static void build_stubs(void)
+{
+ JW *ovd, *ovr, *at;
+ cp = code;
+ x_enter = cp; /* int enter(long *S, void *code) */
+ push_r(r_bp); push_r(r_bx); push_r(r_12); push_r(r_13); push_r(r_14); push_r(r_15);
+ alu_ri(a_sub, r_sp, 8); /* six pushes and the return address: 16-byte aligned again */
+ mov_rr(rST, r_di);
+ mov_rm(rMB, rST, 0);
+ mov_rm(r_ax, rST, 8); lea(rSP, rMB, r_ax, 8, 0); /* sp = m + s*8 */
+ mov_rm(r_ax, rST, 16); lea(rRP, rMB, r_ax, 8, 0);
+ mov_rm(rTAB, rST, 24);
+ jmp_r(r_si);
+ x_exit = cp; /* eax = next vm address */
+ mov_rr(r_cx, rSP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 8, r_cx);
+ mov_rr(r_cx, rRP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 16, r_cx);
+ alu_ri(a_add, r_sp, 8);
+ pop_r(r_15); pop_r(r_14); pop_r(r_13); pop_r(r_12); pop_r(r_bx); pop_r(r_bp);
+ xb(0xc3);
+ ovd = cp;
+ mov_r32i(r_ax, PC_DATA_OVERFLOW); jmp_to(x_exit);
+ ovr = cp;
+ mov_r32i(r_ax, PC_RETURN_OVERFLOW); jmp_to(x_exit);
+ x_disp = cp; /* eax = target: check cancellation and stacks before chaining */
+ if (context_current) {
+ mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx);
+ at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp);
+ }
+ mov_mr(rST, 104, r_ax);
+ alu_rm(a_cmp, rSP, rST, 88); jcc_to(c_b, ovd); /* sp below its limit */
+ alu_rm(a_cmp, rRP, rST, 96); jcc_to(c_ae, ovr); /* rp at its limit */
+ mov_ri(r_cx, (Cell)(uintptr_t)&code_words); /* the current code reservation */
+ alu_rm(a_cmp, r_ax, r_cx, 0); jcc_to(c_ae, x_exit);
+ mov_rm(rTAB, rST, 24);
+ mov_rx(r_cx, rTAB, r_ax, 8, 0);
+ test_rr(r_cx, r_cx); jcc_to(c_e, x_exit);
+ jmp_r(r_cx);
+ x_first = cp;
+}
+#endif
static void flush(void)
{
@@ -3424,7 +3599,7 @@ static void jitwrite(int on)
#endif
}
-static void syncicache(unsigned *a, unsigned *b)
+static void syncicache(JW *a, JW *b)
{
#ifdef __APPLE__
sys_icache_invalidate(a, (char *)b - (char *)a);
@@ -3457,7 +3632,11 @@ static int hook(Cell a, Cell v)
if (hit) drop_epoch++;
return hit;
}
+static int inst_hook(int d) { return install(d); }
+static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
+#endif
+#ifdef ZYJIT
/* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */
static void fjit(int k)
{
@@ -4240,9 +4419,6 @@ static void rcompile(Rg *R, int eager)
if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1; /* registers, not space: give up on it */
}
-static int inst_hook(int d) { return install(d); }
-static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
-
static void *translate(int p)
{
unsigned *start;
@@ -5032,7 +5208,7 @@ static Blk lcompile(int p)
#endif
/* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */
-#ifndef ZYLLVM
+#if !defined(ZYLLVM) && !defined(ZYJIT_X64)
static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */
#endif
static VM_TLS Cell rvals[MAXUOP * 4 + 16];
@@ -5052,6 +5228,9 @@ static int rdrop(int a) /* a store landed on region code: drop those regions */
#ifdef ZYLLVM
static int lhook(Cell a, Cell v);
#define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0)
+#elif defined(ZYJIT_X64)
+/* a region store may land on translated code too: the jit's hook drops those. the gate word is a plain store, as in the region interpreter anywhere */
+#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (cov[a] && (a) != SYSGATE) hook(a, b); else m[a] = (b); } } while (0)
#else
#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0)
#endif
@@ -5127,9 +5306,21 @@ static int rexec(Rg *R, int e, int *ps, int *pr)
return (int)t;
}
#ifndef ZYLLVM
+#ifndef ZYJIT_X64
static void be_drop(int j) { (void)j; }
+#endif
static void rforget(Rg *R) { (void)R; }
+#ifdef ZYJIT_X64
+/* a region entry is never chained to: drop any translation that started there */
+static void rmark(Rg *R)
+{
+ int e;
+ R->marked = 1; cinc(rcov, R->lo, R->hi);
+ for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0;
+}
+#else
static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); }
+#endif
static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); }
static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
#endif
@@ -5692,6 +5883,382 @@ static void rcompile(Rg *R, int eager)
if (!(R->nat = lregion(R))) R->nonat = 1;
}
#endif
+#ifdef ZYJIT_X64
+/* ---- the x86-64 jit: block translation, and the loop that runs it ----
+ the rules are the arm64 jit's: a block ends at call, jump, ret or halt; a jz
+ does not end it (the not-taken side keeps compiling); a literal followed by
+ call, jump or jz is a constant target; every exit goes through x_disp, which
+ checks both stacks against their limits, looks the target up in tab and
+ chains to it when it is already translated, and otherwise returns to c. the
+ instruction selection differs: within a block the data stack pointer is not
+ moved for every push and pop. vd counts the words pushed (negative) or
+ popped since r13 was last written, stack slot k from the top is
+ [r13 + 8 * (vd + k)], and r13 is brought up to date before every call into
+ c and every exit. o3 regions are not compiled here: a region entry is never
+ translated (the c loop sees it first), and the region interpreter runs it. */
+static VM_TLS int vd;
+static VM_TLS int x64_noo3; /* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */
+#define SL(k) (8 * (vd + (k)))
+static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); }
+static void stv(int k, int r) { mov_mr(rSP, SL(k), r); }
+static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); } /* r13 up to date; vd stays for the other path */
+static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); }
+static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); } /* a constant target: the low 32 bits, like the interpreter's int pc */
+static void disp_ax(void) { sp_keep(); jmp_to(x_disp); }
+static void publish_stacks(void) /* s and r into S, as x_exit does, for hooks that call back into the vm */
+{
+ lea(r_ax, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 8, r_ax);
+ mov_rr(r_ax, rRP); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 16, r_ax);
+}
+static void reload_stacks(void) /* r13, r14, r15 from S after c may have moved the stacks or the table */
+{
+ mov_rm(r_cx, rST, 8); lea(rSP, rMB, r_cx, 8, 0);
+ mov_rm(r_cx, rST, 16); lea(rRP, rMB, r_cx, 8, 0);
+ mov_rm(rTAB, rST, 24);
+}
+/* sp = (r13 + 8 * vd - r12) / 8, the data stack index, into r */
+static void sp_index(int r) { lea(r, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r, rMB); shift_ri(5, r, 3); }
+
+/* a store of rdx to the word rax. known: the address is known to be inside memory. on a
+ covered word the hook runs and, if it dropped a translation, the block resumes in c at i + 1 */
+static void emit_store(int i, int known)
+{
+ JW *done1 = 0, *slow, *done2, *z;
+ if (!known) { alu_ri(a_cmp, r_ax, MW); done1 = j32(c_ae); }
+ mov_rm(r_cx, rST, 32);
+ mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); /* cmp byte [rcx + rax], 0 */
+ slow = j32(c_ne);
+ mov_xr(rMB, r_ax, 8, 0, r_dx);
+ done2 = jm32();
+ fix32(slow, cp);
+ mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
+ publish_stacks();
+ call_m(rST, 64);
+ reload_stacks();
+ test32(r_ax, r_ax);
+ z = j32(c_e);
+ mov_r32i(r_ax, (unsigned)(i + 1)); jmp_to(x_exit); /* a translation died: resume at i + 1 in c; r13 is exact */
+ fix32(z, cp);
+ if (vd) lea(rSP, rSP, -1, 1, -8 * vd);
+ if (done1) fix32(done1, cp);
+ fix32(done2, cp);
+}
+/* the word at rax, or 0 outside memory, into rcx */
+static void emit_load(void)
+{
+ JW *over;
+ xor32(r_cx);
+ alu_ri(a_cmp, r_ax, MW);
+ over = j8(c_ae);
+ mov_rx(r_cx, rMB, r_ax, 8, 0);
+ fix8(over, cp);
+}
+/* byte loads: zero extended, 0 unless the whole access lies in memory */
+static void emit_ld_bytes(int n)
+{
+ JW *over;
+ ldv(r_ax, 0);
+ xor32(r_cx);
+ mov_ri(r_dx, ((Cell)MW << 3) - (n - 1));
+ alu_rr(a_cmp, r_ax, r_dx);
+ over = j8(c_ae);
+ if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0);
+ else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0);
+ else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0);
+ fix8(over, cp);
+ stv(0, r_cx);
+}
+/* c helper for byte stores: 1 if a translation died */
+static long xbst(Cell a, Cell v, long n)
+{
+ int e = drop_epoch;
+ b_store(a, v, (int)n);
+ return drop_epoch != e;
+}
+static void float_op(int i, int k)
+{
+ JW *ok;
+ if (k >= 601 && k <= 604) {
+ static const unsigned char opc[] = { 0x58, 0x5c, 0x59, 0x5e };
+ sd_m(0x10, 0, rSP, SL(1));
+ sd_m(opc[k - 601], 0, rSP, SL(0));
+ vd++;
+ sd_m(0x11, 0, rSP, SL(0));
+ } else if (k >= 605 && k <= 607) {
+ xor32(r_ax);
+ sd_m(0x10, 0, rSP, SL(0));
+ mem_op(0x66, 0, 1, 0x2e, 0, rSP, -1, 1, SL(1)); /* ucomisd xmm0, [b]: a against b, unordered sets cf */
+ if (k == 605) { setcc(c_e, r_ax); setcc(c_np, r_cx); reg_op(0, 0, 0, 0x20, r_cx, r_ax); } /* eq and not unordered */
+ else setcc(k == 606 ? c_a : c_ae, r_ax); /* b < a is a > b; b <= a is a >= b */
+ vd++;
+ stv(0, r_ax);
+ } else if (k == 608) {
+ mem_op(0xf2, 1, 1, 0x2a, 0, rSP, -1, 1, SL(0)); /* cvtsi2sd xmm0, qword [sp] */
+ sd_m(0x11, 0, rSP, SL(0));
+ } else if (k == 609) { /* truncate toward zero; the indefinite result goes to the c helper, which saturates and maps a nan to 0 */
+ mem_op(0xf2, 1, 1, 0x2c, r_ax, rSP, -1, 1, SL(0)); /* cvttsd2si rax, [sp] */
+ mov_ri(r_cx, (Cell)((UCell)1 << 63));
+ alu_rr(a_cmp, r_ax, r_cx);
+ ok = j8(c_ne);
+ ldv(r_di, 0); call_abs((void *)fop_f2i);
+ fix8(ok, cp);
+ stv(0, r_ax);
+ } else if (k == 610) {
+ ldv(r_di, 0); vd++; call_abs((void *)fop_print);
+ } else if (k == 618) { /* (bits prec mode --) */
+ ldv(r_di, 2); ldv(r_si, 1); ldv(r_dx, 0); vd += 3; call_abs((void *)fop_print_mode);
+ } else { /* 611 */
+ ldv(r_di, 0); call_abs((void *)fop_parse_cell); stv(0, r_ax);
+ }
+ (void)i;
+}
+
+static void *translate(int p)
+{
+ JW *start;
+ int i = p, n = 0, k, t, used[64], nused = 0, j2, ju;
+ Cell w;
+ if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush();
+ start = cp;
+ vd = 0;
+ for (;;) {
+ if (i >= code_words) { exit_with(-1); break; }
+ if (n++ >= BLKW || (i != p && o3ent[i])) { disp_imm(i); break; } /* a region entry is entered from c, never by chaining */
+ w = m[i];
+ if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
+ t = (int)((w - 1) / 2); /* literal + call/jmp/jz with a known target */
+ if (k == 8) {
+ JW *skip;
+ ldv(r_cx, 0); vd++;
+ test_rr(r_cx, r_cx);
+ skip = j32(c_ne);
+ disp_imm(t);
+ fix32(skip, cp);
+ i += 2;
+ continue;
+ }
+ if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); }
+ disp_imm(t);
+ i += 2;
+ break;
+ }
+ if (w & 1) {
+ Cell v = (w - 1) / 2;
+ int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1;
+ if (i + 1 < code_words && !(m[i + 1] & 1)) ko = (int)(m[i + 1] / 2);
+ if (ko == 1 && v >= 0 && v < MW && v < (1 << 27)) { /* a fixed address: one load */
+ mov_rm(r_ax, rMB, (int)(8 * v)); vd--; stv(0, r_ax);
+ i += 2; continue;
+ }
+ if (small && ko >= 13 && ko <= 20) { /* the literal is an immediate operand */
+ int d = (int)v;
+ switch (ko) {
+ case 13: alu_mi(a_add, rSP, SL(0), d); break;
+ case 14: alu_mi(a_sub, rSP, SL(0), d); break;
+ case 15: imul_rmi(r_ax, rSP, SL(0), d); stv(0, r_ax); break;
+ case 16: alu_mi(a_and, rSP, SL(0), d); break;
+ case 17: alu_mi(a_or, rSP, SL(0), d); break;
+ case 18: alu_mi(a_xor, rSP, SL(0), d); break;
+ case 19: if (d & 63) shift_mi(7, rSP, SL(0), d & 63); break;
+ default: xor32(r_cx); alu_mi(a_cmp, rSP, SL(0), d); setcc(c_l, r_cx); stv(0, r_cx);
+ }
+ i += 2; continue;
+ }
+ vd--;
+ if (small) mov_mi(rSP, SL(0), (int)v);
+ else { mov_ri(r_ax, v); stv(0, r_ax); }
+ i++; continue;
+ }
+ k = (int)(w / 2);
+ if (k == 0) { exit_with(-1); i++; break; }
+ if (k < 0) { exit_with(-1); i++; break; } /* reserved: the interpreter halts */
+ if (k == 6 || k == 7) {
+ mov_r32m(r_ax, rSP, SL(0)); vd++;
+ if (k == 6) { mov_mi(rRP, 0, i + 1); alu_ri(a_add, rRP, 8); }
+ disp_ax();
+ i++;
+ break;
+ }
+ if (k == 8) { /* jz: taken leaves, not taken keeps going */
+ JW *skip;
+ mov_r32m(r_ax, rSP, SL(0)); ldv(r_cx, 1); vd += 2;
+ test_rr(r_cx, r_cx);
+ skip = j32(c_ne);
+ disp_ax();
+ fix32(skip, cp);
+ i++;
+ continue;
+ }
+ if (k == 9) { alu_ri(a_sub, rRP, 8); mov_r32m(r_ax, rRP, 0); disp_ax(); i++; break; }
+ if (k == 21) { /* install: call into c, then return to it so it can flush */
+ mov_r32m(r_di, rSP, SL(0)); call_m(rST, 72);
+ movsxd_rr(r_ax, r_ax); stv(0, r_ax);
+ exit_with(i + 1); i++; break;
+ }
+ if (k == 600) { /* flit: the next word is raw IEEE bits, even when bit 0 is set */
+ if (i + 1 >= code_words) { exit_with(-1); i++; break; }
+ mov_ri(r_ax, m[i + 1]); vd--; stv(0, r_ax);
+ i += 2;
+ continue;
+ }
+ if ((k >= 601 && k <= 611) || k == 618) { float_op(i, k); i++; continue; }
+ if (k == 612 || k == 614 || k == 616) { emit_ld_bytes(k == 612 ? 1 : k == 614 ? 2 : 4); i++; continue; }
+ if (k == 613 || k == 615 || k == 617) {
+ JW *ok;
+ ldv(r_di, 0); ldv(r_si, 1); mov_r32i(r_dx, k == 613 ? 1 : k == 615 ? 2 : 4); vd += 2;
+ call_abs((void *)xbst);
+ test32(r_ax, r_ax);
+ ok = j32(c_e);
+ exit_with(i + 1);
+ fix32(ok, cp);
+ i++; continue;
+ }
+ if (k == 619) { /* push the data-stack index: the word s points at */
+ sp_index(r_ax); vd--; stv(0, r_ax);
+ i++; continue;
+ }
+ if (k == 620) { /* a native call: the helper pops the arguments and pushes the result */
+ lea(r_di, rSP, -1, 1, 8 * vd); mov_rr(r_si, rRP);
+ call_abs((void *)ncall_host);
+ mov_rr(rSP, r_ax); vd = 0;
+ i++; continue;
+ }
+ if (k >= XOP0 && k < XOP0 + NENH) {
+ if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; /* wired or not, an install makes this a different block */
+ if (enh[k - XOP0].live) { /* the interpreter runs it: x0 = data stack, slot */
+ lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0));
+ call_m(rST, 80);
+ mov_rr(rSP, r_ax); vd = 0;
+ i++; continue;
+ }
+ }
+ if (k > 20) { exit_with(-1); i++; break; } /* not wired: halt */
+ switch (k) {
+ case 1:
+ ldv(r_ax, 0); emit_load(); stv(0, r_cx); break;
+ case 2: /* store: inline unless the target word is translated */
+ ldv(r_ax, 0); ldv(r_dx, 1); vd += 2;
+ emit_store(i, 0); break;
+ case 3: { /* pick uses a word address, with the same bounds as load */
+ ldv(r_ax, 0);
+ sp_index(r_cx);
+ lea(r_ax, r_ax, r_cx, 1, 1);
+ emit_load(); stv(0, r_cx); break; }
+ case 4: vd++; break;
+ case 5: ldv(r_ax, 0); ldv(r_cx, 1); stv(0, r_cx); stv(1, r_ax); break;
+ case 10:
+ call_m(rST, 40); movsx64_rr8(r_ax, r_ax); vd--; stv(0, r_ax); break;
+ case 11:
+ mov_rm(r_ax, rST, 56);
+ if (sizeof J == 1) mem_op(0, 1, 1, 0xbe, r_ax, r_ax, -1, 1, 0); /* movsx rax, byte [rax] */
+ else mem_op(0, 1, 0, 0x63, r_ax, r_ax, -1, 1, 0); /* movsxd rax, dword [rax] */
+ vd--; stv(0, r_ax); break;
+ case 12:
+ ldv(r_ax, 0); movsx_rr8(r_di, r_ax); vd++; call_m(rST, 48); break;
+ case 13: case 14: case 16: case 17: case 18:
+ ldv(r_ax, 1);
+ alu_rm(k == 13 ? a_add : k == 14 ? a_sub : k == 16 ? a_and : k == 17 ? a_or : a_xor, r_ax, rSP, SL(0));
+ vd++; stv(0, r_ax); break;
+ case 15: ldv(r_ax, 1); imul_rm(r_ax, rSP, SL(0)); vd++; stv(0, r_ax); break;
+ case 19: ldv(r_cx, 0); ldv(r_ax, 1); shift_cl(7, r_ax); vd++; stv(0, r_ax); break;
+ default: /* 20 */
+ ldv(r_ax, 1); xor32(r_cx); alu_rm(a_cmp, r_ax, rSP, SL(0)); setcc(c_l, r_cx); vd++; stv(0, r_cx);
+ }
+ i++;
+ }
+ j2 = ix_add(p, i);
+ cinc(cov, p, i);
+ if (nused && used[0] < 0) dep_add(NENH, j2);
+ else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2);
+ tab[p] = start;
+ if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start));
+ return start;
+}
+static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); }
+static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr)
+{
+ union { JW *u; int (*f)(long *, void *); } enter;
+ void *saved_eq = 0, *saved_ent[RMAXE];
+ int i, nent = 0, flushed = 0, ep0 = drop_epoch, term = 0;
+ Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1;
+ enter.u = x_enter;
+ if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; }
+ if (re) {
+ nent = re->ne; if (nent > RMAXE) nent = RMAXE;
+ for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; }
+ }
+ S[1] = *ps; S[2] = *pr;
+ x64_limits();
+ while (!term && !context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) {
+ void *c;
+ if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
+ Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW;
+ if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
+ S[1] = sp; S[2] = rp;
+ x64_limits();
+ pc = (int)S[13]; continue;
+ }
+ if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break;
+ if (stop_ge >= 0 && pc >= stop_ge) break;
+ if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break;
+ if (o3ent[pc] && !x64_noo3) { /* an o3 region: the region interpreter runs it */
+ Rg *R = rat(pc);
+ if (R) {
+ int s = (int)S[1], r = (int)S[2];
+ if (s < stack_data_limit() || r > stack_return_limit()) {
+ Cell sp = s, rp = r;
+ int ret = r > stack_return_limit();
+ if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
+ s = (int)sp; r = (int)rp;
+ }
+ pc = rexec(R, o3ei[pc], &s, &r);
+ S[1] = s; S[2] = r;
+ x64_limits();
+ if ((unsigned)pc >= (unsigned)code_words) term = 1; /* a stop, not a jit exit: no saved target to resume at */
+ continue;
+ }
+ }
+ c = tab[pc];
+ if (!c) {
+ jitwrite(1); c = translate(pc); jitwrite(0);
+ if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0;
+ if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0;
+ }
+ if (!c) break;
+ pc = enter.f(S, c);
+ if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
+ }
+ *ps = S[1]; *pr = S[2];
+ flushed = drop_epoch != ep0;
+ if (!flushed) {
+ if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq;
+ if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i];
+ }
+ return pc;
+}
+/* the code area, the stubs and the state block, for this thread */
+static int x64_boot(Cell sp, Cell rp, void *stop)
+{
+ code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (code == MAP_FAILED) { perror("mmap"); return 1; }
+ x64_noo3 = getenv("ZYNOO3") != 0;
+ jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
+ S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
+ S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
+ S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)stop;
+ x64_limits();
+ cov[SYSGATE] = 1;
+ return 0;
+}
+static int run(int p)
+{
+ Cell s = DS0, r = RS0;
+ if (x64_boot(s, r, &context_default_stop)) return 1;
+ p = vm_until(p, -1, -1, 0, &s, &r);
+ halt_at = -1;
+ return stopped(p);
+}
+#else
#define LD(x) (inmem(x) ? m[x] : 0)
static VM_TLS int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest;
static VM_TLS Rg *stop_rg;
@@ -5839,6 +6406,7 @@ static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr)
stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr;
return back;
}
+#endif
static int o3_ready(Rg *C)
{
if (!C || C->dead) return 0;
@@ -5875,7 +6443,7 @@ static void b_commit(Cell w, Cell v)
{
if ((unsigned)w >= (unsigned)MW) return;
if (w == SYSGATE) { m[w] = v; return; }
-#if defined(ZYJIT)
+#if defined(ZYNATIVE)
if (cov[(int)w]) hook(w, v); else m[w] = v;
#elif defined(ZYLLVM)
if (lcov[(int)w]) lhook(w, v); else m[w] = v;
@@ -5915,8 +6483,11 @@ static int code_reserve(Cell end)
#define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0)
CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1);
CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap);
+#ifdef ZYNATIVE
+ CODE_COLUMN(tab, code_words, cap);
#ifdef ZYJIT
- CODE_COLUMN(tab, code_words, cap); CODE_COLUMN(fn_nat, code_words, cap);
+ CODE_COLUMN(fn_nat, code_words, cap);
+#endif
#elif defined(ZYLLVM)
CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap);
#endif
@@ -5928,7 +6499,7 @@ static int code_reserve(Cell end)
}
for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); }
code_words = cap;
-#ifdef ZYJIT
+#ifdef ZYNATIVE
S[3] = (long)tab;
#endif
return (int)cap;
@@ -5940,7 +6511,7 @@ static int code_range_live(Cell start, Cell count)
if (start < 0 || count <= 0 || start >= code_words) return 0;
end = count > code_words - start ? code_words : start + count;
for (i = start; i < end; i++) {
-#ifdef ZYJIT
+#ifdef ZYNATIVE
if (cov[i]) return 1;
#else
if (rcov[i]) return 1;
@@ -6005,8 +6576,11 @@ static void context_engine_free(void) {
for (i = 0; i < nrg; i++) { rclear(rgs[i]); free(rgs[i]); }
for (i = 0; i < NENH; i++) { free(enh[i].u); free(enh[i].reg); }
free(pg_head); free(o3ent); free(o3ei); free(gn_); free(gh_);
+#ifdef ZYNATIVE
#ifdef ZYJIT
- free(tab); free(fn_nat); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1);
+ free(fn_nat);
+#endif
+ free(tab); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1);
#else
free(rcov);
#ifdef ZYLLVM
@@ -6030,7 +6604,7 @@ static void *context_worker(void *opaque) {
memcpy(ffi_cb, context->callback_snapshot, sizeof ffi_cb); memcpy(ffi_sym, context->symbol_snapshot, sizeof ffi_sym); memcpy(ffi_lib, context->library_snapshot, sizeof ffi_lib);
ffi_nsym = context->ffi_symbols; ffi_nlib = context->ffi_libraries; ffi_tried_libm = context->ffi_libm;
#endif
-#ifdef ZYJIT
+#ifdef ZYNATIVE
cov = calloc((size_t)MW, 1);
#else
rcov = calloc((size_t)MW, 1);
@@ -6038,7 +6612,7 @@ static void *context_worker(void *opaque) {
lcov = calloc((size_t)MW, 1);
#endif
#endif
-#ifdef ZYJIT
+#ifdef ZYNATIVE
if (!cov) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
#else
if (!rcov
@@ -6053,6 +6627,10 @@ static void *context_worker(void *opaque) {
nrg = context->region_count; context->region_count = 0;
memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words);
free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0;
+#ifdef ZYJIT_X64
+ if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
+ atomic_fetch_add(&context_code_live, 1);
+#endif
#ifdef ZYJIT
#ifdef __APPLE__
code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
@@ -6184,7 +6762,7 @@ int main(int argc, char **argv)
if (memory_words < 0) return 1;
if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; }
m = calloc((size_t)MW, sizeof *m);
-#ifdef ZYJIT
+#ifdef ZYNATIVE
cov = calloc((size_t)MW, 1);
if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
#else
diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh
index 65f455e..4ef2ca5 100644
--- a/tests/bench/bench.sh
+++ b/tests/bench/bench.sh
@@ -4,8 +4,12 @@
cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT
CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1
make -s build/core/genome.bin || exit 1
-$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy
+$CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy
vms=("$T/zy"); names=("interp")
+# the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions run on the region interpreter)
+if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then
+ $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); }
+fi
if command -v llvm-config >/dev/null; then
$CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); }
fi
diff --git a/tests/drivers/jit_x64_test.sh b/tests/drivers/jit_x64_test.sh
new file mode 100644
index 0000000..ce0438f
--- /dev/null
+++ b/tests/drivers/jit_x64_test.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# runs every program in tests/programs, examples and examples/learn on two cores
+# and compares stdout and the exit status. usage:
+# bash tests/drivers/jit_x64_test.sh <reference core> <jit core> [name-filter]
+# the reference is normally the interpreter (built with -DZYINTERP) and the second
+# core the x86-64 jit. each program is the core's stdin, with nothing after it, and
+# runs with a timeout (JIT_TEST_TIMEOUT seconds, 120 by default). a program that
+# differs is listed with the first lines of the difference; the exit status of
+# this script is the number of programs that differ.
+ref=${1:?usage: jit_x64_test.sh <reference core> <jit core> [filter]}
+jit=${2:?usage: jit_x64_test.sh <reference core> <jit core> [filter]}
+filter=${3:-}
+limit=${JIT_TEST_TIMEOUT:-120}
+root=$(cd "$(dirname "$0")/../.." && pwd)
+cd "$root" || exit 2
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+pass=0 fail=0 total=0
+for f in tests/programs/*.zc examples/*.zc examples/learn/*.zc; do
+ [ -n "$filter" ] && [[ "$f" != *$filter* ]] && continue
+ total=$((total + 1))
+ timeout "$limit" "$ref" < "$f" > "$tmp/ref.out" 2> "$tmp/ref.err"; rr=$?
+ timeout "$limit" "$jit" < "$f" > "$tmp/jit.out" 2> "$tmp/jit.err"; rj=$?
+ if [ "$rr" = "$rj" ] && cmp -s "$tmp/ref.out" "$tmp/jit.out"; then
+ pass=$((pass + 1))
+ else
+ fail=$((fail + 1))
+ echo "DIFF $f (status $rr vs $rj)"
+ diff "$tmp/ref.out" "$tmp/jit.out" | head -6
+ head -3 "$tmp/jit.err"
+ fi
+done
+echo "$pass of $total programs agree, $fail differ"
+exit "$fail"
diff --git a/tests/test.sh b/tests/test.sh
index 298c356..ec93827 100755
--- a/tests/test.sh
+++ b/tests/test.sh
@@ -52,9 +52,10 @@ ROOT=$T
CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1
export CC
-# the language server runs on the interpreter unless the host has the arm64 jit, and a
-# check of a large document then takes many seconds: stretch the drivers' deadlines
-if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -m)" in arm64|aarch64) ZC_TEST_TIMEOUT_SCALE=1;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi
+# the language server runs on the interpreter unless the host has a jit (arm64, or x86-64
+# linux), and a check of a large document then takes many seconds: stretch the drivers'
+# deadlines. the x86-64 jit gets a little room, since it does not compile o3 regions
+if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -s)-$(uname -m)" in *-arm64|*-aarch64) ZC_TEST_TIMEOUT_SCALE=1;; Linux-x86_64) ZC_TEST_TIMEOUT_SCALE=2;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi
export ZC_TEST_TIMEOUT_SCALE
# the default map, generated as make does it, and the tools built against it
$CC -O2 -w -fwrapv src/tools/genmap.c -o $T/genmap || exit 1