diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | TODO.md | 4 | ||||
| -rw-r--r-- | src/core/core.c | 271 | ||||
| -rw-r--r-- | tests/bench/bench.sh | 2 | ||||
| -rwxr-xr-x | tests/test.sh | 6 |
5 files changed, 271 insertions, 14 deletions
@@ -590,7 +590,7 @@ calls push the arguments left to right and the callee's prologue pops them into registers: x19 holds the memory base, x20 the data stack pointer, x21 the return stack pointer, x22 the translation table and x23 the state block (memory, s, r, table, coverage map, Q, G, &J, store hook, install hook, enhancement fallback). o3 regions also use x26..x28, which the entry stub saves. -the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are not compiled to x86-64: a region entry is never chained to, so control reaches it through the c loop, which runs the region on the region interpreter exactly as the interpreter build does, and a store from either side into the other's code drops what it should (the regions mark their words in the same coverage map). the region interpreter is slower than the jit's own code for the same words (the `bench.sh` o3 kernels take 2 to 4 times longer than their plain versions on this jit, where they are 3 to 8 times faster on the interpreter), so `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does, until a region compiler for x86-64 exists. +the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are compiled to x86-64 too, but not as on arm64: there are too few registers to keep the vars in them, so each var lives in its home frame slot (rbp points at the frame, reloaded at every entry), and a block's values get the six registers rsi, rdi, r8..r11. the vars a block changes are written home at its end, and the other slots that share a var get it at every exit. calls are plain vm calls through the dispatch stub, and ops the lowering lacks (floats, popcount) call the evaluator. a region that needs more than six live values, or makes native calls, runs on the region interpreter, entered from the c loop; `ZYO3INTERP=1` puts every region there, and `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does. compilation is lazy. the first time control reaches an address, that block is translated from live memory. a block ends at call, jump, ret or halt. a `z` doesn't end it: the not-taken side keeps compiling, which makes superblocks. a literal followed by `c`, `j` or `z` is fused into a constant target. every exit goes through a dispatch stub that checks both stack pointers against their limits (kept in x24 and x25), bounds-checks the target, loads `tab[t]`, and branches straight to it when it's already compiled. otherwise it returns to c, which compiles the block and re-enters. enhancements are inlined: inputs load from the data stack into x0..x15, the optimized micro-ops run in registers with x16 and x17 as scratch, and the outputs go back. an install calls into c and invalidates translations and o3 regions that depend on its slot. invalidated code is rebuilt on its next entry. when the loop reaches an o3 entry point, it compiles the region (or reuses it) instead of translating a block. @@ -520,8 +520,8 @@ brief someone without re-reading the code. - [x] (done on work/line-tables, see "agent work at the end of the session") runtime source positions. architecture: at each statement start (`stmt()`) record (code address `here`, line); per function a permanent data block [entry, end, name, file, count, pairs...] linked into a runtime list (`rt.line_blocks`), or appended to run-only images by `build_image` like the symbol blob. `lines on;`/`lines off;` like `docs on;`; on for user programs, off for the genome unless the growth is small. a new syscall op returns the return-stack pointer (the return stack is in vm memory, 0x1A4100..0x1B40FF, growing up); implement it in `src/core/core.c`, `src/micro/micro.c`, `src/js/tinyvm.ts` and the wasm port. `rt.trace()` walks the return stack, printing ` at file:line in function` for words inside a known function range. the runtime's fatal reports (`oob`, bad heap pointer, double free, out of memory, frame overflow) print the trace after their unchanged first line. check mode skips all of it. - [ ] borrow returns tied to an argument. inference already works for a callee defined before the call: `char* skip(char* s) { ...; return s + i; }` makes `let r = skip(t); rt.free(t); r[0]` an error. the gap: when the call comes before the callee's definition, nothing is reported and the program reads freed memory. extend the deferred effect requirements (`effect_require`, `effect_finish` at unit end) so a forward-called function whose return turns out to alias a parameter produces the same error. for bodyless declarations, externs and function types, add an unobtrusive spelling, for example a `borrows s` clause after the parameter list (`char* skip(char* s) borrows s;`), checked against the body when one exists. - [ ] variadic arguments and `std.print`. design: a pack is passed as one hidden argument, a caller-built read-only growable-array handle (`fixed T[?]` view: [len, cap, items, element size] in the caller's frame, so `len(a)` and `a[i]` work unchanged and nothing is allocated). `...int a` is a typed pack; several packs per function when their element types differ (`void f(...int a, ...char b)`): each extra argument goes to the pack whose element type matches its static type, an ambiguous match is an error, packs come after the fixed parameters. `...any args` packs each value with a kind code (int, float, char, string, bstr, pointer, ...) and its type record: `any` is a small built-in record { kind, value, type }. `...*any args` makes the call comma-free: after the fixed arguments, the parser reads unary expressions until `)`, commas optional, so `std.print("x = " name ", age " age "\n");` works and binary expressions must be parenthesized. `std.print(...*any args)` prints each argument by its kind (`put_n`, the float printer, `put_s`, `put_bs`, a character for `char`, hex for pointers). -- [ ] (inlining done on work/jit2) x86-64 jit phase 2: compile o3 regions to x86-64 (port `rcompile` and the region register allocation; regions now run on the region interpreter and are 2 to 4 times slower than their plain versions on this jit) and inline enhancements (lower the optimized micro-ops to x86 registers as the arm64 path does, instead of calling `xrun`). validate with `tests/drivers/jit_x64_test.sh` and `tests/bench/bench.sh`, which now has an x86-64 jit column. add the driver to `tests/test.sh` on x86-64 linux. - done so far: enhancements are inlined (`x64_spec_enh`, `x64_enh`, `x64_uop` in core.c; seven value registers rsi, rdi, r8..r11, rbp; rax, rcx, rdx scratch; floats, byte loads and stores, native calls, popcnt and f2s/s2f still go through `xrun`; `ZYNOENH=1` calls everything as before). bench.sh plain kernels on the x86-64 jit, ms before/after: collatz 49.1/32.1, sieve 4.4/4.1, digits 41.9/14.7, fib 0.83/0.77; o3 unchanged (regions still interpreted). a 256-word pointer sweep with bounds checks 291 to 72 ms. `tests/programs/enh_ops.zc` covers every lowered op on edge values; jit_x64_test runs in test.sh's jit section (not in the quick path). next: o3 regions (`rcompile`, region register allocation), then the loop-entry flag. +- [ ] (inlining and o3 regions done on work/jit2) x86-64 jit phase 2: compile o3 regions to x86-64 (port `rcompile` and the region register allocation; regions now run on the region interpreter and are 2 to 4 times slower than their plain versions on this jit) and inline enhancements (lower the optimized micro-ops to x86 registers as the arm64 path does, instead of calling `xrun`). validate with `tests/drivers/jit_x64_test.sh` and `tests/bench/bench.sh`, which now has an x86-64 jit column. add the driver to `tests/test.sh` on x86-64 linux. + done so far: enhancements are inlined (`x64_spec_enh`, `x64_enh`, `x64_uop` in core.c; seven value registers rsi, rdi, r8..r11, rbp; rax, rcx, rdx scratch; floats, byte loads and stores, native calls, popcnt and f2s/s2f still go through `xrun`; `ZYNOENH=1` calls everything as before). bench.sh plain kernels on the x86-64 jit, ms before/after: collatz 49.1/32.1, sieve 4.4/4.1, digits 41.9/14.7, fib 0.83/0.77; o3 unchanged (regions still interpreted). a 256-word pointer sweep with bounds checks 291 to 72 ms. `tests/programs/enh_ops.zc` covers every lowered op on edge values; jit_x64_test runs in test.sh's jit section (not in the quick path). o3 regions now compile (`rjit_x64`, `x64_ralloc`, `remit_x64` in core.c): vars stay in their frame slots, values in six registers, calls go through x_disp; `ZYO3INTERP=1` keeps the region interpreter. bench.sh o3 kernels on the x86-64 jit, ms interpreted/compiled: collatz 289/10.4, sieve 8.7/0.40, digits 89/3.0, fib 3.0/0.92 (fib is still slower than its plain version: calls and returns go through the dispatch stub). still interpreted: regions with more than six live values (one in tests/programs/o3.zc) or native calls. later: vars in registers across a loop, direct o3 calls, floats in xmm registers, spilling instead of giving up. next: the loop-entry flag. - [x] `memdebug`'s `obj_base` used to search only 64 words back for a block header, so `buf[255]` on a 256-word block reported "bad heap pointer". it now rounds the pointer down to each size class's alignment (largest first) and takes the first header whose check word, size and class agree and whose payload holds the pointer, so interiors at any depth work, a deep pointer into a freed block reports use after free, and header words and free space still report "bad heap pointer". regression: `tests/programs/memdebug_interior.zc` (in `tests/test.sh` beside the other memdebug checks). note that a stray pointer that lands inside another live block is accepted, since memdebug cannot know which block it came from. - [ ] (partly done, see "agent work at the end of the session") `defer` follow-ups (small): accept `defer if (...) ...;` and other single statements besides an expression; decide whether deferred code may name an owner declared after the `defer` (now rejected by the lexical rule; one line in `defer_one`). - [ ] build the llvm core by default when `llvm-config` is available (the makefile falls back to the interpreter otherwise), so x86 hosts get a native tier and the language server stops being interpreted; keep `ZYNOLLVM=1` and `-DZYINTERP` as the opt-outs. diff --git a/src/core/core.c b/src/core/core.c index c4306ae..e3895ae 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -3,8 +3,9 @@ is translated to native code the first time it is reached, straight from live memory. stores that land on translated words throw those translations away, so code that rewrites itself, including the resident compiler, just gets - retranslated on the next visit. the arm64 jit also compiles o3 regions; on - x86-64 they run on the region interpreter; enhancements are inlined on both + retranslated on the next visit. both jits compile o3 regions (on x86-64 a + region that does not fit six value registers, or makes native calls, runs on + the region interpreter, as every region does under ZYO3INTERP=1); enhancements are inlined on both (on x86-64 those with float, byte or native-call ops are still called). on other hosts (or with -DZYINTERP) a plain interpreter with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */ #include <stdio.h> @@ -5314,21 +5315,24 @@ static int rexec(Rg *R, int e, int *ps, int *pr) #ifndef ZYJIT_X64 static void be_drop(int j) { (void)j; } #endif -static void rforget(Rg *R) { (void)R; } #ifdef ZYJIT_X64 -/* a region entry is never chained to: drop any translation that started there */ +/* a region entry is never chained to a translation: drop any that started there. + the jit's coverage marks the words too, so a store from translated code to + the region's own code reaches the hook, which drops the region */ static void rmark(Rg *R) { int e; - R->marked = 1; cinc(rcov, R->lo, R->hi); + R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(cov, R->lo, R->hi); for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0; } +static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(cov, R->lo, R->hi); } #else +static void rforget(Rg *R) { (void)R; } static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); } -#endif static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); } static void rcompile(Rg *R, int eager) { (void)R; (void)eager; } #endif +#endif #ifdef ZYLLVM /* ---- o3 regions in llvm: one function per region, vars in allocas (so @@ -6072,7 +6076,21 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; } +static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS Rg *xR; +static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ +static int xv(int v, int scratch) +{ + if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } + if (xvh[v]) { /* a var at block entry: its home slot, or fp for var 0 */ + int j = xvh[v] - 1; + if (j == 0) { mov_rr(scratch, r_bp); alu_rr(a_sub, scratch, rMB); shift_ri(5, scratch, 3); } + else if (xR->nv0 && j >= xR->nv0) { mov_ri(scratch, (Cell)(uintptr_t)&xsyn[j]); mov_rm(scratch, scratch, 0); } + else mov_rm(scratch, r_bp, 8 * xR->slot[j]); + return scratch; + } + return xreg[(int)xr[v]]; +} static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; } static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); } static void x64_uop(Uop *u) @@ -6155,7 +6173,7 @@ static void x64_enh(Enh *e) { int i, j; xr = e->reg; - for (i = 0; i < e->nv; i++) xkc[i] = 0; + for (i = 0; i < e->nv; i++) { xkc[i] = 0; xvh[i] = 0; } for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; } for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j); for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]); @@ -6163,6 +6181,237 @@ static void x64_enh(Enh *e) for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax)); } +/* ---- o3 regions in x86-64. there are too few registers for the arm64 + scheme (vars in registers), so a var lives in its home frame slot, + m[fp + slot], with rbp = m + 8 * fp loaded at every entry. a block's values + get the six caller-saved registers rsi, rdi, r8..r11; rax, rcx, rdx are + scratch. a var read is a load from its home, var 0 is fp itself. the vars a + block changes are written home at its end (old values read first, so a swap + works), then the other slots that share a var get its value at every exit. + ops the inline lowering lacks (floats, popcount and friends) call ev. + calls are plain vm calls (the return point is an entry, so the callee's + return chains back in); everything else leaves through x_disp. a region + with native calls stays on the region interpreter, as does every region + under ZYO3INTERP=1. */ +static VM_TLS int x64_o3interp; +static VM_TLS signed char rxr[MAXUOP * 4 + 16]; +#define RXNREG 6 +static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } +static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); } +static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); } +/* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ +static int x64_ralloc(RB *B) +{ + static VM_TLS int last[MAXUOP * 4 + 16]; + int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; + if (B->nv > MAXUOP * 4) return -1; + for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + if (u->op == U_NARG || u->op == U_NCALL) return -1; + if (u->op == U_VGET || u->op == U_SLD || u->op == U_LI) continue; + na = nargs(u->op); + if (na > 0) last[u->a] = i; + if (na > 1) last[u->b] = i; + if (na > 2) last[u->c] = i; + } + for (j = 0; j < B->nso; j++) last[B->so[2 * j + 1]] = n; + for (j = 1; j < xR->nv; j++) if (B->vs[j] >= 0) last[B->vs[j]] = n; + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) last[B->cv] = n; + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + na = u->op == U_VGET || u->op == U_SLD || u->op == U_LI ? 0 : nargs(u->op); + for (k = 0; k < na; k++) { + int a = k == 0 ? u->a : k == 1 ? u->b : u->c; + if (last[a] == i && rxr[a] >= 0) { fr |= 1 << rxr[a]; last[a] = -2; } + } + if (is_store(u->op)) continue; + if (u->op == U_LI) { xkc[u->d] = 1; xkv[u->d] = u->imm; continue; } + if (u->op == U_VGET) { xvh[u->d] = u->imm + 1; continue; } + if (!fr) return -1; + for (k = 0; !(fr >> k & 1); k++); + fr &= ~(1 << k); rxr[u->d] = (signed char)k; + if (last[u->d] == -1) { fr |= 1 << k; } /* never read: its register is free again at once */ + } + return 0; +} +/* the store of rdx to the word rax; a covered word goes through the hook */ +static void rstore(void) +{ + JW *out, *slow, *done; + alu_ri(a_cmp, r_ax, MW); out = j32(c_ae); + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); + slow = j32(c_ne); + mov_xr(rMB, r_ax, 8, 0, r_dx); + done = jm32(); + fix32(slow, cp); + rsave(); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + publish_stacks(); + call_m(rST, 64); + reload_stacks(); + rrest(); + fix32(out, cp); fix32(done, cp); +} +/* a c call with up to three values: rdi = k, rsi, rdx, rcx = a, b, c; the result into d */ +static void rcallc(void *f, long k, Uop *u, int na, int d) +{ + rsave(); + if (na > 0) mov_rr(r_ax, xv(u->a, r_ax)); + if (na > 1) mov_rr(r_dx, xv(u->b, r_dx)); + if (na > 2) mov_rr(r_cx, xv(u->c, r_cx)); + mov_rr(r_si, r_ax); + mov_ri(r_di, k); + call_abs(f); + rrest(); + if (d >= 0) mov_rr(d, r_ax); +} +static void rlduop(Uop *u, int d) +{ + int n = u->op == U_LD8 ? 1 : u->op == U_LD16 ? 2 : 4; + JW *over; + mov_rr(r_ax, xv(u->a, r_ax)); + xor32(r_cx); + mov_ri(r_dx, ((Cell)MW << 3) - (n - 1)); + alu_rr(a_cmp, r_ax, r_dx); + over = j8(c_ae); + if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0); + else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0); + else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0); + fix8(over, cp); + mov_rr(d, r_cx); +} +/* the other slots a var stands for get its value: the frame is exact again */ +static void rspill(Rg *R) +{ + int k; + for (k = 0; k < R->np; k++) + if (R->pv[k] > 0 && R->ps[k] != R->slot[R->pv[k]]) { mov_rm(r_ax, r_bp, 8 * R->slot[R->pv[k]]); mov_mr(r_bp, 8 * R->ps[k], r_ax); } +} +static void rexit(Rg *R, int t) { rspill(R); mov_r32i(r_ax, (unsigned)t); jmp_to(x_disp); } +/* var j's home gets register r (never rcx) */ +static void rvset(Rg *R, int j, int r) +{ + if (R->nv0 && j >= R->nv0) { mov_ri(r_cx, (Cell)(uintptr_t)&xsyn[j]); mov_mr(r_cx, 0, r); } + else mov_mr(r_bp, 8 * R->slot[j], r); +} +static VM_TLS JW **rfa; static VM_TLS int *rfb, nrf; +static void rjmp(JW *at, int b) { rfa[nrf] = at; rfb[nrf++] = b; } +static void remit_x64(Rg *R, RB *B) +{ + int i, j, d; + JW *at; + if (context_current) { + mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx); + at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp); + } + xr = rxr; + for (i = 0; i < B->n; i++) { + Uop *u = &B->u[i]; + int op = u->op; + if (op == U_LI || op == U_VGET) continue; + d = is_store(op) || rxr[u->d] < 0 ? -1 : xreg[(int)rxr[u->d]]; + if (op == U_SLD) { if (d >= 0) mov_rm(d, rSP, 8 * u->imm); continue; } + if (d < 0 && !is_store(op)) continue; + if (op == U_ST) { mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); rstore(); continue; } + if (op >= U_LD8 && op <= U_ST32) { + if (op == U_LD8 || op == U_LD16 || op == U_LD32) { rlduop(u, d); continue; } + rsave(); + mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + mov_r32i(r_dx, op == U_ST8 ? 1 : op == U_ST16 ? 2 : 4); + call_abs((void *)xbst); + rrest(); + continue; + } + if (x64_op_ok(op)) { x64_uop(u); continue; } + rcallc((void *)xev, op, u, nargs(op), d); + } + for (j = 0; j < B->nso; j++) mov_mr(rSP, 8 * B->so[2 * j], xv(B->so[2 * j + 1], r_ax)); + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) mov_rr(r_dx, xv(B->cv, r_dx)); + /* the vars home: read every var-valued one first */ + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) push_r(xv(B->vs[j], r_ax)); + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && !xvh[B->vs[j]]) rvset(R, j, xv(B->vs[j], r_ax)); + for (j = R->nv - 1; j >= 1; j--) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) { pop_r(r_ax); rvset(R, j, r_ax); } + if (B->delta) lea(rSP, rSP, -1, 1, 8 * B->delta); + switch (B->term) { + case T_BR: rjmp(jm32(), B->s1); break; + case T_CBR: + test_rr(r_dx, r_dx); + if (B->s1 >= 0) rjmp(j32(c_e), B->s1); + else { at = j32(c_ne); rexit(R, B->x1); fix32(at, cp); } + if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2); + break; + case T_CALL: + rspill(R); + mov_mi(rRP, 0, B->x2); alu_ri(a_add, rRP, 8); + mov_rr(r_ax, r_dx); jmp_to(x_disp); + break; + case T_RET: + rspill(R); + alu_ri(a_sub, rRP, 8); mov_rm(r_ax, rRP, 0); jmp_to(x_disp); + break; + default: + rspill(R); mov_rr(r_ax, r_dx); jmp_to(x_disp); + } +} +/* 0: compiled; -1: can't be; -2: no room now */ +static int rjit_x64(Rg *R) +{ + JW *start = cp, **lab; + int b, e, need = 64 * R->ne + 8192, rc = 0; + for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256; + if (cp - code + need > CODEW * 4 - (1 << 19)) return -2; + lab = malloc((size_t)R->nb * sizeof *lab); + rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0; + if (!lab || !rfa || !rfb) { free(lab); free(rfa); free(rfb); return -2; } + xR = R; + for (b = 0; b < R->nb && !rc; b++) if (x64_ralloc(&R->b[b])) rc = -1; +#ifdef ZYRDUMP + for (b = 0; b < R->nb; b++) { + RB *B = &R->b[b]; int i, j; + fprintf(stderr, "b%d: n=%d delta=%d term=%d cv=%d s1=%d s2=%d x1=%d x2=%d\n", b, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2); + for (i = 0; i < B->n; i++) { Uop *u = &B->u[i]; fprintf(stderr, " v%d = %s %d %d %d imm=%ld\n", u->d, u->op < U_N ? unames[u->op] : u->op == U_VGET ? "vget" : u->op == U_SLD ? "sld" : "?", u->a, u->b, u->c, (long)u->imm); } + for (j = 0; j < B->nso; j++) fprintf(stderr, " so %d <- v%d\n", B->so[2*j], B->so[2*j+1]); + for (j = 1; j < R->nv; j++) fprintf(stderr, " var%d(slot %d) <- %d\n", j, R->slot[j], B->vs[j]); + } + for (b = 0; b < R->np; b++) fprintf(stderr, "ps %d -> var %d\n", R->ps[b], R->pv[b]); +#endif + for (e = 0; e < R->ne && !rc; e++) { + R->stub[e] = cp; + mov_rm(r_ax, rMB, 8 * ZFP); lea(r_bp, rMB, r_ax, 8, 0); + rjmp(jm32(), R->entb[e]); + } + for (b = 0; b < R->nb && !rc; b++) { + lab[b] = cp; + x64_ralloc(&R->b[b]); + remit_x64(R, &R->b[b]); + } + if (!rc && cp - start > need) { fprintf(stderr, "o3: x86-64 code overran its estimate\n"); abort(); } + if (!rc) { + int k; + for (k = 0; k < nrf; k++) fix32(rfa[k], lab[rfb[k]]); + R->nat = start; + for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = R->stub[e]; + if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start)); + } else cp = start; + free(lab); free(rfa); free(rfb); + return rc; +} +static void rforget(Rg *R) +{ + int e; + if (R->nat) for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words && tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0; + R->nat = 0; +} +static void rcompile(Rg *R, int eager) +{ + (void)eager; + if (R->nat || R->nonat || x64_o3interp || x64_noo3 || !R->lifted) return; + if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); } +} + static void *translate(int p) { JW *start; @@ -6374,7 +6623,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break; if (stop_ge >= 0 && pc >= stop_ge) break; if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break; - if (o3ent[pc] && !x64_noo3) { /* an o3 region: the region interpreter runs it */ + if (o3ent[pc] && !x64_noo3) { /* an o3 region: its x86-64 code if it has some, else the region interpreter */ Rg *R = rat(pc); if (R) { int s = (int)S[1], r = (int)S[2]; @@ -6384,6 +6633,8 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; } s = (int)sp; r = (int)rp; } + rcompile(R, 0); + if (R->nat && tab[pc]) { S[1] = s; S[2] = r; x64_limits(); goto native; } pc = rexec(R, o3ei[pc], &s, &r); S[1] = s; S[2] = r; x64_limits(); @@ -6391,6 +6642,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr continue; } } + native: c = tab[pc]; if (!c) { jitwrite(1); c = translate(pc); jitwrite(0); @@ -6416,6 +6668,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) if (code == MAP_FAILED) { perror("mmap"); return 1; } x64_noo3 = getenv("ZYNOO3") != 0; x64_noenh = getenv("ZYNOENH") != 0; + x64_o3interp = getenv("ZYO3INTERP") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 4ef2ca5..0a9acef 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -6,7 +6,7 @@ CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 make -s build/core/genome.bin || exit 1 $CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy vms=("$T/zy"); names=("interp") -# the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions run on the region interpreter) +# the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions compiled; ZYO3INTERP=1 runs them on the region interpreter) if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } fi diff --git a/tests/test.sh b/tests/test.sh index bc199e1..9dd04a8 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -54,7 +54,7 @@ CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 export CC # the language server runs on the interpreter unless the host has a jit (arm64, or x86-64 # linux), and a check of a large document then takes many seconds: stretch the drivers' -# deadlines. the x86-64 jit gets a little room, since it does not compile o3 regions +# deadlines. the x86-64 jit gets a little room if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -s)-$(uname -m)" in *-arm64|*-aarch64) ZC_TEST_TIMEOUT_SCALE=1;; Linux-x86_64) ZC_TEST_TIMEOUT_SCALE=2;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi export ZC_TEST_TIMEOUT_SCALE # the default map, generated as make does it, and the tools built against it @@ -511,6 +511,10 @@ s_jit() { grep '^DIFF' $T/jit_x64.out chk $r 0 "x86-64 jit agrees with the interpreter: $(tail -n 1 $T/jit_x64.out)" chk "$(ZYNOENH=1 $T/zyx < tests/programs/enh_ops.zc | md5sum)" "$(md5sum < tests/programs/enh_ops.expected)" "x86-64 jit with enhancements called through xrun (ZYNOENH=1)" + chk "$(ZYTRACE=1 $T/zyx < tests/programs/o3.zc 2>&1 | grep -c 'bytes of x86-64')" "7" "x86-64 jit compiles the o3 regions it can (one has too many live values)" + for p in o3 o3copy o3licm o3push o3bytes fo3 o3call o3deep; do + chk "$($T/zyx < tests/programs/$p.zc 2>&1)" "$(ZYO3INTERP=1 $T/zyx < tests/programs/$p.zc 2>&1)" "x86-64 o3 code matches the region interpreter (ZYO3INTERP=1): $p" + done fi } |