diff options
| -rw-r--r-- | note | 1 | ||||
| -rw-r--r-- | src/core/core.c | 605 | ||||
| -rw-r--r-- | tests/drivers/jit_x64_test.sh | 34 |
3 files changed, 625 insertions, 15 deletions
@@ -0,0 +1 @@ +zy
\ No newline at end of file diff --git a/src/core/core.c b/src/core/core.c index dc50644..51d6b4f 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -53,6 +53,15 @@ #include <libkern/OSCacheControl.h> #endif #endif +/* the x86-64 jit: linux hosts only, and not with the interpreter forced or llvm asked for. + o3 regions run on the region interpreter there. ZYNATIVE is either jit. */ +#if defined(__x86_64__) && defined(__linux__) && !defined(ZYINTERP) && !defined(ZYLLVM) && !defined(ZYJIT) +#define ZYJIT_X64 1 +#include <sys/mman.h> +#endif +#if defined(ZYJIT) || defined(ZYJIT_X64) +#define ZYNATIVE 1 +#endif #if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK) #define ZYBIGSTACK 1 #include <sys/mman.h> @@ -3286,12 +3295,24 @@ static Rg *rat(int p) return R; } -#ifdef ZYJIT +/* ---- the native jits' shared state and rules ---- + both jits (arm64 and x86-64) keep the translation table, the coverage map, the + state block and the store hook here, so a store into translated code, a flush + and a dropped translation mean the same thing on either. the emitters and the + instruction selection are each jit's own. */ +#ifdef ZYNATIVE static VM_TLS unsigned char *cov; /* how many live translations cover the word (255 sticks) */ static VM_TLS void **tab; /* vm address -> translation */ static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); } -static VM_TLS unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first; +#ifdef ZYJIT +typedef unsigned JW; /* code is a string of 32-bit instruction words */ +#else +typedef unsigned char JW; /* code is a string of bytes */ +#define rcov cov /* a lifted o3 region marks its words in the same map */ +#endif +static VM_TLS JW *code, *cp, *x_enter, *x_exit, *x_disp, *x_first; static VM_TLS long S[15]; /* memory, stacks, hooks, limits, and the target saved on overflow */ +#ifdef ZYJIT static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc) { o3_invoke(ps, pr, target, retpc); @@ -3403,6 +3424,158 @@ static void build_stubs(void) E(0xd61f0120); /* br x9 */ x_first = cp; } +#else +/* ---- the x86-64 emitter ---- + registers: r12 holds the memory base, r13 the data stack pointer, r14 the + return stack pointer, r15 the translation table and rbx the state block S + (the arm64 jit's x19..x23). rax, rcx, rdx, rsi, rdi and r8..r11 are scratch. + the callee-saved ones keep their meaning across calls into c, and rsp stays + 16-byte aligned inside translated code so those calls are plain system v + calls. code is built in a byte buffer: a few encoders for the forms the + translator uses (modrm with base, index and displacement, an immediate, + or a register), then one function per vm operation. */ +enum { r_ax, r_cx, r_dx, r_bx, r_sp, r_bp, r_si, r_di, r_8, r_9, r_10, r_11, r_12, r_13, r_14, r_15 }; +enum { a_add = 0, a_or = 1, a_and = 4, a_sub = 5, a_xor = 6, a_cmp = 7 }; /* alu opcode groups */ +enum { c_b = 2, c_ae = 3, c_e = 4, c_ne = 5, c_be = 6, c_a = 7, c_np = 11, c_l = 12, c_ge = 13, c_le = 14, c_g = 15 }; +#define rMB r_12 +#define rSP r_13 +#define rRP r_14 +#define rTAB r_15 +#define rST r_bx + +static void xb(int b) { *cp++ = (unsigned char)b; } +static void xd(unsigned v) { memcpy(cp, &v, 4); cp += 4; } +static void xq(uint64_t v) { memcpy(cp, &v, 8); cp += 8; } +static void rex(int w, int r, int x, int b) +{ + int v = 0x40 | (w ? 8 : 0) | ((r & 8) ? 4 : 0) | ((x & 8) ? 2 : 0) | ((b & 8) ? 1 : 0); + if (v != 0x40) xb(v); +} +/* modrm for [base + index * scale + disp]; index < 0 means none */ +static void modrm_mem(int reg, int base, int index, int scale, int disp) +{ + int mod, sib = index >= 0 || (base & 7) == 4; + if (disp == 0 && (base & 7) != 5) mod = 0; + else if (disp >= -128 && disp <= 127) mod = 1; + else mod = 2; + xb(mod << 6 | (reg & 7) << 3 | (sib ? 4 : (base & 7))); + if (sib) xb((scale == 8 ? 3 : scale == 4 ? 2 : scale == 2 ? 1 : 0) << 6 | ((index >= 0 ? index : 4) & 7) << 3 | (base & 7)); + if (mod == 1) xb(disp); else if (mod == 2) xd((unsigned)disp); +} +/* pre: a mandatory prefix byte or 0; two: the 0f escape */ +static void mem_op(int pre, int w, int two, int opc, int reg, int base, int index, int scale, int disp) +{ + if (pre) xb(pre); + rex(w, reg, index < 0 ? 0 : index, base); + if (two) xb(0x0f); + xb(opc); + modrm_mem(reg, base, index, scale, disp); +} +static void reg_op(int pre, int w, int two, int opc, int reg, int rm) +{ + if (pre) xb(pre); + rex(w, reg, 0, rm); + if (two) xb(0x0f); + xb(opc); + xb(0xc0 | (reg & 7) << 3 | (rm & 7)); +} +static void mov_rr(int d, int s) { reg_op(0, 1, 0, 0x89, s, d); } +static void mov_rm(int r, int b, int disp) { mem_op(0, 1, 0, 0x8b, r, b, -1, 1, disp); } +static void mov_mr(int b, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, -1, 1, disp); } +static void mov_rx(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8b, r, b, x, sc, disp); } +static void mov_xr(int b, int x, int sc, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, x, sc, disp); } +static void mov_r32m(int r, int b, int disp) { mem_op(0, 0, 0, 0x8b, r, b, -1, 1, disp); } /* zero extends */ +static void mov_mi(int b, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, -1, 1, disp); xd((unsigned)imm); } +static void mov_r32i(int r, unsigned imm) { rex(0, 0, 0, r); xb(0xb8 | (r & 7)); xd(imm); } +static void mov_ri(int r, Cell v) +{ + UCell u = (UCell)v; + if (u <= 0xffffffffu) mov_r32i(r, (unsigned)u); + else if (v >= -2147483647 - 1 && v < 0) { reg_op(0, 1, 0, 0xc7, 0, r); xd((unsigned)(int)v); } + else { rex(1, 0, 0, r); xb(0xb8 | (r & 7)); xq(u); } +} +static void lea(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8d, r, b, x, sc, disp); } +static void xor32(int r) { reg_op(0, 0, 0, 0x31, r, r); } +static void alu_rr(int op, int d, int s) { reg_op(0, 1, 0, op * 8 + 1, s, d); } +static void alu_rm(int op, int d, int b, int disp) { mem_op(0, 1, 0, op * 8 + 3, d, b, -1, 1, disp); } +static void alu_ri(int op, int d, int imm) +{ + if (imm >= -128 && imm <= 127) { reg_op(0, 1, 0, 0x83, op, d); xb(imm); } + else { reg_op(0, 1, 0, 0x81, op, d); xd((unsigned)imm); } +} +static void alu_mi(int op, int b, int disp, int imm) +{ + if (imm >= -128 && imm <= 127) { mem_op(0, 1, 0, 0x83, op, b, -1, 1, disp); xb(imm); } + else { mem_op(0, 1, 0, 0x81, op, b, -1, 1, disp); xd((unsigned)imm); } +} +static void test_rr(int a, int b) { reg_op(0, 1, 0, 0x85, b, a); } +static void test32(int a, int b) { reg_op(0, 0, 0, 0x85, b, a); } +static void imul_rm(int d, int b, int disp) { mem_op(0, 1, 1, 0xaf, d, b, -1, 1, disp); } +static void imul_rmi(int d, int b, int disp, int imm) { mem_op(0, 1, 0, 0x69, d, b, -1, 1, disp); xd((unsigned)imm); } +static void shift_cl(int op, int r) { reg_op(0, 1, 0, 0xd3, op, r); } /* 4 shl, 5 shr, 7 sar */ +static void shift_ri(int op, int r, int n) { reg_op(0, 1, 0, 0xc1, op, r); xb(n); } +static void shift_mi(int op, int b, int disp, int n) { mem_op(0, 1, 0, 0xc1, op, b, -1, 1, disp); xb(n); } +static void setcc(int cc, int r) { reg_op(0, 0, 1, 0x90 + cc, 0, r); } /* al, cl or dl only */ +static void movsx_rr8(int d, int s) { reg_op(0, 0, 1, 0xbe, d, s); } /* movsx r32, r8 (al..dl) */ +static void movsx64_rr8(int d, int s) { reg_op(0, 1, 1, 0xbe, d, s); } +static void movsxd_rr(int d, int s) { reg_op(0, 1, 0, 0x63, d, s); } +static void push_r(int r) { rex(0, 0, 0, r); xb(0x50 | (r & 7)); } +static void pop_r(int r) { rex(0, 0, 0, r); xb(0x58 | (r & 7)); } +static void call_m(int b, int disp) { mem_op(0, 0, 0, 0xff, 2, b, -1, 1, disp); } +static void call_r(int r) { reg_op(0, 0, 0, 0xff, 2, r); } +static void jmp_r(int r) { reg_op(0, 0, 0, 0xff, 4, r); } +static void call_abs(void *f) { mov_ri(r_ax, (Cell)(uintptr_t)f); call_r(r_ax); } +/* branches. j8/j32 return the place to patch; the *_to forms go to a known address */ +static JW *j32(int cc) { JW *at; xb(0x0f); xb(0x80 + cc); at = cp; xd(0); return at; } +static JW *jm32(void) { JW *at; xb(0xe9); at = cp; xd(0); return at; } +static JW *j8(int cc) { JW *at; xb(0x70 + cc); at = cp; xb(0); return at; } +static void fix32(JW *at, JW *to) { int d = (int)(to - (at + 4)); memcpy(at, &d, 4); } +static void fix8(JW *at, JW *to) { *at = (JW)(to - (at + 1)); } +static void jcc_to(int cc, JW *to) { JW *at = j32(cc); fix32(at, to); } +static void jmp_to(JW *to) { JW *at = jm32(); fix32(at, to); } +/* sse2 on xmm0 and xmm1 only */ +static void sd_m(int opc, int x, int b, int disp) { mem_op(0xf2, 0, 1, opc, x, b, -1, 1, disp); } /* movsd 10/11, addsd 58, mulsd 59, subsd 5c, divsd 5e */ + +static void build_stubs(void) +{ + JW *ovd, *ovr, *at; + cp = code; + x_enter = cp; /* int enter(long *S, void *code) */ + push_r(r_bp); push_r(r_bx); push_r(r_12); push_r(r_13); push_r(r_14); push_r(r_15); + alu_ri(a_sub, r_sp, 8); /* six pushes and the return address: 16-byte aligned again */ + mov_rr(rST, r_di); + mov_rm(rMB, rST, 0); + mov_rm(r_ax, rST, 8); lea(rSP, rMB, r_ax, 8, 0); /* sp = m + s*8 */ + mov_rm(r_ax, rST, 16); lea(rRP, rMB, r_ax, 8, 0); + mov_rm(rTAB, rST, 24); + jmp_r(r_si); + x_exit = cp; /* eax = next vm address */ + mov_rr(r_cx, rSP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 8, r_cx); + mov_rr(r_cx, rRP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 16, r_cx); + alu_ri(a_add, r_sp, 8); + pop_r(r_15); pop_r(r_14); pop_r(r_13); pop_r(r_12); pop_r(r_bx); pop_r(r_bp); + xb(0xc3); + ovd = cp; + mov_r32i(r_ax, PC_DATA_OVERFLOW); jmp_to(x_exit); + ovr = cp; + mov_r32i(r_ax, PC_RETURN_OVERFLOW); jmp_to(x_exit); + x_disp = cp; /* eax = target: check cancellation and stacks before chaining */ + if (context_current) { + mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx); + at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp); + } + mov_mr(rST, 104, r_ax); + alu_rm(a_cmp, rSP, rST, 88); jcc_to(c_b, ovd); /* sp below its limit */ + alu_rm(a_cmp, rRP, rST, 96); jcc_to(c_ae, ovr); /* rp at its limit */ + mov_ri(r_cx, (Cell)(uintptr_t)&code_words); /* the current code reservation */ + alu_rm(a_cmp, r_ax, r_cx, 0); jcc_to(c_ae, x_exit); + mov_rm(rTAB, rST, 24); + mov_rx(r_cx, rTAB, r_ax, 8, 0); + test_rr(r_cx, r_cx); jcc_to(c_e, x_exit); + jmp_r(r_cx); + x_first = cp; +} +#endif static void flush(void) { @@ -3424,7 +3597,7 @@ static void jitwrite(int on) #endif } -static void syncicache(unsigned *a, unsigned *b) +static void syncicache(JW *a, JW *b) { #ifdef __APPLE__ sys_icache_invalidate(a, (char *)b - (char *)a); @@ -3457,7 +3630,11 @@ static int hook(Cell a, Cell v) if (hit) drop_epoch++; return hit; } +static int inst_hook(int d) { return install(d); } +static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); } +#endif +#ifdef ZYJIT /* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */ static void fjit(int k) { @@ -4240,9 +4417,6 @@ static void rcompile(Rg *R, int eager) if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1; /* registers, not space: give up on it */ } -static int inst_hook(int d) { return install(d); } -static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); } - static void *translate(int p) { unsigned *start; @@ -5032,7 +5206,7 @@ static Blk lcompile(int p) #endif /* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */ -#ifndef ZYLLVM +#if !defined(ZYLLVM) && !defined(ZYJIT_X64) static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */ #endif static VM_TLS Cell rvals[MAXUOP * 4 + 16]; @@ -5052,6 +5226,9 @@ static int rdrop(int a) /* a store landed on region code: drop those regions */ #ifdef ZYLLVM static int lhook(Cell a, Cell v); #define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0) +#elif defined(ZYJIT_X64) +/* a region store may land on translated code too: the jit's hook drops those. the gate word is a plain store, as in the region interpreter anywhere */ +#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (cov[a] && (a) != SYSGATE) hook(a, b); else m[a] = (b); } } while (0) #else #define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0) #endif @@ -5127,9 +5304,21 @@ static int rexec(Rg *R, int e, int *ps, int *pr) return (int)t; } #ifndef ZYLLVM +#ifndef ZYJIT_X64 static void be_drop(int j) { (void)j; } +#endif static void rforget(Rg *R) { (void)R; } +#ifdef ZYJIT_X64 +/* a region entry is never chained to: drop any translation that started there */ +static void rmark(Rg *R) +{ + int e; + R->marked = 1; cinc(rcov, R->lo, R->hi); + for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0; +} +#else static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); } +#endif static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); } static void rcompile(Rg *R, int eager) { (void)R; (void)eager; } #endif @@ -5692,6 +5881,381 @@ static void rcompile(Rg *R, int eager) if (!(R->nat = lregion(R))) R->nonat = 1; } #endif +#ifdef ZYJIT_X64 +/* ---- the x86-64 jit: block translation, and the loop that runs it ---- + the rules are the arm64 jit's: a block ends at call, jump, ret or halt; a jz + does not end it (the not-taken side keeps compiling); a literal followed by + call, jump or jz is a constant target; every exit goes through x_disp, which + checks both stacks against their limits, looks the target up in tab and + chains to it when it is already translated, and otherwise returns to c. the + instruction selection differs: within a block the data stack pointer is not + moved for every push and pop. vd counts the words pushed (negative) or + popped since r13 was last written, stack slot k from the top is + [r13 + 8 * (vd + k)], and r13 is brought up to date before every call into + c and every exit. o3 regions are not compiled here: a region entry is never + translated (the c loop sees it first), and the region interpreter runs it. */ +static VM_TLS int vd; +#define SL(k) (8 * (vd + (k))) +static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); } +static void stv(int k, int r) { mov_mr(rSP, SL(k), r); } +static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); } /* r13 up to date; vd stays for the other path */ +static void sp_sync(void) { sp_keep(); vd = 0; } +static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); } +static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); } /* a constant target: the low 32 bits, like the interpreter's int pc */ +static void disp_ax(void) { sp_keep(); jmp_to(x_disp); } +static void publish_stacks(void) /* s and r into S, as x_exit does, for hooks that call back into the vm */ +{ + lea(r_ax, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 8, r_ax); + mov_rr(r_ax, rRP); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 16, r_ax); +} +static void reload_stacks(void) /* r13, r14, r15 from S after c may have moved the stacks or the table */ +{ + mov_rm(r_cx, rST, 8); lea(rSP, rMB, r_cx, 8, 0); + mov_rm(r_cx, rST, 16); lea(rRP, rMB, r_cx, 8, 0); + mov_rm(rTAB, rST, 24); +} +/* sp = (r13 + 8 * vd - r12) / 8, the data stack index, into r */ +static void sp_index(int r) { lea(r, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r, rMB); shift_ri(5, r, 3); } + +/* a store of rdx to the word rax. known: the address is known to be inside memory. on a + covered word the hook runs and, if it dropped a translation, the block resumes in c at i + 1 */ +static void emit_store(int i, int known) +{ + JW *done1 = 0, *slow, *done2, *z; + if (!known) { alu_ri(a_cmp, r_ax, MW); done1 = j32(c_ae); } + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); /* cmp byte [rcx + rax], 0 */ + slow = j32(c_ne); + mov_xr(rMB, r_ax, 8, 0, r_dx); + done2 = jm32(); + fix32(slow, cp); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + publish_stacks(); + call_m(rST, 64); + reload_stacks(); + test32(r_ax, r_ax); + z = j32(c_e); + mov_r32i(r_ax, (unsigned)(i + 1)); jmp_to(x_exit); /* a translation died: resume at i + 1 in c; r13 is exact */ + fix32(z, cp); + if (vd) lea(rSP, rSP, -1, 1, -8 * vd); + if (done1) fix32(done1, cp); + fix32(done2, cp); +} +/* the word at rax, or 0 outside memory, into rcx */ +static void emit_load(void) +{ + JW *over; + xor32(r_cx); + alu_ri(a_cmp, r_ax, MW); + over = j8(c_ae); + mov_rx(r_cx, rMB, r_ax, 8, 0); + fix8(over, cp); +} +/* byte loads: zero extended, 0 unless the whole access lies in memory */ +static void emit_ld_bytes(int n) +{ + JW *over; + ldv(r_ax, 0); + xor32(r_cx); + mov_ri(r_dx, ((Cell)MW << 3) - (n - 1)); + alu_rr(a_cmp, r_ax, r_dx); + over = j8(c_ae); + if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0); + else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0); + else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0); + fix8(over, cp); + stv(0, r_cx); +} +/* c helper for byte stores: 1 if a translation died */ +static long xbst(Cell a, Cell v, long n) +{ + int e = drop_epoch; + b_store(a, v, (int)n); + return drop_epoch != e; +} +static void float_op(int i, int k) +{ + JW *ok; + if (k >= 601 && k <= 604) { + static const unsigned char opc[] = { 0x58, 0x5c, 0x59, 0x5e }; + sd_m(0x10, 0, rSP, SL(1)); + sd_m(opc[k - 601], 0, rSP, SL(0)); + vd++; + sd_m(0x11, 0, rSP, SL(0)); + } else if (k >= 605 && k <= 607) { + xor32(r_ax); + sd_m(0x10, 0, rSP, SL(0)); + mem_op(0x66, 0, 1, 0x2e, 0, rSP, -1, 1, SL(1)); /* ucomisd xmm0, [b]: a against b, unordered sets cf */ + if (k == 605) { setcc(c_e, r_ax); setcc(c_np, r_cx); reg_op(0, 0, 0, 0x20, r_cx, r_ax); } /* eq and not unordered */ + else setcc(k == 606 ? c_a : c_ae, r_ax); /* b < a is a > b; b <= a is a >= b */ + vd++; + stv(0, r_ax); + } else if (k == 608) { + mem_op(0xf2, 1, 1, 0x2a, 0, rSP, -1, 1, SL(0)); /* cvtsi2sd xmm0, qword [sp] */ + sd_m(0x11, 0, rSP, SL(0)); + } else if (k == 609) { /* truncate toward zero; the indefinite result goes to the c helper, which saturates and maps a nan to 0 */ + mem_op(0xf2, 1, 1, 0x2c, r_ax, rSP, -1, 1, SL(0)); /* cvttsd2si rax, [sp] */ + mov_ri(r_cx, (Cell)((UCell)1 << 63)); + alu_rr(a_cmp, r_ax, r_cx); + ok = j8(c_ne); + ldv(r_di, 0); call_abs((void *)fop_f2i); + fix8(ok, cp); + stv(0, r_ax); + } else if (k == 610) { + ldv(r_di, 0); vd++; call_abs((void *)fop_print); + } else if (k == 618) { /* (bits prec mode --) */ + ldv(r_di, 2); ldv(r_si, 1); ldv(r_dx, 0); vd += 3; call_abs((void *)fop_print_mode); + } else { /* 611 */ + ldv(r_di, 0); call_abs((void *)fop_parse_cell); stv(0, r_ax); + } + (void)i; +} + +static void *translate(int p) +{ + JW *start; + int i = p, n = 0, k, t, used[64], nused = 0, j2, ju; + Cell w, x; + if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush(); + start = cp; + vd = 0; + for (;;) { + if (i >= code_words) { exit_with(-1); break; } + if (n++ >= BLKW || (i != p && o3ent[i])) { disp_imm(i); break; } /* a region entry is entered from c, never by chaining */ + w = m[i]; + if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) { + t = (int)((w - 1) / 2); /* literal + call/jmp/jz with a known target */ + if (k == 8) { + JW *skip; + ldv(r_cx, 0); vd++; + test_rr(r_cx, r_cx); + skip = j32(c_ne); + disp_imm(t); + fix32(skip, cp); + i += 2; + continue; + } + if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); } + disp_imm(t); + i += 2; + break; + } + if (w & 1) { + Cell v = (w - 1) / 2; + int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1; + if (i + 1 < code_words && !(m[i + 1] & 1)) ko = (int)(m[i + 1] / 2); + if (ko == 1 && v >= 0 && v < MW && v < (1 << 27)) { /* a fixed address: one load */ + mov_rm(r_ax, rMB, (int)(8 * v)); vd--; stv(0, r_ax); + i += 2; continue; + } + if (small && ko >= 13 && ko <= 20) { /* the literal is an immediate operand */ + int d = (int)v; + switch (ko) { + case 13: alu_mi(a_add, rSP, SL(0), d); break; + case 14: alu_mi(a_sub, rSP, SL(0), d); break; + case 15: imul_rmi(r_ax, rSP, SL(0), d); stv(0, r_ax); break; + case 16: alu_mi(a_and, rSP, SL(0), d); break; + case 17: alu_mi(a_or, rSP, SL(0), d); break; + case 18: alu_mi(a_xor, rSP, SL(0), d); break; + case 19: if (d & 63) shift_mi(7, rSP, SL(0), d & 63); break; + default: xor32(r_cx); alu_mi(a_cmp, rSP, SL(0), d); setcc(c_l, r_cx); stv(0, r_cx); + } + i += 2; continue; + } + vd--; + if (small) mov_mi(rSP, SL(0), (int)v); + else { mov_ri(r_ax, v); stv(0, r_ax); } + i++; continue; + } + k = (int)(w / 2); + if (k == 0) { exit_with(-1); i++; break; } + if (k < 0) { exit_with(-1); i++; break; } /* reserved: the interpreter halts */ + if (k == 6 || k == 7) { + mov_r32m(r_ax, rSP, SL(0)); vd++; + if (k == 6) { mov_mi(rRP, 0, i + 1); alu_ri(a_add, rRP, 8); } + disp_ax(); + i++; + break; + } + if (k == 8) { /* jz: taken leaves, not taken keeps going */ + JW *skip; + mov_r32m(r_ax, rSP, SL(0)); ldv(r_cx, 1); vd += 2; + test_rr(r_cx, r_cx); + skip = j32(c_ne); + disp_ax(); + fix32(skip, cp); + i++; + continue; + } + if (k == 9) { alu_ri(a_sub, rRP, 8); mov_r32m(r_ax, rRP, 0); disp_ax(); i++; break; } + if (k == 21) { /* install: call into c, then return to it so it can flush */ + mov_r32m(r_di, rSP, SL(0)); call_m(rST, 72); + movsxd_rr(r_ax, r_ax); stv(0, r_ax); + exit_with(i + 1); i++; break; + } + if (k == 600) { /* flit: the next word is raw IEEE bits, even when bit 0 is set */ + if (i + 1 >= code_words) { exit_with(-1); i++; break; } + mov_ri(r_ax, m[i + 1]); vd--; stv(0, r_ax); + i += 2; + continue; + } + if ((k >= 601 && k <= 611) || k == 618) { float_op(i, k); i++; continue; } + if (k == 612 || k == 614 || k == 616) { emit_ld_bytes(k == 612 ? 1 : k == 614 ? 2 : 4); i++; continue; } + if (k == 613 || k == 615 || k == 617) { + JW *ok; + ldv(r_di, 0); ldv(r_si, 1); mov_r32i(r_dx, k == 613 ? 1 : k == 615 ? 2 : 4); vd += 2; + call_abs((void *)xbst); + test32(r_ax, r_ax); + ok = j32(c_e); + exit_with(i + 1); + fix32(ok, cp); + i++; continue; + } + if (k == 619) { /* push the data-stack index: the word s points at */ + sp_index(r_ax); vd--; stv(0, r_ax); + i++; continue; + } + if (k == 620) { /* a native call: the helper pops the arguments and pushes the result */ + lea(r_di, rSP, -1, 1, 8 * vd); mov_rr(r_si, rRP); + call_abs((void *)ncall_host); + mov_rr(rSP, r_ax); vd = 0; + i++; continue; + } + if (k >= XOP0 && k < XOP0 + NENH) { + if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; /* wired or not, an install makes this a different block */ + if (enh[k - XOP0].live) { /* the interpreter runs it: x0 = data stack, slot */ + lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0)); + call_m(rST, 80); + mov_rr(rSP, r_ax); vd = 0; + i++; continue; + } + } + if (k > 20) { exit_with(-1); i++; break; } /* not wired: halt */ + switch (k) { + case 1: + ldv(r_ax, 0); emit_load(); stv(0, r_cx); break; + case 2: /* store: inline unless the target word is translated */ + ldv(r_ax, 0); ldv(r_dx, 1); vd += 2; + emit_store(i, 0); break; + case 3: { /* pick uses a word address, with the same bounds as load */ + ldv(r_ax, 0); + sp_index(r_cx); + lea(r_ax, r_ax, r_cx, 1, 1); + emit_load(); stv(0, r_cx); break; } + case 4: vd++; break; + case 5: ldv(r_ax, 0); ldv(r_cx, 1); stv(0, r_cx); stv(1, r_ax); break; + case 10: + call_m(rST, 40); movsx64_rr8(r_ax, r_ax); vd--; stv(0, r_ax); break; + case 11: + mov_rm(r_ax, rST, 56); + if (sizeof J == 1) mem_op(0, 1, 1, 0xbe, r_ax, r_ax, -1, 1, 0); /* movsx rax, byte [rax] */ + else mem_op(0, 1, 0, 0x63, r_ax, r_ax, -1, 1, 0); /* movsxd rax, dword [rax] */ + vd--; stv(0, r_ax); break; + case 12: + ldv(r_ax, 0); movsx_rr8(r_di, r_ax); vd++; call_m(rST, 48); break; + case 13: case 14: case 16: case 17: case 18: + ldv(r_ax, 1); + alu_rm(k == 13 ? a_add : k == 14 ? a_sub : k == 16 ? a_and : k == 17 ? a_or : a_xor, r_ax, rSP, SL(0)); + vd++; stv(0, r_ax); break; + case 15: ldv(r_ax, 1); imul_rm(r_ax, rSP, SL(0)); vd++; stv(0, r_ax); break; + case 19: ldv(r_cx, 0); ldv(r_ax, 1); shift_cl(7, r_ax); vd++; stv(0, r_ax); break; + default: /* 20 */ + ldv(r_ax, 1); xor32(r_cx); alu_rm(a_cmp, r_ax, rSP, SL(0)); setcc(c_l, r_cx); vd++; stv(0, r_cx); + } + i++; + } + j2 = ix_add(p, i); + cinc(cov, p, i); + if (nused && used[0] < 0) dep_add(NENH, j2); + else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2); + tab[p] = start; + if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start)); + return start; +} +static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); } +static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr) +{ + union { JW *u; int (*f)(long *, void *); } enter; + void *saved_eq = 0, *saved_ent[RMAXE]; + int i, nent = 0, flushed = 0, ep0 = drop_epoch, term = 0; + Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1; + enter.u = x_enter; + if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; } + if (re) { + nent = re->ne; if (nent > RMAXE) nent = RMAXE; + for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; } + } + S[1] = *ps; S[2] = *pr; + x64_limits(); + while (!term && !context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) { + void *c; + if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) { + Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW; + if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break; + S[1] = sp; S[2] = rp; + x64_limits(); + pc = (int)S[13]; continue; + } + if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break; + if (stop_ge >= 0 && pc >= stop_ge) break; + if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break; + if (o3ent[pc]) { /* an o3 region: the region interpreter runs it */ + Rg *R = rat(pc); + if (R) { + int s = (int)S[1], r = (int)S[2]; + if (s < stack_data_limit() || r > stack_return_limit()) { + Cell sp = s, rp = r; + int ret = r > stack_return_limit(); + if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; } + s = (int)sp; r = (int)rp; + } + pc = rexec(R, o3ei[pc], &s, &r); + S[1] = s; S[2] = r; + x64_limits(); + if ((unsigned)pc >= (unsigned)code_words) term = 1; /* a stop, not a jit exit: no saved target to resume at */ + continue; + } + } + c = tab[pc]; + if (!c) { + jitwrite(1); c = translate(pc); jitwrite(0); + if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0; + if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0; + } + if (!c) break; + pc = enter.f(S, c); + if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); } + } + *ps = S[1]; *pr = S[2]; + flushed = drop_epoch != ep0; + if (!flushed) { + if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq; + if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i]; + } + return pc; +} +/* the code area, the stubs and the state block, for this thread */ +static int x64_boot(Cell sp, Cell rp, void *stop) +{ + code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (code == MAP_FAILED) { perror("mmap"); return 1; } + jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); + S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; + S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; + S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)stop; + x64_limits(); + cov[SYSGATE] = 1; + return 0; +} +static int run(int p) +{ + Cell s = DS0, r = RS0; + if (x64_boot(s, r, &context_default_stop)) return 1; + p = vm_until(p, -1, -1, 0, &s, &r); + halt_at = -1; + return stopped(p); +} +#else #define LD(x) (inmem(x) ? m[x] : 0) static VM_TLS int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest; static VM_TLS Rg *stop_rg; @@ -5839,6 +6403,7 @@ static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr) stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr; return back; } +#endif static int o3_ready(Rg *C) { if (!C || C->dead) return 0; @@ -5875,7 +6440,7 @@ static void b_commit(Cell w, Cell v) { if ((unsigned)w >= (unsigned)MW) return; if (w == SYSGATE) { m[w] = v; return; } -#if defined(ZYJIT) +#if defined(ZYNATIVE) if (cov[(int)w]) hook(w, v); else m[w] = v; #elif defined(ZYLLVM) if (lcov[(int)w]) lhook(w, v); else m[w] = v; @@ -5915,8 +6480,11 @@ static int code_reserve(Cell end) #define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0) CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1); CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap); +#ifdef ZYNATIVE + CODE_COLUMN(tab, code_words, cap); #ifdef ZYJIT - CODE_COLUMN(tab, code_words, cap); CODE_COLUMN(fn_nat, code_words, cap); + CODE_COLUMN(fn_nat, code_words, cap); +#endif #elif defined(ZYLLVM) CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap); #endif @@ -5928,7 +6496,7 @@ static int code_reserve(Cell end) } for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); } code_words = cap; -#ifdef ZYJIT +#ifdef ZYNATIVE S[3] = (long)tab; #endif return (int)cap; @@ -5940,7 +6508,7 @@ static int code_range_live(Cell start, Cell count) if (start < 0 || count <= 0 || start >= code_words) return 0; end = count > code_words - start ? code_words : start + count; for (i = start; i < end; i++) { -#ifdef ZYJIT +#ifdef ZYNATIVE if (cov[i]) return 1; #else if (rcov[i]) return 1; @@ -6005,8 +6573,11 @@ static void context_engine_free(void) { for (i = 0; i < nrg; i++) { rclear(rgs[i]); free(rgs[i]); } for (i = 0; i < NENH; i++) { free(enh[i].u); free(enh[i].reg); } free(pg_head); free(o3ent); free(o3ei); free(gn_); free(gh_); +#ifdef ZYNATIVE #ifdef ZYJIT - free(tab); free(fn_nat); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1); + free(fn_nat); +#endif + free(tab); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1); #else free(rcov); #ifdef ZYLLVM @@ -6030,7 +6601,7 @@ static void *context_worker(void *opaque) { memcpy(ffi_cb, context->callback_snapshot, sizeof ffi_cb); memcpy(ffi_sym, context->symbol_snapshot, sizeof ffi_sym); memcpy(ffi_lib, context->library_snapshot, sizeof ffi_lib); ffi_nsym = context->ffi_symbols; ffi_nlib = context->ffi_libraries; ffi_tried_libm = context->ffi_libm; #endif -#ifdef ZYJIT +#ifdef ZYNATIVE cov = calloc((size_t)MW, 1); #else rcov = calloc((size_t)MW, 1); @@ -6038,7 +6609,7 @@ static void *context_worker(void *opaque) { lcov = calloc((size_t)MW, 1); #endif #endif -#ifdef ZYJIT +#ifdef ZYNATIVE if (!cov) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } #else if (!rcov @@ -6053,6 +6624,10 @@ static void *context_worker(void *opaque) { nrg = context->region_count; context->region_count = 0; memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words); free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; +#ifdef ZYJIT_X64 + if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } + atomic_fetch_add(&context_code_live, 1); +#endif #ifdef ZYJIT #ifdef __APPLE__ code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0); @@ -6184,7 +6759,7 @@ int main(int argc, char **argv) if (memory_words < 0) return 1; if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; } m = calloc((size_t)MW, sizeof *m); -#ifdef ZYJIT +#ifdef ZYNATIVE cov = calloc((size_t)MW, 1); if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; } #else diff --git a/tests/drivers/jit_x64_test.sh b/tests/drivers/jit_x64_test.sh new file mode 100644 index 0000000..ce0438f --- /dev/null +++ b/tests/drivers/jit_x64_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# runs every program in tests/programs, examples and examples/learn on two cores +# and compares stdout and the exit status. usage: +# bash tests/drivers/jit_x64_test.sh <reference core> <jit core> [name-filter] +# the reference is normally the interpreter (built with -DZYINTERP) and the second +# core the x86-64 jit. each program is the core's stdin, with nothing after it, and +# runs with a timeout (JIT_TEST_TIMEOUT seconds, 120 by default). a program that +# differs is listed with the first lines of the difference; the exit status of +# this script is the number of programs that differ. +ref=${1:?usage: jit_x64_test.sh <reference core> <jit core> [filter]} +jit=${2:?usage: jit_x64_test.sh <reference core> <jit core> [filter]} +filter=${3:-} +limit=${JIT_TEST_TIMEOUT:-120} +root=$(cd "$(dirname "$0")/../.." && pwd) +cd "$root" || exit 2 +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +pass=0 fail=0 total=0 +for f in tests/programs/*.zc examples/*.zc examples/learn/*.zc; do + [ -n "$filter" ] && [[ "$f" != *$filter* ]] && continue + total=$((total + 1)) + timeout "$limit" "$ref" < "$f" > "$tmp/ref.out" 2> "$tmp/ref.err"; rr=$? + timeout "$limit" "$jit" < "$f" > "$tmp/jit.out" 2> "$tmp/jit.err"; rj=$? + if [ "$rr" = "$rj" ] && cmp -s "$tmp/ref.out" "$tmp/jit.out"; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + echo "DIFF $f (status $rr vs $rj)" + diff "$tmp/ref.out" "$tmp/jit.out" | head -6 + head -3 "$tmp/jit.err" + fi +done +echo "$pass of $total programs agree, $fail differ" +exit "$fail" |