x86-64 jit: translate ordinary vm blocks to x86-64 on linux, o3 regions on the region interpreter

3 files changed, 625 insertions, 15 deletions

diff --git a/note b/note
new file mode 100644
index 0000000..42fe16c
--- /dev/null
+++ b/note
@@ -0,0 +1 @@
+zy
\ No newline at end of file
diff --git a/src/core/core.c b/src/core/core.c
index dc50644..51d6b4f 100644
--- a/src/core/core.c
+++ b/src/core/core.c
@@ -53,6 +53,15 @@
 #include <libkern/OSCacheControl.h>
 #endif
 #endif
+/* the x86-64 jit: linux hosts only, and not with the interpreter forced or llvm asked for.
+   o3 regions run on the region interpreter there. ZYNATIVE is either jit. */
+#if defined(__x86_64__) && defined(__linux__) && !defined(ZYINTERP) && !defined(ZYLLVM) && !defined(ZYJIT)
+#define ZYJIT_X64 1
+#include <sys/mman.h>
+#endif
+#if defined(ZYJIT) || defined(ZYJIT_X64)
+#define ZYNATIVE 1
+#endif
 #if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK)
 #define ZYBIGSTACK 1
 #include <sys/mman.h>
@@ -3286,12 +3295,24 @@ static Rg *rat(int p)
 	return R;
 }
 
-#ifdef ZYJIT
+/* ---- the native jits' shared state and rules ----
+   both jits (arm64 and x86-64) keep the translation table, the coverage map, the
+   state block and the store hook here, so a store into translated code, a flush
+   and a dropped translation mean the same thing on either. the emitters and the
+   instruction selection are each jit's own. */
+#ifdef ZYNATIVE
 static VM_TLS unsigned char *cov;	/* how many live translations cover the word (255 sticks) */
 static VM_TLS void **tab;		/* vm address -> translation */
 static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); }
-static VM_TLS unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
+#ifdef ZYJIT
+typedef unsigned JW;		/* code is a string of 32-bit instruction words */
+#else
+typedef unsigned char JW;	/* code is a string of bytes */
+#define rcov cov		/* a lifted o3 region marks its words in the same map */
+#endif
+static VM_TLS JW *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
 static VM_TLS long S[15];	/* memory, stacks, hooks, limits, and the target saved on overflow */
+#ifdef ZYJIT
 static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc)
 {
 	o3_invoke(ps, pr, target, retpc);
@@ -3403,6 +3424,158 @@ static void build_stubs(void)
 	E(0xd61f0120);			/* br x9 */
 	x_first = cp;
 }
+#else
+/* ---- the x86-64 emitter ----
+   registers: r12 holds the memory base, r13 the data stack pointer, r14 the
+   return stack pointer, r15 the translation table and rbx the state block S
+   (the arm64 jit's x19..x23). rax, rcx, rdx, rsi, rdi and r8..r11 are scratch.
+   the callee-saved ones keep their meaning across calls into c, and rsp stays
+   16-byte aligned inside translated code so those calls are plain system v
+   calls. code is built in a byte buffer: a few encoders for the forms the
+   translator uses (modrm with base, index and displacement, an immediate,
+   or a register), then one function per vm operation. */
+enum { r_ax, r_cx, r_dx, r_bx, r_sp, r_bp, r_si, r_di, r_8, r_9, r_10, r_11, r_12, r_13, r_14, r_15 };
+enum { a_add = 0, a_or = 1, a_and = 4, a_sub = 5, a_xor = 6, a_cmp = 7 };	/* alu opcode groups */
+enum { c_b = 2, c_ae = 3, c_e = 4, c_ne = 5, c_be = 6, c_a = 7, c_np = 11, c_l = 12, c_ge = 13, c_le = 14, c_g = 15 };
+#define rMB r_12
+#define rSP r_13
+#define rRP r_14
+#define rTAB r_15
+#define rST r_bx
+
+static void xb(int b) { *cp++ = (unsigned char)b; }
+static void xd(unsigned v) { memcpy(cp, &v, 4); cp += 4; }
+static void xq(uint64_t v) { memcpy(cp, &v, 8); cp += 8; }
+static void rex(int w, int r, int x, int b)
+{
+	int v = 0x40 | (w ? 8 : 0) | ((r & 8) ? 4 : 0) | ((x & 8) ? 2 : 0) | ((b & 8) ? 1 : 0);
+	if (v != 0x40) xb(v);
+}
+/* modrm for [base + index * scale + disp]; index < 0 means none */
+static void modrm_mem(int reg, int base, int index, int scale, int disp)
+{
+	int mod, sib = index >= 0 || (base & 7) == 4;
+	if (disp == 0 && (base & 7) != 5) mod = 0;
+	else if (disp >= -128 && disp <= 127) mod = 1;
+	else mod = 2;
+	xb(mod << 6 | (reg & 7) << 3 | (sib ? 4 : (base & 7)));
+	if (sib) xb((scale == 8 ? 3 : scale == 4 ? 2 : scale == 2 ? 1 : 0) << 6 | ((index >= 0 ? index : 4) & 7) << 3 | (base & 7));
+	if (mod == 1) xb(disp); else if (mod == 2) xd((unsigned)disp);
+}
+/* pre: a mandatory prefix byte or 0; two: the 0f escape */
+static void mem_op(int pre, int w, int two, int opc, int reg, int base, int index, int scale, int disp)
+{
+	if (pre) xb(pre);
+	rex(w, reg, index < 0 ? 0 : index, base);
+	if (two) xb(0x0f);
+	xb(opc);
+	modrm_mem(reg, base, index, scale, disp);
+}
+static void reg_op(int pre, int w, int two, int opc, int reg, int rm)
+{
+	if (pre) xb(pre);
+	rex(w, reg, 0, rm);
+	if (two) xb(0x0f);
+	xb(opc);
+	xb(0xc0 | (reg & 7) << 3 | (rm & 7));
+}
+static void mov_rr(int d, int s) { reg_op(0, 1, 0, 0x89, s, d); }
+static void mov_rm(int r, int b, int disp) { mem_op(0, 1, 0, 0x8b, r, b, -1, 1, disp); }
+static void mov_mr(int b, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, -1, 1, disp); }
+static void mov_rx(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8b, r, b, x, sc, disp); }
+static void mov_xr(int b, int x, int sc, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, x, sc, disp); }
+static void mov_r32m(int r, int b, int disp) { mem_op(0, 0, 0, 0x8b, r, b, -1, 1, disp); }	/* zero extends */
+static void mov_mi(int b, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, -1, 1, disp); xd((unsigned)imm); }
+static void mov_r32i(int r, unsigned imm) { rex(0, 0, 0, r); xb(0xb8 | (r & 7)); xd(imm); }
+static void mov_ri(int r, Cell v)
+{
+	UCell u = (UCell)v;
+	if (u <= 0xffffffffu) mov_r32i(r, (unsigned)u);
+	else if (v >= -2147483647 - 1 && v < 0) { reg_op(0, 1, 0, 0xc7, 0, r); xd((unsigned)(int)v); }
+	else { rex(1, 0, 0, r); xb(0xb8 | (r & 7)); xq(u); }
+}
+static void lea(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8d, r, b, x, sc, disp); }
+static void xor32(int r) { reg_op(0, 0, 0, 0x31, r, r); }
+static void alu_rr(int op, int d, int s) { reg_op(0, 1, 0, op * 8 + 1, s, d); }
+static void alu_rm(int op, int d, int b, int disp) { mem_op(0, 1, 0, op * 8 + 3, d, b, -1, 1, disp); }
+static void alu_ri(int op, int d, int imm)
+{
+	if (imm >= -128 && imm <= 127) { reg_op(0, 1, 0, 0x83, op, d); xb(imm); }
+	else { reg_op(0, 1, 0, 0x81, op, d); xd((unsigned)imm); }
+}
+static void alu_mi(int op, int b, int disp, int imm)
+{
+	if (imm >= -128 && imm <= 127) { mem_op(0, 1, 0, 0x83, op, b, -1, 1, disp); xb(imm); }
+	else { mem_op(0, 1, 0, 0x81, op, b, -1, 1, disp); xd((unsigned)imm); }
+}
+static void test_rr(int a, int b) { reg_op(0, 1, 0, 0x85, b, a); }
+static void test32(int a, int b) { reg_op(0, 0, 0, 0x85, b, a); }
+static void imul_rm(int d, int b, int disp) { mem_op(0, 1, 1, 0xaf, d, b, -1, 1, disp); }
+static void imul_rmi(int d, int b, int disp, int imm) { mem_op(0, 1, 0, 0x69, d, b, -1, 1, disp); xd((unsigned)imm); }
+static void shift_cl(int op, int r) { reg_op(0, 1, 0, 0xd3, op, r); }	/* 4 shl, 5 shr, 7 sar */
+static void shift_ri(int op, int r, int n) { reg_op(0, 1, 0, 0xc1, op, r); xb(n); }
+static void shift_mi(int op, int b, int disp, int n) { mem_op(0, 1, 0, 0xc1, op, b, -1, 1, disp); xb(n); }
+static void setcc(int cc, int r) { reg_op(0, 0, 1, 0x90 + cc, 0, r); }	/* al, cl or dl only */
+static void movsx_rr8(int d, int s) { reg_op(0, 0, 1, 0xbe, d, s); }		/* movsx r32, r8 (al..dl) */
+static void movsx64_rr8(int d, int s) { reg_op(0, 1, 1, 0xbe, d, s); }
+static void movsxd_rr(int d, int s) { reg_op(0, 1, 0, 0x63, d, s); }
+static void push_r(int r) { rex(0, 0, 0, r); xb(0x50 | (r & 7)); }
+static void pop_r(int r) { rex(0, 0, 0, r); xb(0x58 | (r & 7)); }
+static void call_m(int b, int disp) { mem_op(0, 0, 0, 0xff, 2, b, -1, 1, disp); }
+static void call_r(int r) { reg_op(0, 0, 0, 0xff, 2, r); }
+static void jmp_r(int r) { reg_op(0, 0, 0, 0xff, 4, r); }
+static void call_abs(void *f) { mov_ri(r_ax, (Cell)(uintptr_t)f); call_r(r_ax); }
+/* branches. j8/j32 return the place to patch; the *_to forms go to a known address */
+static JW *j32(int cc) { JW *at; xb(0x0f); xb(0x80 + cc); at = cp; xd(0); return at; }
+static JW *jm32(void) { JW *at; xb(0xe9); at = cp; xd(0); return at; }
+static JW *j8(int cc) { JW *at; xb(0x70 + cc); at = cp; xb(0); return at; }
+static void fix32(JW *at, JW *to) { int d = (int)(to - (at + 4)); memcpy(at, &d, 4); }
+static void fix8(JW *at, JW *to) { *at = (JW)(to - (at + 1)); }
+static void jcc_to(int cc, JW *to) { JW *at = j32(cc); fix32(at, to); }
+static void jmp_to(JW *to) { JW *at = jm32(); fix32(at, to); }
+/* sse2 on xmm0 and xmm1 only */
+static void sd_m(int opc, int x, int b, int disp) { mem_op(0xf2, 0, 1, opc, x, b, -1, 1, disp); }	/* movsd 10/11, addsd 58, mulsd 59, subsd 5c, divsd 5e */
+
+static void build_stubs(void)
+{
+	JW *ovd, *ovr, *at;
+	cp = code;
+	x_enter = cp;			/* int enter(long *S, void *code) */
+	push_r(r_bp); push_r(r_bx); push_r(r_12); push_r(r_13); push_r(r_14); push_r(r_15);
+	alu_ri(a_sub, r_sp, 8);		/* six pushes and the return address: 16-byte aligned again */
+	mov_rr(rST, r_di);
+	mov_rm(rMB, rST, 0);
+	mov_rm(r_ax, rST, 8); lea(rSP, rMB, r_ax, 8, 0);		/* sp = m + s*8 */
+	mov_rm(r_ax, rST, 16); lea(rRP, rMB, r_ax, 8, 0);
+	mov_rm(rTAB, rST, 24);
+	jmp_r(r_si);
+	x_exit = cp;			/* eax = next vm address */
+	mov_rr(r_cx, rSP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 8, r_cx);
+	mov_rr(r_cx, rRP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 16, r_cx);
+	alu_ri(a_add, r_sp, 8);
+	pop_r(r_15); pop_r(r_14); pop_r(r_13); pop_r(r_12); pop_r(r_bx); pop_r(r_bp);
+	xb(0xc3);
+	ovd = cp;
+	mov_r32i(r_ax, PC_DATA_OVERFLOW); jmp_to(x_exit);
+	ovr = cp;
+	mov_r32i(r_ax, PC_RETURN_OVERFLOW); jmp_to(x_exit);
+	x_disp = cp;			/* eax = target: check cancellation and stacks before chaining */
+	if (context_current) {
+		mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx);
+		at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp);
+	}
+	mov_mr(rST, 104, r_ax);
+	alu_rm(a_cmp, rSP, rST, 88); jcc_to(c_b, ovd);		/* sp below its limit */
+	alu_rm(a_cmp, rRP, rST, 96); jcc_to(c_ae, ovr);		/* rp at its limit */
+	mov_ri(r_cx, (Cell)(uintptr_t)&code_words);		/* the current code reservation */
+	alu_rm(a_cmp, r_ax, r_cx, 0); jcc_to(c_ae, x_exit);
+	mov_rm(rTAB, rST, 24);
+	mov_rx(r_cx, rTAB, r_ax, 8, 0);
+	test_rr(r_cx, r_cx); jcc_to(c_e, x_exit);
+	jmp_r(r_cx);
+	x_first = cp;
+}
+#endif
 
 static void flush(void)
 {
@@ -3424,7 +3597,7 @@ static void jitwrite(int on)
 #endif
 }
 
-static void syncicache(unsigned *a, unsigned *b)
+static void syncicache(JW *a, JW *b)
 {
 #ifdef __APPLE__
 	sys_icache_invalidate(a, (char *)b - (char *)a);
@@ -3457,7 +3630,11 @@ static int hook(Cell a, Cell v)
 	if (hit) drop_epoch++;
 	return hit;
 }
+static int inst_hook(int d) { return install(d); }
+static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
+#endif
 
+#ifdef ZYJIT
 /* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */
 static void fjit(int k)
 {
@@ -4240,9 +4417,6 @@ static void rcompile(Rg *R, int eager)
 	if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1;	/* registers, not space: give up on it */
 }
 
-static int inst_hook(int d) { return install(d); }
-static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
-
 static void *translate(int p)
 {
 	unsigned *start;
@@ -5032,7 +5206,7 @@ static Blk lcompile(int p)
 #endif
 
 /* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */
-#ifndef ZYLLVM
+#if !defined(ZYLLVM) && !defined(ZYJIT_X64)
 static VM_TLS unsigned char *rcov;	/* 1 on the words of a lifted region */
 #endif
 static VM_TLS Cell rvals[MAXUOP * 4 + 16];
@@ -5052,6 +5226,9 @@ static int rdrop(int a)		/* a store landed on region code: drop those regions */
 #ifdef ZYLLVM
 static int lhook(Cell a, Cell v);
 #define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0)
+#elif defined(ZYJIT_X64)
+/* a region store may land on translated code too: the jit's hook drops those. the gate word is a plain store, as in the region interpreter anywhere */
+#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (cov[a] && (a) != SYSGATE) hook(a, b); else m[a] = (b); } } while (0)
 #else
 #define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0)
 #endif
@@ -5127,9 +5304,21 @@ static int rexec(Rg *R, int e, int *ps, int *pr)
 	return (int)t;
 }
 #ifndef ZYLLVM
+#ifndef ZYJIT_X64
 static void be_drop(int j) { (void)j; }
+#endif
 static void rforget(Rg *R) { (void)R; }
+#ifdef ZYJIT_X64
+/* a region entry is never chained to: drop any translation that started there */
+static void rmark(Rg *R)
+{
+	int e;
+	R->marked = 1; cinc(rcov, R->lo, R->hi);
+	for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0;
+}
+#else
 static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); }
+#endif
 static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); }
 static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
 #endif
@@ -5692,6 +5881,381 @@ static void rcompile(Rg *R, int eager)
 	if (!(R->nat = lregion(R))) R->nonat = 1;
 }
 #endif
+#ifdef ZYJIT_X64
+/* ---- the x86-64 jit: block translation, and the loop that runs it ----
+   the rules are the arm64 jit's: a block ends at call, jump, ret or halt; a jz
+   does not end it (the not-taken side keeps compiling); a literal followed by
+   call, jump or jz is a constant target; every exit goes through x_disp, which
+   checks both stacks against their limits, looks the target up in tab and
+   chains to it when it is already translated, and otherwise returns to c. the
+   instruction selection differs: within a block the data stack pointer is not
+   moved for every push and pop. vd counts the words pushed (negative) or
+   popped since r13 was last written, stack slot k from the top is
+   [r13 + 8 * (vd + k)], and r13 is brought up to date before every call into
+   c and every exit. o3 regions are not compiled here: a region entry is never
+   translated (the c loop sees it first), and the region interpreter runs it. */
+static VM_TLS int vd;
+#define SL(k) (8 * (vd + (k)))
+static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); }
+static void stv(int k, int r) { mov_mr(rSP, SL(k), r); }
+static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); }	/* r13 up to date; vd stays for the other path */
+static void sp_sync(void) { sp_keep(); vd = 0; }
+static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); }
+static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); }	/* a constant target: the low 32 bits, like the interpreter's int pc */
+static void disp_ax(void) { sp_keep(); jmp_to(x_disp); }
+static void publish_stacks(void)	/* s and r into S, as x_exit does, for hooks that call back into the vm */
+{
+	lea(r_ax, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 8, r_ax);
+	mov_rr(r_ax, rRP); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 16, r_ax);
+}
+static void reload_stacks(void)	/* r13, r14, r15 from S after c may have moved the stacks or the table */
+{
+	mov_rm(r_cx, rST, 8); lea(rSP, rMB, r_cx, 8, 0);
+	mov_rm(r_cx, rST, 16); lea(rRP, rMB, r_cx, 8, 0);
+	mov_rm(rTAB, rST, 24);
+}
+/* sp = (r13 + 8 * vd - r12) / 8, the data stack index, into r */
+static void sp_index(int r) { lea(r, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r, rMB); shift_ri(5, r, 3); }
+
+/* a store of rdx to the word rax. known: the address is known to be inside memory. on a
+   covered word the hook runs and, if it dropped a translation, the block resumes in c at i + 1 */
+static void emit_store(int i, int known)
+{
+	JW *done1 = 0, *slow, *done2, *z;
+	if (!known) { alu_ri(a_cmp, r_ax, MW); done1 = j32(c_ae); }
+	mov_rm(r_cx, rST, 32);
+	mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0);		/* cmp byte [rcx + rax], 0 */
+	slow = j32(c_ne);
+	mov_xr(rMB, r_ax, 8, 0, r_dx);
+	done2 = jm32();
+	fix32(slow, cp);
+	mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
+	publish_stacks();
+	call_m(rST, 64);
+	reload_stacks();
+	test32(r_ax, r_ax);
+	z = j32(c_e);
+	mov_r32i(r_ax, (unsigned)(i + 1)); jmp_to(x_exit);	/* a translation died: resume at i + 1 in c; r13 is exact */
+	fix32(z, cp);
+	if (vd) lea(rSP, rSP, -1, 1, -8 * vd);
+	if (done1) fix32(done1, cp);
+	fix32(done2, cp);
+}
+/* the word at rax, or 0 outside memory, into rcx */
+static void emit_load(void)
+{
+	JW *over;
+	xor32(r_cx);
+	alu_ri(a_cmp, r_ax, MW);
+	over = j8(c_ae);
+	mov_rx(r_cx, rMB, r_ax, 8, 0);
+	fix8(over, cp);
+}
+/* byte loads: zero extended, 0 unless the whole access lies in memory */
+static void emit_ld_bytes(int n)
+{
+	JW *over;
+	ldv(r_ax, 0);
+	xor32(r_cx);
+	mov_ri(r_dx, ((Cell)MW << 3) - (n - 1));
+	alu_rr(a_cmp, r_ax, r_dx);
+	over = j8(c_ae);
+	if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0);
+	else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0);
+	else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0);
+	fix8(over, cp);
+	stv(0, r_cx);
+}
+/* c helper for byte stores: 1 if a translation died */
+static long xbst(Cell a, Cell v, long n)
+{
+	int e = drop_epoch;
+	b_store(a, v, (int)n);
+	return drop_epoch != e;
+}
+static void float_op(int i, int k)
+{
+	JW *ok;
+	if (k >= 601 && k <= 604) {
+		static const unsigned char opc[] = { 0x58, 0x5c, 0x59, 0x5e };
+		sd_m(0x10, 0, rSP, SL(1));
+		sd_m(opc[k - 601], 0, rSP, SL(0));
+		vd++;
+		sd_m(0x11, 0, rSP, SL(0));
+	} else if (k >= 605 && k <= 607) {
+		xor32(r_ax);
+		sd_m(0x10, 0, rSP, SL(0));
+		mem_op(0x66, 0, 1, 0x2e, 0, rSP, -1, 1, SL(1));		/* ucomisd xmm0, [b]: a against b, unordered sets cf */
+		if (k == 605) { setcc(c_e, r_ax); setcc(c_np, r_cx); reg_op(0, 0, 0, 0x20, r_cx, r_ax); }	/* eq and not unordered */
+		else setcc(k == 606 ? c_a : c_ae, r_ax);			/* b < a is a > b; b <= a is a >= b */
+		vd++;
+		stv(0, r_ax);
+	} else if (k == 608) {
+		mem_op(0xf2, 1, 1, 0x2a, 0, rSP, -1, 1, SL(0));		/* cvtsi2sd xmm0, qword [sp] */
+		sd_m(0x11, 0, rSP, SL(0));
+	} else if (k == 609) {	/* truncate toward zero; the indefinite result goes to the c helper, which saturates and maps a nan to 0 */
+		mem_op(0xf2, 1, 1, 0x2c, r_ax, rSP, -1, 1, SL(0));	/* cvttsd2si rax, [sp] */
+		mov_ri(r_cx, (Cell)((UCell)1 << 63));
+		alu_rr(a_cmp, r_ax, r_cx);
+		ok = j8(c_ne);
+		ldv(r_di, 0); call_abs((void *)fop_f2i);
+		fix8(ok, cp);
+		stv(0, r_ax);
+	} else if (k == 610) {
+		ldv(r_di, 0); vd++; call_abs((void *)fop_print);
+	} else if (k == 618) {	/* (bits prec mode --) */
+		ldv(r_di, 2); ldv(r_si, 1); ldv(r_dx, 0); vd += 3; call_abs((void *)fop_print_mode);
+	} else {		/* 611 */
+		ldv(r_di, 0); call_abs((void *)fop_parse_cell); stv(0, r_ax);
+	}
+	(void)i;
+}
+
+static void *translate(int p)
+{
+	JW *start;
+	int i = p, n = 0, k, t, used[64], nused = 0, j2, ju;
+	Cell w, x;
+	if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush();
+	start = cp;
+	vd = 0;
+	for (;;) {
+		if (i >= code_words) { exit_with(-1); break; }
+		if (n++ >= BLKW || (i != p && o3ent[i])) { disp_imm(i); break; }	/* a region entry is entered from c, never by chaining */
+		w = m[i];
+		if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
+			t = (int)((w - 1) / 2);	/* literal + call/jmp/jz with a known target */
+			if (k == 8) {
+				JW *skip;
+				ldv(r_cx, 0); vd++;
+				test_rr(r_cx, r_cx);
+				skip = j32(c_ne);
+				disp_imm(t);
+				fix32(skip, cp);
+				i += 2;
+				continue;
+			}
+			if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); }
+			disp_imm(t);
+			i += 2;
+			break;
+		}
+		if (w & 1) {
+			Cell v = (w - 1) / 2;
+			int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1;
+			if (i + 1 < code_words && !(m[i + 1] & 1)) ko = (int)(m[i + 1] / 2);
+			if (ko == 1 && v >= 0 && v < MW && v < (1 << 27)) {	/* a fixed address: one load */
+				mov_rm(r_ax, rMB, (int)(8 * v)); vd--; stv(0, r_ax);
+				i += 2; continue;
+			}
+			if (small && ko >= 13 && ko <= 20) {	/* the literal is an immediate operand */
+				int d = (int)v;
+				switch (ko) {
+				case 13: alu_mi(a_add, rSP, SL(0), d); break;
+				case 14: alu_mi(a_sub, rSP, SL(0), d); break;
+				case 15: imul_rmi(r_ax, rSP, SL(0), d); stv(0, r_ax); break;
+				case 16: alu_mi(a_and, rSP, SL(0), d); break;
+				case 17: alu_mi(a_or, rSP, SL(0), d); break;
+				case 18: alu_mi(a_xor, rSP, SL(0), d); break;
+				case 19: if (d & 63) shift_mi(7, rSP, SL(0), d & 63); break;
+				default: xor32(r_cx); alu_mi(a_cmp, rSP, SL(0), d); setcc(c_l, r_cx); stv(0, r_cx);
+				}
+				i += 2; continue;
+			}
+			vd--;
+			if (small) mov_mi(rSP, SL(0), (int)v);
+			else { mov_ri(r_ax, v); stv(0, r_ax); }
+			i++; continue;
+		}
+		k = (int)(w / 2);
+		if (k == 0) { exit_with(-1); i++; break; }
+		if (k < 0) { exit_with(-1); i++; break; }	/* reserved: the interpreter halts */
+		if (k == 6 || k == 7) {
+			mov_r32m(r_ax, rSP, SL(0)); vd++;
+			if (k == 6) { mov_mi(rRP, 0, i + 1); alu_ri(a_add, rRP, 8); }
+			disp_ax();
+			i++;
+			break;
+		}
+		if (k == 8) {	/* jz: taken leaves, not taken keeps going */
+			JW *skip;
+			mov_r32m(r_ax, rSP, SL(0)); ldv(r_cx, 1); vd += 2;
+			test_rr(r_cx, r_cx);
+			skip = j32(c_ne);
+			disp_ax();
+			fix32(skip, cp);
+			i++;
+			continue;
+		}
+		if (k == 9) { alu_ri(a_sub, rRP, 8); mov_r32m(r_ax, rRP, 0); disp_ax(); i++; break; }
+		if (k == 21) {	/* install: call into c, then return to it so it can flush */
+			mov_r32m(r_di, rSP, SL(0)); call_m(rST, 72);
+			movsxd_rr(r_ax, r_ax); stv(0, r_ax);
+			exit_with(i + 1); i++; break;
+		}
+		if (k == 600) {	/* flit: the next word is raw IEEE bits, even when bit 0 is set */
+			if (i + 1 >= code_words) { exit_with(-1); i++; break; }
+			mov_ri(r_ax, m[i + 1]); vd--; stv(0, r_ax);
+			i += 2;
+			continue;
+		}
+		if ((k >= 601 && k <= 611) || k == 618) { float_op(i, k); i++; continue; }
+		if (k == 612 || k == 614 || k == 616) { emit_ld_bytes(k == 612 ? 1 : k == 614 ? 2 : 4); i++; continue; }
+		if (k == 613 || k == 615 || k == 617) {
+			JW *ok;
+			ldv(r_di, 0); ldv(r_si, 1); mov_r32i(r_dx, k == 613 ? 1 : k == 615 ? 2 : 4); vd += 2;
+			call_abs((void *)xbst);
+			test32(r_ax, r_ax);
+			ok = j32(c_e);
+			exit_with(i + 1);
+			fix32(ok, cp);
+			i++; continue;
+		}
+		if (k == 619) {	/* push the data-stack index: the word s points at */
+			sp_index(r_ax); vd--; stv(0, r_ax);
+			i++; continue;
+		}
+		if (k == 620) {	/* a native call: the helper pops the arguments and pushes the result */
+			lea(r_di, rSP, -1, 1, 8 * vd); mov_rr(r_si, rRP);
+			call_abs((void *)ncall_host);
+			mov_rr(rSP, r_ax); vd = 0;
+			i++; continue;
+		}
+		if (k >= XOP0 && k < XOP0 + NENH) {
+			if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1;	/* wired or not, an install makes this a different block */
+			if (enh[k - XOP0].live) {	/* the interpreter runs it: x0 = data stack, slot */
+				lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0));
+				call_m(rST, 80);
+				mov_rr(rSP, r_ax); vd = 0;
+				i++; continue;
+			}
+		}
+		if (k > 20) { exit_with(-1); i++; break; }	/* not wired: halt */
+		switch (k) {
+		case 1:
+			ldv(r_ax, 0); emit_load(); stv(0, r_cx); break;
+		case 2:	/* store: inline unless the target word is translated */
+			ldv(r_ax, 0); ldv(r_dx, 1); vd += 2;
+			emit_store(i, 0); break;
+		case 3: {	/* pick uses a word address, with the same bounds as load */
+			ldv(r_ax, 0);
+			sp_index(r_cx);
+			lea(r_ax, r_ax, r_cx, 1, 1);
+			emit_load(); stv(0, r_cx); break; }
+		case 4: vd++; break;
+		case 5: ldv(r_ax, 0); ldv(r_cx, 1); stv(0, r_cx); stv(1, r_ax); break;
+		case 10:
+			call_m(rST, 40); movsx64_rr8(r_ax, r_ax); vd--; stv(0, r_ax); break;
+		case 11:
+			mov_rm(r_ax, rST, 56);
+			if (sizeof J == 1) mem_op(0, 1, 1, 0xbe, r_ax, r_ax, -1, 1, 0);	/* movsx rax, byte [rax] */
+			else mem_op(0, 1, 0, 0x63, r_ax, r_ax, -1, 1, 0);		/* movsxd rax, dword [rax] */
+			vd--; stv(0, r_ax); break;
+		case 12:
+			ldv(r_ax, 0); movsx_rr8(r_di, r_ax); vd++; call_m(rST, 48); break;
+		case 13: case 14: case 16: case 17: case 18:
+			ldv(r_ax, 1);
+			alu_rm(k == 13 ? a_add : k == 14 ? a_sub : k == 16 ? a_and : k == 17 ? a_or : a_xor, r_ax, rSP, SL(0));
+			vd++; stv(0, r_ax); break;
+		case 15: ldv(r_ax, 1); imul_rm(r_ax, rSP, SL(0)); vd++; stv(0, r_ax); break;
+		case 19: ldv(r_cx, 0); ldv(r_ax, 1); shift_cl(7, r_ax); vd++; stv(0, r_ax); break;
+		default:	/* 20 */
+			ldv(r_ax, 1); xor32(r_cx); alu_rm(a_cmp, r_ax, rSP, SL(0)); setcc(c_l, r_cx); vd++; stv(0, r_cx);
+		}
+		i++;
+	}
+	j2 = ix_add(p, i);
+	cinc(cov, p, i);
+	if (nused && used[0] < 0) dep_add(NENH, j2);
+	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2);
+	tab[p] = start;
+	if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start));
+	return start;
+}
+static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); }
+static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr)
+{
+	union { JW *u; int (*f)(long *, void *); } enter;
+	void *saved_eq = 0, *saved_ent[RMAXE];
+	int i, nent = 0, flushed = 0, ep0 = drop_epoch, term = 0;
+	Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1;
+	enter.u = x_enter;
+	if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; }
+	if (re) {
+		nent = re->ne; if (nent > RMAXE) nent = RMAXE;
+		for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; }
+	}
+	S[1] = *ps; S[2] = *pr;
+	x64_limits();
+	while (!term && !context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) {
+		void *c;
+		if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
+			Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW;
+			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
+			S[1] = sp; S[2] = rp;
+			x64_limits();
+			pc = (int)S[13]; continue;
+		}
+		if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break;
+		if (stop_ge >= 0 && pc >= stop_ge) break;
+		if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break;
+		if (o3ent[pc]) {	/* an o3 region: the region interpreter runs it */
+			Rg *R = rat(pc);
+			if (R) {
+				int s = (int)S[1], r = (int)S[2];
+				if (s < stack_data_limit() || r > stack_return_limit()) {
+					Cell sp = s, rp = r;
+					int ret = r > stack_return_limit();
+					if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
+					s = (int)sp; r = (int)rp;
+				}
+				pc = rexec(R, o3ei[pc], &s, &r);
+				S[1] = s; S[2] = r;
+				x64_limits();
+				if ((unsigned)pc >= (unsigned)code_words) term = 1;	/* a stop, not a jit exit: no saved target to resume at */
+				continue;
+			}
+		}
+		c = tab[pc];
+		if (!c) {
+			jitwrite(1); c = translate(pc); jitwrite(0);
+			if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0;
+			if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0;
+		}
+		if (!c) break;
+		pc = enter.f(S, c);
+		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
+	}
+	*ps = S[1]; *pr = S[2];
+	flushed = drop_epoch != ep0;
+	if (!flushed) {
+		if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq;
+		if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i];
+	}
+	return pc;
+}
+/* the code area, the stubs and the state block, for this thread */
+static int x64_boot(Cell sp, Cell rp, void *stop)
+{
+	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (code == MAP_FAILED) { perror("mmap"); return 1; }
+	jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
+	S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
+	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
+	S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)stop;
+	x64_limits();
+	cov[SYSGATE] = 1;
+	return 0;
+}
+static int run(int p)
+{
+	Cell s = DS0, r = RS0;
+	if (x64_boot(s, r, &context_default_stop)) return 1;
+	p = vm_until(p, -1, -1, 0, &s, &r);
+	halt_at = -1;
+	return stopped(p);
+}
+#else
 #define LD(x) (inmem(x) ? m[x] : 0)
 static VM_TLS int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest;
 static VM_TLS Rg *stop_rg;
@@ -5839,6 +6403,7 @@ static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr)
 	stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr;
 	return back;
 }
+#endif
 static int o3_ready(Rg *C)
 {
 	if (!C || C->dead) return 0;
@@ -5875,7 +6440,7 @@ static void b_commit(Cell w, Cell v)
 {
 	if ((unsigned)w >= (unsigned)MW) return;
 	if (w == SYSGATE) { m[w] = v; return; }
-#if defined(ZYJIT)
+#if defined(ZYNATIVE)
 	if (cov[(int)w]) hook(w, v); else m[w] = v;
 #elif defined(ZYLLVM)
 	if (lcov[(int)w]) lhook(w, v); else m[w] = v;
@@ -5915,8 +6480,11 @@ static int code_reserve(Cell end)
 #define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0)
 	CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1);
 	CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap);
+#ifdef ZYNATIVE
+	CODE_COLUMN(tab, code_words, cap);
 #ifdef ZYJIT
-	CODE_COLUMN(tab, code_words, cap); CODE_COLUMN(fn_nat, code_words, cap);
+	CODE_COLUMN(fn_nat, code_words, cap);
+#endif
 #elif defined(ZYLLVM)
 	CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap);
 #endif
@@ -5928,7 +6496,7 @@ static int code_reserve(Cell end)
 	}
 	for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); }
 	code_words = cap;
-#ifdef ZYJIT
+#ifdef ZYNATIVE
 	S[3] = (long)tab;
 #endif
 	return (int)cap;
@@ -5940,7 +6508,7 @@ static int code_range_live(Cell start, Cell count)
 	if (start < 0 || count <= 0 || start >= code_words) return 0;
 	end = count > code_words - start ? code_words : start + count;
 	for (i = start; i < end; i++) {
-#ifdef ZYJIT
+#ifdef ZYNATIVE
 		if (cov[i]) return 1;
 #else
 		if (rcov[i]) return 1;
@@ -6005,8 +6573,11 @@ static void context_engine_free(void) {
  for (i = 0; i < nrg; i++) { rclear(rgs[i]); free(rgs[i]); }
  for (i = 0; i < NENH; i++) { free(enh[i].u); free(enh[i].reg); }
  free(pg_head); free(o3ent); free(o3ei); free(gn_); free(gh_);
+#ifdef ZYNATIVE
 #ifdef ZYJIT
- free(tab); free(fn_nat); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1);
+ free(fn_nat);
+#endif
+ free(tab); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1);
 #else
  free(rcov);
 #ifdef ZYLLVM
@@ -6030,7 +6601,7 @@ static void *context_worker(void *opaque) {
  memcpy(ffi_cb, context->callback_snapshot, sizeof ffi_cb); memcpy(ffi_sym, context->symbol_snapshot, sizeof ffi_sym); memcpy(ffi_lib, context->library_snapshot, sizeof ffi_lib);
  ffi_nsym = context->ffi_symbols; ffi_nlib = context->ffi_libraries; ffi_tried_libm = context->ffi_libm;
 #endif
-#ifdef ZYJIT
+#ifdef ZYNATIVE
  cov = calloc((size_t)MW, 1);
 #else
  rcov = calloc((size_t)MW, 1);
@@ -6038,7 +6609,7 @@ static void *context_worker(void *opaque) {
  lcov = calloc((size_t)MW, 1);
 #endif
 #endif
-#ifdef ZYJIT
+#ifdef ZYNATIVE
  if (!cov) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
 #else
  if (!rcov
@@ -6053,6 +6624,10 @@ static void *context_worker(void *opaque) {
  nrg = context->region_count; context->region_count = 0;
  memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words);
  free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0;
+#ifdef ZYJIT_X64
+ if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
+ atomic_fetch_add(&context_code_live, 1);
+#endif
 #ifdef ZYJIT
 #ifdef __APPLE__
  code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
@@ -6184,7 +6759,7 @@ int main(int argc, char **argv)
 	if (memory_words < 0) return 1;
 	if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; }
 	m = calloc((size_t)MW, sizeof *m);
-#ifdef ZYJIT
+#ifdef ZYNATIVE
 	cov = calloc((size_t)MW, 1);
 	if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
 #else
diff --git a/tests/drivers/jit_x64_test.sh b/tests/drivers/jit_x64_test.sh
new file mode 100644
index 0000000..ce0438f
--- /dev/null
+++ b/tests/drivers/jit_x64_test.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# runs every program in tests/programs, examples and examples/learn on two cores
+# and compares stdout and the exit status. usage:
+#   bash tests/drivers/jit_x64_test.sh <reference core> <jit core> [name-filter]
+# the reference is normally the interpreter (built with -DZYINTERP) and the second
+# core the x86-64 jit. each program is the core's stdin, with nothing after it, and
+# runs with a timeout (JIT_TEST_TIMEOUT seconds, 120 by default). a program that
+# differs is listed with the first lines of the difference; the exit status of
+# this script is the number of programs that differ.
+ref=${1:?usage: jit_x64_test.sh <reference core> <jit core> [filter]}
+jit=${2:?usage: jit_x64_test.sh <reference core> <jit core> [filter]}
+filter=${3:-}
+limit=${JIT_TEST_TIMEOUT:-120}
+root=$(cd "$(dirname "$0")/../.." && pwd)
+cd "$root" || exit 2
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+pass=0 fail=0 total=0
+for f in tests/programs/*.zc examples/*.zc examples/learn/*.zc; do
+	[ -n "$filter" ] && [[ "$f" != *$filter* ]] && continue
+	total=$((total + 1))
+	timeout "$limit" "$ref" < "$f" > "$tmp/ref.out" 2> "$tmp/ref.err"; rr=$?
+	timeout "$limit" "$jit" < "$f" > "$tmp/jit.out" 2> "$tmp/jit.err"; rj=$?
+	if [ "$rr" = "$rj" ] && cmp -s "$tmp/ref.out" "$tmp/jit.out"; then
+		pass=$((pass + 1))
+	else
+		fail=$((fail + 1))
+		echo "DIFF $f (status $rr vs $rj)"
+		diff "$tmp/ref.out" "$tmp/jit.out" | head -6
+		head -3 "$tmp/jit.err"
+	fi
+done
+echo "$pass of $total programs agree, $fail differ"
+exit "$fail"