x86-64 jit: compile o3 regions (vars in their frame slots, six value registers); ZYO3INTERP=1 keeps the region interpreter

1 files changed, 258 insertions, 6 deletions

diff --git a/src/core/core.c b/src/core/core.c
index c4306ae..73be049 100644
--- a/src/core/core.c
+++ b/src/core/core.c
@@ -5314,21 +5314,24 @@ static int rexec(Rg *R, int e, int *ps, int *pr)
 #ifndef ZYJIT_X64
 static void be_drop(int j) { (void)j; }
 #endif
-static void rforget(Rg *R) { (void)R; }
 #ifdef ZYJIT_X64
-/* a region entry is never chained to: drop any translation that started there */
+/* a region entry is never chained to a translation: drop any that started there.
+   the jit's coverage marks the words too, so a store from translated code to
+   the region's own code reaches the hook, which drops the region */
 static void rmark(Rg *R)
 {
 	int e;
-	R->marked = 1; cinc(rcov, R->lo, R->hi);
+	R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(cov, R->lo, R->hi);
 	for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0;
 }
+static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(cov, R->lo, R->hi); }
 #else
+static void rforget(Rg *R) { (void)R; }
 static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); }
-#endif
 static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); }
 static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
 #endif
+#endif
 #ifdef ZYLLVM
 
 /* ---- o3 regions in llvm: one function per region, vars in allocas (so
@@ -6072,7 +6075,21 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te)
 	return 0;
 }
 /* the register value v lives in, or scratch with the constant v loaded into it */
-static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; }
+static VM_TLS int xvh[MAXUOP * 4 + 16];	/* regions: 1 + the var a value reads at block entry, 0 if none */
+static VM_TLS Rg *xR;
+static VM_TLS Cell xsyn[RMAXV];	/* the home of a var the lifter made (a hoisted value), which has no frame slot */
+static int xv(int v, int scratch)
+{
+	if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; }
+	if (xvh[v]) {	/* a var at block entry: its home slot, or fp for var 0 */
+		int j = xvh[v] - 1;
+		if (j == 0) { mov_rr(scratch, r_bp); alu_rr(a_sub, scratch, rMB); shift_ri(5, scratch, 3); }
+		else if (xR->nv0 && j >= xR->nv0) { mov_ri(scratch, (Cell)(uintptr_t)&xsyn[j]); mov_rm(scratch, scratch, 0); }
+		else mov_rm(scratch, r_bp, 8 * xR->slot[j]);
+		return scratch;
+	}
+	return xreg[(int)xr[v]];
+}
 static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; }
 static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); }
 static void x64_uop(Uop *u)
@@ -6155,7 +6172,7 @@ static void x64_enh(Enh *e)
 {
 	int i, j;
 	xr = e->reg;
-	for (i = 0; i < e->nv; i++) xkc[i] = 0;
+	for (i = 0; i < e->nv; i++) { xkc[i] = 0; xvh[i] = 0; }
 	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; }
 	for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j);
 	for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]);
@@ -6163,6 +6180,237 @@ static void x64_enh(Enh *e)
 	for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax));
 }
 
+/* ---- o3 regions in x86-64. there are too few registers for the arm64
+   scheme (vars in registers), so a var lives in its home frame slot,
+   m[fp + slot], with rbp = m + 8 * fp loaded at every entry. a block's values
+   get the six caller-saved registers rsi, rdi, r8..r11; rax, rcx, rdx are
+   scratch. a var read is a load from its home, var 0 is fp itself. the vars a
+   block changes are written home at its end (old values read first, so a swap
+   works), then the other slots that share a var get its value at every exit.
+   ops the inline lowering lacks (floats, popcount and friends) call ev.
+   calls are plain vm calls (the return point is an entry, so the callee's
+   return chains back in); everything else leaves through x_disp. a region
+   with native calls stays on the region interpreter, as does every region
+   under ZYO3INTERP=1. */
+static VM_TLS int x64_o3interp;
+static VM_TLS signed char rxr[MAXUOP * 4 + 16];
+#define RXNREG 6
+static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); }
+static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); }
+static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); }
+/* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */
+static int x64_ralloc(RB *B)
+{
+	static VM_TLS int last[MAXUOP * 4 + 16];
+	int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n;
+	if (B->nv > MAXUOP * 4) return -1;
+	for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; }
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i];
+		if (u->op == U_NARG || u->op == U_NCALL) return -1;
+		if (u->op == U_VGET || u->op == U_SLD || u->op == U_LI) continue;
+		na = nargs(u->op);
+		if (na > 0) last[u->a] = i;
+		if (na > 1) last[u->b] = i;
+		if (na > 2) last[u->c] = i;
+	}
+	for (j = 0; j < B->nso; j++) last[B->so[2 * j + 1]] = n;
+	for (j = 1; j < xR->nv; j++) if (B->vs[j] >= 0) last[B->vs[j]] = n;
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) last[B->cv] = n;
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i];
+		na = u->op == U_VGET || u->op == U_SLD || u->op == U_LI ? 0 : nargs(u->op);
+		for (k = 0; k < na; k++) {
+			int a = k == 0 ? u->a : k == 1 ? u->b : u->c;
+			if (last[a] == i && rxr[a] >= 0) { fr |= 1 << rxr[a]; last[a] = -2; }
+		}
+		if (is_store(u->op)) continue;
+		if (u->op == U_LI) { xkc[u->d] = 1; xkv[u->d] = u->imm; continue; }
+		if (u->op == U_VGET) { xvh[u->d] = u->imm + 1; continue; }
+		if (!fr) return -1;
+		for (k = 0; !(fr >> k & 1); k++);
+		fr &= ~(1 << k); rxr[u->d] = (signed char)k;
+		if (last[u->d] == -1) { fr |= 1 << k; }	/* never read: its register is free again at once */
+	}
+	return 0;
+}
+/* the store of rdx to the word rax; a covered word goes through the hook */
+static void rstore(void)
+{
+	JW *out, *slow, *done;
+	alu_ri(a_cmp, r_ax, MW); out = j32(c_ae);
+	mov_rm(r_cx, rST, 32);
+	mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0);
+	slow = j32(c_ne);
+	mov_xr(rMB, r_ax, 8, 0, r_dx);
+	done = jm32();
+	fix32(slow, cp);
+	rsave();
+	mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
+	publish_stacks();
+	call_m(rST, 64);
+	reload_stacks();
+	rrest();
+	fix32(out, cp); fix32(done, cp);
+}
+/* a c call with up to three values: rdi = k, rsi, rdx, rcx = a, b, c; the result into d */
+static void rcallc(void *f, long k, Uop *u, int na, int d)
+{
+	rsave();
+	if (na > 0) mov_rr(r_ax, xv(u->a, r_ax));
+	if (na > 1) mov_rr(r_dx, xv(u->b, r_dx));
+	if (na > 2) mov_rr(r_cx, xv(u->c, r_cx));
+	mov_rr(r_si, r_ax);
+	mov_ri(r_di, k);
+	call_abs(f);
+	rrest();
+	if (d >= 0) mov_rr(d, r_ax);
+}
+static void rlduop(Uop *u, int d)
+{
+	int n = u->op == U_LD8 ? 1 : u->op == U_LD16 ? 2 : 4;
+	JW *over;
+	mov_rr(r_ax, xv(u->a, r_ax));
+	xor32(r_cx);
+	mov_ri(r_dx, ((Cell)MW << 3) - (n - 1));
+	alu_rr(a_cmp, r_ax, r_dx);
+	over = j8(c_ae);
+	if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0);
+	else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0);
+	else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0);
+	fix8(over, cp);
+	mov_rr(d, r_cx);
+}
+/* the other slots a var stands for get its value: the frame is exact again */
+static void rspill(Rg *R)
+{
+	int k;
+	for (k = 0; k < R->np; k++)
+		if (R->pv[k] > 0 && R->ps[k] != R->slot[R->pv[k]]) { mov_rm(r_ax, r_bp, 8 * R->slot[R->pv[k]]); mov_mr(r_bp, 8 * R->ps[k], r_ax); }
+}
+static void rexit(Rg *R, int t) { rspill(R); mov_r32i(r_ax, (unsigned)t); jmp_to(x_disp); }
+/* var j's home gets register r (never rcx) */
+static void rvset(Rg *R, int j, int r)
+{
+	if (R->nv0 && j >= R->nv0) { mov_ri(r_cx, (Cell)(uintptr_t)&xsyn[j]); mov_mr(r_cx, 0, r); }
+	else mov_mr(r_bp, 8 * R->slot[j], r);
+}
+static VM_TLS JW **rfa; static VM_TLS int *rfb, nrf;
+static void rjmp(JW *at, int b) { rfa[nrf] = at; rfb[nrf++] = b; }
+static void remit_x64(Rg *R, RB *B)
+{
+	int i, j, d;
+	JW *at;
+	if (context_current) {
+		mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx);
+		at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp);
+	}
+	xr = rxr;
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		int op = u->op;
+		if (op == U_LI || op == U_VGET) continue;
+		d = is_store(op) || rxr[u->d] < 0 ? -1 : xreg[(int)rxr[u->d]];
+		if (op == U_SLD) { if (d >= 0) mov_rm(d, rSP, 8 * u->imm); continue; }
+		if (d < 0 && !is_store(op)) continue;
+		if (op == U_ST) { mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); rstore(); continue; }
+		if (op >= U_LD8 && op <= U_ST32) {
+			if (op == U_LD8 || op == U_LD16 || op == U_LD32) { rlduop(u, d); continue; }
+			rsave();
+			mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx));
+			mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
+			mov_r32i(r_dx, op == U_ST8 ? 1 : op == U_ST16 ? 2 : 4);
+			call_abs((void *)xbst);
+			rrest();
+			continue;
+		}
+		if (x64_op_ok(op)) { x64_uop(u); continue; }
+		rcallc((void *)xev, op, u, nargs(op), d);
+	}
+	for (j = 0; j < B->nso; j++) mov_mr(rSP, 8 * B->so[2 * j], xv(B->so[2 * j + 1], r_ax));
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) mov_rr(r_dx, xv(B->cv, r_dx));
+	/* the vars home: read every var-valued one first */
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) push_r(xv(B->vs[j], r_ax));
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && !xvh[B->vs[j]]) rvset(R, j, xv(B->vs[j], r_ax));
+	for (j = R->nv - 1; j >= 1; j--) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) { pop_r(r_ax); rvset(R, j, r_ax); }
+	if (B->delta) lea(rSP, rSP, -1, 1, 8 * B->delta);
+	switch (B->term) {
+	case T_BR: rjmp(jm32(), B->s1); break;
+	case T_CBR:
+		test_rr(r_dx, r_dx);
+		if (B->s1 >= 0) rjmp(j32(c_e), B->s1);
+		else { at = j32(c_ne); rexit(R, B->x1); fix32(at, cp); }
+		if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2);
+		break;
+	case T_CALL:
+		rspill(R);
+		mov_mi(rRP, 0, B->x2); alu_ri(a_add, rRP, 8);
+		mov_rr(r_ax, r_dx); jmp_to(x_disp);
+		break;
+	case T_RET:
+		rspill(R);
+		alu_ri(a_sub, rRP, 8); mov_rm(r_ax, rRP, 0); jmp_to(x_disp);
+		break;
+	default:
+		rspill(R); mov_rr(r_ax, r_dx); jmp_to(x_disp);
+	}
+}
+/* 0: compiled; -1: can't be; -2: no room now */
+static int rjit_x64(Rg *R)
+{
+	JW *start = cp, **lab;
+	int b, e, need = 64 * R->ne + 8192, rc = 0;
+	for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256;
+	if (cp - code + need > CODEW * 4 - (1 << 19)) return -2;
+	lab = malloc((size_t)R->nb * sizeof *lab);
+	rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0;
+	if (!lab || !rfa || !rfb) { free(lab); free(rfa); free(rfb); return -2; }
+	xR = R;
+	for (b = 0; b < R->nb && !rc; b++) if (x64_ralloc(&R->b[b])) rc = -1;
+#ifdef ZYRDUMP
+	for (b = 0; b < R->nb; b++) {
+		RB *B = &R->b[b]; int i, j;
+		fprintf(stderr, "b%d: n=%d delta=%d term=%d cv=%d s1=%d s2=%d x1=%d x2=%d\n", b, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2);
+		for (i = 0; i < B->n; i++) { Uop *u = &B->u[i]; fprintf(stderr, "  v%d = %s %d %d %d imm=%ld\n", u->d, u->op < U_N ? unames[u->op] : u->op == U_VGET ? "vget" : u->op == U_SLD ? "sld" : "?", u->a, u->b, u->c, (long)u->imm); }
+		for (j = 0; j < B->nso; j++) fprintf(stderr, "  so %d <- v%d\n", B->so[2*j], B->so[2*j+1]);
+		for (j = 1; j < R->nv; j++) fprintf(stderr, "  var%d(slot %d) <- %d\n", j, R->slot[j], B->vs[j]);
+	}
+	for (b = 0; b < R->np; b++) fprintf(stderr, "ps %d -> var %d\n", R->ps[b], R->pv[b]);
+#endif
+	for (e = 0; e < R->ne && !rc; e++) {
+		R->stub[e] = cp;
+		mov_rm(r_ax, rMB, 8 * ZFP); lea(r_bp, rMB, r_ax, 8, 0);
+		rjmp(jm32(), R->entb[e]);
+	}
+	for (b = 0; b < R->nb && !rc; b++) {
+		lab[b] = cp;
+		x64_ralloc(&R->b[b]);
+		remit_x64(R, &R->b[b]);
+	}
+	if (!rc && cp - start > need) { fprintf(stderr, "o3: x86-64 code overran its estimate\n"); abort(); }
+	if (!rc) {
+		int k;
+		for (k = 0; k < nrf; k++) fix32(rfa[k], lab[rfb[k]]);
+		R->nat = start;
+		for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = R->stub[e];
+		if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start));
+	} else cp = start;
+	free(lab); free(rfa); free(rfb);
+	return rc;
+}
+static void rforget(Rg *R)
+{
+	int e;
+	if (R->nat) for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words && tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0;
+	R->nat = 0;
+}
+static void rcompile(Rg *R, int eager)
+{
+	(void)eager;
+	if (R->nat || R->nonat || x64_o3interp || x64_noo3 || !R->lifted) return;
+	if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); }
+}
+
 static void *translate(int p)
 {
 	JW *start;
@@ -6384,6 +6632,8 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr
 					if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
 					s = (int)sp; r = (int)rp;
 				}
+				rcompile(R, 0);
+				if (R->nat && tab[pc]) { S[1] = s; S[2] = r; x64_limits(); goto native; }
 				pc = rexec(R, o3ei[pc], &s, &r);
 				S[1] = s; S[2] = r;
 				x64_limits();
@@ -6391,6 +6641,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr
 				continue;
 			}
 		}
+	native:
 		c = tab[pc];
 		if (!c) {
 			jitwrite(1); c = translate(pc); jitwrite(0);
@@ -6416,6 +6667,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop)
 	if (code == MAP_FAILED) { perror("mmap"); return 1; }
 	x64_noo3 = getenv("ZYNOO3") != 0;
 	x64_noenh = getenv("ZYNOENH") != 0;
+	x64_o3interp = getenv("ZYO3INTERP") != 0;
 	jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
 	S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
 	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;