x86-64 jit: inline enhancements (micro-ops lowered to registers, literal specialization, ZYNOENH=1 to call xrun); enh_ops test; jit_x64_test in test.sh

1 files changed, 176 insertions, 5 deletions

diff --git a/src/core/core.c b/src/core/core.c
index 7ffbd02..2a9bdae 100644
--- a/src/core/core.c
+++ b/src/core/core.c
@@ -1014,11 +1014,12 @@ static void xdump(int slot, Enh *e)
 	}
 	for (i = 0; i < e->nout; i++) fprintf(stderr, "  out%d = v%d\n", i, e->out[i]);
 }
-/* registers for the jit: values live in x0..x15, straight-line linear scan.
-   if they don't fit, reg stays 0 and the jit calls the interpreter instead. */
-static void regalloc(Enh *e)
+/* registers for the jit: values live in x0..x15 (registers 0..nr-1 of the
+   x86-64 jit's own set there), straight-line linear scan. if they don't fit,
+   reg stays 0 and the jit calls the interpreter instead. */
+static void regalloc_n(Enh *e, int nr)
 {
-	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = 0xffff;
+	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = (1 << nr) - 1;
 	signed char *r = malloc(e->nv);
 	for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; }
 	for (i = 0; i < e->n; i++) {
@@ -1028,7 +1029,7 @@ static void regalloc(Enh *e)
 		if (uarity[u->op] > 2) last[u->c] = i;
 	}
 	for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n;
-#define TAKE(v) do { for (k = 0; k < 16 && !(fr >> k & 1); k++); if (k == 16) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
+#define TAKE(v) do { for (k = 0; k < nr && !(fr >> k & 1); k++); if (k == nr) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
 	for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j);
 	for (i = 0; i < e->n; i++) {
 		Uop *u = &e->u[i]; int ops[3], no = 0;
@@ -1044,6 +1045,7 @@ static void regalloc(Enh *e)
 fail:
 	free(last); free(r); e->reg = 0;
 }
+static void regalloc(Enh *e) { regalloc_n(e, 16); }
 /* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs,
    outputs, payload address, payload length, parameter names (or 0). kind 2
    is an o3 region (below).
@@ -5901,6 +5903,7 @@ static void rcompile(Rg *R, int eager)
    translated (the c loop sees it first), and the region interpreter runs it. */
 static VM_TLS int vd;
 static VM_TLS int x64_noo3;	/* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */
+static VM_TLS int x64_noenh;	/* ZYNOENH=1: call every enhancement through xrun, as phase 1 did */
 #define SL(k) (8 * (vd + (k)))
 static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); }
 static void stv(int k, int r) { mov_mr(rSP, SL(k), r); }
@@ -6016,6 +6019,150 @@ static void float_op(int i, int k)
 	(void)i;
 }
 
+/* ---- enhancements, inlined ----
+   an enhancement's optimized micro-ops are lowered to x86-64 instead of being
+   run by xrun: values live in rsi, rdi, r8..r11 and rbp (regalloc_n with seven
+   registers), rax, rcx and rdx are the scratch that division, shifts and setcc
+   need, and constants are made where they are used. like the arm64 jit, a run
+   of literals in front of the opcode specializes the enhancement on them
+   (x64_spec_enh), so the optimizer sees them as constants. an enhancement with
+   an op this lowering lacks (floats, byte loads and stores, native calls), one
+   that does not fit the registers, or one too long still goes through xrun. */
+static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp };
+#define XNREG 7
+#define XMAXOP 400
+static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16];
+static VM_TLS signed char *xr;
+static int x64_op_ok(int op)
+{
+	switch (op) {
+	case U_LI: case U_MOV: case U_ADD: case U_SUB: case U_MUL: case U_SDIV: case U_UDIV: case U_SREM: case U_UREM:
+	case U_AND: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR: case U_NEG: case U_NOT:
+	case U_LT: case U_LTU: case U_EQ: case U_NE: case U_MIN: case U_MAX: case U_MINU: case U_MAXU:
+	case U_MULHS: case U_MULHU: case U_CLZ: case U_CTZ: case U_BSWAP: case U_ROTL: case U_ROTR:
+	case U_ABS: case U_SEL: case U_LD: case U_ST:
+		return 1;
+	}
+	return 0;
+}
+/* the enhancement in slot s, specialized on nk literals that feed its last inputs
+   (nk may be 0) and given x86-64 registers. 0 if it can be inlined */
+static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te)
+{
+	Enh *e = &enh[s];
+	int nin = e->nin - nk, i, j, out[8], na;
+	static VM_TLS int mp[MAXUOP * 4 + 16];
+	if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1;
+	for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1;
+	un = 0; unv = nin; uerr = 0;
+	for (j = 0; j < nin; j++) mp[j] = j;
+	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		na = nargs(u->op);
+		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+		if (u->op != U_ST) mp[u->d] = j;
+	}
+	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
+	memset(te, 0, sizeof *te);
+	if (uerr || optimize(te, nin, out, e->nout)) return -1;
+	for (i = 0; i < te->n; i++) if (!x64_op_ok(te->u[i].op)) { free(te->u); return -1; }
+	regalloc_n(te, XNREG);
+	if (!te->reg) { free(te->u); return -1; }
+	return 0;
+}
+/* the register value v lives in, or scratch with the constant v loaded into it */
+static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; }
+static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; }
+static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); }
+static void x64_uop(Uop *u)
+{
+	int op = u->op, d = is_store(op) ? -1 : xreg[(int)xr[u->d]], k;
+	JW *z, *m1, *e1, *e2;
+	switch (op) {
+	case U_MOV: mov_rr(d, xv(u->a, r_ax)); return;
+	case U_ADD: case U_SUB: case U_AND: case U_OR: case U_XOR:
+		k = op == U_ADD ? a_add : op == U_SUB ? a_sub : op == U_AND ? a_and : op == U_OR ? a_or : a_xor;
+		mov_rr(r_ax, xv(u->a, r_ax));
+		if (xsmall(u->b)) alu_ri(k, r_ax, (int)xkv[u->b]); else alu_rr(k, r_ax, xv(u->b, r_cx));
+		mov_rr(d, r_ax); return;
+	case U_MUL:
+		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 1, 0xaf, r_ax, xv(u->b, r_cx)); mov_rr(d, r_ax); return;
+	case U_SHL: case U_SHR: case U_SAR: case U_ROTL: case U_ROTR:
+		k = op == U_SHL ? 4 : op == U_SHR ? 5 : op == U_SAR ? 7 : op == U_ROTL ? 0 : 1;
+		if (xkc[u->b]) { mov_rr(r_ax, xv(u->a, r_ax)); if (xkv[u->b] & 63) shift_ri(k, r_ax, (int)(xkv[u->b] & 63)); }
+		else { mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); shift_cl(k, r_ax); }
+		mov_rr(d, r_ax); return;
+	case U_NEG: case U_NOT:
+		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_NEG ? 3 : 2, r_ax); mov_rr(d, r_ax); return;
+	case U_LT: case U_LTU: case U_EQ: case U_NE:
+		mov_rr(r_ax, xv(u->a, r_ax)); xor32(r_dx);
+		if (xsmall(u->b)) alu_ri(a_cmp, r_ax, (int)xkv[u->b]); else alu_rr(a_cmp, r_ax, xv(u->b, r_cx));
+		setcc(op == U_LT ? c_l : op == U_LTU ? c_b : op == U_EQ ? c_e : c_ne, r_dx); mov_rr(d, r_dx); return;
+	case U_MIN: case U_MAX: case U_MINU: case U_MAXU:	/* keep a unless the other one wins */
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); alu_rr(a_cmp, r_ax, r_dx);
+		cmov(op == U_MIN ? c_ge : op == U_MAX ? c_le : op == U_MINU ? c_ae : c_be, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_SEL:
+		mov_rr(r_ax, xv(u->c, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); mov_rr(r_cx, xv(u->a, r_cx));
+		test_rr(r_cx, r_cx); cmov(c_ne, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_ABS:
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, r_ax); reg_op(0, 1, 0, 0xf7, 3, r_dx);
+		test_rr(r_ax, r_ax); cmov(c_l, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_MULHS: case U_MULHU:
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_MULHS ? 5 : 4, r_cx);
+		mov_rr(d, r_dx); return;
+	case U_SDIV: case U_SREM:	/* by 0: 0 or a; by -1: -a or 0, which also covers the minimum; else idiv */
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
+		test_rr(r_cx, r_cx); z = j32(c_e);
+		alu_ri(a_cmp, r_cx, -1); m1 = j32(c_e);
+		xb(0x48); xb(0x99); reg_op(0, 1, 0, 0xf7, 7, r_cx);	/* cqo; idiv rcx */
+		if (op == U_SREM) mov_rr(r_ax, r_dx);
+		e1 = jm32();
+		fix32(m1, cp);
+		if (op == U_SDIV) reg_op(0, 1, 0, 0xf7, 3, r_ax); else xor32(r_ax);
+		e2 = jm32();
+		fix32(z, cp);
+		if (op == U_SDIV) xor32(r_ax);
+		fix32(e1, cp); fix32(e2, cp);
+		mov_rr(d, r_ax); return;
+	case U_UDIV: case U_UREM:	/* by 0: 0 or a */
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
+		test_rr(r_cx, r_cx); z = j32(c_e);
+		xor32(r_dx); reg_op(0, 1, 0, 0xf7, 6, r_cx);		/* div rcx */
+		if (op == U_UREM) mov_rr(r_ax, r_dx);
+		e1 = jm32();
+		fix32(z, cp);
+		if (op == U_UDIV) xor32(r_ax);
+		fix32(e1, cp);
+		mov_rr(d, r_ax); return;
+	case U_CLZ:	/* bsr leaves zf set for 0: 127 ^ 63 is 64 */
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 127); reg_op(0, 1, 1, 0xbd, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
+		alu_ri(a_xor, r_dx, 63); mov_rr(d, r_dx); return;
+	case U_CTZ:
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 64); reg_op(0, 1, 1, 0xbc, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
+		mov_rr(d, r_dx); return;
+	case U_BSWAP:
+		mov_rr(r_ax, xv(u->a, r_ax)); xb(0x48); xb(0x0f); xb(0xc8); mov_rr(d, r_ax); return;
+	case U_LD:	/* out of range reads 0, like the interpreter */
+		mov_rr(r_ax, xv(u->a, r_ax)); emit_load(); mov_rr(d, r_cx); return;
+	case U_ST:	/* outside memory stores nothing; like xexec, no store hook */
+		mov_rr(r_ax, xv(u->a, r_ax)); alu_ri(a_cmp, r_ax, MW); z = j32(c_ae);
+		mov_xr(rMB, r_ax, 8, 0, xv(u->b, r_dx)); fix32(z, cp); return;
+	}
+}
+/* inputs from the vm stack into their registers, the ops, the outputs back */
+static void x64_enh(Enh *e)
+{
+	int i, j;
+	xr = e->reg;
+	for (i = 0; i < e->nv; i++) xkc[i] = 0;
+	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; }
+	for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j);
+	for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]);
+	vd += e->nin - e->nout;
+	for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax));
+}
+
 static void *translate(int p)
 {
 	JW *start;
@@ -6045,6 +6192,25 @@ static void *translate(int p)
 			i += 2;
 			break;
 		}
+		if (w & 1 && cp - start < (1 << 18) && !x64_noenh) {	/* literals feeding an enhancement: specialize it on them */
+			int L = 0, s, nk, j;
+			Cell ks[8], x;
+			Enh te;
+			while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++;
+			x = i + L < code_words ? m[i + L] : 1;
+			if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
+				nk = L < enh[s].nin ? L : enh[s].nin;
+				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
+				if (!x64_spec_enh(s, ks, nk, &te)) {
+					for (j = 0; j < L - nk; j++) { Cell v = m[i + j] >> 1; vd--; if (v >= -2147483647 - 1 && v <= 2147483647) mov_mi(rSP, SL(0), (int)v); else { mov_ri(r_ax, v); stv(0, r_ax); } }
+					if (nused < 64) used[nused++] = s; else used[0] = -1;
+					x64_enh(&te);
+					free(te.u); free(te.reg);
+					i += L + 1; n += L;
+					continue;
+				}
+			}
+		}
 		if (w & 1) {
 			Cell v = (w - 1) / 2;
 			int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1;
@@ -6128,6 +6294,10 @@ static void *translate(int p)
 		}
 		if (k >= XOP0 && k < XOP0 + NENH) {
 			if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1;	/* wired or not, an install makes this a different block */
+			if (enh[k - XOP0].live && cp - start < (1 << 18) && !x64_noenh) {	/* inline */
+				Enh te;
+				if (!x64_spec_enh(k - XOP0, 0, 0, &te)) { x64_enh(&te); free(te.u); free(te.reg); i++; continue; }
+			}
 			if (enh[k - XOP0].live) {	/* the interpreter runs it: x0 = data stack, slot */
 				lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0));
 				call_m(rST, 80);
@@ -6245,6 +6415,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop)
 	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 	if (code == MAP_FAILED) { perror("mmap"); return 1; }
 	x64_noo3 = getenv("ZYNOO3") != 0;
+	x64_noenh = getenv("ZYNOENH") != 0;
 	jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
 	S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
 	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;