From 25ce902e2d57d9fb3a0b44f71ef258cfb290a9b7 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 16:40:41 +0330 Subject: x86-64 jit: inline enhancements (micro-ops lowered to registers, literal specialization, ZYNOENH=1 to call xrun); enh_ops test; jit_x64_test in test.sh --- src/core/core.c | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 176 insertions(+), 5 deletions(-) (limited to 'src/core/core.c') diff --git a/src/core/core.c b/src/core/core.c index 7ffbd02..2a9bdae 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -1014,11 +1014,12 @@ static void xdump(int slot, Enh *e) } for (i = 0; i < e->nout; i++) fprintf(stderr, " out%d = v%d\n", i, e->out[i]); } -/* registers for the jit: values live in x0..x15, straight-line linear scan. - if they don't fit, reg stays 0 and the jit calls the interpreter instead. */ -static void regalloc(Enh *e) +/* registers for the jit: values live in x0..x15 (registers 0..nr-1 of the + x86-64 jit's own set there), straight-line linear scan. if they don't fit, + reg stays 0 and the jit calls the interpreter instead. */ +static void regalloc_n(Enh *e, int nr) { - int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = 0xffff; + int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = (1 << nr) - 1; signed char *r = malloc(e->nv); for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; } for (i = 0; i < e->n; i++) { @@ -1028,7 +1029,7 @@ static void regalloc(Enh *e) if (uarity[u->op] > 2) last[u->c] = i; } for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n; -#define TAKE(v) do { for (k = 0; k < 16 && !(fr >> k & 1); k++); if (k == 16) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0) +#define TAKE(v) do { for (k = 0; k < nr && !(fr >> k & 1); k++); if (k == nr) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0) for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j); for (i = 0; i < e->n; i++) { Uop *u = &e->u[i]; int ops[3], no = 0; @@ -1044,6 +1045,7 @@ static void regalloc(Enh *e) fail: free(last); free(r); e->reg = 0; } +static void regalloc(Enh *e) { regalloc_n(e, 16); } /* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs, outputs, payload address, payload length, parameter names (or 0). kind 2 is an o3 region (below). @@ -5901,6 +5903,7 @@ static void rcompile(Rg *R, int eager) translated (the c loop sees it first), and the region interpreter runs it. */ static VM_TLS int vd; static VM_TLS int x64_noo3; /* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */ +static VM_TLS int x64_noenh; /* ZYNOENH=1: call every enhancement through xrun, as phase 1 did */ #define SL(k) (8 * (vd + (k))) static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); } static void stv(int k, int r) { mov_mr(rSP, SL(k), r); } @@ -6016,6 +6019,150 @@ static void float_op(int i, int k) (void)i; } +/* ---- enhancements, inlined ---- + an enhancement's optimized micro-ops are lowered to x86-64 instead of being + run by xrun: values live in rsi, rdi, r8..r11 and rbp (regalloc_n with seven + registers), rax, rcx and rdx are the scratch that division, shifts and setcc + need, and constants are made where they are used. like the arm64 jit, a run + of literals in front of the opcode specializes the enhancement on them + (x64_spec_enh), so the optimizer sees them as constants. an enhancement with + an op this lowering lacks (floats, byte loads and stores, native calls), one + that does not fit the registers, or one too long still goes through xrun. */ +static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp }; +#define XNREG 7 +#define XMAXOP 400 +static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16]; +static VM_TLS signed char *xr; +static int x64_op_ok(int op) +{ + switch (op) { + case U_LI: case U_MOV: case U_ADD: case U_SUB: case U_MUL: case U_SDIV: case U_UDIV: case U_SREM: case U_UREM: + case U_AND: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR: case U_NEG: case U_NOT: + case U_LT: case U_LTU: case U_EQ: case U_NE: case U_MIN: case U_MAX: case U_MINU: case U_MAXU: + case U_MULHS: case U_MULHU: case U_CLZ: case U_CTZ: case U_BSWAP: case U_ROTL: case U_ROTR: + case U_ABS: case U_SEL: case U_LD: case U_ST: + return 1; + } + return 0; +} +/* the enhancement in slot s, specialized on nk literals that feed its last inputs + (nk may be 0) and given x86-64 registers. 0 if it can be inlined */ +static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) +{ + Enh *e = &enh[s]; + int nin = e->nin - nk, i, j, out[8], na; + static VM_TLS int mp[MAXUOP * 4 + 16]; + if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1; + for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1; + un = 0; unv = nin; uerr = 0; + for (j = 0; j < nin; j++) mp[j] = j; + for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]); + for (i = 0; i < e->n; i++) { + Uop *u = &e->u[i]; + na = nargs(u->op); + j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm); + if (u->op != U_ST) mp[u->d] = j; + } + for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]]; + memset(te, 0, sizeof *te); + if (uerr || optimize(te, nin, out, e->nout)) return -1; + for (i = 0; i < te->n; i++) if (!x64_op_ok(te->u[i].op)) { free(te->u); return -1; } + regalloc_n(te, XNREG); + if (!te->reg) { free(te->u); return -1; } + return 0; +} +/* the register value v lives in, or scratch with the constant v loaded into it */ +static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; } +static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; } +static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); } +static void x64_uop(Uop *u) +{ + int op = u->op, d = is_store(op) ? -1 : xreg[(int)xr[u->d]], k; + JW *z, *m1, *e1, *e2; + switch (op) { + case U_MOV: mov_rr(d, xv(u->a, r_ax)); return; + case U_ADD: case U_SUB: case U_AND: case U_OR: case U_XOR: + k = op == U_ADD ? a_add : op == U_SUB ? a_sub : op == U_AND ? a_and : op == U_OR ? a_or : a_xor; + mov_rr(r_ax, xv(u->a, r_ax)); + if (xsmall(u->b)) alu_ri(k, r_ax, (int)xkv[u->b]); else alu_rr(k, r_ax, xv(u->b, r_cx)); + mov_rr(d, r_ax); return; + case U_MUL: + mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 1, 0xaf, r_ax, xv(u->b, r_cx)); mov_rr(d, r_ax); return; + case U_SHL: case U_SHR: case U_SAR: case U_ROTL: case U_ROTR: + k = op == U_SHL ? 4 : op == U_SHR ? 5 : op == U_SAR ? 7 : op == U_ROTL ? 0 : 1; + if (xkc[u->b]) { mov_rr(r_ax, xv(u->a, r_ax)); if (xkv[u->b] & 63) shift_ri(k, r_ax, (int)(xkv[u->b] & 63)); } + else { mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); shift_cl(k, r_ax); } + mov_rr(d, r_ax); return; + case U_NEG: case U_NOT: + mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_NEG ? 3 : 2, r_ax); mov_rr(d, r_ax); return; + case U_LT: case U_LTU: case U_EQ: case U_NE: + mov_rr(r_ax, xv(u->a, r_ax)); xor32(r_dx); + if (xsmall(u->b)) alu_ri(a_cmp, r_ax, (int)xkv[u->b]); else alu_rr(a_cmp, r_ax, xv(u->b, r_cx)); + setcc(op == U_LT ? c_l : op == U_LTU ? c_b : op == U_EQ ? c_e : c_ne, r_dx); mov_rr(d, r_dx); return; + case U_MIN: case U_MAX: case U_MINU: case U_MAXU: /* keep a unless the other one wins */ + mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); alu_rr(a_cmp, r_ax, r_dx); + cmov(op == U_MIN ? c_ge : op == U_MAX ? c_le : op == U_MINU ? c_ae : c_be, r_ax, r_dx); mov_rr(d, r_ax); return; + case U_SEL: + mov_rr(r_ax, xv(u->c, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); mov_rr(r_cx, xv(u->a, r_cx)); + test_rr(r_cx, r_cx); cmov(c_ne, r_ax, r_dx); mov_rr(d, r_ax); return; + case U_ABS: + mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, r_ax); reg_op(0, 1, 0, 0xf7, 3, r_dx); + test_rr(r_ax, r_ax); cmov(c_l, r_ax, r_dx); mov_rr(d, r_ax); return; + case U_MULHS: case U_MULHU: + mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_MULHS ? 5 : 4, r_cx); + mov_rr(d, r_dx); return; + case U_SDIV: case U_SREM: /* by 0: 0 or a; by -1: -a or 0, which also covers the minimum; else idiv */ + mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); + test_rr(r_cx, r_cx); z = j32(c_e); + alu_ri(a_cmp, r_cx, -1); m1 = j32(c_e); + xb(0x48); xb(0x99); reg_op(0, 1, 0, 0xf7, 7, r_cx); /* cqo; idiv rcx */ + if (op == U_SREM) mov_rr(r_ax, r_dx); + e1 = jm32(); + fix32(m1, cp); + if (op == U_SDIV) reg_op(0, 1, 0, 0xf7, 3, r_ax); else xor32(r_ax); + e2 = jm32(); + fix32(z, cp); + if (op == U_SDIV) xor32(r_ax); + fix32(e1, cp); fix32(e2, cp); + mov_rr(d, r_ax); return; + case U_UDIV: case U_UREM: /* by 0: 0 or a */ + mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); + test_rr(r_cx, r_cx); z = j32(c_e); + xor32(r_dx); reg_op(0, 1, 0, 0xf7, 6, r_cx); /* div rcx */ + if (op == U_UREM) mov_rr(r_ax, r_dx); + e1 = jm32(); + fix32(z, cp); + if (op == U_UDIV) xor32(r_ax); + fix32(e1, cp); + mov_rr(d, r_ax); return; + case U_CLZ: /* bsr leaves zf set for 0: 127 ^ 63 is 64 */ + mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 127); reg_op(0, 1, 1, 0xbd, r_dx, r_ax); cmov(c_e, r_dx, r_cx); + alu_ri(a_xor, r_dx, 63); mov_rr(d, r_dx); return; + case U_CTZ: + mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 64); reg_op(0, 1, 1, 0xbc, r_dx, r_ax); cmov(c_e, r_dx, r_cx); + mov_rr(d, r_dx); return; + case U_BSWAP: + mov_rr(r_ax, xv(u->a, r_ax)); xb(0x48); xb(0x0f); xb(0xc8); mov_rr(d, r_ax); return; + case U_LD: /* out of range reads 0, like the interpreter */ + mov_rr(r_ax, xv(u->a, r_ax)); emit_load(); mov_rr(d, r_cx); return; + case U_ST: /* outside memory stores nothing; like xexec, no store hook */ + mov_rr(r_ax, xv(u->a, r_ax)); alu_ri(a_cmp, r_ax, MW); z = j32(c_ae); + mov_xr(rMB, r_ax, 8, 0, xv(u->b, r_dx)); fix32(z, cp); return; + } +} +/* inputs from the vm stack into their registers, the ops, the outputs back */ +static void x64_enh(Enh *e) +{ + int i, j; + xr = e->reg; + for (i = 0; i < e->nv; i++) xkc[i] = 0; + for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; } + for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j); + for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]); + vd += e->nin - e->nout; + for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax)); +} + static void *translate(int p) { JW *start; @@ -6045,6 +6192,25 @@ static void *translate(int p) i += 2; break; } + if (w & 1 && cp - start < (1 << 18) && !x64_noenh) { /* literals feeding an enhancement: specialize it on them */ + int L = 0, s, nk, j; + Cell ks[8], x; + Enh te; + while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++; + x = i + L < code_words ? m[i + L] : 1; + if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) { + nk = L < enh[s].nin ? L : enh[s].nin; + for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1; + if (!x64_spec_enh(s, ks, nk, &te)) { + for (j = 0; j < L - nk; j++) { Cell v = m[i + j] >> 1; vd--; if (v >= -2147483647 - 1 && v <= 2147483647) mov_mi(rSP, SL(0), (int)v); else { mov_ri(r_ax, v); stv(0, r_ax); } } + if (nused < 64) used[nused++] = s; else used[0] = -1; + x64_enh(&te); + free(te.u); free(te.reg); + i += L + 1; n += L; + continue; + } + } + } if (w & 1) { Cell v = (w - 1) / 2; int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1; @@ -6128,6 +6294,10 @@ static void *translate(int p) } if (k >= XOP0 && k < XOP0 + NENH) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; /* wired or not, an install makes this a different block */ + if (enh[k - XOP0].live && cp - start < (1 << 18) && !x64_noenh) { /* inline */ + Enh te; + if (!x64_spec_enh(k - XOP0, 0, 0, &te)) { x64_enh(&te); free(te.u); free(te.reg); i++; continue; } + } if (enh[k - XOP0].live) { /* the interpreter runs it: x0 = data stack, slot */ lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0)); call_m(rST, 80); @@ -6245,6 +6415,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (code == MAP_FAILED) { perror("mmap"); return 1; } x64_noo3 = getenv("ZYNOO3") != 0; + x64_noenh = getenv("ZYNOENH") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; -- cgit v1.2.3