From 999a1ffa7c77ce63d1b6f2eb6c42be6057b2fb69 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 16:54:21 +0330 Subject: x86-64 jit: compile o3 regions (vars in their frame slots, six value registers); ZYO3INTERP=1 keeps the region interpreter --- src/core/core.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 258 insertions(+), 6 deletions(-) (limited to 'src/core') diff --git a/src/core/core.c b/src/core/core.c index c4306ae..73be049 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -5314,21 +5314,24 @@ static int rexec(Rg *R, int e, int *ps, int *pr) #ifndef ZYJIT_X64 static void be_drop(int j) { (void)j; } #endif -static void rforget(Rg *R) { (void)R; } #ifdef ZYJIT_X64 -/* a region entry is never chained to: drop any translation that started there */ +/* a region entry is never chained to a translation: drop any that started there. + the jit's coverage marks the words too, so a store from translated code to + the region's own code reaches the hook, which drops the region */ static void rmark(Rg *R) { int e; - R->marked = 1; cinc(rcov, R->lo, R->hi); + R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(cov, R->lo, R->hi); for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0; } +static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(cov, R->lo, R->hi); } #else +static void rforget(Rg *R) { (void)R; } static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); } -#endif static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); } static void rcompile(Rg *R, int eager) { (void)R; (void)eager; } #endif +#endif #ifdef ZYLLVM /* ---- o3 regions in llvm: one function per region, vars in allocas (so @@ -6072,7 +6075,21 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; } +static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS Rg *xR; +static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ +static int xv(int v, int scratch) +{ + if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } + if (xvh[v]) { /* a var at block entry: its home slot, or fp for var 0 */ + int j = xvh[v] - 1; + if (j == 0) { mov_rr(scratch, r_bp); alu_rr(a_sub, scratch, rMB); shift_ri(5, scratch, 3); } + else if (xR->nv0 && j >= xR->nv0) { mov_ri(scratch, (Cell)(uintptr_t)&xsyn[j]); mov_rm(scratch, scratch, 0); } + else mov_rm(scratch, r_bp, 8 * xR->slot[j]); + return scratch; + } + return xreg[(int)xr[v]]; +} static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; } static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); } static void x64_uop(Uop *u) @@ -6155,7 +6172,7 @@ static void x64_enh(Enh *e) { int i, j; xr = e->reg; - for (i = 0; i < e->nv; i++) xkc[i] = 0; + for (i = 0; i < e->nv; i++) { xkc[i] = 0; xvh[i] = 0; } for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; } for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j); for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]); @@ -6163,6 +6180,237 @@ static void x64_enh(Enh *e) for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax)); } +/* ---- o3 regions in x86-64. there are too few registers for the arm64 + scheme (vars in registers), so a var lives in its home frame slot, + m[fp + slot], with rbp = m + 8 * fp loaded at every entry. a block's values + get the six caller-saved registers rsi, rdi, r8..r11; rax, rcx, rdx are + scratch. a var read is a load from its home, var 0 is fp itself. the vars a + block changes are written home at its end (old values read first, so a swap + works), then the other slots that share a var get its value at every exit. + ops the inline lowering lacks (floats, popcount and friends) call ev. + calls are plain vm calls (the return point is an entry, so the callee's + return chains back in); everything else leaves through x_disp. a region + with native calls stays on the region interpreter, as does every region + under ZYO3INTERP=1. */ +static VM_TLS int x64_o3interp; +static VM_TLS signed char rxr[MAXUOP * 4 + 16]; +#define RXNREG 6 +static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } +static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); } +static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); } +/* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ +static int x64_ralloc(RB *B) +{ + static VM_TLS int last[MAXUOP * 4 + 16]; + int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; + if (B->nv > MAXUOP * 4) return -1; + for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + if (u->op == U_NARG || u->op == U_NCALL) return -1; + if (u->op == U_VGET || u->op == U_SLD || u->op == U_LI) continue; + na = nargs(u->op); + if (na > 0) last[u->a] = i; + if (na > 1) last[u->b] = i; + if (na > 2) last[u->c] = i; + } + for (j = 0; j < B->nso; j++) last[B->so[2 * j + 1]] = n; + for (j = 1; j < xR->nv; j++) if (B->vs[j] >= 0) last[B->vs[j]] = n; + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) last[B->cv] = n; + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + na = u->op == U_VGET || u->op == U_SLD || u->op == U_LI ? 0 : nargs(u->op); + for (k = 0; k < na; k++) { + int a = k == 0 ? u->a : k == 1 ? u->b : u->c; + if (last[a] == i && rxr[a] >= 0) { fr |= 1 << rxr[a]; last[a] = -2; } + } + if (is_store(u->op)) continue; + if (u->op == U_LI) { xkc[u->d] = 1; xkv[u->d] = u->imm; continue; } + if (u->op == U_VGET) { xvh[u->d] = u->imm + 1; continue; } + if (!fr) return -1; + for (k = 0; !(fr >> k & 1); k++); + fr &= ~(1 << k); rxr[u->d] = (signed char)k; + if (last[u->d] == -1) { fr |= 1 << k; } /* never read: its register is free again at once */ + } + return 0; +} +/* the store of rdx to the word rax; a covered word goes through the hook */ +static void rstore(void) +{ + JW *out, *slow, *done; + alu_ri(a_cmp, r_ax, MW); out = j32(c_ae); + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); + slow = j32(c_ne); + mov_xr(rMB, r_ax, 8, 0, r_dx); + done = jm32(); + fix32(slow, cp); + rsave(); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + publish_stacks(); + call_m(rST, 64); + reload_stacks(); + rrest(); + fix32(out, cp); fix32(done, cp); +} +/* a c call with up to three values: rdi = k, rsi, rdx, rcx = a, b, c; the result into d */ +static void rcallc(void *f, long k, Uop *u, int na, int d) +{ + rsave(); + if (na > 0) mov_rr(r_ax, xv(u->a, r_ax)); + if (na > 1) mov_rr(r_dx, xv(u->b, r_dx)); + if (na > 2) mov_rr(r_cx, xv(u->c, r_cx)); + mov_rr(r_si, r_ax); + mov_ri(r_di, k); + call_abs(f); + rrest(); + if (d >= 0) mov_rr(d, r_ax); +} +static void rlduop(Uop *u, int d) +{ + int n = u->op == U_LD8 ? 1 : u->op == U_LD16 ? 2 : 4; + JW *over; + mov_rr(r_ax, xv(u->a, r_ax)); + xor32(r_cx); + mov_ri(r_dx, ((Cell)MW << 3) - (n - 1)); + alu_rr(a_cmp, r_ax, r_dx); + over = j8(c_ae); + if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0); + else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0); + else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0); + fix8(over, cp); + mov_rr(d, r_cx); +} +/* the other slots a var stands for get its value: the frame is exact again */ +static void rspill(Rg *R) +{ + int k; + for (k = 0; k < R->np; k++) + if (R->pv[k] > 0 && R->ps[k] != R->slot[R->pv[k]]) { mov_rm(r_ax, r_bp, 8 * R->slot[R->pv[k]]); mov_mr(r_bp, 8 * R->ps[k], r_ax); } +} +static void rexit(Rg *R, int t) { rspill(R); mov_r32i(r_ax, (unsigned)t); jmp_to(x_disp); } +/* var j's home gets register r (never rcx) */ +static void rvset(Rg *R, int j, int r) +{ + if (R->nv0 && j >= R->nv0) { mov_ri(r_cx, (Cell)(uintptr_t)&xsyn[j]); mov_mr(r_cx, 0, r); } + else mov_mr(r_bp, 8 * R->slot[j], r); +} +static VM_TLS JW **rfa; static VM_TLS int *rfb, nrf; +static void rjmp(JW *at, int b) { rfa[nrf] = at; rfb[nrf++] = b; } +static void remit_x64(Rg *R, RB *B) +{ + int i, j, d; + JW *at; + if (context_current) { + mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx); + at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp); + } + xr = rxr; + for (i = 0; i < B->n; i++) { + Uop *u = &B->u[i]; + int op = u->op; + if (op == U_LI || op == U_VGET) continue; + d = is_store(op) || rxr[u->d] < 0 ? -1 : xreg[(int)rxr[u->d]]; + if (op == U_SLD) { if (d >= 0) mov_rm(d, rSP, 8 * u->imm); continue; } + if (d < 0 && !is_store(op)) continue; + if (op == U_ST) { mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); rstore(); continue; } + if (op >= U_LD8 && op <= U_ST32) { + if (op == U_LD8 || op == U_LD16 || op == U_LD32) { rlduop(u, d); continue; } + rsave(); + mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + mov_r32i(r_dx, op == U_ST8 ? 1 : op == U_ST16 ? 2 : 4); + call_abs((void *)xbst); + rrest(); + continue; + } + if (x64_op_ok(op)) { x64_uop(u); continue; } + rcallc((void *)xev, op, u, nargs(op), d); + } + for (j = 0; j < B->nso; j++) mov_mr(rSP, 8 * B->so[2 * j], xv(B->so[2 * j + 1], r_ax)); + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) mov_rr(r_dx, xv(B->cv, r_dx)); + /* the vars home: read every var-valued one first */ + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) push_r(xv(B->vs[j], r_ax)); + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && !xvh[B->vs[j]]) rvset(R, j, xv(B->vs[j], r_ax)); + for (j = R->nv - 1; j >= 1; j--) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) { pop_r(r_ax); rvset(R, j, r_ax); } + if (B->delta) lea(rSP, rSP, -1, 1, 8 * B->delta); + switch (B->term) { + case T_BR: rjmp(jm32(), B->s1); break; + case T_CBR: + test_rr(r_dx, r_dx); + if (B->s1 >= 0) rjmp(j32(c_e), B->s1); + else { at = j32(c_ne); rexit(R, B->x1); fix32(at, cp); } + if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2); + break; + case T_CALL: + rspill(R); + mov_mi(rRP, 0, B->x2); alu_ri(a_add, rRP, 8); + mov_rr(r_ax, r_dx); jmp_to(x_disp); + break; + case T_RET: + rspill(R); + alu_ri(a_sub, rRP, 8); mov_rm(r_ax, rRP, 0); jmp_to(x_disp); + break; + default: + rspill(R); mov_rr(r_ax, r_dx); jmp_to(x_disp); + } +} +/* 0: compiled; -1: can't be; -2: no room now */ +static int rjit_x64(Rg *R) +{ + JW *start = cp, **lab; + int b, e, need = 64 * R->ne + 8192, rc = 0; + for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256; + if (cp - code + need > CODEW * 4 - (1 << 19)) return -2; + lab = malloc((size_t)R->nb * sizeof *lab); + rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0; + if (!lab || !rfa || !rfb) { free(lab); free(rfa); free(rfb); return -2; } + xR = R; + for (b = 0; b < R->nb && !rc; b++) if (x64_ralloc(&R->b[b])) rc = -1; +#ifdef ZYRDUMP + for (b = 0; b < R->nb; b++) { + RB *B = &R->b[b]; int i, j; + fprintf(stderr, "b%d: n=%d delta=%d term=%d cv=%d s1=%d s2=%d x1=%d x2=%d\n", b, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2); + for (i = 0; i < B->n; i++) { Uop *u = &B->u[i]; fprintf(stderr, " v%d = %s %d %d %d imm=%ld\n", u->d, u->op < U_N ? unames[u->op] : u->op == U_VGET ? "vget" : u->op == U_SLD ? "sld" : "?", u->a, u->b, u->c, (long)u->imm); } + for (j = 0; j < B->nso; j++) fprintf(stderr, " so %d <- v%d\n", B->so[2*j], B->so[2*j+1]); + for (j = 1; j < R->nv; j++) fprintf(stderr, " var%d(slot %d) <- %d\n", j, R->slot[j], B->vs[j]); + } + for (b = 0; b < R->np; b++) fprintf(stderr, "ps %d -> var %d\n", R->ps[b], R->pv[b]); +#endif + for (e = 0; e < R->ne && !rc; e++) { + R->stub[e] = cp; + mov_rm(r_ax, rMB, 8 * ZFP); lea(r_bp, rMB, r_ax, 8, 0); + rjmp(jm32(), R->entb[e]); + } + for (b = 0; b < R->nb && !rc; b++) { + lab[b] = cp; + x64_ralloc(&R->b[b]); + remit_x64(R, &R->b[b]); + } + if (!rc && cp - start > need) { fprintf(stderr, "o3: x86-64 code overran its estimate\n"); abort(); } + if (!rc) { + int k; + for (k = 0; k < nrf; k++) fix32(rfa[k], lab[rfb[k]]); + R->nat = start; + for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = R->stub[e]; + if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start)); + } else cp = start; + free(lab); free(rfa); free(rfb); + return rc; +} +static void rforget(Rg *R) +{ + int e; + if (R->nat) for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words && tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0; + R->nat = 0; +} +static void rcompile(Rg *R, int eager) +{ + (void)eager; + if (R->nat || R->nonat || x64_o3interp || x64_noo3 || !R->lifted) return; + if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); } +} + static void *translate(int p) { JW *start; @@ -6384,6 +6632,8 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; } s = (int)sp; r = (int)rp; } + rcompile(R, 0); + if (R->nat && tab[pc]) { S[1] = s; S[2] = r; x64_limits(); goto native; } pc = rexec(R, o3ei[pc], &s, &r); S[1] = s; S[2] = r; x64_limits(); @@ -6391,6 +6641,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr continue; } } + native: c = tab[pc]; if (!c) { jitwrite(1); c = translate(pc); jitwrite(0); @@ -6416,6 +6667,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) if (code == MAP_FAILED) { perror("mmap"); return 1; } x64_noo3 = getenv("ZYNOO3") != 0; x64_noenh = getenv("ZYNOENH") != 0; + x64_o3interp = getenv("ZYO3INTERP") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; -- cgit v1.2.3