diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/core/core.c | 271 |
1 files changed, 262 insertions, 9 deletions
diff --git a/src/core/core.c b/src/core/core.c index c4306ae..e3895ae 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -3,8 +3,9 @@ is translated to native code the first time it is reached, straight from live memory. stores that land on translated words throw those translations away, so code that rewrites itself, including the resident compiler, just gets - retranslated on the next visit. the arm64 jit also compiles o3 regions; on - x86-64 they run on the region interpreter; enhancements are inlined on both + retranslated on the next visit. both jits compile o3 regions (on x86-64 a + region that does not fit six value registers, or makes native calls, runs on + the region interpreter, as every region does under ZYO3INTERP=1); enhancements are inlined on both (on x86-64 those with float, byte or native-call ops are still called). on other hosts (or with -DZYINTERP) a plain interpreter with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */ #include <stdio.h> @@ -5314,21 +5315,24 @@ static int rexec(Rg *R, int e, int *ps, int *pr) #ifndef ZYJIT_X64 static void be_drop(int j) { (void)j; } #endif -static void rforget(Rg *R) { (void)R; } #ifdef ZYJIT_X64 -/* a region entry is never chained to: drop any translation that started there */ +/* a region entry is never chained to a translation: drop any that started there. + the jit's coverage marks the words too, so a store from translated code to + the region's own code reaches the hook, which drops the region */ static void rmark(Rg *R) { int e; - R->marked = 1; cinc(rcov, R->lo, R->hi); + R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(cov, R->lo, R->hi); for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0; } +static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(cov, R->lo, R->hi); } #else +static void rforget(Rg *R) { (void)R; } static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); } -#endif static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); } static void rcompile(Rg *R, int eager) { (void)R; (void)eager; } #endif +#endif #ifdef ZYLLVM /* ---- o3 regions in llvm: one function per region, vars in allocas (so @@ -6072,7 +6076,21 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; } +static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS Rg *xR; +static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ +static int xv(int v, int scratch) +{ + if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } + if (xvh[v]) { /* a var at block entry: its home slot, or fp for var 0 */ + int j = xvh[v] - 1; + if (j == 0) { mov_rr(scratch, r_bp); alu_rr(a_sub, scratch, rMB); shift_ri(5, scratch, 3); } + else if (xR->nv0 && j >= xR->nv0) { mov_ri(scratch, (Cell)(uintptr_t)&xsyn[j]); mov_rm(scratch, scratch, 0); } + else mov_rm(scratch, r_bp, 8 * xR->slot[j]); + return scratch; + } + return xreg[(int)xr[v]]; +} static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; } static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); } static void x64_uop(Uop *u) @@ -6155,7 +6173,7 @@ static void x64_enh(Enh *e) { int i, j; xr = e->reg; - for (i = 0; i < e->nv; i++) xkc[i] = 0; + for (i = 0; i < e->nv; i++) { xkc[i] = 0; xvh[i] = 0; } for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; } for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j); for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]); @@ -6163,6 +6181,237 @@ static void x64_enh(Enh *e) for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax)); } +/* ---- o3 regions in x86-64. there are too few registers for the arm64 + scheme (vars in registers), so a var lives in its home frame slot, + m[fp + slot], with rbp = m + 8 * fp loaded at every entry. a block's values + get the six caller-saved registers rsi, rdi, r8..r11; rax, rcx, rdx are + scratch. a var read is a load from its home, var 0 is fp itself. the vars a + block changes are written home at its end (old values read first, so a swap + works), then the other slots that share a var get its value at every exit. + ops the inline lowering lacks (floats, popcount and friends) call ev. + calls are plain vm calls (the return point is an entry, so the callee's + return chains back in); everything else leaves through x_disp. a region + with native calls stays on the region interpreter, as does every region + under ZYO3INTERP=1. */ +static VM_TLS int x64_o3interp; +static VM_TLS signed char rxr[MAXUOP * 4 + 16]; +#define RXNREG 6 +static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } +static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); } +static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); } +/* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ +static int x64_ralloc(RB *B) +{ + static VM_TLS int last[MAXUOP * 4 + 16]; + int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; + if (B->nv > MAXUOP * 4) return -1; + for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + if (u->op == U_NARG || u->op == U_NCALL) return -1; + if (u->op == U_VGET || u->op == U_SLD || u->op == U_LI) continue; + na = nargs(u->op); + if (na > 0) last[u->a] = i; + if (na > 1) last[u->b] = i; + if (na > 2) last[u->c] = i; + } + for (j = 0; j < B->nso; j++) last[B->so[2 * j + 1]] = n; + for (j = 1; j < xR->nv; j++) if (B->vs[j] >= 0) last[B->vs[j]] = n; + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) last[B->cv] = n; + for (i = 0; i < n; i++) { + Uop *u = &B->u[i]; + na = u->op == U_VGET || u->op == U_SLD || u->op == U_LI ? 0 : nargs(u->op); + for (k = 0; k < na; k++) { + int a = k == 0 ? u->a : k == 1 ? u->b : u->c; + if (last[a] == i && rxr[a] >= 0) { fr |= 1 << rxr[a]; last[a] = -2; } + } + if (is_store(u->op)) continue; + if (u->op == U_LI) { xkc[u->d] = 1; xkv[u->d] = u->imm; continue; } + if (u->op == U_VGET) { xvh[u->d] = u->imm + 1; continue; } + if (!fr) return -1; + for (k = 0; !(fr >> k & 1); k++); + fr &= ~(1 << k); rxr[u->d] = (signed char)k; + if (last[u->d] == -1) { fr |= 1 << k; } /* never read: its register is free again at once */ + } + return 0; +} +/* the store of rdx to the word rax; a covered word goes through the hook */ +static void rstore(void) +{ + JW *out, *slow, *done; + alu_ri(a_cmp, r_ax, MW); out = j32(c_ae); + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); + slow = j32(c_ne); + mov_xr(rMB, r_ax, 8, 0, r_dx); + done = jm32(); + fix32(slow, cp); + rsave(); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + publish_stacks(); + call_m(rST, 64); + reload_stacks(); + rrest(); + fix32(out, cp); fix32(done, cp); +} +/* a c call with up to three values: rdi = k, rsi, rdx, rcx = a, b, c; the result into d */ +static void rcallc(void *f, long k, Uop *u, int na, int d) +{ + rsave(); + if (na > 0) mov_rr(r_ax, xv(u->a, r_ax)); + if (na > 1) mov_rr(r_dx, xv(u->b, r_dx)); + if (na > 2) mov_rr(r_cx, xv(u->c, r_cx)); + mov_rr(r_si, r_ax); + mov_ri(r_di, k); + call_abs(f); + rrest(); + if (d >= 0) mov_rr(d, r_ax); +} +static void rlduop(Uop *u, int d) +{ + int n = u->op == U_LD8 ? 1 : u->op == U_LD16 ? 2 : 4; + JW *over; + mov_rr(r_ax, xv(u->a, r_ax)); + xor32(r_cx); + mov_ri(r_dx, ((Cell)MW << 3) - (n - 1)); + alu_rr(a_cmp, r_ax, r_dx); + over = j8(c_ae); + if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0); + else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0); + else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0); + fix8(over, cp); + mov_rr(d, r_cx); +} +/* the other slots a var stands for get its value: the frame is exact again */ +static void rspill(Rg *R) +{ + int k; + for (k = 0; k < R->np; k++) + if (R->pv[k] > 0 && R->ps[k] != R->slot[R->pv[k]]) { mov_rm(r_ax, r_bp, 8 * R->slot[R->pv[k]]); mov_mr(r_bp, 8 * R->ps[k], r_ax); } +} +static void rexit(Rg *R, int t) { rspill(R); mov_r32i(r_ax, (unsigned)t); jmp_to(x_disp); } +/* var j's home gets register r (never rcx) */ +static void rvset(Rg *R, int j, int r) +{ + if (R->nv0 && j >= R->nv0) { mov_ri(r_cx, (Cell)(uintptr_t)&xsyn[j]); mov_mr(r_cx, 0, r); } + else mov_mr(r_bp, 8 * R->slot[j], r); +} +static VM_TLS JW **rfa; static VM_TLS int *rfb, nrf; +static void rjmp(JW *at, int b) { rfa[nrf] = at; rfb[nrf++] = b; } +static void remit_x64(Rg *R, RB *B) +{ + int i, j, d; + JW *at; + if (context_current) { + mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx); + at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp); + } + xr = rxr; + for (i = 0; i < B->n; i++) { + Uop *u = &B->u[i]; + int op = u->op; + if (op == U_LI || op == U_VGET) continue; + d = is_store(op) || rxr[u->d] < 0 ? -1 : xreg[(int)rxr[u->d]]; + if (op == U_SLD) { if (d >= 0) mov_rm(d, rSP, 8 * u->imm); continue; } + if (d < 0 && !is_store(op)) continue; + if (op == U_ST) { mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); rstore(); continue; } + if (op >= U_LD8 && op <= U_ST32) { + if (op == U_LD8 || op == U_LD16 || op == U_LD32) { rlduop(u, d); continue; } + rsave(); + mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + mov_r32i(r_dx, op == U_ST8 ? 1 : op == U_ST16 ? 2 : 4); + call_abs((void *)xbst); + rrest(); + continue; + } + if (x64_op_ok(op)) { x64_uop(u); continue; } + rcallc((void *)xev, op, u, nargs(op), d); + } + for (j = 0; j < B->nso; j++) mov_mr(rSP, 8 * B->so[2 * j], xv(B->so[2 * j + 1], r_ax)); + if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) mov_rr(r_dx, xv(B->cv, r_dx)); + /* the vars home: read every var-valued one first */ + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) push_r(xv(B->vs[j], r_ax)); + for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && !xvh[B->vs[j]]) rvset(R, j, xv(B->vs[j], r_ax)); + for (j = R->nv - 1; j >= 1; j--) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) { pop_r(r_ax); rvset(R, j, r_ax); } + if (B->delta) lea(rSP, rSP, -1, 1, 8 * B->delta); + switch (B->term) { + case T_BR: rjmp(jm32(), B->s1); break; + case T_CBR: + test_rr(r_dx, r_dx); + if (B->s1 >= 0) rjmp(j32(c_e), B->s1); + else { at = j32(c_ne); rexit(R, B->x1); fix32(at, cp); } + if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2); + break; + case T_CALL: + rspill(R); + mov_mi(rRP, 0, B->x2); alu_ri(a_add, rRP, 8); + mov_rr(r_ax, r_dx); jmp_to(x_disp); + break; + case T_RET: + rspill(R); + alu_ri(a_sub, rRP, 8); mov_rm(r_ax, rRP, 0); jmp_to(x_disp); + break; + default: + rspill(R); mov_rr(r_ax, r_dx); jmp_to(x_disp); + } +} +/* 0: compiled; -1: can't be; -2: no room now */ +static int rjit_x64(Rg *R) +{ + JW *start = cp, **lab; + int b, e, need = 64 * R->ne + 8192, rc = 0; + for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256; + if (cp - code + need > CODEW * 4 - (1 << 19)) return -2; + lab = malloc((size_t)R->nb * sizeof *lab); + rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0; + if (!lab || !rfa || !rfb) { free(lab); free(rfa); free(rfb); return -2; } + xR = R; + for (b = 0; b < R->nb && !rc; b++) if (x64_ralloc(&R->b[b])) rc = -1; +#ifdef ZYRDUMP + for (b = 0; b < R->nb; b++) { + RB *B = &R->b[b]; int i, j; + fprintf(stderr, "b%d: n=%d delta=%d term=%d cv=%d s1=%d s2=%d x1=%d x2=%d\n", b, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2); + for (i = 0; i < B->n; i++) { Uop *u = &B->u[i]; fprintf(stderr, " v%d = %s %d %d %d imm=%ld\n", u->d, u->op < U_N ? unames[u->op] : u->op == U_VGET ? "vget" : u->op == U_SLD ? "sld" : "?", u->a, u->b, u->c, (long)u->imm); } + for (j = 0; j < B->nso; j++) fprintf(stderr, " so %d <- v%d\n", B->so[2*j], B->so[2*j+1]); + for (j = 1; j < R->nv; j++) fprintf(stderr, " var%d(slot %d) <- %d\n", j, R->slot[j], B->vs[j]); + } + for (b = 0; b < R->np; b++) fprintf(stderr, "ps %d -> var %d\n", R->ps[b], R->pv[b]); +#endif + for (e = 0; e < R->ne && !rc; e++) { + R->stub[e] = cp; + mov_rm(r_ax, rMB, 8 * ZFP); lea(r_bp, rMB, r_ax, 8, 0); + rjmp(jm32(), R->entb[e]); + } + for (b = 0; b < R->nb && !rc; b++) { + lab[b] = cp; + x64_ralloc(&R->b[b]); + remit_x64(R, &R->b[b]); + } + if (!rc && cp - start > need) { fprintf(stderr, "o3: x86-64 code overran its estimate\n"); abort(); } + if (!rc) { + int k; + for (k = 0; k < nrf; k++) fix32(rfa[k], lab[rfb[k]]); + R->nat = start; + for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = R->stub[e]; + if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start)); + } else cp = start; + free(lab); free(rfa); free(rfb); + return rc; +} +static void rforget(Rg *R) +{ + int e; + if (R->nat) for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words && tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0; + R->nat = 0; +} +static void rcompile(Rg *R, int eager) +{ + (void)eager; + if (R->nat || R->nonat || x64_o3interp || x64_noo3 || !R->lifted) return; + if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); } +} + static void *translate(int p) { JW *start; @@ -6374,7 +6623,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break; if (stop_ge >= 0 && pc >= stop_ge) break; if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break; - if (o3ent[pc] && !x64_noo3) { /* an o3 region: the region interpreter runs it */ + if (o3ent[pc] && !x64_noo3) { /* an o3 region: its x86-64 code if it has some, else the region interpreter */ Rg *R = rat(pc); if (R) { int s = (int)S[1], r = (int)S[2]; @@ -6384,6 +6633,8 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; } s = (int)sp; r = (int)rp; } + rcompile(R, 0); + if (R->nat && tab[pc]) { S[1] = s; S[2] = r; x64_limits(); goto native; } pc = rexec(R, o3ei[pc], &s, &r); S[1] = s; S[2] = r; x64_limits(); @@ -6391,6 +6642,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr continue; } } + native: c = tab[pc]; if (!c) { jitwrite(1); c = translate(pc); jitwrite(0); @@ -6416,6 +6668,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) if (code == MAP_FAILED) { perror("mmap"); return 1; } x64_noo3 = getenv("ZYNOO3") != 0; x64_noenh = getenv("ZYNOENH") != 0; + x64_o3interp = getenv("ZYO3INTERP") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; |