From 63ff1aacf764e692a7a29a7c0043278f637f1dbb Mon Sep 17 00:00:00 2001 From: zinc Date: Thu, 15 Oct 2026 07:30:53 +0330 Subject: x86-64 jit: inline new_k, recycle_leaf and own fast paths, in o3 regions and plain blocks a call to rt.new_k / rt.recycle_leaf with a constant class, or to rt.own, does its class-list pop or push (or owner store) inline and goes on after the call; an empty list, memdebug, a held range, a block outside memory, a translated word or a type/class mismatch takes the ordinary call. in a region this replaces the o3_invoke round trip. o3 frees of leaf objects now use recycle_leaf too (new_k stamps the type inside o3 since b86db60). ZYNOFASTALLOC=1 turns it off; ZYPERFMAP=1 writes a perf map of translated code. tests/programs/o3_freelist.zc: reuse, zeroing, an empty list, untyped neighbours, memdebug; jit vs ZYNOFASTALLOC vs ZYO3INTERP. --- README.md | 2 +- src/compiler/analyzer.zc | 4 +- src/core/core.c | 212 +++++++++++++++++++++++++++++++++++++++++- src/core/stack.h | 5 + tests/programs/o3_freelist.zc | 84 +++++++++++++++++ tests/test.sh | 6 +- 6 files changed, 306 insertions(+), 7 deletions(-) create mode 100644 tests/programs/o3_freelist.zc diff --git a/README.md b/README.md index 839eac2..6a56945 100644 --- a/README.md +++ b/README.md @@ -590,7 +590,7 @@ calls push the arguments left to right and the callee's prologue pops them into registers: x19 holds the memory base, x20 the data stack pointer, x21 the return stack pointer, x22 the translation table and x23 the state block (memory, s, r, table, coverage map, Q, G, &J, store hook, install hook, enhancement fallback). o3 regions also use x26..x28, which the entry stub saves. -the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are compiled to x86-64 too, but not as on arm64: there are too few registers to keep the vars in them, so each var lives in its home frame slot (rbp points at the frame, reloaded at every entry), and a block's values get the six registers rsi, rdi, r8..r11. the vars a block changes are written home at its end, and the other slots that share a var get it at every exit. calls are plain vm calls through the dispatch stub, and ops the lowering lacks (floats, popcount) call the evaluator. a region that needs more than six live values, or makes native calls, runs on the region interpreter, entered from the c loop; `ZYO3INTERP=1` puts every region there, and `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does. +the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are compiled to x86-64 too, but not as on arm64: there are too few registers to keep the vars in them, so each var lives in its home frame slot (rbp points at the frame, reloaded at every entry), and a block's values get the six registers rsi, rdi, r8..r11. the vars a block changes are written home at its end, and the other slots that share a var get it at every exit. calls are plain vm calls through the dispatch stub, and ops the lowering lacks (floats, popcount) call the evaluator. a region that needs more than six live values, or makes native calls, runs on the region interpreter, entered from the c loop; `ZYO3INTERP=1` puts every region there, and `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does. calls to `rt.new_k` and `rt.recycle_leaf` with a constant size class, and to `rt.own`, are inlined in both plain blocks and compiled regions: the jit pops or pushes the class list itself and goes on after the call, and takes the ordinary call for anything unusual (an empty list, memdebug, a held range, a word it has translated, a type or class that does not match). the runtime hands over the addresses it needs with its stack registration (`frame_control[17..21]`, `SC_FAST` in `src/core/stack.h`); `ZYNOFASTALLOC=1` turns the inlining off. `ZYPERFMAP=1` writes `/tmp/perf-.map`, so `perf report` names translated code by its vm address range (`zc__`, `zc_o3__`). compilation is lazy. the first time control reaches an address, that block is translated from live memory. a block ends at call, jump, ret or halt. a `z` doesn't end it: the not-taken side keeps compiling, which makes superblocks. a literal followed by `c`, `j` or `z` is fused into a constant target. every exit goes through a dispatch stub that checks both stack pointers against their limits (kept in x24 and x25), bounds-checks the target, loads `tab[t]`, and branches straight to it when it's already compiled. otherwise it returns to c, which compiles the block and re-enters. enhancements are inlined: inputs load from the data stack into x0..x15, the optimized micro-ops run in registers with x16 and x17 as scratch, and the outputs go back. an install calls into c and invalidates translations and o3 regions that depend on its slot. invalidated code is rebuilt on its next entry. when the loop reaches an o3 entry point, it compiles the region (or reuses it) instead of translating a block. diff --git a/src/compiler/analyzer.zc b/src/compiler/analyzer.zc index 8ceaf09..42cb7b5 100644 --- a/src/compiler/analyzer.zc +++ b/src/compiler/analyzer.zc @@ -1004,7 +1004,7 @@ int freevar(int i) { if (o3on && fchk && ldroppable(i)) cerrn(i, "has drop and can't end inside an o3 region"); if (local_captured && local_captured[i]) { lit(0); laddr(lslot[i]); op(O_ST); return 0; } vaddr(i); op(O_LD); - if (lleaf[i] && !o3on && !dry && lookup(s_recycle_leaf) >= 0) { + if (lleaf[i] && !dry && lookup(s_recycle_leaf) >= 0) { // o3 too: new_k stamps the type there as well lit(lleaf[i]); lit(szclass(lnsz[i])); callname(s_recycle_leaf); @@ -1434,7 +1434,7 @@ int reassign_free(int i, int rc, int bor) { // freevar without its clearing store: the new value is stored next, through // the address taken before the right side (a captured local's slot holds its cell) vaddr(i); op(O_LD); - if (lleaf[i] && !o3on && !dry && lookup(s_recycle_leaf) >= 0) { lit(lleaf[i]); lit(szclass(lnsz[i])); callname(s_recycle_leaf); } + if (lleaf[i] && !dry && lookup(s_recycle_leaf) >= 0) { lit(lleaf[i]); lit(szclass(lnsz[i])); callname(s_recycle_leaf); } else callname(s_recycle); op(O_DROP); end_owner(i); diff --git a/src/core/core.c b/src/core/core.c index 3133c55..ce9e5ca 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -6249,6 +6249,7 @@ static void x64_enh(Enh *e) with native calls stays on the region interpreter, as does every region under ZYO3INTERP=1. */ static VM_TLS int x64_o3interp; +static void perfmap_note(JW *start, JW *end, const char *kind, int a, int b); static VM_TLS signed char rxr[MAXUOP * 4 + 16]; #define RXNREG 6 static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } @@ -6470,6 +6471,173 @@ static void rcall_x64(Rg *R, RB *B) for (e = 0; e < R->ne; e++) if (R->ent[e] == B->x2) { jmp_to((JW *)R->stub[e]); hopped = 1; break; } if (!hopped) { mov_r32i(r_ax, (unsigned)B->x2); jmp_to(x_disp); } } +/* ---- the allocator's fast paths, inline ---- + a call to rt.new_k or rt.recycle_leaf with a constant size class, or to + rt.own, is a dozen loads and stores when the class list can serve it, far less + than the call: from a compiled region the call even leaves through o3_invoke, + which runs the callee on the vm and comes back. so the jit does those stores + itself, in a region and in plain translated code, and goes on at the call's + return point; anything unusual (an empty list, memdebug, a held range, a block + outside memory, a word the jit has translated, a type or class that doesn't + match, a size new_k would clamp) takes the ordinary call, which does exactly + what it did before. no collection can happen on the fast paths, so the o3 + rules hold. the runtime hands over the addresses after its stack registration + (stack.h SC_FAST); ZYNOFASTALLOC=1 turns all of it off. */ +static VM_TLS int x64_nofast; +typedef struct { Cell freel, dbg, hlo, hhi, bump, newk, rleaf, own; } FastCfg; +static int x64_fastcfg(FastCfg *f) +{ + Cell c = stack_control; + if (x64_nofast || c <= 0 || !inmem(c + SC_FAST_END - 1) || m[c + SC_FAST] != SC_FAST_TAG) return 0; + f->freel = m[c + SC_FREE]; f->dbg = m[c + SC_DBG]; f->hlo = m[c + SC_HOLD_LOW]; f->hhi = m[c + SC_HOLD_HIGH]; + f->bump = m[c + SC_BUMP]; f->newk = m[c + SC_NEW_K]; f->rleaf = m[c + SC_RECYCLE_LEAF]; f->own = m[c + SC_OWN]; + /* every word it touches by address goes in a 32-bit displacement */ + if (!inmem(f->freel + 31) || !inmem(f->dbg) || !inmem(f->hlo) || !inmem(f->hhi) || !inmem(f->bump)) return 0; + if (8 * (f->freel + 31) > 0x7fffffff || 8 * f->dbg > 0x7fffffff || 8 * f->hlo > 0x7fffffff || 8 * f->hhi > 0x7fffffff || 8 * f->bump > 0x7fffffff) return 0; + return 1; +} +static void mov_xi(int b, int x, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, x, 8, disp); xd((unsigned)imm); } /* qword [b + x*8 + disp] = imm */ +#define FAST_MAXJ 24 +/* new_k(n, t, k) for a constant k in 3..5: n at [rSP + sd + 16], t at +8, k + on top. the block goes in the slot n came from; the caller pops the other two. + every branch to the ordinary call is added to slow. */ +static void x64_fast_newk(FastCfg *f, int k, int sd, JW **slow, int *ns) +{ + int fk = (int)(f->freel + k), j; + JW *at, *below; + /* rax = the block, rsi = the next one */ + mov_rm(r_ax, rMB, 8 * fk); + test_rr(r_ax, r_ax); slow[(*ns)++] = j32(c_e); + mov_rm(r_cx, rMB, (int)(8 * f->dbg)); test_rr(r_cx, r_cx); slow[(*ns)++] = j32(c_ne); + mov_ri(r_cx, MW - ((Cell)1 << k)); alu_rr(a_cmp, r_ax, r_cx); slow[(*ns)++] = j32(c_a); + mov_rm(r_cx, rMB, (int)(8 * f->hhi)); test_rr(r_cx, r_cx); at = j32(c_e); /* new_k's held-range rule */ + alu_rr(a_cmp, r_ax, r_cx); below = j32(c_ge); + lea(r_cx, r_ax, -1, 1, 1 << k); alu_rm(a_cmp, r_cx, rMB, (int)(8 * f->hlo)); slow[(*ns)++] = j32(c_g); + fix32(below, cp); fix32(at, cp); + mov_rx(r_si, rMB, r_ax, 8, 40); + test_rr(r_si, r_si); at = j32(c_e); + mov_ri(r_cx, MW - 7); alu_rr(a_cmp, r_si, r_cx); slow[(*ns)++] = j32(c_a); + fix32(at, cp); + mov_rm(r_di, rSP, sd + 16); /* n: new_k clamps a negative one, the call can do that */ + mov_ri(r_cx, 0x7fffffff); alu_rr(a_cmp, r_di, r_cx); slow[(*ns)++] = j32(c_a); + /* none of the words written may be translated code: the block, the list head, the next block's back link */ + mov_rm(r_cx, rST, 32); + mov_rx(r_8, r_cx, r_ax, 1, 0); + for (j = 1; j < (1 << k) / 8; j++) mem_op(0, 1, 0, a_or * 8 + 3, r_8, r_cx, r_ax, 1, 8 * j); + test_rr(r_8, r_8); slow[(*ns)++] = j32(c_ne); + mem_op(0, 0, 0, 0x80, 7, r_cx, -1, 1, fk); xb(0); slow[(*ns)++] = j32(c_ne); + test_rr(r_si, r_si); at = j32(c_e); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_si, 1, 6); xb(0); slow[(*ns)++] = j32(c_ne); + fix32(at, cp); + /* pop it */ + mov_mr(rMB, 8 * fk, r_si); + test_rr(r_si, r_si); at = j32(c_e); + mov_xi(rMB, r_si, 48, 0); + fix32(at, cp); + shift_ri(4, r_di, 32); alu_ri(a_or, r_di, (k << 16) | 1); mov_xr(rMB, r_ax, 8, 8, r_di); /* b[1]: class, live, n */ + lea(r_cx, r_ax, -1, 1, 5); + mov_rr(r_di, r_cx); alu_ri(a_xor, r_di, 0x2B7E1516); mov_xr(rMB, r_ax, 8, 16, r_di); /* b[2]: the check word */ + mov_xi(rMB, r_ax, 24, 0); /* b[3]: no owner */ + mov_rm(r_di, rSP, sd + 8); mov_xr(rMB, r_ax, 8, 32, r_di); /* b[4]: the type */ + xor32(r_di); + for (j = 5; j < (1 << k); j++) mov_xr(rMB, r_ax, 8, 8 * j, r_di); /* a zeroed payload */ + mov_mr(rSP, sd + 16, r_cx); +} +/* recycle_leaf(p, t, k) for a constant k: p at [rSP + sd + 16], t at +8, k on + top. its 0 goes in p's slot; the caller pops the other two */ +static void x64_fast_rleaf(FastCfg *f, int k, int sd, JW **slow, int *ns) +{ + int fk = (int)(f->freel + k); + JW *at; + /* rax = p, the block is p - 5; rsi = the list head */ + mov_rm(r_cx, rMB, (int)(8 * f->dbg)); test_rr(r_cx, r_cx); slow[(*ns)++] = j32(c_ne); + mov_rm(r_ax, rSP, sd + 16); + alu_ri(a_cmp, r_ax, HEAP_LO + 5); slow[(*ns)++] = j32(c_l); + alu_rm(a_cmp, r_ax, rMB, (int)(8 * f->bump)); slow[(*ns)++] = j32(c_ge); + mov_ri(r_cx, MW - 8); alu_rr(a_cmp, r_ax, r_cx); slow[(*ns)++] = j32(c_a); + mov_rr(r_di, r_ax); alu_ri(a_xor, r_di, 0x2B7E1516); + mem_op(0, 1, 0, a_cmp * 8 + 3, r_di, rMB, r_ax, 8, -24); slow[(*ns)++] = j32(c_ne); /* the check word */ + mov_rx(r_si, rMB, r_ax, 8, -32); + mov_r32i(r_di, 0xffff); alu_rr(a_and, r_di, r_si); alu_ri(a_cmp, r_di, 1); slow[(*ns)++] = j32(c_ne); /* live, no other flag */ + mov_rr(r_di, r_si); shift_ri(5, r_di, 16); alu_ri(a_and, r_di, 31); alu_ri(a_cmp, r_di, k); slow[(*ns)++] = j32(c_ne); + mov_rm(r_di, rSP, sd + 8); mem_op(0, 1, 0, a_cmp * 8 + 3, r_di, rMB, r_ax, 8, -8); slow[(*ns)++] = j32(c_ne); /* the type */ + mov_rm(r_si, rMB, 8 * fk); + test_rr(r_si, r_si); at = j32(c_e); + mov_ri(r_cx, MW - 7); alu_rr(a_cmp, r_si, r_cx); slow[(*ns)++] = j32(c_a); + fix32(at, cp); + mov_rm(r_cx, rST, 32); + mov_rx(r_8, r_cx, r_ax, 1, -5); test_rr(r_8, r_8); slow[(*ns)++] = j32(c_ne); + mem_op(0, 0, 0, 0x80, 7, r_cx, -1, 1, fk); xb(0); slow[(*ns)++] = j32(c_ne); + test_rr(r_si, r_si); at = j32(c_e); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_si, 1, 6); xb(0); slow[(*ns)++] = j32(c_ne); + fix32(at, cp); + /* push it */ + mov_xi(rMB, r_ax, -24, 0); + mov_xi(rMB, r_ax, -32, k << 16); + mov_xr(rMB, r_ax, 8, 0, r_si); + mov_xi(rMB, r_ax, 8, 0); + lea(r_di, r_ax, -1, 1, -5); + test_rr(r_si, r_si); at = j32(c_e); + mov_xr(rMB, r_si, 8, 48, r_di); + fix32(at, cp); + mov_mr(rMB, 8 * fk, r_di); + mov_mi(rSP, sd + 16, 0); +} +/* own(c, p): c at [rSP + sd + 8], p on top. c stays in its slot; the caller pops p. + a c that isn't a live unshared object is left alone, as own does; only a + translated owner word needs the call */ +static void x64_fast_own(FastCfg *f, int sd, JW **slow, int *ns) +{ + JW *skip[6]; + int n = 0, j; + mov_rm(r_ax, rSP, sd + 8); + alu_ri(a_cmp, r_ax, HEAP_LO + 5); skip[n++] = j32(c_l); + alu_rm(a_cmp, r_ax, rMB, (int)(8 * f->bump)); skip[n++] = j32(c_ge); + mov_ri(r_cx, MW - 8); alu_rr(a_cmp, r_ax, r_cx); skip[n++] = j32(c_a); + mov_rr(r_di, r_ax); alu_ri(a_xor, r_di, 0x2B7E1516); + mem_op(0, 1, 0, a_cmp * 8 + 3, r_di, rMB, r_ax, 8, -24); skip[n++] = j32(c_ne); + mov_rx(r_si, rMB, r_ax, 8, -32); alu_ri(a_and, r_si, 9); alu_ri(a_cmp, r_si, 1); skip[n++] = j32(c_ne); /* live, not shared */ + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, -2); xb(0); slow[(*ns)++] = j32(c_ne); + mov_rm(r_si, rSP, sd); mov_xr(rMB, r_ax, 8, -16, r_si); + for (j = 0; j < n; j++) fix32(skip[j], cp); +} +/* jump on to the code after the call: its entry stub, or through the dispatcher */ +static void rhop(Rg *R, int to) +{ + int e; + for (e = 0; e < R->ne; e++) if (R->ent[e] == to) { jmp_to((JW *)R->stub[e]); return; } + mov_r32i(r_ax, (unsigned)to); jmp_to(x_disp); +} +/* the constant the block stored to stack slot `slot` (before its delta), or -1 */ +static Cell rso_const(RB *B, int slot) +{ + int j; + for (j = 0; j < B->nso; j++) if (B->so[2 * j] == slot) { int v = B->so[2 * j + 1]; return xkc[v] ? xkv[v] : -1; } + return -1; +} +/* in a region, the fast path of the call that ends block B, if it has one. it + falls through (rdx, the target, untouched) to the ordinary call when it can't + serve it */ +static void rfast_x64(Rg *R, RB *B) +{ + FastCfg f; + Cell target, k; + JW *slow[FAST_MAXJ]; + int ns = 0, j; + if (!xkc[B->cv] || !x64_fastcfg(&f)) return; + target = xkv[B->cv]; + if (target != f.newk && target != f.rleaf) return; + k = rso_const(B, B->delta); /* the class is the last argument, on top */ + if (k < 3 || k > 30 || (target == f.newk && k > 5)) return; + if (trace) fprintf(stderr, "o3: %s class %d inline at %d\n", target == f.newk ? "new_k" : "recycle_leaf", (int)k, B->x2 - 1); + rspill(R); + if (target == f.newk) x64_fast_newk(&f, (int)k, 0, slow, &ns); + else x64_fast_rleaf(&f, (int)k, 0, slow, &ns); + lea(rSP, rSP, -1, 1, 16); + rhop(R, B->x2); + for (j = 0; j < ns; j++) fix32(slow[j], cp); +} static void remit_x64(Rg *R, RB *B) { int i, j, d; @@ -6516,6 +6684,7 @@ static void remit_x64(Rg *R, RB *B) if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2); break; case T_CALL: + rfast_x64(R, B); rcall_x64(R, B); break; case T_RET: @@ -6539,7 +6708,7 @@ static int rjit_x64(Rg *R) { JW *start = cp, **lab; int b, e, need = 64 * R->ne + 8192, rc = 0; - for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256; + for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256 + (R->b[b].term == T_CALL ? 1024 : 0); if (cp - code + need > CODEW * 4 - (1 << 19)) return -2; lab = malloc((size_t)R->nb * sizeof *lab); rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0; @@ -6590,6 +6759,7 @@ static int rjit_x64(Rg *R) fn_nat[R->fn].argc = R->argc; } if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start)); + perfmap_note(start, cp, "o3_", R->lo, R->hi - 1); } else cp = start; free(lab); free(rfa); free(rfb); return rc; @@ -6608,11 +6778,25 @@ static void rcompile(Rg *R, int eager) if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); } } +/* ZYPERFMAP=1 writes /tmp/perf-.map, so `perf report` names translated + code by the vm addresses it came from (zc__, zc_o3__) */ +static FILE *perfmap; +static void perfmap_note(JW *start, JW *end, const char *kind, int a, int b) +{ + static int tried; + if (!tried) { + char path[64]; + tried = 1; + if (getenv("ZYPERFMAP")) { snprintf(path, sizeof path, "/tmp/perf-%d.map", (int)getpid()); perfmap = fopen(path, "w"); } + } + if (perfmap) { fprintf(perfmap, "%lx %lx zc_%s%d_%d\n", (unsigned long)(uintptr_t)start, (unsigned long)((char *)end - (char *)start), kind, a, b); fflush(perfmap); } +} static void *translate(int p) { JW *start; - int i = p, n = 0, k, t, used[64], nused = 0, j2, ju; - Cell w; + int i = p, n = 0, k, t, used[64], nused = 0, j2, ju, lastlit = -1; + Cell w, lastv = 0; + FastCfg fc; if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush(); start = cp; vd = 0; @@ -6632,6 +6816,25 @@ static void *translate(int p) i += 2; continue; } + if (k == 6 && x64_fastcfg(&fc)) { /* new_k, recycle_leaf or own: the fast path inline, the call beside it */ + int kind = t == fc.newk ? 1 : t == fc.rleaf ? 2 : t == fc.own ? 3 : 0; + Cell kc = lastlit == i - 1 ? lastv : -1; + if (kind == 3 || (kind == 1 && kc >= 3 && kc <= 5) || (kind == 2 && kc >= 3 && kc <= 30)) { + JW *slow[FAST_MAXJ], *join; + int ns = 0, j; + if (kind == 1) x64_fast_newk(&fc, (int)kc, 8 * vd, slow, &ns); + else if (kind == 2) x64_fast_rleaf(&fc, (int)kc, 8 * vd, slow, &ns); + else x64_fast_own(&fc, 8 * vd, slow, &ns); + join = jm32(); + for (j = 0; j < ns; j++) fix32(slow[j], cp); + mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); + disp_imm(t); + fix32(join, cp); + vd += kind == 3 ? 1 : 2; /* the arguments are gone, the result is in the first one's slot */ + i += 2; + continue; + } + } if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); } disp_imm(t); i += 2; @@ -6681,6 +6884,7 @@ static void *translate(int p) vd--; if (small) mov_mi(rSP, SL(0), (int)v); else { mov_ri(r_ax, v); stv(0, r_ax); } + lastlit = i; lastv = v; i++; continue; } k = (int)(w / 2); @@ -6790,6 +6994,7 @@ static void *translate(int p) else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2); tab[p] = start; if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start)); + perfmap_note(start, cp, "", p, i - 1); return start; } static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); } @@ -6874,6 +7079,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) x64_noo3 = getenv("ZYNOO3") != 0; x64_noenh = getenv("ZYNOENH") != 0; x64_o3interp = getenv("ZYO3INTERP") != 0; + x64_nofast = getenv("ZYNOFASTALLOC") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; diff --git a/src/core/stack.h b/src/core/stack.h index 8c7f7a2..dbc718a 100644 --- a/src/core/stack.h +++ b/src/core/stack.h @@ -15,6 +15,11 @@ enum { SC_RETURN_LOW, SC_RETURN_TOP, SC_RETURN_NODE, SC_FREE, SC_STOP, SC_BUMP, SC_MAX, SC_HOLD_LOW, SC_HOLD_HIGH, SC_CUT, SC_GC_ACTIVE, SC_COUNT }; +/* optional words after the registered ones, for the native allocator fast paths + in compiled o3 regions: a tag (a runtime without them never has it there), the + address of rt.dbg, and the entry addresses of rt.new_k, rt.recycle_leaf and rt.own */ +enum { SC_FAST = SC_COUNT, SC_DBG, SC_NEW_K, SC_RECYCLE_LEAF, SC_OWN, SC_FAST_END }; +#define SC_FAST_TAG 0x2B7E1517 /* the including core supplies stack_memory_grow and refreshes m if it moves. */ static Cell stack_get(int slot) { Cell p = m[stack_control + slot]; return inmem(p) ? m[p] : 0; } diff --git a/tests/programs/o3_freelist.zc b/tests/programs/o3_freelist.zc new file mode 100644 index 0000000..7836987 --- /dev/null +++ b/tests/programs/o3_freelist.zc @@ -0,0 +1,84 @@ +namespace; +include rt; + +// allocation inside a compiled o3 region. on the x86-64 jit a constant-size +// new(T) pops its size-class list inline and a leaf's free pushes it back +// inline; anything the inline code can't serve (an empty list, a mismatch) +// takes the ordinary rt.new_k / rt.recycle_leaf call. every line must print the +// same with ZYNOFASTALLOC=1, under ZYO3INTERP=1 and on the other backends. +struct point { int x, y; }; // 2 words: class 3 +struct wide { int a, b, c, d, e, f, g, h, i, j; }; // 10 words: class 4, eleven zeroed words +struct cell { cell* next; int v; }; // has a pointer field: not a leaf + +// a block freed at the end of a pass comes back on the next: the sums show the +// payload was zeroed every time +o3 int churn(int n) { + int total = 0, i = 0; + while (i < n) { + point* p = rt.new(point); + total = total + p.x + p.y; + p.x = i; p.y = 7; + i++; + } + return total; +} +o3 int churn_wide(int n) { + int total = 0, i = 0; + while (i < n) { + wide* w = rt.new(wide); + total = total + w.a + w.e + w.j; + w.a = 1; w.e = 2; w.j = 3; + i++; + } + return total; +} +// the second new gets the block the first one freed +o3 int addr(point* p) { int a = 0; unchecked { a = p; } return a; } +o3 int reuse() { + int a = 0, b = 0, z = 0; + { point* p = rt.new(point); a = addr(p); p.x = 41; p.y = 42; } + { point* q = rt.new(point); b = addr(q); z = q.x + q.y; } + if (a == b && z == 0) return 1; + return 0; +} +// more blocks than the class list holds: the list runs empty and new falls back +// to the general allocator, then the whole chain goes back +o3 cell*.u build(int n) { + cell* c = rt.new(cell); + c.v = n; + if (n > 0) c.next = build(n - 1); + return u*c; +} +o3 int sum(cell* c) { + if (!c.next) return c.v; + return c.v + sum(c.next); +} +o3 int chain(int n) { + cell* head = build(n); + int s = sum(head); + rt.free(d*head); + return s; +} +// a header the free can't vouch for: n words of untyped new have no type stamp, +// so this free is the general recycle, and a typed new after it still works +o3 int mixed(int n) { + int i = 0, s = 0; + while (i < n) { + point* p = rt.new(point); + int* w = rt.new(2); + p.x = i; w[0] = i; w[1] = 1; + s = s + p.x - w[0] + w[1]; + i++; + } + return s; +} + +int main() { + rt.put_n(churn(1000)); rt.put_c(' '); + rt.put_n(churn_wide(1000)); rt.put_c(' '); + rt.put_n(reuse()); rt.put_c(' '); + rt.put_n(chain(5000)); rt.put_c(' '); + rt.put_n(mixed(500)); rt.put_c(' '); + rt.put_n(churn(1000) + reuse()); rt.put_c(10); + return 0; +} diff --git a/tests/test.sh b/tests/test.sh index 6992c0d..8bf7a9b 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -103,6 +103,8 @@ run(){ # $1 = vm command, $2 = label chk "$($vm < examples/typed.zc | tr '\n' '/')" "last: 4/point is 2 words: x int, y int/vec is 3 words: len int, cap int, items point**/push is bool(vec*, T.d)/the heap says v is a vec, and v.items[0] is a point/" "$l typed example" chk "$($vm < examples/wiring.zc)" "16 1234 25 4/9" "$l enhancement example" chk "$($vm < tests/programs/allocfast.zc)" "0 0 499500 0" "$l constant-size allocation fast path" + chk "$($vm < tests/programs/o3_freelist.zc)" "0 0 1 12502500 500 1" "$l o3 new and free: class-list reuse, zeroing, an empty list, untyped neighbours" + chk "$(sed 's/^include rt;$/include rt;\nmemdebug on;/' tests/programs/o3_freelist.zc | $vm)" "0 0 1 12502500 500 1" "$l o3 new and free under memdebug take the general path" chk "$($vm < tests/programs/basics.zc | tr '\n' '/')" "285 610 42 512/hey 65 -3 -2 1024 -8/12456/757/9 100-101/5/ok/" "$l language basics" chk "$($vm < tests/programs/umd.zc)" "100 6 0 15 99 5 5" "$l u m d lifetimes" chk "$($vm < tests/programs/names.zc)" "12 1 8" "$l u, m and d are ordinary names" @@ -523,7 +525,9 @@ s_jit() { chk $r 0 "x86-64 jit agrees with the interpreter: $(tail -n 1 $T/jit_x64.out)" chk "$(ZYNOENH=1 $T/zyx < tests/programs/enh_ops.zc | md5sum)" "$(md5sum < tests/programs/enh_ops.expected)" "x86-64 jit with enhancements called through xrun (ZYNOENH=1)" chk "$(ZYTRACE=1 $T/zyx < tests/programs/o3.zc 2>&1 | grep -c 'bytes of x86-64')" "7" "x86-64 jit compiles the o3 regions it can (one has too many live values)" - for p in o3 o3copy o3licm o3push o3bytes fo3 o3call o3deep; do + chk "$(ZYTRACE=1 $T/zyx < tests/programs/o3_freelist.zc 2>&1 | grep -c ' inline at ')" "11" "x86-64 jit inlines new_k and recycle_leaf in compiled o3 regions" + chk "$(ZYNOFASTALLOC=1 $T/zyx < tests/programs/o3_freelist.zc 2>&1)" "$($T/zyx < tests/programs/o3_freelist.zc 2>&1)" "x86-64 o3 inline allocation matches the plain calls (ZYNOFASTALLOC=1)" + for p in o3 o3copy o3licm o3push o3bytes fo3 o3call o3deep o3_freelist; do chk "$($T/zyx < tests/programs/$p.zc 2>&1)" "$(ZYO3INTERP=1 $T/zyx < tests/programs/$p.zc 2>&1)" "x86-64 o3 code matches the region interpreter (ZYO3INTERP=1): $p" done fi -- cgit v1.2.3