From e52a9a399762e5715c261c45d7ecd8e3a68ad50c Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 10:00:13 +0000 Subject: x86-64 jit: translate ordinary vm blocks to x86-64 on linux, o3 regions on the region interpreter --- src/core/core.c | 605 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 590 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/core/core.c b/src/core/core.c index dc50644..51d6b4f 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -53,6 +53,15 @@ #include #endif #endif +/* the x86-64 jit: linux hosts only, and not with the interpreter forced or llvm asked for. + o3 regions run on the region interpreter there. ZYNATIVE is either jit. */ +#if defined(__x86_64__) && defined(__linux__) && !defined(ZYINTERP) && !defined(ZYLLVM) && !defined(ZYJIT) +#define ZYJIT_X64 1 +#include +#endif +#if defined(ZYJIT) || defined(ZYJIT_X64) +#define ZYNATIVE 1 +#endif #if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK) #define ZYBIGSTACK 1 #include @@ -3286,12 +3295,24 @@ static Rg *rat(int p) return R; } -#ifdef ZYJIT +/* ---- the native jits' shared state and rules ---- + both jits (arm64 and x86-64) keep the translation table, the coverage map, the + state block and the store hook here, so a store into translated code, a flush + and a dropped translation mean the same thing on either. the emitters and the + instruction selection are each jit's own. */ +#ifdef ZYNATIVE static VM_TLS unsigned char *cov; /* how many live translations cover the word (255 sticks) */ static VM_TLS void **tab; /* vm address -> translation */ static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); } -static VM_TLS unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first; +#ifdef ZYJIT +typedef unsigned JW; /* code is a string of 32-bit instruction words */ +#else +typedef unsigned char JW; /* code is a string of bytes */ +#define rcov cov /* a lifted o3 region marks its words in the same map */ +#endif +static VM_TLS JW *code, *cp, *x_enter, *x_exit, *x_disp, *x_first; static VM_TLS long S[15]; /* memory, stacks, hooks, limits, and the target saved on overflow */ +#ifdef ZYJIT static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc) { o3_invoke(ps, pr, target, retpc); @@ -3403,6 +3424,158 @@ static void build_stubs(void) E(0xd61f0120); /* br x9 */ x_first = cp; } +#else +/* ---- the x86-64 emitter ---- + registers: r12 holds the memory base, r13 the data stack pointer, r14 the + return stack pointer, r15 the translation table and rbx the state block S + (the arm64 jit's x19..x23). rax, rcx, rdx, rsi, rdi and r8..r11 are scratch. + the callee-saved ones keep their meaning across calls into c, and rsp stays + 16-byte aligned inside translated code so those calls are plain system v + calls. code is built in a byte buffer: a few encoders for the forms the + translator uses (modrm with base, index and displacement, an immediate, + or a register), then one function per vm operation. */ +enum { r_ax, r_cx, r_dx, r_bx, r_sp, r_bp, r_si, r_di, r_8, r_9, r_10, r_11, r_12, r_13, r_14, r_15 }; +enum { a_add = 0, a_or = 1, a_and = 4, a_sub = 5, a_xor = 6, a_cmp = 7 }; /* alu opcode groups */ +enum { c_b = 2, c_ae = 3, c_e = 4, c_ne = 5, c_be = 6, c_a = 7, c_np = 11, c_l = 12, c_ge = 13, c_le = 14, c_g = 15 }; +#define rMB r_12 +#define rSP r_13 +#define rRP r_14 +#define rTAB r_15 +#define rST r_bx + +static void xb(int b) { *cp++ = (unsigned char)b; } +static void xd(unsigned v) { memcpy(cp, &v, 4); cp += 4; } +static void xq(uint64_t v) { memcpy(cp, &v, 8); cp += 8; } +static void rex(int w, int r, int x, int b) +{ + int v = 0x40 | (w ? 8 : 0) | ((r & 8) ? 4 : 0) | ((x & 8) ? 2 : 0) | ((b & 8) ? 1 : 0); + if (v != 0x40) xb(v); +} +/* modrm for [base + index * scale + disp]; index < 0 means none */ +static void modrm_mem(int reg, int base, int index, int scale, int disp) +{ + int mod, sib = index >= 0 || (base & 7) == 4; + if (disp == 0 && (base & 7) != 5) mod = 0; + else if (disp >= -128 && disp <= 127) mod = 1; + else mod = 2; + xb(mod << 6 | (reg & 7) << 3 | (sib ? 4 : (base & 7))); + if (sib) xb((scale == 8 ? 3 : scale == 4 ? 2 : scale == 2 ? 1 : 0) << 6 | ((index >= 0 ? index : 4) & 7) << 3 | (base & 7)); + if (mod == 1) xb(disp); else if (mod == 2) xd((unsigned)disp); +} +/* pre: a mandatory prefix byte or 0; two: the 0f escape */ +static void mem_op(int pre, int w, int two, int opc, int reg, int base, int index, int scale, int disp) +{ + if (pre) xb(pre); + rex(w, reg, index < 0 ? 0 : index, base); + if (two) xb(0x0f); + xb(opc); + modrm_mem(reg, base, index, scale, disp); +} +static void reg_op(int pre, int w, int two, int opc, int reg, int rm) +{ + if (pre) xb(pre); + rex(w, reg, 0, rm); + if (two) xb(0x0f); + xb(opc); + xb(0xc0 | (reg & 7) << 3 | (rm & 7)); +} +static void mov_rr(int d, int s) { reg_op(0, 1, 0, 0x89, s, d); } +static void mov_rm(int r, int b, int disp) { mem_op(0, 1, 0, 0x8b, r, b, -1, 1, disp); } +static void mov_mr(int b, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, -1, 1, disp); } +static void mov_rx(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8b, r, b, x, sc, disp); } +static void mov_xr(int b, int x, int sc, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, x, sc, disp); } +static void mov_r32m(int r, int b, int disp) { mem_op(0, 0, 0, 0x8b, r, b, -1, 1, disp); } /* zero extends */ +static void mov_mi(int b, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, -1, 1, disp); xd((unsigned)imm); } +static void mov_r32i(int r, unsigned imm) { rex(0, 0, 0, r); xb(0xb8 | (r & 7)); xd(imm); } +static void mov_ri(int r, Cell v) +{ + UCell u = (UCell)v; + if (u <= 0xffffffffu) mov_r32i(r, (unsigned)u); + else if (v >= -2147483647 - 1 && v < 0) { reg_op(0, 1, 0, 0xc7, 0, r); xd((unsigned)(int)v); } + else { rex(1, 0, 0, r); xb(0xb8 | (r & 7)); xq(u); } +} +static void lea(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8d, r, b, x, sc, disp); } +static void xor32(int r) { reg_op(0, 0, 0, 0x31, r, r); } +static void alu_rr(int op, int d, int s) { reg_op(0, 1, 0, op * 8 + 1, s, d); } +static void alu_rm(int op, int d, int b, int disp) { mem_op(0, 1, 0, op * 8 + 3, d, b, -1, 1, disp); } +static void alu_ri(int op, int d, int imm) +{ + if (imm >= -128 && imm <= 127) { reg_op(0, 1, 0, 0x83, op, d); xb(imm); } + else { reg_op(0, 1, 0, 0x81, op, d); xd((unsigned)imm); } +} +static void alu_mi(int op, int b, int disp, int imm) +{ + if (imm >= -128 && imm <= 127) { mem_op(0, 1, 0, 0x83, op, b, -1, 1, disp); xb(imm); } + else { mem_op(0, 1, 0, 0x81, op, b, -1, 1, disp); xd((unsigned)imm); } +} +static void test_rr(int a, int b) { reg_op(0, 1, 0, 0x85, b, a); } +static void test32(int a, int b) { reg_op(0, 0, 0, 0x85, b, a); } +static void imul_rm(int d, int b, int disp) { mem_op(0, 1, 1, 0xaf, d, b, -1, 1, disp); } +static void imul_rmi(int d, int b, int disp, int imm) { mem_op(0, 1, 0, 0x69, d, b, -1, 1, disp); xd((unsigned)imm); } +static void shift_cl(int op, int r) { reg_op(0, 1, 0, 0xd3, op, r); } /* 4 shl, 5 shr, 7 sar */ +static void shift_ri(int op, int r, int n) { reg_op(0, 1, 0, 0xc1, op, r); xb(n); } +static void shift_mi(int op, int b, int disp, int n) { mem_op(0, 1, 0, 0xc1, op, b, -1, 1, disp); xb(n); } +static void setcc(int cc, int r) { reg_op(0, 0, 1, 0x90 + cc, 0, r); } /* al, cl or dl only */ +static void movsx_rr8(int d, int s) { reg_op(0, 0, 1, 0xbe, d, s); } /* movsx r32, r8 (al..dl) */ +static void movsx64_rr8(int d, int s) { reg_op(0, 1, 1, 0xbe, d, s); } +static void movsxd_rr(int d, int s) { reg_op(0, 1, 0, 0x63, d, s); } +static void push_r(int r) { rex(0, 0, 0, r); xb(0x50 | (r & 7)); } +static void pop_r(int r) { rex(0, 0, 0, r); xb(0x58 | (r & 7)); } +static void call_m(int b, int disp) { mem_op(0, 0, 0, 0xff, 2, b, -1, 1, disp); } +static void call_r(int r) { reg_op(0, 0, 0, 0xff, 2, r); } +static void jmp_r(int r) { reg_op(0, 0, 0, 0xff, 4, r); } +static void call_abs(void *f) { mov_ri(r_ax, (Cell)(uintptr_t)f); call_r(r_ax); } +/* branches. j8/j32 return the place to patch; the *_to forms go to a known address */ +static JW *j32(int cc) { JW *at; xb(0x0f); xb(0x80 + cc); at = cp; xd(0); return at; } +static JW *jm32(void) { JW *at; xb(0xe9); at = cp; xd(0); return at; } +static JW *j8(int cc) { JW *at; xb(0x70 + cc); at = cp; xb(0); return at; } +static void fix32(JW *at, JW *to) { int d = (int)(to - (at + 4)); memcpy(at, &d, 4); } +static void fix8(JW *at, JW *to) { *at = (JW)(to - (at + 1)); } +static void jcc_to(int cc, JW *to) { JW *at = j32(cc); fix32(at, to); } +static void jmp_to(JW *to) { JW *at = jm32(); fix32(at, to); } +/* sse2 on xmm0 and xmm1 only */ +static void sd_m(int opc, int x, int b, int disp) { mem_op(0xf2, 0, 1, opc, x, b, -1, 1, disp); } /* movsd 10/11, addsd 58, mulsd 59, subsd 5c, divsd 5e */ + +static void build_stubs(void) +{ + JW *ovd, *ovr, *at; + cp = code; + x_enter = cp; /* int enter(long *S, void *code) */ + push_r(r_bp); push_r(r_bx); push_r(r_12); push_r(r_13); push_r(r_14); push_r(r_15); + alu_ri(a_sub, r_sp, 8); /* six pushes and the return address: 16-byte aligned again */ + mov_rr(rST, r_di); + mov_rm(rMB, rST, 0); + mov_rm(r_ax, rST, 8); lea(rSP, rMB, r_ax, 8, 0); /* sp = m + s*8 */ + mov_rm(r_ax, rST, 16); lea(rRP, rMB, r_ax, 8, 0); + mov_rm(rTAB, rST, 24); + jmp_r(r_si); + x_exit = cp; /* eax = next vm address */ + mov_rr(r_cx, rSP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 8, r_cx); + mov_rr(r_cx, rRP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 16, r_cx); + alu_ri(a_add, r_sp, 8); + pop_r(r_15); pop_r(r_14); pop_r(r_13); pop_r(r_12); pop_r(r_bx); pop_r(r_bp); + xb(0xc3); + ovd = cp; + mov_r32i(r_ax, PC_DATA_OVERFLOW); jmp_to(x_exit); + ovr = cp; + mov_r32i(r_ax, PC_RETURN_OVERFLOW); jmp_to(x_exit); + x_disp = cp; /* eax = target: check cancellation and stacks before chaining */ + if (context_current) { + mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx); + at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp); + } + mov_mr(rST, 104, r_ax); + alu_rm(a_cmp, rSP, rST, 88); jcc_to(c_b, ovd); /* sp below its limit */ + alu_rm(a_cmp, rRP, rST, 96); jcc_to(c_ae, ovr); /* rp at its limit */ + mov_ri(r_cx, (Cell)(uintptr_t)&code_words); /* the current code reservation */ + alu_rm(a_cmp, r_ax, r_cx, 0); jcc_to(c_ae, x_exit); + mov_rm(rTAB, rST, 24); + mov_rx(r_cx, rTAB, r_ax, 8, 0); + test_rr(r_cx, r_cx); jcc_to(c_e, x_exit); + jmp_r(r_cx); + x_first = cp; +} +#endif static void flush(void) { @@ -3424,7 +3597,7 @@ static void jitwrite(int on) #endif } -static void syncicache(unsigned *a, unsigned *b) +static void syncicache(JW *a, JW *b) { #ifdef __APPLE__ sys_icache_invalidate(a, (char *)b - (char *)a); @@ -3457,7 +3630,11 @@ static int hook(Cell a, Cell v) if (hit) drop_epoch++; return hit; } +static int inst_hook(int d) { return install(d); } +static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); } +#endif +#ifdef ZYJIT /* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */ static void fjit(int k) { @@ -4240,9 +4417,6 @@ static void rcompile(Rg *R, int eager) if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1; /* registers, not space: give up on it */ } -static int inst_hook(int d) { return install(d); } -static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); } - static void *translate(int p) { unsigned *start; @@ -5032,7 +5206,7 @@ static Blk lcompile(int p) #endif /* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */ -#ifndef ZYLLVM +#if !defined(ZYLLVM) && !defined(ZYJIT_X64) static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */ #endif static VM_TLS Cell rvals[MAXUOP * 4 + 16]; @@ -5052,6 +5226,9 @@ static int rdrop(int a) /* a store landed on region code: drop those regions */ #ifdef ZYLLVM static int lhook(Cell a, Cell v); #define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0) +#elif defined(ZYJIT_X64) +/* a region store may land on translated code too: the jit's hook drops those. the gate word is a plain store, as in the region interpreter anywhere */ +#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (cov[a] && (a) != SYSGATE) hook(a, b); else m[a] = (b); } } while (0) #else #define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0) #endif @@ -5127,9 +5304,21 @@ static int rexec(Rg *R, int e, int *ps, int *pr) return (int)t; } #ifndef ZYLLVM +#ifndef ZYJIT_X64 static void be_drop(int j) { (void)j; } +#endif static void rforget(Rg *R) { (void)R; } +#ifdef ZYJIT_X64 +/* a region entry is never chained to: drop any translation that started there */ +static void rmark(Rg *R) +{ + int e; + R->marked = 1; cinc(rcov, R->lo, R->hi); + for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0; +} +#else static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); } +#endif static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); } static void rcompile(Rg *R, int eager) { (void)R; (void)eager; } #endif @@ -5692,6 +5881,381 @@ static void rcompile(Rg *R, int eager) if (!(R->nat = lregion(R))) R->nonat = 1; } #endif +#ifdef ZYJIT_X64 +/* ---- the x86-64 jit: block translation, and the loop that runs it ---- + the rules are the arm64 jit's: a block ends at call, jump, ret or halt; a jz + does not end it (the not-taken side keeps compiling); a literal followed by + call, jump or jz is a constant target; every exit goes through x_disp, which + checks both stacks against their limits, looks the target up in tab and + chains to it when it is already translated, and otherwise returns to c. the + instruction selection differs: within a block the data stack pointer is not + moved for every push and pop. vd counts the words pushed (negative) or + popped since r13 was last written, stack slot k from the top is + [r13 + 8 * (vd + k)], and r13 is brought up to date before every call into + c and every exit. o3 regions are not compiled here: a region entry is never + translated (the c loop sees it first), and the region interpreter runs it. */ +static VM_TLS int vd; +#define SL(k) (8 * (vd + (k))) +static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); } +static void stv(int k, int r) { mov_mr(rSP, SL(k), r); } +static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); } /* r13 up to date; vd stays for the other path */ +static void sp_sync(void) { sp_keep(); vd = 0; } +static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); } +static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); } /* a constant target: the low 32 bits, like the interpreter's int pc */ +static void disp_ax(void) { sp_keep(); jmp_to(x_disp); } +static void publish_stacks(void) /* s and r into S, as x_exit does, for hooks that call back into the vm */ +{ + lea(r_ax, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 8, r_ax); + mov_rr(r_ax, rRP); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 16, r_ax); +} +static void reload_stacks(void) /* r13, r14, r15 from S after c may have moved the stacks or the table */ +{ + mov_rm(r_cx, rST, 8); lea(rSP, rMB, r_cx, 8, 0); + mov_rm(r_cx, rST, 16); lea(rRP, rMB, r_cx, 8, 0); + mov_rm(rTAB, rST, 24); +} +/* sp = (r13 + 8 * vd - r12) / 8, the data stack index, into r */ +static void sp_index(int r) { lea(r, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r, rMB); shift_ri(5, r, 3); } + +/* a store of rdx to the word rax. known: the address is known to be inside memory. on a + covered word the hook runs and, if it dropped a translation, the block resumes in c at i + 1 */ +static void emit_store(int i, int known) +{ + JW *done1 = 0, *slow, *done2, *z; + if (!known) { alu_ri(a_cmp, r_ax, MW); done1 = j32(c_ae); } + mov_rm(r_cx, rST, 32); + mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0); /* cmp byte [rcx + rax], 0 */ + slow = j32(c_ne); + mov_xr(rMB, r_ax, 8, 0, r_dx); + done2 = jm32(); + fix32(slow, cp); + mov_rr(r_di, r_ax); mov_rr(r_si, r_dx); + publish_stacks(); + call_m(rST, 64); + reload_stacks(); + test32(r_ax, r_ax); + z = j32(c_e); + mov_r32i(r_ax, (unsigned)(i + 1)); jmp_to(x_exit); /* a translation died: resume at i + 1 in c; r13 is exact */ + fix32(z, cp); + if (vd) lea(rSP, rSP, -1, 1, -8 * vd); + if (done1) fix32(done1, cp); + fix32(done2, cp); +} +/* the word at rax, or 0 outside memory, into rcx */ +static void emit_load(void) +{ + JW *over; + xor32(r_cx); + alu_ri(a_cmp, r_ax, MW); + over = j8(c_ae); + mov_rx(r_cx, rMB, r_ax, 8, 0); + fix8(over, cp); +} +/* byte loads: zero extended, 0 unless the whole access lies in memory */ +static void emit_ld_bytes(int n) +{ + JW *over; + ldv(r_ax, 0); + xor32(r_cx); + mov_ri(r_dx, ((Cell)MW << 3) - (n - 1)); + alu_rr(a_cmp, r_ax, r_dx); + over = j8(c_ae); + if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0); + else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0); + else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0); + fix8(over, cp); + stv(0, r_cx); +} +/* c helper for byte stores: 1 if a translation died */ +static long xbst(Cell a, Cell v, long n) +{ + int e = drop_epoch; + b_store(a, v, (int)n); + return drop_epoch != e; +} +static void float_op(int i, int k) +{ + JW *ok; + if (k >= 601 && k <= 604) { + static const unsigned char opc[] = { 0x58, 0x5c, 0x59, 0x5e }; + sd_m(0x10, 0, rSP, SL(1)); + sd_m(opc[k - 601], 0, rSP, SL(0)); + vd++; + sd_m(0x11, 0, rSP, SL(0)); + } else if (k >= 605 && k <= 607) { + xor32(r_ax); + sd_m(0x10, 0, rSP, SL(0)); + mem_op(0x66, 0, 1, 0x2e, 0, rSP, -1, 1, SL(1)); /* ucomisd xmm0, [b]: a against b, unordered sets cf */ + if (k == 605) { setcc(c_e, r_ax); setcc(c_np, r_cx); reg_op(0, 0, 0, 0x20, r_cx, r_ax); } /* eq and not unordered */ + else setcc(k == 606 ? c_a : c_ae, r_ax); /* b < a is a > b; b <= a is a >= b */ + vd++; + stv(0, r_ax); + } else if (k == 608) { + mem_op(0xf2, 1, 1, 0x2a, 0, rSP, -1, 1, SL(0)); /* cvtsi2sd xmm0, qword [sp] */ + sd_m(0x11, 0, rSP, SL(0)); + } else if (k == 609) { /* truncate toward zero; the indefinite result goes to the c helper, which saturates and maps a nan to 0 */ + mem_op(0xf2, 1, 1, 0x2c, r_ax, rSP, -1, 1, SL(0)); /* cvttsd2si rax, [sp] */ + mov_ri(r_cx, (Cell)((UCell)1 << 63)); + alu_rr(a_cmp, r_ax, r_cx); + ok = j8(c_ne); + ldv(r_di, 0); call_abs((void *)fop_f2i); + fix8(ok, cp); + stv(0, r_ax); + } else if (k == 610) { + ldv(r_di, 0); vd++; call_abs((void *)fop_print); + } else if (k == 618) { /* (bits prec mode --) */ + ldv(r_di, 2); ldv(r_si, 1); ldv(r_dx, 0); vd += 3; call_abs((void *)fop_print_mode); + } else { /* 611 */ + ldv(r_di, 0); call_abs((void *)fop_parse_cell); stv(0, r_ax); + } + (void)i; +} + +static void *translate(int p) +{ + JW *start; + int i = p, n = 0, k, t, used[64], nused = 0, j2, ju; + Cell w, x; + if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush(); + start = cp; + vd = 0; + for (;;) { + if (i >= code_words) { exit_with(-1); break; } + if (n++ >= BLKW || (i != p && o3ent[i])) { disp_imm(i); break; } /* a region entry is entered from c, never by chaining */ + w = m[i]; + if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) { + t = (int)((w - 1) / 2); /* literal + call/jmp/jz with a known target */ + if (k == 8) { + JW *skip; + ldv(r_cx, 0); vd++; + test_rr(r_cx, r_cx); + skip = j32(c_ne); + disp_imm(t); + fix32(skip, cp); + i += 2; + continue; + } + if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); } + disp_imm(t); + i += 2; + break; + } + if (w & 1) { + Cell v = (w - 1) / 2; + int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1; + if (i + 1 < code_words && !(m[i + 1] & 1)) ko = (int)(m[i + 1] / 2); + if (ko == 1 && v >= 0 && v < MW && v < (1 << 27)) { /* a fixed address: one load */ + mov_rm(r_ax, rMB, (int)(8 * v)); vd--; stv(0, r_ax); + i += 2; continue; + } + if (small && ko >= 13 && ko <= 20) { /* the literal is an immediate operand */ + int d = (int)v; + switch (ko) { + case 13: alu_mi(a_add, rSP, SL(0), d); break; + case 14: alu_mi(a_sub, rSP, SL(0), d); break; + case 15: imul_rmi(r_ax, rSP, SL(0), d); stv(0, r_ax); break; + case 16: alu_mi(a_and, rSP, SL(0), d); break; + case 17: alu_mi(a_or, rSP, SL(0), d); break; + case 18: alu_mi(a_xor, rSP, SL(0), d); break; + case 19: if (d & 63) shift_mi(7, rSP, SL(0), d & 63); break; + default: xor32(r_cx); alu_mi(a_cmp, rSP, SL(0), d); setcc(c_l, r_cx); stv(0, r_cx); + } + i += 2; continue; + } + vd--; + if (small) mov_mi(rSP, SL(0), (int)v); + else { mov_ri(r_ax, v); stv(0, r_ax); } + i++; continue; + } + k = (int)(w / 2); + if (k == 0) { exit_with(-1); i++; break; } + if (k < 0) { exit_with(-1); i++; break; } /* reserved: the interpreter halts */ + if (k == 6 || k == 7) { + mov_r32m(r_ax, rSP, SL(0)); vd++; + if (k == 6) { mov_mi(rRP, 0, i + 1); alu_ri(a_add, rRP, 8); } + disp_ax(); + i++; + break; + } + if (k == 8) { /* jz: taken leaves, not taken keeps going */ + JW *skip; + mov_r32m(r_ax, rSP, SL(0)); ldv(r_cx, 1); vd += 2; + test_rr(r_cx, r_cx); + skip = j32(c_ne); + disp_ax(); + fix32(skip, cp); + i++; + continue; + } + if (k == 9) { alu_ri(a_sub, rRP, 8); mov_r32m(r_ax, rRP, 0); disp_ax(); i++; break; } + if (k == 21) { /* install: call into c, then return to it so it can flush */ + mov_r32m(r_di, rSP, SL(0)); call_m(rST, 72); + movsxd_rr(r_ax, r_ax); stv(0, r_ax); + exit_with(i + 1); i++; break; + } + if (k == 600) { /* flit: the next word is raw IEEE bits, even when bit 0 is set */ + if (i + 1 >= code_words) { exit_with(-1); i++; break; } + mov_ri(r_ax, m[i + 1]); vd--; stv(0, r_ax); + i += 2; + continue; + } + if ((k >= 601 && k <= 611) || k == 618) { float_op(i, k); i++; continue; } + if (k == 612 || k == 614 || k == 616) { emit_ld_bytes(k == 612 ? 1 : k == 614 ? 2 : 4); i++; continue; } + if (k == 613 || k == 615 || k == 617) { + JW *ok; + ldv(r_di, 0); ldv(r_si, 1); mov_r32i(r_dx, k == 613 ? 1 : k == 615 ? 2 : 4); vd += 2; + call_abs((void *)xbst); + test32(r_ax, r_ax); + ok = j32(c_e); + exit_with(i + 1); + fix32(ok, cp); + i++; continue; + } + if (k == 619) { /* push the data-stack index: the word s points at */ + sp_index(r_ax); vd--; stv(0, r_ax); + i++; continue; + } + if (k == 620) { /* a native call: the helper pops the arguments and pushes the result */ + lea(r_di, rSP, -1, 1, 8 * vd); mov_rr(r_si, rRP); + call_abs((void *)ncall_host); + mov_rr(rSP, r_ax); vd = 0; + i++; continue; + } + if (k >= XOP0 && k < XOP0 + NENH) { + if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; /* wired or not, an install makes this a different block */ + if (enh[k - XOP0].live) { /* the interpreter runs it: x0 = data stack, slot */ + lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0)); + call_m(rST, 80); + mov_rr(rSP, r_ax); vd = 0; + i++; continue; + } + } + if (k > 20) { exit_with(-1); i++; break; } /* not wired: halt */ + switch (k) { + case 1: + ldv(r_ax, 0); emit_load(); stv(0, r_cx); break; + case 2: /* store: inline unless the target word is translated */ + ldv(r_ax, 0); ldv(r_dx, 1); vd += 2; + emit_store(i, 0); break; + case 3: { /* pick uses a word address, with the same bounds as load */ + ldv(r_ax, 0); + sp_index(r_cx); + lea(r_ax, r_ax, r_cx, 1, 1); + emit_load(); stv(0, r_cx); break; } + case 4: vd++; break; + case 5: ldv(r_ax, 0); ldv(r_cx, 1); stv(0, r_cx); stv(1, r_ax); break; + case 10: + call_m(rST, 40); movsx64_rr8(r_ax, r_ax); vd--; stv(0, r_ax); break; + case 11: + mov_rm(r_ax, rST, 56); + if (sizeof J == 1) mem_op(0, 1, 1, 0xbe, r_ax, r_ax, -1, 1, 0); /* movsx rax, byte [rax] */ + else mem_op(0, 1, 0, 0x63, r_ax, r_ax, -1, 1, 0); /* movsxd rax, dword [rax] */ + vd--; stv(0, r_ax); break; + case 12: + ldv(r_ax, 0); movsx_rr8(r_di, r_ax); vd++; call_m(rST, 48); break; + case 13: case 14: case 16: case 17: case 18: + ldv(r_ax, 1); + alu_rm(k == 13 ? a_add : k == 14 ? a_sub : k == 16 ? a_and : k == 17 ? a_or : a_xor, r_ax, rSP, SL(0)); + vd++; stv(0, r_ax); break; + case 15: ldv(r_ax, 1); imul_rm(r_ax, rSP, SL(0)); vd++; stv(0, r_ax); break; + case 19: ldv(r_cx, 0); ldv(r_ax, 1); shift_cl(7, r_ax); vd++; stv(0, r_ax); break; + default: /* 20 */ + ldv(r_ax, 1); xor32(r_cx); alu_rm(a_cmp, r_ax, rSP, SL(0)); setcc(c_l, r_cx); vd++; stv(0, r_cx); + } + i++; + } + j2 = ix_add(p, i); + cinc(cov, p, i); + if (nused && used[0] < 0) dep_add(NENH, j2); + else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2); + tab[p] = start; + if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start)); + return start; +} +static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); } +static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr) +{ + union { JW *u; int (*f)(long *, void *); } enter; + void *saved_eq = 0, *saved_ent[RMAXE]; + int i, nent = 0, flushed = 0, ep0 = drop_epoch, term = 0; + Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1; + enter.u = x_enter; + if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; } + if (re) { + nent = re->ne; if (nent > RMAXE) nent = RMAXE; + for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; } + } + S[1] = *ps; S[2] = *pr; + x64_limits(); + while (!term && !context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) { + void *c; + if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) { + Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW; + if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break; + S[1] = sp; S[2] = rp; + x64_limits(); + pc = (int)S[13]; continue; + } + if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break; + if (stop_ge >= 0 && pc >= stop_ge) break; + if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break; + if (o3ent[pc]) { /* an o3 region: the region interpreter runs it */ + Rg *R = rat(pc); + if (R) { + int s = (int)S[1], r = (int)S[2]; + if (s < stack_data_limit() || r > stack_return_limit()) { + Cell sp = s, rp = r; + int ret = r > stack_return_limit(); + if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; } + s = (int)sp; r = (int)rp; + } + pc = rexec(R, o3ei[pc], &s, &r); + S[1] = s; S[2] = r; + x64_limits(); + if ((unsigned)pc >= (unsigned)code_words) term = 1; /* a stop, not a jit exit: no saved target to resume at */ + continue; + } + } + c = tab[pc]; + if (!c) { + jitwrite(1); c = translate(pc); jitwrite(0); + if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0; + if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0; + } + if (!c) break; + pc = enter.f(S, c); + if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); } + } + *ps = S[1]; *pr = S[2]; + flushed = drop_epoch != ep0; + if (!flushed) { + if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq; + if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i]; + } + return pc; +} +/* the code area, the stubs and the state block, for this thread */ +static int x64_boot(Cell sp, Cell rp, void *stop) +{ + code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (code == MAP_FAILED) { perror("mmap"); return 1; } + jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); + S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; + S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; + S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)stop; + x64_limits(); + cov[SYSGATE] = 1; + return 0; +} +static int run(int p) +{ + Cell s = DS0, r = RS0; + if (x64_boot(s, r, &context_default_stop)) return 1; + p = vm_until(p, -1, -1, 0, &s, &r); + halt_at = -1; + return stopped(p); +} +#else #define LD(x) (inmem(x) ? m[x] : 0) static VM_TLS int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest; static VM_TLS Rg *stop_rg; @@ -5839,6 +6403,7 @@ static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr) stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr; return back; } +#endif static int o3_ready(Rg *C) { if (!C || C->dead) return 0; @@ -5875,7 +6440,7 @@ static void b_commit(Cell w, Cell v) { if ((unsigned)w >= (unsigned)MW) return; if (w == SYSGATE) { m[w] = v; return; } -#if defined(ZYJIT) +#if defined(ZYNATIVE) if (cov[(int)w]) hook(w, v); else m[w] = v; #elif defined(ZYLLVM) if (lcov[(int)w]) lhook(w, v); else m[w] = v; @@ -5915,8 +6480,11 @@ static int code_reserve(Cell end) #define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0) CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1); CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap); +#ifdef ZYNATIVE + CODE_COLUMN(tab, code_words, cap); #ifdef ZYJIT - CODE_COLUMN(tab, code_words, cap); CODE_COLUMN(fn_nat, code_words, cap); + CODE_COLUMN(fn_nat, code_words, cap); +#endif #elif defined(ZYLLVM) CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap); #endif @@ -5928,7 +6496,7 @@ static int code_reserve(Cell end) } for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); } code_words = cap; -#ifdef ZYJIT +#ifdef ZYNATIVE S[3] = (long)tab; #endif return (int)cap; @@ -5940,7 +6508,7 @@ static int code_range_live(Cell start, Cell count) if (start < 0 || count <= 0 || start >= code_words) return 0; end = count > code_words - start ? code_words : start + count; for (i = start; i < end; i++) { -#ifdef ZYJIT +#ifdef ZYNATIVE if (cov[i]) return 1; #else if (rcov[i]) return 1; @@ -6005,8 +6573,11 @@ static void context_engine_free(void) { for (i = 0; i < nrg; i++) { rclear(rgs[i]); free(rgs[i]); } for (i = 0; i < NENH; i++) { free(enh[i].u); free(enh[i].reg); } free(pg_head); free(o3ent); free(o3ei); free(gn_); free(gh_); +#ifdef ZYNATIVE #ifdef ZYJIT - free(tab); free(fn_nat); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1); + free(fn_nat); +#endif + free(tab); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1); #else free(rcov); #ifdef ZYLLVM @@ -6030,7 +6601,7 @@ static void *context_worker(void *opaque) { memcpy(ffi_cb, context->callback_snapshot, sizeof ffi_cb); memcpy(ffi_sym, context->symbol_snapshot, sizeof ffi_sym); memcpy(ffi_lib, context->library_snapshot, sizeof ffi_lib); ffi_nsym = context->ffi_symbols; ffi_nlib = context->ffi_libraries; ffi_tried_libm = context->ffi_libm; #endif -#ifdef ZYJIT +#ifdef ZYNATIVE cov = calloc((size_t)MW, 1); #else rcov = calloc((size_t)MW, 1); @@ -6038,7 +6609,7 @@ static void *context_worker(void *opaque) { lcov = calloc((size_t)MW, 1); #endif #endif -#ifdef ZYJIT +#ifdef ZYNATIVE if (!cov) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } #else if (!rcov @@ -6053,6 +6624,10 @@ static void *context_worker(void *opaque) { nrg = context->region_count; context->region_count = 0; memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words); free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; +#ifdef ZYJIT_X64 + if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } + atomic_fetch_add(&context_code_live, 1); +#endif #ifdef ZYJIT #ifdef __APPLE__ code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0); @@ -6184,7 +6759,7 @@ int main(int argc, char **argv) if (memory_words < 0) return 1; if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; } m = calloc((size_t)MW, sizeof *m); -#ifdef ZYJIT +#ifdef ZYNATIVE cov = calloc((size_t)MW, 1); if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; } #else -- cgit v1.2.3 From 44e60a7afc4673c88890997d5779aa7c6c06f330 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 10:30:45 +0000 Subject: x86-64 jit: zyi target, ZYNOO3, bench and test timeouts, docs; drop a stray file --- Makefile | 6 +++++- README.md | 12 +++++++----- note | 1 - src/core/core.c | 21 ++++++++++++--------- tests/bench/bench.sh | 6 +++++- tests/test.sh | 7 ++++--- 6 files changed, 33 insertions(+), 20 deletions(-) delete mode 100644 note (limited to 'src') diff --git a/Makefile b/Makefile index d5caa92..56894e4 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ # make check the minimal suite, for iterating (tests/test.sh -q) # make test JOBS=4 the same, four sections at a time # make genome build/core/genome.bin from src/, with the fixpoint check +# make build/bin/zyi the interpreter alone (zy has a jit on arm64 and on x86-64 linux) # make lsp build/bin/zclsp, the language server # make web build/web/: the page, its script and zc.wasm # make js build/js/: the javascript vm, its layout and genome @@ -87,9 +88,12 @@ $(B)/core/genome.bin: $(ZC_DEPS) $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) src/core cp $(B)/s1/genome.bin $@ @echo "fixpoint: $$(( $$(wc -c < $@) / 8 )) words, $@ updated" -# the interpreter: the vm core with the compiler genome inside +# the vm core with the compiler genome inside: a jit on arm64 and x86-64 linux, else the interpreter $(BIN)/zy$(EXE): src/core/core.c $(IO) $(B)/core/genome.bin $(CORE_H) | $(BIN) $(CC) $(CFLAGS) -DJTYPE=int src/core/core.c $(IO) $(COREINC) -o $@ +# the interpreter alone, even on a host with a jit: make build/bin/zyi, to compare against zy +$(BIN)/zyi$(EXE): src/core/core.c $(IO) $(B)/core/genome.bin $(CORE_H) | $(BIN) + $(CC) $(CFLAGS) -DZYINTERP -DJTYPE=int src/core/core.c $(IO) $(COREINC) -o $@ # the portable microcore $(BIN)/zym$(EXE): src/micro/micro.c src/micro/micro_port.h $(PORT) $(B)/core/genome.bin $(CORE_H) | $(BIN) $(CC) $(CFLAGS) $(MICRO_THREADS) src/micro/micro.c $(PORT) $(COREINC) -Isrc/micro -o $@ diff --git a/README.md b/README.md index 0b59a6b..bf46a01 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # tinyvm zygote -a small stack cpu began as a C89 code golf entry and became a self-hosting runtime. a native core runs vm code with an arm64 jit, through llvm on supported builds, or in the interpreter. the genome contains a compiler for zc, a C-like language with forth influences. the compiler resides in vm memory alongside the program; code can call its functions, modify its tables, add syntax, and compile new code at runtime. heap objects carry an ownership tag (u, m or d) checked by a static analyzer, with deterministic frees at scope ends and a mark-sweep collector underneath. types are a compiler primitive, with structs, generics, reflection and an optional typechecker. programs can install up to 512 custom opcodes (virtual enhancements), which the jits inline as native code. code marked `o3` bypasses the stack machine: the core compiles the region as a unit and keeps its variables in registers. +a small stack cpu began as a C89 code golf entry and became a self-hosting runtime. a native core runs vm code with an arm64 or x86-64 jit, through llvm on supported builds, or in the interpreter. the genome contains a compiler for zc, a C-like language with forth influences. the compiler resides in vm memory alongside the program; code can call its functions, modify its tables, add syntax, and compile new code at runtime. heap objects carry an ownership tag (u, m or d) checked by a static analyzer, with deterministic frees at scope ends and a mark-sweep collector underneath. types are a compiler primitive, with structs, generics, reflection and an optional typechecker. programs can install up to 512 custom opcodes (virtual enhancements), which the jits inline as native code. code marked `o3` bypasses the stack machine: the core compiles the region as a unit and keeps its variables in registers. this folder has the latest code. `docs/LEARN.md` is a tour of the language. `docs/history/` is the scrapbook: notes for every stage, plus frozen code from the stages that still have it, archived as `docs/history/code-snapshots.tar.gz`. ## files -`src/core/core.c` is the native core. it contains the arm64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; `src/compiler/compiler.zc` is the compiler. both are written in zc and begin with an overview. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by `src/compiler/compiler.zc`. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` checks that image against the memory map before it becomes `build/core/genome.bin`. an image is a file of little-endian 64-bit words, and every core takes its genome in at compile time with c23 `#embed` (`src/core/genome.h`). `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept: +`src/core/core.c` is the native core. it contains the arm64 jit, the x86-64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; `src/compiler/compiler.zc` is the compiler. both are written in zc and begin with an overview. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by `src/compiler/compiler.zc`. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` checks that image against the memory map before it becomes `build/core/genome.bin`. an image is a file of little-endian 64-bit words, and every core takes its genome in at compile time with c23 `#embed` (`src/core/genome.h`). `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept: - `mandel.zc`: the mandelbrot set in 16.16 fixed point, with the fixed-point multiply wired in as a custom opcode. - `bf.zc`: a brainfuck compiler. it turns brainfuck into zc source, has the resident compiler compile it, and runs it (native code on the jit). it takes a program after a backtick, or runs hello world. @@ -71,7 +71,7 @@ windows needs clang or gcc and powershell for the tests. the batch file is the s ### native backends -on arm64, `src/core/core.c` uses its jit by default. other hosts use the interpreter. `-DZYINTERP` forces interpretation; `-DZYLLVM` enables the llvm backend when its headers and libraries are available. direct builds from the root are: +on arm64, `src/core/core.c` uses its arm64 jit by default, and on x86-64 linux its x86-64 jit. other hosts use the interpreter. `-DZYINTERP` forces interpretation (`make build/bin/zyi` builds that core beside `zy`); `-DZYLLVM` enables the llvm backend when its headers and libraries are available. direct builds from the root are: ```sh make genome @@ -433,7 +433,7 @@ going into o3 is free: o3 code reads normal objects the same way and ignores the ## the llvm backend -a tiered jit for hosts the arm64 backend doesn't cover. the interpreter counts how often each block start gets reached, and a hot block is translated to llvm ir using the same block rules as the arm64 jit: it ends at call, jump, ret or halt, keeps going past a jz, and literal targets are fused. the ir goes through the O2 pipeline and gets compiled by orc's lljit. stores into compiled words drop the blocks covering them, like on arm64. it pays off for long-running code: a loop-and-recursion benchmark runs about 3.3x faster than the interpreter. short bursty work like the self-compile is slower, since compile time dominates there. `ZYHOT=1` compiles every block, which is a good stress test, and the test suite runs that way too. +a tiered jit for hosts the native jits don't cover, or where it is asked for explicitly (`-DZYLLVM` replaces the x86-64 jit). the interpreter counts how often each block start gets reached, and a hot block is translated to llvm ir using the same block rules as the arm64 jit: it ends at call, jump, ret or halt, keeps going past a jz, and literal targets are fused. the ir goes through the O2 pipeline and gets compiled by orc's lljit. stores into compiled words drop the blocks covering them, like on arm64. it pays off for long-running code: a loop-and-recursion benchmark runs about 3.3x faster than the interpreter. short bursty work like the self-compile is slower, since compile time dominates there. `ZYHOT=1` compiles every block, which is a good stress test, and the test suite runs that way too. ## the heap @@ -443,7 +443,7 @@ the heap is a buddy allocator over [HEAP_LO, top), where top starts at HEAP_HI a ## c abi -zc can call native c functions and hand zc functions to native code. it is built on the syscall gate (ops 14 to 17), so it needs a host with `dlsym`: the interpreter, the llvm backend and the arm64 jit on linux and macos (the jit has to be a dynamic binary, since a static one has no `dlsym`). the microcore, the javascript vm, the web build and the windows scripts answer -1, and a native call there gives -1 and nothing else. +zc can call native c functions and hand zc functions to native code. it is built on the syscall gate (ops 14 to 17), so it needs a host with `dlsym`: the interpreter, the llvm backend and the arm64 and x86-64 jits on linux and macos (the jit has to be a dynamic binary, since a static one has no `dlsym`). the microcore, the javascript vm, the web build and the windows scripts answer -1, and a native call there gives -1 and nothing else. ``` include rt; @@ -568,6 +568,8 @@ calls push the arguments left to right and the callee's prologue pops them into registers: x19 holds the memory base, x20 the data stack pointer, x21 the return stack pointer, x22 the translation table and x23 the state block (memory, s, r, table, coverage map, Q, G, &J, store hook, install hook, enhancement fallback). o3 regions also use x26..x28, which the entry stub saves. +the x86-64 jit (linux hosts, `ZYJIT_X64`) keeps the same state in callee-saved registers: r12 is the memory base, r13 the data stack pointer, r14 the return stack pointer, r15 the translation table and rbx the state block, with rax, rcx, rdx, rsi, rdi and r8..r11 as scratch and rsp 16-byte aligned inside translated code so calls into c are plain system v calls. it shares everything but the emitter with the arm64 jit: the translation table, the coverage map, the store hook, the dropped-translation bookkeeping, the flush, the block rules below and the dispatch stubs' policy. inside a block it does not move r13 for every push and pop: the translator counts the words pushed and popped and addresses stack slots from r13 with that offset, and brings r13 up to date before a call into c and at every exit. it translates the base opcodes, the float and byte opcodes (the byte loads inline, the byte stores through a c helper), install, and the c abi call; every installed enhancement is a call to the interpreter's evaluator for its slot (`xrun`). o3 regions are not compiled to x86-64: a region entry is never chained to, so control reaches it through the c loop, which runs the region on the region interpreter exactly as the interpreter build does, and a store from either side into the other's code drops what it should (the regions mark their words in the same coverage map). the region interpreter is slower than the jit's own code for the same words (the `bench.sh` o3 kernels take 2 to 4 times longer than their plain versions on this jit, where they are 3 to 8 times faster on the interpreter), so `ZYNOO3=1` leaves regions alone and runs their vm code through the jit, as the microcore does, until a region compiler for x86-64 exists. + compilation is lazy. the first time control reaches an address, that block is translated from live memory. a block ends at call, jump, ret or halt. a `z` doesn't end it: the not-taken side keeps compiling, which makes superblocks. a literal followed by `c`, `j` or `z` is fused into a constant target. every exit goes through a dispatch stub that checks both stack pointers against their limits (kept in x24 and x25), bounds-checks the target, loads `tab[t]`, and branches straight to it when it's already compiled. otherwise it returns to c, which compiles the block and re-enters. enhancements are inlined: inputs load from the data stack into x0..x15, the optimized micro-ops run in registers with x16 and x17 as scratch, and the outputs go back. an install calls into c and invalidates translations and o3 regions that depend on its slot. invalidated code is rebuilt on its next entry. when the loop reaches an o3 entry point, it compiles the region (or reuses it) instead of translating a block. self-modification is handled with a counted coverage map: one byte per word, the number of live translations that cover it, sticky at 255. a store checks that byte inline: an uncovered target, the usual case, gets a plain store. a covered target calls the hook, which looks the word up in a page index (64-word pages, each with a list of the blocks that touch it) and drops only the translations containing that word, decrementing the coverage of just their words. the jit code then returns to c at the next instruction, so even patches to the very next word take effect. writing back the same value is a no-op. installing an enhancement does not flush: every translation records which enhancement slots it inlined, and an install drops only those (and the o3 regions that depend on the slot). if the 16 mb code buffer or the block index fills, everything is flushed. on llvm each block has its own resource tracker, so dropped code is freed once no compiled code is running. diff --git a/note b/note deleted file mode 100644 index 42fe16c..0000000 --- a/note +++ /dev/null @@ -1 +0,0 @@ -zy \ No newline at end of file diff --git a/src/core/core.c b/src/core/core.c index 51d6b4f..b571981 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -1,10 +1,12 @@ /* zygote core: puts the genome (the zc compiler and its runtime, both vm code) - into memory and runs it. on arm64, every block of vm code is translated to - arm64 the first time it is reached, straight from live memory. stores that - land on translated words throw those translations away, so code that - rewrites itself, including the resident compiler, just gets retranslated - on the next visit. on other hosts (or with -DZYINTERP) a plain interpreter - with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */ + into memory and runs it. on arm64 and on x86-64 linux, every block of vm code + is translated to native code the first time it is reached, straight from live + memory. stores that land on translated words throw those translations away, + so code that rewrites itself, including the resident compiler, just gets + retranslated on the next visit. the arm64 jit also compiles o3 regions; on + x86-64 they run on the region interpreter, and enhancements are called, not + inlined. on other hosts (or with -DZYINTERP) a plain interpreter with the same + semantics runs instead. ZYTRACE=1 logs translations on stderr. */ #include #include #include @@ -5895,11 +5897,11 @@ static void rcompile(Rg *R, int eager) c and every exit. o3 regions are not compiled here: a region entry is never translated (the c loop sees it first), and the region interpreter runs it. */ static VM_TLS int vd; +static VM_TLS int x64_noo3; /* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */ #define SL(k) (8 * (vd + (k))) static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); } static void stv(int k, int r) { mov_mr(rSP, SL(k), r); } static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); } /* r13 up to date; vd stays for the other path */ -static void sp_sync(void) { sp_keep(); vd = 0; } static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); } static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); } /* a constant target: the low 32 bits, like the interpreter's int pc */ static void disp_ax(void) { sp_keep(); jmp_to(x_disp); } @@ -6015,7 +6017,7 @@ static void *translate(int p) { JW *start; int i = p, n = 0, k, t, used[64], nused = 0, j2, ju; - Cell w, x; + Cell w; if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush(); start = cp; vd = 0; @@ -6199,7 +6201,7 @@ static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break; if (stop_ge >= 0 && pc >= stop_ge) break; if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break; - if (o3ent[pc]) { /* an o3 region: the region interpreter runs it */ + if (o3ent[pc] && !x64_noo3) { /* an o3 region: the region interpreter runs it */ Rg *R = rat(pc); if (R) { int s = (int)S[1], r = (int)S[2]; @@ -6239,6 +6241,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop) { code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (code == MAP_FAILED) { perror("mmap"); return 1; } + x64_noo3 = getenv("ZYNOO3") != 0; jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp); S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov; S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook; diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index 65f455e..4ef2ca5 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -4,8 +4,12 @@ cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 make -s build/core/genome.bin || exit 1 -$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy +$CC -O2 -w -DJTYPE=int -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy vms=("$T/zy"); names=("interp") +# the host's own jit, where there is one (x86-64 linux: blocks are translated, o3 regions run on the region interpreter) +if [ "$(uname -s)" = Linux ] && [ "$(uname -m)" = x86_64 ]; then + $CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyn && { vms+=("$T/zyn"); names+=("x86-64 jit"); } +fi if command -v llvm-config >/dev/null; then $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } fi diff --git a/tests/test.sh b/tests/test.sh index 709d09c..809da6e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -50,9 +50,10 @@ ROOT=$T CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 export CC -# the language server runs on the interpreter unless the host has the arm64 jit, and a -# check of a large document then takes many seconds: stretch the drivers' deadlines -if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -m)" in arm64|aarch64) ZC_TEST_TIMEOUT_SCALE=1;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi +# the language server runs on the interpreter unless the host has a jit (arm64, or x86-64 +# linux), and a check of a large document then takes many seconds: stretch the drivers' +# deadlines. the x86-64 jit gets a little room, since it does not compile o3 regions +if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -s)-$(uname -m)" in *-arm64|*-aarch64) ZC_TEST_TIMEOUT_SCALE=1;; Linux-x86_64) ZC_TEST_TIMEOUT_SCALE=2;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi export ZC_TEST_TIMEOUT_SCALE # the default map, generated as make does it, and the tools built against it $CC -O2 -w -fwrapv src/tools/genmap.c -o $T/genmap || exit 1 -- cgit v1.2.3