a

1 files changed, 2019 insertions, 0 deletions

diff --git a/docs/history/15-o3/code/core.c b/docs/history/15-o3/code/core.c
new file mode 100644
index 0000000..4e74c93
--- /dev/null
+++ b/docs/history/15-o3/code/core.c
@@ -0,0 +1,2019 @@
+/* zygote core: puts the genome (the zc compiler and its runtime, both vm code)
+   into memory and runs it. on arm64, every block of vm code is translated to
+   arm64 the first time it is reached, straight from live memory. stores that
+   land on translated words throw those translations away, so code that
+   rewrites itself, including the resident compiler, just gets retranslated
+   on the next visit. on other hosts (or with -DZYINTERP) a plain interpreter
+   with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#if defined(__aarch64__) && !defined(ZYINTERP)
+#define ZYJIT 1
+#include <sys/mman.h>
+#ifdef __APPLE__
+#include <pthread.h>
+#include <libkern/OSCacheControl.h>
+#endif
+#endif
+#include "genome.h"
+
+#ifndef JTYPE
+#define JTYPE char
+#endif
+char Q(void);
+void G(char);
+extern JTYPE J;
+
+#define MW (1 << 20)	/* memory, in words */
+#define DS0 (MW - 16)	/* data stack grows down from here */
+#define RS0 0xF0000	/* return stack grows up from here */
+#define DSLIM (0xFC000 + 1100)	/* data stack overflows below this (a block pushes at most ~1024) */
+#define RSLIM (0xF8000 - 8)	/* return stack overflows above this */
+/* why the cpu stopped, when it wasn't a halt */
+static int stopped(int p)
+{
+	if (p == MW + 1) { fprintf(stderr, "zy: data stack overflow\n"); return 2; }
+	if (p == MW + 2) { fprintf(stderr, "zy: return stack overflow\n"); return 2; }
+	return 0;
+}
+#define CODEW (4 << 20)	/* 16 MB of code, in instruction words */
+#define MAXB 65536
+#define BLKW 1024	/* longest block, in vm words */
+
+static int m[MW];
+static int trace;
+
+/* ---- virtual enhancements ----
+   up to 512 custom opcodes (22 .. 533), wired in at runtime by the install
+   opcode (21). an enhancement is a short program over native micro-ops,
+   written either as a recipe (text) or as straight-line vm code that gets
+   lifted. both become ssa, go through the optimizer below, and then run in
+   the interpreter or get inlined by the jit as native code. */
+#define NENH 512
+#define XOP0 22
+enum { U_LI, U_MOV, U_ADD, U_SUB, U_MUL, U_SDIV, U_UDIV, U_SREM, U_UREM, U_AND, U_OR, U_XOR,
+	U_SHL, U_SHR, U_SAR, U_NEG, U_NOT, U_LT, U_LTU, U_EQ, U_NE, U_MIN, U_MAX, U_MINU, U_MAXU,
+	U_MULHS, U_MULHU, U_CLZ, U_ABS, U_SEL, U_LD, U_ST, U_N };
+static const char *unames[U_N] = { "li", "mov", "add", "sub", "mul", "sdiv", "udiv", "srem", "urem",
+	"and", "or", "xor", "shl", "shr", "sar", "neg", "not", "lt", "ltu", "eq", "ne", "min", "max",
+	"minu", "maxu", "mulhs", "mulhu", "clz", "abs", "sel", "ld", "st" };
+static const signed char uarity[U_N] = { 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2,
+	2, 2, 2, 2, 2, 2, 1, 1, 3, 1, 2 };
+typedef struct { int op, d, a, b, c, imm; } Uop;	/* ssa: d is a fresh value, a b c are values */
+typedef struct {
+	int live, kind, nin, nout, n, nv, out[8];
+	Uop *u;
+	signed char *reg;	/* physical register per value, for the jit; 0 if it didn't fit */
+} Enh;
+static Enh enh[NENH];
+static int need_flush;
+
+#define MAXUOP 4096
+static Uop ub[MAXUOP]; static int un, unv, uerr;	/* the program being built */
+static int unew(int op, int a, int b, int c, int imm)
+{
+	Uop *u;
+	if (un >= MAXUOP) { uerr = -5; return 0; }
+	u = &ub[un++]; u->op = op; u->d = unv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
+	return u->d;
+}
+static int mread(int a) { return (unsigned)a < MW ? m[a] : 0; }
+
+/* recipe text: one statement per line or per ';'
+     r0 = sdiv r0 r1      dst = op operands, operands are rN, parameter names or numbers
+     r2 = r0              a copy, or a number
+     st r1 r0             memory[r1] = r0
+   inputs arrive in r0.. (and under the parameter names), outputs leave from r0.. */
+static int parse_recipe(int src, int names, int nin, int nout, int *out)
+{
+	int cur[16], i, p = src, n, k, op, args[3], na, dst, isst;
+	char tok[8][32]; int nt;
+	for (i = 0; i < 16; i++) cur[i] = i < nin ? i : -1;
+	for (;;) {
+		/* read one statement into tokens */
+		nt = 0;
+		for (;;) {
+			int c = mread(p);
+			while (c == ' ' || c == '\t' || c == '\r') c = mread(++p);
+			if (!c || c == '\n' || c == ';') break;
+			if (c == '=') { if (nt < 8) strcpy(tok[nt++], "="); p++; continue; }
+			n = 0;
+			while (c && c != ' ' && c != '\t' && c != '\n' && c != ';' && c != '=' && c != '\r') { if (n < 31) tok[nt < 8 ? nt : 7][n++] = (char)c; c = mread(++p); }
+			if (nt < 8) tok[nt++][n] = 0;
+		}
+		if (nt) {
+			isst = !strcmp(tok[0], "st");
+			if (!isst && (nt < 3 || strcmp(tok[1], "="))) return -2;
+			/* destination register */
+			dst = -1;
+			if (!isst) {
+				if (tok[0][0] == 'r' && tok[0][1] >= '0' && tok[0][1] <= '9') dst = atoi(tok[0] + 1);
+				else for (i = 0; i < nin && names; i++) {
+					int s = mread(names + i), j = 0;
+					while (mread(s + j) && mread(s + j) == (unsigned char)tok[0][j]) j++;
+					if (!mread(s + j) && !tok[0][j]) dst = i;
+				}
+				if (dst < 0 || dst > 15) return -2;
+			}
+			k = isst ? 1 : 2;	/* first token after "=" (or after st) */
+			op = -1;
+			if (!isst && nt > 3) { for (i = 0; i < U_N; i++) if (!strcmp(tok[k], unames[i])) op = i; if (op < 0) return -2; k++; }
+			if (isst) op = U_ST;
+			na = 0;
+			for (; k < nt; k++) {
+				char *t = tok[k], *e; int v = -1;
+				if (na >= 3) return -2;
+				if (t[0] == 'r' && t[1] >= '0' && t[1] <= '9') { int r = atoi(t + 1); if (r > 15) return -2; if (cur[r] < 0) cur[r] = unew(U_LI, 0, 0, 0, 0); v = cur[r]; }
+				else if ((t[0] >= '0' && t[0] <= '9') || t[0] == '-') { long x = strtol(t, &e, 0); if (*e) return -2; v = unew(U_LI, 0, 0, 0, (int)x); }
+				else {
+					for (i = 0; i < nin && names; i++) {
+						int s = mread(names + i), j = 0;
+						while (mread(s + j) && mread(s + j) == (unsigned char)t[j]) j++;
+						if (!mread(s + j) && !t[j]) v = cur[i] < 0 ? (cur[i] = unew(U_LI, 0, 0, 0, 0)) : cur[i];
+					}
+					if (v < 0) return -2;
+				}
+				args[na++] = v;
+			}
+			if (op < 0) {	/* x = y */
+				if (na != 1) return -2;
+				cur[dst] = args[0];
+			} else {
+				if (na != (op == U_LI ? 0 : uarity[op])) {
+					if (!(op == U_LI && na == 1)) return -2;
+				}
+				if (op == U_LI) cur[dst] = args[0];
+				else if (op == U_ST) unew(U_ST, args[0], args[1], 0, 0);
+				else cur[dst] = unew(op, args[0], na > 1 ? args[1] : 0, na > 2 ? args[2] : 0, 0);
+			}
+			if (uerr) return uerr;
+		}
+		if (!mread(p)) break;
+		p++;
+	}
+	for (i = 0; i < nout; i++) { if (cur[i] < 0) cur[i] = unew(U_LI, 0, 0, 0, 0); out[i] = cur[i]; }
+	return uerr;
+}
+
+/* straight-line vm code from addr up to its ret, run symbolically */
+static int lift_vm(int a, int len, int nin, int nout, int *out)
+{
+	int st[256], sp = 0, i, w, k, x, y;
+	for (i = 0; i < nin; i++) st[sp++] = i;
+	for (i = 0; i < len; i++) {
+		w = mread(a + i);
+		if (w & 1) { if (sp >= 256) return -4; st[sp++] = unew(U_LI, 0, 0, 0, w >> 1); continue; }
+		k = w / 2;
+		if (k == 9) break;
+		switch (k) {
+		case 1: if (sp < 1) return -4; st[sp - 1] = unew(U_LD, st[sp - 1], 0, 0, 0); break;
+		case 2: if (sp < 2) return -4; unew(U_ST, st[sp - 1], st[sp - 2], 0, 0); sp -= 2; break;
+		case 3:	/* pick needs a constant index */
+			if (sp < 1) return -4;
+			x = st[sp - 1];
+			if (x < nin) return -4;
+			{ int j; for (j = 0; j < un && ub[j].d != x; j++); if (j == un || ub[j].op != U_LI) return -4; y = ub[j].imm; }
+			if (y < 0 || y >= sp - 1) return -4;
+			st[sp - 1] = st[sp - 2 - y];
+			break;
+		case 4: if (sp < 1) return -4; sp--; break;
+		case 5: if (sp < 2) return -4; x = st[sp - 1]; st[sp - 1] = st[sp - 2]; st[sp - 2] = x; break;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: {
+			static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
+			if (sp < 2) return -4;
+			x = st[sp - 1]; y = st[sp - 2]; sp -= 2;
+			st[sp++] = unew(map[k - 13], y, x, 0, 0);
+			break; }
+		default: return -4;	/* control flow, i/o and other enhancements don't lift */
+		}
+		if (uerr) return uerr;
+	}
+	if (sp != nout) return -4;
+	for (i = 0; i < nout; i++) out[i] = st[i];
+	return uerr;
+}
+
+/* ---- the optimizer ----
+   one forward pass of copy propagation, constant folding, algebraic
+   simplification, division by constants (the multiply-high trick) and
+   common subexpression elimination, then dead code removal. */
+static int ev(int op, int a, int b, int c)
+{
+	unsigned ua = (unsigned)a, ub_ = (unsigned)b;
+	switch (op) {
+	case U_ADD: return (int)(ua + ub_);
+	case U_SUB: return (int)(ua - ub_);
+	case U_MUL: return (int)(ua * ub_);
+	case U_SDIV: return b == 0 ? 0 : (a == (int)0x80000000 && b == -1) ? a : a / b;
+	case U_UDIV: return b == 0 ? 0 : (int)(ua / ub_);
+	case U_SREM: return b == 0 ? a : (a == (int)0x80000000 && b == -1) ? 0 : a % b;
+	case U_UREM: return b == 0 ? a : (int)(ua % ub_);
+	case U_AND: return a & b;
+	case U_OR: return a | b;
+	case U_XOR: return a ^ b;
+	case U_SHL: return (int)(ua << (b & 31));
+	case U_SHR: return (int)(ua >> (b & 31));
+	case U_SAR: return a >> (b & 31);
+	case U_NEG: return (int)(0u - ua);
+	case U_NOT: return ~a;
+	case U_LT: return a < b;
+	case U_LTU: return ua < ub_;
+	case U_EQ: return a == b;
+	case U_NE: return a != b;
+	case U_MIN: return a < b ? a : b;
+	case U_MAX: return a > b ? a : b;
+	case U_MINU: return ua < ub_ ? a : b;
+	case U_MAXU: return ua > ub_ ? a : b;
+	case U_MULHS: return (int)(((long long)a * (long long)b) >> 32);
+	case U_MULHU: return (int)(((unsigned long long)ua * ub_) >> 32);
+	case U_CLZ: { int n = 0; if (!ua) return 32; while (!(ua & 0x80000000u)) { ua <<= 1; n++; } return n; }
+	case U_ABS: return a < 0 ? (int)(0u - ua) : a;
+	case U_SEL: return a ? b : c;
+	}
+	return 0;
+}
+static void smagic(int d, int *mul, int *sh)	/* hacker's delight, signed */
+{
+	const unsigned two31 = 0x80000000u;
+	unsigned ad = d < 0 ? 0u - (unsigned)d : (unsigned)d, t = two31 + ((unsigned)d >> 31);
+	unsigned anc = t - 1 - t % ad, q1 = two31 / anc, r1 = two31 - q1 * anc, q2 = two31 / ad, r2 = two31 - q2 * ad, delta;
+	int p = 31;
+	do {
+		p++;
+		q1 *= 2; r1 *= 2; if (r1 >= anc) { q1++; r1 -= anc; }
+		q2 *= 2; r2 *= 2; if (r2 >= ad) { q2++; r2 -= ad; }
+		delta = ad - r2;
+	} while (q1 < delta || (q1 == delta && r1 == 0));
+	*mul = (int)(q2 + 1); if (d < 0) *mul = -*mul;
+	*sh = p - 32;
+}
+static void umagic(unsigned d, unsigned *mul, int *add, int *sh)	/* hacker's delight, unsigned */
+{
+	unsigned nc = 0xFFFFFFFFu - (0u - d) % d, q1, r1, q2, r2, delta;
+	int p = 31;
+	*add = 0;
+	q1 = 0x80000000u / nc; r1 = 0x80000000u - q1 * nc;
+	q2 = 0x7FFFFFFFu / d; r2 = 0x7FFFFFFFu - q2 * d;
+	do {
+		p++;
+		if (r1 >= nc - r1) { q1 = 2 * q1 + 1; r1 = 2 * r1 - nc; } else { q1 = 2 * q1; r1 = 2 * r1; }
+		if (r2 + 1 >= d - r2) { if (q2 >= 0x7FFFFFFFu) *add = 1; q2 = 2 * q2 + 1; r2 = 2 * r2 + 1 - d; }
+		else { if (q2 >= 0x80000000u) *add = 1; q2 = 2 * q2; r2 = 2 * r2 + 1; }
+		delta = d - 1 - r2;
+	} while (p < 64 && (q1 < delta || (q1 == delta && r1 == 0)));
+	*mul = q2 + 1; *sh = p - 32;
+}
+
+static Uop ob[MAXUOP]; static int on, onv;
+static int *omap, *isk, *kv;	/* old value -> new value; constness of new values */
+static int pure(int op) { return op != U_LD && op != U_ST; }
+static int ispow2(unsigned x) { return x && !(x & (x - 1)); }
+static int lg2(unsigned x) { int n = 0; while (x > 1) { x >>= 1; n++; } return n; }
+static int oe(int op, int a, int b, int c, int imm);
+static int K(int v) { return oe(U_LI, 0, 0, 0, v); }
+static int raw(int op, int a, int b, int c, int imm)
+{
+	Uop *u; int i;
+	if (pure(op))	/* cse */
+		for (i = 0; i < on; i++) {
+			u = &ob[i];
+			if (u->op == op && u->a == a && u->b == b && u->c == c && u->imm == imm) return u->d;
+		}
+	if (on >= MAXUOP) { uerr = -5; return 0; }
+	u = &ob[on++]; u->op = op; u->d = onv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
+	isk[u->d] = op == U_LI; kv[u->d] = imm;
+	return u->d;
+}
+static int sdivk(int a, int d)	/* a / d for a constant d, no divide instruction */
+{
+	int mul, sh, q, t;
+	if (d == 1) return a;
+	if (d == -1) return oe(U_NEG, a, 0, 0, 0);
+	if (d == (int)0x80000000) return oe(U_EQ, a, K(d), 0, 0);
+	if (ispow2((unsigned)(d < 0 ? -d : d))) {
+		int k = lg2((unsigned)(d < 0 ? -d : d));
+		t = oe(U_SAR, a, K(31), 0, 0);
+		t = oe(U_SHR, t, K(32 - k), 0, 0);
+		q = oe(U_SAR, oe(U_ADD, a, t, 0, 0), K(k), 0, 0);
+		return d < 0 ? oe(U_NEG, q, 0, 0, 0) : q;
+	}
+	smagic(d, &mul, &sh);
+	q = oe(U_MULHS, a, K(mul), 0, 0);
+	if (d > 0 && mul < 0) q = oe(U_ADD, q, a, 0, 0);
+	if (d < 0 && mul > 0) q = oe(U_SUB, q, a, 0, 0);
+	if (sh) q = oe(U_SAR, q, K(sh), 0, 0);
+	return oe(U_ADD, q, oe(U_SHR, q, K(31), 0, 0), 0, 0);
+}
+static int udivk(int a, unsigned d)
+{
+	unsigned mul; int add, sh, t;
+	if (d == 1) return a;
+	if (ispow2(d)) return oe(U_SHR, a, K(lg2(d)), 0, 0);
+	if (d > 0x80000000u) return oe(U_XOR, oe(U_LTU, a, K((int)d), 0, 0), K(1), 0, 0);	/* a >= d */
+	umagic(d, &mul, &add, &sh);
+	t = oe(U_MULHU, a, K((int)mul), 0, 0);
+	if (!add) return sh ? oe(U_SHR, t, K(sh), 0, 0) : t;
+	t = oe(U_ADD, oe(U_SHR, oe(U_SUB, a, t, 0, 0), K(1), 0, 0), t, 0, 0);
+	return sh > 1 ? oe(U_SHR, t, K(sh - 1), 0, 0) : t;
+}
+static int oe(int op, int a, int b, int c, int imm)
+{
+	int ka, kb, A, B;
+	if (op == U_LI) return raw(U_LI, 0, 0, 0, imm);
+	if (op == U_MOV) return a;
+	if (op == U_LD || op == U_ST) return raw(op, a, b, 0, 0);
+	ka = isk[a]; kb = uarity[op] > 1 && isk[b];
+	A = kv[a]; B = kv[b];
+	/* everything constant: fold */
+	if (ka && (uarity[op] < 2 || kb) && (uarity[op] < 3 || isk[c])) return K(ev(op, A, B, kv[c]));
+	/* constants go right for commutative ops */
+	if (ka && !kb && (op == U_ADD || op == U_MUL || op == U_AND || op == U_OR || op == U_XOR || op == U_EQ || op == U_NE ||
+		op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU || op == U_MULHS || op == U_MULHU)) {
+		int t = a; a = b; b = t; kb = 1; B = A; ka = 0;
+	}
+	if (op == U_SEL && ka) return A ? b : c;
+	if (op == U_SEL && b == c) return b;
+	if (kb) switch (op) {
+	case U_ADD: case U_SUB: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR:
+		if (!B || (op >= U_SHL && op <= U_SAR && !(B & 31))) return a;
+		break;
+	case U_MUL:
+		if (B == 0) return K(0);
+		if (B == 1) return a;
+		if (B == -1) return oe(U_NEG, a, 0, 0, 0);
+		if (ispow2((unsigned)B)) return oe(U_SHL, a, K(lg2((unsigned)B)), 0, 0);
+		break;
+	case U_AND: if (B == 0) return K(0); if (B == -1) return a; break;
+	case U_SDIV: if (B == 0) return K(0); return sdivk(a, B);
+	case U_SREM: if (B == 0) return a; if (B == 1 || B == -1) return K(0);
+		return oe(U_SUB, a, oe(U_MUL, sdivk(a, B), b, 0, 0), 0, 0);
+	case U_UDIV: if (B == 0) return K(0); return udivk(a, (unsigned)B);
+	case U_UREM: if (B == 0) return a;
+		if (ispow2((unsigned)B)) return oe(U_AND, a, K(B - 1), 0, 0);
+		return oe(U_SUB, a, oe(U_MUL, udivk(a, (unsigned)B), b, 0, 0), 0, 0);
+	}
+	if (a == b && uarity[op] == 2) {
+		if (op == U_SUB || op == U_XOR || op == U_LT || op == U_LTU || op == U_NE) return K(0);
+		if (op == U_AND || op == U_OR || op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU) return a;
+		if (op == U_EQ) return K(1);
+	}
+	return raw(op, a, b, c, 0);
+}
+/* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */
+static int optimize(Enh *e, int nin, int *out, int nout)
+{
+	int i, *live, n = 0;
+	static int omap_[MAXUOP + 16], isk_[MAXUOP * 4], kv_[MAXUOP * 4];
+	omap = omap_; isk = isk_; kv = kv_;
+	on = 0; onv = nin;
+	for (i = 0; i < nin; i++) { omap[i] = i; isk[i] = 0; }
+	for (i = 0; i < un && !uerr; i++) {
+		Uop *u = &ub[i];
+		int a = uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD ? omap[u->a] : 0;
+		int b = uarity[u->op] > 1 || u->op == U_ST ? omap[u->b] : 0;
+		int c = uarity[u->op] > 2 ? omap[u->c] : 0;
+		if (u->op == U_ST) oe(U_ST, a, b, 0, 0);
+		else omap[u->d] = oe(u->op, a, b, c, u->imm);
+	}
+	if (uerr) return uerr;
+	/* dead code: outputs and stores are live */
+	live = calloc(onv + 1, sizeof *live);
+	for (i = 0; i < nout; i++) { e->out[i] = omap[out[i]]; live[e->out[i]] = 1; }
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (u->op != U_ST && !live[u->d]) { u->op = -1; continue; }
+		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) live[u->a] = 1;
+		if (uarity[u->op] > 1 || u->op == U_ST) live[u->b] = 1;
+		if (uarity[u->op] > 2) live[u->c] = 1;
+	}
+	free(live);
+	e->u = malloc((on + 1) * sizeof(Uop));
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) e->u[n++] = ob[i];
+	e->n = n; e->nv = onv; e->nin = nin; e->nout = nout;
+	return 0;
+}
+/* run an installed enhancement on a downward-growing stack; returns the new top */
+static int *xexec(Enh *e, int *sp)
+{
+	int vb[256], *v = e->nv <= 256 ? vb : malloc(e->nv * sizeof(int)), i, j;
+	for (j = 0; j < e->nin; j++) v[j] = sp[e->nin - 1 - j];
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		switch (u->op) {
+		case U_LI: v[u->d] = u->imm; break;
+		case U_LD: v[u->d] = mread(v[u->a]); break;
+		case U_ST: if ((unsigned)v[u->a] < MW) m[v[u->a]] = v[u->b]; break;
+		default: v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
+		}
+	}
+	sp += e->nin - e->nout;
+	for (j = 0; j < e->nout; j++) sp[e->nout - 1 - j] = v[e->out[j]];
+	if (v != vb) free(v);
+	return sp;
+}
+static void xdump(int slot, Enh *e)
+{
+	int i;
+	fprintf(stderr, "enh %d: %d in, %d out, %d ops\n", slot, e->nin, e->nout, e->n);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		if (u->op == U_ST) fprintf(stderr, "  st v%d v%d\n", u->a, u->b);
+		else if (u->op == U_LI) fprintf(stderr, "  v%d = %d\n", u->d, u->imm);
+		else { fprintf(stderr, "  v%d = %s v%d", u->d, unames[u->op], u->a); if (uarity[u->op] > 1) fprintf(stderr, " v%d", u->b); if (uarity[u->op] > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
+	}
+	for (i = 0; i < e->nout; i++) fprintf(stderr, "  out%d = v%d\n", i, e->out[i]);
+}
+/* registers for the jit: values live in w0..w15, straight-line linear scan.
+   if they don't fit, reg stays 0 and the jit calls the interpreter instead. */
+static void regalloc(Enh *e)
+{
+	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = 0xffff;
+	signed char *r = malloc(e->nv);
+	for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; }
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) last[u->a] = i;
+		if (uarity[u->op] > 1 || u->op == U_ST) last[u->b] = i;
+		if (uarity[u->op] > 2) last[u->c] = i;
+	}
+	for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n;
+#define TAKE(v) do { for (k = 0; k < 16 && !(fr >> k & 1); k++); if (k == 16) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
+	for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i]; int ops[3], no = 0;
+		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) ops[no++] = u->a;
+		if (uarity[u->op] > 1 || u->op == U_ST) ops[no++] = u->b;
+		if (uarity[u->op] > 2) ops[no++] = u->c;
+		for (j = 0; j < no; j++) if (last[ops[j]] == i && r[ops[j]] >= 0 && (fr >> r[ops[j]] & 1) == 0) fr |= 1 << r[ops[j]];
+		if (u->op != U_ST) TAKE(u->d);
+		if (u->op != U_ST && last[u->d] == -2) fr |= 1 << r[u->d];	/* never used */
+	}
+#undef TAKE
+	free(last); e->reg = r; return;
+fail:
+	free(last); free(r); e->reg = 0;
+}
+/* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs,
+   outputs, payload address, payload length, parameter names (or 0). kind 2
+   is an o3 region (below).
+   returns 0, or -1 bad descriptor, -2 recipe syntax, -4 code doesn't lift,
+   -5 too big. */
+static int o3install(int d);
+static int install(int d)
+{
+	int slot = mread(d), kind = mread(d + 1), nin = mread(d + 2), nout = mread(d + 3);
+	int pay = mread(d + 4), len = mread(d + 5), names = mread(d + 6), out[8], r;
+	Enh ne;
+	if (kind == 2) return o3install(d);
+	if ((unsigned)slot >= NENH || (unsigned)nin > 8 || (unsigned)nout > 8 || (kind != 0 && kind != 1)) return -1;
+	un = 0; unv = nin; uerr = 0;
+	r = kind == 0 ? parse_recipe(pay, names, nin, nout, out) : lift_vm(pay, len, nin, nout, out);
+	if (r) { if (trace) fprintf(stderr, "enh %d: install failed (%d)\n", slot, r); return r; }
+	memset(&ne, 0, sizeof ne);
+	if ((r = optimize(&ne, nin, out, nout))) return r;
+	ne.kind = kind; ne.live = 1;
+	regalloc(&ne);
+	if (enh[slot].live) { free(enh[slot].u); free(enh[slot].reg); }
+	enh[slot] = ne;
+	need_flush = 1;
+	if (trace) xdump(slot, &enh[slot]);
+	return 0;
+}
+
+/* ---- o3: optimized regions ----
+   o3 code comes from the compiler as a region: a range [lo, hi) of vm words
+   plus up to 7 frame slots that nothing takes the address of (install kind 2:
+   [-, 2, lo, hi, n, slots...]). the region is cut into basic blocks. each
+   block is lifted to ssa: the vm stack is tracked symbolically, so only what
+   is left on it at the end of the block touches memory, the slots live in
+   registers across blocks (VGET reads one at block entry, the block's end
+   says what each one holds now), the frame pointer is var 0, and enhancements
+   are spliced in. every block then goes through the optimizer above. a
+   backend compiles the whole region at once: arm64, llvm at O3, or the
+   region interpreter below. the region is entered at lo and at the return
+   points of its calls. calls, returns, jumps out, and anything a block can't
+   lift leave it: the slots go back to the frame first, so the code outside
+   sees the usual memory. stores to its own code drop it; it lifts again on
+   its next entry. */
+#define ZFP 0xA0000	/* the frame pointer word */
+#define U_VGET U_N	/* a var at block entry; imm = var */
+#define U_SLD (U_N + 1)	/* the stack word at offset imm from the top at block entry */
+#define RMAXV 8
+#define RMAXE 256
+enum { T_BR, T_CBR, T_JMP, T_CALL, T_RET };
+typedef struct {
+	int a, n, nv, delta, nso, term, cv, s1, s2, x1, x2;
+	int vs[RMAXV];	/* each var's value at the end, -1 if unchanged */
+	int *so;	/* stack words to write at the end: offset, value */
+	Uop *u;
+} RB;
+typedef struct {
+	int idx, lo, hi, nv, slot[RMAXV], lifted, dead, nonat, nb, ne, maxnv;
+	int ent[RMAXE], entb[RMAXE];
+	RB *b;
+	void *nat, *stub[RMAXE];	/* the backend's code */
+} Rg;
+#define MAXRG 4096
+static Rg *rgs[MAXRG];
+static int nrg;
+static unsigned short o3ent[MW];	/* region + 1 at an entry point */
+static unsigned char o3ei[MW];		/* which of its entries */
+static void rforget(Rg *R);		/* the backend drops its code */
+static void rmark(Rg *R);		/* the backend watches the region's words */
+static void rcompile(Rg *R, int eager);
+
+static int nargs(int op) { return op >= U_N ? 0 : uarity[op]; }
+static void rclear(Rg *R)
+{
+	int b;
+	rforget(R);
+	for (b = 0; b < R->nb; b++) { free(R->b[b].u); free(R->b[b].so); }
+	free(R->b); R->b = 0; R->nb = 0; R->lifted = 0;
+}
+static void rdrop_all(void) { int j; for (j = 0; j < nrg; j++) if (rgs[j]->lifted) rclear(rgs[j]); }
+
+/* lifting one block */
+static Rg *R_;
+static unsigned char *lead_;
+static int *bidx_, *wl_, nw_;
+static int lv_[RMAXV], lw_[RMAXV];
+#define SB 1100
+static int skv[2 * SB], stp, slo, shi;
+static unsigned char skk[2 * SB];
+static int afp[MAXUOP * 4 + 16];	/* value -> 1 + its offset from fp, if it is fp + k */
+static int emap[MAXUOP * 4 + 16];
+
+static int rvar(int v)
+{
+	if (lv_[v] < 0) { lv_[v] = raw(U_VGET, 0, 0, 0, v); if (!v) afp[lv_[v]] = 1; }
+	return lv_[v];
+}
+static int slotvar(int off) { int v; for (v = 1; v < R_->nv; v++) if (R_->slot[v] == off) return v; return 0; }
+static int rbin(int op, int x, int y, int z)
+{
+	int r = oe(op, x, y, z, 0);
+	if (op == U_ADD && !afp[r]) {
+		if (afp[x] && isk[y]) afp[r] = afp[x] + kv[y];
+		else if (afp[y] && isk[x]) afp[r] = afp[y] + kv[x];
+	}
+	return r;
+}
+static int rld(int a)
+{
+	int v;
+	if (isk[a] && kv[a] == ZFP) return rvar(0);
+	if (afp[a] && (v = slotvar(afp[a] - 1))) return rvar(v);
+	return oe(U_LD, a, 0, 0, 0);
+}
+static int rst(int a, int b)
+{
+	int v;
+	if (isk[a] && kv[a] == ZFP) return 0;	/* moving the frame: not in here */
+	if (afp[a] && (v = slotvar(afp[a] - 1))) { lv_[v] = b; lw_[v] = 1; return 1; }
+	oe(U_ST, a, b, 0, 0);
+	return 1;
+}
+static int sget(int o)
+{
+	if (o < -SB || o >= SB) { uerr = -6; return 0; }
+	if (!skk[o + SB]) {
+		if (o < 0) { uerr = -6; return 0; }
+		skv[o + SB] = raw(U_SLD, 0, 0, 0, o); skk[o + SB] = 1;
+	}
+	if (o < slo) slo = o;
+	if (o >= shi) shi = o + 1;
+	return skv[o + SB];
+}
+static void sset(int o, int v)
+{
+	if (o < -SB || o >= SB) { uerr = -6; return; }
+	skv[o + SB] = v; skk[o + SB] = 1;
+	if (o < slo) slo = o;
+	if (o >= shi) shi = o + 1;
+}
+static int spop(void) { return sget(stp++); }
+static void spush(int v) { sset(--stp, v); }
+static int rblk(int t) { return t >= R_->lo && t < R_->hi && lead_[t - R_->lo] ? bidx_[t - R_->lo] : -1; }
+/* an enhancement, spliced into the block */
+static int rsplice(Enh *e)
+{
+	int i, j;
+	for (j = e->nin; j--;) emap[j] = spop();
+	for (i = 0; i < e->n && !uerr; i++) {
+		Uop *u = &e->u[i];
+		int na = nargs(u->op), a = na > 0 ? emap[u->a] : 0, b = na > 1 ? emap[u->b] : 0, c = na > 2 ? emap[u->c] : 0;
+		if (u->op == U_LI) emap[u->d] = K(u->imm);
+		else if (u->op == U_LD) emap[u->d] = rld(a);
+		else if (u->op == U_ST) { if (!rst(a, b)) uerr = -4; }
+		else emap[u->d] = rbin(u->op, a, b, c);
+	}
+	for (j = 0; j < e->nout; j++) spush(emap[e->out[j]]);
+	return uerr;
+}
+static int lift_block(int bi)
+{
+	static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
+	RB *B = &R_->b[bi];
+	int i = B->a, w, k, x, y, t, j, o, n = 0, *live;
+	on = 0; onv = 0; uerr = 0;
+	memset(afp, 0, sizeof afp); memset(skk, 0, sizeof skk);
+	stp = slo = shi = 0;
+	for (j = 0; j < RMAXV; j++) { lv_[j] = -1; lw_[j] = 0; }
+	B->term = T_JMP; B->cv = -1; B->s1 = B->s2 = -1; B->x1 = B->x2 = 0;
+	for (;;) {
+		if (uerr) return uerr;
+		if (i != B->a && (t = rblk(i)) >= 0) { B->term = T_BR; B->s1 = t; goto done; }
+		if (i < R_->lo || i >= R_->hi) goto leave_at;
+		w = m[i];
+		if (w & 1) { spush(K(w >> 1)); i++; continue; }
+		k = w / 2;
+		switch (k) {
+		case 0: B->cv = K(-1); goto done;	/* halt */
+		case 1: x = spop(); spush(rld(x)); break;
+		case 2:
+			x = sget(stp);
+			if (isk[x] && kv[x] == ZFP) goto leave_at;
+			x = spop(); y = spop(); rst(x, y);
+			break;
+		case 3:	/* pick: a constant depth */
+			x = sget(stp);
+			if (!isk[x] || kv[x] < 0 || kv[x] > 1024) goto leave_at;
+			stp++; spush(sget(stp + kv[x]));
+			break;
+		case 4: stp++; break;
+		case 5: x = sget(stp); y = sget(stp + 1); sset(stp, y); sset(stp + 1, x); break;
+		case 6: B->cv = spop(); B->term = T_CALL; B->x2 = i + 1; goto done;
+		case 7:
+			x = spop();
+			if (isk[x] && (t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; }
+			else B->cv = x;
+			goto done;
+		case 8:
+			x = sget(stp);
+			if (!isk[x]) goto leave_at;
+			stp++; y = spop();
+			if (isk[y]) {	/* a known condition */
+				if (kv[y]) { i++; continue; }
+				if ((t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; } else B->cv = K(kv[x]);
+				goto done;
+			}
+			B->term = T_CBR; B->cv = y;
+			B->x1 = kv[x]; B->s1 = rblk(kv[x]);
+			B->x2 = i + 1; B->s2 = rblk(i + 1);
+			goto done;
+		case 9: B->term = T_RET; goto done;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
+			x = spop(); y = spop(); spush(rbin(map[k - 13], y, x, 0));
+			break;
+		default:
+			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { if (rsplice(&enh[k - XOP0])) return uerr; break; }
+			goto leave_at;	/* i/o, install, unwired: done the usual way */
+		}
+		i++;
+	}
+leave_at:
+	B->term = T_JMP; B->cv = K(i);
+done:
+	if (uerr) return uerr;
+	B->delta = stp;
+	B->so = malloc((shi - slo + 1) * 2 * sizeof(int));
+	B->nso = 0;
+	for (o = stp > slo ? stp : slo; o < shi; o++)
+		if (skk[o + SB]) {
+			x = skv[o + SB];
+			if (ob[x].op == U_SLD && ob[x].imm == o) continue;	/* still the word that was there */
+			if (afp[x] && slotvar(afp[x] - 1)) return -7;	/* a register var's address can't wait in memory */
+			B->so[2 * B->nso] = o; B->so[2 * B->nso + 1] = x; B->nso++;
+		}
+	for (j = 0; j < RMAXV; j++) B->vs[j] = -1;
+	for (j = 1; j < R_->nv; j++)
+		if (lw_[j] && !(ob[lv_[j]].op == U_VGET && ob[lv_[j]].imm == j)) B->vs[j] = lv_[j];
+	/* dead code: what the block leaves behind and its stores are live */
+	live = calloc(onv + 1, sizeof *live);
+	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
+	for (j = 1; j < R_->nv; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) live[B->cv] = 1;
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (u->op != U_ST && !live[u->d]) { u->op = -1; continue; }
+		k = nargs(u->op);
+		if (k > 0) live[u->a] = 1;
+		if (k > 1) live[u->b] = 1;
+		if (k > 2) live[u->c] = 1;
+	}
+	free(live);
+	B->u = malloc((on + 1) * sizeof(Uop));
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) B->u[n++] = ob[i];
+	B->n = n; B->nv = onv;
+	if (onv > R_->maxnv) R_->maxnv = onv;
+	return 0;
+}
+static void addl(int x, int en)
+{
+	int e;
+	if (x < R_->lo || x >= R_->hi) return;
+	if (!lead_[x - R_->lo]) { lead_[x - R_->lo] = 1; wl_[nw_++] = x; }
+	if (en) { for (e = 0; e < R_->ne && R_->ent[e] != x; e++); if (e == R_->ne && e < RMAXE) R_->ent[R_->ne++] = x; }
+}
+/* blocks, then each one lifted and optimized */
+static int rlift(Rg *R)
+{
+	int n = R->hi - R->lo, L, i, w, k, t, b, e, r = 0, cnt;
+	if (R->lifted) rclear(R);
+	R_ = R;
+	lead_ = calloc(n, 1); bidx_ = malloc(n * sizeof(int)); wl_ = malloc(n * sizeof(int)); nw_ = 0;
+	R->ne = 0;
+	addl(R->lo, 1);
+	while (nw_) {
+		L = wl_[--nw_];
+		for (i = L, cnt = 0;;) {
+			if (i >= R->hi || (i != L && lead_[i - R->lo])) break;
+			if (cnt++ >= 1000) { addl(i, 0); break; }
+			w = m[i];
+			if (w & 1) {
+				if (i + 1 < R->hi && !(m[i + 1] & 1) && (k = m[i + 1] / 2) >= 6 && k <= 8) {
+					t = w >> 1;
+					if (k == 6) addl(i + 2, 1);	/* a call comes back here */
+					else if (k == 7) addl(t, 0);
+					else { addl(t, 0); addl(i + 2, 0); }
+					break;
+				}
+				i++;
+				continue;
+			}
+			k = w / 2;
+			if (k == 6) { addl(i + 1, 1); break; }
+			if (k == 8) { addl(i + 1, 0); break; }
+			if (k == 0 || k == 7 || k == 9) break;
+			i++;
+		}
+	}
+	for (i = 0, b = 0; i < n; i++) if (lead_[i]) bidx_[i] = b++;
+	R->nb = b; R->b = calloc(b, sizeof(RB)); R->maxnv = 0;
+	for (i = 0; i < n; i++) if (lead_[i]) R->b[bidx_[i]].a = R->lo + i;
+	for (b = 0; b < R->nb && !r; b++) r = lift_block(b);
+	for (e = 0; e < R->ne; e++) R->entb[e] = bidx_[R->ent[e] - R->lo];
+	free(lead_); free(bidx_); free(wl_);
+	if (r) {
+		if (trace) fprintf(stderr, "o3: %d..%d doesn't lift (%d)\n", R->lo, R->hi - 1, r);
+		R->lifted = 1; rclear(R); return r;
+	}
+	for (e = 0; e < R->ne; e++) { o3ent[R->ent[e]] = R->idx + 1; o3ei[R->ent[e]] = e; }
+	R->lifted = 1;
+	rmark(R);
+	if (trace) {
+		int ops = 0;
+		for (b = 0; b < R->nb; b++) ops += R->b[b].n;
+		fprintf(stderr, "o3: %d..%d: %d blocks, %d entries, %d vars, %d micro-ops\n", R->lo, R->hi - 1, R->nb, R->ne, R->nv - 1, ops);
+		if (getenv("ZYO3DUMP"))
+			for (b = 0; b < R->nb; b++) {
+				RB *B = &R->b[b];
+				fprintf(stderr, " b%d @%d: %d ops, sp %+d, term %d cv v%d s %d/%d x %d/%d\n", b, B->a, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2);
+				for (i = 0; i < B->n; i++) {
+					Uop *u = &B->u[i];
+					if (u->op == U_ST) fprintf(stderr, "   st v%d v%d\n", u->a, u->b);
+					else if (u->op == U_LI) fprintf(stderr, "   v%d = %d\n", u->d, u->imm);
+					else if (u->op == U_VGET) fprintf(stderr, "   v%d = var%d\n", u->d, u->imm);
+					else if (u->op == U_SLD) fprintf(stderr, "   v%d = s[%d]\n", u->d, u->imm);
+					else { fprintf(stderr, "   v%d = %s v%d", u->d, unames[u->op], u->a); if (nargs(u->op) > 1) fprintf(stderr, " v%d", u->b); if (nargs(u->op) > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
+				}
+				for (i = 0; i < B->nso; i++) fprintf(stderr, "   s[%d] = v%d\n", B->so[2 * i], B->so[2 * i + 1]);
+				for (i = 1; i < R->nv; i++) if (B->vs[i] >= 0) fprintf(stderr, "   var%d = v%d\n", i, B->vs[i]);
+			}
+	}
+	return 0;
+}
+/* install kind 2 */
+static int o3install(int d)
+{
+	int lo = mread(d + 2), hi = mread(d + 3), n = mread(d + 4), i, r;
+	Rg *R;
+	if (lo <= 0 || hi <= lo || hi > MW || hi - lo > 1000000 || (unsigned)n > 7 || nrg >= MAXRG) return -1;
+	R = calloc(1, sizeof *R);
+	R->idx = nrg; rgs[nrg++] = R;
+	R->lo = lo; R->hi = hi; R->nv = n + 1;
+	for (i = 0; i < n; i++) { R->slot[i + 1] = mread(d + 5 + i); if ((unsigned)R->slot[i + 1] >= 4096) { R->dead = 1; return -1; } }
+	if ((r = rlift(R))) { R->dead = 1; return r; }
+	rcompile(R, 1);
+	return 0;
+}
+/* the region entered at p, lifted again if it has to be, or 0 */
+static Rg *rat(int p)
+{
+	Rg *R = rgs[o3ent[p] - 1];
+	if (R->dead) { o3ent[p] = 0; return 0; }
+	if (!R->lifted && rlift(R)) { R->dead = 1; o3ent[p] = 0; return 0; }
+	if (o3ei[p] >= R->ne || R->ent[o3ei[p]] != p) { o3ent[p] = 0; return 0; }
+	return R;
+}
+
+#ifdef ZYJIT
+static unsigned char cov[MW];	/* 1 if a live translation covers the word */
+static void *tab[MW];		/* vm address -> translation */
+static struct { int a, b; } bl[MAXB];	/* live translations cover [a,b); a < 0 is dead */
+static int nbl;
+static unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
+static long S[13];	/* m, s, r, tab, cov, Q, G, &J, store hook, install, enhancement fallback, stack limits */
+
+enum { W0 = 0, W1 = 1, X9 = 9, X10, X11, M = 19, SP = 20, RP = 21, TAB = 22, ST = 23 };
+
+static void E(unsigned x) { *cp++ = x; }
+static void li(int d, int v)
+{
+	unsigned u = (unsigned)v;
+	E(0x52800000 | (u & 0xffff) << 5 | d);
+	if (u >> 16) E(0x72a00000 | (u >> 16) << 5 | d);
+}
+static void ldr_post(int t, int n, int i) { E(0xb8400400 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void ldr_pre(int t, int n, int i) { E(0xb8400c00 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void str_pre(int t, int n, int i) { E(0xb8000c00 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void str_post(int t, int n, int i) { E(0xb8000400 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void ldr_u(int t, int n, int o) { E(0xb9400000 | (o / 4) << 10 | n << 5 | t); }
+static void str_u(int t, int n, int o) { E(0xb9000000 | (o / 4) << 10 | n << 5 | t); }
+static void ldrx_u(int t, int n, int o) { E(0xf9400000 | (o / 8) << 10 | n << 5 | t); }
+static void ldr_rr(int t, int n, int r, int ext) { E(0xb8600800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
+static void str_rr(int t, int n, int r, int ext) { E(0xb8200800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
+static void ldp_post(int t, int u, int n, int i) { E(0x28c00000 | ((i / 4) & 0x7f) << 15 | u << 10 | n << 5 | t); }
+static void op3(unsigned op, int d, int n, int r) { E(op | r << 16 | n << 5 | d); }
+static void addx_ext(int d, int n, int r, int ext) { E(0x8b200000 | r << 16 | ext << 13 | 2 << 10 | n << 5 | d); }
+static void movw(int d, int r) { E(0x2a0003e0 | r << 16 | d); }
+static void blr(int n) { E(0xd63f0000 | n << 5); }
+static void b_to(unsigned *t) { long o = t - cp; E(0x14000000 | (o & 0x3ffffff)); }
+static void cb_fix(unsigned *at, unsigned *to) { *at |= ((to - at) & 0x7ffff) << 5; }
+#define SXTW 6
+#define UXTW 2
+
+static void build_stubs(void)
+{
+	cp = code;
+	x_enter = cp;			/* int enter(long *S, void *code) */
+	E(0xa9b97bfd);			/* stp x29, x30, [sp, #-112]! */
+	E(0x910003fd);			/* mov x29, sp */
+	E(0xa90153f3);			/* stp x19, x20, [sp, #16] */
+	E(0xa9025bf5);			/* stp x21, x22, [sp, #32] */
+	E(0xf9001bf7);			/* str x23, [sp, #48] */
+	E(0xa90467f8);			/* stp x24, x25, [sp, #64] */
+	E(0xa9056ffa);			/* stp x26, x27, [sp, #80] */
+	E(0xf90033fc);			/* str x28, [sp, #96] */
+	E(0xaa0003f7);			/* mov x23, x0 */
+	ldrx_u(M, ST, 0);
+	ldrx_u(24, ST, 88); ldrx_u(25, ST, 96);	/* x24, x25: the stack limits, as addresses */
+	ldr_u(X9, ST, 8); addx_ext(SP, M, X9, UXTW);
+	ldr_u(X9, ST, 16); addx_ext(RP, M, X9, UXTW);
+	ldrx_u(TAB, ST, 24);
+	E(0xd61f0020);			/* br x1 */
+	x_exit = cp;			/* w0 = next vm address */
+	op3(0xcb000000, X9, SP, M); E(0xd342fd29); str_u(X9, ST, 8);	/* s = (x20 - x19) >> 2 */
+	op3(0xcb000000, X9, RP, M); E(0xd342fd29); str_u(X9, ST, 16);
+	E(0xa94153f3);			/* ldp x19, x20, [sp, #16] */
+	E(0xa9425bf5);			/* ldp x21, x22, [sp, #32] */
+	E(0xf9401bf7);			/* ldr x23, [sp, #48] */
+	E(0xa94467f8);			/* ldp x24, x25, [sp, #64] */
+	E(0xa9456ffa);			/* ldp x26, x27, [sp, #80] */
+	E(0xf94033fc);			/* ldr x28, [sp, #96] */
+	E(0xa8c77bfd);			/* ldp x29, x30, [sp], #112 */
+	E(0xd65f03c0);			/* ret */
+	{
+	unsigned *ovd = cp, *ovr;
+	li(W0, MW + 1); b_to(x_exit);	/* data stack overflow */
+	ovr = cp;
+	li(W0, MW + 2); b_to(x_exit);	/* return stack overflow */
+	x_disp = cp;			/* w0 = target: check the stacks, chain through tab[] or fall back to c */
+	E(0xeb18029f);			/* cmp x20, x24 */
+	E(0x54000003 | ((ovd - cp) & 0x7ffff) << 5);	/* b.lo data overflow */
+	E(0xeb1902bf);			/* cmp x21, x25 */
+	E(0x54000002 | ((ovr - cp) & 0x7ffff) << 5);	/* b.hs return overflow */
+	}
+	E(0x7144001f);			/* cmp w0, #MW */
+	E(0x54000002 | ((x_exit - cp) & 0x7ffff) << 5);	/* b.hs exit */
+	E(0xf8605ac9);			/* ldr x9, [x22, w0, uxtw #3] */
+	E(0xb4000009 | ((x_exit - cp) & 0x7ffff) << 5);	/* cbz x9, exit */
+	E(0xd61f0120);			/* br x9 */
+	x_first = cp;
+}
+
+static void flush(void)
+{
+	memset(tab, 0, sizeof tab);
+	memset(cov, 0, sizeof cov);
+	nbl = 0;
+	cp = x_first;
+	rdrop_all();
+	if (trace) fprintf(stderr, "jit: flushed\n");
+}
+
+static void jitwrite(int on)
+{
+#ifdef __APPLE__
+	pthread_jit_write_protect_np(!on);
+#else
+	(void)on;
+#endif
+}
+
+static void syncicache(unsigned *a, unsigned *b)
+{
+#ifdef __APPLE__
+	sys_icache_invalidate(a, (char *)b - (char *)a);
+#else
+	__builtin___clear_cache((char *)a, (char *)b);
+#endif
+}
+
+/* store hook for words that are covered by a translation */
+static int hook(int a, int v)
+{
+	int j, x, hit = 0;
+	if (m[a] == v) return 0;
+	m[a] = v;
+	for (j = 0; j < nbl; j++)
+		if (bl[j].a >= 0 && bl[j].a <= a && a < bl[j].b) {
+			tab[bl[j].a] = 0;
+			for (x = bl[j].a; x < bl[j].b; x++) cov[x] = 0;
+			if (trace) fprintf(stderr, "jit: dropped %d..%d\n", bl[j].a, bl[j].b - 1);
+			bl[j].a = -1;
+			hit = 1;
+		}
+	for (j = 0; j < nrg; j++)
+		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
+			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d)\n", rgs[j]->lo, rgs[j]->hi - 1, a);
+			rclear(rgs[j]);
+			memset(cov, 0, sizeof cov);
+			hit = 1;
+		}
+	for (j = 0; j < nbl; j++)
+		if (bl[j].a >= 0) for (x = bl[j].a; x < bl[j].b; x++) cov[x] = 1;
+	for (j = 0; j < nrg; j++) if (rgs[j]->lifted) rmark(rgs[j]);
+	return hit;
+}
+
+static void op(int i, int k)	/* everything except control flow */
+{
+	unsigned *a, *b, *c, *z;
+	switch (k) {
+	case 1: ldr_u(X9, SP, 0); ldr_rr(X9, M, X9, SXTW); str_u(X9, SP, 0); return;
+	case 2:	/* store: inline unless the target word is translated */
+		ldp_post(X9, X10, SP, 8);
+		E(0x7144013f);			/* cmp w9, #MW */
+		a = cp; E(0x54000002);		/* b.hs done */
+		ldrx_u(X11, ST, 32);
+		E(0x3869496b);			/* ldrb w11, [x11, w9, uxtw] */
+		b = cp; E(0x3500000b);		/* cbnz w11, slow */
+		str_rr(X10, M, X9, UXTW);
+		c = cp; E(0x14000000);		/* b done */
+		cb_fix(b, cp);			/* slow: hook(a, v) */
+		movw(W0, X9); movw(W1, X10); ldrx_u(X11, ST, 64); blr(X11);
+		z = cp; E(0x34000000);		/* cbz w0, done */
+		li(W0, i + 1); b_to(x_exit);	/* a translation died: resume at i+1 in c */
+		cb_fix(a, cp); cb_fix(z, cp);
+		*c |= (cp - c) & 0x3ffffff;
+		return;
+	case 3: ldr_u(X9, SP, 0); E(0x9100128a); ldr_rr(X9, X10, X9, SXTW); str_u(X9, SP, 0); return; /* add x10, x20, #4 */
+	case 4: E(0x91001294); return;	/* add x20, x20, #4 */
+	case 5: E(0x29402a89); E(0x2900268a); return;	/* ldp w9, w10, [x20]; stp w10, w9, [x20] */
+	case 10: ldrx_u(X11, ST, 40); blr(X11); E(0x13001c09); str_pre(X9, SP, -4); return; /* sxtb w9, w0 */
+	case 11:
+		ldrx_u(X9, ST, 56);
+		if (sizeof J == 1) E(0x39400129); else ldr_u(X9, X9, 0);	/* ldrb w9, [x9] */
+		str_pre(X9, SP, -4);
+		return;
+	case 12: ldr_post(W0, SP, 4); E(0x13001c00); ldrx_u(X11, ST, 48); blr(X11); return; /* sxtb w0, w0 */
+	}
+	ldr_post(X9, SP, 4);
+	ldr_u(X10, SP, 0);
+	switch (k) {
+	case 13: op3(0x0b000000, X10, X10, X9); break;
+	case 14: op3(0x4b000000, X10, X10, X9); break;
+	case 15: op3(0x1b007c00, X10, X10, X9); break;
+	case 16: op3(0x0a000000, X10, X10, X9); break;
+	case 17: op3(0x2a000000, X10, X10, X9); break;
+	case 18: op3(0x4a000000, X10, X10, X9); break;
+	case 19: op3(0x1ac02800, X10, X10, X9); break;
+	default: E(0x6b09015f); E(0x1a9fa7ea);	/* cmp w10, w9; cset w10, lt */
+	}
+	str_u(X10, SP, 0);
+}
+
+/* one micro-op, operands in registers. tmp is a scratch register for the
+   remainders; x16 and x17 are scratch too. */
+static void emit_uop(int op, int d, int a, int b, int c, int tmp)
+{
+	switch (op) {
+	case U_ADD: op3(0x0b000000, d, a, b); break;
+	case U_SUB: op3(0x4b000000, d, a, b); break;
+	case U_MUL: op3(0x1b007c00, d, a, b); break;
+	case U_SDIV: op3(0x1ac00c00, d, a, b); break;
+	case U_UDIV: op3(0x1ac00800, d, a, b); break;
+	case U_SREM: op3(0x1ac00c00, tmp, a, b); E(0x1b008000 | b << 16 | a << 10 | tmp << 5 | d); break;	/* msub */
+	case U_UREM: op3(0x1ac00800, tmp, a, b); E(0x1b008000 | b << 16 | a << 10 | tmp << 5 | d); break;
+	case U_AND: op3(0x0a000000, d, a, b); break;
+	case U_OR: op3(0x2a000000, d, a, b); break;
+	case U_XOR: op3(0x4a000000, d, a, b); break;
+	case U_SHL: op3(0x1ac02000, d, a, b); break;
+	case U_SHR: op3(0x1ac02400, d, a, b); break;
+	case U_SAR: op3(0x1ac02800, d, a, b); break;
+	case U_NEG: E(0x4b0003e0 | a << 16 | d); break;
+	case U_NOT: E(0x2a2003e0 | a << 16 | d); break;
+	case U_LT: E(0x6b00001f | b << 16 | a << 5); E(0x1a9fa7e0 | d); break;	/* cmp; cset lt */
+	case U_LTU: E(0x6b00001f | b << 16 | a << 5); E(0x1a9f27e0 | d); break;	/* cset lo */
+	case U_EQ: E(0x6b00001f | b << 16 | a << 5); E(0x1a9f17e0 | d); break;
+	case U_NE: E(0x6b00001f | b << 16 | a << 5); E(0x1a9f07e0 | d); break;
+	case U_MIN: E(0x6b00001f | b << 16 | a << 5); E(0x1a800000 | b << 16 | 11 << 12 | a << 5 | d); break;
+	case U_MAX: E(0x6b00001f | b << 16 | a << 5); E(0x1a800000 | b << 16 | 12 << 12 | a << 5 | d); break;
+	case U_MINU: E(0x6b00001f | b << 16 | a << 5); E(0x1a800000 | b << 16 | 3 << 12 | a << 5 | d); break;
+	case U_MAXU: E(0x6b00001f | b << 16 | a << 5); E(0x1a800000 | b << 16 | 8 << 12 | a << 5 | d); break;
+	case U_MULHS: E(0x9b207c00 | b << 16 | a << 5 | 16); E(0xd360fc00 | 16 << 5 | d); break;	/* smull; lsr #32 */
+	case U_MULHU: E(0x9ba07c00 | b << 16 | a << 5 | 16); E(0xd360fc00 | 16 << 5 | d); break;
+	case U_CLZ: E(0x5ac01000 | a << 5 | d); break;
+	case U_ABS: E(0x7100001f | a << 5); E(0x5a800400 | a << 16 | 10 << 12 | a << 5 | d); break;	/* cmp #0; cneg */
+	case U_SEL: E(0x7100001f | a << 5); E(0x1a800000 | c << 16 | 1 << 12 | b << 5 | d); break;	/* csel ne */
+	case U_LD:	/* out of range reads 0, like the interpreter */
+		E(0x52800011); E(0x7144001f | a << 5); E(0x54000042); ldr_rr(17, M, a, UXTW); movw(d, 17);
+		break;
+	case U_ST: E(0x7144001f | a << 5); E(0x54000042); str_rr(b, M, a, UXTW); break;
+	}
+}
+/* an enhancement, inlined: inputs from the data stack into w0..w15, the
+   optimized micro-ops, outputs back. constants are made where they're used,
+   or folded into the instruction (a fixed address, a small addend). */
+static int ekc[MAXUOP * 4 + 16], ekv[MAXUOP * 4 + 16];
+static void emit_enh_e(Enh *e, int s)
+{
+	signed char *r = e->reg;
+	int i, j, delta, na;
+	if (!r) {	/* didn't fit in registers: x0 = data stack, w1 = slot, call the interpreter */
+		E(0xaa1403e0); li(W1, s); ldrx_u(X11, ST, 80); blr(X11); E(0xaa0003f4);
+		return;
+	}
+	for (i = 0; i < e->nv; i++) ekc[i] = 0;
+	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { ekc[e->u[i].d] = 1; ekv[e->u[i].d] = e->u[i].imm; }
+#define EOP(x) (ekc[x] ? (li(r[x], ekv[x]), r[x]) : r[x])
+	for (j = 0; j < e->nin; j++) if (r[j] >= 0) ldr_u(r[j], SP, 4 * (e->nin - 1 - j));
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		int d = u->op == U_ST ? 0 : r[u->d], a = 0, b = 0, c = 0;
+		if (u->op == U_LI) continue;
+		if (u->op == U_LD && ekc[u->a] && (unsigned)ekv[u->a] < MW) {
+			if (ekv[u->a] < 4096) ldr_u(d, M, ekv[u->a] * 4); else ldr_rr(d, M, EOP(u->a), UXTW);
+			continue;
+		}
+		if ((u->op == U_ADD || u->op == U_SUB) && ekc[u->b] && ekv[u->b] >= 0 && ekv[u->b] < 4096) {
+			E((u->op == U_ADD ? 0x11000000 : 0x51000000) | ekv[u->b] << 10 | EOP(u->a) << 5 | d);
+			continue;
+		}
+		na = nargs(u->op);
+		if (na > 0) a = EOP(u->a);
+		if (na > 1) b = EOP(u->b);
+		if (na > 2) c = EOP(u->c);
+		emit_uop(u->op, d, a, b, c, 16);
+	}
+	delta = 4 * (e->nin - e->nout);
+	if (delta > 0) E(0x91000294 | delta << 10);
+	if (delta < 0) E(0xd1000294 | (-delta) << 10);
+	for (j = 0; j < e->nout; j++) str_u(EOP(e->out[j]), SP, 4 * (e->nout - 1 - j));
+#undef EOP
+}
+static void emit_enh(int s) { emit_enh_e(&enh[s], s); }
+/* lit .. lit xop: the enhancement specialized on the literals that feed its
+   last inputs, so the optimizer sees them as constants (x / 7 becomes a
+   multiply, lfa(k) a load and an add). 0 if it worked. */
+static int spec_enh(int s, int *ks, int nk, Enh *te)
+{
+	Enh *e = &enh[s];
+	int nin = e->nin - nk, i, j, out[8], na;
+	static int mp[MAXUOP * 4 + 16];
+	if (e->nv > MAXUOP * 4) return -1;
+	un = 0; unv = nin; uerr = 0;
+	for (j = 0; j < nin; j++) mp[j] = j;
+	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		na = nargs(u->op);
+		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+		if (u->op != U_ST) mp[u->d] = j;
+	}
+	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
+	memset(te, 0, sizeof *te);
+	if (uerr || optimize(te, nin, out, e->nout)) return -1;
+	regalloc(te);
+	if (!te->reg) { free(te->u); return -1; }
+	return 0;
+}
+
+/* ---- o3 regions on arm64 ----
+   vars live in w0..w7 (w0 is the frame pointer), block values in w8..w14,
+   w26..w28 and the homes of unused vars. constants have no register: they
+   are made in a scratch register where they're used (w16, w17, w15), or
+   folded into the instruction. x15..x17 are scratch. */
+static signed char rreg[MAXUOP * 4 + 16];
+static unsigned char rk[MAXUOP * 4 + 16], rhome[MAXUOP * 4 + 16];
+static int rkv[MAXUOP * 4 + 16], rlast[MAXUOP * 4 + 16];
+static unsigned **rfa; static int *rfb, rnf;	/* branches to blocks, fixed up at the end */
+
+static void rforget(Rg *R)
+{
+	int e;
+	if (R->nat) for (e = 0; e < R->ne; e++) if (tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0;
+	R->nat = 0;
+}
+static void rmark(Rg *R) { int x; for (x = R->lo; x < R->hi; x++) cov[x] = 1; }
+static int ralloc(Rg *R, RB *B)
+{
+	int i, j, k, n = B->n;
+	unsigned fr = 0x7f00u | 0x1c000000u;	/* w8..w14, w26..w28 */
+	for (j = R->nv; j < 8; j++) fr |= 1u << j;
+	for (i = 0; i < B->nv; i++) { rlast[i] = -2; rk[i] = 0; rhome[i] = 0; rreg[i] = -1; }
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i]; int na = nargs(u->op);
+		if (na > 0) rlast[u->a] = i;
+		if (na > 1) rlast[u->b] = i;
+		if (na > 2) rlast[u->c] = i;
+	}
+	for (j = 0; j < B->nso; j++) rlast[B->so[2 * j + 1]] = n;
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) rlast[B->vs[j]] = n;
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) rlast[B->cv] = n;
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i]; int na = nargs(u->op), ops[3], d = u->d;
+		ops[0] = u->a; ops[1] = u->b; ops[2] = u->c;
+		for (j = 0; j < na; j++) { int x = ops[j]; if (rlast[x] == i && rreg[x] >= 0 && !rhome[x]) fr |= 1u << rreg[x]; }
+		if (u->op == U_ST) continue;
+		if (u->op == U_LI) { rk[d] = 1; rkv[d] = u->imm; continue; }
+		if (u->op == U_VGET) { rreg[d] = (signed char)u->imm; rhome[d] = 1; continue; }
+		for (k = 0; k < 32 && !(fr >> k & 1); k++);
+		if (k == 32) return -1;
+		fr &= ~(1u << k); rreg[d] = (signed char)k;
+		if (rlast[d] == -2) fr |= 1u << k;
+	}
+	return 0;
+}
+static int ropr(int x, int s) { if (rk[x]) { li(s, rkv[x]); return s; } return rreg[x]; }
+/* w t <-> the data stack word o places from the top at block entry */
+static void rsw(int t, int o, int store)
+{
+	int by = o * 4;
+	if (by >= 0 && by < 16384) { if (store) str_u(t, SP, by); else ldr_u(t, SP, by); }
+	else if (by >= -256 && by < 256) E((store ? 0xb8000000u : 0xb8400000u) | (unsigned)(by & 0x1ff) << 12 | SP << 5 | t);	/* stur, ldur */
+	else { li(16, by); addx_ext(16, SP, 16, SXTW); if (store) str_u(t, 16, 0); else ldr_u(t, 16, 0); }
+}
+static void addsp(int by)
+{
+	if (by > 0 && by < 4096) E(0x91000294 | by << 10);
+	else if (by < 0 && -by < 4096) E(0xd1000294 | (-by) << 10);
+	else if (by) { li(16, by); addx_ext(SP, SP, 16, SXTW); }
+}
+static void rspill(Rg *R)
+{
+	int v;
+	for (v = 1; v < R->nv; v++) { E(0x11000000 | R->slot[v] << 10 | 16); str_rr(v, M, 16, UXTW); }	/* add w16, w0, #slot */
+}
+static void rload(Rg *R)
+{
+	int v;
+	li(16, ZFP); ldr_rr(0, M, 16, UXTW);
+	for (v = 1; v < R->nv; v++) { E(0x11000000 | R->slot[v] << 10 | 16); ldr_rr(v, M, 16, UXTW); }
+}
+static void rexitk(Rg *R, int t) { rspill(R); li(W0, t); b_to(x_disp); }
+static void rbranch(int b) { rfa[rnf] = cp; rfb[rnf++] = b; E(0x14000000); }
+/* m[a] = v, checked like the jit's own stores: a word some translation covers goes through hook */
+static void rstore(int a, int v)
+{
+	unsigned *p1, *p2, *p3;
+	int j;
+	E(0x7144001f | a << 5);		/* cmp wa, #MW */
+	p1 = cp; E(0x54000002);		/* b.hs done */
+	ldrx_u(15, ST, 32);
+	E(0x38604800 | a << 16 | 15 << 5 | 15);	/* ldrb w15, [x15, wa, uxtw] */
+	p2 = cp; E(0x35000000 | 15);	/* cbnz w15, slow */
+	str_rr(v, M, a, UXTW);
+	p3 = cp; E(0x14000000);
+	cb_fix(p2, cp);
+	E(0xd10203ff);			/* sub sp, sp, #128 */
+	for (j = 0; j < 16; j += 2) E(0xa9000000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);	/* stp xj, xj+1, [sp, #8j] */
+	movw(15, a); movw(1, v); movw(0, 15);
+	ldrx_u(16, ST, 64); blr(16);
+	for (j = 0; j < 16; j += 2) E(0xa9400000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
+	E(0x910203ff);			/* add sp, sp, #128 */
+	cb_fix(p1, cp);
+	*p3 |= (cp - p3) & 0x3ffffff;
+}
+static void rpmove(Rg *R, RB *B)
+{
+	int dst[RMAXV], src[RMAXV], n = 0, j, k, x;
+	for (j = 1; j < R->nv; j++)
+		if (B->vs[j] >= 0 && !rk[B->vs[j]] && rreg[B->vs[j]] != j) { dst[n] = j; src[n] = rreg[B->vs[j]]; n++; }
+	while (n) {
+		for (j = 0; j < n; j++) { for (k = 0; k < n && src[k] != dst[j]; k++); if (k == n) break; }
+		if (j < n) { movw(dst[j], src[j]); n--; dst[j] = dst[n]; src[j] = src[n]; continue; }
+		x = src[0]; movw(16, x);	/* a cycle: park one */
+		for (k = 0; k < n; k++) if (src[k] == x) src[k] = 16;
+	}
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && rk[B->vs[j]]) li(j, rkv[B->vs[j]]);
+}
+static void emit_uop(int op, int d, int a, int b, int c, int tmp);
+static int remit(Rg *R, int bi)
+{
+	RB *B = &R->b[bi];
+	int i, j, d, a, b, c;
+	unsigned *p;
+	if (ralloc(R, B)) return -1;
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		switch (u->op) {
+		case U_LI: case U_VGET: continue;
+		case U_SLD: rsw(rreg[u->d], u->imm, 0); continue;
+		case U_LD:
+			if (rk[u->a] && (unsigned)rkv[u->a] < MW) {	/* a fixed address */
+				if (rkv[u->a] < 4096) ldr_u(rreg[u->d], M, rkv[u->a] * 4);
+				else { li(16, rkv[u->a]); ldr_rr(rreg[u->d], M, 16, UXTW); }
+				continue;
+			}
+			break;
+		case U_ST: a = ropr(u->a, 16); rstore(a, ropr(u->b, 17)); continue;
+		case U_ADD: case U_SUB:
+			if (rk[u->b] && rkv[u->b] >= 0 && rkv[u->b] < 4096) {
+				E((u->op == U_ADD ? 0x11000000 : 0x51000000) | rkv[u->b] << 10 | ropr(u->a, 16) << 5 | rreg[u->d]);
+				continue;
+			}
+			break;
+		}
+		d = rreg[u->d];
+		a = nargs(u->op) > 0 ? ropr(u->a, 16) : 0;
+		b = nargs(u->op) > 1 ? ropr(u->b, 17) : 0;
+		c = nargs(u->op) > 2 ? ropr(u->c, 15) : 0;
+		emit_uop(u->op, d, a, b, c, 15);
+	}
+	for (j = 0; j < B->nso; j++) rsw(ropr(B->so[2 * j + 1], 17), B->so[2 * j], 1);
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) {
+		if (rk[B->cv]) li(17, rkv[B->cv]); else movw(17, rreg[B->cv]);
+	}
+	addsp(B->delta * 4);
+	rpmove(R, B);
+	switch (B->term) {
+	case T_BR: rbranch(B->s1); break;
+	case T_CBR:
+		if (B->s1 >= 0) { rfa[rnf] = cp; rfb[rnf++] = B->s1; E(0x34000000 | 17); }	/* cbz w17, taken */
+		else { p = cp; E(0x35000000 | 17); rexitk(R, B->x1); cb_fix(p, cp); }
+		if (B->s2 >= 0) rbranch(B->s2); else rexitk(R, B->x2);
+		break;
+	case T_CALL: rspill(R); li(16, B->x2); str_post(16, RP, 4); movw(W0, 17); b_to(x_disp); break;
+	case T_RET: rspill(R); ldr_pre(W0, RP, -4); b_to(x_disp); break;
+	default: rspill(R); movw(W0, 17); b_to(x_disp);
+	}
+	return 0;
+}
+/* the whole region: entry stubs that load the vars, then the blocks */
+static int rjit(Rg *R, int wr)
+{
+	unsigned *start = cp, **lab;
+	int b, e, j, bad = 0;
+	long need = (long)(R->hi - R->lo) * 96 + (long)R->ne * 32 + 4096;
+	if (need > (code + CODEW) - cp) return -1;
+	if (wr) jitwrite(1);
+	lab = malloc((R->nb + 1) * sizeof *lab);
+	rfa = malloc((2 * R->nb + R->ne + 4) * sizeof *rfa); rfb = malloc((2 * R->nb + R->ne + 4) * sizeof *rfb); rnf = 0;
+	for (e = 0; e < R->ne; e++) { R->stub[e] = cp; rload(R); rbranch(R->entb[e]); }
+	for (b = 0; b < R->nb && !bad; b++) { lab[b] = cp; bad = remit(R, b); }
+	if (!bad) {
+		for (j = 0; j < rnf; j++) {
+			unsigned *at = rfa[j], *to = lab[rfb[j]];
+			if ((*at & 0xfc000000u) == 0x14000000u) *at |= (to - at) & 0x3ffffff; else cb_fix(at, to);
+		}
+		R->nat = start;
+		for (e = 0; e < R->ne; e++) tab[R->ent[e]] = R->stub[e];
+		syncicache(start, cp);
+		if (trace) fprintf(stderr, "o3: %d..%d -> %ld bytes of arm64\n", R->lo, R->hi - 1, (long)((char *)cp - (char *)start));
+	} else {
+		cp = start;
+		if (trace) fprintf(stderr, "o3: %d..%d: a block needs too many registers\n", R->lo, R->hi - 1);
+	}
+	free(lab); free(rfa); free(rfb);
+	if (wr) jitwrite(0);
+	return bad;
+}
+static void rcompile(Rg *R, int eager)
+{
+	if (R->nat || R->nonat) return;
+	if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1;	/* registers, not space: give up on it */
+}
+
+static int inst_hook(int d) { return install(d); }
+static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (int *)sp); }
+
+static void *translate(int p)
+{
+	unsigned *start;
+	int i = p, n = 0, w, k, t;
+	if (cp - code > CODEW - 65536 || nbl >= MAXB) flush();
+	if (o3ent[p]) {	/* an o3 region starts here */
+		Rg *R = rat(p);
+		if (R) { rcompile(R, 0); if (R->nat && tab[p]) return tab[p]; }
+	}
+	start = cp;
+	for (;;) {
+		if (i >= MW) { E(0x12800000); b_to(x_exit); break; }	/* movn w0, #0: halt */
+		if (n++ >= BLKW) { li(W0, i); b_to(x_disp); break; }
+		w = m[i];
+		cov[i] = 1;
+		if (w & 1 && i + 1 < MW && !(m[i + 1] & 1) && (k = m[i + 1] / 2) >= 6 && k <= 8) {
+			t = (w - 1) / 2;	/* literal + call/jmp/jz with a known target */
+			cov[i + 1] = 1;
+			if (k == 8) {
+				unsigned *a;
+				ldr_post(X10, SP, 4);
+				a = cp; E(0x3500000a);	/* cbnz w10, fallthrough */
+				li(W0, t); b_to(x_disp);
+				cb_fix(a, cp);
+				i += 2;
+				continue;
+			}
+			li(X10, i + 2);
+			if (k == 6) str_post(X10, RP, 4); else str_u(X10, RP, 0);
+			li(W0, t); b_to(x_disp);
+			i += 2;
+			break;
+		}
+		if (w & 1) {
+			int L = 0, x, s, nk, ks[8], j;
+			Enh te;
+			while (L < 8 && i + L < MW && (m[i + L] & 1)) L++;
+			x = i + L < MW ? m[i + L] : 1;
+			if (!(x & 1) && (s = x / 2 - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
+				nk = L < enh[s].nin ? L : enh[s].nin;
+				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
+				if (!spec_enh(s, ks, nk, &te)) {
+					for (j = 0; j < L - nk; j++) { li(X9, m[i + j] >> 1); str_pre(X9, SP, -4); }
+					for (j = 0; j <= L; j++) cov[i + j] = 1;
+					emit_enh_e(&te, s);
+					free(te.u); free(te.reg);
+					i += L + 1; n += L;
+					continue;
+				}
+			}
+			li(X9, (w - 1) / 2); str_pre(X9, SP, -4); i++; continue;
+		}
+		k = w / 2;
+		if (k == 0) { E(0x12800000); b_to(x_exit); i++; break; }
+		if (k < 0 || k == 6 || k == 7) {
+			ldr_post(W0, SP, 4);
+			li(X10, i + 1);
+			if (k == 6) str_post(X10, RP, 4); else str_u(X10, RP, 0);
+			if (k < 0) { li(X11, 7 - k); addx_ext(RP, RP, X11, SXTW); }
+			b_to(x_disp);
+			i++;
+			break;
+		}
+		if (k == 8) {	/* jz: taken leaves, not taken keeps going */
+			ldp_post(X9, X10, SP, 8);
+			E(0x3500006a);		/* cbnz w10, +3 */
+			movw(W0, X9);
+			b_to(x_disp);
+			i++;
+			continue;
+		}
+		if (k == 9) { ldr_pre(W0, RP, -4); b_to(x_disp); i++; break; }
+		if (k == 21) {	/* install: call into c, then return to it so it can flush */
+			ldr_u(W0, SP, 0); ldrx_u(X11, ST, 72); blr(X11); str_u(W0, SP, 0);
+			li(W0, i + 1); b_to(x_exit); i++; break;
+		}
+		if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { emit_enh(k - XOP0); i++; continue; }
+		if (k > 20) { E(0x12800000); b_to(x_exit); i++; break; }	/* not wired: halt */
+		op(i, k);
+		i++;
+	}
+	bl[nbl].a = p;
+	bl[nbl++].b = i;
+	tab[p] = start;
+	syncicache(start, cp);
+	if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)((char *)cp - (char *)start));
+	return start;
+}
+
+#else
+/* portable path for hosts without the arm64 jit: a plain interpreter with
+   the same semantics (shift counts wrap at 32, stores outside memory are
+   ignored, loads outside memory read 0, leaving memory halts). */
+#ifdef ZYLLVM
+/* ---- llvm backend ----
+   a tiered jit for hosts the arm64 backend doesn't cover. the interpreter
+   counts how often each block start is reached; a hot block is translated to
+   llvm ir (same block rules as the arm64 jit: it ends at call, jump, ret or
+   halt, and keeps going past a jz), optimized at O2 and compiled with orc.
+   enhancements are inlined as ir. stores into compiled words drop the blocks
+   that cover them, so self-modifying code still works. */
+#include <llvm-c/Core.h>
+#include <llvm-c/Error.h>
+#include <llvm-c/LLJIT.h>
+#include <llvm-c/Orc.h>
+#include <llvm-c/Target.h>
+#include <llvm-c/Transforms/PassBuilder.h>
+static int HOT = 5000;		/* runs before a block gets compiled: ZYHOT */
+static const char *lpasses = "default<O2>";	/* ZYPASSES */
+#define LMAXB 65536
+typedef int (*Blk)(int *mem, int *regs);
+static Blk lt[MW];			/* address -> compiled block */
+static unsigned short hot[MW];
+static unsigned char lcov[MW];		/* 1 if a compiled block covers the word */
+static struct { int a, b; } lbl[LMAXB];
+static int nlbl, llvm_ok, nblocks;
+static LLVMOrcLLJITRef jit;
+static LLVMOrcThreadSafeContextRef tsc;
+static LLVMContextRef ctx;
+static LLVMTypeRef i8t, i32t, i64t, ptrt, i1t;
+static LLVMBuilderRef bd;
+static LLVMValueRef fn_, vm_, vregs, vs, vr;	/* function, memory, regs, s and r slots */
+static LLVMModuleRef mod_;
+
+static unsigned char rcov[MW];	/* 1 on the words of a lifted o3 region */
+static int rdrop(int a);
+static void ldrop_all(void)
+{
+	rdrop_all(); memset(rcov, 0, sizeof rcov);
+	memset(lt, 0, sizeof lt); memset(lcov, 0, sizeof lcov); memset(hot, 0, sizeof hot); nlbl = 0;
+	if (trace) fprintf(stderr, "llvm: dropped every block\n");
+}
+/* a store to a word some block covers: write it, drop those blocks */
+static int lhook(int a, int v)
+{
+	int j, x, hit = 0;
+	if (m[a] == v) return 0;
+	m[a] = v;
+	for (j = 0; j < nlbl; j++)
+		if (lbl[j].a >= 0 && lbl[j].a <= a && a < lbl[j].b) {
+			lt[lbl[j].a] = 0; hot[lbl[j].a] = 0;
+			if (trace) fprintf(stderr, "llvm: dropped %d..%d\n", lbl[j].a, lbl[j].b - 1);
+			lbl[j].a = -1; hit = 1;
+		}
+	if (rcov[a] && rdrop(a)) hit = 1;
+	if (hit) {
+		memset(lcov, 0, sizeof lcov);
+		for (j = 0; j < nlbl; j++) if (lbl[j].a >= 0) for (x = lbl[j].a; x < lbl[j].b; x++) lcov[x] = 1;
+		for (j = 0; j < nrg; j++) if (rgs[j]->lifted) for (x = rgs[j]->lo; x < rgs[j]->hi; x++) lcov[x] = 1;
+	}
+	return hit;
+}
+static int linstall(int d) { return install(d); }
+
+static LLVMValueRef K32(int v) { return LLVMConstInt(i32t, (unsigned)v, 0); }
+static LLVMValueRef cptr(void *p) { return LLVMConstIntToPtr(LLVMConstInt(i64t, (unsigned long long)(size_t)p, 0), ptrt); }
+static LLVMValueRef mslot(LLVMValueRef idx) { return LLVMBuildGEP2(bd, i32t, vm_, &idx, 1, ""); }
+static LLVMValueRef ld(LLVMValueRef idx) { return LLVMBuildLoad2(bd, i32t, mslot(idx), ""); }
+static void st(LLVMValueRef idx, LLVMValueRef v) { LLVMBuildStore(bd, v, mslot(idx)); }
+static LLVMValueRef gets(void) { return LLVMBuildLoad2(bd, i32t, vs, ""); }
+static void sets(LLVMValueRef v) { LLVMBuildStore(bd, v, vs); }
+static LLVMValueRef add(LLVMValueRef a, int k) { return LLVMBuildAdd(bd, a, K32(k), ""); }
+/* a load that reads 0 outside memory */
+static LLVMValueRef safeld(LLVMValueRef a)
+{
+	LLVMValueRef in = LLVMBuildICmp(bd, LLVMIntULT, a, K32(MW), "");
+	LLVMValueRef v = ld(LLVMBuildSelect(bd, in, a, K32(0), ""));
+	return LLVMBuildSelect(bd, in, v, K32(0), "");
+}
+static void push(LLVMValueRef v) { LLVMValueRef s = add(gets(), -1); sets(s); st(s, v); }
+/* leave the block: write s and r back, return the next vm address */
+static void leave(LLVMValueRef next)
+{
+	LLVMValueRef r0 = K32(0), r1 = K32(1);
+	LLVMBuildStore(bd, gets(), LLVMBuildGEP2(bd, i32t, vregs, &r0, 1, ""));
+	LLVMBuildStore(bd, LLVMBuildLoad2(bd, i32t, vr, ""), LLVMBuildGEP2(bd, i32t, vregs, &r1, 1, ""));
+	LLVMBuildRet(bd, next);
+}
+static LLVMBasicBlockRef newbb(void) { return LLVMAppendBasicBlockInContext(ctx, fn_, ""); }
+static LLVMValueRef callp(void *f, LLVMTypeRef ret, LLVMTypeRef *at, int n, LLVMValueRef *args)
+{
+	LLVMTypeRef ft = LLVMFunctionType(ret, at, (unsigned)n, 0);
+	return LLVMBuildCall2(bd, ft, cptr(f), args, (unsigned)n, "");
+}
+/* one micro-op as ir: the same semantics as ev() */
+static LLVMValueRef ir_op(int op, LLVMValueRef a, LLVMValueRef b, LLVMValueRef c)
+{
+	LLVMValueRef z = K32(0), q, d, t, c1, c2, r = 0;
+	LLVMTypeRef i64 = i64t;
+		switch (op) {
+	case U_ADD: r = LLVMBuildAdd(bd, a, b, ""); break;
+	case U_SUB: r = LLVMBuildSub(bd, a, b, ""); break;
+	case U_MUL: r = LLVMBuildMul(bd, a, b, ""); break;
+	case U_SDIV: case U_SREM:	/* /0 and INT_MIN/-1 are defined, like the cpu */
+		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
+		c2 = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, K32((int)0x80000000), ""), LLVMBuildICmp(bd, LLVMIntEQ, b, K32(-1), ""), "");
+		d = LLVMBuildSelect(bd, LLVMBuildOr(bd, c1, c2, ""), K32(1), b, "");
+		if (op == U_SDIV) { q = LLVMBuildSDiv(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, z, q, ""); }
+		else { q = LLVMBuildSRem(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, a, LLVMBuildSelect(bd, c2, z, q, ""), ""); }
+		break;
+	case U_UDIV: case U_UREM:
+		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
+		d = LLVMBuildSelect(bd, c1, K32(1), b, "");
+		if (op == U_UDIV) r = LLVMBuildSelect(bd, c1, z, LLVMBuildUDiv(bd, a, d, ""), "");
+		else r = LLVMBuildSelect(bd, c1, a, LLVMBuildURem(bd, a, d, ""), "");
+		break;
+	case U_AND: r = LLVMBuildAnd(bd, a, b, ""); break;
+	case U_OR: r = LLVMBuildOr(bd, a, b, ""); break;
+	case U_XOR: r = LLVMBuildXor(bd, a, b, ""); break;
+	case U_SHL: r = LLVMBuildShl(bd, a, LLVMBuildAnd(bd, b, K32(31), ""), ""); break;
+	case U_SHR: r = LLVMBuildLShr(bd, a, LLVMBuildAnd(bd, b, K32(31), ""), ""); break;
+	case U_SAR: r = LLVMBuildAShr(bd, a, LLVMBuildAnd(bd, b, K32(31), ""), ""); break;
+	case U_NEG: r = LLVMBuildSub(bd, z, a, ""); break;
+	case U_NOT: r = LLVMBuildNot(bd, a, ""); break;
+	case U_LT: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), i32t, ""); break;
+	case U_LTU: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), i32t, ""); break;
+	case U_EQ: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, b, ""), i32t, ""); break;
+	case U_NE: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntNE, a, b, ""), i32t, ""); break;
+	case U_MIN: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), a, b, ""); break;
+	case U_MAX: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSGT, a, b, ""), a, b, ""); break;
+	case U_MINU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), a, b, ""); break;
+	case U_MAXU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntUGT, a, b, ""), a, b, ""); break;
+	case U_MULHS: case U_MULHU: {
+		LLVMValueRef x = op == U_MULHS ? LLVMBuildSExt(bd, a, i64, "") : LLVMBuildZExt(bd, a, i64, "");
+		LLVMValueRef y = op == U_MULHS ? LLVMBuildSExt(bd, b, i64, "") : LLVMBuildZExt(bd, b, i64, "");
+		r = LLVMBuildTrunc(bd, LLVMBuildLShr(bd, LLVMBuildMul(bd, x, y, ""), LLVMConstInt(i64, 32, 0), ""), i32t, "");
+		break; }
+	case U_CLZ: {
+		unsigned id = LLVMLookupIntrinsicID("llvm.ctlz", 9);
+		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i32t, 1), args[2];
+		args[0] = a; args[1] = LLVMConstInt(i1t, 0, 0);
+		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i32t, 1), f, args, 2, "");
+		break; }
+	case U_ABS: t = LLVMBuildICmp(bd, LLVMIntSLT, a, z, ""); r = LLVMBuildSelect(bd, t, LLVMBuildSub(bd, z, a, ""), a, ""); break;
+	case U_SEL: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntNE, a, z, ""), b, c, ""); break;
+	}
+	return r;
+}
+/* an enhancement as ir */
+static void ir_enh(Enh *e)
+{
+	LLVMValueRef *v = calloc(e->nv + 1, sizeof *v), s = gets();
+	int i, j;
+	for (j = 0; j < e->nin; j++) v[j] = ld(add(s, e->nin - 1 - j));
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		LLVMValueRef a = u->op == U_LI ? 0 : v[u->a], b = uarity[u->op] > 1 || u->op == U_ST ? v[u->b] : 0, c = uarity[u->op] > 2 ? v[u->c] : 0, r = 0;
+		switch (u->op) {
+		case U_LI: r = K32(u->imm); break;
+		case U_LD: r = safeld(a); break;
+		case U_ST: {
+			LLVMBasicBlockRef yes = newbb(), done = newbb();
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, K32(MW), ""), yes, done);
+			LLVMPositionBuilderAtEnd(bd, yes); st(a, b); LLVMBuildBr(bd, done);
+			LLVMPositionBuilderAtEnd(bd, done);
+			break; }
+		default: r = ir_op(u->op, a, b, c);
+		}
+		if (u->op != U_ST) v[u->d] = r;
+	}
+	s = add(s, e->nin - e->nout); sets(s);
+	for (j = 0; j < e->nout; j++) st(add(s, e->nout - 1 - j), v[e->out[j]]);
+	free(v);
+}
+static void llvm_init(void)
+{
+	LLVMErrorRef err;
+	if (getenv("ZYHOT")) HOT = atoi(getenv("ZYHOT"));
+	if (HOT < 1) HOT = 1;
+	if (HOT > 65000) HOT = 65000;
+	if (getenv("ZYPASSES")) lpasses = getenv("ZYPASSES");
+	LLVMInitializeNativeTarget(); LLVMInitializeNativeAsmPrinter();
+	if ((err = LLVMOrcCreateLLJIT(&jit, 0))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return; }
+	tsc = LLVMOrcCreateNewThreadSafeContext();
+	ctx = LLVMOrcThreadSafeContextGetContext(tsc);
+	i1t = LLVMInt1TypeInContext(ctx); i8t = LLVMInt8TypeInContext(ctx); i32t = LLVMInt32TypeInContext(ctx);
+	i64t = LLVMInt64TypeInContext(ctx); ptrt = LLVMPointerTypeInContext(ctx, 0);
+	bd = LLVMCreateBuilderInContext(ctx);
+	llvm_ok = 1;
+}
+/* compile the block at p; 0 if it couldn't be */
+static Blk lcompile(int p)
+{
+	char name[32]; LLVMTypeRef pt[2], ft; LLVMValueRef s, a, b, v, r, args[2]; LLVMTypeRef at[2];
+	LLVMOrcExecutorAddress addr; LLVMErrorRef err; LLVMPassBuilderOptionsRef po;
+	int i = p, n = 0, w, k, t, done = 0;
+	if (!llvm_ok || nlbl >= LMAXB) return 0;
+	sprintf(name, "b%d_%d", p, nblocks++);
+	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
+	pt[0] = ptrt; pt[1] = ptrt;
+	ft = LLVMFunctionType(i32t, pt, 2, 0);
+	fn_ = LLVMAddFunction(mod_, name, ft);
+	vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1);
+	LLVMPositionBuilderAtEnd(bd, newbb());
+	vs = LLVMBuildAlloca(bd, i32t, "s"); vr = LLVMBuildAlloca(bd, i32t, "r");
+	{ LLVMValueRef r0 = K32(0), r1 = K32(1);
+	  sets(LLVMBuildLoad2(bd, i32t, LLVMBuildGEP2(bd, i32t, vregs, &r0, 1, ""), ""));
+	  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i32t, LLVMBuildGEP2(bd, i32t, vregs, &r1, 1, ""), ""), vr); }
+	while (!done) {
+		if (i >= MW) { leave(K32(MW)); break; }
+		if (n++ >= 1024) { leave(K32(i)); break; }
+		w = m[i];
+		lcov[i] = 1;
+		if (w & 1) {
+			/* a literal followed by call, jump or jz: a known target */
+			if (i + 1 < MW && !(m[i + 1] & 1) && (k = m[i + 1] / 2) >= 6 && k <= 8) {
+				t = w >> 1; lcov[i + 1] = 1;
+				if (k == 8) {
+					LLVMBasicBlockRef tk = newbb(), go = newbb();
+					s = gets(); a = ld(s); sets(add(s, 1));
+					LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, K32(0), ""), tk, go);
+					LLVMPositionBuilderAtEnd(bd, tk); leave(K32(t));
+					LLVMPositionBuilderAtEnd(bd, go);
+					i += 2; continue;
+				}
+				if (k == 6) { r = LLVMBuildLoad2(bd, i32t, vr, ""); st(r, K32(i + 2)); LLVMBuildStore(bd, add(r, 1), vr); }
+				leave(K32(t)); i += 2; break;
+			}
+			push(K32(w >> 1)); i++; continue;
+		}
+		k = w / 2;
+		s = gets();
+		switch (k) {
+		case 0: leave(K32(MW)); done = 1; break;
+		case 1: st(s, safeld(ld(s))); break;
+		case 2: {
+			LLVMBasicBlockRef in = newbb(), cv = newbb(), plain = newbb(), out = newbb(), quit = newbb();
+			a = ld(s); b = ld(add(s, 1)); sets(add(s, 2));
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, K32(MW), ""), in, out);
+			LLVMPositionBuilderAtEnd(bd, in);
+			v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
+			LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
+			LLVMPositionBuilderAtEnd(bd, cv);
+			at[0] = i32t; at[1] = i32t; args[0] = a; args[1] = b;
+			v = callp((void *)lhook, i32t, at, 2, args);
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, K32(0), ""), quit, out);
+			LLVMPositionBuilderAtEnd(bd, quit); leave(K32(i + 1));	/* a block died: back to the loop */
+			LLVMPositionBuilderAtEnd(bd, out);
+			break; }
+		case 3: a = ld(s); st(s, safeld(LLVMBuildAdd(bd, add(s, 1), a, ""))); break;
+		case 4: sets(add(s, 1)); break;
+		case 5: a = ld(s); b = ld(add(s, 1)); st(s, b); st(add(s, 1), a); break;
+		case 6: case 7:
+			a = ld(s); sets(add(s, 1));
+			if (k == 6) { r = LLVMBuildLoad2(bd, i32t, vr, ""); st(r, K32(i + 1)); LLVMBuildStore(bd, add(r, 1), vr); }
+			leave(a); done = 1; break;
+		case 8: {
+			LLVMBasicBlockRef tk = newbb(), go = newbb();
+			a = ld(s); b = ld(add(s, 1)); sets(add(s, 2));
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, b, K32(0), ""), tk, go);
+			LLVMPositionBuilderAtEnd(bd, tk); leave(a);
+			LLVMPositionBuilderAtEnd(bd, go);
+			break; }
+		case 9:
+			r = add(LLVMBuildLoad2(bd, i32t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
+			leave(ld(r)); done = 1; break;
+		case 10: v = callp((void *)Q, i8t, 0, 0, 0); push(LLVMBuildSExt(bd, v, i32t, "")); break;
+		case 11:
+			if (sizeof J == 1) push(LLVMBuildZExt(bd, LLVMBuildLoad2(bd, i8t, cptr(&J), ""), i32t, ""));
+			else push(LLVMBuildLoad2(bd, i32t, cptr(&J), ""));
+			break;
+		case 12: at[0] = i8t; args[0] = LLVMBuildTrunc(bd, ld(s), i8t, ""); callp((void *)G, LLVMVoidTypeInContext(ctx), at, 1, args); sets(add(s, 1)); break;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
+			a = ld(s); b = ld(add(s, 1)); s = add(s, 1); sets(s);
+			switch (k) {
+			case 13: v = LLVMBuildAdd(bd, b, a, ""); break;
+			case 14: v = LLVMBuildSub(bd, b, a, ""); break;
+			case 15: v = LLVMBuildMul(bd, b, a, ""); break;
+			case 16: v = LLVMBuildAnd(bd, b, a, ""); break;
+			case 17: v = LLVMBuildOr(bd, b, a, ""); break;
+			case 18: v = LLVMBuildXor(bd, b, a, ""); break;
+			case 19: v = LLVMBuildAShr(bd, b, LLVMBuildAnd(bd, a, K32(31), ""), ""); break;
+			default: v = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, b, a, ""), i32t, "");
+			}
+			st(s, v); break;
+		case 21:	/* install: then back to the loop, which drops every block */
+			at[0] = i32t; args[0] = ld(s);
+			st(s, callp((void *)linstall, i32t, at, 1, args));
+			leave(K32(i + 1)); done = 1; break;
+		default:
+			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { ir_enh(&enh[k - XOP0]); break; }
+			leave(K32(MW)); done = 1;	/* not wired: halt */
+		}
+		i++;
+	}
+	lbl[nlbl].a = p; lbl[nlbl++].b = i;
+	po = LLVMCreatePassBuilderOptions();
+	if ((err = LLVMRunPasses(mod_, lpasses, 0, po))) { LLVMConsumeError(err); }
+	LLVMDisposePassBuilderOptions(po);
+	if ((err = LLVMOrcLLJITAddLLVMIRModule(jit, LLVMOrcLLJITGetMainJITDylib(jit), LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
+		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0;
+	}
+	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
+	if (trace) fprintf(stderr, "llvm: %d..%d compiled\n", p, i - 1);
+	return (Blk)(size_t)addr;
+}
+#endif
+
+/* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */
+#ifndef ZYLLVM
+static unsigned char rcov[MW];	/* 1 on the words of a lifted region */
+#endif
+static int rvals[MAXUOP * 4 + 16];
+static int rdrop(int a)		/* a store landed on region code: drop those regions */
+{
+	int j, x, hit = 0;
+	for (j = 0; j < nrg; j++)
+		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
+			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d: %d)\n", rgs[j]->lo, rgs[j]->hi - 1, a, m[a]);
+			rclear(rgs[j]); hit = 1;
+		}
+	if (hit) {
+		memset(rcov, 0, sizeof rcov);
+		for (j = 0; j < nrg; j++) if (rgs[j]->lifted) for (x = rgs[j]->lo; x < rgs[j]->hi; x++) rcov[x] = 1;
+	}
+	return hit;
+}
+#ifdef ZYLLVM
+static int lhook(int a, int v);
+#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (lcov[a]) lhook(a, b); else m[a] = b; } } while (0)
+#else
+#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0)
+#endif
+/* the region interpreter: ssa values in an array, vars in locals */
+static int rexec(Rg *R, int e, int *ps, int *pr)
+{
+	int var[RMAXV], nv2[RMAXV], *v = rvals, s = *ps, r = *pr, b = R->entb[e], fp, j, i, t, o;
+	var[0] = fp = mread(ZFP);
+	for (j = 1; j < R->nv; j++) var[j] = mread(fp + R->slot[j]);
+	for (;;) {
+		RB *B = &R->b[b];
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			switch (u->op) {
+			case U_LI: v[u->d] = u->imm; break;
+			case U_VGET: v[u->d] = var[u->imm]; break;
+			case U_SLD: v[u->d] = mread(s + u->imm); break;
+			case U_LD: v[u->d] = mread(v[u->a]); break;
+			case U_ST: RSTORE(v[u->a], v[u->b]); break;
+			default: v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
+			}
+		}
+		for (j = 0; j < B->nso; j++) { o = s + B->so[2 * j]; if ((unsigned)o < MW) m[o] = v[B->so[2 * j + 1]]; }
+		s += B->delta;
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) nv2[j] = v[B->vs[j]];
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) var[j] = nv2[j];
+		if (B->term == T_BR) { b = B->s1; continue; }
+		if (B->term == T_CBR) {
+			if (!v[B->cv]) { if (B->s1 >= 0) { b = B->s1; continue; } t = B->x1; }
+			else { if (B->s2 >= 0) { b = B->s2; continue; } t = B->x2; }
+		} else if (B->term == T_JMP) t = v[B->cv];
+		else if (B->term == T_CALL) { m[r++] = B->x2; t = v[B->cv]; }
+		else t = m[--r];
+		break;
+	}
+	for (j = 1; j < R->nv; j++) if ((unsigned)(fp + R->slot[j]) < MW) m[fp + R->slot[j]] = var[j];
+	*ps = s; *pr = r;
+	return t;
+}
+#ifndef ZYLLVM
+static void rforget(Rg *R) { (void)R; }
+static void rmark(Rg *R) { int x; for (x = R->lo; x < R->hi; x++) rcov[x] = 1; }
+static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
+#endif
+#ifdef ZYLLVM
+
+/* ---- o3 regions in llvm: one function per region, vars in allocas (so
+   mem2reg makes them ssa across blocks), each block's micro-ops as ir, then
+   the O3 pipeline. f(mem, regs, entry) returns the next vm address. */
+typedef int (*RFn)(int *mem, int *regs, int ent);
+static int nregions;
+static LLVMValueRef rvv[RMAXV];
+static void rforget(Rg *R) { R->nat = 0; }
+static void rmark(Rg *R) { int x; for (x = R->lo; x < R->hi; x++) rcov[x] = lcov[x] = 1; }
+static void rspill_ir(Rg *R)
+{
+	LLVMValueRef fp = LLVMBuildLoad2(bd, i32t, rvv[0], "");
+	int j;
+	for (j = 1; j < R->nv; j++) st(add(fp, R->slot[j]), LLVMBuildLoad2(bd, i32t, rvv[j], ""));
+}
+static LLVMBasicBlockRef rexit_bb(Rg *R, int t)
+{
+	LLVMBasicBlockRef here = LLVMGetInsertBlock(bd), x = newbb();
+	LLVMPositionBuilderAtEnd(bd, x);
+	rspill_ir(R); leave(K32(t));
+	LLVMPositionBuilderAtEnd(bd, here);
+	return x;
+}
+/* m[a] = v, through lhook if some compiled code covers the word */
+static void rstore_ir(LLVMValueRef a, LLVMValueRef b)
+{
+	LLVMBasicBlockRef in = newbb(), cv = newbb(), plain = newbb(), out = newbb();
+	LLVMValueRef v, args[2]; LLVMTypeRef at[2];
+	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, K32(MW), ""), in, out);
+	LLVMPositionBuilderAtEnd(bd, in);
+	v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
+	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
+	LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, cv);
+	at[0] = i32t; at[1] = i32t; args[0] = a; args[1] = b;
+	callp((void *)lhook, i32t, at, 2, args);
+	LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, out);
+}
+static void *lregion(Rg *R)
+{
+	char name[40]; LLVMTypeRef pt[3], ft; LLVMValueRef ent, sw, *val, s0, fp, r;
+	LLVMBasicBlockRef *bb, eb, dflt;
+	LLVMOrcExecutorAddress addr; LLVMErrorRef err; LLVMPassBuilderOptionsRef po;
+	int b, i, j, e;
+	sprintf(name, "o3_%d_%d", R->lo, nregions++);
+	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
+	pt[0] = ptrt; pt[1] = ptrt; pt[2] = i32t;
+	ft = LLVMFunctionType(i32t, pt, 3, 0);
+	fn_ = LLVMAddFunction(mod_, name, ft);
+	vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1); ent = LLVMGetParam(fn_, 2);
+	LLVMPositionBuilderAtEnd(bd, newbb());
+	vs = LLVMBuildAlloca(bd, i32t, "s"); vr = LLVMBuildAlloca(bd, i32t, "r");
+	for (j = 0; j < R->nv; j++) rvv[j] = LLVMBuildAlloca(bd, i32t, "");
+	{ LLVMValueRef r0 = K32(0), r1 = K32(1);
+	  sets(LLVMBuildLoad2(bd, i32t, LLVMBuildGEP2(bd, i32t, vregs, &r0, 1, ""), ""));
+	  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i32t, LLVMBuildGEP2(bd, i32t, vregs, &r1, 1, ""), ""), vr); }
+	bb = malloc(R->nb * sizeof *bb);
+	for (b = 0; b < R->nb; b++) bb[b] = newbb();
+	dflt = newbb();
+	sw = LLVMBuildSwitch(bd, ent, dflt, (unsigned)R->ne);
+	for (e = 0; e < R->ne; e++) {
+		eb = newbb();
+		LLVMAddCase(sw, K32(e), eb);
+		LLVMPositionBuilderAtEnd(bd, eb);
+		fp = ld(K32(ZFP));
+		LLVMBuildStore(bd, fp, rvv[0]);
+		for (j = 1; j < R->nv; j++) LLVMBuildStore(bd, safeld(add(fp, R->slot[j])), rvv[j]);
+		LLVMBuildBr(bd, bb[R->entb[e]]);
+	}
+	LLVMPositionBuilderAtEnd(bd, dflt); leave(K32(-1));
+	val = calloc(R->maxnv + 1, sizeof *val);
+	for (b = 0; b < R->nb; b++) {
+		RB *B = &R->b[b];
+		LLVMPositionBuilderAtEnd(bd, bb[b]);
+		s0 = gets();
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			LLVMValueRef a = nargs(u->op) > 0 ? val[u->a] : 0, x = nargs(u->op) > 1 ? val[u->b] : 0, c = nargs(u->op) > 2 ? val[u->c] : 0;
+			switch (u->op) {
+			case U_LI: val[u->d] = K32(u->imm); break;
+			case U_VGET: val[u->d] = LLVMBuildLoad2(bd, i32t, rvv[u->imm], ""); break;
+			case U_SLD: val[u->d] = ld(add(s0, u->imm)); break;
+			case U_LD: val[u->d] = safeld(a); break;
+			case U_ST: rstore_ir(a, x); break;
+			default: val[u->d] = ir_op(u->op, a, x, c);
+			}
+		}
+		for (j = 0; j < B->nso; j++) st(add(s0, B->so[2 * j]), val[B->so[2 * j + 1]]);
+		sets(add(s0, B->delta));
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) LLVMBuildStore(bd, val[B->vs[j]], rvv[j]);
+		switch (B->term) {
+		case T_BR: LLVMBuildBr(bd, bb[B->s1]); break;
+		case T_CBR:
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, val[B->cv], K32(0), ""),
+				B->s1 >= 0 ? bb[B->s1] : rexit_bb(R, B->x1), B->s2 >= 0 ? bb[B->s2] : rexit_bb(R, B->x2));
+			break;
+		case T_CALL:
+			r = LLVMBuildLoad2(bd, i32t, vr, ""); st(r, K32(B->x2)); LLVMBuildStore(bd, add(r, 1), vr);
+			rspill_ir(R); leave(val[B->cv]);
+			break;
+		case T_RET:
+			r = add(LLVMBuildLoad2(bd, i32t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
+			rspill_ir(R); leave(ld(r));
+			break;
+		default: rspill_ir(R); leave(val[B->cv]);
+		}
+	}
+	free(val); free(bb);
+	po = LLVMCreatePassBuilderOptions();
+	if ((err = LLVMRunPasses(mod_, "default<O3>", 0, po))) LLVMConsumeError(err);
+	LLVMDisposePassBuilderOptions(po);
+	if ((err = LLVMOrcLLJITAddLLVMIRModule(jit, LLVMOrcLLJITGetMainJITDylib(jit), LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
+		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0;
+	}
+	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
+	if (trace) fprintf(stderr, "o3: %d..%d compiled by llvm\n", R->lo, R->hi - 1);
+	return (void *)(size_t)addr;
+}
+static void rcompile(Rg *R, int eager)
+{
+	(void)eager;
+	if (R->nat || R->nonat || !llvm_ok) return;
+	if (!(R->nat = lregion(R))) R->nonat = 1;
+}
+#endif
+#define LD(x) ((unsigned)(x) < MW ? m[x] : 0)
+static int run(int p)
+{
+	int s = DS0, r = RS0, w, a, b, xfer = 1;
+	unsigned long long steps = 0;
+#ifdef ZYLLVM
+	int regs[2];
+	if (!getenv("ZYNOLLVM")) llvm_init();
+#endif
+	while ((unsigned)p < MW) {
+		if (xfer && (s < DSLIM || r > RSLIM)) { p = s < DSLIM ? MW + 1 : MW + 2; break; }
+		if (xfer && o3ent[p]) {	/* an o3 region */
+			Rg *R = rat(p);
+			if (R) {
+#ifdef ZYLLVM
+				rcompile(R, 0);
+				if (R->nat) { regs[0] = s; regs[1] = r; p = ((RFn)R->nat)(m, regs, o3ei[p]); s = regs[0]; r = regs[1]; }
+				else
+#endif
+				p = rexec(R, o3ei[p], &s, &r);
+				continue;
+			}
+		}
+#ifdef ZYLLVM
+		if (xfer && llvm_ok) {	/* at a block start: run it compiled, or count it */
+			xfer = 0;
+			if (!lt[p] && hot[p] < 65535 && ++hot[p] == HOT && !(lt[p] = lcompile(p))) hot[p] = 65535;
+			if (lt[p]) {
+				regs[0] = s; regs[1] = r;
+				p = lt[p](m, regs);
+				s = regs[0]; r = regs[1];
+				if (need_flush) { need_flush = 0; ldrop_all(); }
+				xfer = 1;
+				continue;
+			}
+		}
+#endif
+		w = m[p++];
+		steps++;
+		if (w & 1) { m[--s] = w >> 1; continue; }
+		a = m[s]; b = m[s + 1];
+		switch (w / 2) {
+		case 1: m[s] = LD(a); break;
+		case 2:
+			if ((unsigned)a < MW) {
+#ifdef ZYLLVM
+				if (lcov[a]) lhook(a, b); else
+#else
+				if (rcov[a]) { m[a] = b; rdrop(a); } else
+#endif
+				m[a] = b;
+			}
+			s += 2; break;
+		case 3: m[s] = LD(s + 1 + a); break;
+		case 4: s++; break;
+		case 5: m[s] = b; m[s + 1] = a; break;
+		case 6: m[r++] = p; p = a; s++; xfer = 1; break;
+		case 7: p = a; s++; xfer = 1; break;
+		case 8: if (!b) { p = a; xfer = 1; } s += 2; break;
+		case 9: p = m[--r]; xfer = 1; break;
+		case 10: m[--s] = (signed char)Q(); break;
+		case 11: m[--s] = J; break;
+		case 12: G((char)a); s++; break;
+		case 13: m[++s] = b + a; break;
+		case 14: m[++s] = (int)((unsigned)b - (unsigned)a); break;
+		case 15: m[++s] = (int)((unsigned)b * (unsigned)a); break;
+		case 16: m[++s] = b & a; break;
+		case 17: m[++s] = b | a; break;
+		case 18: m[++s] = b ^ a; break;
+		case 19: m[++s] = b >> (a & 31); break;
+		case 20: m[++s] = b < a; break;
+		case 21:
+			m[s] = install(a);
+#ifdef ZYLLVM
+			if (need_flush) { need_flush = 0; ldrop_all(); }
+#else
+			if (need_flush) { need_flush = 0; rdrop_all(); memset(rcov, 0, sizeof rcov); }
+#endif
+			break;
+		default:
+			if (w / 2 >= XOP0 && w / 2 < XOP0 + NENH && enh[w / 2 - XOP0].live) s = (int)(xexec(&enh[w / 2 - XOP0], m + s) - m);
+			else p = MW;	/* halt: unwired opcodes and the reserved negative ones */
+		}
+	}
+	if (trace) fprintf(stderr, "interp: %llu steps\n", steps);
+	return stopped(p);
+}
+#endif
+
+int main(void)
+{
+	trace = getenv("ZYTRACE") != 0;
+	memcpy(m + GORG, genome, sizeof genome);
+#ifdef ZYJIT
+	{
+	int p = GORG;
+	union { unsigned *u; int (*f)(long *, void *); } enter;
+#ifdef __APPLE__
+	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
+#else
+	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#endif
+	if (code == MAP_FAILED) { perror("mmap"); return 1; }
+	jitwrite(1);
+	build_stubs();
+	jitwrite(0);
+	syncicache(code, cp);
+	S[0] = (long)m; S[1] = DS0; S[2] = RS0; S[3] = (long)tab; S[4] = (long)cov;
+	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
+	S[9] = (long)inst_hook; S[10] = (long)xrun;
+	S[11] = (long)(m + DSLIM); S[12] = (long)(m + RSLIM);
+	enter.u = x_enter;
+	while ((unsigned)p < MW) {
+		void *c = tab[p];
+		if (!c) { jitwrite(1); c = translate(p); jitwrite(0); }
+		p = enter.f(S, c);
+		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
+	}
+	return stopped(p);
+	}
+#else
+	return run(GORG);
+#endif
+	return 0;
+}