a

1 files changed, 5889 insertions, 0 deletions

diff --git a/src/core/core.c b/src/core/core.c
new file mode 100644
index 0000000..2f6032a
--- /dev/null
+++ b/src/core/core.c
@@ -0,0 +1,5889 @@
+/* zygote core: puts the genome (the zc compiler and its runtime, both vm code)
+   into memory and runs it. on arm64, every block of vm code is translated to
+   arm64 the first time it is reached, straight from live memory. stores that
+   land on translated words throw those translations away, so code that
+   rewrites itself, including the resident compiler, just gets retranslated
+   on the next visit. on other hosts (or with -DZYINTERP) a plain interpreter
+   with the same semantics runs instead. ZYTRACE=1 logs translations on stderr. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#ifdef _WIN32
+#include <io.h>
+#include <process.h>
+#ifndef O_BINARY
+#define O_BINARY _O_BINARY
+#endif
+#if defined(_MSC_VER)
+#define open _open
+#define close _close
+#define read _read
+#define write _write
+#endif
+#else
+#include <unistd.h>
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#endif
+#include <time.h>
+#if (defined(__linux__) || defined(__APPLE__)) && !defined(__wasm__) && !defined(ZY_NOFFI)
+#define ZYFFI 1
+#include <dlfcn.h>
+#endif
+#if defined(__aarch64__) && !defined(ZYINTERP)
+#define ZYJIT 1
+#include <sys/mman.h>
+#ifdef __APPLE__
+#include <pthread.h>
+#include <libkern/OSCacheControl.h>
+#endif
+#endif
+#if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK)
+#define ZYBIGSTACK 1
+#include <sys/mman.h>
+#include <ucontext.h>
+#endif
+#ifdef USE_LSP_GENOME
+#include "lsp_genome.h"
+#elif defined(GENOME_H)
+#include GENOME_H
+#else
+#include "genome.h"
+#endif
+#include "memmap.h"
+#include "fops.h"
+#include "bops.h"
+
+#ifndef JTYPE
+#define JTYPE char
+#endif
+char Q(void);
+void G(char);
+extern JTYPE J;
+
+typedef int64_t Cell;	/* one vm word. memory stays word-addressed. */
+typedef uint64_t UCell;
+enum { DEFAULT_MEMORY_LIMIT = MW };
+static int memory_limit = DEFAULT_MEMORY_LIMIT;
+static int memory_words;
+static int memory_growth_limit;
+/* bounds baked into native blocks use the reservation, whose base never moves. */
+#undef MW
+#define MW memory_limit
+/* MW, the stacks and SYSGATE come from memmap.h. the default is 1<<20. */
+/* stop values stay outside every executable address reservation. */
+#define PC_HALT 0x7ffffff0
+#define PC_DATA_OVERFLOW (PC_HALT + 1)
+#define PC_RETURN_OVERFLOW (PC_HALT + 2)
+#define PC_FRAME_OVERFLOW (PC_HALT + 3)
+static Cell code_words;
+static int code_reserve(Cell end);
+static int code_range_live(Cell start, Cell count);
+/* why the cpu stopped, when it wasn't a halt */
+static int stopped(int p)
+{
+	if (p == PC_DATA_OVERFLOW) { fprintf(stderr, "zy: data stack overflow\n"); return 2; }
+	if (p == PC_RETURN_OVERFLOW) { fprintf(stderr, "zy: return stack overflow\n"); return 2; }
+	if (p == PC_FRAME_OVERFLOW) { fprintf(stderr, "zy: frame stack overflow\n"); return 2; }
+	return 0;
+}
+#define CODEW (4 << 20)	/* 16 MB of code, in instruction words */
+#define MAXB 65536
+#define BLKW 1024	/* longest block, in vm words */
+
+static Cell *m;
+static Cell frame_floor_word = ZFP + 6, frame_top_word = ZFP + 7, frame_guard_word = ZFP + 8;
+static Cell frame_bound(void) { return m[frame_floor_word] + m[frame_guard_word]; }
+
+static int b_ok(Cell addr, int n) { return bop_ok(addr, n, MW); }
+static Cell b_load(Cell addr, int n)
+{
+	if (!b_ok(addr, n)) return 0;
+	return bop_load(m, addr, n);
+}
+static void b_commit(Cell w, Cell v);
+static void b_store(Cell addr, Cell val, int n)
+{
+	Cell out[2], w0;
+	int nw, i;
+	if (!b_ok(addr, n)) return;
+	nw = bop_merge(m, addr, val, n, out, &w0);
+	for (i = 0; i < nw; i++) b_commit(w0 + i, out[i]);
+}
+static Cell bld8(Cell a) { return b_load(a, 1); }
+static Cell bld16(Cell a) { return b_load(a, 2); }
+static Cell bld32(Cell a) { return b_load(a, 4); }
+static void bst8(Cell a, Cell v) { b_store(a, v, 1); }
+static void bst16(Cell a, Cell v) { b_store(a, v, 2); }
+static void bst32(Cell a, Cell v) { b_store(a, v, 4); }
+
+static void fop_print(Cell bits)
+{
+	char buf[400];
+	int n = fop_format((uint64_t)bits, buf), i;
+	for (i = 0; i < n; i++) G(buf[i]);
+}
+/* opcode 618: (bits prec mode --). mode 0 shortest, 1 fixed, 2 scientific. */
+static void fop_print_mode(Cell bits, Cell prec, Cell mode)
+{
+	char buf[400];
+	int n = fop_format_mode((uint64_t)bits, mode, prec, buf), i;
+	for (i = 0; i < n; i++) G(buf[i]);
+}
+static Cell fop_parse_cell(Cell addr)
+{
+	if ((UCell)addr >= (UCell)MW) return 0;
+	return (Cell)fop_parse_words(m + addr, (UCell)MW - (UCell)addr);
+}
+static int trace;
+
+/* the command line and the environment, kept for ops 21 to 24. a program read
+   from stdin has no file name, so argv[0] is <eval>, as in zy(1). the rest of
+   the host's argv follows it. the environment is a bare char **, so its count
+   is counted once at startup. */
+#ifndef _WIN32
+extern char **environ;
+#endif
+static int zy_argc;
+static char **zy_argv;
+static int zy_envc;
+static char **zy_envp;
+
+static void zy_args(int argc, char **argv)
+{
+	char **e;
+	int n = 0;
+#ifdef _WIN32
+	e = _environ;
+#else
+	e = environ;
+#endif
+	while (e && e[n]) n++;
+	zy_envp = e; zy_envc = n;
+	zy_argv = argv;
+	zy_argc = argc > 0 ? argc : 1;
+}
+
+/* syscall gate: a store of the op to SYSGATE runs hostcall. args sit in the
+   next three words, and the result replaces the op. see rt.zc. */
+static int zcstr(int addr, char *dst, int max)
+{
+	int i = 0, c;
+	if (max < 1) return -1;
+	if ((unsigned)addr >= MW) { dst[0] = 0; return -1; }
+	while (i < max - 1 && (unsigned)(addr + i) < MW) {
+		c = (int)m[addr + i];
+		if (!c) break;
+		dst[i++] = (char)c;
+	}
+	dst[i] = 0;
+	return i;
+}
+static int inmem(Cell a) { return (UCell)a < (UCell)MW; }
+static int stack_memory_grow(UCell words)
+{
+	if (words > (UCell)memory_growth_limit) return 0;
+	if (words > (UCell)memory_words) memory_words = (int)words;
+	return 1;
+}
+#include "stack.h"
+
+/* copy entry a of a char ** into vm words, one character per word, terminated
+   like SYS_GETENV. a zero buffer asks for the length only, which is how the
+   caller sizes the string before fetching it. lead, when given, replaces entry
+   zero: a program read from stdin has no file name, so argv[0] is <eval>. */
+static Cell vecstr(char **v, int count, Cell a, Cell b, Cell c, const char *lead)
+{
+	char *e;
+	int i = 0;
+	if (a < 0 || a >= (Cell)count) return -1;
+	e = (a == 0 && lead) ? (char *)lead : v[a];
+	if (!e) return -1;
+	if (!b) { while (e[i]) i++; return i; }
+	if (c < 1 || !inmem(b)) return -1;
+	while (e[i] && i < c - 1 && inmem(b + i)) { m[b + i] = (unsigned char)e[i]; i++; }
+	if (inmem(b + i)) m[b + i] = 0;
+	return i;
+}
+static Cell host_rand(void)
+{
+	static uint64_t s;
+	if (!s) s = (uint64_t)time(0) ^ 0x9E3779B97F4A7C15ull;
+	if (!s) s = 0x9E3779B97F4A7C15ull;
+	s ^= s << 13; s ^= s >> 7; s ^= s << 17;
+	if ((Cell)s == -1) s ^= 0x100;
+	return (Cell)s;
+}
+/* c abi: ops 14..18 of the gate. see the ffi section further down. cb_s and
+   cb_r are the vm stack pointers as of the gate store, so that a callback from
+   native code can run vm code on top of them. */
+static Cell cb_s, cb_r;
+static Cell ffi_call(Cell blk);
+static Cell ffi_find(Cell name);
+static Cell ffi_cbmake(Cell entry, Cell sig);
+static Cell ffi_load(Cell name);
+static Cell hostcall(Cell op, Cell a, Cell b, Cell c)
+{
+	char path[1024], tmp[512];
+	int n, i, got;
+#ifdef ZY_SANDBOX
+	/* a build for checking code the user has not run yet (the language server): compile-time
+	   code may read what it includes, but it may not write files, exit, load or call C code,
+	   or make raw system calls */
+	if (op == 4 || op == 7 || op == 12 || (op >= 14 && op <= 17) || op == 40) return -1;
+	if ((op == 1 || op == 10) && b != 0) return -1;
+#endif
+	switch (op) {
+	case 1:
+		if (zcstr(a, path, 1024) < 0) return -1;
+		if (b == 1) return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
+		return open(path, O_RDONLY | O_BINARY);
+	case 2:
+		return close((int)a);
+	case 3:
+		if (c < 0 || !inmem(b)) return -1;
+		n = 0;
+		while (n < c) {
+			i = c - n;
+			if (i > 512) i = 512;
+			if (!inmem(b + n)) break;
+			if (!inmem(b + n + i - 1)) i = (int)(MW - (b + n));
+			if (i <= 0) break;
+			got = (int)read((int)a, tmp, (size_t)i);
+			if (got <= 0) break;
+			for (i = 0; i < got; i++) m[b + n + i] = (unsigned char)tmp[i];
+			n += got;
+		}
+		return n;
+	case 4:
+		if (c < 0 || !inmem(b)) return -1;
+		n = 0;
+		while (n < c) {
+			i = c - n;
+			if (i > 512) i = 512;
+			if (!inmem(b + n)) break;
+			if (!inmem(b + n + i - 1)) i = (int)(MW - (b + n));
+			if (i <= 0) break;
+			for (got = 0; got < i; got++) tmp[got] = (char)m[b + n + got];
+			got = (int)write((int)a, tmp, (size_t)i);
+			if (got <= 0) break;
+			n += got;
+		}
+		return n;
+	case 5: {
+		char *e;
+		if (zcstr(a, path, 1024) < 0) return -1;
+		e = getenv(path);
+		if (!e) return -1;
+		if (c < 1 || !inmem(b)) return -1;
+		i = 0;
+		while (e[i] && i < c - 1 && inmem(b + i)) { m[b + i] = (unsigned char)e[i]; i++; }
+		if (inmem(b + i)) m[b + i] = 0;
+		return i;
+	}
+	case 6:
+		return (Cell)time(0);
+	case 7:
+		exit((int)a);
+	case 8:
+		return host_rand();
+	case 9:
+		if (a == 0) return memory_words;
+		if (a == 1) return HEAP_LO;
+		if (a == 2) return HEAP_HI;
+		if (a == 3) return (Cell)memory_words * 8;
+		if (a == 4) {
+			if (b < 0 || b > memory_growth_limit) return -1;
+			if (b > memory_words) memory_words = (int)b;
+			return memory_words;
+		}
+		if (a == 5) {
+			if (b < memory_words || b > MW) return -1;
+			memory_growth_limit = (int)b; return b;
+		}
+		if (a == 6) {
+			if (!stack_register(b)) return -1;
+			frame_floor_word = m[b]; frame_top_word = m[b + 1]; frame_guard_word = m[b + 2];
+			return 0;
+		}
+		if (a == 7 || a == 8) return stack_expand(a == 8, b, &cb_s, &cb_r);
+		if (a == 9) return code_reserve(b);
+		return -1;
+	case 10:
+		if (bop_cstr(m, a, path, 1024, MW) < 0) return -1;
+		if (b == 1) return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
+		return open(path, O_RDONLY | O_BINARY);
+	case 11:
+		if (c < 0 || (c > 0 && !bop_ok(b, 1, MW))) return -1;
+		n = 0;
+		while (n < c) {
+			i = (int)(c - n);
+			if (i > 512) i = 512;
+			i = bop_span(b + n, i, MW);
+			if (i <= 0) break;
+			got = (int)read((int)a, tmp, (size_t)i);
+			if (got <= 0) break;
+			for (i = 0; i < got; i++) bop_put1(m, b + n + i, (unsigned char)tmp[i]);
+			n += got;
+		}
+		return n;
+	case 12:
+		if (c < 0 || (c > 0 && !bop_ok(b, 1, MW))) return -1;
+		n = 0;
+		while (n < c) {
+			i = (int)(c - n);
+			if (i > 512) i = 512;
+			i = bop_span(b + n, i, MW);
+			if (i <= 0) break;
+			for (got = 0; got < i; got++) tmp[got] = (char)bop_load(m, b + n + got, 1);
+			got = (int)write((int)a, tmp, (size_t)i);
+			if (got <= 0) break;
+			n += got;
+		}
+		return n;
+	case 13: {
+		char *e;
+		if (bop_cstr(m, a, path, 1024, MW) < 0) return -1;
+		e = getenv(path);
+		if (!e) return -1;
+		if (c < 1 || !bop_ok(b, 1, MW)) return -1;
+		i = 0;
+		while (e[i] && i < c - 1 && bop_ok(b + i, 1, MW)) { bop_put1(m, b + i, (unsigned char)e[i]); i++; }
+		if (bop_ok(b + i, 1, MW)) bop_put1(m, b + i, 0);
+		return i;
+	}
+	case 14: return ffi_call(a);
+	case 15: return ffi_find(a);
+	case 16: return ffi_cbmake(a, b);
+	case 17: return ffi_load(a);
+	case 18: {	/* fill words: a = dst, b = value, c = count. above the program area only
+		   (no translation covers it), and -1 means "do it yourself" */
+		Cell i;
+		if (a < PW || code_range_live(a, c) || c <= 0 || c > MW - a) return -1;
+		for (i = 0; i < c; i++) m[a + i] = b;
+		return c; }
+	case 19:	/* copy words: a = dst, b = src, c = count. a forward copy that overlaps
+		   itself (dst inside the source) is left to the caller's loop */
+		if (a < PW || code_range_live(a, c) || b < 0 || c <= 0 || c > MW - a || c > MW - b) return -1;
+		if (a > b && a < b + c) return -1;
+		memmove(m + a, m + b, (size_t)c * sizeof(Cell));
+		return c;
+	case 20: {	/* scan for heap pointers: a = first word, b = count, c = hbump. the index of
+		   the first word that could be the payload address of an allocated block (a
+		   heap address whose check word at w[-3] is w ^ MAGIC), or b if there is none.
+		   the allocator's header is 5 words and its magic 0x2B7E1516 (src/runtime/rt.zc); the
+		   caller still checks the flags, this only skips the words that are plainly
+		   not pointers. -1 if the range is not in memory */
+		Cell i, w, lim = c > MW ? MW : c;
+		if (a < 0 || b <= 0 || b > MW - a) return -1;
+		for (i = 0; i < b; i++) {
+			w = m[a + i];
+			if (w >= HEAP_LO + 5 && w < lim && m[w - 3] == (w ^ 0x2B7E1516)) return i;
+		}
+		return b; }
+	case 21:	/* argc: the host's argument count, or 1 for <eval> alone */
+		return (Cell)zy_argc;
+	case 22:	/* argv: a = index, b = buffer, c = capacity. a zero buffer asks for the
+		   length only, which is how the caller sizes the string before it fetches
+		   it. the string is copied one character per word and terminated, like
+		   SYS_GETENV. -1 if there is no such argument */
+		return vecstr(zy_argv, zy_argc, a, b, c, "<eval>");
+	case 23:	/* envc: how many entries the environment has */
+		return (Cell)zy_envc;
+	case 24:	/* envp: one entry of the environment, shaped like argv */
+		return vecstr(zy_envp, zy_envc, a, b, c, 0);
+	case 40:	/* raw host syscall: a = address of seven words, the syscall number and then its six arguments. the result is the kernel's, or -errno. linux only; elsewhere -1 */
+#ifdef __linux__
+	{
+		long w[7], r;
+		int k;
+		if (!inmem(a) || !inmem(a + 6)) return -1;
+		for (k = 0; k < 7; k++) w[k] = (long)m[a + k];
+		r = syscall(w[0], w[1], w[2], w[3], w[4], w[5], w[6]);
+		if (r == -1) return -(Cell)errno;
+		return (Cell)r;
+	}
+#else
+		return -1;
+#endif
+	default:
+		return -1;
+	}
+}
+
+/* ---- virtual enhancements ----
+   up to 512 custom opcodes (22 .. 533), wired in at runtime by the install
+   opcode (21). an enhancement is a short program over native micro-ops,
+   written either as a recipe (text) or as straight-line vm code that gets
+   lifted. both become ssa, go through the optimizer below, and then run in
+   the interpreter or get inlined by the jit as native code. */
+#define NENH 512
+#define XOP0 22
+enum { U_LI, U_MOV, U_ADD, U_SUB, U_MUL, U_SDIV, U_UDIV, U_SREM, U_UREM, U_AND, U_OR, U_XOR,
+	U_SHL, U_SHR, U_SAR, U_NEG, U_NOT, U_LT, U_LTU, U_EQ, U_NE, U_MIN, U_MAX, U_MINU, U_MAXU,
+	U_MULHS, U_MULHU, U_CLZ, U_ABS, U_SEL, U_LD, U_ST,
+	U_FADD, U_FSUB, U_FMUL, U_FDIV, U_FEQ, U_FLT, U_FLE, U_I2F, U_F2I,
+	U_LD8, U_LD16, U_LD32, U_ST8, U_ST16, U_ST32,
+	U_NARG, U_NCALL,	/* a native call is one narg per argument (imm = its position) and then the ncall (imm = the call's descriptor) */
+	U_CTZ, U_POPCNT, U_BSWAP, U_ROTL, U_ROTR,	/* bit tricks: count trailing zeros, population count, byte swap, rotates */
+	U_F2S, U_S2F,		/* binary64 bits to binary32 bits (rounded) and back (exact) */
+	U_N };
+static const char *unames[U_N] = { "li", "mov", "add", "sub", "mul", "sdiv", "udiv", "srem", "urem",
+	"and", "or", "xor", "shl", "shr", "sar", "neg", "not", "lt", "ltu", "eq", "ne", "min", "max",
+	"minu", "maxu", "mulhs", "mulhu", "clz", "abs", "sel", "ld", "st",
+	"fadd", "fsub", "fmul", "fdiv", "feq", "flt", "fle", "i2f", "f2i",
+	"ld8", "ld16", "ld32", "st8", "st16", "st32", "narg", "ncall",
+	"ctz", "popcnt", "bswap", "rotl", "rotr", "f2s", "s2f" };
+static const signed char uarity[U_N] = { 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2,
+	2, 2, 2, 2, 2, 2, 1, 1, 3, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 2, 2, 1, 1 };
+static int is_store(int op)
+{
+	return op == U_ST || op == U_ST8 || op == U_ST16 || op == U_ST32;
+}
+/* ops that matter even when nothing reads their value: a store, an argument
+   handed to a native call, the call. none of them has a result a later pass
+   may reuse or drop */
+static int is_eff(int op)
+{
+	return is_store(op) || op == U_NARG || op == U_NCALL;
+}
+static int is_memop(int op)
+{
+	return is_eff(op) || op == U_LD || op == U_LD8 || op == U_LD16 || op == U_LD32;
+}
+static int bwidth(int op)
+{
+	if (op == U_LD8 || op == U_ST8) return 1;
+	if (op == U_LD16 || op == U_ST16) return 2;
+	if (op == U_LD32 || op == U_ST32) return 4;
+	return 0;
+}
+typedef struct { int op, d, a, b, c; Cell imm; } Uop;	/* ssa: d is a fresh value, a b c are values */
+typedef struct {
+	int live, kind, nin, nout, n, nv, out[8];
+	Uop *u;
+	signed char *reg;	/* physical register per value, for the jit; 0 if it didn't fit */
+} Enh;
+static Enh enh[NENH];
+static int need_flush;
+
+#define MAXUOP 4096
+static Uop ub[MAXUOP]; static int un, unv, uerr;	/* the program being built */
+static int unew(int op, int a, int b, int c, Cell imm)
+{
+	Uop *u;
+	if (un >= MAXUOP) { uerr = -5; return 0; }
+	u = &ub[un++]; u->op = op; u->d = unv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
+	return u->d;
+}
+static Cell mread(int a) { return (unsigned)a < MW ? m[a] : 0; }
+
+/* recipe text: one statement per line or per ';'
+     r0 = sdiv r0 r1      dst = op operands, operands are rN, parameter names or numbers
+     r2 = r0              a copy, or a number
+     st r1 r0             memory[r1] = r0
+   inputs arrive in r0.. (and under the parameter names), outputs leave from r0.. */
+static int parse_recipe(int src, int names, int nin, int nout, int *out)
+{
+	int cur[16], i, p = src, n, k, op, args[3], na, dst, isst;
+	char tok[8][32]; int nt;
+	for (i = 0; i < 16; i++) cur[i] = i < nin ? i : -1;
+	for (;;) {
+		/* read one statement into tokens */
+		nt = 0;
+		for (;;) {
+			int c = mread(p);
+			while (c == ' ' || c == '\t' || c == '\r') c = mread(++p);
+			if (!c || c == '\n' || c == ';') break;
+			if (c == '=') { if (nt < 8) strcpy(tok[nt++], "="); p++; continue; }
+			n = 0;
+			while (c && c != ' ' && c != '\t' && c != '\n' && c != ';' && c != '=' && c != '\r') { if (n < 31) tok[nt < 8 ? nt : 7][n++] = (char)c; c = mread(++p); }
+			if (nt < 8) tok[nt++][n] = 0;
+		}
+		if (nt) {
+			isst = !strcmp(tok[0], "st");
+			if (!isst && (nt < 3 || strcmp(tok[1], "="))) return -2;
+			/* destination register */
+			dst = -1;
+			if (!isst) {
+				if (tok[0][0] == 'r' && tok[0][1] >= '0' && tok[0][1] <= '9') dst = atoi(tok[0] + 1);
+				else for (i = 0; i < nin && names; i++) {
+					int s = mread(names + i), j = 0;
+					while (mread(s + j) && mread(s + j) == (unsigned char)tok[0][j]) j++;
+					if (!mread(s + j) && !tok[0][j]) dst = i;
+				}
+				if (dst < 0 || dst > 15) return -2;
+			}
+			k = isst ? 1 : 2;	/* first token after "=" (or after st) */
+			op = -1;
+			if (!isst && nt > 3) { for (i = 0; i < U_N; i++) if (!strcmp(tok[k], unames[i])) op = i; if (op < 0) return -2; k++; }
+			if (isst) op = U_ST;
+			na = 0;
+			for (; k < nt; k++) {
+				char *t = tok[k]; int v = -1;
+				if (na >= 3) return -2;
+				if (t[0] == 'r' && t[1] >= '0' && t[1] <= '9') { int r = atoi(t + 1); if (r > 15) return -2; if (cur[r] < 0) cur[r] = unew(U_LI, 0, 0, 0, 0); v = cur[r]; }
+				else if ((t[0] >= '0' && t[0] <= '9') || t[0] == '-') { char *e; Cell x = (Cell)strtoll(t, &e, 0); if (*e) return -2; v = unew(U_LI, 0, 0, 0, x); }
+				else {
+					for (i = 0; i < nin && names; i++) {
+						int s = mread(names + i), j = 0;
+						while (mread(s + j) && mread(s + j) == (unsigned char)t[j]) j++;
+						if (!mread(s + j) && !t[j]) v = cur[i] < 0 ? (cur[i] = unew(U_LI, 0, 0, 0, 0)) : cur[i];
+					}
+					if (v < 0) return -2;
+				}
+				args[na++] = v;
+			}
+			if (op < 0) {	/* x = y */
+				if (na != 1) return -2;
+				cur[dst] = args[0];
+			} else {
+				if (na != (op == U_LI ? 0 : uarity[op])) {
+					if (!(op == U_LI && na == 1)) return -2;
+				}
+				if (op == U_LI) cur[dst] = args[0];
+				else if (is_store(op)) unew(op, args[0], args[1], 0, 0);
+				else cur[dst] = unew(op, args[0], na > 1 ? args[1] : 0, na > 2 ? args[2] : 0, 0);
+			}
+			if (uerr) return uerr;
+		}
+		if (!mread(p)) break;
+		p++;
+	}
+	for (i = 0; i < nout; i++) { if (cur[i] < 0) cur[i] = unew(U_LI, 0, 0, 0, 0); out[i] = cur[i]; }
+	return uerr;
+}
+
+/* straight-line vm code from addr up to its ret, run symbolically */
+static int lift_vm(int a, int len, int nin, int nout, int *out)
+{
+	int st[256], sp = 0, i, k, x, y; Cell w;
+	for (i = 0; i < nin; i++) st[sp++] = i;
+	for (i = 0; i < len; i++) {
+		w = mread(a + i);
+		if (w & 1) { if (sp >= 256) return -4; st[sp++] = unew(U_LI, 0, 0, 0, w >> 1); continue; }
+		k = w / 2;
+		if (k == 9) break;
+		switch (k) {
+		case 1: if (sp < 1) return -4; st[sp - 1] = unew(U_LD, st[sp - 1], 0, 0, 0); break;
+		case 2: if (sp < 2) return -4; unew(U_ST, st[sp - 1], st[sp - 2], 0, 0); sp -= 2; break;
+		case 3:	/* pick needs a constant index */
+			if (sp < 1) return -4;
+			x = st[sp - 1];
+			if (x < nin) return -4;
+			{ int j; for (j = 0; j < un && ub[j].d != x; j++); if (j == un || ub[j].op != U_LI) return -4; y = (int)ub[j].imm; }
+			if (y < 0 || y >= sp - 1) return -4;
+			st[sp - 1] = st[sp - 2 - y];
+			break;
+		case 4: if (sp < 1) return -4; sp--; break;
+		case 5: if (sp < 2) return -4; x = st[sp - 1]; st[sp - 1] = st[sp - 2]; st[sp - 2] = x; break;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: {
+			static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
+			if (sp < 2) return -4;
+			x = st[sp - 1]; y = st[sp - 2]; sp -= 2;
+			st[sp++] = unew(map[k - 13], y, x, 0, 0);
+			break; }
+		default: return -4;	/* control flow, i/o and other enhancements don't lift */
+		}
+		if (uerr) return uerr;
+	}
+	if (sp != nout) return -4;
+	for (i = 0; i < nout; i++) out[i] = st[i];
+	return uerr;
+}
+
+/* ---- the optimizer ----
+   one forward pass of copy propagation, constant folding, algebraic
+   simplification, division by constants (the multiply-high trick) and
+   common subexpression elimination, then dead code removal. */
+static Cell ev(int op, Cell a, Cell b, Cell c)
+{
+	UCell ua = (UCell)a, ub_ = (UCell)b;
+	switch (op) {
+	case U_ADD: return (Cell)(ua + ub_);
+	case U_SUB: return (Cell)(ua - ub_);
+	case U_MUL: return (Cell)(ua * ub_);
+	case U_SDIV: return b == 0 ? 0 : (a == INT64_MIN && b == -1) ? a : a / b;
+	case U_UDIV: return b == 0 ? 0 : (Cell)(ua / ub_);
+	case U_SREM: return b == 0 ? a : (a == INT64_MIN && b == -1) ? 0 : a % b;
+	case U_UREM: return b == 0 ? a : (Cell)(ua % ub_);
+	case U_AND: return a & b;
+	case U_OR: return a | b;
+	case U_XOR: return a ^ b;
+	case U_SHL: return (Cell)(ua << (b & 63));
+	case U_SHR: return (Cell)(ua >> (b & 63));
+	case U_SAR: return a >> (b & 63);
+	case U_NEG: return (Cell)(0ull - ua);
+	case U_NOT: return ~a;
+	case U_LT: return a < b;
+	case U_LTU: return ua < ub_;
+	case U_EQ: return a == b;
+	case U_NE: return a != b;
+	case U_MIN: return a < b ? a : b;
+	case U_MAX: return a > b ? a : b;
+	case U_MINU: return ua < ub_ ? a : b;
+	case U_MAXU: return ua > ub_ ? a : b;
+	case U_MULHS: return (Cell)(((__int128)a * (__int128)b) >> 64);
+	case U_MULHU: return (Cell)(((unsigned __int128)ua * ub_) >> 64);
+	case U_CLZ: { int n = 0; if (!ua) return 64; while (!(ua & (1ull << 63))) { ua <<= 1; n++; } return n; }
+	case U_ABS: return a < 0 ? (Cell)(0ull - ua) : a;
+	case U_CTZ: return ua ? __builtin_ctzll(ua) : 64;
+	case U_POPCNT: return __builtin_popcountll(ua);
+	case U_BSWAP: return (Cell)__builtin_bswap64(ua);
+	case U_ROTL: return (Cell)((ua << (b & 63)) | (ua >> ((64 - (b & 63)) & 63)));
+	case U_ROTR: return (Cell)((ua >> (b & 63)) | (ua << ((64 - (b & 63)) & 63)));
+	case U_F2S: return (Cell)fop_f2s((uint64_t)a);
+	case U_S2F: return (Cell)fop_s2f((uint64_t)a);
+	case U_SEL: return a ? b : c;
+	case U_FADD: return (Cell)fop_add((uint64_t)a, (uint64_t)b);
+	case U_FSUB: return (Cell)fop_sub((uint64_t)a, (uint64_t)b);
+	case U_FMUL: return (Cell)fop_mul((uint64_t)a, (uint64_t)b);
+	case U_FDIV: return (Cell)fop_div((uint64_t)a, (uint64_t)b);
+	case U_FEQ: return fop_eq((uint64_t)a, (uint64_t)b);
+	case U_FLT: return fop_lt((uint64_t)a, (uint64_t)b);
+	case U_FLE: return fop_le((uint64_t)a, (uint64_t)b);
+	case U_I2F: return (Cell)fop_i2f(a);
+	case U_F2I: return fop_f2i((uint64_t)a);
+	}
+	return 0;
+}
+static void smagic(Cell d, Cell *mul, int *sh)	/* hacker's delight, signed, 64-bit */
+{
+	const UCell two63 = 1ull << 63;
+	UCell ad = d < 0 ? 0ull - (UCell)d : (UCell)d, t = two63 + ((UCell)d >> 63);
+	UCell anc = t - 1 - t % ad, q1 = two63 / anc, r1 = two63 - q1 * anc, q2 = two63 / ad, r2 = two63 - q2 * ad, delta;
+	int p = 63;
+	do {
+		p++;
+		q1 *= 2; r1 *= 2; if (r1 >= anc) { q1++; r1 -= anc; }
+		q2 *= 2; r2 *= 2; if (r2 >= ad) { q2++; r2 -= ad; }
+		delta = ad - r2;
+	} while (q1 < delta || (q1 == delta && r1 == 0));
+	*mul = (Cell)(q2 + 1); if (d < 0) *mul = -*mul;
+	*sh = p - 64;
+}
+static void umagic(UCell d, UCell *mul, int *add, int *sh)	/* hacker's delight, unsigned, 64-bit */
+{
+	UCell nc = ~0ull - (0ull - d) % d, q1, r1, q2, r2, delta;
+	const UCell two63 = 1ull << 63;
+	int p = 63;
+	*add = 0;
+	q1 = two63 / nc; r1 = two63 - q1 * nc;
+	q2 = (two63 - 1) / d; r2 = (two63 - 1) - q2 * d;
+	do {
+		p++;
+		if (r1 >= nc - r1) { q1 = 2 * q1 + 1; r1 = 2 * r1 - nc; } else { q1 = 2 * q1; r1 = 2 * r1; }
+		if (r2 + 1 >= d - r2) { if (q2 >= two63 - 1) *add = 1; q2 = 2 * q2 + 1; r2 = 2 * r2 + 1 - d; }
+		else { if (q2 >= two63) *add = 1; q2 = 2 * q2; r2 = 2 * r2 + 1; }
+		delta = d - 1 - r2;
+	} while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0)));
+	*mul = q2 + 1; *sh = p - 64;
+}
+
+static Uop ob[MAXUOP]; static int on, onv;
+static int *omap, *isk; static Cell *kv;	/* old value -> new value; constness of new values */
+static int pure(int op) { return !is_memop(op); }
+static int ispow2(UCell x) { return x && !(x & (x - 1)); }
+static int lg2(UCell x) { int n = 0; while (x > 1) { x >>= 1; n++; } return n; }
+static int oe(int op, int a, int b, int c, Cell imm);
+static int K(Cell v) { return oe(U_LI, 0, 0, 0, v); }
+static int raw(int op, int a, int b, int c, Cell imm)
+{
+	Uop *u; int i;
+	if (pure(op))	/* cse */
+		for (i = 0; i < on; i++) {
+			u = &ob[i];
+			if (u->op == op && u->a == a && u->b == b && u->c == c && u->imm == imm) return u->d;
+		}
+	if (on >= MAXUOP) { uerr = -5; return 0; }
+	u = &ob[on++]; u->op = op; u->d = onv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
+	isk[u->d] = op == U_LI; kv[u->d] = imm;
+	return u->d;
+}
+static int sdivk(int a, Cell d)	/* a / d for a constant d, no divide instruction */
+{
+	Cell mul; int sh, q, t, k;
+	UCell ad;
+	if (d == 1) return a;
+	if (d == -1) return oe(U_NEG, a, 0, 0, 0);
+	if (d == INT64_MIN) return oe(U_EQ, a, K(d), 0, 0);
+	ad = (UCell)(d < 0 ? -d : d);
+	if (ispow2(ad)) {
+		k = lg2(ad);
+		t = oe(U_SAR, a, K(63), 0, 0);
+		t = oe(U_SHR, t, K(64 - k), 0, 0);
+		q = oe(U_SAR, oe(U_ADD, a, t, 0, 0), K(k), 0, 0);
+		return d < 0 ? oe(U_NEG, q, 0, 0, 0) : q;
+	}
+	smagic(d, &mul, &sh);
+	q = oe(U_MULHS, a, K(mul), 0, 0);
+	if (d > 0 && mul < 0) q = oe(U_ADD, q, a, 0, 0);
+	if (d < 0 && mul > 0) q = oe(U_SUB, q, a, 0, 0);
+	if (sh) q = oe(U_SAR, q, K(sh), 0, 0);
+	return oe(U_ADD, q, oe(U_SHR, q, K(63), 0, 0), 0, 0);
+}
+static int udivk(int a, UCell d)
+{
+	UCell mul; int add, sh, t;
+	if (d == 1) return a;
+	if (ispow2(d)) return oe(U_SHR, a, K(lg2(d)), 0, 0);
+	if (d > (1ull << 63)) return oe(U_XOR, oe(U_LTU, a, K((Cell)d), 0, 0), K(1), 0, 0);	/* a >= d */
+	umagic(d, &mul, &add, &sh);
+	t = oe(U_MULHU, a, K((Cell)mul), 0, 0);
+	if (!add) return sh ? oe(U_SHR, t, K(sh), 0, 0) : t;
+	t = oe(U_ADD, oe(U_SHR, oe(U_SUB, a, t, 0, 0), K(1), 0, 0), t, 0, 0);
+	return sh > 1 ? oe(U_SHR, t, K(sh - 1), 0, 0) : t;
+}
+static int oe(int op, int a, int b, int c, Cell imm)
+{
+	int ka, kb; Cell A, B;
+	if (op == U_LI) return raw(U_LI, 0, 0, 0, imm);
+	if (op == U_MOV) return a;
+	if (is_memop(op)) return raw(op, a, b, 0, 0);
+	ka = isk[a]; kb = uarity[op] > 1 && isk[b];
+	A = kv[a]; B = kv[b];
+	/* everything constant: fold */
+	if (ka && (uarity[op] < 2 || kb) && (uarity[op] < 3 || isk[c])) return K(ev(op, A, B, kv[c]));
+	/* constants go right for commutative ops */
+	if (ka && !kb && (op == U_ADD || op == U_MUL || op == U_AND || op == U_OR || op == U_XOR || op == U_EQ || op == U_NE ||
+		op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU || op == U_MULHS || op == U_MULHU ||
+		op == U_FADD || op == U_FMUL || op == U_FEQ)) {
+		int t = a; a = b; b = t; kb = 1; B = A; ka = 0;
+	}
+	if (op == U_SEL && ka) return A ? b : c;
+	if (op == U_SEL && b == c) return b;
+	if (kb) switch (op) {
+	case U_ADD: case U_SUB: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR:
+		if (!B || (op >= U_SHL && op <= U_SAR && !(B & 63))) return a;
+		break;
+	case U_MUL:
+		if (B == 0) return K(0);
+		if (B == 1) return a;
+		if (B == -1) return oe(U_NEG, a, 0, 0, 0);
+		if (B > 0 && ispow2((UCell)B)) return oe(U_SHL, a, K(lg2((UCell)B)), 0, 0);
+		break;
+	case U_AND: if (B == 0) return K(0); if (B == -1) return a; break;
+	case U_SDIV: if (B == 0) return K(0); return sdivk(a, B);
+	case U_SREM: if (B == 0) return a; if (B == 1 || B == -1) return K(0);
+		return oe(U_SUB, a, oe(U_MUL, sdivk(a, B), b, 0, 0), 0, 0);
+	case U_UDIV: if (B == 0) return K(0); return udivk(a, (UCell)B);
+	case U_UREM: if (B == 0) return a;
+		if (B > 0 && ispow2((UCell)B)) return oe(U_AND, a, K(B - 1), 0, 0);
+		return oe(U_SUB, a, oe(U_MUL, udivk(a, (UCell)B), b, 0, 0), 0, 0);
+	}
+	/* two equal operands. not applied to the float ops: x - x is a NaN when x is,
+	   and x == x is false for a NaN. a pair of constant floats already folded. */
+	if (a == b && uarity[op] == 2) {
+		if (op == U_SUB || op == U_XOR || op == U_LT || op == U_LTU || op == U_NE) return K(0);
+		if (op == U_AND || op == U_OR || op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU) return a;
+		if (op == U_EQ) return K(1);
+	}
+	return raw(op, a, b, c, 0);
+}
+/* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */
+static int optimize(Enh *e, int nin, int *out, int nout)
+{
+	int i, *live, n = 0;
+	static int omap_[MAXUOP + 16], isk_[MAXUOP * 4]; static Cell kv_[MAXUOP * 4];
+	omap = omap_; isk = isk_; kv = kv_;
+	on = 0; onv = nin;
+	for (i = 0; i < nin; i++) { omap[i] = i; isk[i] = 0; }
+	for (i = 0; i < un && !uerr; i++) {
+		Uop *u = &ub[i];
+		int a = uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD ? omap[u->a] : 0;
+		int b = uarity[u->op] > 1 || u->op == U_ST ? omap[u->b] : 0;
+		int c = uarity[u->op] > 2 ? omap[u->c] : 0;
+		if (u->op == U_ST) oe(U_ST, a, b, 0, 0);
+		else omap[u->d] = oe(u->op, a, b, c, u->imm);
+	}
+	if (uerr) return uerr;
+	/* dead code: outputs and stores are live */
+	live = calloc(onv + 1, sizeof *live);
+	for (i = 0; i < nout; i++) { e->out[i] = omap[out[i]]; live[e->out[i]] = 1; }
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (!is_store(u->op) && !live[u->d]) { u->op = -1; continue; }
+		if (uarity[u->op] > 0 || is_memop(u->op)) live[u->a] = 1;
+		if (uarity[u->op] > 1 || is_store(u->op)) live[u->b] = 1;
+		if (uarity[u->op] > 2) live[u->c] = 1;
+	}
+	free(live);
+	e->u = malloc((on + 1) * sizeof(Uop));
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) e->u[n++] = ob[i];
+	e->n = n; e->nv = onv; e->nin = nin; e->nout = nout;
+	return 0;
+}
+/* run an installed enhancement on a downward-growing stack; returns the new top */
+static Cell *xexec(Enh *e, Cell *sp)
+{
+	Cell vb[256], *v = e->nv <= 256 ? vb : malloc(e->nv * sizeof(Cell)), i, j;
+	for (j = 0; j < e->nin; j++) v[j] = sp[e->nin - 1 - j];
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		switch (u->op) {
+		case U_LI: v[u->d] = u->imm; break;
+		case U_LD: v[u->d] = mread(v[u->a]); break;
+		case U_ST: if (inmem(v[u->a])) m[v[u->a]] = v[u->b]; break;
+		default:
+			if (u->op >= U_LD8 && u->op <= U_ST32) {
+				if (u->op == U_LD8) v[u->d] = bld8(v[u->a]);
+				else if (u->op == U_LD16) v[u->d] = bld16(v[u->a]);
+				else if (u->op == U_LD32) v[u->d] = bld32(v[u->a]);
+				else if (u->op == U_ST8) bst8(v[u->a], v[u->b]);
+				else if (u->op == U_ST16) bst16(v[u->a], v[u->b]);
+				else if (u->op == U_ST32) bst32(v[u->a], v[u->b]);
+				break;
+			}
+			v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
+		}
+	}
+	sp += e->nin - e->nout;
+	for (j = 0; j < e->nout; j++) sp[e->nout - 1 - j] = v[e->out[j]];
+	if (v != vb) free(v);
+	return sp;
+}
+static void xdump(int slot, Enh *e)
+{
+	int i;
+	fprintf(stderr, "enh %d: %d in, %d out, %d ops\n", slot, e->nin, e->nout, e->n);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		if (u->op == U_ST) fprintf(stderr, "  st v%d v%d\n", u->a, u->b);
+		else if (u->op == U_LI) fprintf(stderr, "  v%d = %lld\n", u->d, (long long)u->imm);
+		else { fprintf(stderr, "  v%d = %s v%d", u->d, unames[u->op], u->a); if (uarity[u->op] > 1) fprintf(stderr, " v%d", u->b); if (uarity[u->op] > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
+	}
+	for (i = 0; i < e->nout; i++) fprintf(stderr, "  out%d = v%d\n", i, e->out[i]);
+}
+/* registers for the jit: values live in x0..x15, straight-line linear scan.
+   if they don't fit, reg stays 0 and the jit calls the interpreter instead. */
+static void regalloc(Enh *e)
+{
+	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = 0xffff;
+	signed char *r = malloc(e->nv);
+	for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; }
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) last[u->a] = i;
+		if (uarity[u->op] > 1 || u->op == U_ST) last[u->b] = i;
+		if (uarity[u->op] > 2) last[u->c] = i;
+	}
+	for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n;
+#define TAKE(v) do { for (k = 0; k < 16 && !(fr >> k & 1); k++); if (k == 16) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
+	for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i]; int ops[3], no = 0;
+		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) ops[no++] = u->a;
+		if (uarity[u->op] > 1 || u->op == U_ST) ops[no++] = u->b;
+		if (uarity[u->op] > 2) ops[no++] = u->c;
+		for (j = 0; j < no; j++) if (last[ops[j]] == i && r[ops[j]] >= 0 && (fr >> r[ops[j]] & 1) == 0) fr |= 1 << r[ops[j]];
+		if (!is_store(u->op)) TAKE(u->d);
+		if (!is_store(u->op) && last[u->d] == -2) fr |= 1 << r[u->d];	/* never used */
+	}
+#undef TAKE
+	free(last); e->reg = r; return;
+fail:
+	free(last); free(r); e->reg = 0;
+}
+/* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs,
+   outputs, payload address, payload length, parameter names (or 0). kind 2
+   is an o3 region (below).
+   returns 0, or -1 bad descriptor, -2 recipe syntax, -4 code doesn't lift,
+   -5 too big. */
+static int o3install(int d);
+static void slot_changed(int slot);
+static int install(int d)
+{
+	int slot = mread(d), kind = mread(d + 1), nin = mread(d + 2), nout = mread(d + 3);
+	int pay = mread(d + 4), len = mread(d + 5), names = mread(d + 6), out[8], r;
+	Enh ne;
+	if (kind == 2) return o3install(d);
+	if ((unsigned)slot >= NENH || (unsigned)nin > 8 || (unsigned)nout > 8 || (kind != 0 && kind != 1)) return -1;
+	un = 0; unv = nin; uerr = 0;
+	r = kind == 0 ? parse_recipe(pay, names, nin, nout, out) : lift_vm(pay, len, nin, nout, out);
+	if (r) { if (trace) fprintf(stderr, "enh %d: install failed (%d)\n", slot, r); return r; }
+	memset(&ne, 0, sizeof ne);
+	if ((r = optimize(&ne, nin, out, nout))) return r;
+	ne.kind = kind; ne.live = 1;
+	regalloc(&ne);
+	if (enh[slot].live) { free(enh[slot].u); free(enh[slot].reg); }
+	enh[slot] = ne;
+	slot_changed(slot);
+	if (trace) xdump(slot, &enh[slot]);
+	return 0;
+}
+
+/* ---- o3: optimized regions ----
+   o3 code comes from the compiler as a region: a range [lo, hi) of vm words
+   plus the frame slots that nothing takes the address of, each with one of 7
+   registers (install kind 2: [-, 2, lo, hi, n, slot, register, ...]). the region is cut into basic blocks. each
+   block is lifted to ssa: the vm stack is tracked symbolically, so only what
+   is left on it at the end of the block touches memory, the slots live in
+   registers across blocks (VGET reads one at block entry, the block's end
+   says what each one holds now), the frame pointer is var 0, and enhancements
+   are spliced in. empty jumps are folded and a straight loop is rotated, so
+   the backends see one block per iteration when the body itself doesn't
+   branch. a backend compiles the whole region at once: arm64, llvm at O3, or the
+   region interpreter below. the region is entered at lo and at the return
+   points of its calls. a register can stand for several slots (variables
+   whose scopes don't overlap), and spilling writes it to all of them. calls,
+   returns, jumps out, and anything a block can't
+   lift leave it: the slots go back to the frame first, so the code outside
+   sees the usual memory. stores to its own code drop it; it lifts again on
+   its next entry. */
+/* ZFP is the frame pointer word from memmap.h */
+#define U_VGET U_N	/* a var at block entry; imm = var */
+#define U_SLD (U_N + 1)	/* the stack word at offset imm from the top at block entry */
+#define RMAXV 8
+#define RMAXE 256
+enum { T_BR, T_CBR, T_JMP, T_CALL, T_RET };
+typedef struct {
+	int a, n, nv, delta, nso, term, cv, s1, s2, x1, x2;
+	int vs[RMAXV];	/* each var's value at the end, -1 if unchanged */
+	int *so;	/* stack words to write at the end: offset, value */
+	Uop *u;
+} RB;
+typedef struct {
+	int idx, lo, hi, nv, nv0, slot[RMAXV], lifted, dead, nonat, nb, ne, maxnv;
+	int np, ps[32], pv[32];	/* frame slot -> var; vars in scopes that never overlap share one */
+	unsigned char fv[RMAXV];	/* 1: this var's home is a d register, not an x register */
+	unsigned char seen[RMAXV];
+	unsigned char sc[RMAXV];	/* 1: every local that lives in this var is a plain number (int, char, bool, float), never a pointer */
+	int ent[RMAXE], entb[RMAXE];
+	RB *b;
+	void *nat, *dnat, *stub[RMAXE];	/* the backend's code; dnat is the register-passing entry of a function region */
+	int fslots, argc, fn;		/* callable o3 function: frame, arity, prologue address */
+	int marked;			/* the backend is watching the region's words */
+	unsigned char dep[NENH / 8];	/* enhancement slots spliced into the lifted code */
+	void *rt;			/* llvm: the resource tracker that owns the region's code */
+	unsigned gen;			/* bumped every time the region's code is dropped: a copy made from an older one is stale */
+} Rg;
+#define MAXRG 4096
+static Rg *rgs[MAXRG];
+static int nrg;
+static int fn_pc[MAXRG];
+static Rg *fn_rg_[MAXRG];
+static Rg *fn_cache[1024];
+static int nfn;
+
+/* ---- bookkeeping for translated code, shared by the backends ----
+   a translation covers a range of vm words [a,b). three things have to stay
+   cheap when code is rewritten while it runs. which translation a store
+   hits: ranges are indexed by 64-word page, so a covered store looks at the
+   handful of ranges on its page. which words are still covered after one
+   goes: the coverage bytes are counters (a word two translations share counts
+   2; 255 sticks), so dropping a translation subtracts its own range and
+   nothing is rebuilt. and what an install invalidates: a translation records
+   the enhancement slots it inlined, or found unwired, so an install drops
+   those and keeps the rest. */
+#define PGSH 6
+#define NPG ((code_words >> PGSH) + 1)
+#define MAXPGN (1 << 19)
+#define MAXDEP (1 << 18)
+static struct { int a, b; } ix_blk[MAXB];
+static int ix_n;
+static int *pg_head, pg_free, pg_top, pg_live;		/* node numbers are stored plus one: 0 is none */
+static struct { int blk, next; } pg_node[MAXPGN];
+static int dep_head[NENH + 1], dep_n;				/* slot NENH: depends on every install */
+static struct { int blk, next; } dep_node[MAXDEP];
+static int drop_epoch;						/* changes whenever any translation dies */
+static void be_drop(int j);					/* the backend forgets block j's code and coverage */
+static void runmark(Rg *R);					/* the backend stops watching the region's words */
+static void rclear(Rg *R);
+static void cinc(unsigned char *c, int lo, int hi) { if (hi > code_words) hi = (int)code_words; for (; lo < hi; lo++) if (c[lo] != 255) c[lo]++; }
+static void cdec(unsigned char *c, int lo, int hi) { if (hi > code_words) hi = (int)code_words; for (; lo < hi; lo++) if (c[lo] && c[lo] != 255) c[lo]--; }
+static void ix_reset(void)
+{
+	memset(pg_head, 0, NPG * sizeof *pg_head); memset(dep_head, 0, sizeof dep_head);
+	pg_free = pg_top = pg_live = dep_n = ix_n = 0;
+	drop_epoch++;
+}
+static int ix_room(void) { return ix_n < MAXB && pg_live + 24 < MAXPGN && dep_n + 300 < MAXDEP; }
+static int ix_add(int a, int b)
+{
+	int j = ix_n++, pg, nd;
+	ix_blk[j].a = a; ix_blk[j].b = b;
+	for (pg = a >> PGSH; pg <= (b - 1) >> PGSH; pg++) {
+		if (pg_free) { nd = pg_free - 1; pg_free = pg_node[nd].next; } else nd = pg_top++;
+		pg_node[nd].blk = j; pg_node[nd].next = pg_head[pg]; pg_head[pg] = nd + 1; pg_live++;
+	}
+	return j;
+}
+static void ix_kill(int j)
+{
+	if (ix_blk[j].a < 0) return;
+	be_drop(j);
+	ix_blk[j].a = -1;
+	drop_epoch++;
+}
+/* drop every translation whose range holds addr; dead entries on the page are unlinked on the way */
+static int ix_drop_at(int addr)
+{
+	int *pp = &pg_head[addr >> PGSH], hit = 0, nd, j;
+	while (*pp) {
+		nd = *pp - 1; j = pg_node[nd].blk;
+		if (ix_blk[j].a < 0) { *pp = pg_node[nd].next; pg_node[nd].next = pg_free; pg_free = nd + 1; pg_live--; continue; }
+		if (ix_blk[j].a <= addr && addr < ix_blk[j].b) { ix_kill(j); hit++; continue; }
+		pp = &pg_node[nd].next;
+	}
+	return hit;
+}
+static void dep_add(int slot, int j)
+{
+	if ((unsigned)slot > NENH) slot = NENH;
+	dep_node[dep_n].blk = j; dep_node[dep_n].next = dep_head[slot]; dep_head[slot] = ++dep_n;
+}
+/* slot was (re)wired: drop what used it, or found it unwired, and nothing else */
+static void slot_changed(int slot)
+{
+	int d, h, j;
+	for (h = 0; h < 2; h++) {
+		int *hp = &dep_head[h ? NENH : slot];
+		for (d = *hp; d; d = dep_node[d - 1].next) ix_kill(dep_node[d - 1].blk);
+		*hp = 0;
+	}
+	for (j = 0; j < nrg; j++)
+		if (rgs[j]->lifted && (rgs[j]->dep[slot >> 3] & (1 << (slot & 7)))) { rclear(rgs[j]); drop_epoch++; }
+}
+static Cell halt_at = -1;	/* a nested call stopped the machine: operand or return overflow */
+static unsigned short *o3ent;	/* region + 1 at an entry point */
+static unsigned char *o3ei;		/* which of its entries */
+static int o3_ready(Rg *C);
+static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr);
+static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr);
+static void o3_bind(Rg *R)
+{
+	int i;
+	if (R->fn < 0) return;	/* -1: a bare block. 0 is a real function at address 0 */
+	fn_cache[R->fn & 1023] = R;
+	for (i = 0; i < nfn; i++) if (fn_pc[i] == R->fn) { fn_rg_[i] = R; return; }
+	if (nfn < MAXRG) { fn_pc[nfn] = R->fn; fn_rg_[nfn++] = R; }
+}
+static Rg *fnlookup(int pc)
+{
+	int i;
+	Rg *R = fn_cache[pc & 1023];
+	if (R && R->fn == pc && !R->dead) return R;
+	for (i = 0; i < nfn; i++) if (fn_pc[i] == pc) {
+		R = fn_rg_[i];
+		if (R && !R->dead) { fn_cache[pc & 1023] = R; return R; }
+	}
+	return 0;
+}
+/* a call made from inside a region. an o3 callee gets a frame and is entered
+   at its region, and its epilogue is not run: fp comes back here. anything
+   else is an ordinary call, pushed on the vm return stack and run until that
+   address is the pc again. */
+/* what is left of a call once the callee's region has returned `back`: a
+   return address at or past the end of the function is the normal exit (the
+   epilogue's lit 0 is pushed when it fell off the end), anything else is
+   code to run on the vm until the function is left */
+static void o3_finish(Rg *C, int back, Cell *ps, Cell *pr)
+{
+	for (;;) {
+		if (halt_at >= 0 || back < 0 || (unsigned)back >= (unsigned)MW) {
+			if (halt_at < 0) halt_at = back < 0 ? PC_HALT : back;
+			m[ZFP] = mread(ZFP) + C->fslots;
+			return;
+		}
+		if (back >= C->hi) {
+			if (back == C->hi) {	/* fell off the end: the epilogue's lit 0 */
+				Cell sp = *ps - 1;
+				if ((unsigned)sp < (unsigned)MW) { m[sp] = 0; *ps = sp; }
+			}
+			m[ZFP] = mread(ZFP) + C->fslots;
+			return;
+		}
+		if (back >= C->lo && o3ent[back] == C->idx + 1) {
+			back = o3_exec_region(C, o3ei[back], ps, pr);
+			continue;
+		}
+		back = vm_until(back, -1, C->hi, C, ps, pr);
+	}
+}
+/* the llvm code calls this one with the callee and the address it returned,
+   when the quick exit did not apply */
+static void o3_finish_c(Cell *ps, Cell *pr, Rg *C, Cell back) { o3_finish(C, (int)back, ps, pr); }
+static void o3_invoke(Cell *ps, Cell *pr, Cell target, Cell retpc)
+{
+	Rg *C = fnlookup((int)target);
+	int back, fp, i, s;
+	if (halt_at >= 0) return;
+	if (*ps < stack_data_limit() + 16) {
+		if (stack_expand(0, stack_get(SC_DATA_TOP) - stack_get(SC_DATA_LOW) + 1, ps, pr) < 0) { halt_at = PC_DATA_OVERFLOW; return; }
+	}
+	if (!C || mread(ZFP) - C->fslots < frame_bound() || !o3_ready(C)) {
+		if ((unsigned)*pr < (unsigned)MW) m[(*pr)++] = retpc;
+		back = vm_until((int)target, (int)retpc, -1, 0, ps, pr);
+		if (back < 0 || (unsigned)back >= (unsigned)code_words) halt_at = back < 0 ? PC_HALT : back;
+		return;
+	}
+	fp = (int)mread(ZFP) - C->fslots;
+	m[ZFP] = fp;
+	s = (int)*ps;
+	for (i = C->argc; i-- > 0; ) {
+		if ((unsigned)s < (unsigned)MW && (unsigned)(fp + i) < (unsigned)MW) m[fp + i] = m[s];
+		s++;
+	}
+	*ps = s;
+	back = o3_exec_region(C, 0, ps, pr);
+	o3_finish(C, back, ps, pr);
+}
+/* ---- c abi ----
+   gate ops 14..17 and opcode 620 let vm code call native functions and hand
+   native code functions of its own. vm memory is one flat array of words, so
+   a vm byte address b is the host pointer (char *)m + b and a word address w
+   is (char *)m + 8 * w. an argument or result of pointer kind is converted
+   that way when it lies inside the array, and passed through untouched (an
+   opaque host pointer, a malloc result) when it does not. 0 is NULL both
+   ways.
+
+   a signature is a word of 4-bit kinds: the return kind first, then one per
+   argument (at most 15), and a 0 kind ends the list.
+     0 void   1 long (any 64-bit integer)   2 double   3 byte pointer
+     4 int (32-bit, sign extended)   5 float (32-bit, widened to a double word)
+     6 word pointer   7 unsigned int   8 short   9 unsigned short
+     10 char   11 unsigned char   (small kinds are extended on the way back)
+     12 byte string object (the word address of its length word; the
+     native side gets the address of its first byte)
+   integers fill the platform's integer argument registers (6 on x86-64, 8 on
+   arm64), doubles the 8 fp registers, and whatever does not fit goes on the
+   native stack in argument order: the call goes through a prototype of that
+   many integer and fp parameters followed by nine stack words, so the
+   platform's own rules place every argument.
+   op 14 (blk): blk is a block of words: [0] the symbol name as a byte
+   address, [1] the signature, [2] a lookup cache the host keeps, [3...] the
+   arguments in order. the result is the native result as a word (the bits,
+   for a double). a missing symbol prints a message and gives -1.
+   op 15 (name): 1 if the symbol resolves, else 0.
+   op 16 (entry, sig): a native function pointer that calls the vm function at
+   entry with the arguments the signature names, or 0. the vm code runs on top
+   of the stacks as they were when the native call that is running started, so
+   the pointer is only good while some native call from the vm is running.
+   op 17 (path): dlopen a library so later lookups can find its symbols.
+   opcode 620 (a0 ... an-1 blk -- r) is op 14 with the arguments on the stack
+   and the block's signature saying how many; o3 lifts it to the micro-ops
+   narg and ncall, and the llvm backend turns those into a direct typed call. */
+#ifdef ZYFFI
+#if defined(__aarch64__)
+#define FFI_NI 8
+#define FFI_IT Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell
+#define FFI_IN Cell a, Cell b, Cell c, Cell d, Cell e, Cell f, Cell g, Cell h
+#define FFI_IV a, b, c, d, e, f, g, h
+#define FFI_IC(iv) iv[0], iv[1], iv[2], iv[3], iv[4], iv[5], iv[6], iv[7]
+#else
+#define FFI_NI 6
+#define FFI_IT Cell, Cell, Cell, Cell, Cell, Cell
+#define FFI_IN Cell a, Cell b, Cell c, Cell d, Cell e, Cell f
+#define FFI_IV a, b, c, d, e, f
+#define FFI_IC(iv) iv[0], iv[1], iv[2], iv[3], iv[4], iv[5]
+#endif
+#define FFI_NS 15
+#define FFI_DT double, double, double, double, double, double, double, double
+#define FFI_ST Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell
+typedef Cell (*FfiI)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef double (*FfiD)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef float (*FfiF)(FFI_IT, FFI_DT, FFI_ST, ...);
+#define FFI_SYMS 256
+#define FFI_LIBS 16
+static struct { char name[96]; void *p; } ffi_sym[FFI_SYMS];
+static int ffi_nsym;
+static void *ffi_lib[FFI_LIBS];
+static int ffi_nlib, ffi_tried_libm;
+static void *ffi_lookup(const char *name)
+{
+	int i;
+	void *p;
+	for (i = 0; i < ffi_nsym; i++) if (!strcmp(ffi_sym[i].name, name)) return ffi_sym[i].p;
+	p = dlsym(RTLD_DEFAULT, name);
+	for (i = 0; !p && i < ffi_nlib; i++) p = dlsym(ffi_lib[i], name);
+	if (!p && !ffi_tried_libm) {	/* the math library is not linked into every host */
+		void *h;
+		ffi_tried_libm = 1;
+		h = dlopen("libm.so.6", RTLD_NOW | RTLD_GLOBAL);
+		if (!h) h = dlopen("libm.dylib", RTLD_NOW | RTLD_GLOBAL);
+		if (h && ffi_nlib < FFI_LIBS) ffi_lib[ffi_nlib++] = h;
+		for (i = 0; !p && i < ffi_nlib; i++) p = dlsym(ffi_lib[i], name);
+	}
+	if (p && strlen(name) < sizeof ffi_sym[0].name && ffi_nsym < FFI_SYMS) {
+		strcpy(ffi_sym[ffi_nsym].name, name);
+		ffi_sym[ffi_nsym++].p = p;
+	}
+	return p;
+}
+static Cell ffi_name(Cell a, char *dst, int max)
+{
+	return bop_cstr(m, a, dst, max, MW);
+}
+static Cell ffi_find(Cell a)
+{
+	char name[96];
+	if (ffi_name(a, name, 96) <= 0) return 0;
+	return ffi_lookup(name) ? 1 : 0;
+}
+static Cell ffi_load(Cell a)
+{
+	char path[512];
+	void *h;
+	if (ffi_name(a, path, 512) <= 0 || ffi_nlib >= FFI_LIBS) return 0;
+	h = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
+	if (!h) return 0;
+	ffi_lib[ffi_nlib++] = h;
+	return 1;
+}
+static Cell ffi_in(Cell v, int kind)	/* a vm value as the native argument */
+{
+	if (kind == 3 && v && (UCell)v < (UCell)MW * 8) return (Cell)((char *)m + v);
+	if (kind == 6 && v && (UCell)v < (UCell)MW) return (Cell)((char *)m + v * 8);
+	if (kind == 12 && v && (UCell)v + 1 < (UCell)MW) return (Cell)((char *)m + (v + 1) * 8);
+	return v;
+}
+static Cell ffi_out(Cell v, int kind)	/* a native result as a vm value */
+{
+	char *p = (char *)v, *lo = (char *)m, *hi = (char *)m + (size_t)MW * 8;
+	if (kind == 4) return (Cell)(int)v;
+	if (kind == 7) return (Cell)(uint32_t)v;
+	if (kind == 8) return (Cell)(int16_t)v;
+	if (kind == 9) return (Cell)(uint16_t)v;
+	if (kind == 10) return (Cell)(int8_t)v;
+	if (kind == 11) return (Cell)(uint8_t)v;
+	if ((kind == 3 || kind == 6) && p >= lo && p < hi) return kind == 3 ? (Cell)(p - lo) : (Cell)((p - lo) >> 3);
+	return v;
+}
+static double ffi_d(Cell v) { double d; memcpy(&d, &v, 8); return d; }
+static Cell ffi_bits(double d) { Cell v; memcpy(&v, &d, 8); return v; }
+/* a signature's arguments are nibbles after the result. code 13 (a struct in
+   registers) and code 14 (a struct in memory) start a group: the next nibble is
+   its length in eightbytes, and each eightbyte is one more nibble (1 integer,
+   2 sse). every other code is one argument. on the stack a group is one item:
+   the vm address of the struct, and its eightbytes are the words from there on.
+   every argument, a group included, is one item (one word of the stack or of a
+   call block's slots) */
+#define FFI_GROUP_REG 13
+#define FFI_GROUP_MEM 14
+static int ffi_hasgroup(Cell sig)
+{
+	int p;
+	for (p = 1; p <= 15; p++) {
+		int k = (int)((sig >> (4 * p)) & 15);
+		if (!k) break;
+		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) return 1;
+	}
+	return 0;
+}
+static int ffi_nargs(Cell sig)
+{
+	int n = 0, p = 1;
+	while (p <= 15) {
+		int k = (int)((sig >> (4 * p)) & 15);
+		if (!k) break;
+		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) {
+			int len = (int)((sig >> (4 * (p + 1))) & 15);
+			n++;
+			p += 2 + len;
+		} else { n++; p++; }
+	}
+	return n;
+}
+/* the symbol a block names. word 2 remembers which table slot answered last
+   time (plus one), so a repeated call costs one string compare. a stale
+   number from another run fails the compare and goes through the lookup */
+static void *ffi_blkfn(Cell blk)
+{
+	char name[96];
+	void *f;
+	int k;
+	if (blk < 0 || blk + 18 >= MW) return 0;
+	if (ffi_name(m[blk], name, 96) <= 0) return 0;
+	k = (int)(m[blk + 2] & 0xffffffffLL);
+	if (k > 0 && k <= ffi_nsym && !strcmp(ffi_sym[k - 1].name, name)) return ffi_sym[k - 1].p;
+	f = ffi_lookup(name);
+	for (k = 0; f && k < ffi_nsym; k++) if (ffi_sym[k].p == f && !strcmp(ffi_sym[k].name, name)) { m[blk + 2] = (m[blk + 2] & ~0xffffffffLL) | (k + 1); break; }
+	if (!f) fprintf(stderr, "zy: no native symbol %s\n", name);
+	return f;
+}
+/* a struct result. the call block's word 2 holds the host's cache in its low half
+   and the result's descriptor in its high half: the struct's length in words, then
+   one class nibble per word (1 integer, 2 float). the result's memory is the first
+   item of the call (the stub makes it), and the words go there. which words travel
+   in memory is the host's rule (ffi_retmem); the rest come back in registers, and
+   the c function type that returns the matching struct gets them there */
+typedef struct { Cell a, b; } FfiSII;
+typedef struct { double a, b; } FfiSDD;
+typedef struct { Cell a; double b; } FfiSID;
+typedef struct { double a; Cell b; } FfiSDI;
+typedef struct { double d[1]; } FfiSF1;
+typedef struct { double d[2]; } FfiSF2;
+typedef struct { double d[3]; } FfiSF3;
+typedef struct { double d[4]; } FfiSF4;
+typedef struct { Cell w[7]; } FfiSBIG;
+typedef FfiSII (*FfiRII)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSDD (*FfiRDD)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSID (*FfiRID)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSDI (*FfiRDI)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSF1 (*FfiRF1)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSF2 (*FfiRF2)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSF3 (*FfiRF3)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSF4 (*FfiRF4)(FFI_IT, FFI_DT, FFI_ST, ...);
+typedef FfiSBIG (*FfiRBIG)(FFI_IT, FFI_DT, FFI_ST, ...);
+#define FFI_CALL(T) ((T)f)(FFI_IC(iv), dv[0], dv[1], dv[2], dv[3], dv[4], dv[5], dv[6], dv[7], sv[0], sv[1], sv[2], sv[3], sv[4], sv[5], sv[6], sv[7], sv[8], sv[9], sv[10], sv[11], sv[12], sv[13], sv[14])
+/* does the host send this result through memory? sysv: more than two words. aapcs64:
+   more than two words unless it is a float struct of up to four words, which comes
+   back in the v registers, and an integer struct of up to two */
+static int ffi_retmem(Cell rdesc)
+{
+	int n = (int)(rdesc & 15), g;
+	if (n <= 2) return 0;
+#if defined(__aarch64__)
+	if (n > 4) return 1;
+	for (g = 0; g < n; g++) if (((rdesc >> (4 + 4 * g)) & 15) != 2) return 1;
+	return 0;
+#else
+	return 1;
+#endif
+}
+/* the call that returns a struct: the words land in m at res */
+static void ffi_call_ret(void *f, Cell rdesc, Cell *iv, double *dv, Cell *sv, Cell res)
+{
+	int n = (int)(rdesc & 15), g, c[8], hfa = 1;
+	for (g = 0; g < n; g++) { c[g] = (int)((rdesc >> (4 + 4 * g)) & 15); if (c[g] != 2) hfa = 0; }
+#if defined(__x86_64__) || defined(__aarch64__)
+	if (ffi_retmem(rdesc)) {
+#if defined(__x86_64__)
+		FFI_CALL(FfiI);	/* iv[0] is the hidden pointer (ffi_invoke set it); the callee writes there */
+#else
+		{ FfiSBIG s = FFI_CALL(FfiRBIG); for (g = 0; g < n; g++) m[res + g] = s.w[g]; }
+#endif
+		return;
+	}
+#if defined(__aarch64__)
+	if (hfa && n == 1) { FfiSF1 s = FFI_CALL(FfiRF1); m[res] = ffi_bits(s.d[0]); }
+	else if (hfa && n == 2) { FfiSF2 s = FFI_CALL(FfiRF2); m[res] = ffi_bits(s.d[0]); m[res + 1] = ffi_bits(s.d[1]); }
+	else if (hfa && n == 3) { FfiSF3 s = FFI_CALL(FfiRF3); for (g = 0; g < 3; g++) m[res + g] = ffi_bits(s.d[g]); }
+	else if (hfa && n == 4) { FfiSF4 s = FFI_CALL(FfiRF4); for (g = 0; g < 4; g++) m[res + g] = ffi_bits(s.d[g]); }
+	else { FfiSII s = FFI_CALL(FfiRII); m[res] = s.a; if (n > 1) m[res + 1] = s.b; }
+#else
+	if (n == 1) { if (c[0] == 2) m[res] = ffi_bits(FFI_CALL(FfiD)); else m[res] = FFI_CALL(FfiI); }
+	else if (c[0] == 1 && c[1] == 1) { FfiSII s = FFI_CALL(FfiRII); m[res] = s.a; m[res + 1] = s.b; }
+	else if (c[0] == 2 && c[1] == 2) { FfiSDD s = FFI_CALL(FfiRDD); m[res] = ffi_bits(s.a); m[res + 1] = ffi_bits(s.b); }
+	else if (c[0] == 1) { FfiSID s = FFI_CALL(FfiRID); m[res] = s.a; m[res + 1] = ffi_bits(s.b); }
+	else { FfiSDI s = FFI_CALL(FfiRDI); m[res] = ffi_bits(s.a); m[res + 1] = s.b; }
+#endif
+#else
+	(void)f; (void)rdesc; (void)iv; (void)dv; (void)sv; (void)res; (void)hfa;
+	fprintf(stderr, "zy: a struct result needs x86-64 or arm64 in this build\n");
+#endif
+}
+/* call f with the vm values av, by the signature. integer-class arguments take
+   the integer registers in turn, fp ones the fp registers, and what is left
+   over lands in the stack words in order (a float32 is the low half of a
+   word, as it would sit in a register) */
+/* darwin puts unnamed arguments on the stack. these standard stdio entry
+   points have known fixed prefixes even when an extern uses a zc alias. */
+static int ffi_varargs(void *f)
+{
+#if defined(__APPLE__) && defined(__aarch64__)
+	static const char *names[] = { "printf", "fprintf", "sprintf", "snprintf" };
+	static const int fixed[] = { 1, 2, 2, 3 };
+	static void *entry[4]; static int ready;
+	int i;
+	if (!ready) { for (i = 0; i < 4; i++) entry[i] = dlsym(RTLD_DEFAULT, names[i]); ready = 1; }
+	for (i = 0; i < 4; i++) if (f == entry[i]) return fixed[i];
+#else
+	(void)f;
+#endif
+	return -1;
+}
+static Cell ffi_invoke(void *f, Cell sig, const Cell *av, Cell rdesc)
+{
+	Cell iv[8] = {0, 0, 0, 0, 0, 0, 0, 0}, sv[FFI_NS] = {0, 0, 0, 0, 0, 0, 0, 0, 0}, r, w;
+	Cell cbuf[225];	/* copies of the structs passed by reference (aapcs64) */
+	int nc = 0;
+	double dv[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+	int ni = 0, nd = 0, ns = 0, i = 0, j = 0, p, k, rk = (int)(sig & 15), fixed = ffi_varargs(f);
+	int sret = rk == FFI_GROUP_REG || rk == FFI_GROUP_MEM, rn = (int)(rdesc & 15);
+	Cell res = 0;
+	if (sret) {
+		if (rn < 1 || rn > 7) return -1;
+		res = av[0];
+		if (res < 0 || res + rn > MW) return -1;
+		j = 1;
+#if defined(__x86_64__)
+		if (ffi_retmem(rdesc)) { iv[0] = (Cell)(intptr_t)(m + res); ni = 1; }
+#endif
+	}
+	for (p = 1; p <= 15; p++) {
+		k = (int)((sig >> (4 * p)) & 15);
+		if (!k) break;
+		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) {
+			/* a struct: one item, its vm address. its eightbytes take registers only as
+			   a whole, else all of them go on the stack, in order (the sysv rule). only
+			   the sysv classes are done so far */
+#if defined(__x86_64__) || defined(__aarch64__)
+			int len = (int)((sig >> (4 * (p + 1))) & 15), g, need_i = 0, need_d = 0;
+			Cell base = av[j];
+			if (base < 0 || base + len > MW || len < 1 || len > 15) return -1;
+			for (g = 0; g < len; g++) {
+				if (((sig >> (4 * (p + 2 + g))) & 15) == 2) need_d++; else need_i++;
+			}
+#if defined(__aarch64__)
+			/* aapcs64: a homogeneous float struct of up to four words goes in the v
+			   registers, one word each; another struct of two words or less goes in
+			   the x registers; a bigger one is passed by reference (a copy, whose
+			   address is the integer argument). a struct that does not fit takes the
+			   rest of its register class, so the later arguments go on the stack */
+			if (need_i == 0 && len <= 4) {
+				if (nd + len <= 8) { for (g = 0; g < len; g++) dv[nd++] = ffi_d(m[base + g]); }
+				else { nd = 8; for (g = 0; g < len; g++) { if (ns >= FFI_NS) return -1; sv[ns++] = m[base + g]; } }
+			} else if (len <= 2) {
+				if (ni + len <= FFI_NI) { for (g = 0; g < len; g++) iv[ni++] = m[base + g]; }
+				else { ni = FFI_NI; for (g = 0; g < len; g++) { if (ns >= FFI_NS) return -1; sv[ns++] = m[base + g]; } }
+			} else {
+				Cell *cp = cbuf + nc;
+				for (g = 0; g < len; g++) { if (nc >= 225) return -1; cbuf[nc++] = m[base + g]; }
+				if (ni < FFI_NI) iv[ni++] = (Cell)(intptr_t)cp;
+				else if (ns < FFI_NS) sv[ns++] = (Cell)(intptr_t)cp;
+				else return -1;
+			}
+#else
+			if (k == FFI_GROUP_REG && ni + need_i <= FFI_NI && nd + need_d <= 8) {
+				for (g = 0; g < len; g++) {
+					if (((sig >> (4 * (p + 2 + g))) & 15) == 2) dv[nd++] = ffi_d(m[base + g]);
+					else iv[ni++] = m[base + g];
+				}
+			} else {
+				for (g = 0; g < len; g++) {
+					if (ns >= FFI_NS) return -1;
+					sv[ns++] = m[base + g];
+				}
+			}
+#endif
+			j++;
+			p += 1 + len;
+			continue;
+#else
+			fprintf(stderr, "zy: a struct by value needs x86-64 or arm64 in this build\n");
+			return -1;
+#endif
+		}
+		if (fixed >= 0 && p > fixed) {
+			if (ns >= FFI_NS) return -1;
+			if (k == 5) w = ffi_bits((double)(float)ffi_d(av[j]));
+			else if (k == 2) w = av[j];
+			else w = ffi_in(av[j], k);
+			sv[ns++] = w; j++; continue;
+		}
+		if (k == 2 || k == 5) {
+			if (k == 5) {
+				float fl = (float)ffi_d(av[j]);
+				uint32_t u;
+				memcpy(&u, &fl, 4);
+				w = (Cell)u;
+			} else w = av[j];
+			if (nd < 8) dv[nd++] = ffi_d(w);
+			else if (ns < FFI_NS) sv[ns++] = w;
+			else return -1;
+		} else {
+			w = ffi_in(av[j], k);
+			if (ni < FFI_NI) iv[ni++] = w;
+			else if (ns < FFI_NS) sv[ns++] = w;
+			else return -1;
+		}
+		j++;
+	}
+	(void)i;
+	if (sret) { ffi_call_ret(f, rdesc, iv, dv, sv, res); return 0; }
+	if (rk == 2) r = ffi_bits(FFI_CALL(FfiD));
+	else if (rk == 5) r = ffi_bits((double)FFI_CALL(FfiF));
+	else {
+		r = FFI_CALL(FfiI);
+		r = rk ? ffi_out(r, rk) : 0;
+	}
+	return r;
+}
+static Cell ffi_call(Cell blk)
+{
+	void *f = ffi_blkfn(blk);
+	Cell av[15];
+	int i;
+	if (!f) return -1;
+	for (i = 0; i < 15; i++) av[i] = m[blk + 3 + i];
+	return ffi_invoke(f, m[blk + 1], av, (Cell)((UCell)m[blk + 2] >> 32));
+}
+/* opcode 620 on the data stack: blk on top, then the arguments, the last one
+   next. returns the new stack top, with the result on it. cb_r has to be set
+   by the caller (the interpreter's r, an llvm block's r) */
+static Cell ncall_exec(Cell s)
+{
+	Cell blk, sig, av[15], res = -1;
+	int n, i;
+	void *f;
+	if ((UCell)s >= (UCell)MW) return s;
+	blk = m[s];
+	if (blk < 0 || blk + 18 >= MW) return s + 1;
+	sig = m[blk + 1];
+	n = ffi_nargs(sig);
+	if (s + n >= MW) return s + 1;
+	for (i = 0; i < n; i++) av[i] = m[s + n - i];
+	f = ffi_blkfn(blk);
+	cb_s = s + n + 1;
+	if ((sig & 15) >= 13) res = -1;	/* a struct result is only made by the stub's syscall */
+	else if (f) res = ffi_invoke(f, sig, av, 0);
+	m[s + n] = res;
+	return s + n;
+}
+/* the jit's form: sp and rp are host pointers into m */
+static long ncall_host(long sp, long rp)
+{
+	Cell s = ((Cell *)sp - m);
+	cb_r = (Cell *)rp - m;
+	return (long)(m + ncall_exec(s));
+}
+
+/* callbacks. each slot is a native function of the c shapes (the integer and
+   fp registers and nine stack words in, one result out), one set per result
+   class. calling it pushes the arguments on the vm data stack as vm code
+   expects them, pushes a return address that stops the nested run, runs the
+   vm function and pops its result. */
+#define FFI_CBS 16
+static struct { Cell entry, sig; int used; } ffi_cb[3][FFI_CBS];
+static Cell ffi_enter(int cls, int k, Cell *iv, double *dv, Cell *sv)
+{
+	Cell s = cb_s, r = cb_r, sig = ffi_cb[cls][k].sig, res = 0, sent = 1, w;
+	int i, kind, ni = 0, nd = 0, ns = 0, n = 0, back;
+	Cell args[16];
+	if (halt_at >= 0 || (unsigned)(s - 20) >= (unsigned)MW || (unsigned)r >= (unsigned)MW) return 0;
+	for (i = 0; i < 15; i++) {
+		kind = (int)((sig >> (4 * (i + 1))) & 15);
+		if (!kind) break;
+		if (kind == 2 || kind == 5) {
+			w = nd < 8 ? ffi_bits(dv[nd++]) : ns < FFI_NS ? sv[ns++] : 0;
+			if (kind == 5) {
+				uint32_t u = (uint32_t)w;
+				float fl;
+				memcpy(&fl, &u, 4);
+				w = ffi_bits((double)fl);
+			}
+			args[n++] = w;
+		} else {
+			w = ni < FFI_NI ? iv[ni++] : ns < FFI_NS ? sv[ns++] : 0;
+			args[n++] = ffi_out(w, kind);
+		}
+	}
+	for (i = 0; i < n; i++) m[--s] = args[i];
+	m[r++] = sent;
+	back = vm_until((int)ffi_cb[cls][k].entry, (int)sent, -1, 0, &s, &r);
+	if (back < 0 || (unsigned)back >= (unsigned)code_words) halt_at = back < 0 ? PC_HALT : back;
+	if (halt_at >= 0) return 0;
+	if ((unsigned)s < (unsigned)MW) res = m[s++];
+	cb_s = s; cb_r = r;
+	kind = (int)(sig & 15);
+	if (cls == 0) return kind ? ffi_in(res, kind) : 0;
+	return res;
+}
+#define FFI_ARGS FFI_IN, double x0, double x1, double x2, double x3, double x4, double x5, double x6, double x7, Cell s0, Cell s1, Cell s2, Cell s3, Cell s4, Cell s5, Cell s6, Cell s7, Cell s8
+#define FFI_PACK Cell iv[8] = {FFI_IV}; double dv[8] = {x0, x1, x2, x3, x4, x5, x6, x7}; Cell sv[FFI_NS] = {s0, s1, s2, s3, s4, s5, s6, s7, s8}
+#define FFI_SLOT(K) \
+	static Cell ffi_i##K(FFI_ARGS) { FFI_PACK; return ffi_enter(0, K, iv, dv, sv); } \
+	static double ffi_d##K(FFI_ARGS) { FFI_PACK; return ffi_d(ffi_enter(1, K, iv, dv, sv)); } \
+	static float ffi_f##K(FFI_ARGS) { FFI_PACK; return (float)ffi_d(ffi_enter(2, K, iv, dv, sv)); }
+#define FFI_ALL(X) X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15)
+FFI_ALL(FFI_SLOT)
+#define FFI_PICK(K) (void *)ffi_i##K,
+#define FFI_PICKD(K) (void *)ffi_d##K,
+#define FFI_PICKF(K) (void *)ffi_f##K,
+static void *ffi_fn[3][FFI_CBS] = { { FFI_ALL(FFI_PICK) }, { FFI_ALL(FFI_PICKD) }, { FFI_ALL(FFI_PICKF) } };
+static Cell ffi_cbmake(Cell entry, Cell sig)
+{
+	int cls = (sig & 15) == 2 ? 1 : (sig & 15) == 5 ? 2 : 0, k;
+	for (k = 0; k < FFI_CBS; k++)
+		if (ffi_cb[cls][k].used && ffi_cb[cls][k].entry == entry && ffi_cb[cls][k].sig == sig) return (Cell)ffi_fn[cls][k];
+	for (k = 0; k < FFI_CBS; k++)
+		if (!ffi_cb[cls][k].used) {
+			ffi_cb[cls][k].used = 1; ffi_cb[cls][k].entry = entry; ffi_cb[cls][k].sig = sig;
+			return (Cell)ffi_fn[cls][k];
+		}
+	return 0;
+}
+/* a native call the o3 lifter has resolved: the micro-ops narg and ncall carry
+   an index into this table. imm of narg is the argument's position */
+typedef struct { void *f; Cell sig; int n; char name[48]; } Ncd;
+static Ncd ncd[256];
+static int nncd;
+static int ncd_get(Cell blk)
+{
+	void *f;
+	Cell sig;
+	char name[96];
+	int i;
+	if (blk < 0 || blk + 18 >= MW) return -1;
+	if (ffi_name(m[blk], name, 96) <= 0) return -1;
+	f = ffi_lookup(name);
+	if (!f) return -1;
+	sig = m[blk + 1];
+	if (ffi_hasgroup(sig) || (sig & 15) >= 13) return -1;	/* the o3 lifter reads one nibble per argument; a struct goes through ncall_exec */
+	for (i = 0; i < nncd; i++) if (ncd[i].f == f && ncd[i].sig == sig) return i;
+	if (nncd >= 256) return -1;
+	ncd[nncd].f = f; ncd[nncd].sig = sig; ncd[nncd].n = ffi_nargs(sig);
+	strncpy(ncd[nncd].name, name, 47);
+	ncd[nncd].name[47] = 0;
+	return nncd++;
+}
+#else
+static Cell ffi_call(Cell blk) { return -1; }
+static Cell ffi_find(Cell name) { return 0; }
+static Cell ffi_cbmake(Cell entry, Cell sig) { return 0; }
+static Cell ffi_load(Cell name) { return 0; }
+static Cell ncall_exec(Cell s)
+{
+	Cell blk, sig;
+	int n = 0;
+	if ((UCell)s >= (UCell)MW) return s;
+	blk = m[s];
+	if (blk < 0 || blk + 18 >= MW) return s + 1;
+	sig = m[blk + 1];
+	while (n < 15 && ((sig >> (4 * (n + 1))) & 15)) n++;
+	m[s + n] = -1;
+	return s + n;
+}
+static long ncall_host(long sp, long rp) { return (long)(m + ncall_exec((Cell *)sp - m)); }
+static int ncd_get(Cell blk) { return -1; }
+#endif
+
+static void rforget(Rg *R);		/* the backend drops its code */
+static void rmark(Rg *R);		/* the backend watches the region's words */
+static void rcompile(Rg *R, int eager);
+
+static int nargs(int op) { return op >= U_N ? 0 : uarity[op]; }
+static int op_fpdest(int op)
+{
+	return op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_I2F || op == U_S2F;
+}
+/* operand 0 is a, 1 is b. comparisons read floats and write an integer. */
+static int op_fpuse(int op, int which)
+{
+	if (op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_FEQ || op == U_FLT || op == U_FLE) return 1;
+	if ((op == U_F2I || op == U_F2S) && which == 0) return 1;
+	return 0;
+}
+/* 1 when the value should stay in an FP register: a float result, a float
+   variable, or a load/constant that is only ever used as a float. a use that
+   also wants an integer (a branch, an int variable) keeps the bits in a GPR
+   and the float use converts at that instruction. */
+static void class_fp(Rg *R, RB *B, unsigned char *fp)
+{
+	int i, j, *fu, *gu, n = B->nv;
+	fu = calloc(n + 1, sizeof(int));
+	gu = calloc(n + 1, sizeof(int));
+	memset(fp, 0, n);
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		int na = nargs(u->op);
+		if (na > 0) { if (op_fpuse(u->op, 0)) fu[u->a]++; else gu[u->a]++; }
+		if (na > 1) { if (op_fpuse(u->op, 1)) fu[u->b]++; else gu[u->b]++; }
+		if (na > 2) gu[u->c]++;
+	}
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
+		if (R->fv[j]) fu[B->vs[j]]++; else gu[B->vs[j]]++;
+	}
+	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) gu[B->cv]++;
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		int d = u->d;
+		if (is_store(u->op) || u->op == U_NARG) continue;
+		if (u->op == U_VGET) { if ((unsigned)u->imm < RMAXV && R->fv[u->imm]) fp[d] = 1; continue; }
+		if (op_fpdest(u->op) || (fu[d] && !gu[d])) fp[d] = 1;
+	}
+	free(fu); free(gu);
+}
+static void rclear(Rg *R)
+{
+	int b;
+	if (R->marked) { runmark(R); R->marked = 0; }
+	memset(R->dep, 0, sizeof R->dep);
+	rforget(R);
+	for (b = 0; b < R->nb; b++) { free(R->b[b].u); free(R->b[b].so); }
+	free(R->b); R->b = 0; R->nb = 0; R->lifted = 0;
+	R->gen++;
+}
+static void rdrop_all(void) { int j; for (j = 0; j < nrg; j++) if (rgs[j]->lifted) rclear(rgs[j]); }
+
+/* lifting one block */
+static Rg *R_;
+static unsigned char *lead_;
+static int *bidx_, *wl_, nw_;
+static int lv_[RMAXV], lw_[RMAXV];
+#define SB 1100
+static int skv[2 * SB], stp, slo, shi;
+static unsigned char skk[2 * SB];
+static int afp[MAXUOP * 4 + 16];	/* value -> 1 + its offset from fp, if it is fp + k */
+static int emap[MAXUOP * 4 + 16];
+
+static int rvar(int v)
+{
+	if (lv_[v] < 0) { lv_[v] = raw(U_VGET, 0, 0, 0, v); if (!v) afp[lv_[v]] = 1; }
+	return lv_[v];
+}
+static int slotvar(int off) { int j; for (j = 0; j < R_->np; j++) if (R_->ps[j] == off) return R_->pv[j]; return 0; }
+static int rbin(int op, int x, int y, int z)
+{
+	int r = oe(op, x, y, z, 0);
+	if (op == U_ADD && !afp[r]) {
+		if (afp[x] && isk[y]) afp[r] = afp[x] + kv[y];
+		else if (afp[y] && isk[x]) afp[r] = afp[y] + kv[x];
+	}
+	return r;
+}
+static int rld(int a)
+{
+	int v;
+	if (isk[a] && kv[a] == ZFP) return rvar(0);
+	if (afp[a] && (v = slotvar(afp[a] - 1))) return rvar(v);
+	return oe(U_LD, a, 0, 0, 0);
+}
+static int rst(int a, int b)
+{
+	int v;
+	if (isk[a] && kv[a] == ZFP) return 0;	/* moving the frame: not in here */
+	if (afp[a] && (v = slotvar(afp[a] - 1))) { lv_[v] = b; lw_[v] = 1; return 1; }
+	oe(U_ST, a, b, 0, 0);
+	return 1;
+}
+static int sget(int o)
+{
+	if (o < -SB || o >= SB) { uerr = -6; return 0; }
+	if (!skk[o + SB]) {
+		if (o < 0) { uerr = -6; return 0; }
+		skv[o + SB] = raw(U_SLD, 0, 0, 0, o); skk[o + SB] = 1;
+	}
+	if (o < slo) slo = o;
+	if (o >= shi) shi = o + 1;
+	return skv[o + SB];
+}
+static void sset(int o, int v)
+{
+	if (o < -SB || o >= SB) { uerr = -6; return; }
+	skv[o + SB] = v; skk[o + SB] = 1;
+	if (o < slo) slo = o;
+	if (o >= shi) shi = o + 1;
+}
+static int spop(void) { return sget(stp++); }
+static void spush(int v) { sset(--stp, v); }
+static int rblk(int t) { return t >= R_->lo && t < R_->hi && lead_[t - R_->lo] ? bidx_[t - R_->lo] : -1; }
+/* an enhancement, spliced into the block */
+static int rsplice(Enh *e)
+{
+	int i, j;
+	for (j = e->nin; j--;) emap[j] = spop();
+	for (i = 0; i < e->n && !uerr; i++) {
+		Uop *u = &e->u[i];
+		int na = nargs(u->op), a = na > 0 ? emap[u->a] : 0, b = na > 1 ? emap[u->b] : 0, c = na > 2 ? emap[u->c] : 0;
+		if (u->op == U_LI) emap[u->d] = K(u->imm);
+		else if (u->op == U_LD) emap[u->d] = rld(a);
+		else if (u->op == U_ST) { if (!rst(a, b)) uerr = -4; }
+		else emap[u->d] = rbin(u->op, a, b, c);
+	}
+	for (j = 0; j < e->nout; j++) spush(emap[e->out[j]]);
+	return uerr;
+}
+static int lift_block(int bi)
+{
+	static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
+	RB *B = &R_->b[bi];
+	int i = B->a, k, x, y, t, j, o, n = 0, *live;
+	Cell w;
+	on = 0; onv = 0; uerr = 0;
+	memset(afp, 0, sizeof afp); memset(skk, 0, sizeof skk);
+	stp = slo = shi = 0;
+	for (j = 0; j < RMAXV; j++) { lv_[j] = -1; lw_[j] = 0; }
+	B->term = T_JMP; B->cv = -1; B->s1 = B->s2 = -1; B->x1 = B->x2 = 0;
+	for (;;) {
+		if (uerr) return uerr;
+		if (i != B->a && (t = rblk(i)) >= 0) { B->term = T_BR; B->s1 = t; goto done; }
+		if (i < R_->lo || i >= R_->hi) goto leave_at;
+		w = m[i];
+		if (w & 1) { spush(K(w >> 1)); i++; continue; }
+		k = (int)(w >> 1);
+		switch (k) {
+		case 0: B->cv = K(-1); goto done;	/* halt */
+		case 1: x = spop(); spush(rld(x)); break;
+		case 2:
+			x = sget(stp);
+			if (isk[x] && kv[x] == ZFP) goto leave_at;
+			x = spop(); y = spop(); rst(x, y);
+			break;
+		case 3:	/* pick: a constant depth */
+			x = sget(stp);
+			if (!isk[x] || kv[x] < 0 || kv[x] > 1024) goto leave_at;
+			stp++; spush(sget(stp + kv[x]));
+			break;
+		case 4: stp++; break;
+		case 5: x = sget(stp); y = sget(stp + 1); sset(stp, y); sset(stp + 1, x); break;
+		case 6: B->cv = spop(); B->term = T_CALL; B->x2 = i + 1; goto done;
+		case 7:
+			x = spop();
+			if (isk[x] && (t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; }
+			else B->cv = x;
+			goto done;
+		case 8:
+			x = sget(stp);
+			if (!isk[x]) goto leave_at;
+			stp++; y = spop();
+			if (isk[y]) {	/* a known condition */
+				if (kv[y]) { i++; continue; }
+				if ((t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; } else B->cv = K(kv[x]);
+				goto done;
+			}
+			B->term = T_CBR; B->cv = y;
+			B->x1 = kv[x]; B->s1 = rblk(kv[x]);
+			B->x2 = i + 1; B->s2 = rblk(i + 1);
+			goto done;
+		case 9: B->term = T_RET; goto done;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
+			x = spop(); y = spop(); spush(rbin(map[k - 13], y, x, 0));
+			break;
+		case 600:	/* flit: the next word is the raw bits, even when it looks like halt */
+			if (i + 1 >= code_words) goto leave_at;
+			spush(K(m[i + 1]));
+			i += 2;
+			continue;
+		case 601: case 602: case 603: case 604: case 605: case 606: case 607: {
+			static const int fmap[] = { U_FADD, U_FSUB, U_FMUL, U_FDIV, U_FEQ, U_FLT, U_FLE };
+			x = spop(); y = spop(); spush(rbin(fmap[k - 601], y, x, 0));
+			break; }
+		case 608: x = spop(); spush(rbin(U_I2F, x, 0, 0)); break;
+		case 609: x = spop(); spush(rbin(U_F2I, x, 0, 0)); break;
+		case 620: {	/* a native call: the block address is a constant, so the callee and its signature are known now */
+#ifdef ZYJIT
+			goto leave_at;
+#else
+			int di, na, av[15];
+			x = sget(stp);
+			if (!isk[x] || (di = ncd_get(kv[x])) < 0) goto leave_at;
+			na = ncd[di].n;
+			stp++;
+			for (j = na; j-- > 0;) av[j] = spop();
+			for (j = 0; j < na; j++) raw(U_NARG, av[j], 0, 0, j);
+			spush(raw(U_NCALL, 0, 0, 0, di));
+			break;
+#endif
+		}
+		case 612: x = spop(); spush(raw(U_LD8, x, 0, 0, 0)); break;
+		case 614: x = spop(); spush(raw(U_LD16, x, 0, 0, 0)); break;
+		case 616: x = spop(); spush(raw(U_LD32, x, 0, 0, 0)); break;
+		case 613: x = spop(); y = spop(); raw(U_ST8, x, y, 0, 0); break;
+		case 615: x = spop(); y = spop(); raw(U_ST16, x, y, 0, 0); break;
+		case 617: x = spop(); y = spop(); raw(U_ST32, x, y, 0, 0); break;
+		default:
+			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { R_->dep[(k - XOP0) >> 3] |= 1 << ((k - XOP0) & 7); if (rsplice(&enh[k - XOP0])) return uerr; break; }
+			goto leave_at;	/* i/o, install, unwired: done the usual way */
+		}
+		i++;
+	}
+leave_at:
+	B->term = T_JMP; B->cv = K(i);
+done:
+	if (uerr) return uerr;
+	B->delta = stp;
+	B->so = malloc((shi - slo + 1) * 2 * sizeof(int));
+	B->nso = 0;
+	for (o = stp > slo ? stp : slo; o < shi; o++)
+		if (skk[o + SB]) {
+			x = skv[o + SB];
+			if (ob[x].op == U_SLD && ob[x].imm == o) continue;	/* still the word that was there */
+			if (afp[x] && slotvar(afp[x] - 1)) return -7;	/* a register var's address can't wait in memory */
+			B->so[2 * B->nso] = o; B->so[2 * B->nso + 1] = x; B->nso++;
+		}
+	for (j = 0; j < RMAXV; j++) B->vs[j] = -1;
+	for (j = 1; j < R_->nv; j++)
+		if (lw_[j] && !(ob[lv_[j]].op == U_VGET && ob[lv_[j]].imm == j)) B->vs[j] = lv_[j];
+	/* dead code: what the block leaves behind and its stores are live */
+	live = calloc(onv + 1, sizeof *live);
+	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
+	for (j = 1; j < R_->nv; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) live[B->cv] = 1;
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
+		k = nargs(u->op);
+		if (k > 0) live[u->a] = 1;
+		if (k > 1) live[u->b] = 1;
+		if (k > 2) live[u->c] = 1;
+	}
+	free(live);
+	B->u = malloc((on + 1) * sizeof(Uop));
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) B->u[n++] = ob[i];
+	B->n = n; B->nv = onv;
+	if (onv > R_->maxnv) R_->maxnv = onv;
+	return 0;
+}
+/* ---- shape: empty jumps, straight-line merges, rotated loops ----
+   the for-loop the compiler emits is a test, an empty jump, a body and a
+   step: four dispatches per iteration. fold the empty jump, paste a block
+   onto its only predecessor, and rotate a straight loop so the hot path is
+   one block that branches to itself. a call, a return, a stack word, or a
+   second branch stays where it is. an entry is never pasted over. */
+static int bempty(Rg *R, int b)
+{
+	RB *B = &R->b[b];
+	int j;
+	if (B->n || B->nso || B->delta || B->term != T_BR) return 0;
+	if (B->s1 < 0 || B->s1 == b) return 0;
+	for (j = 0; j < R->nv; j++) if (B->vs[j] >= 0) return 0;
+	return 1;
+}
+static int bfollow(Rg *R, int b)
+{
+	unsigned char *seen = calloc(R->nb, 1);
+	int n = 0;
+	if (!seen) return b;
+	while ((unsigned)b < (unsigned)R->nb && bempty(R, b) && !seen[b] && n++ < R->nb) {
+		seen[b] = 1;
+		b = R->b[b].s1;
+	}
+	free(seen);
+	return b;
+}
+static int breach(Rg *R, unsigned char *reach)
+{
+	int *st = malloc((R->nb + 1) * sizeof(int)), n = 0, e, b;
+	if (!st) return -1;
+	for (e = 0; e < R->ne; e++) {
+		b = R->entb[e];
+		if ((unsigned)b < (unsigned)R->nb && !reach[b]) { reach[b] = 1; st[n++] = b; }
+	}
+	while (n) {
+		RB *B;
+		b = st[--n];
+		B = &R->b[b];
+		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb && !reach[B->s1]) {
+			reach[B->s1] = 1; st[n++] = B->s1;
+		}
+		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb && !reach[B->s2]) {
+			reach[B->s2] = 1; st[n++] = B->s2;
+		}
+	}
+	free(st);
+	return 0;
+}
+static int rempty(Rg *R)
+{
+	int b, e, ch = 0, *to = malloc(R->nb * sizeof(int));
+	if (!to) return 0;
+	for (b = 0; b < R->nb; b++) {
+		int d = bfollow(R, b);
+		to[b] = (d != b && (unsigned)d < (unsigned)R->nb && !bempty(R, d)) ? d : b;
+	}
+	for (b = 0; b < R->nb; b++) {
+		RB *B = &R->b[b];
+		if ((unsigned)B->s1 < (unsigned)R->nb && to[B->s1] != B->s1) { B->s1 = to[B->s1]; ch = 1; }
+		if ((unsigned)B->s2 < (unsigned)R->nb && to[B->s2] != B->s2) { B->s2 = to[B->s2]; ch = 1; }
+	}
+	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb && to[R->entb[e]] != R->entb[e]) {
+		R->entb[e] = to[R->entb[e]]; ch = 1;
+	}
+	free(to);
+	return ch;
+}
+static int bmapat(int *map, int nv, int x, int *ok)
+{
+	if ((unsigned)x >= (unsigned)nv || map[x] < 0) { *ok = 0; return 0; }
+	return map[x];
+}
+/* paste the blocks in order. a VGET of a var this paste already wrote is the
+   written value, not a fresh read of the block's entry. the last block's
+   branch is the branch of the result. */
+static int rb_build(Rg *R, int *ix, int nix, RB *dst)
+{
+	int cap = 0, i, j, n = 0, nid = 0, cur[RMAXV], wrote[RMAXV];
+	Uop *o;
+	for (i = 0; i < nix; i++) {
+		if ((unsigned)ix[i] >= (unsigned)R->nb) return -1;
+		cap += R->b[ix[i]].n;
+	}
+	if (cap > 2048 || !nix) return -1;
+	o = malloc((cap + 1) * sizeof(Uop));
+	if (!o) return -1;
+	for (j = 0; j < RMAXV; j++) { cur[j] = -1; wrote[j] = 0; }
+	for (i = 0; i < nix; i++) {
+		RB *S = &R->b[ix[i]];
+		int nvmap = S->nv > 0 ? S->nv : 1, k, ok;
+		int *map = malloc(nvmap * sizeof(int));
+		if (!map) { free(o); return -1; }
+		for (k = 0; k < nvmap; k++) map[k] = -1;
+		for (k = 0; k < S->n; k++) {
+			Uop u = S->u[k];
+			int na = nargs(u.op);
+			ok = 1;
+			if (u.op == U_VGET && (unsigned)u.imm < RMAXV && cur[u.imm] >= 0) {
+				if ((unsigned)u.d >= (unsigned)nvmap) { free(map); free(o); return -1; }
+				map[u.d] = cur[u.imm];
+				continue;
+			}
+			if (na > 0) u.a = bmapat(map, nvmap, u.a, &ok);
+			if (na > 1) u.b = bmapat(map, nvmap, u.b, &ok);
+			if (na > 2) u.c = bmapat(map, nvmap, u.c, &ok);
+			if (!ok || n >= cap) { free(map); free(o); return -1; }
+			if (!is_store(u.op)) {
+				if ((unsigned)u.d >= (unsigned)nvmap || nid >= 4096) { free(map); free(o); return -1; }
+				map[u.d] = nid;
+				u.d = nid++;
+			} else u.d = 0;
+			o[n++] = u;
+		}
+		for (j = 0; j < R->nv; j++) if (S->vs[j] >= 0) {
+			if ((unsigned)S->vs[j] >= (unsigned)nvmap || map[S->vs[j]] < 0) { free(map); free(o); return -1; }
+			cur[j] = map[S->vs[j]]; wrote[j] = 1;
+		}
+		if (i + 1 == nix) {
+			dst->term = S->term;
+			dst->s1 = S->s1; dst->s2 = S->s2; dst->x1 = S->x1; dst->x2 = S->x2;
+			if (S->cv >= 0) {
+				if ((unsigned)S->cv >= (unsigned)nvmap || map[S->cv] < 0) { free(map); free(o); return -1; }
+				dst->cv = map[S->cv];
+			} else dst->cv = -1;
+		}
+		free(map);
+	}
+	dst->u = o; dst->n = n; dst->nv = nid > 0 ? nid : 1;
+	dst->nso = 0; dst->so = 0; dst->delta = 0; dst->a = R->b[ix[0]].a;
+	for (j = 0; j < RMAXV; j++) dst->vs[j] = wrote[j] ? cur[j] : -1;
+	if (dst->nv > R->maxnv) R->maxnv = dst->nv;
+	return 0;
+}
+static void addpred(int *np, int *pr, int s, int from)
+{
+	np[s]++;
+	if (np[s] == 1) pr[s] = from;
+	else if (pr[s] != from) pr[s] = -2;
+}
+static int rmerge_one(Rg *R)
+{
+	unsigned char *reach = calloc(R->nb, 1), *ent = calloc(R->nb, 1);
+	int *np = calloc(R->nb, sizeof(int)), *pr = malloc(R->nb * sizeof(int));
+	int b, e, did = 0;
+	if (!reach || !ent || !np || !pr || breach(R, reach)) goto out;
+	for (b = 0; b < R->nb; b++) pr[b] = -1;
+	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) ent[R->entb[e]] = 1;
+	for (b = 0; b < R->nb; b++) if (reach[b]) {
+		RB *B = &R->b[b];
+		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) addpred(np, pr, B->s1, b);
+		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) addpred(np, pr, B->s2, b);
+	}
+	for (b = 0; b < R->nb && !did; b++) {
+		RB *A = &R->b[b], built;
+		int s, ix[2];
+		if (!reach[b] || A->term != T_BR) continue;
+		s = A->s1;
+		if ((unsigned)s >= (unsigned)R->nb || s == b || !reach[s] || ent[s]) continue;
+		if (np[s] != 1 || pr[s] != b) continue;
+		if (A->nso || A->delta || R->b[s].nso || R->b[s].delta) continue;
+		ix[0] = b; ix[1] = s;
+		memset(&built, 0, sizeof built);
+		if (rb_build(R, ix, 2, &built)) continue;
+		free(A->u); free(A->so);
+		built.a = A->a;
+		R->b[b] = built;
+		did = 1;
+	}
+out:
+	free(reach); free(ent); free(np); free(pr);
+	return did;
+}
+/* length of the straight run of single-predecessor jumps from s back to h,
+   or 0 when s is not that kind of back edge. the other edge may leave the
+   loop and come back in later through some other header; that is still an
+   exit of this loop, and the straight edge can be rotated. */
+static int try_chain(Rg *R, int h, int s, int *ch, int *np, int *pr, unsigned char *ent)
+{
+	int n = 0, prev = h, guard = 0;
+	if (s < 0 || s == h) return 0;
+	while (s != h) {
+		RB *B;
+		if ((unsigned)s >= (unsigned)R->nb || ++guard > R->nb) return 0;
+		B = &R->b[s];
+		if (ent[s] || np[s] != 1 || pr[s] != prev || B->nso || B->delta || B->term != T_BR) return 0;
+		ch[n++] = s;
+		prev = s;
+		s = B->s1;
+	}
+	return n;
+}
+static int rrotate_one(Rg *R)
+{
+	unsigned char *reach = calloc(R->nb, 1), *ent = calloc(R->nb, 1);
+	int *np = calloc(R->nb, sizeof(int)), *pr = malloc(R->nb * sizeof(int));
+	int *ch1 = malloc(R->nb * sizeof(int)), *ch2 = malloc(R->nb * sizeof(int));
+	int b, e, did = 0;
+	if (!reach || !ent || !np || !pr || !ch1 || !ch2 || breach(R, reach)) goto out;
+	for (b = 0; b < R->nb; b++) pr[b] = -1;
+	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) ent[R->entb[e]] = 1;
+	for (b = 0; b < R->nb; b++) if (reach[b]) {
+		RB *B = &R->b[b];
+		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) addpred(np, pr, B->s1, b);
+		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) addpred(np, pr, B->s2, b);
+	}
+	for (b = 0; b < R->nb && !did; b++) {
+		RB *H = &R->b[b], built, *grown;
+		int n1, n2, n, back, L, *ch, *ix, i;
+		if (!reach[b] || H->term != T_CBR || H->nso || H->delta || H->cv < 0) continue;
+		n1 = try_chain(R, b, H->s1, ch1, np, pr, ent);
+		n2 = try_chain(R, b, H->s2, ch2, np, pr, ent);
+		if (n2 > 0 && n1 == 0) { n = n2; ch = ch2; back = 2; }
+		else if (n1 > 0 && n2 == 0) { n = n1; ch = ch1; back = 1; }
+		else continue;
+		ix = malloc((n + 1) * sizeof(int));
+		if (!ix) continue;
+		for (i = 0; i < n; i++) ix[i] = ch[i];
+		ix[n] = b;
+		memset(&built, 0, sizeof built);
+		if (rb_build(R, ix, n + 1, &built)) { free(ix); continue; }
+		free(ix);
+		grown = realloc(R->b, (R->nb + 1) * sizeof(RB));
+		if (!grown) { free(built.u); continue; }
+		R->b = grown;
+		L = R->nb;
+		R->b[L] = built;
+		R->nb++;
+		if (back == 2) { R->b[L].s2 = L; R->b[b].s2 = L; }
+		else { R->b[L].s1 = L; R->b[b].s1 = L; }
+		did = 1;
+	}
+out:
+	free(reach); free(ent); free(np); free(pr); free(ch1); free(ch2);
+	return did;
+}
+static void rdead(Rg *R)
+{
+	unsigned char *reach = calloc(R->nb, 1);
+	int b;
+	if (!reach || breach(R, reach)) { free(reach); return; }
+	for (b = 0; b < R->nb; b++) if (!reach[b]) {
+		R->b[b].term = T_BR;
+		R->b[b].s1 = b;
+		R->b[b].s2 = -1;
+		R->b[b].cv = -1;
+	}
+	free(reach);
+}
+static void rshape(Rg *R)
+{
+	int i, guard;
+	if (R->nb < 1) return;
+	for (i = 0; i < 4; i++) if (!rempty(R)) break;
+	guard = R->nb + 4;
+	for (i = 0; i < guard; i++) if (!rmerge_one(R)) break;
+	guard = R->nb + 4;
+	for (i = 0; i < guard; i++) if (!rrotate_one(R)) break;
+	guard = R->nb + 4;
+	for (i = 0; i < guard; i++) if (!rmerge_one(R)) break;
+	for (i = 0; i < 4; i++) if (!rempty(R)) break;
+	rdead(R);
+}
+/* ---- the global pass ----
+   a block only knows its own values, but the variables carry facts from one
+   block to the next. this pass runs a forward dataflow over the blocks for the
+   one fact that pays off everywhere: a variable that holds the same constant
+   on every path into a block. such a read becomes a constant, the block is
+   emitted again through the same folding and strength reduction the lifter
+   uses (so x / step turns into a multiply once step is known, and an add of
+   two known values disappears), and a branch on a known value becomes a jump.
+   the second fact is a copy: on every path a variable holds what another
+   variable holds, and neither has been written since. a read of the copy
+   becomes a read of its source, so the copy can stop being kept at all. a
+   block that writes the source drops the copies of it. the blocks that no
+   longer have a way in are dropped by the shaping that follows, and it
+   repeats while something changes. */
+enum { C_UNSEEN, C_CONST, C_VAR, C_COPY };
+/* C_COPY: the variable holds the same value as variable v (both at the same point) */
+static int copy_ok(const Rg *R, int j, int k) { return j > 0 && j < RMAXV && k > 0 && k < R->nv && !R->fv[j] && !R->fv[k]; }
+typedef struct { unsigned char st[RMAXV]; Cell v[RMAXV]; } CIn;
+static int g_isk[MAXUOP * 4]; static Cell g_kv[MAXUOP * 4]; static int g_omap[MAXUOP + 16];
+static int cjoin(CIn *d, const CIn *s)
+{
+	int j, ch = 0;
+	for (j = 1; j < RMAXV; j++) {
+		if (s->st[j] == C_UNSEEN) continue;
+		if (d->st[j] == C_UNSEEN) { d->st[j] = s->st[j]; d->v[j] = s->v[j]; ch = 1; }
+		else if (d->st[j] != C_VAR && (s->st[j] == C_VAR || s->st[j] != d->st[j] || s->v[j] != d->v[j])) { d->st[j] = C_VAR; ch = 1; }
+	}
+	return ch;
+}
+/* does the block write variable k (a write other than the plain entry read)? */
+static int written_here(const RB *B, int k)
+{
+	if (k <= 0 || k >= RMAXV) return 1;
+	if (B->vs[k] < 0) return 0;
+	return !(B->u[B->vs[k]].op == U_VGET && B->u[B->vs[k]].imm == k);
+}
+/* what each variable holds when the block ends, given what it held at the start */
+static void cblock(Rg *R, RB *B, const CIn *in, CIn *out)
+{
+	int i, j, na;
+	unsigned char *kc = calloc(B->nv + 1, 1);
+	Cell *kw = calloc(B->nv + 1, sizeof(Cell));
+	int *cpk = calloc(B->nv + 1, sizeof(int));	/* cpk[v] = k + 1 when v is a plain read of the entry value of k */
+	*out = *in;
+	if (!kc || !kw || !cpk) { for (j = 1; j < RMAXV; j++) out->st[j] = C_VAR; free(kc); free(kw); free(cpk); return; }
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		if (is_store(u->op) || u->op == U_NARG) continue;
+		if ((unsigned)u->d >= (unsigned)B->nv) continue;
+		if (u->op == U_LI) { kc[u->d] = 1; kw[u->d] = u->imm; continue; }
+		if (u->op == U_VGET) {
+			j = (int)u->imm;
+			if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) { kc[u->d] = 1; kw[u->d] = in->v[j]; }
+			if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) cpk[u->d] = (int)in->v[j] + 1;
+			else if (j > 0 && j < RMAXV) cpk[u->d] = j + 1;
+			continue;
+		}
+		if (u->op >= U_N || is_memop(u->op)) continue;
+		na = nargs(u->op);
+		if ((na > 0 && !kc[u->a]) || (na > 1 && !kc[u->b]) || (na > 2 && !kc[u->c])) continue;
+		kc[u->d] = 1; kw[u->d] = ev(u->op, na > 0 ? kw[u->a] : 0, na > 1 ? kw[u->b] : 0, na > 2 ? kw[u->c] : 0);
+	}
+	/* a variable the block writes is a copy of k when its value is the entry value of k and k is not written here */
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
+		int w = B->vs[j], k;
+		if (kc[w] && !R->fv[j]) { out->st[j] = C_CONST; out->v[j] = kw[w]; }
+		else if (w < B->nv && cpk[w] && (k = cpk[w] - 1) != j && copy_ok(R, j, k) && !written_here(B, k)) { out->st[j] = C_COPY; out->v[j] = k; }
+		else out->st[j] = C_VAR;
+	}
+	/* a copy of a variable this block writes is gone; so is a copy whose variable the block writes */
+	for (j = 1; j < RMAXV; j++) if (out->st[j] == C_COPY && written_here(B, (int)out->v[j])) out->st[j] = C_VAR;
+	free(kc); free(kw); free(cpk);
+}
+/* emit the block again with the known variables as constants. returns 1 if it changed, -1 if it could not be done */
+static int rb_refold(Rg *R, RB *B, const CIn *in)
+{
+	int i, j, na, any = 0, n = 0, *live, *mp, bad = 0;
+	int *sisk = isk, *somap = omap; Cell *skv = kv;
+	Uop *nu;
+	for (i = 0; i < B->n; i++) if (B->u[i].op == U_VGET) {
+		j = (int)B->u[i].imm;
+		if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) any = 1;
+		if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) any = 1;
+	}
+	if (!any) return 0;
+	for (j = 0; j < B->nso; j++) if ((unsigned)B->so[2 * j + 1] >= (unsigned)B->nv) bad = 1;
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= B->nv) bad = 1;
+	if (B->cv >= B->nv) bad = 1;
+	for (i = 0; i < B->n; i++) if ((unsigned)B->u[i].d >= (unsigned)B->nv && !is_store(B->u[i].op) && B->u[i].op != U_NARG) bad = 1;
+	if (bad) return 0;
+	mp = malloc((B->nv + 1) * sizeof(int));
+	if (!mp) return 0;
+	isk = g_isk; kv = g_kv; omap = g_omap;
+	on = 0; onv = 0; uerr = 0;
+	for (i = 0; i < B->n && !uerr; i++) {
+		Uop *u = &B->u[i];
+		na = u->op >= U_N ? 0 : nargs(u->op);
+		if (u->op == U_VGET) {
+			j = (int)u->imm;
+			if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) mp[u->d] = K(in->v[j]);
+			else if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) mp[u->d] = raw(U_VGET, 0, 0, 0, in->v[j]);
+			else mp[u->d] = raw(U_VGET, 0, 0, 0, u->imm);
+		} else if (u->op == U_SLD) mp[u->d] = raw(U_SLD, 0, 0, 0, u->imm);
+		else if (u->op == U_LI) mp[u->d] = K(u->imm);
+		else if (is_store(u->op)) oe(u->op, mp[u->a], mp[u->b], 0, 0);
+		else if (u->op == U_NARG) raw(U_NARG, mp[u->a], 0, 0, u->imm);
+		else if (u->op == U_NCALL) mp[u->d] = raw(U_NCALL, 0, 0, 0, u->imm);
+		else mp[u->d] = oe(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+	}
+	if (uerr) { isk = sisk; kv = skv; omap = somap; free(mp); return 0; }
+	live = calloc(onv + 2, sizeof *live);
+	nu = malloc((on + 1) * sizeof(Uop));
+	if (!live || !nu) { free(live); free(nu); isk = sisk; kv = skv; omap = somap; free(mp); return 0; }
+	for (j = 0; j < B->nso; j++) B->so[2 * j + 1] = mp[B->so[2 * j + 1]];
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) B->vs[j] = mp[B->vs[j]];
+	if (B->cv >= 0) B->cv = mp[B->cv];
+	/* a branch on a known value is a jump, when the way it goes stays in the region; otherwise the exit */
+	if (B->term == T_CBR && B->cv >= 0 && g_isk[B->cv]) {
+		int zero = g_kv[B->cv] == 0, to = zero ? B->s1 : B->s2, x = zero ? B->x1 : B->x2;
+		if (to >= 0) { B->term = T_BR; B->s1 = to; B->s2 = -1; B->cv = -1; }
+		else { B->cv = K(x); B->term = T_JMP; B->s1 = B->s2 = -1; if (onv + 2 > 0) { free(live); live = calloc(onv + 2, sizeof *live); free(nu); nu = malloc((on + 1) * sizeof(Uop)); } }
+	}
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && ob[B->vs[j]].op == U_VGET && ob[B->vs[j]].imm == j) B->vs[j] = -1;
+	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
+	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) live[B->cv] = 1;
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
+		na = u->op >= U_N ? 0 : nargs(u->op);
+		if (is_store(u->op)) na = 2;
+		if (u->op == U_NARG) na = 1;
+		if (na > 0) live[u->a] = 1;
+		if (na > 1) live[u->b] = 1;
+		if (na > 2) live[u->c] = 1;
+	}
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) nu[n++] = ob[i];
+	free(B->u); B->u = nu; B->n = n; B->nv = onv;
+	if (onv > R->maxnv) R->maxnv = onv;
+	free(live); free(mp);
+	isk = sisk; kv = skv; omap = somap;
+	return 1;
+}
+static void rglobal(Rg *R)
+{
+	int round, b, e, s, changed, sweeps, did;
+	if (R->nb < 1 || R->nb > 600) return;
+	for (round = 0; round < 4; round++) {
+		CIn *in = calloc(R->nb, sizeof(CIn)), out;
+		unsigned char *reached = calloc(R->nb, 1);
+		if (!in || !reached) { free(in); free(reached); return; }
+		for (e = 0; e < R->ne; e++) {
+			b = R->entb[e];
+			if ((unsigned)b < (unsigned)R->nb) {
+				int j;
+				reached[b] = 1;
+				for (j = 1; j < RMAXV; j++) in[b].st[j] = C_VAR;
+			}
+		}
+		sweeps = 0; changed = 1;
+		while (changed && sweeps++ < 64) {
+			changed = 0;
+			for (b = 0; b < R->nb; b++) {
+				RB *B = &R->b[b];
+				if (!reached[b]) continue;
+				cblock(R, B, &in[b], &out);
+				if (B->term != T_BR && B->term != T_CBR) continue;
+				for (s = 0; s < 2; s++) {
+					int t = s ? B->s2 : B->s1;
+					if (s && B->term != T_CBR) break;
+					if ((unsigned)t >= (unsigned)R->nb) continue;
+					if (!reached[t]) { reached[t] = 1; in[t] = out; changed = 1; }
+					else if (cjoin(&in[t], &out)) changed = 1;
+				}
+			}
+		}
+		did = 0;
+		for (b = 0; b < R->nb; b++) if (reached[b]) did |= rb_refold(R, &R->b[b], &in[b]);
+		free(in); free(reached);
+		if (!did) return;
+		rshape(R);
+	}
+}
+/* ---- loops, shared values and dead variables ----
+   what the per-block lifter and the constant pass can't see is work that is
+   done again: an expression inside a loop whose operands don't change in it,
+   the same expression (or the same load, when nothing stored in between) in
+   two blocks one of which always runs before the other, and a variable that
+   is written and then written again before anyone reads it. blocks share
+   only variables, so a value that has to cross a block edge needs one: the
+   passes below take a spare register for it (a region has at most 7). each
+   step moves one expression and the analysis is run again, so a later step
+   sees what the earlier one did. */
+typedef struct {
+	int nb, *idom, *rn, *rpo, nrpo, *dep, nl;
+	int *pp, *pl;			/* predecessors of b: pl[pp[b] .. pp[b + 1]); the root, nb, is left out */
+	unsigned char *reach, *isent;
+	struct { int h, n, bad; unsigned char *in; } *lp;
+} Cfg;
+static int csucc(Rg *R, int b, int *o)
+{
+	int n = 0, e;
+	if (b == R->nb) {
+		for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) o[n++] = R->entb[e];
+		return n;
+	}
+	{
+		RB *B = &R->b[b];
+		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) o[n++] = B->s1;
+		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) o[n++] = B->s2;
+	}
+	return n;
+}
+static void cfg_free(Cfg *G)
+{
+	int i;
+	if (!G) return;
+	for (i = 0; i < G->nl; i++) free(G->lp[i].in);
+	free(G->lp); free(G->idom); free(G->rn); free(G->rpo); free(G->dep); free(G->pp); free(G->pl); free(G->reach); free(G->isent); free(G);
+}
+static int cdom(const Cfg *G, int a, int b)
+{
+	int n = 0;
+	while (b != a) {
+		if (b >= G->nb || b < 0 || G->idom[b] < 0 || ++n > G->nb + 2) return 0;
+		b = G->idom[b];
+	}
+	return 1;
+}
+static Cfg *cfg_build(Rg *R)
+{
+	Cfg *G = calloc(1, sizeof *G);
+	int nb = R->nb, b, i, n, e, k, sp, succ[RMAXE + 2], changed, *po, *stn, *sti, npo = 0, guard;
+	if (!G) return 0;
+	G->nb = nb;
+	G->idom = malloc((nb + 1) * sizeof(int)); G->rn = malloc((nb + 1) * sizeof(int)); G->rpo = malloc((nb + 2) * sizeof(int));
+	G->dep = calloc(nb + 1, sizeof(int)); G->pp = calloc(nb + 2, sizeof(int));
+	G->reach = calloc(nb + 1, 1); G->isent = calloc(nb + 1, 1);
+	po = malloc((nb + 2) * sizeof(int)); stn = malloc((nb + 2) * sizeof(int)); sti = malloc((nb + 2) * sizeof(int));
+	if (!G->idom || !G->rn || !G->rpo || !G->dep || !G->pp || !G->reach || !G->isent || !po || !stn || !sti) { free(po); free(stn); free(sti); cfg_free(G); return 0; }
+	for (b = 0; b <= nb; b++) { G->idom[b] = -1; G->rn[b] = -1; }
+	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)nb) G->isent[R->entb[e]] = 1;
+	/* postorder from the root */
+	sp = 0; stn[0] = nb; sti[0] = 0; G->reach[nb] = 1;
+	while (sp >= 0) {
+		n = csucc(R, stn[sp], succ);
+		if (sti[sp] < n) {
+			int s = succ[sti[sp]++];
+			if (!G->reach[s]) { G->reach[s] = 1; sp++; stn[sp] = s; sti[sp] = 0; }
+		} else { po[npo++] = stn[sp]; sp--; }
+	}
+	G->nrpo = npo;
+	for (i = 0; i < npo; i++) { G->rpo[i] = po[npo - 1 - i]; G->rn[G->rpo[i]] = i; }
+	free(po); free(stn); free(sti);
+	/* predecessors (the root is not one) */
+	for (b = 0; b < nb; b++) if (G->reach[b]) { n = csucc(R, b, succ); for (i = 0; i < n; i++) G->pp[succ[i] + 1]++; }
+	for (b = 0; b < nb; b++) G->pp[b + 1] += G->pp[b];
+	G->pl = malloc((G->pp[nb] + 1) * sizeof(int));
+	{
+		int *fill = calloc(nb + 1, sizeof(int));
+		if (!G->pl || !fill) { free(fill); cfg_free(G); return 0; }
+		for (b = 0; b < nb; b++) if (G->reach[b]) { n = csucc(R, b, succ); for (i = 0; i < n; i++) { int s = succ[i]; G->pl[G->pp[s] + fill[s]++] = b; } }
+		free(fill);
+	}
+	/* dominators: the root dominates the entries, and the entries are roots of the graph */
+	G->idom[nb] = nb;
+	guard = 0; changed = 1;
+	while (changed && guard++ < 64) {
+		changed = 0;
+		for (k = 1; k < npo; k++) {
+			int nw = -1, p, j;
+			b = G->rpo[k];
+			if (G->isent[b]) nw = nb;
+			for (j = G->pp[b]; j < G->pp[b + 1]; j++) {
+				p = G->pl[j];
+				if (G->idom[p] < 0) continue;
+				if (nw < 0) { nw = p; continue; }
+				{
+					int x = p, y = nw, g2 = 0;
+					while (x != y && g2++ < 2 * nb + 4) {
+						while (G->rn[x] > G->rn[y] && g2++ < 2 * nb + 4) x = G->idom[x];
+						while (G->rn[y] > G->rn[x] && g2++ < 2 * nb + 4) y = G->idom[y];
+					}
+					nw = x;
+				}
+			}
+			if (nw >= 0 && G->idom[b] != nw) { G->idom[b] = nw; changed = 1; }
+		}
+	}
+	if (changed) { cfg_free(G); return 0; }
+	/* natural loops: a back edge is one whose target dominates its source */
+	G->lp = calloc(nb + 1, sizeof *G->lp);
+	if (!G->lp) { cfg_free(G); return 0; }
+	for (b = 0; b < nb; b++) if (G->reach[b]) {
+		n = csucc(R, b, succ);
+		for (i = 0; i < n; i++) {
+			int h = succ[i], li, *stk, ns = 0, x, j;
+			if (!cdom(G, h, b)) continue;
+			for (li = 0; li < G->nl && G->lp[li].h != h; li++);
+			if (li == G->nl) {
+				G->lp[li].h = h; G->lp[li].in = calloc(nb + 1, 1); G->nl++;
+				if (!G->lp[li].in) { cfg_free(G); return 0; }
+				G->lp[li].in[h] = 1; G->lp[li].n = 1;
+			}
+			stk = malloc((nb + 1) * sizeof(int));
+			if (!stk) { cfg_free(G); return 0; }
+			if (!G->lp[li].in[b]) { G->lp[li].in[b] = 1; G->lp[li].n++; stk[ns++] = b; }
+			while (ns) {
+				x = stk[--ns];
+				for (j = G->pp[x]; j < G->pp[x + 1]; j++) {
+					int p = G->pl[j];
+					if (G->reach[p] && !G->lp[li].in[p]) { G->lp[li].in[p] = 1; G->lp[li].n++; stk[ns++] = p; }
+				}
+			}
+			free(stk);
+		}
+	}
+	for (i = 0; i < G->nl; i++) for (b = 0; b < nb; b++) if (G->lp[i].in[b]) {
+		G->dep[b]++;
+		if (G->isent[b]) G->lp[i].bad = 1;
+	}
+	return G;
+}
+static int w10(int d) { return d <= 0 ? 1 : d == 1 ? 10 : d == 2 ? 100 : d == 3 ? 1000 : 10000; }
+static int opcost(int op)
+{
+	switch (op) {
+	case U_LI: case U_VGET: case U_SLD: return 0;
+	case U_LD: case U_LD8: case U_LD16: case U_LD32: case U_MUL: case U_MULHS: case U_MULHU: case U_FMUL: return 3;
+	case U_SDIV: case U_UDIV: case U_SREM: case U_UREM: case U_FDIV: return 10;
+	default: return 1;
+	}
+}
+/* value numbering: a table of what each number stands for */
+#define GOP_MEM (U_N + 2)
+#define GOP_PHI (U_N + 3)
+#define GOP_ENT (U_N + 4)
+#define GOP_PHIM (U_N + 5)
+#define GNMAX (1 << 17)
+typedef struct { int op, a, b, c, next; Cell imm; } GN;
+static GN *gn_; static int gn_n, *gh_, gn_over;
+static int gn_init(void)
+{
+	if (!gn_) { gn_ = malloc(GNMAX * sizeof(GN)); gh_ = malloc(GNMAX * sizeof(int)); }
+	if (!gn_ || !gh_) return -1;
+	gn_n = 0; gn_over = 0; memset(gh_, 0, GNMAX * sizeof(int));
+	return 0;
+}
+static int gnum(int op, int a, int b, int c, Cell imm)
+{
+	unsigned h; int i;
+	if (op == U_ADD || op == U_MUL || op == U_AND || op == U_OR || op == U_XOR || op == U_EQ || op == U_NE ||
+		op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU || op == U_MULHS || op == U_MULHU ||
+		op == U_FADD || op == U_FMUL || op == U_FEQ) if (a > b) { int t = a; a = b; b = t; }
+	h = ((unsigned)op * 2654435761u) ^ ((unsigned)a * 40503u) ^ ((unsigned)b * 69069u) ^ ((unsigned)c * 1103515245u) ^ (unsigned)(imm * 0x9e3779b1u) ^ (unsigned)((UCell)imm >> 32);
+	h %= GNMAX;
+	for (i = gh_[h]; i; i = gn_[i - 1].next)
+		if (gn_[i - 1].op == op && gn_[i - 1].a == a && gn_[i - 1].b == b && gn_[i - 1].c == c && gn_[i - 1].imm == imm) return i;
+	if (gn_n >= GNMAX - 1) { gn_over = 1; return 1; }
+	gn_[gn_n].op = op; gn_[gn_n].a = a; gn_[gn_n].b = b; gn_[gn_n].c = c; gn_[gn_n].imm = imm;
+	gn_[gn_n].next = gh_[h]; gh_[h] = ++gn_n;
+	return gn_n;
+}
+typedef struct { RB *src; int val, var; } Hoist;	/* write the value src computes into var, in this block */
+typedef struct { int idx, var; } Export;		/* after the block is built, var := what its idx-th op computed */
+/* the value val of block S built again inside the block being emitted */
+static int matv(const RB *P, const int *mp, RB *S, const int *def, int *memo, int val)
+{
+	Uop *u = &S->u[def[val]];
+	int na, a = 0, b = 0, c = 0, r;
+	if (memo[val] >= 0) return memo[val];
+	if (u->op == U_LI) r = K(u->imm);
+	else if (u->op == U_VGET) {
+		int j = (int)u->imm;
+		r = j >= 0 && j < RMAXV && P->vs[j] >= 0 ? mp[P->vs[j]] : raw(U_VGET, 0, 0, 0, u->imm);
+	} else {
+		na = nargs(u->op);
+		if (na > 0) a = matv(P, mp, S, def, memo, u->a);
+		if (na > 1) b = matv(P, mp, S, def, memo, u->b);
+		if (na > 2) c = matv(P, mp, S, def, memo, u->c);
+		r = oe(u->op, a, b, c, u->imm);
+	}
+	memo[val] = r;
+	return r;
+}
+/* the block again through the folding emitter. repl[i] (when nonzero) makes the i-th op a read of that
+   variable; the hoists and exports give variables new values at the end. returns 1 if it was rebuilt */
+static int rb_emit(Rg *R, RB *B, const int *repl, const Hoist *H, int nh, const Export *X, int nx)
+{
+	int i, j, na, n = 0, *live, *mp, bad = 0, *hv = 0, *xv = 0;
+	int *sisk = isk, *somap = omap; Cell *skv = kv;
+	Uop *nu;
+	for (j = 0; j < B->nso; j++) if ((unsigned)B->so[2 * j + 1] >= (unsigned)B->nv) bad = 1;
+	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= B->nv) bad = 1;
+	if (B->cv >= B->nv) bad = 1;
+	for (i = 0; i < B->n; i++) if ((unsigned)B->u[i].d >= (unsigned)B->nv && !is_store(B->u[i].op) && B->u[i].op != U_NARG) bad = 1;
+	if (bad) return 0;
+	mp = malloc((B->nv + 1) * sizeof(int));
+	hv = malloc((nh + 1) * sizeof(int)); xv = malloc((nx + 1) * sizeof(int));
+	if (!mp || !hv || !xv) { free(mp); free(hv); free(xv); return 0; }
+	isk = g_isk; kv = g_kv; omap = g_omap;
+	on = 0; onv = 0; uerr = 0;
+	for (i = 0; i < B->n && !uerr; i++) {
+		Uop *u = &B->u[i];
+		na = u->op >= U_N ? 0 : nargs(u->op);
+		if (repl && repl[i] && !is_store(u->op) && u->op != U_NARG && u->op != U_NCALL) mp[u->d] = raw(U_VGET, 0, 0, 0, repl[i]);
+		else if (u->op == U_VGET) mp[u->d] = raw(U_VGET, 0, 0, 0, u->imm);
+		else if (u->op == U_SLD) mp[u->d] = raw(U_SLD, 0, 0, 0, u->imm);
+		else if (u->op == U_LI) mp[u->d] = K(u->imm);
+		else if (is_store(u->op)) oe(u->op, mp[u->a], mp[u->b], 0, 0);
+		else if (u->op == U_NARG) raw(U_NARG, mp[u->a], 0, 0, u->imm);
+		else if (u->op == U_NCALL) mp[u->d] = raw(U_NCALL, 0, 0, 0, u->imm);
+		else mp[u->d] = oe(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+	}
+	for (j = 0; j < nh && !uerr; j++) {
+		RB *S = H[j].src;
+		int *def = malloc((S->nv + 1) * sizeof(int)), *memo = malloc((S->nv + 1) * sizeof(int));
+		if (!def || !memo) { free(def); free(memo); uerr = -5; break; }
+		for (i = 0; i < S->nv; i++) { def[i] = 0; memo[i] = -1; }
+		for (i = 0; i < S->n; i++) if (!is_store(S->u[i].op) && S->u[i].op != U_NARG && (unsigned)S->u[i].d < (unsigned)S->nv) def[S->u[i].d] = i;
+		hv[j] = matv(B, mp, S, def, memo, H[j].val);
+		free(def); free(memo);
+	}
+	for (j = 0; j < nx; j++) xv[j] = mp[B->u[X[j].idx].d];
+	if (uerr) { isk = sisk; kv = skv; omap = somap; free(mp); free(hv); free(xv); return 0; }
+	live = calloc(onv + 2, sizeof *live);
+	nu = malloc((on + 1) * sizeof(Uop));
+	if (!live || !nu) { free(live); free(nu); isk = sisk; kv = skv; omap = somap; free(mp); free(hv); free(xv); return 0; }
+	for (j = 0; j < B->nso; j++) B->so[2 * j + 1] = mp[B->so[2 * j + 1]];
+	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) B->vs[j] = mp[B->vs[j]];
+	if (B->cv >= 0) B->cv = mp[B->cv];
+	for (j = 0; j < nh; j++) B->vs[H[j].var] = hv[j];
+	for (j = 0; j < nx; j++) B->vs[X[j].var] = xv[j];
+	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && ob[B->vs[j]].op == U_VGET && ob[B->vs[j]].imm == j) B->vs[j] = -1;
+	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
+	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
+	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) live[B->cv] = 1;
+	for (i = on; i--;) {
+		Uop *u = &ob[i];
+		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
+		na = u->op >= U_N ? 0 : nargs(u->op);
+		if (is_store(u->op)) na = 2;
+		if (u->op == U_NARG) na = 1;
+		if (na > 0) live[u->a] = 1;
+		if (na > 1) live[u->b] = 1;
+		if (na > 2) live[u->c] = 1;
+	}
+	for (i = 0; i < on; i++) if (ob[i].op >= 0) nu[n++] = ob[i];
+	free(B->u); B->u = nu; B->n = n; B->nv = onv;
+	if (onv > R->maxnv) R->maxnv = onv;
+	free(live); free(mp); free(hv); free(xv);
+	isk = sisk; kv = skv; omap = somap;
+	return 1;
+}
+/* a new variable for a value that has to cross blocks: 0 if the region has no register left */
+static int newcarry(Rg *R)
+{
+	int v;
+	if (R->nv >= RMAXV) return 0;
+	v = R->nv++;
+	R->slot[v] = R->np ? R->ps[0] : 0; R->fv[v] = 0; R->sc[v] = 1;
+	return v;
+}
+/* ---- hoisting: an expression in a loop whose operands the loop doesn't change is built once, in front of it ---- */
+static int licm_loop(Rg *R, Cfg *G, int li)
+{
+	int nb = R->nb, b, i, j, k, W = 0, hasmem = 0, h = G->lp[li].h, nc = 0, best = -1, P = -1, nout = 0, outp[64];
+	unsigned char *in = G->lp[li].in;
+	struct { int kid, cost, b, d; long long save; } cand[64];
+	long long bestsave = 0;
+	if (G->lp[li].bad || R->nv >= RMAXV) return 0;
+	for (b = 0; b < nb; b++) if (in[b]) {
+		RB *B = &R->b[b];
+		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) W |= 1 << j;
+		for (i = 0; i < B->n; i++) if (is_store(B->u[i].op) || B->u[i].op == U_NARG || B->u[i].op == U_NCALL) hasmem = 1;
+		if (B->term == T_CALL) return 0;
+	}
+	for (j = G->pp[h]; j < G->pp[h + 1]; j++) if (!in[G->pl[j]]) { if (nout < 64) outp[nout] = G->pl[j]; nout++; }
+	if (nout < 1 || nout > 64) return 0;
+	if (gn_init()) return 0;
+	for (b = 0; b < nb; b++) if (in[b]) {
+		RB *B = &R->b[b];
+		int n = B->nv + 1, *kid = calloc(n, sizeof(int)), *cst = calloc(n, sizeof(int)), *used = calloc(n, sizeof(int));
+		unsigned char *inv = calloc(n, 1), *fpv = calloc(n, 1);
+		if (!kid || !cst || !used || !inv || !fpv) { free(kid); free(cst); free(used); free(inv); free(fpv); return 0; }
+		class_fp(R, B, fpv);
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			int d = u->d, na;
+			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
+			if (u->op == U_LI) { inv[d] = 1; kid[d] = gnum(U_LI, 0, 0, 0, u->imm); }
+			else if (u->op == U_VGET) {
+				j = (int)u->imm;
+				inv[d] = j >= 0 && j < RMAXV && !(W >> j & 1) && !fpv[d];
+				kid[d] = gnum(U_VGET, 0, 0, 0, u->imm);
+			} else if (u->op == U_SLD || u->op >= U_N) inv[d] = 0;
+			else {
+				na = nargs(u->op);
+				inv[d] = (na < 1 || inv[u->a]) && (na < 2 || inv[u->b]) && (na < 3 || inv[u->c]) && !fpv[d];
+				if (is_memop(u->op) && hasmem) inv[d] = 0;
+				if (inv[d]) {
+					kid[d] = gnum(u->op, na > 0 ? kid[u->a] : 0, na > 1 ? kid[u->b] : 0, na > 2 ? kid[u->c] : 0, u->imm);
+					cst[d] = opcost(u->op) + (na > 0 ? cst[u->a] : 0) + (na > 1 ? cst[u->b] : 0) + (na > 2 ? cst[u->c] : 0);
+				}
+			}
+		}
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			int na;
+			if (is_store(u->op)) { used[u->a] = 1; used[u->b] = 1; continue; }
+			if (u->op == U_NARG) { used[u->a] = 1; continue; }
+			if (u->op == U_NCALL || (unsigned)u->d >= (unsigned)B->nv || inv[u->d]) continue;
+			na = u->op >= U_N ? 0 : nargs(u->op);
+			if (na > 0) used[u->a] = 1;
+			if (na > 1) used[u->b] = 1;
+			if (na > 2) used[u->c] = 1;
+		}
+		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) used[B->vs[j]] = 1;
+		for (j = 0; j < B->nso; j++) used[B->so[2 * j + 1]] = 1;
+		if (B->cv >= 0 && (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL)) used[B->cv] = 1;
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			int d = u->d;
+			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
+			if (!inv[d] || !used[d] || cst[d] < 2 || u->op == U_LI || u->op == U_VGET) continue;
+			for (k = 0; k < nc && cand[k].kid != kid[d]; k++);
+			if (k == nc) {
+				if (nc >= 64) continue;
+				cand[nc].kid = kid[d]; cand[nc].cost = cst[d]; cand[nc].b = b; cand[nc].d = d; cand[nc].save = 0; nc++;
+			}
+			cand[k].save += (long long)cst[d] * w10(G->dep[b]);
+		}
+		free(kid); free(cst); free(used); free(inv); free(fpv);
+	}
+	if (gn_over) return 0;
+	for (k = 0; k < nc; k++) {
+		long long net = cand[k].save - (long long)(cand[k].cost + 1) * w10(G->dep[h] - 1);
+		if (net > bestsave) { bestsave = net; best = k; }
+	}
+	if (best < 0 || bestsave < 3) return 0;
+	/* where it goes: the one block outside the loop that falls into it, or a new one */
+	if (nout == 1) {
+		RB *Pb = &R->b[outp[0]];
+		if ((Pb->term == T_BR && Pb->s1 == h) || Pb->term == T_CBR) P = outp[0];
+	}
+	if (P < 0) {
+		RB *grown = realloc(R->b, (nb + 1) * sizeof(RB)), *N;
+		if (!grown) return 0;
+		R->b = grown;
+		N = &R->b[nb];
+		memset(N, 0, sizeof *N);
+		N->a = R->b[h].a; N->u = malloc(sizeof(Uop)); N->so = malloc(sizeof(int)); N->n = 0; N->nv = 1; N->term = T_BR; N->s1 = h; N->s2 = -1; N->cv = -1;
+		for (j = 0; j < RMAXV; j++) N->vs[j] = -1;
+		for (j = 0; j < nout; j++) {
+			RB *Q = &R->b[outp[j]];
+			if (Q->s1 == h) Q->s1 = nb;
+			if (Q->term == T_CBR && Q->s2 == h) Q->s2 = nb;
+		}
+		R->nb = nb + 1;
+		P = nb;
+	}
+	{
+		int s = newcarry(R);
+		Hoist hs;
+		RB *src = &R->b[cand[best].b];
+		if (!s) return 0;
+		hs.src = src; hs.val = cand[best].d; hs.var = s;
+		if (!rb_emit(R, &R->b[P], 0, &hs, 1, 0, 0)) { R->nv--; return 0; }
+		/* every block of the loop that builds the same thing reads the new variable instead */
+		for (b = 0; b < R->nb; b++) if (b < nb && in[b]) {
+			RB *B = &R->b[b];
+			int n = B->nv + 1, *kid = calloc(n, sizeof(int)), *rp = calloc(B->n + 1, sizeof(int)), any = 0;
+			unsigned char *inv = calloc(n, 1), *fpv = calloc(n, 1);
+			if (!kid || !rp || !inv || !fpv) { free(kid); free(rp); free(inv); free(fpv); continue; }
+			class_fp(R, B, fpv);
+			for (i = 0; i < B->n; i++) {
+				Uop *u = &B->u[i];
+				int d = u->d, na;
+				if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
+				if (u->op == U_LI) { inv[d] = 1; kid[d] = gnum(U_LI, 0, 0, 0, u->imm); }
+				else if (u->op == U_VGET) { j = (int)u->imm; inv[d] = j >= 0 && j < RMAXV && !(W >> j & 1) && !fpv[d]; kid[d] = gnum(U_VGET, 0, 0, 0, u->imm); }
+				else if (u->op == U_SLD || u->op >= U_N) inv[d] = 0;
+				else {
+					na = nargs(u->op);
+					inv[d] = (na < 1 || inv[u->a]) && (na < 2 || inv[u->b]) && (na < 3 || inv[u->c]) && !fpv[d];
+					if (is_memop(u->op) && hasmem) inv[d] = 0;
+					if (inv[d]) {
+						kid[d] = gnum(u->op, na > 0 ? kid[u->a] : 0, na > 1 ? kid[u->b] : 0, na > 2 ? kid[u->c] : 0, u->imm);
+						if (kid[d] == cand[best].kid && u->op != U_LI && u->op != U_VGET) { rp[i] = s; any = 1; }
+					}
+				}
+			}
+			if (any) rb_emit(R, B, rp, 0, 0, 0, 0);
+			free(kid); free(rp); free(inv); free(fpv);
+		}
+	}
+	return 1;
+}
+
+/* ---- shared expressions: the same value built in two blocks, one of which always runs first ---- */
+typedef struct { int g[RMAXV]; int mem; } GS;
+static void gtrans(Rg *R, int b, const GS *in, GS *out, int *vg, int *vc)
+{
+	RB *B = &R->b[b];
+	int i, j, mem = in->mem;
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		int op = u->op, d = u->d, na;
+		if (op == U_NARG) continue;
+		if (is_store(op)) { mem = gnum(GOP_MEM, b, i, 0, 0); continue; }
+		if ((unsigned)d >= (unsigned)B->nv) continue;
+		vc[d] = 0;
+		if (op == U_LI) vg[d] = gnum(U_LI, 0, 0, 0, u->imm);
+		else if (op == U_VGET) vg[d] = in->g[(int)u->imm & (RMAXV - 1)];
+		else if (op == U_SLD) vg[d] = gnum(U_SLD, b, 0, 0, u->imm);
+		else if (op == U_NCALL) { vg[d] = gnum(U_NCALL, b, i, 0, 0); mem = gnum(GOP_MEM, b, i, 1, 0); }
+		else if (op == U_LD || op == U_LD8 || op == U_LD16 || op == U_LD32) { vg[d] = gnum(op, vg[u->a], mem, 0, 0); vc[d] = opcost(op) + vc[u->a]; }
+		else {
+			na = nargs(op);
+			vg[d] = gnum(op, na > 0 ? vg[u->a] : 0, na > 1 ? vg[u->b] : 0, na > 2 ? vg[u->c] : 0, u->imm);
+			vc[d] = opcost(op) + (na > 0 ? vc[u->a] : 0) + (na > 1 ? vc[u->b] : 0) + (na > 2 ? vc[u->c] : 0);
+		}
+	}
+	*out = *in;
+	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && (unsigned)B->vs[j] < (unsigned)B->nv) out->g[j] = vg[B->vs[j]];
+	out->mem = mem;
+}
+static int gjoin(GS *d, const GS *s, int blk)
+{
+	int v, ch = 0, p;
+	for (v = 0; v < RMAXV; v++) if (d->g[v] != s->g[v]) {
+		p = gnum(GOP_PHI, blk, v, 0, 0);
+		if (d->g[v] != p) { d->g[v] = p; ch = 1; }
+	}
+	if (d->mem != s->mem) {
+		p = gnum(GOP_PHIM, blk, 0, 0, 0);
+		if (d->mem != p) { d->mem = p; ch = 1; }
+	}
+	return ch;
+}
+typedef struct { int g, b, i, cost; } Site;
+static int sitecmp(const void *x, const void *y)
+{
+	const Site *a = x, *b = y;
+	if (a->g != b->g) return a->g < b->g ? -1 : 1;
+	if (a->b != b->b) return a->b < b->b ? -1 : 1;
+	return a->i - b->i;
+}
+static int cse_one(Rg *R, Cfg *G)
+{
+	int nb = R->nb, b, i, e, k, sweeps, changed, ns = 0, capn = 0, s, did = 0;
+	GS *in = 0, *out = 0;
+	unsigned char *seen = 0;
+	int **vgs = 0, **vcs = 0;
+	Site *sites = 0;
+	if (R->nv >= RMAXV || gn_init()) return 0;
+	in = calloc(nb, sizeof(GS)); out = calloc(nb, sizeof(GS)); seen = calloc(nb, 1);
+	vgs = calloc(nb, sizeof(int *)); vcs = calloc(nb, sizeof(int *));
+	if (!in || !out || !seen || !vgs || !vcs) goto fin;
+	for (b = 0; b < nb; b++) {
+		vgs[b] = calloc(R->b[b].nv + 1, sizeof(int)); vcs[b] = calloc(R->b[b].nv + 1, sizeof(int));
+		if (!vgs[b] || !vcs[b]) goto fin;
+	}
+	for (e = 0; e < R->ne; e++) {
+		b = R->entb[e];
+		if ((unsigned)b >= (unsigned)nb) continue;
+		for (k = 0; k < RMAXV; k++) in[b].g[k] = gnum(GOP_ENT, b, k, 0, 0);
+		in[b].mem = gnum(GOP_ENT, b, RMAXV, 0, 0);
+		seen[b] = 1;
+	}
+	for (sweeps = 0, changed = 1; changed && sweeps < 80; sweeps++) {
+		changed = 0;
+		for (k = 0; k < G->nrpo; k++) {
+			int sc[2], nsu, t;
+			b = G->rpo[k];
+			if (b >= nb || !seen[b]) continue;
+			gtrans(R, b, &in[b], &out[b], vgs[b], vcs[b]);
+			nsu = 0;
+			if (R->b[b].term == T_BR || R->b[b].term == T_CBR) if ((unsigned)R->b[b].s1 < (unsigned)nb) sc[nsu++] = R->b[b].s1;
+			if (R->b[b].term == T_CBR && (unsigned)R->b[b].s2 < (unsigned)nb) sc[nsu++] = R->b[b].s2;
+			for (t = 0; t < nsu; t++) {
+				int sx = sc[t];
+				if (!seen[sx]) { seen[sx] = 1; in[sx] = out[b]; changed = 1; }
+				else if (gjoin(&in[sx], &out[b], sx)) changed = 1;
+			}
+		}
+		if (gn_over) goto fin;
+	}
+	if (changed) goto fin;
+	for (b = 0; b < nb; b++) if (seen[b]) gtrans(R, b, &in[b], &out[b], vgs[b], vcs[b]);
+	if (gn_over) goto fin;
+	/* every place that builds something worth sharing */
+	for (b = 0; b < nb; b++) if (seen[b]) {
+		RB *B = &R->b[b];
+		unsigned char *fpv = calloc(B->nv + 1, 1);
+		if (!fpv) goto fin;
+		class_fp(R, B, fpv);
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			int d = u->d;
+			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || u->op == U_LI || u->op == U_VGET || u->op == U_SLD || u->op >= U_N) continue;
+			if ((unsigned)d >= (unsigned)B->nv || fpv[d] || vcs[b][d] < 3) continue;
+			if (ns >= capn) {
+				Site *g2;
+				capn = capn ? capn * 2 : 256;
+				g2 = realloc(sites, capn * sizeof(Site));
+				if (!g2) { free(fpv); goto fin; }
+				sites = g2;
+			}
+			sites[ns].g = vgs[b][d]; sites[ns].b = b; sites[ns].i = i; sites[ns].cost = vcs[b][d]; ns++;
+		}
+		free(fpv);
+	}
+	if (ns > 1) qsort(sites, ns, sizeof(Site), sitecmp);	/* no sites: sites may be null, which qsort must not get */
+	{
+		long long bestsave = 0;
+		int bs = -1, be = -1, bd = -1;
+		for (s = 0; s < ns; s = e) {
+			long long save = 0;
+			int d0 = -1, q;
+			for (e = s; e < ns && sites[e].g == sites[s].g; e++);
+			if (e - s < 2) continue;
+			/* the site that comes first on every path to the others */
+			for (q = s; q < e && d0 < 0; q++) {
+				int r2, good = 1;
+				for (r2 = s; r2 < e && good; r2++) {
+					if (r2 == q) continue;
+					if (sites[r2].b == sites[q].b) { if (sites[r2].i < sites[q].i) good = 0; }
+					else if (!cdom(G, sites[q].b, sites[r2].b)) good = 0;
+				}
+				if (good) d0 = q;
+			}
+			if (d0 < 0) continue;
+			for (q = s; q < e; q++) if (q != d0 && sites[q].b != sites[d0].b) save += (long long)sites[q].cost * w10(G->dep[sites[q].b]);
+			save -= (long long)(sites[d0].cost + 1) * w10(G->dep[sites[d0].b]) / 4 + w10(G->dep[sites[d0].b]);
+			if (save > bestsave) { bestsave = save; bs = s; be = e; bd = d0; }
+		}
+		if (bs >= 0 && bestsave >= 3) {
+			int v = newcarry(R), q;
+			if (v) {
+				Export ex;
+				ex.idx = sites[bd].i; ex.var = v;
+				if (!rb_emit(R, &R->b[sites[bd].b], 0, 0, 0, &ex, 1)) { R->nv--; goto fin; }
+				for (q = bs; q < be; q++) {
+					RB *B = &R->b[sites[q].b];
+					int *rp, q2;
+					if (sites[q].b == sites[bd].b) continue;
+					if (q > bs && sites[q - 1].b == sites[q].b) continue;	/* a block is rebuilt once, for all its sites */
+					rp = calloc(B->n + 1, sizeof(int));
+					if (!rp) continue;
+					for (q2 = q; q2 < be && sites[q2].b == sites[q].b; q2++) rp[sites[q2].i] = v;
+					rb_emit(R, B, rp, 0, 0, 0, 0);
+					free(rp);
+				}
+				did = 1;
+			}
+		}
+	}
+fin:
+	if (vgs) for (b = 0; b < nb; b++) free(vgs[b]);
+	if (vcs) for (b = 0; b < nb; b++) free(vcs[b]);
+	free(vgs); free(vcs); free(in); free(out); free(seen); free(sites);
+	return did;
+}
+/* ---- variables written and never read ---- */
+static int rdead_vars(Rg *R)
+{
+	int nb = R->nb, b, i, j, sweeps, changed, did = 0, nreal = R->nv0 ? R->nv0 : R->nv;
+	unsigned char *gen, *kill, *lin, *lout;
+	unsigned allreal = 0;
+	for (j = 1; j < nreal; j++) allreal |= 1u << j;
+	gen = calloc(nb, 1); kill = calloc(nb, 1); lin = calloc(nb, 1); lout = calloc(nb, 1);
+	if (!gen || !kill || !lin || !lout) { free(gen); free(kill); free(lin); free(lout); return 0; }
+	for (b = 0; b < nb; b++) {
+		RB *B = &R->b[b];
+		for (i = 0; i < B->n; i++) if (B->u[i].op == U_VGET && B->u[i].imm >= 0 && B->u[i].imm < RMAXV) gen[b] |= 1u << B->u[i].imm;
+		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) kill[b] |= 1u << j;
+	}
+	for (sweeps = 0, changed = 1; changed && sweeps < 64; sweeps++) {
+		changed = 0;
+		for (b = nb; b--;) {
+			RB *B = &R->b[b];
+			unsigned o = 0;
+			int s, x;
+			if (B->term == T_BR) o = (unsigned)B->s1 < (unsigned)nb ? lin[B->s1] : allreal;
+			else if (B->term == T_CBR) {
+				for (s = 0; s < 2; s++) {
+					int t = s ? B->s2 : B->s1, xa = s ? B->x2 : B->x1;
+					if ((unsigned)t < (unsigned)nb) o |= lin[t];
+					else o |= (R->fn >= 0 && xa >= R->hi) ? 0 : allreal;
+				}
+			} else if (B->term == T_JMP) {
+				int dead = 0;
+				if (R->fn >= 0 && B->cv >= 0) for (x = 0; x < B->n; x++) if (B->u[x].d == B->cv && B->u[x].op == U_LI && B->u[x].imm >= R->hi) dead = 1;
+				o = dead ? 0 : allreal;
+			} else if (B->term == T_CALL) {
+				int ee;
+				o = allreal;
+				for (ee = 0; ee < R->ne; ee++) if (R->ent[ee] == B->x2 && (unsigned)R->entb[ee] < (unsigned)nb) { o = lin[R->entb[ee]]; break; }
+			} else o = allreal;
+			if (o != lout[b]) { lout[b] = (unsigned char)o; changed = 1; }
+			x = gen[b] | (lout[b] & ~kill[b]);
+			if (x != lin[b]) { lin[b] = (unsigned char)x; changed = 1; }
+		}
+	}
+	if (changed) { free(gen); free(kill); free(lin); free(lout); return 0; }
+	for (b = 0; b < nb; b++) {
+		RB *B = &R->b[b];
+		int any = 0;
+		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && !(lout[b] >> j & 1)) { B->vs[j] = -1; any = 1; }
+		if (any) { rb_emit(R, B, 0, 0, 0, 0, 0); did = 1; }
+	}
+	free(gen); free(kill); free(lin); free(lout);
+	return did;
+}
+static int rshare(Rg *R)
+{
+	int iter, did = 0;
+	if (R->nb < 1 || R->nb > 300 || getenv("ZYNOSHARE")) return 0;
+	for (iter = 0; iter < 8; iter++) {
+		Cfg *G = cfg_build(R);
+		int li, one = 0, ord[256], n, x, y;
+		if (!G) break;
+		n = G->nl < 256 ? G->nl : 256;
+		for (x = 0; x < n; x++) ord[x] = x;
+		for (x = 1; x < n; x++) for (y = x; y > 0 && G->lp[ord[y]].n < G->lp[ord[y - 1]].n; y--) { int t = ord[y]; ord[y] = ord[y - 1]; ord[y - 1] = t; }
+		for (li = 0; li < n && !one; li++) one = licm_loop(R, G, ord[li]);
+		if (!one) one = cse_one(R, G);
+		cfg_free(G);
+		if (!one) break;
+		did = 1;
+	}
+	if (rdead_vars(R)) did = 1;
+	return did;
+}
+static void addl(int x, int en)
+{
+	int e;
+	if (x < R_->lo || x >= R_->hi) return;
+	if (!lead_[x - R_->lo]) { lead_[x - R_->lo] = 1; wl_[nw_++] = x; }
+	if (en) { for (e = 0; e < R_->ne && R_->ent[e] != x; e++); if (e == R_->ne && e < RMAXE) R_->ent[R_->ne++] = x; }
+}
+/* blocks, then each one lifted and optimized */
+static int rlift(Rg *R)
+{
+	int n = R->hi - R->lo, L, i, k, t, b, e, r = 0, cnt;
+	Cell w;
+	if (R->lifted) rclear(R);
+	if (R->nv0) R->nv = R->nv0;
+	R_ = R;
+	lead_ = calloc(n, 1); bidx_ = malloc(n * sizeof(int)); wl_ = malloc(n * sizeof(int)); nw_ = 0;
+	R->ne = 0;
+	addl(R->lo, 1);
+	while (nw_) {
+		L = wl_[--nw_];
+		for (i = L, cnt = 0;;) {
+			if (i >= R->hi || (i != L && lead_[i - R->lo])) break;
+			if (cnt++ >= 1000) { addl(i, 0); break; }
+			w = m[i];
+			if (w & 1) {
+				if (i + 1 < R->hi && !(m[i + 1] & 1) && (k = (int)(m[i + 1] >> 1)) >= 6 && k <= 8) {
+					t = (int)(w >> 1);
+					if (k == 6) addl(i + 2, 1);	/* a call comes back here */
+					else if (k == 7) addl(t, 0);
+					else { addl(t, 0); addl(i + 2, 0); }
+					break;
+				}
+				i++;
+				continue;
+			}
+			k = (int)(w >> 1);
+			if (k == 600) { i += 2; continue; }	/* payload is data, not an opcode */
+			if (k == 6) { addl(i + 1, 1); break; }
+			if (k == 8) { addl(i + 1, 0); break; }
+			if (k == 0 || k == 7 || k == 9) break;
+			i++;
+		}
+	}
+	for (i = 0, b = 0; i < n; i++) if (lead_[i]) bidx_[i] = b++;
+	R->nb = b; R->b = calloc(b, sizeof(RB)); R->maxnv = 0;
+	for (i = 0; i < n; i++) if (lead_[i]) R->b[bidx_[i]].a = R->lo + i;
+	for (b = 0; b < R->nb && !r; b++) r = lift_block(b);
+	for (e = 0; e < R->ne; e++) R->entb[e] = bidx_[R->ent[e] - R->lo];
+	free(lead_); free(bidx_); free(wl_);
+	if (r) {
+		if (trace) fprintf(stderr, "o3: %d..%d doesn't lift (%d)\n", R->lo, R->hi - 1, r);
+		R->lifted = 1; rclear(R); return r;
+	}
+	rshape(R);
+	if (!getenv("ZYNOGLOBAL")) rglobal(R);
+	if (rshare(R)) rshape(R);
+	for (e = 0; e < R->ne; e++) { o3ent[R->ent[e]] = R->idx + 1; o3ei[R->ent[e]] = e; }
+	R->lifted = 1;
+	rmark(R);
+	if (trace) {
+		int ops = 0;
+		for (b = 0; b < R->nb; b++) ops += R->b[b].n;
+		fprintf(stderr, "o3: %d..%d: %d blocks, %d entries, %d vars, %d micro-ops\n", R->lo, R->hi - 1, R->nb, R->ne, R->nv - 1, ops);
+		if (getenv("ZYO3DUMP"))
+			for (b = 0; b < R->nb; b++) {
+				RB *B = &R->b[b];
+				fprintf(stderr, " b%d @%d: %d ops, sp %+d, term %d cv v%d s %d/%d x %d/%d\n", b, B->a, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2);
+				for (i = 0; i < B->n; i++) {
+					Uop *u = &B->u[i];
+					if (u->op == U_ST) fprintf(stderr, "   st v%d v%d\n", u->a, u->b);
+					else if (u->op == U_LI) fprintf(stderr, "   v%d = %lld\n", u->d, (long long)u->imm);
+					else if (u->op == U_VGET) fprintf(stderr, "   v%d = var%d\n", u->d, u->imm);
+					else if (u->op == U_SLD) fprintf(stderr, "   v%d = s[%d]\n", u->d, u->imm);
+					else { fprintf(stderr, "   v%d = %s v%d", u->d, unames[u->op], u->a); if (nargs(u->op) > 1) fprintf(stderr, " v%d", u->b); if (nargs(u->op) > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
+				}
+				for (i = 0; i < B->nso; i++) fprintf(stderr, "   s[%d] = v%d\n", B->so[2 * i], B->so[2 * i + 1]);
+				for (i = 1; i < R->nv; i++) if (B->vs[i] >= 0) fprintf(stderr, "   var%d = v%d\n", i, B->vs[i]);
+			}
+	}
+	return 0;
+}
+/* install kind 2 */
+static int o3install(int d)
+{
+	int lo = mread(d + 2), hi = mread(d + 3), n = mread(d + 4), i, r;
+	Rg *R;
+	if (lo <= 0 || hi <= lo || hi > code_words || hi - lo > 1000000 || (unsigned)n > 32 || nrg >= MAXRG) return -1;
+	R = calloc(1, sizeof *R);
+	R->idx = nrg; rgs[nrg++] = R;
+	R->lo = lo; R->hi = hi; R->nv = 1; R->np = n;
+	for (i = 1; i < RMAXV; i++) R->slot[i] = -1;
+	for (i = 0; i < n; i++) {	/* pairs: frame slot, register 1..7 */
+		int s = mread(d + 5 + 2 * i), v = mread(d + 6 + 2 * i), fp = 0, pl = 0;
+		if (v < 0) { fp = 1; v = -v; }	/* a negative register is a d-register home */
+		if (v >= 16) { pl = 1; v -= 16; }
+		if ((unsigned)s >= 4096 || v < 1 || v >= RMAXV) { R->dead = 1; return -1; }
+		R->fv[v] = (unsigned char)fp;
+		R->sc[v] = (unsigned char)((R->sc[v] || !R->seen[v]) && pl);
+		R->seen[v] = 1;
+		R->ps[i] = s; R->pv[i] = v;
+		if (R->slot[v] < 0) R->slot[v] = s;
+		if (v >= R->nv) R->nv = v + 1;
+	}
+	for (i = 1; i < R->nv; i++) if (R->slot[i] < 0) R->slot[i] = R->ps[0];
+	R->nv0 = R->nv;
+	R->fslots = mread(d + 5 + 2 * n);
+	R->argc = mread(d + 6 + 2 * n);
+	R->fn = mread(d + 7 + 2 * n);
+	if ((unsigned)R->argc > 32 || R->fslots < 0) { R->dead = 1; return -1; }
+	if ((r = rlift(R))) { R->dead = 1; return r; }
+	o3_bind(R);
+	rcompile(R, 1);
+	return 0;
+}
+/* the region entered at p, lifted again if it has to be, or 0 */
+static Rg *rat(int p)
+{
+	Rg *R = rgs[o3ent[p] - 1];
+	if (R->dead) { o3ent[p] = 0; return 0; }
+	if (!R->lifted && rlift(R)) { R->dead = 1; o3ent[p] = 0; return 0; }
+	if (o3ei[p] >= R->ne || R->ent[o3ei[p]] != p) { o3ent[p] = 0; return 0; }
+	return R;
+}
+
+#ifdef ZYJIT
+static unsigned char *cov;	/* how many live translations cover the word (255 sticks) */
+static void **tab;		/* vm address -> translation */
+static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); }
+static unsigned *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
+static long S[14];	/* memory, stacks, hooks, limits, and the target saved on overflow */
+static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc)
+{
+	o3_invoke(ps, pr, target, retpc);
+	S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
+}
+
+enum { W0 = 0, W1 = 1, X9 = 9, X10, X11, M = 19, SP = 20, RP = 21, TAB = 22, ST = 23 };
+
+static void E(unsigned x) { *cp++ = x; }
+static void li(int d, Cell v)	/* movz/movk x, four halfwords; a vm immediate */
+{
+	UCell u = (UCell)v;
+	E(0xd2800000 | ((unsigned)(u & 0xffff) << 5) | d);
+	if (u >> 16) E(0xf2a00000 | ((unsigned)((u >> 16) & 0xffff) << 5) | d);
+	if (u >> 32) E(0xf2c00000 | ((unsigned)((u >> 32) & 0xffff) << 5) | d);
+	if (u >> 48) E(0xf2e00000 | ((unsigned)((u >> 48) & 0xffff) << 5) | d);
+}
+/* executable bounds are read again after a compiler grows its code storage. */
+static void cmp_code(unsigned rn)
+{
+	int tmp = X9;
+	li(tmp, (Cell)(uintptr_t)&code_words); E(0xf9400000u | ((unsigned)tmp << 5) | tmp);
+	E(0xeb00001fu | ((unsigned)tmp << 16) | (rn << 5));
+}
+/* cmp xn, #MW through a scratch register: a data address is in range. MW can be
+   far past what an immediate encodes, since the heap grows into it. */
+static void cmp_mw(unsigned rn, int tmp)
+{
+	li(tmp, MW);
+	E(0xeb00001fu | ((unsigned)tmp << 16) | (rn << 5));
+}
+static void ldr_post(int t, int n, int i) { E(0xf8400400 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void ldr_pre(int t, int n, int i) { E(0xf8400c00 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void str_pre(int t, int n, int i) { E(0xf8000c00 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void str_post(int t, int n, int i) { E(0xf8000400 | (i & 0x1ff) << 12 | n << 5 | t); }
+static void ldr_u(int t, int n, int o) { E(0xf9400000 | (o / 8) << 10 | n << 5 | t); }	/* 8-byte cells */
+static void str_u(int t, int n, int o) { E(0xf9000000 | (o / 8) << 10 | n << 5 | t); }
+static void ldr_d(int t, int n, int o) { E(0xfd400000 | (o / 8) << 10 | n << 5 | t); }	/* ldr Dt, [Xn, #o] */
+static void str_d(int t, int n, int o) { E(0xfd000000 | (o / 8) << 10 | n << 5 | t); }
+static void ldr_d_rr(int t, int n, int r) { E(0xfc605800 | r << 16 | n << 5 | t); }	/* ldr Dt, [Xn, Wr, uxtw #3] */
+static void str_d_rr(int t, int n, int r) { E(0xfc205800 | r << 16 | n << 5 | t); }
+static void fmov_dx(int d, int n) { E(0x9e670000 | n << 5 | d); }	/* fmov Dd, Xn */
+static void fmov_xd(int d, int n) { E(0x9e660000 | n << 5 | d); }	/* fmov Xd, Dn */
+static void fmov_dd(int d, int n) { E(0x1e604000 | n << 5 | d); }	/* fmov Dd, Dn */
+static void ldrx_u(int t, int n, int o) { E(0xf9400000 | (o / 8) << 10 | n << 5 | t); }
+static void ldr_rr(int t, int n, int r, int ext) { E(0xf8600800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
+static void str_rr(int t, int n, int r, int ext) { E(0xf8200800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
+static void ldp_post(int t, int u, int n, int i) { E(0xa8c00000 | ((i / 8) & 0x7f) << 15 | u << 10 | n << 5 | t); }
+static void op3(unsigned op, int d, int n, int r) { E(op | r << 16 | n << 5 | d); }
+static void addx_ext(int d, int n, int r, int ext, int sh) { E(0x8b200000 | r << 16 | ext << 13 | sh << 10 | n << 5 | d); }
+static void movw(int d, int r) { E(0xaa0003e0 | r << 16 | d); }	/* mov xd, xr */
+static void blr(int n) { E(0xd63f0000 | n << 5); }
+static void b_to(unsigned *t) { long o = t - cp; E(0x14000000 | (o & 0x3ffffff)); }
+static void cb_fix(unsigned *at, unsigned *to) { *at |= ((to - at) & 0x7ffff) << 5; }
+#define SXTW 6
+#define UXTW 2
+
+static void build_stubs(void)
+{
+	cp = code;
+	x_enter = cp;			/* int enter(long *S, void *code) */
+	E(0xa9b97bfd);			/* stp x29, x30, [sp, #-112]! */
+	E(0x910003fd);			/* mov x29, sp */
+	E(0xa90153f3);			/* stp x19, x20, [sp, #16] */
+	E(0xa9025bf5);			/* stp x21, x22, [sp, #32] */
+	E(0xf9001bf7);			/* str x23, [sp, #48] */
+	E(0xa90467f8);			/* stp x24, x25, [sp, #64] */
+	E(0xa9056ffa);			/* stp x26, x27, [sp, #80] */
+	E(0xf90033fc);			/* str x28, [sp, #96] */
+	E(0xaa0003f7);			/* mov x23, x0 */
+	ldrx_u(M, ST, 0);
+	ldrx_u(24, ST, 88); ldrx_u(25, ST, 96);	/* x24, x25: the stack limits, as addresses */
+	ldr_u(X9, ST, 8); addx_ext(SP, M, X9, UXTW, 3);	/* sp = m + s*8 */
+	ldr_u(X9, ST, 16); addx_ext(RP, M, X9, UXTW, 3);
+	ldrx_u(TAB, ST, 24);
+	E(0xd61f0020);			/* br x1 */
+	x_exit = cp;			/* x0 = next vm address */
+	op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);	/* s = (x20 - x19) >> 3 */
+	op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
+	E(0xa94153f3);			/* ldp x19, x20, [sp, #16] */
+	E(0xa9425bf5);			/* ldp x21, x22, [sp, #32] */
+	E(0xf9401bf7);			/* ldr x23, [sp, #48] */
+	E(0xa94467f8);			/* ldp x24, x25, [sp, #64] */
+	E(0xa9456ffa);			/* ldp x26, x27, [sp, #80] */
+	E(0xf94033fc);			/* ldr x28, [sp, #96] */
+	E(0xa8c77bfd);			/* ldp x29, x30, [sp], #112 */
+	E(0xd65f03c0);			/* ret */
+	{
+	unsigned *ovd = cp, *ovr;
+	li(W0, PC_DATA_OVERFLOW); b_to(x_exit);	/* data stack overflow */
+	ovr = cp;
+	li(W0, PC_RETURN_OVERFLOW); b_to(x_exit);	/* return stack overflow */
+	x_disp = cp;			/* w0 = target: check the stacks, chain through tab[] or fall back to c */
+	str_u(0, ST, 104);
+	E(0xeb18029f);			/* cmp x20, x24 */
+	E(0x54000003 | ((ovd - cp) & 0x7ffff) << 5);	/* b.lo data overflow */
+	E(0xeb1902bf);			/* cmp x21, x25 */
+	E(0x54000002 | ((ovr - cp) & 0x7ffff) << 5);	/* b.hs return overflow */
+	}
+	cmp_code(0);			/* compare the current code reservation */
+	E(0x54000002 | ((x_exit - cp) & 0x7ffff) << 5);	/* b.hs exit */
+	ldrx_u(TAB, ST, 24);
+	E(0xf8605ac9);			/* ldr x9, [x22, w0, uxtw #3] */
+	E(0xb4000009 | ((x_exit - cp) & 0x7ffff) << 5);	/* cbz x9, exit */
+	E(0xd61f0120);			/* br x9 */
+	x_first = cp;
+}
+
+static void flush(void)
+{
+	memset(tab, 0, (size_t)code_words * sizeof *tab);
+	rdrop_all();
+	memset(cov, 0, (size_t)code_words);
+	cov[SYSGATE] = 255;
+	ix_reset();
+	cp = x_first;
+	if (trace) fprintf(stderr, "jit: flushed\n");
+}
+
+static void jitwrite(int on)
+{
+#ifdef __APPLE__
+	pthread_jit_write_protect_np(!on);
+#else
+	(void)on;
+#endif
+}
+
+static void syncicache(unsigned *a, unsigned *b)
+{
+#ifdef __APPLE__
+	sys_icache_invalidate(a, (char *)b - (char *)a);
+#else
+	__builtin___clear_cache((char *)a, (char *)b);
+#endif
+}
+
+/* store hook for words that are covered by a translation */
+static int hook(Cell a, Cell v)
+{
+	int j, hit = 0;
+	if (a == SYSGATE) {
+		int moving = v == 9 && (m[a + 1] == 7 || m[a + 1] == 8);
+		cb_s = S[1]; cb_r = S[2]; m[a] = hostcall(v, m[a + 1], m[a + 2], m[a + 3]);
+		if (moving) { S[1] = cb_s; S[2] = cb_r; }
+		S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
+		S[3] = (long)tab;
+		return 0;
+	}
+	if (m[a] == v) return 0;
+	m[a] = v;
+	hit = ix_drop_at((int)a);
+	for (j = 0; j < nrg; j++)
+		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
+			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d)\n", rgs[j]->lo, rgs[j]->hi - 1, (int)a);
+			rclear(rgs[j]);
+			hit = 1;
+		}
+	if (hit) drop_epoch++;
+	return hit;
+}
+
+/* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */
+static void fjit(int k)
+{
+	unsigned opw = 0;
+	if (k == 601) opw = 0x1e602820;		/* fadd d0, d1, d0 */
+	else if (k == 602) opw = 0x1e603820;	/* fsub */
+	else if (k == 603) opw = 0x1e600820;	/* fmul */
+	else if (k == 604) opw = 0x1e601820;	/* fdiv */
+	if (opw) {
+		E(0xfd400280);	/* ldr d0, [x20] */
+		E(0xfd400681);	/* ldr d1, [x20, #8] */
+		E(opw);
+		E(0x91002294);	/* add x20, x20, #8 */
+		E(0xfd000280);	/* str d0, [x20] */
+		return;
+	}
+	if (k >= 605 && k <= 607) {
+		E(0xfd400280);
+		E(0xfd400681);
+		E(0x1e602020);	/* fcmp d1, d0: left ? right. mi/ls, not lt/le: unordered is not less */
+		if (k == 605) E(0x9a9f17ea);		/* cset x10, eq */
+		else if (k == 606) E(0x9a9f57ea);	/* cset x10, mi */
+		else E(0x9a9f87ea);			/* cset x10, ls */
+		E(0x91002294);
+		str_u(X10, SP, 0);
+		return;
+	}
+	if (k == 608) { ldr_u(X9, SP, 0); E(0x9e620120); E(0xfd000280); return; }	/* scvtf d0, x9; str d0 */
+	if (k == 609) { E(0xfd400280); E(0x9e78000a); str_u(X10, SP, 0); return; }	/* fcvtzs x10, d0 */
+	if (k == 610) { ldr_post(0, SP, 8); li(X11, (Cell)(uintptr_t)fop_print); blr(X11); return; }
+	if (k == 618) {					/* (bits prec mode --) */
+		ldr_u(2, SP, 0);
+		ldr_u(1, SP, 8);
+		ldr_u(0, SP, 16);
+		E(0x91006294);				/* add x20, x20, #24 */
+		li(X11, (Cell)(uintptr_t)fop_print_mode);
+		blr(X11);
+		return;
+	}
+	ldr_u(0, SP, 0); li(X11, (Cell)(uintptr_t)fop_parse_cell); blr(X11); str_u(0, SP, 0);
+}
+
+/* byte ops 612..617. the address is a byte index into m[], little-endian. */
+static void bjit(int k)
+{
+	Cell fn;
+	if (k == 612 || k == 614 || k == 616) {
+		ldr_u(0, SP, 0);
+		if (k == 612) fn = (Cell)(uintptr_t)bld8;
+		else if (k == 614) fn = (Cell)(uintptr_t)bld16;
+		else fn = (Cell)(uintptr_t)bld32;
+		li(X11, fn); blr(X11); str_u(0, SP, 0);
+		return;
+	}
+	ldp_post(0, 1, SP, 16);
+	if (k == 613) fn = (Cell)(uintptr_t)bst8;
+	else if (k == 615) fn = (Cell)(uintptr_t)bst16;
+	else fn = (Cell)(uintptr_t)bst32;
+	li(X11, fn); blr(X11);
+}
+
+static void op(int i, int k)	/* everything except control flow */
+{
+	unsigned *a, *b, *c, *z;
+	switch (k) {
+	case 1:
+		ldr_u(X9, SP, 0); cmp_mw(9, X11);
+		a = cp; E(0x54000002);		/* b.hs zero */
+		ldr_rr(X9, M, X9, UXTW);
+		b = cp; E(0x14000000);
+		cb_fix(a, cp); li(X9, 0);
+		*b |= (cp - b) & 0x3ffffff;
+		str_u(X9, SP, 0); return;
+	case 2:	/* store: inline unless the target word is translated */
+		ldp_post(X9, X10, SP, 16);
+		cmp_mw(9, X11);			/* cmp x9, MW */
+		a = cp; E(0x54000002);		/* b.hs done */
+		ldrx_u(X11, ST, 32);
+		E(0x3869496b);			/* ldrb w11, [x11, w9, uxtw] */
+		b = cp; E(0x3500000b);		/* cbnz w11, slow */
+		str_rr(X10, M, X9, UXTW);
+		c = cp; E(0x14000000);		/* b done */
+		cb_fix(b, cp);			/* slow: hook(a, v) */
+		movw(W0, X9); movw(W1, X10);
+		/* publish s and r (as exit does) so a syscall that calls back into the vm
+		   can find the stacks. the hook reads them for a gate store */
+		op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);
+		op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
+		ldrx_u(X11, ST, 64); blr(X11);
+		ldr_u(X9, ST, 8); addx_ext(SP, M, X9, UXTW, 3);
+		ldr_u(X9, ST, 16); addx_ext(RP, M, X9, UXTW, 3);
+		ldrx_u(24, ST, 88); ldrx_u(25, ST, 96); ldrx_u(TAB, ST, 24);
+		z = cp; E(0x34000000);		/* cbz w0, done: hook's int result */
+		li(W0, i + 1); b_to(x_exit);	/* a translation died: resume at i+1 in c */
+		cb_fix(a, cp); cb_fix(z, cp);
+		*c |= (cp - c) & 0x3ffffff;
+		return;
+	case 3:	/* pick uses a word address, with the same bounds as load */
+		ldr_u(X9, SP, 0);
+		op3(0xcb000000, X10, SP, M); E(0xd343fd4a); /* (sp - m) / 8 */
+		E(0x9100054a); op3(0x8b000000, X9, X10, X9);
+		cmp_mw(9, X11);
+		a = cp; E(0x54000002);
+		ldr_rr(X9, M, X9, UXTW);
+		b = cp; E(0x14000000);
+		cb_fix(a, cp); li(X9, 0);
+		*b |= (cp - b) & 0x3ffffff;
+		str_u(X9, SP, 0); return;
+	case 4: E(0x91002294); return;	/* add x20, x20, #8 */
+	case 5: E(0xa9402a89); E(0xa900268a); return;	/* ldp x9, x10, [x20]; stp x10, x9, [x20] */
+	case 10: ldrx_u(X11, ST, 40); blr(X11); E(0x93401c09); str_pre(X9, SP, -8); return; /* sxtb x9, w0 */
+	case 11:
+		ldrx_u(X9, ST, 56);
+		if (sizeof J == 1) E(0x39400129); else E(0xb9400129);	/* ldrb or ldr w: J is a host flag, not a vm word */
+		str_pre(X9, SP, -8);
+		return;
+	case 12: ldr_post(W0, SP, 8); E(0x93401c00); ldrx_u(X11, ST, 48); blr(X11); return; /* sxtb x0, w0 */
+	}
+	ldr_post(X9, SP, 8);
+	ldr_u(X10, SP, 0);
+	switch (k) {
+	case 13: op3(0x8b000000, X10, X10, X9); break;
+	case 14: op3(0xcb000000, X10, X10, X9); break;
+	case 15: op3(0x9b007c00, X10, X10, X9); break;
+	case 16: op3(0x8a000000, X10, X10, X9); break;
+	case 17: op3(0xaa000000, X10, X10, X9); break;
+	case 18: op3(0xca000000, X10, X10, X9); break;
+	case 19: op3(0x9ac02800, X10, X10, X9); break;
+	default: E(0xeb09015f); E(0x9a9fa7ea);	/* cmp x10, x9; cset x10, lt */
+	}
+	str_u(X10, SP, 0);
+}
+
+/* one micro-op, operands in registers. tmp is a scratch register for the
+   remainders; x16 and x17 are scratch too. */
+/* a byte address is an index into the flat image. out of range loads 0 and
+   stores nothing. a store into a word some translation covers calls the
+   same helper as opcode 2, so the block is dropped. x16, x17 and the saved link register are scratch,
+   and the address and value are parked on the machine stack across that. */
+static void emit_byte(int op, int d, int a, int b)
+{
+	int n = bwidth(op), j, ar;
+	unsigned *oob, *cov1, *nocross, *cov2, *over;
+	Cell fn;
+	if (!n) return;
+	str_pre(30, 31, -16);	/* x18 is reserved on darwin; borrow the saved link register */
+	if (!is_store(op)) {
+		ar = a;
+		if (d == a) { movw(17, a); ar = 17; }
+		E(0xd2800000 | d);				/* mov xd, #0 */
+		li(30, ((Cell)MW << 3) - (n - 1));
+		E(0xeb1e001f | (ar << 5));			/* cmp xAr, x30 */
+		oob = cp; E(0x54000002);			/* b.hs done */
+		E(0x8b000000 | (ar << 16) | (M << 5) | 17);	/* add x17, x19, xAr */
+		if (n == 1) E(0x39400000 | (17 << 5) | d);
+		else if (n == 2) E(0x79400000 | (17 << 5) | d);
+		else E(0xb9400000 | (17 << 5) | d);
+		cb_fix(oob, cp);
+		ldr_post(30, 31, 16);
+		return;
+	}
+	str_pre(a, 31, -16);
+	str_u(b, 31, 8);
+	li(16, ((Cell)MW << 3) - (n - 1));
+	ldr_u(17, 31, 0);
+	E(0xeb10023f);					/* cmp x17, x16 */
+	oob = cp; E(0x54000002);			/* b.hs pop */
+	E(0xd343fe30);					/* lsr x16, x17, #3 */
+	ldrx_u(30, ST, 32);
+	E(0x38704bde);					/* ldrb w30, [x30, w16, uxtw] */
+	cov1 = cp; E(0x35000000 | 30);			/* cbnz w30, slow */
+	E(0x92400a3e);					/* and x30, x17, #7 */
+	E(0x91000000 | (n << 10) | (30 << 5) | 30);	/* add x30, x30, #n */
+	E(0xf10023df);					/* cmp x30, #8 */
+	nocross = cp; E(0x54000009);			/* b.ls inline */
+	E(0x91000610);					/* add x16, x16, #1 */
+	ldrx_u(30, ST, 32);
+	E(0x38704bde);					/* ldrb w30, [x30, w16, uxtw] */
+	cov2 = cp; E(0x35000000 | 30);			/* cbnz w30, slow */
+	cb_fix(nocross, cp);				/* inline */
+	ldr_u(17, 31, 0);
+	ldr_u(16, 31, 8);
+	E(0x8b11027e);					/* add x30, x19, x17 */
+	if (n == 1) E(0x390003d0);
+	else if (n == 2) E(0x790003d0);
+	else E(0xb90003d0);				/* str w16, [x30] */
+	over = cp; E(0x14000000);			/* b pop */
+	cb_fix(cov1, cp);
+	cb_fix(cov2, cp);				/* slow */
+	E(0xd10203ff);					/* sub sp, sp, #128 */
+	for (j = 0; j < 16; j += 2) E(0xa9000000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
+	ldr_u(0, 31, 128);
+	ldr_u(1, 31, 136);
+	fn = n == 1 ? (Cell)(uintptr_t)bst8 : n == 2 ? (Cell)(uintptr_t)bst16 : (Cell)(uintptr_t)bst32;
+	li(16, fn); blr(16);
+	for (j = 0; j < 16; j += 2) E(0xa9400000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
+	E(0x910203ff);					/* add sp, sp, #128 */
+	*over |= (cp - over) & 0x3ffffff;
+	cb_fix(oob, cp);				/* pop */
+	E(0x910043ff);					/* add sp, sp, #16 */
+	ldr_post(30, 31, 16);
+	(void)d;
+}
+static void emit_uop(int op, int d, int a, int b, int c, int tmp)
+{
+	switch (op) {
+	case U_ADD: op3(0x8b000000, d, a, b); break;
+	case U_SUB: op3(0xcb000000, d, a, b); break;
+	case U_MUL: op3(0x9b007c00, d, a, b); break;
+	case U_SDIV: op3(0x9ac00c00, d, a, b); break;
+	case U_UDIV: op3(0x9ac00800, d, a, b); break;
+	case U_SREM: op3(0x9ac00c00, tmp, a, b); E(0x9b008000 | b << 16 | a << 10 | tmp << 5 | d); break;	/* msub */
+	case U_UREM: op3(0x9ac00800, tmp, a, b); E(0x9b008000 | b << 16 | a << 10 | tmp << 5 | d); break;
+	case U_AND: op3(0x8a000000, d, a, b); break;
+	case U_OR: op3(0xaa000000, d, a, b); break;
+	case U_XOR: op3(0xca000000, d, a, b); break;
+	case U_SHL: op3(0x9ac02000, d, a, b); break;
+	case U_SHR: op3(0x9ac02400, d, a, b); break;
+	case U_SAR: op3(0x9ac02800, d, a, b); break;
+	case U_NEG: E(0xcb0003e0 | a << 16 | d); break;
+	case U_NOT: E(0xaa2003e0 | a << 16 | d); break;
+	case U_LT: E(0xeb00001f | b << 16 | a << 5); E(0x9a9fa7e0 | d); break;	/* cmp; cset lt */
+	case U_LTU: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f27e0 | d); break;	/* cset lo */
+	case U_EQ: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f17e0 | d); break;
+	case U_NE: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f07e0 | d); break;
+	case U_MIN: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 11 << 12 | a << 5 | d); break;
+	case U_MAX: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 12 << 12 | a << 5 | d); break;
+	case U_MINU: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 3 << 12 | a << 5 | d); break;
+	case U_MAXU: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 8 << 12 | a << 5 | d); break;
+	case U_MULHS: E(0x9b407c00 | b << 16 | a << 5 | d); break;	/* smulh */
+	case U_MULHU: E(0x9bc07c00 | b << 16 | a << 5 | d); break;	/* umulh */
+	case U_CLZ: E(0xdac01000 | a << 5 | d); break;
+	case U_CTZ: E(0xdac00000 | a << 5 | 16); E(0xdac01000 | 16 << 5 | d); break;	/* rbit; clz */
+	case U_BSWAP: E(0xdac00c00 | a << 5 | d); break;				/* rev */
+	case U_POPCNT:
+		fmov_dx(16, a); E(0x0e205800 | 16 << 5 | 16);			/* cnt v16.8b */
+		E(0x0e31b800 | 16 << 5 | 16);					/* addv b16, v16.8b */
+		E(0x0e013c00 | 16 << 5 | d); break;				/* umov w, v16.b[0] */
+	case U_ROTR: op3(0x9ac02c00, d, a, b); break;
+	case U_ROTL: E(0xcb0003e0 | b << 16 | 16); op3(0x9ac02c00, d, a, 16); break;	/* neg x16, b; ror */
+	case U_F2S: fmov_dx(16, a); E(0x1e624000 | 16 << 5 | 16); E(0x1e260000 | 16 << 5 | d); break;	/* fcvt s16, d16; fmov w, s */
+	case U_S2F: E(0x1e270000 | a << 5 | 16); E(0x1e22c000 | 16 << 5 | 16); E(0x9e660000 | 16 << 5 | d); break;	/* fmov s16, w; fcvt d16, s16; fmov x, d16 */
+	case U_ABS: E(0xf100001f | a << 5); E(0xda800400 | a << 16 | 10 << 12 | a << 5 | d); break;	/* cmp #0; cneg */
+	case U_SEL: E(0xf100001f | a << 5); E(0x9a800000 | c << 16 | 1 << 12 | b << 5 | d); break;	/* csel ne */
+	case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: {	/* operands arrive in GPRs; enhancements have no d homes */
+		unsigned base = op == U_FADD ? 0x1e602800u : op == U_FSUB ? 0x1e603800u : op == U_FMUL ? 0x1e600800u : 0x1e601800u;
+		fmov_dx(16, a); fmov_dx(17, b); E(base | 17u << 16 | 16u << 5 | 16u); fmov_xd(d, 16); break; }
+	case U_FEQ: case U_FLT: case U_FLE:
+		fmov_dx(16, a); fmov_dx(17, b); E(0x1e602000 | 17u << 16 | 16u << 5);
+		E((op == U_FEQ ? 0x9a9f17e0u : op == U_FLT ? 0x9a9f57e0u : 0x9a9f87e0u) | d); break;
+	case U_I2F: E(0x9e620000 | a << 5 | 16); fmov_xd(d, 16); break;	/* scvtf d16, Xa */
+	case U_F2I: fmov_dx(16, a); E(0x9e780000 | 16 << 5 | d); break;	/* fcvtzs Xd, d16 */
+	case U_LD:	/* out of range reads 0, like the interpreter */
+		E(0xd2800011); cmp_mw(a, 16); E(0x54000042); ldr_rr(17, M, a, UXTW); movw(d, 17);
+		break;
+	case U_ST: cmp_mw(a, 16); E(0x54000042); str_rr(b, M, a, UXTW); break;
+	case U_LD8: case U_LD16: case U_LD32: case U_ST8: case U_ST16: case U_ST32:
+		emit_byte(op, d, a, b); break;
+	}
+}
+/* an enhancement, inlined: inputs from the data stack into x0..x15, the
+   optimized micro-ops, outputs back. constants are made where they're used,
+   or folded into the instruction (a fixed address, a small addend). */
+static int ekc[MAXUOP * 4 + 16]; static Cell ekv[MAXUOP * 4 + 16];
+static void emit_enh_e(Enh *e, int s)
+{
+	signed char *r = e->reg;
+	int i, j, delta, na;
+	if (!r) {	/* didn't fit in registers: x0 = data stack, w1 = slot, call the interpreter */
+		E(0xaa1403e0); li(W1, s); ldrx_u(X11, ST, 80); blr(X11); E(0xaa0003f4);
+		return;
+	}
+	for (i = 0; i < e->nv; i++) ekc[i] = 0;
+	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { ekc[e->u[i].d] = 1; ekv[e->u[i].d] = e->u[i].imm; }
+#define EOP(x) (ekc[x] ? (li(r[x], ekv[x]), r[x]) : r[x])
+	for (j = 0; j < e->nin; j++) if (r[j] >= 0) ldr_u(r[j], SP, 8 * (e->nin - 1 - j));
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		int d = is_store(u->op) ? 0 : r[u->d], a = 0, b = 0, c = 0;
+		if (u->op == U_LI) continue;
+		if (u->op == U_LD && ekc[u->a] && inmem(ekv[u->a])) {
+			if (ekv[u->a] < 4096) ldr_u(d, M, (int)ekv[u->a] * 8); else ldr_rr(d, M, EOP(u->a), UXTW);
+			continue;
+		}
+		if ((u->op == U_ADD || u->op == U_SUB) && ekc[u->b] && ekv[u->b] >= 0 && ekv[u->b] < 4096) {
+			E((u->op == U_ADD ? 0x91000000 : 0xd1000000) | (unsigned)ekv[u->b] << 10 | EOP(u->a) << 5 | d);
+			continue;
+		}
+		na = nargs(u->op);
+		if (na > 0) a = EOP(u->a);
+		if (na > 1) b = EOP(u->b);
+		if (na > 2) c = EOP(u->c);
+		emit_uop(u->op, d, a, b, c, 16);
+	}
+	delta = 8 * (e->nin - e->nout);
+	if (delta > 0) E(0x91000294 | delta << 10);
+	if (delta < 0) E(0xd1000294 | (-delta) << 10);
+	for (j = 0; j < e->nout; j++) str_u(EOP(e->out[j]), SP, 8 * (e->nout - 1 - j));
+#undef EOP
+}
+static void emit_enh(int s) { emit_enh_e(&enh[s], s); }
+/* lit .. lit xop: the enhancement specialized on the literals that feed its
+   last inputs, so the optimizer sees them as constants (x / 7 becomes a
+   multiply, lfa(k) a load and an add). 0 if it worked. */
+static int spec_enh(int s, Cell *ks, int nk, Enh *te)
+{
+	Enh *e = &enh[s];
+	int nin = e->nin - nk, i, j, out[8], na;
+	static int mp[MAXUOP * 4 + 16];
+	if (e->nv > MAXUOP * 4) return -1;
+	un = 0; unv = nin; uerr = 0;
+	for (j = 0; j < nin; j++) mp[j] = j;
+	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		na = nargs(u->op);
+		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+		if (u->op != U_ST) mp[u->d] = j;
+	}
+	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
+	memset(te, 0, sizeof *te);
+	if (uerr || optimize(te, nin, out, e->nout)) return -1;
+	regalloc(te);
+	if (!te->reg) { free(te->u); return -1; }
+	return 0;
+}
+
+/* ---- o3 regions on arm64 ----
+   int vars live in x0..x7 (x0 is the frame pointer) and float vars in the
+   d register of the same number. block values use x8..x14 and w26..w28, or
+   d8..d15 when the value is a float, so a chain of float ops does not move
+   through an integer register. constants have no register: they are made in
+   a scratch (x16/x17 or d16/d17) where they're used. */
+static signed char rreg[MAXUOP * 4 + 16];
+static unsigned char rk[MAXUOP * 4 + 16], rhome[MAXUOP * 4 + 16], rfp[MAXUOP * 4 + 16];
+static Cell rkv[MAXUOP * 4 + 16]; static int rlast[MAXUOP * 4 + 16];
+static unsigned **rfa; static int *rfb, rnf;	/* branches to blocks, fixed up at the end */
+
+#define LINKN 1024
+static void *linkstk[LINKN];
+static Cell jit_depth;
+typedef struct { void *stub; int fslots; int argc; } FnNat;
+static FnNat *fn_nat;
+static void rforget(Rg *R)
+{
+	int e;
+	if (R->nat) for (e = 0; e < R->ne; e++) if (tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0;
+	if ((unsigned)R->fn < (unsigned)code_words) fn_nat[R->fn].stub = 0;
+	R->nat = 0;
+}
+static void rmark(Rg *R) { R->marked = 1; cinc(cov, R->lo, R->hi); }
+static void runmark(Rg *R) { cdec(cov, R->lo, R->hi); }
+static int ralloc(Rg *R, RB *B)
+{
+	int i, j, k, n = B->n;
+	unsigned fr = 0x7f00u | 0x1c000000u;	/* w8..w14, w26..w28 */
+	unsigned ffr = 0xff00u;			/* d8..d15 */
+	unsigned char *want;
+	for (j = R->nv; j < 8; j++) fr |= 1u << j;
+	for (j = 1; j < R->nv && j < 8; j++) if (R->fv[j]) fr |= 1u << j;	/* that x register is free */
+	for (j = 1; j < 8; j++) if (!R->fv[j]) ffr |= 1u << j;
+	if (B->nv > MAXUOP * 4) return -1;
+	want = calloc(B->nv + 1, 1);
+	class_fp(R, B, want);
+	for (i = 0; i < B->nv; i++) { rlast[i] = -2; rk[i] = 0; rhome[i] = 0; rreg[i] = -1; rfp[i] = want[i]; }
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i]; int na = nargs(u->op);
+		if (na > 0) rlast[u->a] = i;
+		if (na > 1) rlast[u->b] = i;
+		if (na > 2) rlast[u->c] = i;
+	}
+	for (j = 0; j < B->nso; j++) rlast[B->so[2 * j + 1]] = n;
+	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) rlast[B->vs[j]] = n;
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) rlast[B->cv] = n;
+	for (i = 0; i < n; i++) {
+		Uop *u = &B->u[i]; int na = nargs(u->op), ops[3], d = u->d, asfp;
+		ops[0] = u->a; ops[1] = u->b; ops[2] = u->c;
+		for (j = 0; j < na; j++) {
+			int x = ops[j];
+			if (rlast[x] == i && rreg[x] >= 0 && !rhome[x]) {
+				if (rfp[x]) ffr |= 1u << rreg[x]; else fr |= 1u << rreg[x];
+			}
+		}
+		if (is_store(u->op)) continue;
+		if (u->op == U_LI) { rk[d] = 1; rkv[d] = u->imm; continue; }
+		if (u->op == U_VGET) {
+			rreg[d] = (signed char)u->imm; rhome[d] = 1;
+			rfp[d] = (unsigned)u->imm < RMAXV && R->fv[u->imm];
+			continue;
+		}
+		asfp = rfp[d];
+		if (asfp) {
+			for (k = 0; k < 32 && !(ffr >> k & 1); k++);
+			if (k == 32) { free(want); return -1; }
+			ffr &= ~(1u << k);
+		} else {
+			for (k = 0; k < 32 && !(fr >> k & 1); k++);
+			if (k == 32) { free(want); return -1; }
+			fr &= ~(1u << k);
+		}
+		rreg[d] = (signed char)k;
+		if (rlast[d] == -2) { if (asfp) ffr |= 1u << k; else fr |= 1u << k; }
+	}
+	free(want);
+	return 0;
+}
+/* a GPR holding the bits. a float-resident value is moved across. */
+static int rgpr(int x, int s)
+{
+	if (rk[x]) { li(s, rkv[x]); return s; }
+	if (rfp[x]) { fmov_xd(s, rreg[x]); return s; }
+	return rreg[x];
+}
+/* a d register holding the bits. scratch s is d16 or d17. */
+static int rdf(int x, int s)
+{
+	if (rfp[x] && !rk[x]) return rreg[x];
+	if (rk[x]) { li(16, rkv[x]); fmov_dx(s, 16); return s; }
+	fmov_dx(s, rreg[x]);
+	return s;
+}
+static int ropr(int x, int s) { return rgpr(x, s); }
+/* w t <-> the data stack word o places from the top at block entry */
+static void rsw(int t, int o, int store)
+{
+	int by = o * 8;
+	if (by >= 0 && by < 32768) { if (store) str_u(t, SP, by); else ldr_u(t, SP, by); }
+	else if (by >= -256 && by < 256) E((store ? 0xf8000000u : 0xf8400000u) | (unsigned)(by & 0x1ff) << 12 | SP << 5 | t);	/* stur, ldur */
+	else { li(16, by); addx_ext(16, SP, 16, SXTW, 0); if (store) str_u(t, 16, 0); else ldr_u(t, 16, 0); }
+}
+static void rswd(int t, int o, int store)
+{
+	int by = o * 8;
+	if (by >= 0 && by < 32768) { if (store) str_d(t, SP, by); else ldr_d(t, SP, by); }
+	else { li(16, by); addx_ext(16, SP, 16, SXTW, 0); if (store) str_d(t, 16, 0); else ldr_d(t, 16, 0); }
+}
+static void addsp(int by)
+{
+	if (by > 0 && by < 4096) E(0x91000294 | by << 10);
+	else if (by < 0 && -by < 4096) E(0xd1000294 | (-by) << 10);
+	else if (by) { li(16, by); addx_ext(SP, SP, 16, SXTW, 0); }
+}
+static void rspill(Rg *R)
+{
+	int v;
+	for (v = 0; v < R->np; v++) {	/* add x16, x0, #slot */
+		E(0x91000000 | R->ps[v] << 10 | 16);
+		if (R->fv[R->pv[v]]) str_d_rr(R->pv[v], M, 16); else str_rr(R->pv[v], M, 16, UXTW);
+	}
+}
+static void rload(Rg *R)
+{
+	int v;
+	li(16, ZFP); ldr_rr(0, M, 16, UXTW);
+	for (v = 1; v < R->nv; v++) {
+		E(0x91000000 | R->slot[v] << 10 | 16);
+		if (R->fv[v]) ldr_d_rr(v, M, 16); else ldr_rr(v, M, 16, UXTW);
+	}
+}
+/* a direct o3 call blrs the callee's entry stub. the stub saves x30 in linkstk
+   when jit_depth is nonzero, and the return reloads it and rets. depth 0 leaves
+   through x_exit, which is how a region entered from bytecode returns.
+   x17 is the vm address being left. vars are already spilled. a fall off the
+   end (pc == hi) pushes the implicit 0. the frame is given back either way. */
+static void rfinish(Rg *R)
+{
+	unsigned *ne, *z;
+	/* entered from bytecode: the epilogue is still ahead of us and it
+	   pushes the implicit 0 and puts the frame back. a direct caller
+	   skipped that epilogue, so do both here and ret. */
+	li(16, (Cell)(uintptr_t)&jit_depth);
+	ldr_u(X9, 16, 0);
+	z = cp; E(0xb4000000 | X9);		/* cbz x9, leave to the epilogue */
+	li(16, R->hi);
+	E(0xeb10023f);				/* cmp x17, x16 */
+	ne = cp; E(0x54000001);			/* b.ne value already pushed */
+	E(0xd1002294);				/* sub x20, x20, #8 */
+	E(0xf900029f);				/* str xzr, [x20] */
+	cb_fix(ne, cp);
+	if (R->fslots) {
+		li(16, ZFP);
+		ldr_rr(10, M, 16, UXTW);
+		if (R->fslots > 0 && R->fslots < 4096) E(0x91000000 | (R->fslots << 10) | (10 << 5) | 10);
+		else { li(11, R->fslots); E(0x8b0b014a); }	/* add x10, x10, x11 */
+		str_rr(10, M, 16, UXTW);
+	}
+	li(16, (Cell)(uintptr_t)&jit_depth);
+	ldr_u(X9, 16, 0);
+	E(0xd1000529);				/* sub x9, x9, #1 */
+	str_u(X9, 16, 0);
+	li(16, (Cell)(uintptr_t)linkstk);
+	E(0x8b090e10);				/* add x16, x16, x9, lsl #3 */
+	ldr_u(30, 16, 0);
+	E(0xd65f03c0);				/* ret */
+	cb_fix(z, cp);
+	movw(W0, 17);
+	b_to(x_exit);
+}
+static void rexitk(Rg *R, int t)
+{
+	rspill(R);
+	if (t >= R->hi) { li(17, t); rfinish(R); }
+	else { li(W0, t); b_to(x_disp); }
+}
+/* o3-to-o3 is a blr from this region to the callee's entry, with the frame
+   built here. anything else is an ordinary vm call: the return address goes
+   on the vm return stack and x_disp runs the callee. */
+static void rcall(Rg *R, RB *B)
+{
+	unsigned *to_slow, *is_null, *too_deep, *too_small, *cz;
+	int e, hopped;
+	rspill(R);
+	li(16, (Cell)(uintptr_t)&fn_nat); ldr_u(16, 16, 0);
+	cmp_code(17);
+	to_slow = cp; E(0x54000002);		/* b.hs slow */
+	E(0x8b11120c);				/* add x12, x16, x17, lsl #4 */
+	ldr_u(X9, 12, 0);
+	is_null = cp; E(0xb4000000 | X9);	/* cbz x9, slow */
+	li(16, (Cell)(uintptr_t)&jit_depth);
+	ldr_u(10, 16, 0);
+	E(0xf110015f);				/* cmp x10, #1024 */
+	too_deep = cp; E(0x54000002);		/* b.hs slow */
+	E(0xb940098a);				/* ldr w10, [x12, #8] fslots */
+	E(0xb9400d8b);				/* ldr w11, [x12, #12] argc */
+	li(16, ZFP);
+	ldr_rr(13, M, 16, UXTW);
+	E(0xcb0a01ad);				/* sub x13, x13, x10 */
+	li(16, (Cell)(uintptr_t)&frame_floor_word); ldr_u(16, 16, 0); ldr_rr(14, M, 16, UXTW);
+	li(16, (Cell)(uintptr_t)&frame_guard_word); ldr_u(16, 16, 0); ldr_rr(15, M, 16, UXTW);
+	op3(0x8b000000, 14, 14, 15); E(0xeb0e01bf); /* cmp x13, x14 */
+	too_small = cp; E(0x54000003);	/* b.lo slow: bytecode can grow the frame */
+	str_pre(X9, 31, -16);
+	li(16, ZFP);
+	str_rr(13, M, 16, UXTW);
+	cz = cp; E(0x34000000 | 11);		/* cbz w11, copied */
+	E(0x5100056b);				/* sub w11, w11, #1 */
+	ldr_post(14, SP, 8);
+	E(0x8b0b01af);				/* add x15, x13, x11 */
+	str_rr(14, M, 15, UXTW);
+	b_to(cz);
+	cb_fix(cz, cp);
+	ldr_post(X9, 31, 16);			/* stub */
+	li(16, (Cell)(uintptr_t)&jit_depth);
+	ldr_u(10, 16, 0);
+	E(0x9100054a);				/* add x10, x10, #1 */
+	str_u(10, 16, 0);
+	blr(X9);
+	hopped = 0;
+	for (e = 0; e < R->ne; e++) if (R->ent[e] == B->x2) { b_to((unsigned *)R->stub[e]); hopped = 1; break; }
+	if (!hopped) { li(W0, B->x2); b_to(x_disp); }
+	cb_fix(to_slow, cp);
+	cb_fix(is_null, cp);
+	cb_fix(too_deep, cp);
+	cb_fix(too_small, cp);
+	/* nested bytecode runs with its own direct-call depth and returns here. */
+	op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);
+	op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
+	E(0x91000000 | (8 << 10) | (ST << 5));
+	E(0x91000000 | (16 << 10) | (ST << 5) | 1);
+	movw(2, 17); li(3, B->x2); li(16, (Cell)(uintptr_t)o3_invoke_jit); blr(16);
+	ldr_u(X9, ST, 8); E(0x8b090e74); /* sp = m + state[1] * 8 */
+	ldr_u(X9, ST, 16); E(0x8b090e75); /* rp = m + state[2] * 8 */
+	ldrx_u(24, ST, 88); ldrx_u(25, ST, 96); ldrx_u(TAB, ST, 24);
+	li(16, (Cell)(uintptr_t)&halt_at); ldr_u(X9, 16, 0); E(0xf100013f);
+	cz = cp; E(0x5400000b); /* b.lt resumed */
+	movw(W0, X9); b_to(x_exit); cb_fix(cz, cp);
+	hopped = 0;
+	for (e = 0; e < R->ne; e++) if (R->ent[e] == B->x2) { b_to((unsigned *)R->stub[e]); hopped = 1; break; }
+	if (!hopped) { li(W0, B->x2); b_to(x_disp); }
+}
+static void rbranch(int b) { rfa[rnf] = cp; rfb[rnf++] = b; E(0x14000000); }
+/* m[a] = v, checked like the jit's own stores: a word some translation covers goes through hook */
+static void rstore(int a, int v, int fp)
+{
+	unsigned *p1, *p2, *p3;
+	int j;
+	cmp_mw(a, 15);			/* cmp xa, MW */
+	p1 = cp; E(0x54000002);		/* b.hs done */
+	ldrx_u(15, ST, 32);
+	E(0x38604800 | a << 16 | 15 << 5 | 15);	/* ldrb w15, [x15, wa, uxtw] */
+	p2 = cp; E(0x35000000 | 15);	/* cbnz w15, slow */
+	if (fp) str_d_rr(v, M, a); else str_rr(v, M, a, UXTW);
+	p3 = cp; E(0x14000000);
+	cb_fix(p2, cp);
+	E(0xd10203ff);			/* sub sp, sp, #128 */
+	for (j = 0; j < 16; j += 2) E(0xa9000000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);	/* stp xj, xj+1, [sp, #8j] */
+	movw(15, a); if (fp) fmov_xd(1, v); else movw(1, v); movw(0, 15);
+	ldrx_u(16, ST, 64); blr(16);
+	for (j = 0; j < 16; j += 2) E(0xa9400000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
+	E(0x910203ff);			/* add sp, sp, #128 */
+	cb_fix(p1, cp);
+	*p3 |= (cp - p3) & 0x3ffffff;
+}
+static void rpmove(Rg *R, RB *B)
+{
+	int dst[RMAXV], src[RMAXV], n = 0, j, k, x, s;
+	for (j = 1; j < R->nv; j++) if (R->fv[j] && B->vs[j] >= 0) {
+		s = B->vs[j];
+		if (rk[s]) { li(16, rkv[s]); fmov_dx(j, 16); continue; }
+		if (!rfp[s]) { fmov_dx(j, rreg[s]); continue; }
+		if (rreg[s] == j) continue;
+		dst[n] = j; src[n] = rreg[s]; n++;
+	}
+	while (n) {
+		for (j = 0; j < n; j++) { for (k = 0; k < n && src[k] != dst[j]; k++); if (k == n) break; }
+		if (j < n) { fmov_dd(dst[j], src[j]); n--; dst[j] = dst[n]; src[j] = src[n]; continue; }
+		x = src[0]; fmov_dd(16, x);
+		for (k = 0; k < n; k++) if (src[k] == x) src[k] = 16;
+	}
+	n = 0;
+	for (j = 1; j < R->nv; j++) if (!R->fv[j] && B->vs[j] >= 0) {
+		s = B->vs[j];
+		if (rk[s]) { li(j, rkv[s]); continue; }
+		if (rfp[s]) { fmov_xd(j, rreg[s]); continue; }
+		if (rreg[s] == j) continue;
+		dst[n] = j; src[n] = rreg[s]; n++;
+	}
+	while (n) {
+		for (j = 0; j < n; j++) { for (k = 0; k < n && src[k] != dst[j]; k++); if (k == n) break; }
+		if (j < n) { movw(dst[j], src[j]); n--; dst[j] = dst[n]; src[j] = src[n]; continue; }
+		x = src[0]; movw(16, x);
+		for (k = 0; k < n; k++) if (src[k] == x) src[k] = 16;
+	}
+}
+static void emit_uop(int op, int d, int a, int b, int c, int tmp);
+static int remit(Rg *R, int bi)
+{
+	RB *B = &R->b[bi];
+	int i, j, d, a, b, c;
+	unsigned *p;
+	if (ralloc(R, B)) return -1;
+	for (i = 0; i < B->n; i++) {
+		Uop *u = &B->u[i];
+		switch (u->op) {
+		case U_LI: case U_VGET: continue;
+		case U_SLD:
+			if (rfp[u->d] && !rk[u->d]) rswd(rreg[u->d], (int)u->imm, 0);
+			else rsw(rreg[u->d], (int)u->imm, 0);
+			continue;
+		case U_LD:
+			if (rfp[u->d] && !rk[u->d]) {
+				if (rk[u->a] && inmem(rkv[u->a])) {
+					if (rkv[u->a] < 4096) ldr_d(rreg[u->d], M, (int)rkv[u->a] * 8);
+					else { li(16, rkv[u->a]); ldr_d_rr(rreg[u->d], M, 16); }
+				} else ldr_d_rr(rreg[u->d], M, rgpr(u->a, 16));
+				continue;
+			}
+			if (rk[u->a] && inmem(rkv[u->a])) {	/* a fixed address */
+				if (rkv[u->a] < 4096) ldr_u(rreg[u->d], M, (int)rkv[u->a] * 8);
+				else { li(16, rkv[u->a]); ldr_rr(rreg[u->d], M, 16, UXTW); }
+				continue;
+			}
+			break;
+		case U_ST:
+			a = rgpr(u->a, 16);
+			if (rfp[u->b] && !rk[u->b]) rstore(a, rreg[u->b], 1);
+			else rstore(a, rgpr(u->b, 17), 0);
+			continue;
+		case U_LD8: case U_LD16: case U_LD32:
+			emit_byte(u->op, rreg[u->d], rgpr(u->a, 16), 0);
+			continue;
+		case U_ST8: case U_ST16: case U_ST32:
+			emit_byte(u->op, 0, rgpr(u->a, 16), rgpr(u->b, 17));
+			continue;
+		case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: {
+			unsigned base = u->op == U_FADD ? 0x1e602800u : u->op == U_FSUB ? 0x1e603800u : u->op == U_FMUL ? 0x1e600800u : 0x1e601800u;
+			int da = rdf(u->a, 16), db = rdf(u->b, 17);
+			E(base | (unsigned)db << 16 | (unsigned)da << 5 | (unsigned)rreg[u->d]);
+			continue; }
+		case U_FEQ: case U_FLT: case U_FLE: {
+			unsigned cset = u->op == U_FEQ ? 0x9a9f17e0u : u->op == U_FLT ? 0x9a9f57e0u : 0x9a9f87e0u;
+			int da = rdf(u->a, 16), db = rdf(u->b, 17);
+			E(0x1e602000u | (unsigned)db << 16 | (unsigned)da << 5);
+			E(cset | (unsigned)rreg[u->d]);
+			continue; }
+		case U_I2F: E(0x9e620000u | (unsigned)rgpr(u->a, 16) << 5 | (unsigned)rreg[u->d]); continue;
+		case U_F2I: E(0x9e780000u | (unsigned)rdf(u->a, 16) << 5 | (unsigned)rreg[u->d]); continue;
+		case U_F2S: E(0x1e624000u | (unsigned)rdf(u->a, 16) << 5 | 16u); E(0x1e260000u | 16u << 5 | (unsigned)rreg[u->d]); continue;	/* fcvt s16, d; fmov w, s16 */
+		case U_S2F: E(0x1e270000u | (unsigned)rgpr(u->a, 16) << 5 | 16u); E(0x1e22c000u | 16u << 5 | (unsigned)rreg[u->d]); continue;	/* fmov s16, w; fcvt d, s16 */
+		case U_ADD: case U_SUB:
+			if (rk[u->b] && rkv[u->b] >= 0 && rkv[u->b] < 4096) {
+				E((u->op == U_ADD ? 0x91000000 : 0xd1000000) | (unsigned)rkv[u->b] << 10 | ropr(u->a, 16) << 5 | rreg[u->d]);
+				continue;
+			}
+			break;
+		}
+		d = rreg[u->d];
+		a = nargs(u->op) > 0 ? ropr(u->a, 16) : 0;
+		b = nargs(u->op) > 1 ? ropr(u->b, 17) : 0;
+		c = nargs(u->op) > 2 ? ropr(u->c, 15) : 0;
+		emit_uop(u->op, d, a, b, c, 15);
+	}
+	for (j = 0; j < B->nso; j++) {
+		int v = B->so[2 * j + 1];
+		if (rfp[v] && !rk[v]) rswd(rreg[v], B->so[2 * j], 1);
+		else rsw(rgpr(v, 17), B->so[2 * j], 1);
+	}
+	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) {
+		if (rk[B->cv]) li(17, rkv[B->cv]);
+		else if (rfp[B->cv]) fmov_xd(17, rreg[B->cv]);
+		else movw(17, rreg[B->cv]);
+	}
+	addsp(B->delta * 8);
+	rpmove(R, B);
+	switch (B->term) {
+	case T_BR: rbranch(B->s1); break;
+	case T_CBR:
+		if (B->s1 >= 0) { rfa[rnf] = cp; rfb[rnf++] = B->s1; E(0xb4000000 | 17); }	/* cbz x17, taken */
+		else { p = cp; E(0xb5000000 | 17); rexitk(R, B->x1); cb_fix(p, cp); }
+		if (B->s2 >= 0) rbranch(B->s2); else rexitk(R, B->x2);
+		break;
+	case T_CALL: rcall(R, B); break;
+	case T_RET: rspill(R); ldr_pre(W0, RP, -8); b_to(x_disp); break;
+	default: {
+		unsigned *hs;
+		rspill(R); li(16, R->hi); E(0xeb10023f);	/* cmp x17, x16 */
+		hs = cp; E(0x54000002);		/* b.hs finish: the epilogue, or past the region */
+		movw(W0, 17); b_to(x_disp);
+		cb_fix(hs, cp);
+		rfinish(R);
+		break; }
+	}
+	return 0;
+}
+/* the whole region: entry stubs that load the vars, then the blocks */
+static int rjit(Rg *R, int wr)
+{
+	unsigned *start = cp, **lab;
+	int b, e, j, bad = 0;
+	long need = (long)(R->hi - R->lo) * 160 + (long)R->ne * 64 + 8192;
+	if (need > (code + CODEW) - cp) return -1;
+	if (wr) jitwrite(1);
+	lab = malloc((R->nb + 1) * sizeof *lab);
+	rfa = malloc((2 * R->nb + R->ne + 4) * sizeof *rfa); rfb = malloc((2 * R->nb + R->ne + 4) * sizeof *rfb); rnf = 0;
+	for (e = 0; e < R->ne; e++) {
+		unsigned *skip;
+		R->stub[e] = cp;
+		if (e == 0) {	/* a direct caller left its return address in x30 and bumped jit_depth */
+			li(16, (Cell)(uintptr_t)&jit_depth);
+			ldr_u(X9, 16, 0);
+			skip = cp; E(0xb4000000 | X9);		/* cbz x9, plain entry */
+			E(0xd100052a);				/* sub x10, x9, #1 */
+			li(16, (Cell)(uintptr_t)linkstk);
+			E(0x8b0a0e10);				/* add x16, x16, x10, lsl #3 */
+			str_u(30, 16, 0);
+			cb_fix(skip, cp);
+		}
+		rload(R); rbranch(R->entb[e]);
+	}
+	for (b = 0; b < R->nb && !bad; b++) { lab[b] = cp; bad = remit(R, b); }
+	if (!bad) {
+		for (j = 0; j < rnf; j++) {
+			unsigned *at = rfa[j], *to = lab[rfb[j]];
+			if ((*at & 0xfc000000u) == 0x14000000u) *at |= (to - at) & 0x3ffffff; else cb_fix(at, to);
+		}
+		R->nat = start;
+		for (e = 0; e < R->ne; e++) tab[R->ent[e]] = R->stub[e];
+		if ((unsigned)R->fn < (unsigned)code_words) {
+			fn_nat[R->fn].stub = R->stub[0];
+			fn_nat[R->fn].fslots = R->fslots;
+			fn_nat[R->fn].argc = R->argc;
+		}
+		syncicache(start, cp);
+		if (trace) fprintf(stderr, "o3: %d..%d -> %ld bytes of arm64\n", R->lo, R->hi - 1, (long)((char *)cp - (char *)start));
+	} else {
+		cp = start;
+		if (trace) fprintf(stderr, "o3: %d..%d: a block needs too many registers\n", R->lo, R->hi - 1);
+	}
+	free(lab); free(rfa); free(rfb);
+	if (wr) jitwrite(0);
+	return bad;
+}
+static void rcompile(Rg *R, int eager)
+{
+	if (R->nat || R->nonat) return;
+	if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1;	/* registers, not space: give up on it */
+}
+
+static int inst_hook(int d) { return install(d); }
+static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
+
+static void *translate(int p)
+{
+	unsigned *start;
+	int i = p, n = 0, k, t, used[64], nused = 0, j2, ju;
+	Cell w, x;
+	if (cp - code > CODEW - 65536 || !ix_room()) flush();
+	if (o3ent[p]) {	/* an o3 region starts here */
+		Rg *R = rat(p);
+		if (R) { rcompile(R, 0); if (R->nat && tab[p]) return tab[p]; }
+	}
+	start = cp;
+	for (;;) {
+		if (i >= code_words) { E(0x92800000); b_to(x_exit); break; }	/* movn x0, #0: halt */
+		if (n++ >= BLKW) { li(W0, i); b_to(x_disp); break; }
+		w = m[i];
+		if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
+			t = (int)((w - 1) / 2);	/* literal + call/jmp/jz with a known target */
+			if (k == 8) {
+				unsigned *a;
+				ldr_post(X10, SP, 8);
+				a = cp; E(0xb500000a);	/* cbnz x10, fallthrough */
+				li(W0, t); b_to(x_disp);
+				cb_fix(a, cp);
+				i += 2;
+				continue;
+			}
+			li(X10, i + 2);
+			if (k == 6) str_post(X10, RP, 8); else str_u(X10, RP, 0);
+			li(W0, t); b_to(x_disp);
+			i += 2;
+			break;
+		}
+		if (w & 1) {
+			int L = 0, s, nk, j;
+			Cell ks[8];
+			Enh te;
+			while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++;
+			x = i + L < code_words ? m[i + L] : 1;
+			if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
+				nk = L < enh[s].nin ? L : enh[s].nin;
+				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
+				if (!spec_enh(s, ks, nk, &te)) {
+					for (j = 0; j < L - nk; j++) { li(X9, m[i + j] >> 1); str_pre(X9, SP, -8); }
+					if (nused < 64) used[nused++] = s; else used[0] = -1;
+					emit_enh_e(&te, s);
+					free(te.u); free(te.reg);
+					i += L + 1; n += L;
+					continue;
+				}
+			}
+			li(X9, (w - 1) / 2); str_pre(X9, SP, -8); i++; continue;
+		}
+		k = (int)(w / 2);
+		if (k == 0) { E(0x92800000); b_to(x_exit); i++; break; }
+		if (k < 0 || k == 6 || k == 7) {
+			ldr_post(W0, SP, 8);
+			li(X10, i + 1);
+			if (k == 6) str_post(X10, RP, 8); else str_u(X10, RP, 0);
+			if (k < 0) { li(X11, 7 - k); addx_ext(RP, RP, X11, SXTW, 3); }
+			b_to(x_disp);
+			i++;
+			break;
+		}
+		if (k == 8) {	/* jz: taken leaves, not taken keeps going */
+			ldp_post(X9, X10, SP, 16);
+			E(0xb500006a);		/* cbnz x10, +3 */
+			movw(W0, X9);
+			b_to(x_disp);
+			i++;
+			continue;
+		}
+		if (k == 9) { ldr_pre(W0, RP, -8); b_to(x_disp); i++; break; }
+		if (k == 21) {	/* install: call into c, then return to it so it can flush */
+			ldr_u(W0, SP, 0); ldrx_u(X11, ST, 72); blr(X11);
+			E(0x93407c00);		/* sxtw x0, w0: Wn writes zero the high half, so -4 would look positive */
+			str_u(W0, SP, 0);
+			li(W0, i + 1); b_to(x_exit); i++; break;
+		}
+		if (k == 600) {	/* flit: the next word is raw IEEE bits, even when bit 0 is set */
+			if (i + 1 >= code_words) { E(0x92800000); b_to(x_exit); i++; break; }
+			li(X9, m[i + 1]);
+			str_pre(X9, SP, -8);
+			i += 2;
+			continue;
+		}
+		if ((k >= 601 && k <= 611) || k == 618) { fjit(k); i++; continue; }
+		if (k >= 612 && k <= 617) { bjit(k); i++; continue; }
+		if (k == 619) {	/* push the data-stack index: the word s points at */
+			op3(0xcb000000, X9, SP, M);	/* sub x9, x20, x19 */
+			E(0xd343fd29);			/* lsr x9, x9, #3 */
+			str_pre(X9, SP, -8);
+			i++;
+			continue;
+		}
+		if (k == 620) {	/* a native call: the helper pops the arguments and pushes the result */
+			movw(0, SP); movw(1, RP);
+			li(X11, (Cell)(uintptr_t)ncall_host); blr(X11);
+			movw(SP, 0);
+			i++;
+			continue;
+		}
+		if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; emit_enh(k - XOP0); i++; continue; }
+		if (k >= XOP0 && k < XOP0 + NENH) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; }	/* unwired now, but an install makes this a different block */
+		if (k > 20) { E(0x92800000); b_to(x_exit); i++; break; }	/* not wired: halt */
+		op(i, k);
+		i++;
+	}
+	j2 = ix_add(p, i);
+	cinc(cov, p, i);
+	if (nused && used[0] < 0) dep_add(NENH, j2);
+	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2);
+	tab[p] = start;
+	syncicache(start, cp);
+	if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)((char *)cp - (char *)start));
+	return start;
+}
+static int o3_ready(Rg *C)
+{
+	if (!C || C->dead) return 0;
+	if (!C->lifted && rlift(C)) { C->dead = 1; return 0; }
+	if (!C->lifted || C->dead) return 0;
+	rcompile(C, 1);
+	return C->nat != 0;
+}
+static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr)
+{
+	union { unsigned *u; int (*f)(long *, void *); } enter;
+	void *saved_eq = 0, *saved_ent[RMAXE];
+	int i, nent = 0, flushed = 0, ep0 = drop_epoch;
+	Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1;
+	Cell saved_depth = jit_depth, *saved_links = 0;
+	if (saved_depth) {
+		saved_links = malloc((size_t)saved_depth * sizeof *saved_links);
+		if (!saved_links) return PC_FRAME_OVERFLOW;
+		memcpy(saved_links, linkstk, (size_t)saved_depth * sizeof *saved_links);
+	}
+	jit_depth = 0;
+	enter.u = x_enter;
+	if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; }
+	if (re) {
+		nent = re->ne; if (nent > RMAXE) nent = RMAXE;
+		for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; }
+	}
+	S[1] = *ps; S[2] = *pr;
+	while ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
+		void *c;
+		if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
+			Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW;
+			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
+			S[1] = sp; S[2] = rp; S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
+			pc = (int)S[13]; continue;
+		}
+		if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break;
+		if (stop_ge >= 0 && pc >= stop_ge) break;
+		if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break;
+		c = tab[pc];
+		if (!c) {
+			jitwrite(1); c = translate(pc); jitwrite(0);
+			if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0;
+			if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0;
+		}
+		if (!c) break;
+		pc = enter.f(S, c);
+		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
+	}
+	*ps = S[1]; *pr = S[2];
+	jit_depth = saved_depth;
+	if (saved_links) { memcpy(linkstk, saved_links, (size_t)saved_depth * sizeof *saved_links); free(saved_links); }
+	flushed = drop_epoch != ep0;
+	if (!flushed) {
+		if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq;
+		if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i];
+	}
+	return pc;
+}
+static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr)
+{
+	union { unsigned *u; int (*f)(long *, void *); } enter;
+	void *codep;
+	int back;
+	Cell saved_depth, *saved_links = 0;
+	if (!C || C->dead || entry < 0 || entry >= C->ne) return -1;
+	if (!C->lifted && rlift(C)) { C->dead = 1; return -1; }
+	rcompile(C, 1);
+	codep = C->nat ? C->stub[entry] : 0;
+	if (!codep) return -1;
+	enter.u = x_enter;
+	S[1] = *ps; S[2] = *pr;
+	saved_depth = jit_depth;
+	if (saved_depth) {
+		saved_links = malloc((size_t)saved_depth * sizeof *saved_links);
+		if (!saved_links) return PC_FRAME_OVERFLOW;
+		memcpy(saved_links, linkstk, (size_t)saved_depth * sizeof *saved_links);
+	}
+	jit_depth = 0;
+	back = enter.f(S, codep);
+	jit_depth = saved_depth;
+	if (saved_links) { memcpy(linkstk, saved_links, (size_t)saved_depth * sizeof *saved_links); free(saved_links); }
+	*ps = S[1]; *pr = S[2];
+	return back;
+}
+
+#else
+/* portable path for hosts without the arm64 jit: a plain interpreter with
+   the same semantics (shift counts wrap at 64, stores outside memory are
+   ignored, loads outside memory read 0, leaving memory halts). */
+#ifdef ZYLLVM
+/* ---- llvm backend ----
+   a tiered jit for hosts the arm64 backend doesn't cover. the interpreter
+   counts how often each block start is reached; a hot block is translated to
+   llvm ir (same block rules as the arm64 jit: it ends at call, jump, ret or
+   halt, and keeps going past a jz), optimized at O2 and compiled with orc.
+   enhancements are inlined as ir. stores into compiled words drop the blocks
+   that cover them, so self-modifying code still works. */
+#include <llvm-c/Core.h>
+#include <llvm/Config/llvm-config.h>
+#include <llvm-c/Error.h>
+#include <llvm-c/LLJIT.h>
+#include <llvm-c/Orc.h>
+#include <llvm-c/Target.h>
+#include <llvm-c/Transforms/PassBuilder.h>
+#include <llvm-c/TargetMachine.h>
+static int HOT = 5000;		/* runs before a block gets compiled: ZYHOT */
+static const char *lpasses = "default<O2>";	/* ZYPASSES */
+#define LMAXB 65536
+typedef Cell (*Blk)(Cell *mem, Cell *regs);
+static Blk *lt;			/* address -> compiled block */
+static unsigned short *hot;
+static unsigned char *lcov;		/* how many compiled blocks cover the word (255 sticks) */
+static void *lrt[MAXB];			/* each block's resource tracker: removing it frees the block's code */
+static int llvm_ok, nblocks;
+static LLVMOrcLLJITRef jit;
+static LLVMTargetMachineRef tm_;	/* the host cpu, so the pass pipeline's cost models know the target */
+static LLVMOrcThreadSafeContextRef tsc;
+static LLVMContextRef ctx;
+static LLVMTypeRef i8t, i32t, i64t, ptrt, i1t, f64t;
+static LLVMBuilderRef bd;
+static LLVMValueRef rvv[RMAXV];
+static LLVMValueRef fn_, vm_, vregs, vs, vr;	/* function, memory, regs, s and r slots */
+static LLVMTypeRef l_dret;	/* what a register-passing function returns: the next address and s */
+static int l_dm;		/* the function being built is the register-passing one */
+static Cell d_r;		/* r, left here by a register-passing function that exits abnormally */
+static LLVMModuleRef mod_;
+
+static unsigned char *rcov;	/* how many lifted o3 regions cover the word */
+static int rdrop(int a);
+/* code that is dropped is freed when no compiled code is running: a block can
+   drop itself (it stores into its own words), and a callback can nest */
+static int ccode;
+static LLVMOrcResourceTrackerRef pend_rt[1024];
+static int npend_rt;
+static void rt_sweep(void)
+{
+	LLVMErrorRef err;
+	if (ccode) return;
+	while (npend_rt > 0) {
+		LLVMOrcResourceTrackerRef rt = pend_rt[--npend_rt];
+		if ((err = LLVMOrcResourceTrackerRemove(rt))) LLVMConsumeError(err);
+		LLVMOrcReleaseResourceTracker(rt);
+	}
+}
+static void rt_queue(void *rt)
+{
+	if (!rt) return;
+	if (npend_rt >= 1024) { rt_sweep(); if (npend_rt >= 1024) { LLVMOrcReleaseResourceTracker((LLVMOrcResourceTrackerRef)rt); return; } }
+	pend_rt[npend_rt++] = (LLVMOrcResourceTrackerRef)rt;
+}
+static void be_drop(int j)
+{
+	int a = ix_blk[j].a;
+	lt[a] = 0; hot[a] = 0;
+	cdec(lcov, a, ix_blk[j].b);
+	rt_queue(lrt[j]); lrt[j] = 0;
+	if (trace) fprintf(stderr, "llvm: dropped %d..%d\n", a, ix_blk[j].b - 1);
+}
+static void ldrop_all(void)
+{
+	int j;
+	rdrop_all();
+	for (j = 0; j < ix_n; j++) if (ix_blk[j].a >= 0) { rt_queue(lrt[j]); lrt[j] = 0; }
+	memset(rcov, 0, (size_t)code_words);
+	memset(lt, 0, (size_t)code_words * sizeof *lt); memset(lcov, 0, (size_t)code_words); memset(hot, 0, (size_t)code_words * sizeof *hot); ix_reset();
+	if (trace) fprintf(stderr, "llvm: dropped every block\n");
+}
+/* a store to a word some block covers: write it, drop those blocks */
+static int lhook(Cell a, Cell v)
+{
+	int hit;
+	if (m[a] == v) return 0;
+	m[a] = v;
+	hit = ix_drop_at((int)a);
+	if (rcov[a] && rdrop((int)a)) hit = 1;
+	if (hit) drop_epoch++;
+	return hit;
+}
+static Cell linstall(Cell d) { return install((int)d); }
+
+static LLVMValueRef K32(int v) { return LLVMConstInt(i32t, (unsigned)v, 0); }
+static LLVMValueRef Kw(Cell v) { return LLVMConstInt(i64t, (unsigned long long)v, 0); }
+static LLVMValueRef cptr(void *p) { return LLVMConstIntToPtr(LLVMConstInt(i64t, (unsigned long long)(size_t)p, 0), ptrt); }
+static LLVMValueRef mslot(LLVMValueRef idx) { return LLVMBuildInBoundsGEP2(bd, i64t, vm_, &idx, 1, ""); }
+static LLVMValueRef ld(LLVMValueRef idx) { return LLVMBuildLoad2(bd, i64t, mslot(idx), ""); }
+static void st(LLVMValueRef idx, LLVMValueRef v) { LLVMBuildStore(bd, v, mslot(idx)); }
+static LLVMValueRef stack_ir(void) { return LLVMBuildLoad2(bd, i64t, vs, ""); }
+static void set_stack_ir(LLVMValueRef v) { LLVMBuildStore(bd, v, vs); }
+static LLVMValueRef add(LLVMValueRef a, int k) { return LLVMBuildAdd(bd, a, Kw(k), ""); }
+/* a load that reads 0 outside memory */
+static LLVMValueRef safeld(LLVMValueRef a)
+{
+	LLVMValueRef in = LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), "");
+	LLVMValueRef v = ld(LLVMBuildSelect(bd, in, a, Kw(0), ""));
+	return LLVMBuildSelect(bd, in, v, Kw(0), "");
+}
+static void push(LLVMValueRef v) { LLVMValueRef s = add(stack_ir(), -1); set_stack_ir(s); st(s, v); }
+/* leave the block: write s and r back, return the next vm address */
+static void leave(LLVMValueRef next)
+{
+	LLVMValueRef r0 = Kw(0), r1 = Kw(1);
+	if (l_dm) {
+		LLVMValueRef agg;
+		LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&d_r));
+		agg = LLVMBuildInsertValue(bd, LLVMGetUndef(l_dret), next, 0, "");
+		agg = LLVMBuildInsertValue(bd, agg, stack_ir(), 1, "");
+		LLVMBuildRet(bd, agg);
+		return;
+	}
+	LLVMBuildStore(bd, stack_ir(), LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""));
+	LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""));
+	LLVMBuildRet(bd, next);
+}
+/* a function region returns for good: its frame is dead, so its variables are not written back */
+static void leave_ok(LLVMValueRef next)
+{
+	if (l_dm) {
+		LLVMValueRef agg = LLVMBuildInsertValue(bd, LLVMGetUndef(l_dret), next, 0, "");
+		agg = LLVMBuildInsertValue(bd, agg, stack_ir(), 1, "");
+		LLVMBuildRet(bd, agg);
+		return;
+	}
+	LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, rvv[0], ""), mslot(Kw(ZFP)));
+	leave(next);
+}
+static LLVMBasicBlockRef newbb(void) { return LLVMAppendBasicBlockInContext(ctx, fn_, ""); }
+static LLVMValueRef callp(void *f, LLVMTypeRef ret, LLVMTypeRef *at, int n, LLVMValueRef *args)
+{
+	LLVMTypeRef ft = LLVMFunctionType(ret, at, (unsigned)n, 0);
+	return LLVMBuildCall2(bd, ft, cptr(f), args, (unsigned)n, "");
+}
+/* one micro-op as ir: the same semantics as ev() */
+static LLVMValueRef ir_op(int op, LLVMValueRef a, LLVMValueRef b, LLVMValueRef c)
+{
+	LLVMValueRef z = Kw(0), q, d, t, c1, c2, r = 0;
+	LLVMTypeRef i128 = LLVMIntTypeInContext(ctx, 128);
+		switch (op) {
+	case U_ADD: r = LLVMBuildAdd(bd, a, b, ""); break;
+	case U_SUB: r = LLVMBuildSub(bd, a, b, ""); break;
+	case U_MUL: r = LLVMBuildMul(bd, a, b, ""); break;
+	case U_SDIV: case U_SREM:	/* /0 and INT64_MIN/-1 are defined, like the cpu */
+		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
+		c2 = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(INT64_MIN), ""), LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(-1), ""), "");
+		d = LLVMBuildSelect(bd, LLVMBuildOr(bd, c1, c2, ""), Kw(1), b, "");
+		if (op == U_SDIV) { q = LLVMBuildSDiv(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, z, q, ""); }
+		else { q = LLVMBuildSRem(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, a, LLVMBuildSelect(bd, c2, z, q, ""), ""); }
+		break;
+	case U_UDIV: case U_UREM:
+		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
+		d = LLVMBuildSelect(bd, c1, Kw(1), b, "");
+		if (op == U_UDIV) r = LLVMBuildSelect(bd, c1, z, LLVMBuildUDiv(bd, a, d, ""), "");
+		else r = LLVMBuildSelect(bd, c1, a, LLVMBuildURem(bd, a, d, ""), "");
+		break;
+	case U_AND: r = LLVMBuildAnd(bd, a, b, ""); break;
+	case U_OR: r = LLVMBuildOr(bd, a, b, ""); break;
+	case U_XOR: r = LLVMBuildXor(bd, a, b, ""); break;
+	case U_SHL: r = LLVMBuildShl(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
+	case U_SHR: r = LLVMBuildLShr(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
+	case U_SAR: r = LLVMBuildAShr(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
+	case U_NEG: r = LLVMBuildSub(bd, z, a, ""); break;
+	case U_NOT: r = LLVMBuildNot(bd, a, ""); break;
+	case U_LT: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), i64t, ""); break;
+	case U_LTU: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), i64t, ""); break;
+	case U_EQ: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, b, ""), i64t, ""); break;
+	case U_NE: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntNE, a, b, ""), i64t, ""); break;
+	case U_MIN: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), a, b, ""); break;
+	case U_MAX: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSGT, a, b, ""), a, b, ""); break;
+	case U_MINU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), a, b, ""); break;
+	case U_MAXU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntUGT, a, b, ""), a, b, ""); break;
+	case U_MULHS: case U_MULHU: {
+		LLVMValueRef x = op == U_MULHS ? LLVMBuildSExt(bd, a, i128, "") : LLVMBuildZExt(bd, a, i128, "");
+		LLVMValueRef y = op == U_MULHS ? LLVMBuildSExt(bd, b, i128, "") : LLVMBuildZExt(bd, b, i128, "");
+		r = LLVMBuildTrunc(bd, LLVMBuildLShr(bd, LLVMBuildMul(bd, x, y, ""), LLVMConstInt(i128, 64, 0), ""), i64t, "");
+		break; }
+	case U_CLZ: {
+		unsigned id = LLVMLookupIntrinsicID("llvm.ctlz", 9);
+		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[2];
+		args[0] = a; args[1] = LLVMConstInt(i1t, 0, 0);
+		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, 2, "");
+		break; }
+	case U_CTZ: case U_POPCNT: case U_BSWAP: {
+		const char *nm = op == U_CTZ ? "llvm.cttz" : op == U_POPCNT ? "llvm.ctpop" : "llvm.bswap";
+		unsigned id = LLVMLookupIntrinsicID(nm, strlen(nm));
+		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[2];
+		args[0] = a; args[1] = LLVMConstInt(i1t, 0, 0);
+		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, op == U_CTZ ? 2 : 1, "");
+		break; }
+	case U_ROTL: case U_ROTR: {
+		const char *nm = op == U_ROTL ? "llvm.fshl" : "llvm.fshr";
+		unsigned id = LLVMLookupIntrinsicID(nm, strlen(nm));
+		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[3];
+		args[0] = a; args[1] = a; args[2] = LLVMBuildAnd(bd, b, Kw(63), "");
+		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, 3, "");
+		break; }
+	case U_F2S: {
+		LLVMTypeRef f32t = LLVMFloatTypeInContext(ctx), i32t = LLVMInt32TypeInContext(ctx);
+		LLVMValueRef s = LLVMBuildFPTrunc(bd, LLVMBuildBitCast(bd, a, f64t, ""), f32t, "");
+		r = LLVMBuildZExt(bd, LLVMBuildBitCast(bd, s, i32t, ""), i64t, "");
+		break; }
+	case U_S2F: {
+		LLVMTypeRef f32t = LLVMFloatTypeInContext(ctx), i32t = LLVMInt32TypeInContext(ctx);
+		LLVMValueRef s = LLVMBuildBitCast(bd, LLVMBuildTrunc(bd, a, i32t, ""), f32t, "");
+		r = LLVMBuildBitCast(bd, LLVMBuildFPExt(bd, s, f64t, ""), i64t, "");
+		break; }
+	case U_ABS: t = LLVMBuildICmp(bd, LLVMIntSLT, a, z, ""); r = LLVMBuildSelect(bd, t, LLVMBuildSub(bd, z, a, ""), a, ""); break;
+	case U_SEL: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntNE, a, z, ""), b, c, ""); break;
+	case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: case U_FEQ: case U_FLT: case U_FLE: case U_I2F: case U_F2I: {
+		LLVMValueRef da = LLVMBuildBitCast(bd, a, f64t, ""), db = b ? LLVMBuildBitCast(bd, b, f64t, "") : 0, dv = 0;
+		if (op == U_FADD) dv = LLVMBuildFAdd(bd, da, db, "");
+		else if (op == U_FSUB) dv = LLVMBuildFSub(bd, da, db, "");
+		else if (op == U_FMUL) dv = LLVMBuildFMul(bd, da, db, "");
+		else if (op == U_FDIV) dv = LLVMBuildFDiv(bd, da, db, "");
+		else if (op == U_FEQ) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, da, db, ""), i64t, "");
+		else if (op == U_FLT) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, da, db, ""), i64t, "");
+		else if (op == U_FLE) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, da, db, ""), i64t, "");
+		else if (op == U_I2F) dv = LLVMBuildSIToFP(bd, a, f64t, "");
+		else dv = LLVMBuildFPToSI(bd, da, i64t, "");
+		if (op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_I2F) dv = LLVMBuildBitCast(bd, dv, i64t, "");
+		r = dv;
+		break; }
+	}
+	return r;
+}
+/* byte address into the flat image. an out-of-range load is 0. a store that
+   misses the image does nothing, and one that touches translated code goes
+   through the same helper as a word store so the translation is dropped. */
+static LLVMValueRef bload_ir(LLVMValueRef addr, int n)
+{
+	LLVMTypeRef ty = LLVMIntTypeInContext(ctx, (unsigned)n * 8);
+	LLVMValueRef lim = Kw(((Cell)MW << 3) - (n - 1));
+	LLVMValueRef in = LLVMBuildICmp(bd, LLVMIntULT, addr, lim, "");
+	LLVMValueRef safe = LLVMBuildSelect(bd, in, addr, Kw(0), "");
+	LLVMValueRef p = LLVMBuildGEP2(bd, i8t, vm_, &safe, 1, "");
+	LLVMValueRef ldv = LLVMBuildLoad2(bd, ty, p, "");
+	LLVMValueRef z;
+	LLVMSetAlignment(ldv, 1);
+	z = LLVMBuildZExt(bd, ldv, i64t, "");
+	return LLVMBuildSelect(bd, in, z, Kw(0), "");
+}
+static void bstore_ir(LLVMValueRef addr, LLVMValueRef val, int n)
+{
+	LLVMTypeRef ty = LLVMIntTypeInContext(ctx, (unsigned)n * 8);
+	LLVMBasicBlockRef inb, callb, inl, out;
+	LLVMValueRef lim, word, endw, c1, c2, covered, p, stv, args[2];
+	LLVMTypeRef at[2];
+	void *fn = n == 1 ? (void *)bst8 : n == 2 ? (void *)bst16 : (void *)bst32;
+	lim = Kw(((Cell)MW << 3) - (n - 1));
+	inb = newbb(); callb = newbb(); inl = newbb(); out = newbb();
+	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, addr, lim, ""), inb, out);
+	LLVMPositionBuilderAtEnd(bd, inb);
+	word = LLVMBuildLShr(bd, addr, Kw(3), "");
+	endw = LLVMBuildLShr(bd, LLVMBuildAdd(bd, addr, Kw(n - 1), ""), Kw(3), "");
+	c1 = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &word, 1, ""), "");
+	c2 = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &endw, 1, ""), "");
+	covered = LLVMBuildOr(bd, LLVMBuildICmp(bd, LLVMIntNE, c1, LLVMConstInt(i8t, 0, 0), ""),
+		LLVMBuildICmp(bd, LLVMIntNE, c2, LLVMConstInt(i8t, 0, 0), ""), "");
+	LLVMBuildCondBr(bd, covered, callb, inl);
+	LLVMPositionBuilderAtEnd(bd, callb);
+	at[0] = i64t; at[1] = i64t; args[0] = addr; args[1] = val;
+	callp(fn, LLVMVoidTypeInContext(ctx), at, 2, args);
+	LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, inl);
+	p = LLVMBuildGEP2(bd, i8t, vm_, &addr, 1, "");
+	stv = LLVMBuildStore(bd, LLVMBuildTrunc(bd, val, ty, ""), p);
+	LLVMSetAlignment(stv, 1);
+	LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, out);
+}
+/* an enhancement as ir */
+static void ir_enh(Enh *e)
+{
+	LLVMValueRef *v = calloc(e->nv + 1, sizeof *v), s = stack_ir();
+	int i, j;
+	for (j = 0; j < e->nin; j++) v[j] = ld(add(s, e->nin - 1 - j));
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		LLVMValueRef a = u->op == U_LI ? 0 : v[u->a], b = uarity[u->op] > 1 || is_store(u->op) ? v[u->b] : 0, c = uarity[u->op] > 2 ? v[u->c] : 0, r = 0;
+		switch (u->op) {
+		case U_LI: r = Kw(u->imm); break;
+		case U_LD: r = safeld(a); break;
+		case U_ST: {
+			LLVMBasicBlockRef yes = newbb(), done = newbb();
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), yes, done);
+			LLVMPositionBuilderAtEnd(bd, yes); st(a, b); LLVMBuildBr(bd, done);
+			LLVMPositionBuilderAtEnd(bd, done);
+			break; }
+		case U_LD8: case U_LD16: case U_LD32: r = bload_ir(a, bwidth(u->op)); break;
+		case U_ST8: case U_ST16: case U_ST32: bstore_ir(a, b, bwidth(u->op)); break;
+		default: r = ir_op(u->op, a, b, c);
+		}
+		if (!is_store(u->op)) v[u->d] = r;
+	}
+	s = add(s, e->nin - e->nout); set_stack_ir(s);
+	for (j = 0; j < e->nout; j++) st(add(s, e->nout - 1 - j), v[e->out[j]]);
+	free(v);
+}
+static void llvm_init(void)
+{
+	LLVMErrorRef err;
+	if (llvm_ok) return;	/* a nested run() must keep the jit it already built */
+	if (getenv("ZYHOT")) HOT = atoi(getenv("ZYHOT"));
+	if (HOT < 1) HOT = 1;
+	if (HOT > 65000) HOT = 65000;
+	if (getenv("ZYPASSES")) lpasses = getenv("ZYPASSES");
+	LLVMInitializeNativeTarget(); LLVMInitializeNativeAsmPrinter();
+	if ((err = LLVMOrcCreateLLJIT(&jit, 0))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return; }
+#if LLVM_VERSION_MAJOR >= 23
+	ctx = LLVMContextCreate();
+	tsc = LLVMOrcCreateNewThreadSafeContextFromLLVMContext(ctx);
+#else
+	tsc = LLVMOrcCreateNewThreadSafeContext();
+	ctx = LLVMOrcThreadSafeContextGetContext(tsc);
+#endif
+	{	/* a target machine for the same host the jit generates for: llvm's vectorizer,
+		   unroller and inliner price instructions from it, and the passes run before
+		   the module reaches the jit, so without one they would price for nothing */
+		LLVMTargetRef tg; char *e = 0, *cpu = LLVMGetHostCPUName(), *feat = LLVMGetHostCPUFeatures();
+		const char *tr = LLVMOrcLLJITGetTripleString(jit);
+		if (!LLVMGetTargetFromTriple(tr, &tg, &e))
+			tm_ = LLVMCreateTargetMachine(tg, tr, cpu, feat, LLVMCodeGenLevelAggressive, LLVMRelocDefault, LLVMCodeModelJITDefault);
+		else LLVMDisposeMessage(e);
+		LLVMDisposeMessage(cpu); LLVMDisposeMessage(feat);
+	}
+	i1t = LLVMInt1TypeInContext(ctx); i8t = LLVMInt8TypeInContext(ctx); i32t = LLVMInt32TypeInContext(ctx);
+	i64t = LLVMInt64TypeInContext(ctx); f64t = LLVMDoubleTypeInContext(ctx); ptrt = LLVMPointerTypeInContext(ctx, 0);
+	bd = LLVMCreateBuilderInContext(ctx);
+	llvm_ok = 1;
+}
+/* a new module for the jit's triple and data layout, and a function with what is
+   known about its two pointers: the vm memory is MW aligned words, the register
+   pair is two. no noalias: the host helpers it calls read and write that memory. */
+static void lfunc_attrs(LLVMValueRef f)
+{
+	static unsigned kn, ka, kd, knu;
+	LLVMValueRef p0 = LLVMGetParam(f, 0);
+	(void)p0;
+	if (!kn) {
+		kn = LLVMGetEnumAttributeKindForName("nonnull", 7);
+		ka = LLVMGetEnumAttributeKindForName("align", 5);
+		kd = LLVMGetEnumAttributeKindForName("dereferenceable", 15);
+		knu = LLVMGetEnumAttributeKindForName("nounwind", 8);
+	}
+	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, kn, 0));
+	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, ka, 8));
+	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, kd, (unsigned long long)MW * 8));
+	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, kn, 0));
+	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, ka, 8));
+	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, kd, 16));
+	LLVMAddAttributeAtIndex(f, (LLVMAttributeIndex)-1, LLVMCreateEnumAttribute(ctx, knu, 0));
+}
+static void lmodule_target(void)
+{
+	LLVMSetTarget(mod_, LLVMOrcLLJITGetTripleString(jit));
+	LLVMSetDataLayout(mod_, LLVMOrcLLJITGetDataLayoutStr(jit));
+}
+static void lrun_passes(const char *pipeline)
+{
+	LLVMErrorRef err;
+	LLVMPassBuilderOptionsRef po = LLVMCreatePassBuilderOptions();
+	if ((err = LLVMRunPasses(mod_, pipeline, tm_, po))) LLVMConsumeError(err);
+	LLVMDisposePassBuilderOptions(po);
+}
+/* compile the block at p; 0 if it couldn't be */
+static Blk lcompile(int p)
+{
+	char name[32]; LLVMTypeRef pt[2], ft; LLVMValueRef s, a, b, v, r, args[2]; LLVMTypeRef at[2];
+	LLVMOrcExecutorAddress addr; LLVMErrorRef err; LLVMPassBuilderOptionsRef po;
+	int i = p, n = 0, k, t, done = 0, used[64], nused = 0, jb, ju; Cell w;
+	LLVMOrcResourceTrackerRef rt;
+	if (!llvm_ok) return 0;
+	if (!ix_room()) ldrop_all();
+	sprintf(name, "b%d_%d", p, nblocks++);
+	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
+	lmodule_target();
+	pt[0] = ptrt; pt[1] = ptrt;
+	ft = LLVMFunctionType(i64t, pt, 2, 0);
+	fn_ = LLVMAddFunction(mod_, name, ft);
+	lfunc_attrs(fn_);
+	vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1);
+	LLVMPositionBuilderAtEnd(bd, newbb());
+	vs = LLVMBuildAlloca(bd, i64t, "s"); vr = LLVMBuildAlloca(bd, i64t, "r");
+	{ LLVMValueRef r0 = Kw(0), r1 = Kw(1);
+	  set_stack_ir(LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""), ""));
+	  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""), ""), vr); }
+	while (!done) {
+		if (i >= code_words) { leave(Kw(PC_HALT)); break; }
+		if (n++ >= 1024) { leave(Kw(i)); break; }
+		w = m[i];
+		if (w & 1) {
+			/* a literal followed by call, jump or jz: a known target */
+			if (i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
+				t = (int)(w >> 1);
+				if (k == 8) {
+					LLVMBasicBlockRef tk = newbb(), go = newbb();
+					s = stack_ir(); a = ld(s); set_stack_ir(add(s, 1));
+					LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(0), ""), tk, go);
+					LLVMPositionBuilderAtEnd(bd, tk); leave(Kw(t));
+					LLVMPositionBuilderAtEnd(bd, go);
+					i += 2; continue;
+				}
+				if (k == 6) { r = LLVMBuildLoad2(bd, i64t, vr, ""); st(r, Kw(i + 2)); LLVMBuildStore(bd, add(r, 1), vr); }
+				leave(Kw(t)); i += 2; break;
+			}
+			push(Kw(w >> 1)); i++; continue;
+		}
+		k = (int)(w / 2);
+		s = stack_ir();
+		switch (k) {
+		case 0: leave(Kw(PC_HALT)); done = 1; break;
+		case 1: st(s, safeld(ld(s))); break;
+		case 2: {
+			LLVMBasicBlockRef rng = newbb(), in = newbb(), cv = newbb(), plain = newbb(), out = newbb(), quit = newbb(), gate = newbb();
+			LLVMTypeRef gt[4];
+			LLVMValueRef gv[4];
+			a = ld(s); b = ld(add(s, 1)); set_stack_ir(add(s, 2));
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(SYSGATE), ""), gate, rng);
+			LLVMPositionBuilderAtEnd(bd, gate);
+			gt[0] = gt[1] = gt[2] = gt[3] = i64t;
+			gv[0] = b; gv[1] = ld(Kw(SYSGATE + 1)); gv[2] = ld(Kw(SYSGATE + 2)); gv[3] = ld(Kw(SYSGATE + 3));
+			LLVMBuildStore(bd, stack_ir(), cptr(&cb_s));
+			LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&cb_r));
+			st(Kw(SYSGATE), callp((void *)hostcall, i64t, gt, 4, gv));
+			{
+				LLVMBasicBlockRef moved = newbb();
+				LLVMValueRef moving = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(9), ""),
+					LLVMBuildOr(bd, LLVMBuildICmp(bd, LLVMIntEQ, gv[1], Kw(7), ""), LLVMBuildICmp(bd, LLVMIntEQ, gv[1], Kw(8), ""), ""), "");
+				LLVMBuildCondBr(bd, moving, moved, out);
+				LLVMPositionBuilderAtEnd(bd, moved);
+				set_stack_ir(LLVMBuildLoad2(bd, i64t, cptr(&cb_s), ""));
+				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, cptr(&cb_r), ""), vr);
+				LLVMBuildBr(bd, out);
+			}
+			LLVMPositionBuilderAtEnd(bd, rng);
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), in, out);
+			LLVMPositionBuilderAtEnd(bd, in);
+			v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
+			LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
+			LLVMPositionBuilderAtEnd(bd, cv);
+			at[0] = i64t; at[1] = i64t; args[0] = a; args[1] = b;
+			v = callp((void *)lhook, i32t, at, 2, args);
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, K32(0), ""), quit, out);
+			LLVMPositionBuilderAtEnd(bd, quit); leave(Kw(i + 1));	/* a block died: back to the loop */
+			LLVMPositionBuilderAtEnd(bd, out);
+			break; }
+		case 3: a = ld(s); st(s, safeld(LLVMBuildAdd(bd, add(s, 1), a, ""))); break;
+		case 4: set_stack_ir(add(s, 1)); break;
+		case 5: a = ld(s); b = ld(add(s, 1)); st(s, b); st(add(s, 1), a); break;
+		case 6: case 7:
+			a = ld(s); set_stack_ir(add(s, 1));
+			if (k == 6) { r = LLVMBuildLoad2(bd, i64t, vr, ""); st(r, Kw(i + 1)); LLVMBuildStore(bd, add(r, 1), vr); }
+			leave(a); done = 1; break;
+		case 8: {
+			LLVMBasicBlockRef tk = newbb(), go = newbb();
+			a = ld(s); b = ld(add(s, 1)); set_stack_ir(add(s, 2));
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(0), ""), tk, go);
+			LLVMPositionBuilderAtEnd(bd, tk); leave(a);
+			LLVMPositionBuilderAtEnd(bd, go);
+			break; }
+		case 9:
+			r = add(LLVMBuildLoad2(bd, i64t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
+			leave(ld(r)); done = 1; break;
+		case 10: v = callp((void *)Q, i8t, 0, 0, 0); push(LLVMBuildSExt(bd, v, i64t, "")); break;
+		case 11:
+			if (sizeof J == 1) push(LLVMBuildZExt(bd, LLVMBuildLoad2(bd, i8t, cptr(&J), ""), i64t, ""));
+			else push(LLVMBuildSExt(bd, LLVMBuildLoad2(bd, i32t, cptr(&J), ""), i64t, ""));
+			break;
+		case 12: at[0] = i8t; args[0] = LLVMBuildTrunc(bd, ld(s), i8t, ""); callp((void *)G, LLVMVoidTypeInContext(ctx), at, 1, args); set_stack_ir(add(s, 1)); break;
+		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
+			a = ld(s); b = ld(add(s, 1)); s = add(s, 1); set_stack_ir(s);
+			switch (k) {
+			case 13: v = LLVMBuildAdd(bd, b, a, ""); break;
+			case 14: v = LLVMBuildSub(bd, b, a, ""); break;
+			case 15: v = LLVMBuildMul(bd, b, a, ""); break;
+			case 16: v = LLVMBuildAnd(bd, b, a, ""); break;
+			case 17: v = LLVMBuildOr(bd, b, a, ""); break;
+			case 18: v = LLVMBuildXor(bd, b, a, ""); break;
+			case 19: v = LLVMBuildAShr(bd, b, LLVMBuildAnd(bd, a, Kw(63), ""), ""); break;
+			default: v = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, b, a, ""), i64t, "");
+			}
+			st(s, v); break;
+		case 21:	/* install: then back to the loop, which drops every block */
+			at[0] = i64t; args[0] = ld(s);
+			st(s, callp((void *)linstall, i64t, at, 1, args));
+			leave(Kw(i + 1)); done = 1; break;
+		case 600:	/* flit: next word is raw bits */
+			if (i + 1 >= code_words) { leave(Kw(PC_HALT)); done = 1; break; }
+			i++; push(Kw(m[i])); break;
+		case 601: case 602: case 603: case 604: case 605: case 606: case 607: {
+			LLVMValueRef da, db, dv;
+			a = ld(s); b = ld(add(s, 1)); s = add(s, 1); set_stack_ir(s);
+			da = LLVMBuildBitCast(bd, a, f64t, "");
+			db = LLVMBuildBitCast(bd, b, f64t, "");
+			if (k == 601) dv = LLVMBuildFAdd(bd, db, da, "");
+			else if (k == 602) dv = LLVMBuildFSub(bd, db, da, "");
+			else if (k == 603) dv = LLVMBuildFMul(bd, db, da, "");
+			else if (k == 604) dv = LLVMBuildFDiv(bd, db, da, "");
+			else if (k == 605) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, db, da, ""), i64t, "");
+			else if (k == 606) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, db, da, ""), i64t, "");
+			else dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, db, da, ""), i64t, "");
+			if (k <= 604) dv = LLVMBuildBitCast(bd, dv, i64t, "");
+			st(s, dv);
+			break; }
+		case 608:
+			st(s, LLVMBuildBitCast(bd, LLVMBuildSIToFP(bd, ld(s), f64t, ""), i64t, ""));
+			break;
+		case 609:
+			st(s, LLVMBuildFPToSI(bd, LLVMBuildBitCast(bd, ld(s), f64t, ""), i64t, ""));
+			break;
+		case 610: {
+			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
+			callp((void *)fop_print, LLVMVoidTypeInContext(ctx), &ft, 1, &arg);
+			set_stack_ir(add(s, 1));
+			break; }
+		case 611: {
+			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
+			st(s, callp((void *)fop_parse_cell, i64t, &ft, 1, &arg));
+			break; }
+		case 618: {					/* (bits prec mode --) */
+			LLVMTypeRef ft[3];
+			LLVMValueRef args3[3];
+			ft[0] = ft[1] = ft[2] = i64t;
+			args3[0] = ld(add(s, 2));
+			args3[1] = ld(add(s, 1));
+			args3[2] = ld(s);
+			callp((void *)fop_print_mode, LLVMVoidTypeInContext(ctx), ft, 3, args3);
+			set_stack_ir(add(s, 3));
+			break; }
+		case 612: case 614: case 616: {
+			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
+			void *f = k == 612 ? (void *)bld8 : k == 614 ? (void *)bld16 : (void *)bld32;
+			st(s, callp(f, i64t, &ft, 1, &arg));
+			break; }
+		case 619: push(s); break;
+		case 620: {
+			LLVMTypeRef ft = i64t; LLVMValueRef arg = s;
+			LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&cb_r));
+			set_stack_ir(callp((void *)ncall_exec, i64t, &ft, 1, &arg));
+			break; }
+		case 613: case 615: case 617: {
+			LLVMTypeRef ft[2], vt = LLVMVoidTypeInContext(ctx);
+			LLVMValueRef args[2];
+			void *f = k == 613 ? (void *)bst8 : k == 615 ? (void *)bst16 : (void *)bst32;
+			ft[0] = i64t; ft[1] = i64t;
+			args[0] = ld(s); args[1] = ld(add(s, 1));
+			callp(f, vt, ft, 2, args);
+			set_stack_ir(add(s, 2));
+			break; }
+		default:
+			if (k >= XOP0 && k < XOP0 + NENH) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; }
+			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { ir_enh(&enh[k - XOP0]); break; }
+			leave(Kw(PC_HALT)); done = 1;	/* not wired: halt */
+		}
+		i++;
+	}
+	jb = ix_add(p, i);
+	cinc(lcov, p, i);
+	if (nused && used[0] < 0) dep_add(NENH, jb);
+	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], jb);
+	lrun_passes(lpasses);
+	rt = LLVMOrcJITDylibCreateResourceTracker(LLVMOrcLLJITGetMainJITDylib(jit));
+	if ((err = LLVMOrcLLJITAddLLVMIRModuleWithRT(jit, rt, LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
+		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); LLVMOrcReleaseResourceTracker(rt); return 0;
+	}
+	lrt[jb] = rt;
+	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
+	if (trace) fprintf(stderr, "llvm: %d..%d compiled\n", p, i - 1);
+	return (Blk)(size_t)addr;
+}
+#endif
+
+/* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */
+#ifndef ZYLLVM
+static unsigned char *rcov;	/* 1 on the words of a lifted region */
+#endif
+static Cell rvals[MAXUOP * 4 + 16];
+#define RPOOL 40
+static Cell rpool[RPOOL][MAXUOP * 4 + 16];
+static int rpool_n;
+static int rdrop(int a)		/* a store landed on region code: drop those regions */
+{
+	int j, hit = 0;
+	for (j = 0; j < nrg; j++)
+		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
+			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d: %d)\n", rgs[j]->lo, rgs[j]->hi - 1, a, m[a]);
+			rclear(rgs[j]); hit = 1;
+		}
+	return hit;
+}
+#ifdef ZYLLVM
+static int lhook(Cell a, Cell v);
+#define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0)
+#else
+#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0)
+#endif
+/* the region interpreter: ssa values in an array, vars in locals */
+static int rexec(Rg *R, int e, int *ps, int *pr)
+{
+	Cell var[RMAXV], nv2[RMAXV], *v, t = 0, npend[16];
+	int s = *ps, r = *pr, b = R->entb[e], fp, j, i, o, pooled = 0, mine = 0;
+	if (rpool_n < RPOOL) { v = rpool[rpool_n++]; pooled = 1; }
+	else { v = malloc((MAXUOP * 4 + 16) * sizeof(Cell)); if (v) mine = 1; else v = rvals; }
+	var[0] = fp = mread(ZFP);
+	for (j = 1; j < R->nv; j++) var[j] = mread(fp + R->slot[j]);
+	for (;;) {
+		RB *B = &R->b[b];
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			switch (u->op) {
+			case U_LI: v[u->d] = u->imm; break;
+			case U_VGET: v[u->d] = var[u->imm]; break;
+			case U_SLD: v[u->d] = mread(s + u->imm); break;
+			case U_LD: v[u->d] = mread(v[u->a]); break;
+			case U_ST: RSTORE(v[u->a], v[u->b]); break;
+			case U_NARG: npend[u->imm] = v[u->a]; break;
+			case U_NCALL:
+#ifdef ZYFFI
+				cb_s = s; cb_r = r;
+				v[u->d] = ffi_invoke(ncd[u->imm].f, ncd[u->imm].sig, npend, 0);
+#endif
+				break;
+			default:
+				if (u->op >= U_LD8 && u->op <= U_ST32) {
+					if (u->op == U_LD8) v[u->d] = bld8(v[u->a]);
+					else if (u->op == U_LD16) v[u->d] = bld16(v[u->a]);
+					else if (u->op == U_LD32) v[u->d] = bld32(v[u->a]);
+					else if (u->op == U_ST8) bst8(v[u->a], v[u->b]);
+					else if (u->op == U_ST16) bst16(v[u->a], v[u->b]);
+					else if (u->op == U_ST32) bst32(v[u->a], v[u->b]);
+					break;
+				}
+				v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
+			}
+		}
+		for (j = 0; j < B->nso; j++) { o = s + B->so[2 * j]; if ((unsigned)o < MW) m[o] = v[B->so[2 * j + 1]]; }
+		s += B->delta;
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) nv2[j] = v[B->vs[j]];
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) var[j] = nv2[j];
+		if (B->term == T_BR) { b = B->s1; continue; }
+		if (B->term == T_CBR) {
+			if (!v[B->cv]) { if (B->s1 >= 0) { b = B->s1; continue; } t = B->x1; }
+			else { if (B->s2 >= 0) { b = B->s2; continue; } t = B->x2; }
+		} 		else if (B->term == T_JMP) t = v[B->cv];
+		else if (B->term == T_CALL) {
+			Cell cs, cr;
+			for (j = 0; j < R->np; j++) if ((unsigned)(fp + R->ps[j]) < MW) m[fp + R->ps[j]] = var[R->pv[j]];
+			cs = s; cr = r;
+			o3_invoke(&cs, &cr, v[B->cv], B->x2);
+			s = (int)cs; r = (int)cr;
+			if (halt_at >= 0) { t = halt_at; break; }
+			var[0] = fp = mread(ZFP);
+			for (j = 1; j < R->nv; j++) var[j] = mread(fp + R->slot[j]);
+			if ((unsigned)B->x2 < (unsigned)code_words && o3ent[B->x2] == R->idx + 1) { b = R->entb[o3ei[B->x2]]; continue; }
+			t = B->x2;
+			break;
+		}
+		else t = m[--r];
+		break;
+	}
+	for (j = 0; j < R->np; j++) if ((unsigned)(fp + R->ps[j]) < MW) m[fp + R->ps[j]] = var[R->pv[j]];
+	*ps = s; *pr = r;
+	if (pooled) rpool_n--;
+	if (mine) free(v);
+	return (int)t;
+}
+#ifndef ZYLLVM
+static void be_drop(int j) { (void)j; }
+static void rforget(Rg *R) { (void)R; }
+static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); }
+static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); }
+static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
+#endif
+#ifdef ZYLLVM
+
+/* ---- o3 regions in llvm: one function per region, vars in allocas (so
+   mem2reg makes them ssa across blocks), each block's micro-ops as ir, then
+   the O3 pipeline. f(mem, regs, entry) returns the next vm address. */
+typedef Cell (*RFn)(Cell *mem, Cell *regs, int ent);
+static int nregions;
+static void rforget(Rg *R) { rt_queue(R->rt); R->rt = 0; R->nat = 0; R->dnat = 0; }
+static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(lcov, R->lo, R->hi); }
+static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(lcov, R->lo, R->hi); }
+static LLVMValueRef as_f64(LLVMValueRef v, int fp) { return fp ? v : LLVMBuildBitCast(bd, v, f64t, ""); }
+static LLVMValueRef as_i64(LLVMValueRef v, int fp) { return fp ? LLVMBuildBitCast(bd, v, i64t, "") : v; }
+/* the variables go back to their frame slots. before a call from inside the
+   region only the ones that may hold a pointer need to (the rest are never
+   looked at again by anything outside: the region goes on with its own
+   copies), and the frame pointer word is not touched */
+static void rspill_vars(Rg *R, int ptr_only)
+{
+	LLVMValueRef fp = LLVMBuildLoad2(bd, i64t, rvv[0], "");
+	int j;
+	for (j = 0; j < R->np; j++) {
+		int v = R->pv[j];
+		if (ptr_only && R->sc[v]) continue;
+		LLVMValueRef bits = R->fv[v] ? LLVMBuildBitCast(bd, LLVMBuildLoad2(bd, f64t, rvv[v], ""), i64t, "") : LLVMBuildLoad2(bd, i64t, rvv[v], "");
+		st(add(fp, R->ps[j]), bits);
+	}
+}
+static void rspill_ir(Rg *R)
+{
+	st(Kw(ZFP), LLVMBuildLoad2(bd, i64t, rvv[0], ""));
+	rspill_vars(R, 0);
+}
+static LLVMBasicBlockRef rexit_bb(Rg *R, int t)
+{
+	LLVMBasicBlockRef here = LLVMGetInsertBlock(bd), x = newbb();
+	LLVMPositionBuilderAtEnd(bd, x);
+	if (R->fn >= 0 && t >= R->hi) leave_ok(Kw(t));
+	else { rspill_ir(R); leave(Kw(t)); }
+	LLVMPositionBuilderAtEnd(bd, here);
+	return x;
+}
+/* m[a] = v, through lhook if some compiled code covers the word */
+static void rstore_ir(LLVMValueRef a, LLVMValueRef b)
+{
+	LLVMBasicBlockRef in = newbb(), cv = newbb(), plain = newbb(), out = newbb();
+	LLVMValueRef v, args[2]; LLVMTypeRef at[2];
+	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), in, out);
+	LLVMPositionBuilderAtEnd(bd, in);
+	v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
+	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
+	LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, cv);
+	at[0] = i64t; at[1] = i64t; args[0] = a; args[1] = b;
+	callp((void *)lhook, i32t, at, 2, args);
+	LLVMBuildBr(bd, out);
+	LLVMPositionBuilderAtEnd(bd, out);
+}
+#ifdef ZYFFI
+/* a native call as ir. the callee and its signature are known, so llvm sees a
+   call with the real argument types and lays out registers and stack by the
+   platform's rules itself (no limit on the number of arguments). pointer
+   kinds are converted from vm addresses to host addresses inline. the
+   common libm functions on doubles become llvm's own intrinsics, which the
+   backend can turn into one instruction (sqrt) or fold. ai and af are each
+   argument as an i64 and as a double. s0 and rcur are the stacks as of the
+   start of the block, for callbacks. */
+static LLVMValueRef lptr_in(LLVMValueRef v, int kind)
+{
+	LLVMValueRef mb = Kw((Cell)(uintptr_t)m), host, inr, nz = LLVMBuildICmp(bd, LLVMIntNE, v, Kw(0), "");
+	if (kind == 3) {
+		inr = LLVMBuildICmp(bd, LLVMIntULT, v, Kw((Cell)MW * 8), "");
+		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, v, ""), v, "");
+	} else if (kind == 6) {
+		inr = LLVMBuildICmp(bd, LLVMIntULT, v, Kw(MW), "");
+		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, LLVMBuildShl(bd, v, Kw(3), ""), ""), v, "");
+	} else {
+		LLVMValueRef v1 = LLVMBuildAdd(bd, v, Kw(1), "");
+		inr = LLVMBuildICmp(bd, LLVMIntULT, v1, Kw(MW), "");
+		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, LLVMBuildShl(bd, v1, Kw(3), ""), ""), v, "");
+	}
+	return LLVMBuildSelect(bd, nz, host, Kw(0), "");
+}
+static LLVMValueRef lptr_out(LLVMValueRef p, int kind)
+{
+	LLVMValueRef mb = Kw((Cell)(uintptr_t)m), off = LLVMBuildSub(bd, p, mb, "");
+	LLVMValueRef inr = LLVMBuildICmp(bd, LLVMIntULT, off, Kw((Cell)MW * 8), "");
+	if (kind == 6) off = LLVMBuildLShr(bd, off, Kw(3), "");
+	return LLVMBuildSelect(bd, inr, off, p, "");
+}
+static void lext_attr(LLVMValueRef call, unsigned idx, int sign)
+{
+	const char *nm = sign ? "signext" : "zeroext";
+	unsigned kid = LLVMGetEnumAttributeKindForName(nm, strlen(nm));
+	LLVMAddCallSiteAttribute(call, idx, LLVMCreateEnumAttribute(ctx, kid, 0));
+}
+static LLVMValueRef lncall(int di, LLVMValueRef *ai, LLVMValueRef *af, LLVMValueRef s0, LLVMValueRef rcur)
+{
+	Ncd *d = &ncd[di];
+	LLVMTypeRef pt[15], rt, ft, f32t = LLVMFloatTypeInContext(ctx), i16t = LLVMInt16TypeInContext(ctx);
+	LLVMValueRef av[15], call, res;
+	int i, k, rk = (int)(d->sig & 15), allf = rk == 2, na = d->n;
+	LLVMBuildStore(bd, s0, cptr(&cb_s));
+	LLVMBuildStore(bd, rcur, cptr(&cb_r));
+	for (i = 0; i < na; i++) {
+		k = (int)((d->sig >> (4 * (i + 1))) & 15);
+		if (k != 2) allf = 0;
+		switch (k) {
+		case 2: pt[i] = f64t; av[i] = af[i]; break;
+		case 5: pt[i] = f32t; av[i] = LLVMBuildFPTrunc(bd, af[i], f32t, ""); break;
+		case 4: case 7: pt[i] = i32t; av[i] = LLVMBuildTrunc(bd, ai[i], i32t, ""); break;
+		case 8: case 9: pt[i] = i16t; av[i] = LLVMBuildTrunc(bd, ai[i], i16t, ""); break;
+		case 10: case 11: pt[i] = i8t; av[i] = LLVMBuildTrunc(bd, ai[i], i8t, ""); break;
+		case 3: case 6: case 12: pt[i] = i64t; av[i] = lptr_in(ai[i], k); break;
+		default: pt[i] = i64t; av[i] = ai[i]; break;
+		}
+	}
+	rt = rk == 0 ? LLVMVoidTypeInContext(ctx) : rk == 2 ? f64t : rk == 5 ? f32t : (rk == 4 || rk == 7) ? i32t : (rk == 8 || rk == 9) ? i16t : (rk == 10 || rk == 11) ? i8t : i64t;
+	if (allf && na >= 1 && na <= 3) {	/* libm on doubles: the intrinsic */
+		static const struct { const char *c, *ll; int n; } lm[] = {
+			{ "sqrt", "llvm.sqrt.f64", 1 }, { "fabs", "llvm.fabs.f64", 1 }, { "floor", "llvm.floor.f64", 1 },
+			{ "ceil", "llvm.ceil.f64", 1 }, { "trunc", "llvm.trunc.f64", 1 }, { "round", "llvm.round.f64", 1 },
+			{ "sin", "llvm.sin.f64", 1 }, { "cos", "llvm.cos.f64", 1 }, { "exp", "llvm.exp.f64", 1 },
+			{ "exp2", "llvm.exp2.f64", 1 }, { "log", "llvm.log.f64", 1 }, { "log2", "llvm.log2.f64", 1 },
+			{ "log10", "llvm.log10.f64", 1 }, { "pow", "llvm.pow.f64", 2 }, { "copysign", "llvm.copysign.f64", 2 },
+			{ "fma", "llvm.fma.f64", 3 } };
+		for (i = 0; i < (int)(sizeof lm / sizeof lm[0]); i++)
+			if (lm[i].n == na && !strcmp(lm[i].c, d->name)) {
+				LLVMTypeRef it = LLVMFunctionType(f64t, pt, (unsigned)na, 0);
+				LLVMValueRef fn = LLVMGetNamedFunction(mod_, lm[i].ll);
+				if (!fn) fn = LLVMAddFunction(mod_, lm[i].ll, it);
+				return LLVMBuildBitCast(bd, LLVMBuildCall2(bd, it, fn, av, (unsigned)na, ""), i64t, "");
+			}
+	}
+#if defined(__x86_64__)
+	ft = LLVMFunctionType(rt, pt, (unsigned)na, 1);	/* a vararg type makes llvm set al, which a variadic callee reads */
+#else
+	{
+		int fixed = ffi_varargs(d->f);
+		if (fixed >= 0 && fixed <= na) {
+			for (i = fixed; i < na; i++) {
+				if (pt[i] == f32t) { av[i] = LLVMBuildFPExt(bd, av[i], f64t, ""); pt[i] = f64t; }
+				else if (pt[i] == i8t || pt[i] == i16t) {
+					int kind = (int)((d->sig >> (4 * (i + 1))) & 15);
+					av[i] = kind == 9 || kind == 11 ? LLVMBuildZExt(bd, av[i], i32t, "") : LLVMBuildSExt(bd, av[i], i32t, ""); pt[i] = i32t;
+				}
+			}
+			ft = LLVMFunctionType(rt, pt, (unsigned)fixed, 1);
+		} else ft = LLVMFunctionType(rt, pt, (unsigned)na, 0);
+	}
+#endif
+	call = LLVMBuildCall2(bd, ft, cptr(d->f), av, (unsigned)na, "");
+	for (i = 0; i < na; i++) {
+		k = (int)((d->sig >> (4 * (i + 1))) & 15);
+		if (k == 4 || k == 8 || k == 10) lext_attr(call, (unsigned)i + 1, 1);
+		if (k == 7 || k == 9 || k == 11) lext_attr(call, (unsigned)i + 1, 0);
+	}
+	switch (rk) {
+	case 0: res = Kw(0); break;
+	case 2: res = LLVMBuildBitCast(bd, call, i64t, ""); break;
+	case 5: res = LLVMBuildBitCast(bd, LLVMBuildFPExt(bd, call, f64t, ""), i64t, ""); break;
+	case 4: res = LLVMBuildSExt(bd, call, i64t, ""); break;
+	case 8: case 10: res = LLVMBuildSExt(bd, call, i64t, ""); break;
+	case 7: case 9: case 11: res = LLVMBuildZExt(bd, call, i64t, ""); break;
+	case 3: case 6: res = lptr_out(call, rk); break;
+	default: res = call; break;
+	}
+	return res;
+}
+#endif
+static LLVMValueRef l_dfn;	/* the register-passing function being built or declared for this region */
+static int has_ret(Rg *R)
+{
+	int b;
+	for (b = 0; b < R->nb; b++) if (R->b[b].term == T_RET) return 1;
+	return 0;
+}
+/* an o3 call to a register-passing callee is inlined: the callee's body is built again into
+   the caller's module as an internal function, so llvm can inline it there. the copy is good
+   for one generation of the callee (rclear bumps gen whenever the callee's code is dropped), and
+   the caller checks that generation before the fast path, so a stale copy is never entered. */
+#define INL_MAX 8		/* copies in one module */
+#define INL_BLOCKS 24		/* largest callee copied */
+typedef struct { Rg *C; unsigned gen; LLVMValueRef fn; } InlEnt;
+static InlEnt inl_tab[INL_MAX];
+static int ninl, inl_depth;
+static void lbody(Rg *R, int direct, LLVMValueRef fnv);
+static LLVMValueRef inline_fn(Rg *C, unsigned *gen)
+{
+	static int off = -1, inl_n;
+	LLVMValueRef sfn, sdfn, svm, svregs, svs, svr, nf, srv[RMAXV];
+	LLVMBasicBlockRef at;
+	LLVMTypeRef dpt[6];
+	char nm[24];
+	int i, j, sdm;
+	if (off < 0) off = getenv("ZYNOINLINE") != 0;
+	if (off || inl_depth || !C->lifted || !C->dnat || C->nb > INL_BLOCKS || has_ret(C)) return 0;
+	for (i = 0; i < ninl; i++) if (inl_tab[i].C == C && inl_tab[i].gen == C->gen) { *gen = C->gen; return inl_tab[i].fn; }
+	if (ninl >= INL_MAX) return 0;
+	at = LLVMGetInsertBlock(bd); sfn = fn_; sdfn = l_dfn; sdm = l_dm;
+	svm = vm_; svregs = vregs; svs = vs; svr = vr;
+	memcpy(srv, rvv, sizeof rvv);
+	for (j = 0; j < 6; j++) dpt[j] = i64t;
+	sprintf(nm, "inl_%d", inl_n++);
+	nf = LLVMAddFunction(mod_, nm, LLVMFunctionType(l_dret, dpt, 6, 0));
+	LLVMSetLinkage(nf, LLVMInternalLinkage);
+	l_dfn = nf;
+	inl_depth++;
+	lbody(C, 1, nf);
+	inl_depth--;
+	fn_ = sfn; l_dfn = sdfn; l_dm = sdm;
+	vm_ = svm; vregs = svregs; vs = svs; vr = svr;
+	memcpy(rvv, srv, sizeof rvv);
+	LLVMPositionBuilderAtEnd(bd, at);
+	inl_tab[ninl].C = C; inl_tab[ninl].gen = C->gen; inl_tab[ninl].fn = nf; ninl++;
+	*gen = C->gen;
+	return nf;
+}
+/* one function for the region. the plain one is f(mem, regs, entry) and takes
+   the vm's own protocol: s and r in regs, variables from the frame. the
+   register-passing one is fd(s, r, fp, a0, a1, a2), entered only at the top of
+   a function by another region: the arguments are the variables, the frame
+   pointer is a value, and it returns the next address and s. */
+/* the callee of a call to a known address, if it is a function region that takes its
+   arguments in registers */
+static Rg *dcallee(LLVMValueRef tgt)
+{
+	Rg *C;
+	if (!LLVMIsAConstantInt(tgt)) return 0;
+	C = fnlookup((int)LLVMConstIntGetZExtValue(tgt));
+	if (!C || C->dead || C->fn < 0 || C->argc > 3 || C->fslots < 0) return 0;
+	return C;
+}
+static void lbody(Rg *R, int direct, LLVMValueRef fnv)
+{
+	LLVMTypeRef pt[3], ft; LLVMValueRef sw = 0, *val, s0, fp, r, ent = 0, rg2;
+	LLVMBasicBlockRef *bb, *ebs, eb, dflt = 0;
+	int b, i, j, e;
+	LLVMValueRef npi[16], npf[16], dev[4];
+	int deo[4], nde = 0;
+	Rg *Cp = 0;
+	(void)pt; (void)ft;
+	fn_ = fnv;
+	l_dm = direct;
+	LLVMPositionBuilderAtEnd(bd, newbb());
+	vs = LLVMBuildAlloca(bd, i64t, "s"); vr = LLVMBuildAlloca(bd, i64t, "r");
+	rg2 = LLVMBuildAlloca(bd, LLVMArrayType(i64t, 2), "regs");
+	for (j = 0; j < R->nv; j++) rvv[j] = LLVMBuildAlloca(bd, j && R->fv[j] ? f64t : i64t, "");
+	bb = malloc(R->nb * sizeof *bb);
+	ebs = calloc(R->ne + 1, sizeof *ebs);
+	for (b = 0; b < R->nb; b++) bb[b] = newbb();
+	if (direct) {
+		vm_ = cptr(m); vregs = 0;
+		LLVMBuildStore(bd, LLVMGetParam(fn_, 0), vs);
+		LLVMBuildStore(bd, LLVMGetParam(fn_, 1), vr);
+		LLVMBuildStore(bd, LLVMGetParam(fn_, 2), rvv[0]);
+		for (j = 1; j < R->nv; j++) {
+			LLVMValueRef bits = Kw(0);
+			if (R->slot[j] >= 0 && R->slot[j] < R->argc && R->slot[j] < 3) bits = LLVMGetParam(fn_, 3 + R->slot[j]);
+			if (R->fv[j]) bits = LLVMBuildBitCast(bd, bits, f64t, "");
+			LLVMBuildStore(bd, bits, rvv[j]);
+		}
+		LLVMBuildBr(bd, bb[R->entb[0]]);
+	} else {
+		vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1); ent = LLVMGetParam(fn_, 2);
+		{ LLVMValueRef r0 = Kw(0), r1 = Kw(1);
+		  set_stack_ir(LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""), ""));
+		  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""), ""), vr); }
+		dflt = newbb();
+		sw = LLVMBuildSwitch(bd, ent, dflt, (unsigned)R->ne);
+	}
+	for (e = direct ? 1 : 0; e < R->ne; e++) {
+		eb = ebs[e] = newbb();
+		if (!direct) LLVMAddCase(sw, K32(e), eb);
+		LLVMPositionBuilderAtEnd(bd, eb);
+		fp = ld(Kw(ZFP));
+		LLVMBuildStore(bd, fp, rvv[0]);
+		for (j = 1; j < R->nv; j++) {
+			LLVMValueRef bits = safeld(add(fp, R->slot[j]));
+			if (R->fv[j]) bits = LLVMBuildBitCast(bd, bits, f64t, "");
+			LLVMBuildStore(bd, bits, rvv[j]);
+		}
+		LLVMBuildBr(bd, bb[R->entb[e]]);
+	}
+	if (!direct) { LLVMPositionBuilderAtEnd(bd, dflt); leave(Kw(-1)); }
+	val = calloc(R->maxnv + 1, sizeof *val);
+	for (b = 0; b < R->nb; b++) {
+		RB *B = &R->b[b];
+		unsigned char *fpv = calloc(B->nv + 1, 1);
+		class_fp(R, B, fpv);
+		LLVMPositionBuilderAtEnd(bd, bb[b]);
+		s0 = stack_ir();
+		for (i = 0; i < B->n; i++) {
+			Uop *u = &B->u[i];
+			int na = nargs(u->op);
+			LLVMValueRef a = na > 0 ? val[u->a] : 0, x = na > 1 ? val[u->b] : 0, c = na > 2 ? val[u->c] : 0;
+			LLVMValueRef ai = na > 0 ? as_i64(a, fpv[u->a]) : 0, af = na > 0 ? as_f64(a, fpv[u->a]) : 0;
+			LLVMValueRef xi = na > 1 ? as_i64(x, fpv[u->b]) : 0, xf = na > 1 ? as_f64(x, fpv[u->b]) : 0;
+			LLVMValueRef ci = na > 2 ? as_i64(c, fpv[u->c]) : 0;
+			switch (u->op) {
+			case U_LI: val[u->d] = Kw(u->imm); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
+			case U_VGET:
+				val[u->d] = LLVMBuildLoad2(bd, (unsigned)u->imm < RMAXV && R->fv[u->imm] ? f64t : i64t, rvv[u->imm], "");
+				break;
+			case U_SLD: val[u->d] = ld(add(s0, (int)u->imm)); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
+			case U_LD: val[u->d] = safeld(ai); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
+			case U_ST: rstore_ir(ai, xi); break;
+			case U_NARG: npi[u->imm] = ai; npf[u->imm] = af; break;
+			case U_NCALL:
+#ifdef ZYFFI
+				val[u->d] = lncall((int)u->imm, npi, npf, s0, LLVMBuildLoad2(bd, i64t, vr, ""));
+				if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, "");
+#endif
+				break;
+			case U_LD8: case U_LD16: case U_LD32: val[u->d] = bload_ir(ai, bwidth(u->op)); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
+			case U_ST8: case U_ST16: case U_ST32: bstore_ir(ai, xi, bwidth(u->op)); break;
+			case U_FADD: val[u->d] = LLVMBuildFAdd(bd, af, xf, ""); break;
+			case U_FSUB: val[u->d] = LLVMBuildFSub(bd, af, xf, ""); break;
+			case U_FMUL: val[u->d] = LLVMBuildFMul(bd, af, xf, ""); break;
+			case U_FDIV: val[u->d] = LLVMBuildFDiv(bd, af, xf, ""); break;
+			case U_FEQ: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, af, xf, ""), i64t, ""); break;
+			case U_FLT: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, af, xf, ""), i64t, ""); break;
+			case U_FLE: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, af, xf, ""), i64t, ""); break;
+			case U_I2F: val[u->d] = LLVMBuildSIToFP(bd, ai, f64t, ""); break;
+			case U_F2I: val[u->d] = LLVMBuildFPToSI(bd, af, i64t, ""); break;
+			default: val[u->d] = ir_op(u->op, ai, xi, ci); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
+			}
+		}
+		Cp = B->term == T_CALL && B->cv >= 0 ? dcallee(as_i64(val[B->cv], fpv[B->cv])) : 0;
+		nde = 0;
+		for (j = 0; j < B->nso; j++) {
+			int off = B->so[2 * j];
+			LLVMValueRef sv_ = as_i64(val[B->so[2 * j + 1]], fpv[B->so[2 * j + 1]]);
+			if (Cp && off >= B->delta && off < B->delta + Cp->argc) { deo[nde] = off; dev[nde++] = sv_; continue; }	/* the register call does not need them in memory */
+			st(add(s0, off), sv_);
+		}
+		set_stack_ir(add(s0, B->delta));
+		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
+			LLVMValueRef bits = R->fv[j] ? as_f64(val[B->vs[j]], fpv[B->vs[j]]) : as_i64(val[B->vs[j]], fpv[B->vs[j]]);
+			LLVMBuildStore(bd, bits, rvv[j]);
+		}
+		switch (B->term) {
+		case T_BR: LLVMBuildBr(bd, bb[B->s1]); break;
+		case T_CBR:
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, as_i64(val[B->cv], fpv[B->cv]), Kw(0), ""),
+				B->s1 >= 0 ? bb[B->s1] : rexit_bb(R, B->x1), B->s2 >= 0 ? bb[B->s2] : rexit_bb(R, B->x2));
+			break;
+		case T_CALL: {
+			LLVMTypeRef at[4]; LLVMValueRef args[4]; LLVMBasicBlockRef bad, okb, slowb, fastb, helpb, finb, eqb, doneb, chk; int ee;
+			LLVMValueRef tgt = as_i64(val[B->cv], fpv[B->cv]), natp = 0, fpn = 0, sv = 0, ret = 0;
+			Rg *C = 0;
+			rspill_vars(R, 1);
+			for (ee = 0; ee < R->ne; ee++) if (R->ent[ee] == B->x2) break;
+			if (LLVMIsAConstantInt(tgt)) {
+				C = fnlookup((int)LLVMConstIntGetZExtValue(tgt));
+				if (C && (C->dead || C->argc > 12 || C->fslots < 0)) C = 0;
+			}
+			slowb = newbb(); okb = newbb(); chk = newbb();
+			if (C && C->fn >= 0 && C->argc <= 3) {
+				/* a callee that is a function region: its arguments go in registers, its frame is a value,
+				   and it comes back with the next address and s. the callee may not be compiled yet, or the
+				   frame or the stack may be about to run out: the helper takes those */
+				LLVMBasicBlockRef dfast = newbb(), dnorm = newbb(), dabn = newbb(), deq = newbb(), dcont = newbb();
+				LLVMValueRef dn, cond, fpc, a3[3], da[6], res, rret, rs2, okf, tmp0, tmp1;
+				LLVMTypeRef dt = LLVMFunctionType(l_dret, (LLVMTypeRef[]){ i64t, i64t, i64t, i64t, i64t, i64t }, 6, 0);
+				int ai2;
+				unsigned igen = 0;
+				LLVMValueRef inl = C == R ? 0 : inline_fn(C, &igen);
+				if (C == R) dn = l_dfn;
+				else if (inl) dn = inl;
+				else dn = LLVMBuildLoad2(bd, ptrt, cptr(&C->dnat), "");
+				fpc = LLVMBuildLoad2(bd, i64t, rvv[0], "");
+				fpn = LLVMBuildSub(bd, fpc, Kw(C->fslots), "");
+				sv = stack_ir();
+				cond = LLVMBuildICmp(bd, LLVMIntUGE, fpn,
+					LLVMBuildAdd(bd, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_floor_word), "")), ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_guard_word), "")), ""), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, fpn, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_top_word), "")), ""), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntUGE, add(sv, -16), Kw(DSLIM), ""), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, sv, Kw(DSHI), ""), "");
+				if (inl) cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntEQ, LLVMBuildLoad2(bd, i32t, cptr(&C->gen), ""), K32((int)igen), ""), "");
+				else if (C != R) cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntNE, LLVMBuildPtrToInt(bd, dn, i64t, ""), Kw(0), ""), "");
+				LLVMBuildCondBr(bd, cond, dfast, slowb);
+				LLVMPositionBuilderAtEnd(bd, dfast);
+				for (ai2 = 0; ai2 < 3; ai2++) {
+					int off = B->delta + C->argc - 1 - ai2, q;
+					a3[ai2] = Kw(0);
+					if (ai2 >= C->argc) continue;
+					a3[ai2] = 0;
+					for (q = 0; q < nde; q++) if (deo[q] == off) a3[ai2] = dev[q];
+					if (!a3[ai2]) a3[ai2] = ld(add(sv, C->argc - 1 - ai2));
+				}
+				da[0] = add(sv, C->argc); da[1] = LLVMBuildLoad2(bd, i64t, vr, "");
+				da[2] = fpn; da[3] = a3[0]; da[4] = a3[1]; da[5] = a3[2];
+				res = LLVMBuildCall2(bd, dt, dn, da, 6, "");
+				rret = LLVMBuildExtractValue(bd, res, 0, "");
+				rs2 = LLVMBuildExtractValue(bd, res, 1, "");
+				okf = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntUGE, rret, Kw(C->hi), ""), LLVMBuildICmp(bd, LLVMIntULT, rret, Kw(MW), ""), "");
+				LLVMBuildCondBr(bd, okf, dnorm, dabn);
+				LLVMPositionBuilderAtEnd(bd, dnorm);
+				set_stack_ir(rs2);
+				LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, rret, Kw(C->hi), ""), deq, dcont);
+				LLVMPositionBuilderAtEnd(bd, deq);
+				{ LLVMValueRef sn = add(stack_ir(), -1); st(sn, Kw(0)); set_stack_ir(sn); }
+				LLVMBuildBr(bd, dcont);
+				LLVMPositionBuilderAtEnd(bd, dcont);
+				if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
+				LLVMPositionBuilderAtEnd(bd, dabn);
+				tmp0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, "");
+				tmp1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
+				LLVMBuildStore(bd, rs2, tmp0);
+				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, cptr(&d_r), ""), tmp1);
+				at[0] = ptrt; at[1] = ptrt; at[2] = ptrt; at[3] = i64t;
+				args[0] = tmp0; args[1] = tmp1; args[2] = cptr(C); args[3] = rret;
+				callp((void *)o3_finish_c, LLVMVoidTypeInContext(ctx), at, 4, args);
+				set_stack_ir(LLVMBuildLoad2(bd, i64t, tmp0, ""));
+				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, tmp1, ""), vr);
+				LLVMBuildBr(bd, chk);
+				(void)fastb; (void)helpb; (void)finb; (void)eqb; (void)doneb; (void)natp; (void)ret;
+			} else if (C) {
+				/* a callee that is already compiled is entered from here: a frame below this one,
+				   the arguments copied across, a call through its entry. the other cases (not
+				   compiled yet, a frame outside memory) take the helper */
+				LLVMValueRef cond, fz;
+				fastb = newbb(); helpb = newbb(); finb = newbb(); eqb = newbb(); doneb = newbb();
+				natp = LLVMBuildLoad2(bd, ptrt, cptr(&C->nat), "");
+				fz = LLVMBuildLoad2(bd, i64t, rvv[0], "");
+				fpn = LLVMBuildSub(bd, fz, Kw(C->fslots), "");
+				sv = stack_ir();
+				cond = LLVMBuildICmp(bd, LLVMIntNE, LLVMBuildPtrToInt(bd, natp, i64t, ""), Kw(0), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntUGE, fpn,
+					LLVMBuildAdd(bd, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_floor_word), "")), ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_guard_word), "")), ""), ""), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, fpn, Kw(MW - 64), ""), "");
+				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, add(sv, -1), Kw(MW - 64), ""), "");
+				LLVMBuildCondBr(bd, cond, fastb, slowb);
+				LLVMPositionBuilderAtEnd(bd, fastb);
+				{
+					LLVMTypeRef ct[3]; LLVMValueRef ca[3], r1, haltv, okf, sn, rv;
+					int i2;
+					st(Kw(ZFP), fpn);
+					for (i2 = 0; i2 < C->argc; i2++) st(add(fpn, i2), ld(add(sv, C->argc - 1 - i2)));
+					r1 = Kw(1);
+					LLVMBuildStore(bd, add(sv, C->argc), rg2);
+					LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), LLVMBuildGEP2(bd, i64t, rg2, &r1, 1, ""));
+					ct[0] = ptrt; ct[1] = ptrt; ct[2] = i32t;
+					ca[0] = vm_; ca[1] = rg2; ca[2] = K32(0);
+					ret = LLVMBuildCall2(bd, LLVMFunctionType(i64t, ct, 3, 0), natp, ca, 3, "");
+					set_stack_ir(LLVMBuildLoad2(bd, i64t, rg2, ""));
+					LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, rg2, &r1, 1, ""), ""), vr);
+					haltv = LLVMBuildLoad2(bd, i64t, cptr(&halt_at), "");
+					okf = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntUGE, ret, Kw(C->hi), ""), LLVMBuildICmp(bd, LLVMIntULT, ret, Kw(MW), ""), "");
+					okf = LLVMBuildAnd(bd, okf, LLVMBuildICmp(bd, LLVMIntSLT, haltv, Kw(0), ""), "");
+					LLVMBuildCondBr(bd, okf, finb, helpb);
+					LLVMPositionBuilderAtEnd(bd, finb);
+					LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, ret, Kw(C->hi), ""), eqb, doneb);
+					LLVMPositionBuilderAtEnd(bd, eqb);
+					sn = add(stack_ir(), -1); st(sn, Kw(0)); set_stack_ir(sn);
+					LLVMBuildBr(bd, doneb);
+					LLVMPositionBuilderAtEnd(bd, doneb);
+					rv = ld(Kw(ZFP));
+					st(Kw(ZFP), add(rv, C->fslots));
+					if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
+					LLVMPositionBuilderAtEnd(bd, helpb);
+					{
+						LLVMValueRef t0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, ""), t1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
+						LLVMBuildStore(bd, stack_ir(), t0); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), t1);
+						at[0] = ptrt; at[1] = ptrt; at[2] = ptrt; at[3] = i64t;
+						args[0] = t0; args[1] = t1; args[2] = cptr(C); args[3] = ret;
+						callp((void *)o3_finish_c, LLVMVoidTypeInContext(ctx), at, 4, args);
+						set_stack_ir(LLVMBuildLoad2(bd, i64t, t0, "")); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, t1, ""), vr);
+					}
+					LLVMBuildBr(bd, chk);
+				}
+			} else LLVMBuildBr(bd, slowb);
+			LLVMPositionBuilderAtEnd(bd, slowb);
+			{ int q; for (q = 0; q < nde; q++) st(add(s0, deo[q]), dev[q]); }
+			st(Kw(ZFP), LLVMBuildLoad2(bd, i64t, rvv[0], ""));
+			{
+				LLVMValueRef t0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, ""), t1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
+				LLVMBuildStore(bd, stack_ir(), t0); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), t1);
+				at[0] = ptrt; at[1] = ptrt; at[2] = i64t; at[3] = i64t;
+				args[0] = t0; args[1] = t1;
+				args[2] = tgt; args[3] = Kw(B->x2);
+				callp((void *)o3_invoke, LLVMVoidTypeInContext(ctx), at, 4, args);
+				set_stack_ir(LLVMBuildLoad2(bd, i64t, t0, "")); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, t1, ""), vr);
+			}
+			LLVMBuildBr(bd, chk);
+			LLVMPositionBuilderAtEnd(bd, chk);
+			bad = newbb();
+			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntSGE, LLVMBuildLoad2(bd, i64t, cptr(&halt_at), ""), Kw(0), ""), bad, okb);
+			LLVMPositionBuilderAtEnd(bd, bad); leave(l_dm ? Kw(-1) : LLVMBuildLoad2(bd, i64t, cptr(&halt_at), ""));	/* a direct callee says -1: any address could look like a return */
+			LLVMPositionBuilderAtEnd(bd, okb);
+			if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
+			(void)fpn; (void)sv;
+			break; }
+		case T_RET:
+			r = add(LLVMBuildLoad2(bd, i64t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
+			rspill_ir(R); leave(ld(r));
+			break;
+		default: {
+			LLVMValueRef tv = as_i64(val[B->cv], fpv[B->cv]);
+			if (R->fn >= 0 && LLVMIsAConstantInt(tv) && (long long)LLVMConstIntGetSExtValue(tv) >= R->hi) leave_ok(tv);
+			else { rspill_ir(R); leave(tv); }
+		}
+		}
+		free(fpv);
+	}
+	free(val); free(bb); free(ebs);
+	l_dm = 0;
+}
+static void *lregion(Rg *R)
+{
+	char name[40], dname[48]; LLVMTypeRef pt[3], ft, dpt[6]; LLVMValueRef f1, f2 = 0;
+	LLVMOrcExecutorAddress addr; LLVMErrorRef err;
+	int dir, i;
+	sprintf(name, "o3_%d_%d", R->lo, nregions++);
+	sprintf(dname, "%s_d", name);
+	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
+	ninl = 0;
+	lmodule_target();
+	pt[0] = ptrt; pt[1] = ptrt; pt[2] = i32t;
+	ft = LLVMFunctionType(i64t, pt, 3, 0);
+	f1 = LLVMAddFunction(mod_, name, ft);
+	lfunc_attrs(f1);
+	dir = R->fn >= 0 && R->argc <= 3 && R->fslots >= 0 && !has_ret(R);
+	if (dir) {
+		static unsigned knu;
+		LLVMTypeRef rt2[2];
+		rt2[0] = i64t; rt2[1] = i64t;
+		l_dret = LLVMStructTypeInContext(ctx, rt2, 2, 0);
+		for (i = 0; i < 6; i++) dpt[i] = i64t;
+		f2 = LLVMAddFunction(mod_, dname, LLVMFunctionType(l_dret, dpt, 6, 0));
+		if (!knu) knu = LLVMGetEnumAttributeKindForName("nounwind", 8);
+		LLVMAddAttributeAtIndex(f2, (LLVMAttributeIndex)-1, LLVMCreateEnumAttribute(ctx, knu, 0));
+	}
+	l_dfn = f2;
+	lbody(R, 0, f1);
+	if (dir) lbody(R, 1, f2);
+	l_dfn = 0;
+	lrun_passes("default<O3>");
+	if (getenv("ZYLLIR")) LLVMDumpModule(mod_);
+	{ LLVMOrcResourceTrackerRef rt = LLVMOrcJITDylibCreateResourceTracker(LLVMOrcLLJITGetMainJITDylib(jit));
+	  if ((err = LLVMOrcLLJITAddLLVMIRModuleWithRT(jit, rt, LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
+		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); LLVMOrcReleaseResourceTracker(rt); return 0;
+	  }
+	  R->rt = rt; }
+	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
+	if (dir) {
+		LLVMOrcExecutorAddress da;
+		if (!(err = LLVMOrcLLJITLookup(jit, &da, dname))) R->dnat = (void *)(size_t)da; else LLVMConsumeError(err);
+	}
+	if (trace) fprintf(stderr, "o3: %d..%d compiled by llvm%s\n", R->lo, R->hi - 1, R->dnat ? " (with a register entry)" : "");
+	return (void *)(size_t)addr;
+}
+static void rcompile(Rg *R, int eager)
+{
+	(void)eager;
+	if (R->nat || R->nonat || !llvm_ok) return;
+	if (!(R->nat = lregion(R))) R->nonat = 1;
+}
+#endif
+#define LD(x) (inmem(x) ? m[x] : 0)
+static int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest;
+static Rg *stop_rg;
+static Cell run_s, run_r;
+static int run(int p)
+{
+	int s = run_nest ? (int)run_s : DS0, r = run_nest ? (int)run_r : RS0, xfer = 1;
+	Cell w, a, b;
+	unsigned long long steps = 0;
+#ifdef ZYLLVM
+	Cell regs[2];
+	if (!getenv("ZYNOLLVM")) llvm_init();
+#endif
+	while ((unsigned)p < (unsigned)code_words) {
+		if (run_nest) {
+			if (stop_eq >= 0 && p == stop_eq && r - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == stop_return_depth) break;
+			if (stop_ge >= 0 && p >= stop_ge) break;
+			if (stop_rg && p >= stop_rg->lo && p < stop_rg->hi && o3ent[p] == stop_rg->idx + 1) break;
+		}
+		if (xfer && (s < stack_data_limit() || r > stack_return_limit())) {
+			Cell sp = s, rp = r;
+			int ret = r > stack_return_limit();
+			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { p = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
+			s = (int)sp; r = (int)rp;
+		}
+		if (xfer && o3ent[p]) {	/* an o3 region */
+			Rg *R = rat(p);
+			if (R) {
+#ifdef ZYLLVM
+				rcompile(R, 0);
+				if (R->nat) { regs[0] = s; regs[1] = r; ccode++; p = (int)((RFn)R->nat)(m, regs, o3ei[p]); ccode--; rt_sweep(); s = (int)regs[0]; r = (int)regs[1]; }
+				else
+#endif
+				p = rexec(R, o3ei[p], &s, &r);
+				continue;
+			}
+		}
+#ifdef ZYLLVM
+		if (xfer && llvm_ok) {	/* at a block start: run it compiled, or count it */
+			xfer = 0;
+			if (!lt[p] && hot[p] < 65535 && ++hot[p] == HOT && !(lt[p] = lcompile(p))) hot[p] = 65535;
+			if (lt[p]) {
+				regs[0] = s; regs[1] = r;
+				ccode++; p = lt[p](m, regs); ccode--;
+				s = regs[0]; r = regs[1];
+				rt_sweep();
+				if (need_flush) { need_flush = 0; ldrop_all(); }
+				xfer = 1;
+				continue;
+			}
+		}
+#endif
+		w = m[p++];
+		steps++;
+		if (w & 1) { m[--s] = w >> 1; continue; }
+		a = m[s]; b = m[s + 1];
+		switch ((int)(w >> 1)) {
+		case 1: m[s] = LD(a); break;
+		case 2:
+			if (a == SYSGATE) {
+				int moving = b == 9 && (m[a + 1] == 7 || m[a + 1] == 8);
+				cb_s = s + 2; cb_r = r; m[a] = hostcall(b, m[a + 1], m[a + 2], m[a + 3]);
+				if (moving) { s = (int)cb_s; r = (int)cb_r; } else s += 2;
+				break;
+			}
+			if (inmem(a)) {
+#ifdef ZYLLVM
+				if (lcov[a]) lhook(a, b); else
+#else
+				if (rcov[a]) { m[a] = b; rdrop(a); } else
+#endif
+				m[a] = b;
+			}
+			s += 2; break;
+		case 3: m[s] = LD(s + 1 + a); break;
+		case 4: s++; break;
+		case 5: m[s] = b; m[s + 1] = a; break;
+		case 6: m[r++] = p; p = a; s++; xfer = 1; break;
+		case 7: p = a; s++; xfer = 1; break;
+		case 8: if (!b) { p = a; xfer = 1; } s += 2; break;
+		case 9: p = m[--r]; xfer = 1; break;
+		case 10: m[--s] = (signed char)Q(); break;
+		case 11: m[--s] = J; break;
+		case 12: G((char)a); s++; break;
+		case 13: m[++s] = b + a; break;
+		case 14: m[++s] = b - a; break;
+		case 15: m[++s] = b * a; break;
+		case 16: m[++s] = b & a; break;
+		case 17: m[++s] = b | a; break;
+		case 18: m[++s] = b ^ a; break;
+		case 19: m[++s] = b >> (a & 63); break;
+		case 20: m[++s] = b < a; break;
+		case 600:
+			if (p >= code_words) { p = PC_HALT; break; }
+			m[--s] = m[p++];
+			break;
+		case 601: m[++s] = (Cell)fop_add((uint64_t)b, (uint64_t)a); break;
+		case 602: m[++s] = (Cell)fop_sub((uint64_t)b, (uint64_t)a); break;
+		case 603: m[++s] = (Cell)fop_mul((uint64_t)b, (uint64_t)a); break;
+		case 604: m[++s] = (Cell)fop_div((uint64_t)b, (uint64_t)a); break;
+		case 605: m[++s] = fop_eq((uint64_t)b, (uint64_t)a); break;
+		case 606: m[++s] = fop_lt((uint64_t)b, (uint64_t)a); break;
+		case 607: m[++s] = fop_le((uint64_t)b, (uint64_t)a); break;
+		case 608: m[s] = (Cell)fop_i2f(a); break;
+		case 609: m[s] = fop_f2i((uint64_t)a); break;
+		case 610: fop_print(a); s++; break;
+		case 611: m[s] = fop_parse_cell(a); break;
+		case 618: fop_print_mode(m[s + 2], b, a); s += 3; break;
+		case 612: m[s] = bld8(a); break;
+		case 613: bst8(a, b); s += 2; break;
+		case 614: m[s] = bld16(a); break;
+		case 615: bst16(a, b); s += 2; break;
+		case 616: m[s] = bld32(a); break;
+		case 617: bst32(a, b); s += 2; break;
+		case 619: { Cell t = s; m[--s] = t; break; }
+		case 620: cb_r = r; s = (int)ncall_exec(s); break;
+		case 21:
+			m[s] = install(a);
+#ifdef ZYLLVM
+			if (need_flush) { need_flush = 0; ldrop_all(); }
+#else
+			if (need_flush) { need_flush = 0; rdrop_all(); memset(rcov, 0, (size_t)code_words); }
+#endif
+			break;
+		default:
+			if ((int)(w >> 1) >= XOP0 && (int)(w >> 1) < XOP0 + NENH && enh[(int)(w >> 1) - XOP0].live) s = (int)(xexec(&enh[(int)(w >> 1) - XOP0], m + s) - m);
+			else p = PC_HALT;	/* halt: unwired opcodes and the reserved negative ones */
+		}
+	}
+	if (run_nest) { run_s = s; run_r = r; return p; }
+	halt_at = -1;
+	if (trace) fprintf(stderr, "interp: %llu steps\n", steps);
+	return stopped(p);
+}
+static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr)
+{
+	int pe = stop_eq, pg = stop_ge, pd = stop_return_depth, pn = run_nest, back;
+	Rg *old = stop_rg;
+	Cell ss = run_s, rr = run_r;
+	stop_eq = seq; stop_ge = sge; stop_return_depth = (int)(*pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1); stop_rg = re;
+	run_nest = 1; run_s = *ps; run_r = *pr;
+	back = run(pc);
+	*ps = run_s; *pr = run_r;
+	stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr;
+	return back;
+}
+static int o3_ready(Rg *C)
+{
+	if (!C || C->dead) return 0;
+	if (!C->lifted && rlift(C)) { C->dead = 1; return 0; }
+	return C->lifted && !C->dead;
+}
+static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr)
+{
+	int s, r, back;
+	if (!C || C->dead || entry < 0 || entry >= C->ne) return -1;
+	if (!C->lifted && rlift(C)) { C->dead = 1; return -1; }
+#ifdef ZYLLVM
+	rcompile(C, 0);
+	if (C->nat) {
+		typedef Cell (*RFn2)(Cell *, Cell *, int);
+		Cell regs[2];
+		regs[0] = *ps; regs[1] = *pr;
+		back = (int)((RFn2)C->nat)(m, regs, entry);
+		*ps = regs[0]; *pr = regs[1];
+		return back;
+	}
+#endif
+	s = (int)*ps; r = (int)*pr;
+	back = rexec(C, entry, &s, &r);
+	*ps = s; *pr = r;
+	return back;
+}
+#endif
+
+/* a byte store writes whole words through the same hook as opcode 2, so a
+   translation covering either word is dropped. it does not run a syscall:
+   the gate is a word store at SYSGATE. */
+static void b_commit(Cell w, Cell v)
+{
+	if ((unsigned)w >= (unsigned)MW) return;
+	if (w == SYSGATE) { m[w] = v; return; }
+#if defined(ZYJIT)
+	if (cov[(int)w]) hook(w, v); else m[w] = v;
+#elif defined(ZYLLVM)
+	if (lcov[(int)w]) lhook(w, v); else m[w] = v;
+#else
+	if (rcov[(int)w]) { m[w] = v; rdrop((int)w); } else m[w] = v;
+#endif
+}
+
+#ifdef ZYBIGSTACK
+static ucontext_t uc_main, uc_vm;
+static int vm_rc;
+static void vm_entry(void) { vm_rc = run(GORG); }
+#endif
+static int memory_option(const char *name, int fallback, int minimum)
+{
+	const char *s = getenv(name);
+	char *end;
+	unsigned long long words;
+	if (!s) return fallback;
+	errno = 0; words = strtoull(s, &end, 0);
+	if (errno || !*s || *end || words < (unsigned)minimum || words > 0x7fffffff) {
+		fprintf(stderr, "zy: invalid %s (minimum %d words)\n", name, minimum);
+		return -1;
+	}
+	return (int)words;
+}
+/* reserve all address-indexed metadata together; a denied request leaves it intact. */
+static int code_reserve(Cell end)
+{
+	Cell cap = code_words ? code_words : PW;
+	void *slots[6], *previous[6], *fresh[6]; size_t widths[6], oldn[6], newn[6]; int count = 0, i, j;
+	if (end <= 0 || end > memory_words || end >= PC_HALT) return -1;
+	if (end <= code_words) return (int)code_words;
+	while (cap < end) { if (cap > memory_words / 2) { cap = memory_words; break; } cap *= 2; }
+	if (cap >= PC_HALT) cap = PC_HALT - 1;
+	if (cap < end) return -1;
+#define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0)
+	CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1);
+	CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap);
+#ifdef ZYJIT
+	CODE_COLUMN(tab, code_words, cap); CODE_COLUMN(fn_nat, code_words, cap);
+#elif defined(ZYLLVM)
+	CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap);
+#endif
+#undef CODE_COLUMN
+	for (i = 0; i < count; i++) {
+		fresh[i] = calloc(newn[i], widths[i]);
+		if (!fresh[i]) { for (j = 0; j < i; j++) free(fresh[j]); return -1; }
+		if (oldn[i]) memcpy(fresh[i], previous[i], oldn[i] * widths[i]);
+	}
+	for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); }
+	code_words = cap;
+#ifdef ZYJIT
+	S[3] = (long)tab;
+#endif
+	return (int)cap;
+}
+/* bulk writes use the ordinary store path when they would rewrite live code. */
+static int code_range_live(Cell start, Cell count)
+{
+	Cell end, i;
+	if (start < 0 || count <= 0 || start >= code_words) return 0;
+	end = count > code_words - start ? code_words : start + count;
+	for (i = start; i < end; i++) {
+#ifdef ZYJIT
+		if (cov[i]) return 1;
+#else
+		if (rcov[i]) return 1;
+#ifdef ZYLLVM
+		if (lcov[i]) return 1;
+#endif
+#endif
+	}
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+#ifdef _WIN32
+	_setmode(_fileno(stdin), _O_BINARY);
+	_setmode(_fileno(stdout), _O_BINARY);
+#endif
+	zy_args(argc, argv);
+	memory_limit = memory_option("TINYVM_MAX_MEMORY_WORDS", DEFAULT_MEMORY_LIMIT, HEAP_HI > PW ? HEAP_HI : PW);
+	if (memory_limit < 0) return 1;
+	memory_growth_limit = memory_limit;
+	memory_words = memory_option("TINYVM_MEMORY_WORDS", HEAP_HI > PW ? HEAP_HI : PW, HEAP_HI > PW ? HEAP_HI : PW);
+	if (memory_words < 0) return 1;
+	if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; }
+	m = calloc((size_t)MW, sizeof *m);
+#ifdef ZYJIT
+	cov = calloc((size_t)MW, 1);
+	if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
+#else
+	rcov = calloc((size_t)MW, 1);
+#ifdef ZYLLVM
+	lcov = calloc((size_t)MW, 1);
+	if (!lcov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
+#endif
+	if (!rcov) { fprintf(stderr, "zy: cannot reserve region coverage\n"); return 1; }
+#endif
+	if (!m) { fprintf(stderr, "zy: cannot reserve vm memory\n"); return 1; }
+	if (code_reserve(PW) < 0) { fprintf(stderr, "zy: cannot reserve code metadata\n"); return 1; }
+	trace = getenv("ZYTRACE") != 0;
+	if ((unsigned)GORG + (unsigned)GLEN > (unsigned)PW || (unsigned)GORG + (unsigned)GLEN > (unsigned)FBOT) {
+		fprintf(stderr, "zy: genome does not fit\n");
+		return 1;
+	}
+	memcpy(m + GORG, genome, sizeof genome);
+	m[ZFP + 6] = FBOT; m[ZFP + 7] = FTOP; m[ZFP + 8] = 1024;
+#ifdef ZYJIT
+	{
+	int p = GORG;
+	union { unsigned *u; int (*f)(long *, void *); } enter;
+#ifdef __APPLE__
+	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
+#else
+	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#endif
+	if (code == MAP_FAILED) { perror("mmap"); return 1; }
+	jitwrite(1);
+	build_stubs();
+	jitwrite(0);
+	syncicache(code, cp);
+	S[0] = (long)m; S[1] = DS0; S[2] = RS0; S[3] = (long)tab; S[4] = (long)cov;
+	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
+	S[9] = (long)inst_hook; S[10] = (long)xrun;
+	S[11] = (long)(m + DSLIM); S[12] = (long)(m + RSLIM);
+	cov[SYSGATE] = 1;
+	enter.u = x_enter;
+		while ((unsigned)p < (unsigned)code_words || p == PC_DATA_OVERFLOW || p == PC_RETURN_OVERFLOW) {
+			if (p == PC_DATA_OVERFLOW || p == PC_RETURN_OVERFLOW) {
+				Cell sp = S[1], rp = S[2]; int ret = p == PC_RETURN_OVERFLOW;
+				if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
+				S[1] = sp; S[2] = rp;
+				S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
+				p = (int)S[13]; continue;
+			}
+		void *c = tab[p];
+		if (!c) { jitwrite(1); c = translate(p); jitwrite(0); }
+		p = enter.f(S, c);
+		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
+	}
+	return stopped(p);
+	}
+#elif defined(ZYBIGSTACK)
+	{
+		/* o3 calls nest on the host stack, one native frame per call: give the machine a
+		   stack as deep as its own frame stack can go (untouched pages cost nothing) */
+		size_t sz = (size_t)1 << 29;
+		void *stk = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+		if (stk != MAP_FAILED) {
+			getcontext(&uc_vm);
+			uc_vm.uc_stack.ss_sp = stk; uc_vm.uc_stack.ss_size = sz; uc_vm.uc_link = &uc_main;
+			makecontext(&uc_vm, vm_entry, 0);
+			swapcontext(&uc_main, &uc_vm);
+			return vm_rc;
+		}
+		return run(GORG);
+	}
+#else
+	return run(GORG);
+#endif
+	return 0;
+}