path: root/src/core/core.c

/* zygote core: puts the genome (the zc compiler and its runtime, both vm code)
   into memory and runs it. on arm64 and on x86-64 linux, every block of vm code
   is translated to native code the first time it is reached, straight from live
   memory. stores that land on translated words throw those translations away,
   so code that rewrites itself, including the resident compiler, just gets
   retranslated on the next visit. both jits compile o3 regions (on x86-64 a
   region that does not fit six value registers, or makes native calls, runs on
   the region interpreter, as every region does under ZYO3INTERP=1); enhancements are inlined on both
   (on x86-64 those with float, byte or native-call ops are still called). on other hosts (or with -DZYINTERP) a plain interpreter with the same
   semantics runs instead. ZYTRACE=1 logs translations on stderr. */
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#ifdef _WIN32
#include <io.h>
#include <process.h>
#ifndef O_BINARY
#define O_BINARY _O_BINARY
#endif
#if defined(_MSC_VER)
#define open _open
#define close _close
#define read _read
#define write _write
#endif
#else
#include <unistd.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <poll.h>
#include <signal.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
#endif
#include <time.h>
#if !defined(_WIN32) && !defined(__wasm__)
#include <pthread.h>
#include <stdatomic.h>
#define VM_TLS _Thread_local
#else
#define VM_TLS
#endif
#if (defined(__linux__) || defined(__APPLE__)) && !defined(__wasm__) && !defined(ZY_NOFFI)
#define ZYFFI 1
#include <dlfcn.h>
#endif
#if defined(__aarch64__) && !defined(ZYINTERP)
#define ZYJIT 1
#include <sys/mman.h>
#ifdef __APPLE__
#include <pthread.h>
#include <libkern/OSCacheControl.h>
#endif
#endif
/* the x86-64 jit: linux hosts only, and not with the interpreter forced or llvm asked for.
   o3 regions run on the region interpreter there. ZYNATIVE is either jit. */
#if defined(__x86_64__) && defined(__linux__) && !defined(ZYINTERP) && !defined(ZYLLVM) && !defined(ZYJIT)
#define ZYJIT_X64 1
#include <sys/mman.h>
#endif
#if defined(ZYJIT) || defined(ZYJIT_X64)
#define ZYNATIVE 1
#endif
#if defined(__linux__) && defined(__GLIBC__) && !defined(__wasm__) && !defined(ZYJIT) && !defined(ZY_NOBIGSTACK)
#define ZYBIGSTACK 1
#include <sys/mman.h>
#include <ucontext.h>
#endif
#include "genome.h"
#include "memmap.h"
#include "fops.h"
#include "bops.h"

#ifndef JTYPE
#define JTYPE char
#endif
char Q(void);
int input_pending(void);
void G(char);
extern VM_TLS JTYPE J;

typedef int64_t Cell;	/* one vm word. memory stays word-addressed. */
typedef uint64_t UCell;
enum { DEFAULT_MEMORY_LIMIT = MW };
static VM_TLS int memory_limit = DEFAULT_MEMORY_LIMIT;
static VM_TLS int memory_words;
static VM_TLS int memory_growth_limit;
/* bounds baked into native blocks use the reservation, whose base never moves. */
#undef MW
#define MW memory_limit
/* MW, the stacks and SYSGATE come from memmap.h. the default is 1<<20. */
/* stop values stay outside every executable address reservation. */
#define PC_HALT 0x7ffffff0
#define PC_DATA_OVERFLOW (PC_HALT + 1)
#define PC_RETURN_OVERFLOW (PC_HALT + 2)
#define PC_FRAME_OVERFLOW (PC_HALT + 3)
static VM_TLS Cell code_words;
static int code_reserve(Cell end);
static int code_range_live(Cell start, Cell count);
/* why the cpu stopped, when it wasn't a halt */
static int stopped(int p)
{
	if (p == PC_DATA_OVERFLOW) { fprintf(stderr, "zy: data stack overflow\n"); return 2; }
	if (p == PC_RETURN_OVERFLOW) { fprintf(stderr, "zy: return stack overflow\n"); return 2; }
	if (p == PC_FRAME_OVERFLOW) { fprintf(stderr, "zy: frame stack overflow\n"); return 2; }
	return 0;
}
#define CODEW (4 << 20)	/* 16 MB of code, in instruction words */
#define MAXB 65536
#define BLKW 1024	/* longest block, in vm words */

static VM_TLS Cell *m;
static VM_TLS Cell frame_floor_word = ZFP + 6, frame_top_word = ZFP + 7, frame_guard_word = ZFP + 8;
static Cell frame_bound(void) { return m[frame_floor_word] + m[frame_guard_word]; }

static int b_ok(Cell addr, int n) { return bop_ok(addr, n, MW); }
static Cell b_load(Cell addr, int n)
{
	if (!b_ok(addr, n)) return 0;
	return bop_load(m, addr, n);
}
static void b_commit(Cell w, Cell v);
static void b_store(Cell addr, Cell val, int n)
{
	Cell out[2], w0;
	int nw, i;
	if (!b_ok(addr, n)) return;
	nw = bop_merge(m, addr, val, n, out, &w0);
	for (i = 0; i < nw; i++) b_commit(w0 + i, out[i]);
}
static Cell bld8(Cell a) { return b_load(a, 1); }
static Cell bld16(Cell a) { return b_load(a, 2); }
static Cell bld32(Cell a) { return b_load(a, 4); }
static void bst8(Cell a, Cell v) { b_store(a, v, 1); }
static void bst16(Cell a, Cell v) { b_store(a, v, 2); }
static void bst32(Cell a, Cell v) { b_store(a, v, 4); }

static void fop_print(Cell bits)
{
	char buf[400];
	int n = fop_format((uint64_t)bits, buf), i;
	for (i = 0; i < n; i++) G(buf[i]);
}
/* opcode 618: (bits prec mode --). mode 0 shortest, 1 fixed, 2 scientific. */
static void fop_print_mode(Cell bits, Cell prec, Cell mode)
{
	char buf[400];
	int n = fop_format_mode((uint64_t)bits, mode, prec, buf), i;
	for (i = 0; i < n; i++) G(buf[i]);
}
static Cell fop_parse_cell(Cell addr)
{
	if ((UCell)addr >= (UCell)MW) return 0;
	return (Cell)fop_parse_words(m + addr, (UCell)MW - (UCell)addr);
}
static VM_TLS int trace;

/* the command line and the environment, kept for ops 21 to 24. a program read
   from stdin has no file name, so argv[0] is <eval>, as in zy(1). the rest of
   the host's argv follows it. the environment is a bare char **, so its count
   is counted once at startup. */
#ifndef _WIN32
extern char **environ;
#endif
static VM_TLS int zy_argc;
static VM_TLS char **zy_argv;
static VM_TLS int zy_envc;
static VM_TLS char **zy_envp;

static void zy_args(int argc, char **argv)
{
	char **e;
	int n = 0;
#ifdef _WIN32
	e = _environ;
#else
	e = environ;
#endif
	while (e && e[n]) n++;
	zy_envp = e; zy_envc = n;
	zy_argv = argv;
	zy_argc = argc > 0 ? argc : 1;
}

/* syscall gate: a store of the op to SYSGATE runs hostcall. args sit in the
   next three words, and the result replaces the op. see rt.zc. */
static int zcstr(int addr, char *dst, int max)
{
	int i = 0, c;
	if (max < 1) return -1;
	if ((unsigned)addr >= MW) { dst[0] = 0; return -1; }
	while (i < max - 1 && (unsigned)(addr + i) < MW) {
		c = (int)m[addr + i];
		if (!c) break;
		dst[i++] = (char)c;
	}
	dst[i] = 0;
	return i;
}
static int inmem(Cell a) { return (UCell)a < (UCell)MW; }
static int stack_memory_grow(UCell words)
{
	if (words > (UCell)memory_growth_limit) return 0;
	if (words > (UCell)memory_words) memory_words = (int)words;
	return 1;
}
#include "stack.h"

/* copy entry a of a char ** into vm words, one character per word, terminated
   like SYS_GETENV. a zero buffer asks for the length only, which is how the
   caller sizes the string before fetching it. lead, when given, replaces entry
   zero: a program read from stdin has no file name, so argv[0] is <eval>. */
static Cell vecstr(char **v, int count, Cell a, Cell b, Cell c, const char *lead)
{
	char *e;
	int i = 0;
	if (a < 0 || a >= (Cell)count) return -1;
	e = (a == 0 && lead) ? (char *)lead : v[a];
	if (!e) return -1;
	if (!b) { while (e[i]) i++; return i; }
	if (c < 1 || !inmem(b)) return -1;
	while (e[i] && i < c - 1 && inmem(b + i)) { m[b + i] = (unsigned char)e[i]; i++; }
	if (inmem(b + i)) m[b + i] = 0;
	return i;
}
static Cell host_rand(void)
{
	static VM_TLS uint64_t s;
	if (!s) s = (uint64_t)time(0) ^ 0x9E3779B97F4A7C15ull;
	if (!s) s = 0x9E3779B97F4A7C15ull;
	s ^= s << 13; s ^= s >> 7; s ^= s << 17;
	if ((Cell)s == -1) s ^= 0x100;
	return (Cell)s;
}
/* c abi: ops 14..18 of the gate. see the ffi section further down. cb_s and
   cb_r are the vm stack pointers as of the gate store, so that a callback from
   native code can run vm code on top of them. */
static VM_TLS Cell cb_s, cb_r;
static Cell ffi_call(Cell blk);
static Cell ffi_find(Cell name);
static Cell ffi_cbmake(Cell entry, Cell sig);
static Cell ffi_load(Cell name);
typedef struct VmContext VmContext;
static VM_TLS VmContext *context_current, *context_children;
static int context_cancelled(void);
static void context_exit(int value);
static Cell context_start(Cell entry, Cell channel);
static Cell context_join(Cell token, int cancel);
#if !defined(_WIN32) && !defined(__wasm__)
static VM_TLS atomic_int context_default_stop;
static atomic_int context_records_live, context_code_live, context_memory_live;
size_t io_context_size(void);
void io_context_snapshot(void *destination);
void io_context_restore(const void *source);
#else
static VM_TLS int context_default_stop;
#endif
static int context_descriptor(Cell handle);
static int context_channel_close(Cell handle);
#define CONTEXT_HANDLE_LO 0x40000000
static Cell hostcall(Cell op, Cell a, Cell b, Cell c)
{
	char path[1024], tmp[512];
	int n, i, got;
#ifdef ZY_SANDBOX
	/* a build for checking code the user has not run yet (the language server): compile-time
	   code may read what it includes, but it may not write files, exit, load or call C code,
	   or make raw system calls */
	if (op == 4 || op == 7 || op == 12 || (op >= 14 && op <= 17) || op == 40 || (op >= 43 && op <= 49) || (op >= 51 && op <= 54)) return -1;
	if ((op == 1 || op == 10) && b != 0) return -1;
#endif
	switch (op) {
 case 51: return context_start(a, b);
 case 52: return context_join(a, (int)b);
 case 53:
#if !defined(_WIN32) && !defined(__wasm__)
  return getpid();
#else
  return -1;
#endif
 case 55:
#if !defined(_WIN32) && !defined(__wasm__)
  return a == 0 ? atomic_load(&context_records_live) : a == 1 ? atomic_load(&context_code_live) : a == 2 ? atomic_load(&context_memory_live) : -1;
#else
  return -1;
#endif
 case 54:
#if !defined(_WIN32) && !defined(__wasm__)
  return (Cell)(uintptr_t)pthread_self();
#else
  return -1;
#endif
 case 56: return cb_r;	/* the return stack pointer as of the gate store: the word after the top return address */
	case 1:
		if (zcstr(a, path, 1024) < 0) return -1;
		if (b == 1) return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
		return open(path, O_RDONLY | O_BINARY);
	case 2:
  if (a >= CONTEXT_HANDLE_LO) return context_channel_close(a);
		return close((int)a);
	case 3:
		if (c < 0 || !inmem(b)) return -1;
		n = 0;
		while (n < c) {
			i = c - n;
			if (i > 512) i = 512;
			if (!inmem(b + n)) break;
			if (!inmem(b + n + i - 1)) i = (int)(MW - (b + n));
			if (i <= 0) break;
			got = (int)read((int)a, tmp, (size_t)i);
			if (got <= 0) break;
			for (i = 0; i < got; i++) m[b + n + i] = (unsigned char)tmp[i];
			n += got;
		}
		return n;
	case 4:
		if (c < 0 || !inmem(b)) return -1;
		n = 0;
		while (n < c) {
			i = c - n;
			if (i > 512) i = 512;
			if (!inmem(b + n)) break;
			if (!inmem(b + n + i - 1)) i = (int)(MW - (b + n));
			if (i <= 0) break;
			for (got = 0; got < i; got++) tmp[got] = (char)m[b + n + got];
			got = (int)write((int)a, tmp, (size_t)i);
			if (got <= 0) break;
			n += got;
		}
		return n;
	case 5: {
		char *e;
		if (zcstr(a, path, 1024) < 0) return -1;
		e = getenv(path);
		if (!e) return -1;
		if (c < 1 || !inmem(b)) return -1;
		i = 0;
		while (e[i] && i < c - 1 && inmem(b + i)) { m[b + i] = (unsigned char)e[i]; i++; }
		if (inmem(b + i)) m[b + i] = 0;
		return i;
	}
	case 6:
		return (Cell)time(0);
	case 7:
  if (context_current) { context_exit((int)a); return 0; }
		exit((int)a);
	case 8:
		return host_rand();
	case 9:
		if (a == 0) return memory_words;
		if (a == 1) return HEAP_LO;
		if (a == 2) return HEAP_HI;
		if (a == 3) return (Cell)memory_words * 8;
		if (a == 4) {
			if (b < 0 || b > memory_growth_limit) return -1;
			if (b > memory_words) memory_words = (int)b;
			return memory_words;
		}
		if (a == 5) {
			if (b < memory_words || b > MW) return -1;
			memory_growth_limit = (int)b; return b;
		}
		if (a == 6) {
			if (!stack_register(b)) return -1;
			frame_floor_word = m[b]; frame_top_word = m[b + 1]; frame_guard_word = m[b + 2];
			return 0;
		}
		if (a == 7 || a == 8) return stack_expand(a == 8, b, &cb_s, &cb_r);
		if (a == 9) return code_reserve(b);
		return -1;
	case 43: { /* clone an isolated execution context with a bidirectional channel */
#if !defined(_WIN32) && !defined(__wasm__)
		int fd[2]; pid_t child;
		if (!inmem(a) || socketpair(AF_UNIX, SOCK_STREAM, 0, fd) < 0) return -1;
		fflush(NULL); child = fork();
		if (child < 0) { close(fd[0]); close(fd[1]); return -1; }
		if (!child) {
			close(fd[0]); m[a] = fd[1];
#if defined(__APPLE__) && defined(ZYJIT)
			/* fork resets the thread's jit permission state; resume executable code. */
			pthread_jit_write_protect_np(1);
#endif
			return 0;
		}
		close(fd[1]); m[a] = fd[0]; return child;
#else
		return -1;
#endif
	}
	case 44: case 45: { /* write/read complete word buffers on a context channel */
  a = context_descriptor(a); if (a < 0) return -1;
#if !defined(_WIN32) && !defined(__wasm__)
		size_t done = 0, length; ssize_t got; unsigned char *buffer;
		if (b < 0 || c < 0 || c > MW - b) return -1;
		buffer = (unsigned char *)(m + b); length = (size_t)c * sizeof(Cell);
		while (done < length) {
			if (op == 44) {
#ifdef __APPLE__
				int enabled = 1; setsockopt((int)a, SOL_SOCKET, SO_NOSIGPIPE, &enabled, sizeof enabled);
				#ifdef MSG_NOSIGNAL
				got = send((int)a, buffer + done, length - done, MSG_NOSIGNAL);
#else
				got = send((int)a, buffer + done, length - done, 0);
#endif
#else
				got = send((int)a, buffer + done, length - done, MSG_NOSIGNAL);
#endif
			} else got = read((int)a, buffer + done, length - done);
			if (got < 0 && errno == EINTR) continue;
			if (got <= 0) return -1;
			done += (size_t)got;
		}
		return c;
#else
		return -1;
#endif
	}
	case 46: { /* channel readiness, with a millisecond timeout */
  a = context_descriptor(a); if (a < 0) return -1;
#if !defined(_WIN32) && !defined(__wasm__)
		struct pollfd fd; int ready; fd.fd = (int)a; fd.events = POLLIN; fd.revents = 0;
		do { ready = poll(&fd, 1, (int)b); } while (ready < 0 && errno == EINTR);
		return ready > 0 ? !!(fd.revents & (POLLIN | POLLHUP)) : ready;
#else
		return -1;
#endif
	}
	case 47: case 49: { /* join a child context, optionally terminating it first */
#if !defined(_WIN32) && !defined(__wasm__)
		int status; pid_t joined;
		if (a <= 0) return -1;
		if (op == 49) kill((pid_t)a, SIGTERM);
		do { joined = waitpid((pid_t)a, &status, 0); } while (joined < 0 && errno == EINTR);
		return joined < 0 ? -1 : WIFEXITED(status) ? WEXITSTATUS(status) : -1;
#else
		return -1;
#endif
	}
	case 50: { /* monotonic milliseconds for elapsed-time measurements */
#if !defined(_WIN32) && !defined(__wasm__)
		struct timespec now; if (clock_gettime(CLOCK_MONOTONIC, &now) < 0) return -1;
		return (Cell)now.tv_sec * 1000 + now.tv_nsec / 1000000;
#else
		return -1;
#endif
	}
	case 48: { /* sleep for a bounded millisecond duration */
#if !defined(_WIN32) && !defined(__wasm__)
		struct timespec duration;
		if (a < 0 || a > 86400000) return -1;
		duration.tv_sec = (time_t)(a / 1000); duration.tv_nsec = (long)(a % 1000) * 1000000;
		while (nanosleep(&duration, &duration) < 0) if (errno != EINTR) return -1;
		return 0;
#else
		return -1;
#endif
	}
	case 42:	/* poll: 1 when input can be read without waiting */
		return input_pending();
	case 41: {	/* shortest float text into a word buffer */
		char buf[400]; int n = fop_format_mode((uint64_t)a, 0, 17, buf), i;
		if (c <= n || b < 0 || b > MW || c > MW - b) return -1;
		for (i = 0; i < n; i++) b_commit(b + i, (unsigned char)buf[i]);
		b_commit(b + n, 0); return n;
	}
	case 10:
		if (bop_cstr(m, a, path, 1024, MW) < 0) return -1;
		if (b == 1) return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
		return open(path, O_RDONLY | O_BINARY);
	case 11:
		if (c < 0 || (c > 0 && !bop_ok(b, 1, MW))) return -1;
		n = 0;
		while (n < c) {
			i = (int)(c - n);
			if (i > 512) i = 512;
			i = bop_span(b + n, i, MW);
			if (i <= 0) break;
			got = (int)read((int)a, tmp, (size_t)i);
			if (got <= 0) break;
			for (i = 0; i < got; i++) bop_put1(m, b + n + i, (unsigned char)tmp[i]);
			n += got;
		}
		return n;
	case 12:
		if (c < 0 || (c > 0 && !bop_ok(b, 1, MW))) return -1;
		n = 0;
		while (n < c) {
			i = (int)(c - n);
			if (i > 512) i = 512;
			i = bop_span(b + n, i, MW);
			if (i <= 0) break;
			for (got = 0; got < i; got++) tmp[got] = (char)bop_load(m, b + n + got, 1);
			got = (int)write((int)a, tmp, (size_t)i);
			if (got <= 0) break;
			n += got;
		}
		return n;
	case 13: {
		char *e;
		if (bop_cstr(m, a, path, 1024, MW) < 0) return -1;
		e = getenv(path);
		if (!e) return -1;
		if (c < 1 || !bop_ok(b, 1, MW)) return -1;
		i = 0;
		while (e[i] && i < c - 1 && bop_ok(b + i, 1, MW)) { bop_put1(m, b + i, (unsigned char)e[i]); i++; }
		if (bop_ok(b + i, 1, MW)) bop_put1(m, b + i, 0);
		return i;
	}
	case 14: return ffi_call(a);
	case 15: return ffi_find(a);
	case 16: return ffi_cbmake(a, b);
	case 17: return ffi_load(a);
	case 18: {	/* fill words: a = dst, b = value, c = count. above the program area only
		   (no translation covers it), and -1 means "do it yourself" */
		Cell i;
		if (a < PW || code_range_live(a, c) || c <= 0 || c > MW - a) return -1;
		for (i = 0; i < c; i++) m[a + i] = b;
		return c; }
	case 19:	/* copy words: a = dst, b = src, c = count. a forward copy that overlaps
		   itself (dst inside the source) is left to the caller's loop */
		if (a < PW || code_range_live(a, c) || b < 0 || c <= 0 || c > MW - a || c > MW - b) return -1;
		if (a > b && a < b + c) return -1;
		memmove(m + a, m + b, (size_t)c * sizeof(Cell));
		return c;
	case 20: {	/* scan for heap pointers: a = first word, b = count, c = hbump. the index of
		   the first word that could be the payload address of an allocated block (a
		   heap address whose check word at w[-3] is w ^ MAGIC), or b if there is none.
		   the allocator's header is 5 words and its magic 0x2B7E1516 (src/runtime/rt.zc); the
		   caller still checks the flags, this only skips the words that are plainly
		   not pointers. -1 if the range is not in memory */
		Cell i, w, lim = c > MW ? MW : c;
		if (a < 0 || b <= 0 || b > MW - a) return -1;
		for (i = 0; i < b; i++) {
			w = m[a + i];
			if (w >= HEAP_LO + 5 && w < lim && m[w - 3] == (w ^ 0x2B7E1516)) return i;
		}
		return b; }
	case 21:	/* argc: the host's argument count, or 1 for <eval> alone */
		return (Cell)zy_argc;
	case 22:	/* argv: a = index, b = buffer, c = capacity. a zero buffer asks for the
		   length only, which is how the caller sizes the string before it fetches
		   it. the string is copied one character per word and terminated, like
		   SYS_GETENV. -1 if there is no such argument */
		return vecstr(zy_argv, zy_argc, a, b, c, "<eval>");
	case 23:	/* envc: how many entries the environment has */
		return (Cell)zy_envc;
	case 24:	/* envp: one entry of the environment, shaped like argv */
		return vecstr(zy_envp, zy_envc, a, b, c, 0);
	case 40:	/* raw host syscall: a = address of seven words, the syscall number and then its six arguments. the result is the kernel's, or -errno. linux only; elsewhere -1 */
#ifdef __linux__
	{
		long w[7], r;
		int k;
		if (!inmem(a) || !inmem(a + 6)) return -1;
		for (k = 0; k < 7; k++) w[k] = (long)m[a + k];
		r = syscall(w[0], w[1], w[2], w[3], w[4], w[5], w[6]);
		if (r == -1) return -(Cell)errno;
		return (Cell)r;
	}
#else
		return -1;
#endif
	default:
		return -1;
	}
}

/* ---- virtual enhancements ----
   up to 512 custom opcodes (22 .. 533), wired in at runtime by the install
   opcode (21). an enhancement is a short program over native micro-ops,
   written either as a recipe (text) or as straight-line vm code that gets
   lifted. both become ssa, go through the optimizer below, and then run in
   the interpreter or get inlined by the jit as native code. */
#define NENH 512
#define XOP0 22
enum { U_LI, U_MOV, U_ADD, U_SUB, U_MUL, U_SDIV, U_UDIV, U_SREM, U_UREM, U_AND, U_OR, U_XOR,
	U_SHL, U_SHR, U_SAR, U_NEG, U_NOT, U_LT, U_LTU, U_EQ, U_NE, U_MIN, U_MAX, U_MINU, U_MAXU,
	U_MULHS, U_MULHU, U_CLZ, U_ABS, U_SEL, U_LD, U_ST,
	U_FADD, U_FSUB, U_FMUL, U_FDIV, U_FEQ, U_FLT, U_FLE, U_I2F, U_F2I,
	U_LD8, U_LD16, U_LD32, U_ST8, U_ST16, U_ST32,
	U_NARG, U_NCALL,	/* a native call is one narg per argument (imm = its position) and then the ncall (imm = the call's descriptor) */
	U_CTZ, U_POPCNT, U_BSWAP, U_ROTL, U_ROTR,	/* bit tricks: count trailing zeros, population count, byte swap, rotates */
	U_F2S, U_S2F,		/* binary64 bits to binary32 bits (rounded) and back (exact) */
	U_N };
static const char *unames[U_N] = { "li", "mov", "add", "sub", "mul", "sdiv", "udiv", "srem", "urem",
	"and", "or", "xor", "shl", "shr", "sar", "neg", "not", "lt", "ltu", "eq", "ne", "min", "max",
	"minu", "maxu", "mulhs", "mulhu", "clz", "abs", "sel", "ld", "st",
	"fadd", "fsub", "fmul", "fdiv", "feq", "flt", "fle", "i2f", "f2i",
	"ld8", "ld16", "ld32", "st8", "st16", "st32", "narg", "ncall",
	"ctz", "popcnt", "bswap", "rotl", "rotr", "f2s", "s2f" };
static const signed char uarity[U_N] = { 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 1, 1, 3, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 2, 2, 1, 1 };
static int is_store(int op)
{
	return op == U_ST || op == U_ST8 || op == U_ST16 || op == U_ST32;
}
/* ops that matter even when nothing reads their value: a store, an argument
   handed to a native call, the call. none of them has a result a later pass
   may reuse or drop */
static int is_eff(int op)
{
	return is_store(op) || op == U_NARG || op == U_NCALL;
}
static int is_memop(int op)
{
	return is_eff(op) || op == U_LD || op == U_LD8 || op == U_LD16 || op == U_LD32;
}
static int bwidth(int op)
{
	if (op == U_LD8 || op == U_ST8) return 1;
	if (op == U_LD16 || op == U_ST16) return 2;
	if (op == U_LD32 || op == U_ST32) return 4;
	return 0;
}
typedef struct { int op, d, a, b, c; Cell imm; } Uop;	/* ssa: d is a fresh value, a b c are values */
typedef struct {
	int live, kind, nin, nout, n, nv, out[8];
	Uop *u;
	signed char *reg;	/* physical register per value, for the jit; 0 if it didn't fit */
} Enh;
static VM_TLS Enh enh[NENH];
static VM_TLS int need_flush;

#define MAXUOP 4096
static VM_TLS Uop ub[MAXUOP]; static VM_TLS int un, unv, uerr;	/* the program being built */
static int unew(int op, int a, int b, int c, Cell imm)
{
	Uop *u;
	if (un >= MAXUOP) { uerr = -5; return 0; }
	u = &ub[un++]; u->op = op; u->d = unv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
	return u->d;
}
static Cell mread(Cell a) { return (UCell)a < MW ? m[a] : 0; }	/* the whole word: an address past 2^32 reads 0, as on the jits and the microcore */

/* recipe text: one statement per line or per ';'
     r0 = sdiv r0 r1      dst = op operands, operands are rN, parameter names or numbers
     r2 = r0              a copy, or a number
     st r1 r0             memory[r1] = r0
   inputs arrive in r0.. (and under the parameter names), outputs leave from r0.. */
static int parse_recipe(int src, int names, int nin, int nout, int *out)
{
	int cur[16], i, p = src, n, k, op, args[3], na, dst, isst;
	char tok[8][32]; int nt;
	for (i = 0; i < 16; i++) cur[i] = i < nin ? i : -1;
	for (;;) {
		/* read one statement into tokens */
		nt = 0;
		for (;;) {
			int c = mread(p);
			while (c == ' ' || c == '\t' || c == '\r') c = mread(++p);
			if (!c || c == '\n' || c == ';') break;
			if (c == '=') { if (nt < 8) strcpy(tok[nt++], "="); p++; continue; }
			n = 0;
			while (c && c != ' ' && c != '\t' && c != '\n' && c != ';' && c != '=' && c != '\r') { if (n < 31) tok[nt < 8 ? nt : 7][n++] = (char)c; c = mread(++p); }
			if (nt < 8) tok[nt++][n] = 0;
		}
		if (nt) {
			isst = !strcmp(tok[0], "st");
			if (!isst && (nt < 3 || strcmp(tok[1], "="))) return -2;
			/* destination register */
			dst = -1;
			if (!isst) {
				if (tok[0][0] == 'r' && tok[0][1] >= '0' && tok[0][1] <= '9') dst = atoi(tok[0] + 1);
				else for (i = 0; i < nin && names; i++) {
					int s = mread(names + i), j = 0;
					while (mread(s + j) && mread(s + j) == (unsigned char)tok[0][j]) j++;
					if (!mread(s + j) && !tok[0][j]) dst = i;
				}
				if (dst < 0 || dst > 15) return -2;
			}
			k = isst ? 1 : 2;	/* first token after "=" (or after st) */
			op = -1;
			if (!isst && nt > 3) { for (i = 0; i < U_N; i++) if (!strcmp(tok[k], unames[i])) op = i; if (op < 0) return -2; k++; }
			if (isst) op = U_ST;
			na = 0;
			for (; k < nt; k++) {
				char *t = tok[k]; int v = -1;
				if (na >= 3) return -2;
				if (t[0] == 'r' && t[1] >= '0' && t[1] <= '9') { int r = atoi(t + 1); if (r > 15) return -2; if (cur[r] < 0) cur[r] = unew(U_LI, 0, 0, 0, 0); v = cur[r]; }
				else if ((t[0] >= '0' && t[0] <= '9') || t[0] == '-') { char *e; Cell x = (Cell)strtoll(t, &e, 0); if (*e) return -2; v = unew(U_LI, 0, 0, 0, x); }
				else {
					for (i = 0; i < nin && names; i++) {
						int s = mread(names + i), j = 0;
						while (mread(s + j) && mread(s + j) == (unsigned char)t[j]) j++;
						if (!mread(s + j) && !t[j]) v = cur[i] < 0 ? (cur[i] = unew(U_LI, 0, 0, 0, 0)) : cur[i];
					}
					if (v < 0) return -2;
				}
				args[na++] = v;
			}
			if (op < 0) {	/* x = y */
				if (na != 1) return -2;
				cur[dst] = args[0];
			} else {
				if (na != (op == U_LI ? 0 : uarity[op])) {
					if (!(op == U_LI && na == 1)) return -2;
				}
				if (op == U_LI) cur[dst] = args[0];
				else if (is_store(op)) unew(op, args[0], args[1], 0, 0);
				else cur[dst] = unew(op, args[0], na > 1 ? args[1] : 0, na > 2 ? args[2] : 0, 0);
			}
			if (uerr) return uerr;
		}
		if (!mread(p)) break;
		p++;
	}
	for (i = 0; i < nout; i++) { if (cur[i] < 0) cur[i] = unew(U_LI, 0, 0, 0, 0); out[i] = cur[i]; }
	return uerr;
}

/* straight-line vm code from addr up to its ret, run symbolically */
static int lift_vm(int a, int len, int nin, int nout, int *out)
{
	int st[256], sp = 0, i, k, x, y; Cell w;
	for (i = 0; i < nin; i++) st[sp++] = i;
	for (i = 0; i < len; i++) {
		w = mread(a + i);
		if (w & 1) { if (sp >= 256) return -4; st[sp++] = unew(U_LI, 0, 0, 0, w >> 1); continue; }
		k = w / 2;
		if (k == 9) break;
		switch (k) {
		case 1: if (sp < 1) return -4; st[sp - 1] = unew(U_LD, st[sp - 1], 0, 0, 0); break;
		case 2: if (sp < 2) return -4; unew(U_ST, st[sp - 1], st[sp - 2], 0, 0); sp -= 2; break;
		case 3:	/* pick needs a constant index */
			if (sp < 1) return -4;
			x = st[sp - 1];
			if (x < nin) return -4;
			{ int j; for (j = 0; j < un && ub[j].d != x; j++); if (j == un || ub[j].op != U_LI) return -4; y = (int)ub[j].imm; }
			if (y < 0 || y >= sp - 1) return -4;
			st[sp - 1] = st[sp - 2 - y];
			break;
		case 4: if (sp < 1) return -4; sp--; break;
		case 5: if (sp < 2) return -4; x = st[sp - 1]; st[sp - 1] = st[sp - 2]; st[sp - 2] = x; break;
		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: {
			static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
			if (sp < 2) return -4;
			x = st[sp - 1]; y = st[sp - 2]; sp -= 2;
			st[sp++] = unew(map[k - 13], y, x, 0, 0);
			break; }
		default: return -4;	/* control flow, i/o and other enhancements don't lift */
		}
		if (uerr) return uerr;
	}
	if (sp != nout) return -4;
	for (i = 0; i < nout; i++) out[i] = st[i];
	return uerr;
}

/* ---- the optimizer ----
   one forward pass of copy propagation, constant folding, algebraic
   simplification, division by constants (the multiply-high trick) and
   common subexpression elimination, then dead code removal. */
static Cell ev(int op, Cell a, Cell b, Cell c)
{
	UCell ua = (UCell)a, ub_ = (UCell)b;
	switch (op) {
	case U_ADD: return (Cell)(ua + ub_);
	case U_SUB: return (Cell)(ua - ub_);
	case U_MUL: return (Cell)(ua * ub_);
	case U_SDIV: return b == 0 ? 0 : (a == INT64_MIN && b == -1) ? a : a / b;
	case U_UDIV: return b == 0 ? 0 : (Cell)(ua / ub_);
	case U_SREM: return b == 0 ? a : (a == INT64_MIN && b == -1) ? 0 : a % b;
	case U_UREM: return b == 0 ? a : (Cell)(ua % ub_);
	case U_AND: return a & b;
	case U_OR: return a | b;
	case U_XOR: return a ^ b;
	case U_SHL: return (Cell)(ua << (b & 63));
	case U_SHR: return (Cell)(ua >> (b & 63));
	case U_SAR: return a >> (b & 63);
	case U_NEG: return (Cell)(0ull - ua);
	case U_NOT: return ~a;
	case U_LT: return a < b;
	case U_LTU: return ua < ub_;
	case U_EQ: return a == b;
	case U_NE: return a != b;
	case U_MIN: return a < b ? a : b;
	case U_MAX: return a > b ? a : b;
	case U_MINU: return ua < ub_ ? a : b;
	case U_MAXU: return ua > ub_ ? a : b;
	case U_MULHS: return (Cell)(((__int128)a * (__int128)b) >> 64);
	case U_MULHU: return (Cell)(((unsigned __int128)ua * ub_) >> 64);
	case U_CLZ: { int n = 0; if (!ua) return 64; while (!(ua & (1ull << 63))) { ua <<= 1; n++; } return n; }
	case U_ABS: return a < 0 ? (Cell)(0ull - ua) : a;
	case U_CTZ: return ua ? __builtin_ctzll(ua) : 64;
	case U_POPCNT: return __builtin_popcountll(ua);
	case U_BSWAP: return (Cell)__builtin_bswap64(ua);
	case U_ROTL: return (Cell)((ua << (b & 63)) | (ua >> ((64 - (b & 63)) & 63)));
	case U_ROTR: return (Cell)((ua >> (b & 63)) | (ua << ((64 - (b & 63)) & 63)));
	case U_F2S: return (Cell)fop_f2s((uint64_t)a);
	case U_S2F: return (Cell)fop_s2f((uint64_t)a);
	case U_SEL: return a ? b : c;
	case U_FADD: return (Cell)fop_add((uint64_t)a, (uint64_t)b);
	case U_FSUB: return (Cell)fop_sub((uint64_t)a, (uint64_t)b);
	case U_FMUL: return (Cell)fop_mul((uint64_t)a, (uint64_t)b);
	case U_FDIV: return (Cell)fop_div((uint64_t)a, (uint64_t)b);
	case U_FEQ: return fop_eq((uint64_t)a, (uint64_t)b);
	case U_FLT: return fop_lt((uint64_t)a, (uint64_t)b);
	case U_FLE: return fop_le((uint64_t)a, (uint64_t)b);
	case U_I2F: return (Cell)fop_i2f(a);
	case U_F2I: return fop_f2i((uint64_t)a);
	}
	return 0;
}
static void smagic(Cell d, Cell *mul, int *sh)	/* hacker's delight, signed, 64-bit */
{
	const UCell two63 = 1ull << 63;
	UCell ad = d < 0 ? 0ull - (UCell)d : (UCell)d, t = two63 + ((UCell)d >> 63);
	UCell anc = t - 1 - t % ad, q1 = two63 / anc, r1 = two63 - q1 * anc, q2 = two63 / ad, r2 = two63 - q2 * ad, delta;
	int p = 63;
	do {
		p++;
		q1 *= 2; r1 *= 2; if (r1 >= anc) { q1++; r1 -= anc; }
		q2 *= 2; r2 *= 2; if (r2 >= ad) { q2++; r2 -= ad; }
		delta = ad - r2;
	} while (q1 < delta || (q1 == delta && r1 == 0));
	*mul = (Cell)(q2 + 1); if (d < 0) *mul = -*mul;
	*sh = p - 64;
}
static void umagic(UCell d, UCell *mul, int *add, int *sh)	/* hacker's delight, unsigned, 64-bit */
{
	UCell nc = ~0ull - (0ull - d) % d, q1, r1, q2, r2, delta;
	const UCell two63 = 1ull << 63;
	int p = 63;
	*add = 0;
	q1 = two63 / nc; r1 = two63 - q1 * nc;
	q2 = (two63 - 1) / d; r2 = (two63 - 1) - q2 * d;
	do {
		p++;
		if (r1 >= nc - r1) { q1 = 2 * q1 + 1; r1 = 2 * r1 - nc; } else { q1 = 2 * q1; r1 = 2 * r1; }
		if (r2 + 1 >= d - r2) { if (q2 >= two63 - 1) *add = 1; q2 = 2 * q2 + 1; r2 = 2 * r2 + 1 - d; }
		else { if (q2 >= two63) *add = 1; q2 = 2 * q2; r2 = 2 * r2 + 1; }
		delta = d - 1 - r2;
	} while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0)));
	*mul = q2 + 1; *sh = p - 64;
}

static VM_TLS Uop ob[MAXUOP]; static VM_TLS int on, onv;
static VM_TLS int *omap, *isk; static VM_TLS Cell *kv;	/* old value -> new value; constness of new values */
static int pure(int op) { return !is_memop(op); }
static int ispow2(UCell x) { return x && !(x & (x - 1)); }
static int lg2(UCell x) { int n = 0; while (x > 1) { x >>= 1; n++; } return n; }
static int oe(int op, int a, int b, int c, Cell imm);
static int K(Cell v) { return oe(U_LI, 0, 0, 0, v); }
static int raw(int op, int a, int b, int c, Cell imm)
{
	Uop *u; int i;
	if (pure(op))	/* cse */
		for (i = 0; i < on; i++) {
			u = &ob[i];
			if (u->op == op && u->a == a && u->b == b && u->c == c && u->imm == imm) return u->d;
		}
	if (on >= MAXUOP) { uerr = -5; return 0; }
	u = &ob[on++]; u->op = op; u->d = onv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
	isk[u->d] = op == U_LI; kv[u->d] = imm;
	return u->d;
}
static int sdivk(int a, Cell d)	/* a / d for a constant d, no divide instruction */
{
	Cell mul; int sh, q, t, k;
	UCell ad;
	if (d == 1) return a;
	if (d == -1) return oe(U_NEG, a, 0, 0, 0);
	if (d == INT64_MIN) return oe(U_EQ, a, K(d), 0, 0);
	ad = (UCell)(d < 0 ? -d : d);
	if (ispow2(ad)) {
		k = lg2(ad);
		t = oe(U_SAR, a, K(63), 0, 0);
		t = oe(U_SHR, t, K(64 - k), 0, 0);
		q = oe(U_SAR, oe(U_ADD, a, t, 0, 0), K(k), 0, 0);
		return d < 0 ? oe(U_NEG, q, 0, 0, 0) : q;
	}
	smagic(d, &mul, &sh);
	q = oe(U_MULHS, a, K(mul), 0, 0);
	if (d > 0 && mul < 0) q = oe(U_ADD, q, a, 0, 0);
	if (d < 0 && mul > 0) q = oe(U_SUB, q, a, 0, 0);
	if (sh) q = oe(U_SAR, q, K(sh), 0, 0);
	return oe(U_ADD, q, oe(U_SHR, q, K(63), 0, 0), 0, 0);
}
static int udivk(int a, UCell d)
{
	UCell mul; int add, sh, t;
	if (d == 1) return a;
	if (ispow2(d)) return oe(U_SHR, a, K(lg2(d)), 0, 0);
	if (d > (1ull << 63)) return oe(U_XOR, oe(U_LTU, a, K((Cell)d), 0, 0), K(1), 0, 0);	/* a >= d */
	umagic(d, &mul, &add, &sh);
	t = oe(U_MULHU, a, K((Cell)mul), 0, 0);
	if (!add) return sh ? oe(U_SHR, t, K(sh), 0, 0) : t;
	t = oe(U_ADD, oe(U_SHR, oe(U_SUB, a, t, 0, 0), K(1), 0, 0), t, 0, 0);
	return sh > 1 ? oe(U_SHR, t, K(sh - 1), 0, 0) : t;
}
static int oe(int op, int a, int b, int c, Cell imm)
{
	int ka, kb; Cell A, B;
	if (op == U_LI) return raw(U_LI, 0, 0, 0, imm);
	if (op == U_MOV) return a;
	if (is_memop(op)) return raw(op, a, b, 0, 0);
	ka = isk[a]; kb = uarity[op] > 1 && isk[b];
	A = kv[a]; B = kv[b];
	/* everything constant: fold */
	if (ka && (uarity[op] < 2 || kb) && (uarity[op] < 3 || isk[c])) return K(ev(op, A, B, kv[c]));
	/* constants go right for commutative ops */
	if (ka && !kb && (op == U_ADD || op == U_MUL || op == U_AND || op == U_OR || op == U_XOR || op == U_EQ || op == U_NE ||
		op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU || op == U_MULHS || op == U_MULHU ||
		op == U_FADD || op == U_FMUL || op == U_FEQ)) {
		int t = a; a = b; b = t; kb = 1; B = A; ka = 0;
	}
	if (op == U_SEL && ka) return A ? b : c;
	if (op == U_SEL && b == c) return b;
	if (kb) switch (op) {
	case U_ADD: case U_SUB: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR:
		if (!B || (op >= U_SHL && op <= U_SAR && !(B & 63))) return a;
		break;
	case U_MUL:
		if (B == 0) return K(0);
		if (B == 1) return a;
		if (B == -1) return oe(U_NEG, a, 0, 0, 0);
		if (B > 0 && ispow2((UCell)B)) return oe(U_SHL, a, K(lg2((UCell)B)), 0, 0);
		break;
	case U_AND: if (B == 0) return K(0); if (B == -1) return a; break;
	case U_SDIV: if (B == 0) return K(0); return sdivk(a, B);
	case U_SREM: if (B == 0) return a; if (B == 1 || B == -1) return K(0);
		return oe(U_SUB, a, oe(U_MUL, sdivk(a, B), b, 0, 0), 0, 0);
	case U_UDIV: if (B == 0) return K(0); return udivk(a, (UCell)B);
	case U_UREM: if (B == 0) return a;
		if (B > 0 && ispow2((UCell)B)) return oe(U_AND, a, K(B - 1), 0, 0);
		return oe(U_SUB, a, oe(U_MUL, udivk(a, (UCell)B), b, 0, 0), 0, 0);
	}
	/* two equal operands. not applied to the float ops: x - x is a NaN when x is,
	   and x == x is false for a NaN. a pair of constant floats already folded. */
	if (a == b && uarity[op] == 2) {
		if (op == U_SUB || op == U_XOR || op == U_LT || op == U_LTU || op == U_NE) return K(0);
		if (op == U_AND || op == U_OR || op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU) return a;
		if (op == U_EQ) return K(1);
	}
	return raw(op, a, b, c, 0);
}
static void optimizer_storage(void) {
 static VM_TLS int omap_[MAXUOP + 16], isk_[MAXUOP * 4]; static VM_TLS Cell kv_[MAXUOP * 4];
 omap = omap_; isk = isk_; kv = kv_;
}
/* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */
static int optimize(Enh *e, int nin, int *out, int nout)
{
	int i, *live, n = 0;
	optimizer_storage();
	on = 0; onv = nin;
	for (i = 0; i < nin; i++) { omap[i] = i; isk[i] = 0; }
	for (i = 0; i < un && !uerr; i++) {
		Uop *u = &ub[i];
		int a = uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD ? omap[u->a] : 0;
		int b = uarity[u->op] > 1 || u->op == U_ST ? omap[u->b] : 0;
		int c = uarity[u->op] > 2 ? omap[u->c] : 0;
		if (u->op == U_ST) oe(U_ST, a, b, 0, 0);
		else omap[u->d] = oe(u->op, a, b, c, u->imm);
	}
	if (uerr) return uerr;
	/* dead code: outputs and stores are live */
	live = calloc(onv + 1, sizeof *live);
	for (i = 0; i < nout; i++) { e->out[i] = omap[out[i]]; live[e->out[i]] = 1; }
	for (i = on; i--;) {
		Uop *u = &ob[i];
		if (!is_store(u->op) && !live[u->d]) { u->op = -1; continue; }
		if (uarity[u->op] > 0 || is_memop(u->op)) live[u->a] = 1;
		if (uarity[u->op] > 1 || is_store(u->op)) live[u->b] = 1;
		if (uarity[u->op] > 2) live[u->c] = 1;
	}
	free(live);
	e->u = malloc((on + 1) * sizeof(Uop));
	for (i = 0; i < on; i++) if (ob[i].op >= 0) e->u[n++] = ob[i];
	e->n = n; e->nv = onv; e->nin = nin; e->nout = nout;
	return 0;
}
/* run an installed enhancement on a downward-growing stack; returns the new top */
static Cell *xexec(Enh *e, Cell *sp)
{
	Cell vb[256], *v = e->nv <= 256 ? vb : malloc(e->nv * sizeof(Cell)), i, j;
	for (j = 0; j < e->nin; j++) v[j] = sp[e->nin - 1 - j];
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		switch (u->op) {
		case U_LI: v[u->d] = u->imm; break;
		case U_LD: v[u->d] = mread(v[u->a]); break;
		case U_ST: if (inmem(v[u->a])) m[v[u->a]] = v[u->b]; break;
		default:
			if (u->op >= U_LD8 && u->op <= U_ST32) {
				if (u->op == U_LD8) v[u->d] = bld8(v[u->a]);
				else if (u->op == U_LD16) v[u->d] = bld16(v[u->a]);
				else if (u->op == U_LD32) v[u->d] = bld32(v[u->a]);
				else if (u->op == U_ST8) bst8(v[u->a], v[u->b]);
				else if (u->op == U_ST16) bst16(v[u->a], v[u->b]);
				else if (u->op == U_ST32) bst32(v[u->a], v[u->b]);
				break;
			}
			v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
		}
	}
	sp += e->nin - e->nout;
	for (j = 0; j < e->nout; j++) sp[e->nout - 1 - j] = v[e->out[j]];
	if (v != vb) free(v);
	return sp;
}
static void xdump(int slot, Enh *e)
{
	int i;
	fprintf(stderr, "enh %d: %d in, %d out, %d ops\n", slot, e->nin, e->nout, e->n);
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		if (u->op == U_ST) fprintf(stderr, "  st v%d v%d\n", u->a, u->b);
		else if (u->op == U_LI) fprintf(stderr, "  v%d = %lld\n", u->d, (long long)u->imm);
		else { fprintf(stderr, "  v%d = %s v%d", u->d, unames[u->op], u->a); if (uarity[u->op] > 1) fprintf(stderr, " v%d", u->b); if (uarity[u->op] > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
	}
	for (i = 0; i < e->nout; i++) fprintf(stderr, "  out%d = v%d\n", i, e->out[i]);
}
/* registers for the jit: values live in x0..x15 (registers 0..nr-1 of the
   x86-64 jit's own set there), straight-line linear scan. if they don't fit,
   reg stays 0 and the jit calls the interpreter instead. */
static void regalloc_n(Enh *e, int nr)
{
	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = (1 << nr) - 1;
	signed char *r = malloc(e->nv);
	for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; }
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) last[u->a] = i;
		if (uarity[u->op] > 1 || u->op == U_ST) last[u->b] = i;
		if (uarity[u->op] > 2) last[u->c] = i;
	}
	for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n;
#define TAKE(v) do { for (k = 0; k < nr && !(fr >> k & 1); k++); if (k == nr) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
	for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j);
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i]; int ops[3], no = 0;
		if (uarity[u->op] > 0 || u->op == U_ST || u->op == U_LD) ops[no++] = u->a;
		if (uarity[u->op] > 1 || u->op == U_ST) ops[no++] = u->b;
		if (uarity[u->op] > 2) ops[no++] = u->c;
		for (j = 0; j < no; j++) if (last[ops[j]] == i && r[ops[j]] >= 0 && (fr >> r[ops[j]] & 1) == 0) fr |= 1 << r[ops[j]];
		if (!is_store(u->op)) TAKE(u->d);
		if (!is_store(u->op) && last[u->d] == -2) fr |= 1 << r[u->d];	/* never used */
	}
#undef TAKE
	free(last); e->reg = r; return;
fail:
	free(last); free(r); e->reg = 0;
}
static void regalloc(Enh *e) { regalloc_n(e, 16); }
/* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs,
   outputs, payload address, payload length, parameter names (or 0). kind 2
   is an o3 region (below).
   returns 0, or -1 bad descriptor, -2 recipe syntax, -4 code doesn't lift,
   -5 too big. */
static int o3install(int d);
static void slot_changed(int slot);
static int install(int d)
{
	int slot = mread(d), kind = mread(d + 1), nin = mread(d + 2), nout = mread(d + 3);
	int pay = mread(d + 4), len = mread(d + 5), names = mread(d + 6), out[8], r;
	Enh ne;
	if (kind == 2) return o3install(d);
	if ((unsigned)slot >= NENH || (unsigned)nin > 8 || (unsigned)nout > 8 || (kind != 0 && kind != 1)) return -1;
	un = 0; unv = nin; uerr = 0;
	r = kind == 0 ? parse_recipe(pay, names, nin, nout, out) : lift_vm(pay, len, nin, nout, out);
	if (r) { if (trace) fprintf(stderr, "enh %d: install failed (%d)\n", slot, r); return r; }
	memset(&ne, 0, sizeof ne);
	if ((r = optimize(&ne, nin, out, nout))) return r;
	ne.kind = kind; ne.live = 1;
	regalloc(&ne);
	if (enh[slot].live) { free(enh[slot].u); free(enh[slot].reg); }
	enh[slot] = ne;
	slot_changed(slot);
	if (trace) xdump(slot, &enh[slot]);
	return 0;
}

/* ---- o3: optimized regions ----
   o3 code comes from the compiler as a region: a range [lo, hi) of vm words
   plus the frame slots that nothing takes the address of, each with one of 7
   registers (install kind 2: [-, 2, lo, hi, n, slot, register, ...]). the region is cut into basic blocks. each
   block is lifted to ssa: the vm stack is tracked symbolically, so only what
   is left on it at the end of the block touches memory, the slots live in
   registers across blocks (VGET reads one at block entry, the block's end
   says what each one holds now), the frame pointer is var 0, and enhancements
   are spliced in. empty jumps are folded and a straight loop is rotated, so
   the backends see one block per iteration when the body itself doesn't
   branch. a backend compiles the whole region at once: arm64, llvm at O3, or the
   region interpreter below. the region is entered at lo and at the return
   points of its calls. a register can stand for several slots (variables
   whose scopes don't overlap), and spilling writes it to all of them. calls,
   returns, jumps out, and anything a block can't
   lift leave it: the slots go back to the frame first, so the code outside
   sees the usual memory. stores to its own code drop it; it lifts again on
   its next entry. */
/* ZFP is the frame pointer word from memmap.h */
#define U_VGET U_N	/* a var at block entry; imm = var */
#define U_SLD (U_N + 1)	/* the stack word at offset imm from the top at block entry */
#define RMAXV 8
#define RMAXE 256
enum { T_BR, T_CBR, T_JMP, T_CALL, T_RET };
typedef struct {
	int a, n, nv, delta, nso, term, cv, s1, s2, x1, x2;
	int vs[RMAXV];	/* each var's value at the end, -1 if unchanged */
	int *so;	/* stack words to write at the end: offset, value */
	Uop *u;
} RB;
typedef struct {
	int idx, lo, hi, nv, nv0, slot[RMAXV], lifted, dead, nonat, nb, ne, maxnv;
	int np, ps[32], pv[32];	/* frame slot -> var; vars in scopes that never overlap share one */
	unsigned char fv[RMAXV];	/* 1: this var's home is a d register, not an x register */
	unsigned char seen[RMAXV];
	unsigned char sc[RMAXV];	/* 1: every local that lives in this var is a plain number (int, char, bool, float), never a pointer */
	int ent[RMAXE], entb[RMAXE];
	RB *b;
	void *nat, *dnat, *stub[RMAXE];	/* the backend's code; dnat is the register-passing entry of a function region */
	int fslots, argc, fn;		/* callable o3 function: frame, arity, prologue address */
	int marked;			/* the backend is watching the region's words */
	unsigned char dep[NENH / 8];	/* enhancement slots spliced into the lifted code */
	void *rt;			/* llvm: the resource tracker that owns the region's code */
	unsigned gen;			/* bumped every time the region's code is dropped: a copy made from an older one is stale */
} Rg;
#define MAXRG 4096
static VM_TLS Rg *rgs[MAXRG];
static VM_TLS int nrg;
static VM_TLS int fn_pc[MAXRG];
static VM_TLS Rg *fn_rg_[MAXRG];
static VM_TLS Rg *fn_cache[1024];
static VM_TLS int nfn;

/* ---- bookkeeping for translated code, shared by the backends ----
   a translation covers a range of vm words [a,b). three things have to stay
   cheap when code is rewritten while it runs. which translation a store
   hits: ranges are indexed by 64-word page, so a covered store looks at the
   handful of ranges on its page. which words are still covered after one
   goes: the coverage bytes are counters (a word two translations share counts
   2; 255 sticks), so dropping a translation subtracts its own range and
   nothing is rebuilt. and what an install invalidates: a translation records
   the enhancement slots it inlined, or found unwired, so an install drops
   those and keeps the rest. */
#define PGSH 6
#define NPG ((code_words >> PGSH) + 1)
#define MAXPGN (1 << 19)
#define MAXDEP (1 << 18)
static VM_TLS struct { int a, b; } ix_blk[MAXB];
static VM_TLS int ix_n;
static VM_TLS int *pg_head, pg_free, pg_top, pg_live;		/* node numbers are stored plus one: 0 is none */
static VM_TLS struct { int blk, next; } pg_node[MAXPGN];
static VM_TLS int dep_head[NENH + 1], dep_n;				/* slot NENH: depends on every install */
static VM_TLS struct { int blk, next; } dep_node[MAXDEP];
static VM_TLS int drop_epoch;						/* changes whenever any translation dies */
static void be_drop(int j);					/* the backend forgets block j's code and coverage */
static void runmark(Rg *R);					/* the backend stops watching the region's words */
static void rclear(Rg *R);
static void cinc(unsigned char *c, int lo, int hi) { if (hi > code_words) hi = (int)code_words; for (; lo < hi; lo++) if (c[lo] != 255) c[lo]++; }
static void cdec(unsigned char *c, int lo, int hi) { if (hi > code_words) hi = (int)code_words; for (; lo < hi; lo++) if (c[lo] && c[lo] != 255) c[lo]--; }
static void ix_reset(void)
{
	memset(pg_head, 0, NPG * sizeof *pg_head); memset(dep_head, 0, sizeof dep_head);
	pg_free = pg_top = pg_live = dep_n = ix_n = 0;
	drop_epoch++;
}
static int ix_room(void) { return ix_n < MAXB && pg_live + 24 < MAXPGN && dep_n + 300 < MAXDEP; }
static int ix_add(int a, int b)
{
	int j = ix_n++, pg, nd;
	ix_blk[j].a = a; ix_blk[j].b = b;
	for (pg = a >> PGSH; pg <= (b - 1) >> PGSH; pg++) {
		if (pg_free) { nd = pg_free - 1; pg_free = pg_node[nd].next; } else nd = pg_top++;
		pg_node[nd].blk = j; pg_node[nd].next = pg_head[pg]; pg_head[pg] = nd + 1; pg_live++;
	}
	return j;
}
static void ix_kill(int j)
{
	if (ix_blk[j].a < 0) return;
	be_drop(j);
	ix_blk[j].a = -1;
	drop_epoch++;
}
/* drop every translation whose range holds addr; dead entries on the page are unlinked on the way */
static int ix_drop_at(int addr)
{
	int *pp = &pg_head[addr >> PGSH], hit = 0, nd, j;
	while (*pp) {
		nd = *pp - 1; j = pg_node[nd].blk;
		if (ix_blk[j].a < 0) { *pp = pg_node[nd].next; pg_node[nd].next = pg_free; pg_free = nd + 1; pg_live--; continue; }
		if (ix_blk[j].a <= addr && addr < ix_blk[j].b) { ix_kill(j); hit++; continue; }
		pp = &pg_node[nd].next;
	}
	return hit;
}
static void dep_add(int slot, int j)
{
	if ((unsigned)slot > NENH) slot = NENH;
	dep_node[dep_n].blk = j; dep_node[dep_n].next = dep_head[slot]; dep_head[slot] = ++dep_n;
}
/* slot was (re)wired: drop what used it, or found it unwired, and nothing else */
static void slot_changed(int slot)
{
	int d, h, j;
	for (h = 0; h < 2; h++) {
		int *hp = &dep_head[h ? NENH : slot];
		for (d = *hp; d; d = dep_node[d - 1].next) ix_kill(dep_node[d - 1].blk);
		*hp = 0;
	}
	for (j = 0; j < nrg; j++)
		if (rgs[j]->lifted && (rgs[j]->dep[slot >> 3] & (1 << (slot & 7)))) { rclear(rgs[j]); drop_epoch++; }
}
static VM_TLS Cell halt_at = -1;	/* a nested call stopped the machine: operand or return overflow */
static VM_TLS unsigned short *o3ent;	/* region + 1 at an entry point */
static VM_TLS unsigned char *o3ei;		/* which of its entries */
static int o3_ready(Rg *C);
static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr);
static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr);
static void o3_bind(Rg *R)
{
	int i;
	if (R->fn < 0) return;	/* -1: a bare block. 0 is a real function at address 0 */
	fn_cache[R->fn & 1023] = R;
	for (i = 0; i < nfn; i++) if (fn_pc[i] == R->fn) { fn_rg_[i] = R; return; }
	if (nfn < MAXRG) { fn_pc[nfn] = R->fn; fn_rg_[nfn++] = R; }
}
static Rg *fnlookup(int pc)
{
	int i;
	Rg *R = fn_cache[pc & 1023];
	if (R && R->fn == pc && !R->dead) return R;
	for (i = 0; i < nfn; i++) if (fn_pc[i] == pc) {
		R = fn_rg_[i];
		if (R && !R->dead) { fn_cache[pc & 1023] = R; return R; }
	}
	return 0;
}
/* a call made from inside a region. an o3 callee gets a frame and is entered
   at its region, and its epilogue is not run: fp comes back here. anything
   else is an ordinary call, pushed on the vm return stack and run until that
   address is the pc again. */
/* what is left of a call once the callee's region has returned `back`: a
   return address at or past the end of the function is the normal exit (the
   epilogue's lit 0 is pushed when it fell off the end), anything else is
   code to run on the vm until the function is left */
static void o3_finish(Rg *C, int back, Cell *ps, Cell *pr)
{
	for (;;) {
		if (halt_at >= 0 || back < 0 || (unsigned)back >= (unsigned)MW) {
			if (halt_at < 0) halt_at = back < 0 ? PC_HALT : back;
			m[ZFP] = mread(ZFP) + C->fslots;
			return;
		}
		if (back >= C->hi) {
			if (back == C->hi) {	/* fell off the end: the epilogue's lit 0 */
				Cell sp = *ps - 1;
				if ((unsigned)sp < (unsigned)MW) { m[sp] = 0; *ps = sp; }
			}
			m[ZFP] = mread(ZFP) + C->fslots;
			return;
		}
		if (back >= C->lo && o3ent[back] == C->idx + 1) {
			back = o3_exec_region(C, o3ei[back], ps, pr);
			continue;
		}
		back = vm_until(back, -1, C->hi, C, ps, pr);
	}
}
/* the llvm code calls this one with the callee and the address it returned,
   when the quick exit did not apply */
static void o3_finish_c(Cell *ps, Cell *pr, Rg *C, Cell back) { o3_finish(C, (int)back, ps, pr); }
static void o3_invoke(Cell *ps, Cell *pr, Cell target, Cell retpc)
{
	Rg *C = fnlookup((int)target);
	int back, fp, i, s;
	if (halt_at >= 0) return;
	if (*ps < stack_data_limit() + 16) {
		if (stack_expand(0, stack_get(SC_DATA_TOP) - stack_get(SC_DATA_LOW) + 1, ps, pr) < 0) { halt_at = PC_DATA_OVERFLOW; return; }
	}
	if (!C || mread(ZFP) - C->fslots < frame_bound() || !o3_ready(C)) {
		if ((unsigned)*pr < (unsigned)MW) m[(*pr)++] = retpc;
		back = vm_until((int)target, (int)retpc, -1, 0, ps, pr);
		if (back < 0 || (unsigned)back >= (unsigned)code_words) halt_at = back < 0 ? PC_HALT : back;
		return;
	}
	fp = (int)mread(ZFP) - C->fslots;
	m[ZFP] = fp;
	s = (int)*ps;
	for (i = C->argc; i-- > 0; ) {
		if ((unsigned)s < (unsigned)MW && (unsigned)(fp + i) < (unsigned)MW) m[fp + i] = m[s];
		s++;
	}
	*ps = s;
	back = o3_exec_region(C, 0, ps, pr);
	o3_finish(C, back, ps, pr);
}
/* ---- c abi ----
   gate ops 14..17 and opcode 620 let vm code call native functions and hand
   native code functions of its own. vm memory is one flat array of words, so
   a vm byte address b is the host pointer (char *)m + b and a word address w
   is (char *)m + 8 * w. an argument or result of pointer kind is converted
   that way when it lies inside the array, and passed through untouched (an
   opaque host pointer, a malloc result) when it does not. 0 is NULL both
   ways.

   a signature is a word of 4-bit kinds: the return kind first, then one per
   argument (at most 15), and a 0 kind ends the list.
     0 void   1 long (any 64-bit integer)   2 double   3 byte pointer
     4 int (32-bit, sign extended)   5 float (32-bit, widened to a double word)
     6 word pointer   7 unsigned int   8 short   9 unsigned short
     10 char   11 unsigned char   (small kinds are extended on the way back)
     12 byte string object (the word address of its length word; the
     native side gets the address of its first byte)
   integers fill the platform's integer argument registers (6 on x86-64, 8 on
   arm64), doubles the 8 fp registers, and whatever does not fit goes on the
   native stack in argument order: the call goes through a prototype of that
   many integer and fp parameters followed by nine stack words, so the
   platform's own rules place every argument.
   op 14 (blk): blk is a block of words: [0] the symbol name as a byte
   address, [1] the signature, [2] a lookup cache the host keeps, [3...] the
   arguments in order. the result is the native result as a word (the bits,
   for a double). a missing symbol prints a message and gives -1.
   op 15 (name): 1 if the symbol resolves, else 0.
   op 16 (entry, sig): a native function pointer that calls the vm function at
   entry with the arguments the signature names, or 0. the vm code runs on top
   of the stacks as they were when the native call that is running started, so
   the pointer is only good while some native call from the vm is running.
   op 17 (path): dlopen a library so later lookups can find its symbols.
   opcode 620 (a0 ... an-1 blk -- r) is op 14 with the arguments on the stack
   and the block's signature saying how many; o3 lifts it to the micro-ops
   narg and ncall, and the llvm backend turns those into a direct typed call. */
#ifdef ZYFFI
#if defined(__aarch64__)
#define FFI_NI 8
#define FFI_IT Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell
#define FFI_IN Cell a, Cell b, Cell c, Cell d, Cell e, Cell f, Cell g, Cell h
#define FFI_IV a, b, c, d, e, f, g, h
#define FFI_IC(iv) iv[0], iv[1], iv[2], iv[3], iv[4], iv[5], iv[6], iv[7]
#else
#define FFI_NI 6
#define FFI_IT Cell, Cell, Cell, Cell, Cell, Cell
#define FFI_IN Cell a, Cell b, Cell c, Cell d, Cell e, Cell f
#define FFI_IV a, b, c, d, e, f
#define FFI_IC(iv) iv[0], iv[1], iv[2], iv[3], iv[4], iv[5]
#endif
#define FFI_NS 15
#define FFI_DT double, double, double, double, double, double, double, double
#define FFI_ST Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell, Cell
typedef Cell (*FfiI)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef double (*FfiD)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef float (*FfiF)(FFI_IT, FFI_DT, FFI_ST, ...);
#define FFI_SYMS 256
#define FFI_LIBS 16
static VM_TLS struct { char name[96]; void *p; } ffi_sym[FFI_SYMS];
static VM_TLS int ffi_nsym;
static VM_TLS void *ffi_lib[FFI_LIBS];
static VM_TLS int ffi_nlib, ffi_tried_libm;
static void *ffi_lookup(const char *name)
{
	int i;
	void *p;
	for (i = 0; i < ffi_nsym; i++) if (!strcmp(ffi_sym[i].name, name)) return ffi_sym[i].p;
	p = dlsym(RTLD_DEFAULT, name);
	for (i = 0; !p && i < ffi_nlib; i++) p = dlsym(ffi_lib[i], name);
	if (!p && !ffi_tried_libm) {	/* the math library is not linked into every host */
		void *h;
		ffi_tried_libm = 1;
		h = dlopen("libm.so.6", RTLD_NOW | RTLD_GLOBAL);
		if (!h) h = dlopen("libm.dylib", RTLD_NOW | RTLD_GLOBAL);
		if (h && ffi_nlib < FFI_LIBS) ffi_lib[ffi_nlib++] = h;
		for (i = 0; !p && i < ffi_nlib; i++) p = dlsym(ffi_lib[i], name);
	}
	if (p && strlen(name) < sizeof ffi_sym[0].name && ffi_nsym < FFI_SYMS) {
		strcpy(ffi_sym[ffi_nsym].name, name);
		ffi_sym[ffi_nsym++].p = p;
	}
	return p;
}
static Cell ffi_name(Cell a, char *dst, int max)
{
	return bop_cstr(m, a, dst, max, MW);
}
static Cell ffi_find(Cell a)
{
	char name[96];
	if (ffi_name(a, name, 96) <= 0) return 0;
	return ffi_lookup(name) ? 1 : 0;
}
static Cell ffi_load(Cell a)
{
	char path[512];
	void *h;
	if (ffi_name(a, path, 512) <= 0 || ffi_nlib >= FFI_LIBS) return 0;
	h = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
	if (!h) return 0;
	ffi_lib[ffi_nlib++] = h;
	return 1;
}
static Cell ffi_in(Cell v, int kind)	/* a vm value as the native argument */
{
	if (kind == 3 && v && (UCell)v < (UCell)MW * 8) return (Cell)((char *)m + v);
	if (kind == 6 && v && (UCell)v < (UCell)MW) return (Cell)((char *)m + v * 8);
	if (kind == 12 && v && (UCell)v + 1 < (UCell)MW) return (Cell)((char *)m + (v + 1) * 8);
	return v;
}
static Cell ffi_out(Cell v, int kind)	/* a native result as a vm value */
{
	char *p = (char *)v, *lo = (char *)m, *hi = (char *)m + (size_t)MW * 8;
	if (kind == 4) return (Cell)(int)v;
	if (kind == 7) return (Cell)(uint32_t)v;
	if (kind == 8) return (Cell)(int16_t)v;
	if (kind == 9) return (Cell)(uint16_t)v;
	if (kind == 10) return (Cell)(int8_t)v;
	if (kind == 11) return (Cell)(uint8_t)v;
	if ((kind == 3 || kind == 6) && p >= lo && p < hi) return kind == 3 ? (Cell)(p - lo) : (Cell)((p - lo) >> 3);
	return v;
}
static double ffi_d(Cell v) { double d; memcpy(&d, &v, 8); return d; }
static Cell ffi_bits(double d) { Cell v; memcpy(&v, &d, 8); return v; }
/* a signature's arguments are nibbles after the result. code 13 (a struct in
   registers) and code 14 (a struct in memory) start a group: the next nibble is
   its length in eightbytes, and each eightbyte is one more nibble (1 integer,
   2 sse). every other code is one argument. on the stack a group is one item:
   the vm address of the struct, and its eightbytes are the words from there on.
   every argument, a group included, is one item (one word of the stack or of a
   call block's slots) */
#define FFI_GROUP_REG 13
#define FFI_GROUP_MEM 14
static int ffi_hasgroup(Cell sig)
{
	int p;
	for (p = 1; p <= 15; p++) {
		int k = (int)((sig >> (4 * p)) & 15);
		if (!k) break;
		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) return 1;
	}
	return 0;
}
static int ffi_nargs(Cell sig)
{
	int n = 0, p = 1;
	while (p <= 15) {
		int k = (int)((sig >> (4 * p)) & 15);
		if (!k) break;
		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) {
			int len = (int)((sig >> (4 * (p + 1))) & 15);
			n++;
			p += 2 + len;
		} else { n++; p++; }
	}
	return n;
}
/* the symbol a block names. word 2 remembers which table slot answered last
   time (plus one), so a repeated call costs one string compare. a stale
   number from another run fails the compare and goes through the lookup */
static void *ffi_blkfn(Cell blk)
{
	char name[96];
	void *f;
	int k;
	if (blk < 0 || blk + 18 >= MW) return 0;
	if (ffi_name(m[blk], name, 96) <= 0) return 0;
	k = (int)(m[blk + 2] & 0xffffffffLL);
	if (k > 0 && k <= ffi_nsym && !strcmp(ffi_sym[k - 1].name, name)) return ffi_sym[k - 1].p;
	f = ffi_lookup(name);
	for (k = 0; f && k < ffi_nsym; k++) if (ffi_sym[k].p == f && !strcmp(ffi_sym[k].name, name)) { m[blk + 2] = (m[blk + 2] & ~0xffffffffLL) | (k + 1); break; }
	if (!f) fprintf(stderr, "zy: no native symbol %s\n", name);
	return f;
}
/* a struct result. the call block's word 2 holds the host's cache in its low half
   and the result's descriptor in its high half: the struct's length in words, then
   one class nibble per word (1 integer, 2 float). the result's memory is the first
   item of the call (the stub makes it), and the words go there. which words travel
   in memory is the host's rule (ffi_retmem); the rest come back in registers, and
   the c function type that returns the matching struct gets them there */
typedef struct { Cell a, b; } FfiSII;
typedef struct { double a, b; } FfiSDD;
typedef struct { Cell a; double b; } FfiSID;
typedef struct { double a; Cell b; } FfiSDI;
typedef struct { double d[1]; } FfiSF1;
typedef struct { double d[2]; } FfiSF2;
typedef struct { double d[3]; } FfiSF3;
typedef struct { double d[4]; } FfiSF4;
typedef struct { Cell w[7]; } FfiSBIG;
typedef FfiSII (*FfiRII)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSDD (*FfiRDD)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSID (*FfiRID)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSDI (*FfiRDI)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSF1 (*FfiRF1)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSF2 (*FfiRF2)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSF3 (*FfiRF3)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSF4 (*FfiRF4)(FFI_IT, FFI_DT, FFI_ST, ...);
typedef FfiSBIG (*FfiRBIG)(FFI_IT, FFI_DT, FFI_ST, ...);
#define FFI_CALL(T) ((T)f)(FFI_IC(iv), dv[0], dv[1], dv[2], dv[3], dv[4], dv[5], dv[6], dv[7], sv[0], sv[1], sv[2], sv[3], sv[4], sv[5], sv[6], sv[7], sv[8], sv[9], sv[10], sv[11], sv[12], sv[13], sv[14])
/* does the host send this result through memory? sysv: more than two words. aapcs64:
   more than two words unless it is a float struct of up to four words, which comes
   back in the v registers, and an integer struct of up to two */
static int ffi_retmem(Cell rdesc)
{
	int n = (int)(rdesc & 15), g;
	if (n <= 2) return 0;
#if defined(__aarch64__)
	if (n > 4) return 1;
	for (g = 0; g < n; g++) if (((rdesc >> (4 + 4 * g)) & 15) != 2) return 1;
	return 0;
#else
	return 1;
#endif
}
/* the call that returns a struct: the words land in m at res */
static void ffi_call_ret(void *f, Cell rdesc, Cell *iv, double *dv, Cell *sv, Cell res)
{
	int n = (int)(rdesc & 15), g, c[8], hfa = 1;
	for (g = 0; g < n; g++) { c[g] = (int)((rdesc >> (4 + 4 * g)) & 15); if (c[g] != 2) hfa = 0; }
#if defined(__x86_64__) || defined(__aarch64__)
	if (ffi_retmem(rdesc)) {
#if defined(__x86_64__)
		FFI_CALL(FfiI);	/* iv[0] is the hidden pointer (ffi_invoke set it); the callee writes there */
#else
		{ FfiSBIG s = FFI_CALL(FfiRBIG); for (g = 0; g < n; g++) m[res + g] = s.w[g]; }
#endif
		return;
	}
#if defined(__aarch64__)
	if (hfa && n == 1) { FfiSF1 s = FFI_CALL(FfiRF1); m[res] = ffi_bits(s.d[0]); }
	else if (hfa && n == 2) { FfiSF2 s = FFI_CALL(FfiRF2); m[res] = ffi_bits(s.d[0]); m[res + 1] = ffi_bits(s.d[1]); }
	else if (hfa && n == 3) { FfiSF3 s = FFI_CALL(FfiRF3); for (g = 0; g < 3; g++) m[res + g] = ffi_bits(s.d[g]); }
	else if (hfa && n == 4) { FfiSF4 s = FFI_CALL(FfiRF4); for (g = 0; g < 4; g++) m[res + g] = ffi_bits(s.d[g]); }
	else { FfiSII s = FFI_CALL(FfiRII); m[res] = s.a; if (n > 1) m[res + 1] = s.b; }
#else
	if (n == 1) { if (c[0] == 2) m[res] = ffi_bits(FFI_CALL(FfiD)); else m[res] = FFI_CALL(FfiI); }
	else if (c[0] == 1 && c[1] == 1) { FfiSII s = FFI_CALL(FfiRII); m[res] = s.a; m[res + 1] = s.b; }
	else if (c[0] == 2 && c[1] == 2) { FfiSDD s = FFI_CALL(FfiRDD); m[res] = ffi_bits(s.a); m[res + 1] = ffi_bits(s.b); }
	else if (c[0] == 1) { FfiSID s = FFI_CALL(FfiRID); m[res] = s.a; m[res + 1] = ffi_bits(s.b); }
	else { FfiSDI s = FFI_CALL(FfiRDI); m[res] = ffi_bits(s.a); m[res + 1] = s.b; }
#endif
#else
	(void)f; (void)rdesc; (void)iv; (void)dv; (void)sv; (void)res; (void)hfa;
	fprintf(stderr, "zy: a struct result needs x86-64 or arm64 in this build\n");
#endif
}
/* call f with the vm values av, by the signature. integer-class arguments take
   the integer registers in turn, fp ones the fp registers, and what is left
   over lands in the stack words in order (a float32 is the low half of a
   word, as it would sit in a register) */
/* darwin puts unnamed arguments on the stack. these standard stdio entry
   points have known fixed prefixes even when an extern uses a zc alias. */
static int ffi_varargs(void *f)
{
#if defined(__APPLE__) && defined(__aarch64__)
	static const char *names[] = { "printf", "fprintf", "sprintf", "snprintf" };
	static const int fixed[] = { 1, 2, 2, 3 };
	static VM_TLS void *entry[4]; static VM_TLS int ready;
	int i;
	if (!ready) { for (i = 0; i < 4; i++) entry[i] = dlsym(RTLD_DEFAULT, names[i]); ready = 1; }
	for (i = 0; i < 4; i++) if (f == entry[i]) return fixed[i];
#else
	(void)f;
#endif
	return -1;
}
static Cell ffi_invoke(void *f, Cell sig, const Cell *av, Cell rdesc)
{
	Cell iv[8] = {0, 0, 0, 0, 0, 0, 0, 0}, sv[FFI_NS] = {0, 0, 0, 0, 0, 0, 0, 0, 0}, r, w;
	Cell cbuf[225];	/* copies of the structs passed by reference (aapcs64) */
	int nc = 0;
	double dv[8] = {0, 0, 0, 0, 0, 0, 0, 0};
	int ni = 0, nd = 0, ns = 0, i = 0, j = 0, p, k, rk = (int)(sig & 15), fixed = ffi_varargs(f);
	int sret = rk == FFI_GROUP_REG || rk == FFI_GROUP_MEM, rn = (int)(rdesc & 15);
	Cell res = 0;
	if (sret) {
		if (rn < 1 || rn > 7) return -1;
		res = av[0];
		if (res < 0 || res + rn > MW) return -1;
		j = 1;
#if defined(__x86_64__)
		if (ffi_retmem(rdesc)) { iv[0] = (Cell)(intptr_t)(m + res); ni = 1; }
#endif
	}
	for (p = 1; p <= 15; p++) {
		k = (int)((sig >> (4 * p)) & 15);
		if (!k) break;
		if (k == FFI_GROUP_REG || k == FFI_GROUP_MEM) {
			/* a struct: one item, its vm address. its eightbytes take registers only as
			   a whole, else all of them go on the stack, in order (the sysv rule). only
			   the sysv classes are done so far */
#if defined(__x86_64__) || defined(__aarch64__)
			int len = (int)((sig >> (4 * (p + 1))) & 15), g, need_i = 0, need_d = 0;
			Cell base = av[j];
			if (base < 0 || base + len > MW || len < 1 || len > 15) return -1;
			for (g = 0; g < len; g++) {
				if (((sig >> (4 * (p + 2 + g))) & 15) == 2) need_d++; else need_i++;
			}
#if defined(__aarch64__)
			/* aapcs64: a homogeneous float struct of up to four words goes in the v
			   registers, one word each; another struct of two words or less goes in
			   the x registers; a bigger one is passed by reference (a copy, whose
			   address is the integer argument). a struct that does not fit takes the
			   rest of its register class, so the later arguments go on the stack */
			if (need_i == 0 && len <= 4) {
				if (nd + len <= 8) { for (g = 0; g < len; g++) dv[nd++] = ffi_d(m[base + g]); }
				else { nd = 8; for (g = 0; g < len; g++) { if (ns >= FFI_NS) return -1; sv[ns++] = m[base + g]; } }
			} else if (len <= 2) {
				if (ni + len <= FFI_NI) { for (g = 0; g < len; g++) iv[ni++] = m[base + g]; }
				else { ni = FFI_NI; for (g = 0; g < len; g++) { if (ns >= FFI_NS) return -1; sv[ns++] = m[base + g]; } }
			} else {
				Cell *cp = cbuf + nc;
				for (g = 0; g < len; g++) { if (nc >= 225) return -1; cbuf[nc++] = m[base + g]; }
				if (ni < FFI_NI) iv[ni++] = (Cell)(intptr_t)cp;
				else if (ns < FFI_NS) sv[ns++] = (Cell)(intptr_t)cp;
				else return -1;
			}
#else
			if (k == FFI_GROUP_REG && ni + need_i <= FFI_NI && nd + need_d <= 8) {
				for (g = 0; g < len; g++) {
					if (((sig >> (4 * (p + 2 + g))) & 15) == 2) dv[nd++] = ffi_d(m[base + g]);
					else iv[ni++] = m[base + g];
				}
			} else {
				for (g = 0; g < len; g++) {
					if (ns >= FFI_NS) return -1;
					sv[ns++] = m[base + g];
				}
			}
#endif
			j++;
			p += 1 + len;
			continue;
#else
			fprintf(stderr, "zy: a struct by value needs x86-64 or arm64 in this build\n");
			return -1;
#endif
		}
		if (fixed >= 0 && p > fixed) {
			if (ns >= FFI_NS) return -1;
			if (k == 5) w = ffi_bits((double)(float)ffi_d(av[j]));
			else if (k == 2) w = av[j];
			else w = ffi_in(av[j], k);
			sv[ns++] = w; j++; continue;
		}
		if (k == 2 || k == 5) {
			if (k == 5) {
				float fl = (float)ffi_d(av[j]);
				uint32_t u;
				memcpy(&u, &fl, 4);
				w = (Cell)u;
			} else w = av[j];
			if (nd < 8) dv[nd++] = ffi_d(w);
			else if (ns < FFI_NS) sv[ns++] = w;
			else return -1;
		} else {
			w = ffi_in(av[j], k);
			if (ni < FFI_NI) iv[ni++] = w;
			else if (ns < FFI_NS) sv[ns++] = w;
			else return -1;
		}
		j++;
	}
	(void)i;
	if (sret) { ffi_call_ret(f, rdesc, iv, dv, sv, res); return 0; }
	if (rk == 2) r = ffi_bits(FFI_CALL(FfiD));
	else if (rk == 5) r = ffi_bits((double)FFI_CALL(FfiF));
	else {
		r = FFI_CALL(FfiI);
		r = rk ? ffi_out(r, rk) : 0;
	}
	return r;
}
static Cell ffi_call(Cell blk)
{
	void *f = ffi_blkfn(blk);
	Cell av[15];
	int i;
	if (!f) return -1;
	for (i = 0; i < 15; i++) av[i] = m[blk + 3 + i];
	return ffi_invoke(f, m[blk + 1], av, (Cell)((UCell)m[blk + 2] >> 32));
}
/* opcode 620 on the data stack: blk on top, then the arguments, the last one
   next. returns the new stack top, with the result on it. cb_r has to be set
   by the caller (the interpreter's r, an llvm block's r) */
static Cell ncall_exec(Cell s)
{
	Cell blk, sig, av[15], res = -1;
	int n, i;
	void *f;
	if ((UCell)s >= (UCell)MW) return s;
	blk = m[s];
	if (blk < 0 || blk + 18 >= MW) return s + 1;
	sig = m[blk + 1];
	n = ffi_nargs(sig);
	if (s + n >= MW) return s + 1;
	for (i = 0; i < n; i++) av[i] = m[s + n - i];
	f = ffi_blkfn(blk);
	cb_s = s + n + 1;
	if ((sig & 15) >= 13) res = -1;	/* a struct result is only made by the stub's syscall */
	else if (f) res = ffi_invoke(f, sig, av, 0);
	m[s + n] = res;
	return s + n;
}
/* the jit's form: sp and rp are host pointers into m */
static long ncall_host(long sp, long rp)
{
	Cell s = ((Cell *)sp - m);
	cb_r = (Cell *)rp - m;
	return (long)(m + ncall_exec(s));
}

/* callbacks. each slot is a native function of the c shapes (the integer and
   fp registers and nine stack words in, one result out), one set per result
   class. calling it pushes the arguments on the vm data stack as vm code
   expects them, pushes a return address that stops the nested run, runs the
   vm function and pops its result. */
#define FFI_CBS 16
static VM_TLS struct { Cell entry, sig; int used; } ffi_cb[3][FFI_CBS];
static Cell ffi_enter(int cls, int k, Cell *iv, double *dv, Cell *sv)
{
	Cell s = cb_s, r = cb_r, sig = ffi_cb[cls][k].sig, res = 0, sent = 1, w;
	int i, kind, ni = 0, nd = 0, ns = 0, n = 0, back;
	Cell args[16];
	if (halt_at >= 0 || (unsigned)(s - 20) >= (unsigned)MW || (unsigned)r >= (unsigned)MW) return 0;
	for (i = 0; i < 15; i++) {
		kind = (int)((sig >> (4 * (i + 1))) & 15);
		if (!kind) break;
		if (kind == 2 || kind == 5) {
			w = nd < 8 ? ffi_bits(dv[nd++]) : ns < FFI_NS ? sv[ns++] : 0;
			if (kind == 5) {
				uint32_t u = (uint32_t)w;
				float fl;
				memcpy(&fl, &u, 4);
				w = ffi_bits((double)fl);
			}
			args[n++] = w;
		} else {
			w = ni < FFI_NI ? iv[ni++] : ns < FFI_NS ? sv[ns++] : 0;
			args[n++] = ffi_out(w, kind);
		}
	}
	for (i = 0; i < n; i++) m[--s] = args[i];
	m[r++] = sent;
	back = vm_until((int)ffi_cb[cls][k].entry, (int)sent, -1, 0, &s, &r);
	if (back < 0 || (unsigned)back >= (unsigned)code_words) halt_at = back < 0 ? PC_HALT : back;
	if (halt_at >= 0) return 0;
	if ((unsigned)s < (unsigned)MW) res = m[s++];
	cb_s = s; cb_r = r;
	kind = (int)(sig & 15);
	if (cls == 0) return kind ? ffi_in(res, kind) : 0;
	return res;
}
#define FFI_ARGS FFI_IN, double x0, double x1, double x2, double x3, double x4, double x5, double x6, double x7, Cell s0, Cell s1, Cell s2, Cell s3, Cell s4, Cell s5, Cell s6, Cell s7, Cell s8
#define FFI_PACK Cell iv[8] = {FFI_IV}; double dv[8] = {x0, x1, x2, x3, x4, x5, x6, x7}; Cell sv[FFI_NS] = {s0, s1, s2, s3, s4, s5, s6, s7, s8}
#define FFI_SLOT(K) \
	static Cell ffi_i##K(FFI_ARGS) { FFI_PACK; return ffi_enter(0, K, iv, dv, sv); } \
	static double ffi_d##K(FFI_ARGS) { FFI_PACK; return ffi_d(ffi_enter(1, K, iv, dv, sv)); } \
	static float ffi_f##K(FFI_ARGS) { FFI_PACK; return (float)ffi_d(ffi_enter(2, K, iv, dv, sv)); }
#define FFI_ALL(X) X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15)
FFI_ALL(FFI_SLOT)
#define FFI_PICK(K) (void *)ffi_i##K,
#define FFI_PICKD(K) (void *)ffi_d##K,
#define FFI_PICKF(K) (void *)ffi_f##K,
static VM_TLS void *ffi_fn[3][FFI_CBS] = { { FFI_ALL(FFI_PICK) }, { FFI_ALL(FFI_PICKD) }, { FFI_ALL(FFI_PICKF) } };
static Cell ffi_cbmake(Cell entry, Cell sig)
{
	int cls = (sig & 15) == 2 ? 1 : (sig & 15) == 5 ? 2 : 0, k;
	for (k = 0; k < FFI_CBS; k++)
		if (ffi_cb[cls][k].used && ffi_cb[cls][k].entry == entry && ffi_cb[cls][k].sig == sig) return (Cell)ffi_fn[cls][k];
	for (k = 0; k < FFI_CBS; k++)
		if (!ffi_cb[cls][k].used) {
			ffi_cb[cls][k].used = 1; ffi_cb[cls][k].entry = entry; ffi_cb[cls][k].sig = sig;
			return (Cell)ffi_fn[cls][k];
		}
	return 0;
}
/* a native call the o3 lifter has resolved: the micro-ops narg and ncall carry
   an index into this table. imm of narg is the argument's position */
typedef struct { void *f; Cell sig; int n; char name[48]; } Ncd;
static VM_TLS Ncd ncd[256];
static VM_TLS int nncd;
static int ncd_get(Cell blk)
{
	void *f;
	Cell sig;
	char name[96];
	int i;
	if (blk < 0 || blk + 18 >= MW) return -1;
	if (ffi_name(m[blk], name, 96) <= 0) return -1;
	f = ffi_lookup(name);
	if (!f) return -1;
	sig = m[blk + 1];
	if (ffi_hasgroup(sig) || (sig & 15) >= 13) return -1;	/* the o3 lifter reads one nibble per argument; a struct goes through ncall_exec */
	for (i = 0; i < nncd; i++) if (ncd[i].f == f && ncd[i].sig == sig) return i;
	if (nncd >= 256) return -1;
	ncd[nncd].f = f; ncd[nncd].sig = sig; ncd[nncd].n = ffi_nargs(sig);
	strncpy(ncd[nncd].name, name, 47);
	ncd[nncd].name[47] = 0;
	return nncd++;
}
#else
static Cell ffi_call(Cell blk) { return -1; }
static Cell ffi_find(Cell name) { return 0; }
static Cell ffi_cbmake(Cell entry, Cell sig) { return 0; }
static Cell ffi_load(Cell name) { return 0; }
static Cell ncall_exec(Cell s)
{
	Cell blk, sig;
	int n = 0;
	if ((UCell)s >= (UCell)MW) return s;
	blk = m[s];
	if (blk < 0 || blk + 18 >= MW) return s + 1;
	sig = m[blk + 1];
	while (n < 15 && ((sig >> (4 * (n + 1))) & 15)) n++;
	m[s + n] = -1;
	return s + n;
}
static long ncall_host(long sp, long rp) { return (long)(m + ncall_exec((Cell *)sp - m)); }
static int ncd_get(Cell blk) { return -1; }
#endif

static void rforget(Rg *R);		/* the backend drops its code */
static void rmark(Rg *R);		/* the backend watches the region's words */
static void rcompile(Rg *R, int eager);

static int nargs(int op) { return op >= U_N ? 0 : uarity[op]; }
static int op_fpdest(int op)
{
	return op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_I2F || op == U_S2F;
}
/* operand 0 is a, 1 is b. comparisons read floats and write an integer. */
static int op_fpuse(int op, int which)
{
	if (op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_FEQ || op == U_FLT || op == U_FLE) return 1;
	if ((op == U_F2I || op == U_F2S) && which == 0) return 1;
	return 0;
}
/* 1 when the value should stay in an FP register: a float result, a float
   variable, or a load/constant that is only ever used as a float. a use that
   also wants an integer (a branch, an int variable) keeps the bits in a GPR
   and the float use converts at that instruction. */
static void class_fp(Rg *R, RB *B, unsigned char *fp)
{
	int i, j, *fu, *gu, n = B->nv;
	fu = calloc(n + 1, sizeof(int));
	gu = calloc(n + 1, sizeof(int));
	memset(fp, 0, n);
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		int na = nargs(u->op);
		if (na > 0) { if (op_fpuse(u->op, 0)) fu[u->a]++; else gu[u->a]++; }
		if (na > 1) { if (op_fpuse(u->op, 1)) fu[u->b]++; else gu[u->b]++; }
		if (na > 2) gu[u->c]++;
	}
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
		if (R->fv[j]) fu[B->vs[j]]++; else gu[B->vs[j]]++;
	}
	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) gu[B->cv]++;
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		int d = u->d;
		if (is_store(u->op) || u->op == U_NARG) continue;
		if (u->op == U_VGET) { if ((unsigned)u->imm < RMAXV && R->fv[u->imm]) fp[d] = 1; continue; }
		if (op_fpdest(u->op) || (fu[d] && !gu[d])) fp[d] = 1;
	}
	free(fu); free(gu);
}
static void rclear(Rg *R)
{
	int b;
	if (R->marked) { runmark(R); R->marked = 0; }
	memset(R->dep, 0, sizeof R->dep);
	rforget(R);
	for (b = 0; b < R->nb; b++) { free(R->b[b].u); free(R->b[b].so); }
	free(R->b); R->b = 0; R->nb = 0; R->lifted = 0;
	R->gen++;
}
static void rdrop_all(void) { int j; for (j = 0; j < nrg; j++) if (rgs[j]->lifted) rclear(rgs[j]); }

/* lifting one block */
static VM_TLS Rg *R_;
static VM_TLS unsigned char *lead_;
static VM_TLS int *bidx_, *wl_, nw_;
static VM_TLS int lv_[RMAXV], lw_[RMAXV];
#define SB 1100
static VM_TLS int skv[2 * SB], stp, slo, shi;
static VM_TLS unsigned char skk[2 * SB];
static VM_TLS int afp[MAXUOP * 4 + 16];	/* value -> 1 + its offset from fp, if it is fp + k */
static VM_TLS int emap[MAXUOP * 4 + 16];

static int rvar(int v)
{
	if (lv_[v] < 0) { lv_[v] = raw(U_VGET, 0, 0, 0, v); if (!v) afp[lv_[v]] = 1; }
	return lv_[v];
}
static int slotvar(int off) { int j; for (j = 0; j < R_->np; j++) if (R_->ps[j] == off) return R_->pv[j]; return 0; }
static int rbin(int op, int x, int y, int z)
{
	int r = oe(op, x, y, z, 0);
	if (op == U_ADD && !afp[r]) {
		if (afp[x] && isk[y]) afp[r] = afp[x] + kv[y];
		else if (afp[y] && isk[x]) afp[r] = afp[y] + kv[x];
	}
	return r;
}
static int rld(int a)
{
	int v;
	if (isk[a] && kv[a] == ZFP) return rvar(0);
	if (afp[a] && (v = slotvar(afp[a] - 1))) return rvar(v);
	return oe(U_LD, a, 0, 0, 0);
}
static int rst(int a, int b)
{
	int v;
	if (isk[a] && kv[a] == ZFP) return 0;	/* moving the frame: not in here */
	if (afp[a] && (v = slotvar(afp[a] - 1))) { lv_[v] = b; lw_[v] = 1; return 1; }
	oe(U_ST, a, b, 0, 0);
	return 1;
}
static int sget(int o)
{
	if (o < -SB || o >= SB) { uerr = -6; return 0; }
	if (!skk[o + SB]) {
		if (o < 0) { uerr = -6; return 0; }
		skv[o + SB] = raw(U_SLD, 0, 0, 0, o); skk[o + SB] = 1;
	}
	if (o < slo) slo = o;
	if (o >= shi) shi = o + 1;
	return skv[o + SB];
}
static void sset(int o, int v)
{
	if (o < -SB || o >= SB) { uerr = -6; return; }
	skv[o + SB] = v; skk[o + SB] = 1;
	if (o < slo) slo = o;
	if (o >= shi) shi = o + 1;
}
static int spop(void) { return sget(stp++); }
static void spush(int v) { sset(--stp, v); }
static int rblk(int t) { return t >= R_->lo && t < R_->hi && lead_[t - R_->lo] ? bidx_[t - R_->lo] : -1; }
/* an enhancement, spliced into the block */
static int rsplice(Enh *e)
{
	int i, j;
	for (j = e->nin; j--;) emap[j] = spop();
	for (i = 0; i < e->n && !uerr; i++) {
		Uop *u = &e->u[i];
		int na = nargs(u->op), a = na > 0 ? emap[u->a] : 0, b = na > 1 ? emap[u->b] : 0, c = na > 2 ? emap[u->c] : 0;
		if (u->op == U_LI) emap[u->d] = K(u->imm);
		else if (u->op == U_LD) emap[u->d] = rld(a);
		else if (u->op == U_ST) { if (!rst(a, b)) uerr = -4; }
		else emap[u->d] = rbin(u->op, a, b, c);
	}
	for (j = 0; j < e->nout; j++) spush(emap[e->out[j]]);
	return uerr;
}
static int lift_block(int bi)
{
	static const int map[] = { U_ADD, U_SUB, U_MUL, U_AND, U_OR, U_XOR, U_SAR, U_LT };
	RB *B = &R_->b[bi];
	int i = B->a, k, x, y, t, j, o, n = 0, *live;
	Cell w;
	on = 0; onv = 0; uerr = 0;
	memset(afp, 0, sizeof afp); memset(skk, 0, sizeof skk);
	stp = slo = shi = 0;
	for (j = 0; j < RMAXV; j++) { lv_[j] = -1; lw_[j] = 0; }
	B->term = T_JMP; B->cv = -1; B->s1 = B->s2 = -1; B->x1 = B->x2 = 0;
	for (;;) {
		if (uerr) return uerr;
		if (i != B->a && (t = rblk(i)) >= 0) { B->term = T_BR; B->s1 = t; goto done; }
		if (i < R_->lo || i >= R_->hi) goto leave_at;
		w = m[i];
		if (w & 1) { spush(K(w >> 1)); i++; continue; }
		k = (int)(w >> 1);
		switch (k) {
		case 0: B->cv = K(-1); goto done;	/* halt */
		case 1: x = spop(); spush(rld(x)); break;
		case 2:
			x = sget(stp);
			if (isk[x] && kv[x] == ZFP) goto leave_at;
			x = spop(); y = spop(); rst(x, y);
			break;
		case 3:	/* pick: a constant depth */
			x = sget(stp);
			if (!isk[x] || kv[x] < 0 || kv[x] > 1024) goto leave_at;
			stp++; spush(sget(stp + kv[x]));
			break;
		case 4: stp++; break;
		case 5: x = sget(stp); y = sget(stp + 1); sset(stp, y); sset(stp + 1, x); break;
		case 6: B->cv = spop(); B->term = T_CALL; B->x2 = i + 1; goto done;
		case 7:
			x = spop();
			if (isk[x] && (t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; }
			else B->cv = x;
			goto done;
		case 8:
			x = sget(stp);
			if (!isk[x]) goto leave_at;
			stp++; y = spop();
			if (isk[y]) {	/* a known condition */
				if (kv[y]) { i++; continue; }
				if ((t = rblk(kv[x])) >= 0) { B->term = T_BR; B->s1 = t; } else B->cv = K(kv[x]);
				goto done;
			}
			B->term = T_CBR; B->cv = y;
			B->x1 = kv[x]; B->s1 = rblk(kv[x]);
			B->x2 = i + 1; B->s2 = rblk(i + 1);
			goto done;
		case 9: B->term = T_RET; goto done;
		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
			x = spop(); y = spop(); spush(rbin(map[k - 13], y, x, 0));
			break;
		case 600:	/* flit: the next word is the raw bits, even when it looks like halt */
			if (i + 1 >= code_words) goto leave_at;
			spush(K(m[i + 1]));
			i += 2;
			continue;
		case 601: case 602: case 603: case 604: case 605: case 606: case 607: {
			static const int fmap[] = { U_FADD, U_FSUB, U_FMUL, U_FDIV, U_FEQ, U_FLT, U_FLE };
			x = spop(); y = spop(); spush(rbin(fmap[k - 601], y, x, 0));
			break; }
		case 608: x = spop(); spush(rbin(U_I2F, x, 0, 0)); break;
		case 609: x = spop(); spush(rbin(U_F2I, x, 0, 0)); break;
		case 620: {	/* a native call: the block address is a constant, so the callee and its signature are known now */
#ifdef ZYJIT
			goto leave_at;
#else
			int di, na, av[15];
			x = sget(stp);
			if (!isk[x] || (di = ncd_get(kv[x])) < 0) goto leave_at;
			na = ncd[di].n;
			stp++;
			for (j = na; j-- > 0;) av[j] = spop();
			for (j = 0; j < na; j++) raw(U_NARG, av[j], 0, 0, j);
			spush(raw(U_NCALL, 0, 0, 0, di));
			break;
#endif
		}
		case 612: x = spop(); spush(raw(U_LD8, x, 0, 0, 0)); break;
		case 614: x = spop(); spush(raw(U_LD16, x, 0, 0, 0)); break;
		case 616: x = spop(); spush(raw(U_LD32, x, 0, 0, 0)); break;
		case 613: x = spop(); y = spop(); raw(U_ST8, x, y, 0, 0); break;
		case 615: x = spop(); y = spop(); raw(U_ST16, x, y, 0, 0); break;
		case 617: x = spop(); y = spop(); raw(U_ST32, x, y, 0, 0); break;
		default:
			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { R_->dep[(k - XOP0) >> 3] |= 1 << ((k - XOP0) & 7); if (rsplice(&enh[k - XOP0])) return uerr; break; }
			goto leave_at;	/* i/o, install, unwired: done the usual way */
		}
		i++;
	}
leave_at:
	B->term = T_JMP; B->cv = K(i);
done:
	if (uerr) return uerr;
	B->delta = stp;
	B->so = malloc((shi - slo + 1) * 2 * sizeof(int));
	B->nso = 0;
	for (o = stp > slo ? stp : slo; o < shi; o++)
		if (skk[o + SB]) {
			x = skv[o + SB];
			if (ob[x].op == U_SLD && ob[x].imm == o) continue;	/* still the word that was there */
			if (afp[x] && slotvar(afp[x] - 1)) return -7;	/* a register var's address can't wait in memory */
			B->so[2 * B->nso] = o; B->so[2 * B->nso + 1] = x; B->nso++;
		}
	for (j = 0; j < RMAXV; j++) B->vs[j] = -1;
	for (j = 1; j < R_->nv; j++)
		if (lw_[j] && !(ob[lv_[j]].op == U_VGET && ob[lv_[j]].imm == j)) B->vs[j] = lv_[j];
	/* dead code: what the block leaves behind and its stores are live */
	live = calloc(onv + 1, sizeof *live);
	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
	for (j = 1; j < R_->nv; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) live[B->cv] = 1;
	for (i = on; i--;) {
		Uop *u = &ob[i];
		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
		k = nargs(u->op);
		if (k > 0) live[u->a] = 1;
		if (k > 1) live[u->b] = 1;
		if (k > 2) live[u->c] = 1;
	}
	free(live);
	B->u = malloc((on + 1) * sizeof(Uop));
	for (i = 0; i < on; i++) if (ob[i].op >= 0) B->u[n++] = ob[i];
	B->n = n; B->nv = onv;
	if (onv > R_->maxnv) R_->maxnv = onv;
	return 0;
}
/* ---- shape: empty jumps, straight-line merges, rotated loops ----
   the for-loop the compiler emits is a test, an empty jump, a body and a
   step: four dispatches per iteration. fold the empty jump, paste a block
   onto its only predecessor, and rotate a straight loop so the hot path is
   one block that branches to itself. a call, a return, a stack word, or a
   second branch stays where it is. an entry is never pasted over. */
static int bempty(Rg *R, int b)
{
	RB *B = &R->b[b];
	int j;
	if (B->n || B->nso || B->delta || B->term != T_BR) return 0;
	if (B->s1 < 0 || B->s1 == b) return 0;
	for (j = 0; j < R->nv; j++) if (B->vs[j] >= 0) return 0;
	return 1;
}
static int bfollow(Rg *R, int b)
{
	unsigned char *seen = calloc(R->nb, 1);
	int n = 0;
	if (!seen) return b;
	while ((unsigned)b < (unsigned)R->nb && bempty(R, b) && !seen[b] && n++ < R->nb) {
		seen[b] = 1;
		b = R->b[b].s1;
	}
	free(seen);
	return b;
}
static int breach(Rg *R, unsigned char *reach)
{
	int *st = malloc((R->nb + 1) * sizeof(int)), n = 0, e, b;
	if (!st) return -1;
	for (e = 0; e < R->ne; e++) {
		b = R->entb[e];
		if ((unsigned)b < (unsigned)R->nb && !reach[b]) { reach[b] = 1; st[n++] = b; }
	}
	while (n) {
		RB *B;
		b = st[--n];
		B = &R->b[b];
		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb && !reach[B->s1]) {
			reach[B->s1] = 1; st[n++] = B->s1;
		}
		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb && !reach[B->s2]) {
			reach[B->s2] = 1; st[n++] = B->s2;
		}
	}
	free(st);
	return 0;
}
static int rempty(Rg *R)
{
	int b, e, ch = 0, *to = malloc(R->nb * sizeof(int));
	if (!to) return 0;
	for (b = 0; b < R->nb; b++) {
		int d = bfollow(R, b);
		to[b] = (d != b && (unsigned)d < (unsigned)R->nb && !bempty(R, d)) ? d : b;
	}
	for (b = 0; b < R->nb; b++) {
		RB *B = &R->b[b];
		if ((unsigned)B->s1 < (unsigned)R->nb && to[B->s1] != B->s1) { B->s1 = to[B->s1]; ch = 1; }
		if ((unsigned)B->s2 < (unsigned)R->nb && to[B->s2] != B->s2) { B->s2 = to[B->s2]; ch = 1; }
	}
	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb && to[R->entb[e]] != R->entb[e]) {
		R->entb[e] = to[R->entb[e]]; ch = 1;
	}
	free(to);
	return ch;
}
static int bmapat(int *map, int nv, int x, int *ok)
{
	if ((unsigned)x >= (unsigned)nv || map[x] < 0) { *ok = 0; return 0; }
	return map[x];
}
/* paste the blocks in order. a VGET of a var this paste already wrote is the
   written value, not a fresh read of the block's entry. the last block's
   branch is the branch of the result. */
static int rb_build(Rg *R, int *ix, int nix, RB *dst)
{
	int cap = 0, i, j, n = 0, nid = 0, cur[RMAXV], wrote[RMAXV];
	Uop *o;
	for (i = 0; i < nix; i++) {
		if ((unsigned)ix[i] >= (unsigned)R->nb) return -1;
		cap += R->b[ix[i]].n;
	}
	if (cap > 2048 || !nix) return -1;
	o = malloc((cap + 1) * sizeof(Uop));
	if (!o) return -1;
	for (j = 0; j < RMAXV; j++) { cur[j] = -1; wrote[j] = 0; }
	for (i = 0; i < nix; i++) {
		RB *S = &R->b[ix[i]];
		int nvmap = S->nv > 0 ? S->nv : 1, k, ok;
		int *map = malloc(nvmap * sizeof(int));
		if (!map) { free(o); return -1; }
		for (k = 0; k < nvmap; k++) map[k] = -1;
		for (k = 0; k < S->n; k++) {
			Uop u = S->u[k];
			int na = nargs(u.op);
			ok = 1;
			if (u.op == U_VGET && (unsigned)u.imm < RMAXV && cur[u.imm] >= 0) {
				if ((unsigned)u.d >= (unsigned)nvmap) { free(map); free(o); return -1; }
				map[u.d] = cur[u.imm];
				continue;
			}
			if (na > 0) u.a = bmapat(map, nvmap, u.a, &ok);
			if (na > 1) u.b = bmapat(map, nvmap, u.b, &ok);
			if (na > 2) u.c = bmapat(map, nvmap, u.c, &ok);
			if (!ok || n >= cap) { free(map); free(o); return -1; }
			if (!is_store(u.op)) {
				if ((unsigned)u.d >= (unsigned)nvmap || nid >= 4096) { free(map); free(o); return -1; }
				map[u.d] = nid;
				u.d = nid++;
			} else u.d = 0;
			o[n++] = u;
		}
		for (j = 0; j < R->nv; j++) if (S->vs[j] >= 0) {
			if ((unsigned)S->vs[j] >= (unsigned)nvmap || map[S->vs[j]] < 0) { free(map); free(o); return -1; }
			cur[j] = map[S->vs[j]]; wrote[j] = 1;
		}
		if (i + 1 == nix) {
			dst->term = S->term;
			dst->s1 = S->s1; dst->s2 = S->s2; dst->x1 = S->x1; dst->x2 = S->x2;
			if (S->cv >= 0) {
				if ((unsigned)S->cv >= (unsigned)nvmap || map[S->cv] < 0) { free(map); free(o); return -1; }
				dst->cv = map[S->cv];
			} else dst->cv = -1;
		}
		free(map);
	}
	dst->u = o; dst->n = n; dst->nv = nid > 0 ? nid : 1;
	dst->nso = 0; dst->so = 0; dst->delta = 0; dst->a = R->b[ix[0]].a;
	for (j = 0; j < RMAXV; j++) dst->vs[j] = wrote[j] ? cur[j] : -1;
	if (dst->nv > R->maxnv) R->maxnv = dst->nv;
	return 0;
}
static void addpred(int *np, int *pr, int s, int from)
{
	np[s]++;
	if (np[s] == 1) pr[s] = from;
	else if (pr[s] != from) pr[s] = -2;
}
static int rmerge_one(Rg *R)
{
	unsigned char *reach = calloc(R->nb, 1), *ent = calloc(R->nb, 1);
	int *np = calloc(R->nb, sizeof(int)), *pr = malloc(R->nb * sizeof(int));
	int b, e, did = 0;
	if (!reach || !ent || !np || !pr || breach(R, reach)) goto out;
	for (b = 0; b < R->nb; b++) pr[b] = -1;
	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) ent[R->entb[e]] = 1;
	for (b = 0; b < R->nb; b++) if (reach[b]) {
		RB *B = &R->b[b];
		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) addpred(np, pr, B->s1, b);
		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) addpred(np, pr, B->s2, b);
	}
	for (b = 0; b < R->nb && !did; b++) {
		RB *A = &R->b[b], built;
		int s, ix[2];
		if (!reach[b] || A->term != T_BR) continue;
		s = A->s1;
		if ((unsigned)s >= (unsigned)R->nb || s == b || !reach[s] || ent[s]) continue;
		if (np[s] != 1 || pr[s] != b) continue;
		if (A->nso || A->delta || R->b[s].nso || R->b[s].delta) continue;
		ix[0] = b; ix[1] = s;
		memset(&built, 0, sizeof built);
		if (rb_build(R, ix, 2, &built)) continue;
		free(A->u); free(A->so);
		built.a = A->a;
		R->b[b] = built;
		did = 1;
	}
out:
	free(reach); free(ent); free(np); free(pr);
	return did;
}
/* length of the straight run of single-predecessor jumps from s back to h,
   or 0 when s is not that kind of back edge. the other edge may leave the
   loop and come back in later through some other header; that is still an
   exit of this loop, and the straight edge can be rotated. */
static int try_chain(Rg *R, int h, int s, int *ch, int *np, int *pr, unsigned char *ent)
{
	int n = 0, prev = h, guard = 0;
	if (s < 0 || s == h) return 0;
	while (s != h) {
		RB *B;
		if ((unsigned)s >= (unsigned)R->nb || ++guard > R->nb) return 0;
		B = &R->b[s];
		if (ent[s] || np[s] != 1 || pr[s] != prev || B->nso || B->delta || B->term != T_BR) return 0;
		ch[n++] = s;
		prev = s;
		s = B->s1;
	}
	return n;
}
static int rrotate_one(Rg *R)
{
	unsigned char *reach = calloc(R->nb, 1), *ent = calloc(R->nb, 1);
	int *np = calloc(R->nb, sizeof(int)), *pr = malloc(R->nb * sizeof(int));
	int *ch1 = malloc(R->nb * sizeof(int)), *ch2 = malloc(R->nb * sizeof(int));
	int b, e, did = 0;
	if (!reach || !ent || !np || !pr || !ch1 || !ch2 || breach(R, reach)) goto out;
	for (b = 0; b < R->nb; b++) pr[b] = -1;
	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) ent[R->entb[e]] = 1;
	for (b = 0; b < R->nb; b++) if (reach[b]) {
		RB *B = &R->b[b];
		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) addpred(np, pr, B->s1, b);
		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) addpred(np, pr, B->s2, b);
	}
	for (b = 0; b < R->nb && !did; b++) {
		RB *H = &R->b[b], built, *grown;
		int n1, n2, n, back, L, *ch, *ix, i;
		if (!reach[b] || H->term != T_CBR || H->nso || H->delta || H->cv < 0) continue;
		n1 = try_chain(R, b, H->s1, ch1, np, pr, ent);
		n2 = try_chain(R, b, H->s2, ch2, np, pr, ent);
		if (n2 > 0 && n1 == 0) { n = n2; ch = ch2; back = 2; }
		else if (n1 > 0 && n2 == 0) { n = n1; ch = ch1; back = 1; }
		else continue;
		ix = malloc((n + 1) * sizeof(int));
		if (!ix) continue;
		for (i = 0; i < n; i++) ix[i] = ch[i];
		ix[n] = b;
		memset(&built, 0, sizeof built);
		if (rb_build(R, ix, n + 1, &built)) { free(ix); continue; }
		free(ix);
		grown = realloc(R->b, (R->nb + 1) * sizeof(RB));
		if (!grown) { free(built.u); continue; }
		R->b = grown;
		L = R->nb;
		R->b[L] = built;
		R->nb++;
		if (back == 2) { R->b[L].s2 = L; R->b[b].s2 = L; }
		else { R->b[L].s1 = L; R->b[b].s1 = L; }
		did = 1;
	}
out:
	free(reach); free(ent); free(np); free(pr); free(ch1); free(ch2);
	return did;
}
static void rdead(Rg *R)
{
	unsigned char *reach = calloc(R->nb, 1);
	int b;
	if (!reach || breach(R, reach)) { free(reach); return; }
	for (b = 0; b < R->nb; b++) if (!reach[b]) {
		R->b[b].term = T_BR;
		R->b[b].s1 = b;
		R->b[b].s2 = -1;
		R->b[b].cv = -1;
	}
	free(reach);
}
static void rshape(Rg *R)
{
	int i, guard;
	if (R->nb < 1) return;
	for (i = 0; i < 4; i++) if (!rempty(R)) break;
	guard = R->nb + 4;
	for (i = 0; i < guard; i++) if (!rmerge_one(R)) break;
	guard = R->nb + 4;
	for (i = 0; i < guard; i++) if (!rrotate_one(R)) break;
	guard = R->nb + 4;
	for (i = 0; i < guard; i++) if (!rmerge_one(R)) break;
	for (i = 0; i < 4; i++) if (!rempty(R)) break;
	rdead(R);
}
/* ---- the global pass ----
   a block only knows its own values, but the variables carry facts from one
   block to the next. this pass runs a forward dataflow over the blocks for the
   one fact that pays off everywhere: a variable that holds the same constant
   on every path into a block. such a read becomes a constant, the block is
   emitted again through the same folding and strength reduction the lifter
   uses (so x / step turns into a multiply once step is known, and an add of
   two known values disappears), and a branch on a known value becomes a jump.
   the second fact is a copy: on every path a variable holds what another
   variable holds, and neither has been written since. a read of the copy
   becomes a read of its source, so the copy can stop being kept at all. a
   block that writes the source drops the copies of it. the blocks that no
   longer have a way in are dropped by the shaping that follows, and it
   repeats while something changes. */
enum { C_UNSEEN, C_CONST, C_VAR, C_COPY };
/* C_COPY: the variable holds the same value as variable v (both at the same point) */
static int copy_ok(const Rg *R, int j, int k) { return j > 0 && j < RMAXV && k > 0 && k < R->nv && !R->fv[j] && !R->fv[k]; }
typedef struct { unsigned char st[RMAXV]; Cell v[RMAXV]; } CIn;
static VM_TLS int g_isk[MAXUOP * 4]; static VM_TLS Cell g_kv[MAXUOP * 4]; static VM_TLS int g_omap[MAXUOP + 16];
static int cjoin(CIn *d, const CIn *s)
{
	int j, ch = 0;
	for (j = 1; j < RMAXV; j++) {
		if (s->st[j] == C_UNSEEN) continue;
		if (d->st[j] == C_UNSEEN) { d->st[j] = s->st[j]; d->v[j] = s->v[j]; ch = 1; }
		else if (d->st[j] != C_VAR && (s->st[j] == C_VAR || s->st[j] != d->st[j] || s->v[j] != d->v[j])) { d->st[j] = C_VAR; ch = 1; }
	}
	return ch;
}
/* does the block write variable k (a write other than the plain entry read)? */
static int written_here(const RB *B, int k)
{
	if (k <= 0 || k >= RMAXV) return 1;
	if (B->vs[k] < 0) return 0;
	return !(B->u[B->vs[k]].op == U_VGET && B->u[B->vs[k]].imm == k);
}
/* what each variable holds when the block ends, given what it held at the start */
static void cblock(Rg *R, RB *B, const CIn *in, CIn *out)
{
	int i, j, na;
	unsigned char *kc = calloc(B->nv + 1, 1);
	Cell *kw = calloc(B->nv + 1, sizeof(Cell));
	int *cpk = calloc(B->nv + 1, sizeof(int));	/* cpk[v] = k + 1 when v is a plain read of the entry value of k */
	*out = *in;
	if (!kc || !kw || !cpk) { for (j = 1; j < RMAXV; j++) out->st[j] = C_VAR; free(kc); free(kw); free(cpk); return; }
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		if (is_store(u->op) || u->op == U_NARG) continue;
		if ((unsigned)u->d >= (unsigned)B->nv) continue;
		if (u->op == U_LI) { kc[u->d] = 1; kw[u->d] = u->imm; continue; }
		if (u->op == U_VGET) {
			j = (int)u->imm;
			if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) { kc[u->d] = 1; kw[u->d] = in->v[j]; }
			if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) cpk[u->d] = (int)in->v[j] + 1;
			else if (j > 0 && j < RMAXV) cpk[u->d] = j + 1;
			continue;
		}
		if (u->op >= U_N || is_memop(u->op)) continue;
		na = nargs(u->op);
		if ((na > 0 && !kc[u->a]) || (na > 1 && !kc[u->b]) || (na > 2 && !kc[u->c])) continue;
		kc[u->d] = 1; kw[u->d] = ev(u->op, na > 0 ? kw[u->a] : 0, na > 1 ? kw[u->b] : 0, na > 2 ? kw[u->c] : 0);
	}
	/* a variable the block writes is a copy of k when its value is the entry value of k and k is not written here */
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
		int w = B->vs[j], k;
		if (kc[w] && !R->fv[j]) { out->st[j] = C_CONST; out->v[j] = kw[w]; }
		else if (w < B->nv && cpk[w] && (k = cpk[w] - 1) != j && copy_ok(R, j, k) && !written_here(B, k)) { out->st[j] = C_COPY; out->v[j] = k; }
		else out->st[j] = C_VAR;
	}
	/* a copy of a variable this block writes is gone; so is a copy whose variable the block writes */
	for (j = 1; j < RMAXV; j++) if (out->st[j] == C_COPY && written_here(B, (int)out->v[j])) out->st[j] = C_VAR;
	free(kc); free(kw); free(cpk);
}
/* emit the block again with the known variables as constants. returns 1 if it changed, -1 if it could not be done */
static int rb_refold(Rg *R, RB *B, const CIn *in)
{
	int i, j, na, any = 0, n = 0, *live, *mp, bad = 0;
	int *sisk = isk, *somap = omap; Cell *skv = kv;
	Uop *nu;
	for (i = 0; i < B->n; i++) if (B->u[i].op == U_VGET) {
		j = (int)B->u[i].imm;
		if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) any = 1;
		if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) any = 1;
	}
	if (!any) return 0;
	for (j = 0; j < B->nso; j++) if ((unsigned)B->so[2 * j + 1] >= (unsigned)B->nv) bad = 1;
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= B->nv) bad = 1;
	if (B->cv >= B->nv) bad = 1;
	for (i = 0; i < B->n; i++) if ((unsigned)B->u[i].d >= (unsigned)B->nv && !is_store(B->u[i].op) && B->u[i].op != U_NARG) bad = 1;
	if (bad) return 0;
	mp = malloc((B->nv + 1) * sizeof(int));
	if (!mp) return 0;
	isk = g_isk; kv = g_kv; omap = g_omap;
	on = 0; onv = 0; uerr = 0;
	for (i = 0; i < B->n && !uerr; i++) {
		Uop *u = &B->u[i];
		na = u->op >= U_N ? 0 : nargs(u->op);
		if (u->op == U_VGET) {
			j = (int)u->imm;
			if (j > 0 && j < RMAXV && in->st[j] == C_CONST && !R->fv[j]) mp[u->d] = K(in->v[j]);
			else if (j > 0 && j < RMAXV && in->st[j] == C_COPY && copy_ok(R, j, (int)in->v[j])) mp[u->d] = raw(U_VGET, 0, 0, 0, in->v[j]);
			else mp[u->d] = raw(U_VGET, 0, 0, 0, u->imm);
		} else if (u->op == U_SLD) mp[u->d] = raw(U_SLD, 0, 0, 0, u->imm);
		else if (u->op == U_LI) mp[u->d] = K(u->imm);
		else if (is_store(u->op)) oe(u->op, mp[u->a], mp[u->b], 0, 0);
		else if (u->op == U_NARG) raw(U_NARG, mp[u->a], 0, 0, u->imm);
		else if (u->op == U_NCALL) mp[u->d] = raw(U_NCALL, 0, 0, 0, u->imm);
		else mp[u->d] = oe(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
	}
	if (uerr) { isk = sisk; kv = skv; omap = somap; free(mp); return 0; }
	live = calloc(onv + 2, sizeof *live);
	nu = malloc((on + 1) * sizeof(Uop));
	if (!live || !nu) { free(live); free(nu); isk = sisk; kv = skv; omap = somap; free(mp); return 0; }
	for (j = 0; j < B->nso; j++) B->so[2 * j + 1] = mp[B->so[2 * j + 1]];
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) B->vs[j] = mp[B->vs[j]];
	if (B->cv >= 0) B->cv = mp[B->cv];
	/* a branch on a known value is a jump, when the way it goes stays in the region; otherwise the exit */
	if (B->term == T_CBR && B->cv >= 0 && g_isk[B->cv]) {
		int zero = g_kv[B->cv] == 0, to = zero ? B->s1 : B->s2, x = zero ? B->x1 : B->x2;
		if (to >= 0) { B->term = T_BR; B->s1 = to; B->s2 = -1; B->cv = -1; }
		else { B->cv = K(x); B->term = T_JMP; B->s1 = B->s2 = -1; if (onv + 2 > 0) { free(live); live = calloc(onv + 2, sizeof *live); free(nu); nu = malloc((on + 1) * sizeof(Uop)); } }
	}
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && ob[B->vs[j]].op == U_VGET && ob[B->vs[j]].imm == j) B->vs[j] = -1;
	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) live[B->cv] = 1;
	for (i = on; i--;) {
		Uop *u = &ob[i];
		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
		na = u->op >= U_N ? 0 : nargs(u->op);
		if (is_store(u->op)) na = 2;
		if (u->op == U_NARG) na = 1;
		if (na > 0) live[u->a] = 1;
		if (na > 1) live[u->b] = 1;
		if (na > 2) live[u->c] = 1;
	}
	for (i = 0; i < on; i++) if (ob[i].op >= 0) nu[n++] = ob[i];
	free(B->u); B->u = nu; B->n = n; B->nv = onv;
	if (onv > R->maxnv) R->maxnv = onv;
	free(live); free(mp);
	isk = sisk; kv = skv; omap = somap;
	return 1;
}
static void rglobal(Rg *R)
{
	int round, b, e, s, changed, sweeps, did;
	if (R->nb < 1 || R->nb > 600) return;
	for (round = 0; round < 4; round++) {
		CIn *in = calloc(R->nb, sizeof(CIn)), out;
		unsigned char *reached = calloc(R->nb, 1);
		if (!in || !reached) { free(in); free(reached); return; }
		for (e = 0; e < R->ne; e++) {
			b = R->entb[e];
			if ((unsigned)b < (unsigned)R->nb) {
				int j;
				reached[b] = 1;
				for (j = 1; j < RMAXV; j++) in[b].st[j] = C_VAR;
			}
		}
		sweeps = 0; changed = 1;
		while (changed && sweeps++ < 64) {
			changed = 0;
			for (b = 0; b < R->nb; b++) {
				RB *B = &R->b[b];
				if (!reached[b]) continue;
				cblock(R, B, &in[b], &out);
				if (B->term != T_BR && B->term != T_CBR) continue;
				for (s = 0; s < 2; s++) {
					int t = s ? B->s2 : B->s1;
					if (s && B->term != T_CBR) break;
					if ((unsigned)t >= (unsigned)R->nb) continue;
					if (!reached[t]) { reached[t] = 1; in[t] = out; changed = 1; }
					else if (cjoin(&in[t], &out)) changed = 1;
				}
			}
		}
		did = 0;
		for (b = 0; b < R->nb; b++) if (reached[b]) did |= rb_refold(R, &R->b[b], &in[b]);
		free(in); free(reached);
		if (!did) return;
		rshape(R);
	}
}
/* ---- loops, shared values and dead variables ----
   what the per-block lifter and the constant pass can't see is work that is
   done again: an expression inside a loop whose operands don't change in it,
   the same expression (or the same load, when nothing stored in between) in
   two blocks one of which always runs before the other, and a variable that
   is written and then written again before anyone reads it. blocks share
   only variables, so a value that has to cross a block edge needs one: the
   passes below take a spare register for it (a region has at most 7). each
   step moves one expression and the analysis is run again, so a later step
   sees what the earlier one did. */
typedef struct {
	int nb, *idom, *rn, *rpo, nrpo, *dep, nl;
	int *pp, *pl;			/* predecessors of b: pl[pp[b] .. pp[b + 1]); the root, nb, is left out */
	unsigned char *reach, *isent;
	struct { int h, n, bad; unsigned char *in; } *lp;
} Cfg;
static int csucc(Rg *R, int b, int *o)
{
	int n = 0, e;
	if (b == R->nb) {
		for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)R->nb) o[n++] = R->entb[e];
		return n;
	}
	{
		RB *B = &R->b[b];
		if ((B->term == T_BR || B->term == T_CBR) && (unsigned)B->s1 < (unsigned)R->nb) o[n++] = B->s1;
		if (B->term == T_CBR && (unsigned)B->s2 < (unsigned)R->nb) o[n++] = B->s2;
	}
	return n;
}
static void cfg_free(Cfg *G)
{
	int i;
	if (!G) return;
	for (i = 0; i < G->nl; i++) free(G->lp[i].in);
	free(G->lp); free(G->idom); free(G->rn); free(G->rpo); free(G->dep); free(G->pp); free(G->pl); free(G->reach); free(G->isent); free(G);
}
static int cdom(const Cfg *G, int a, int b)
{
	int n = 0;
	while (b != a) {
		if (b >= G->nb || b < 0 || G->idom[b] < 0 || ++n > G->nb + 2) return 0;
		b = G->idom[b];
	}
	return 1;
}
static Cfg *cfg_build(Rg *R)
{
	Cfg *G = calloc(1, sizeof *G);
	int nb = R->nb, b, i, n, e, k, sp, succ[RMAXE + 2], changed, *po, *stn, *sti, npo = 0, guard;
	if (!G) return 0;
	G->nb = nb;
	G->idom = malloc((nb + 1) * sizeof(int)); G->rn = malloc((nb + 1) * sizeof(int)); G->rpo = malloc((nb + 2) * sizeof(int));
	G->dep = calloc(nb + 1, sizeof(int)); G->pp = calloc(nb + 2, sizeof(int));
	G->reach = calloc(nb + 1, 1); G->isent = calloc(nb + 1, 1);
	po = malloc((nb + 2) * sizeof(int)); stn = malloc((nb + 2) * sizeof(int)); sti = malloc((nb + 2) * sizeof(int));
	if (!G->idom || !G->rn || !G->rpo || !G->dep || !G->pp || !G->reach || !G->isent || !po || !stn || !sti) { free(po); free(stn); free(sti); cfg_free(G); return 0; }
	for (b = 0; b <= nb; b++) { G->idom[b] = -1; G->rn[b] = -1; }
	for (e = 0; e < R->ne; e++) if ((unsigned)R->entb[e] < (unsigned)nb) G->isent[R->entb[e]] = 1;
	/* postorder from the root */
	sp = 0; stn[0] = nb; sti[0] = 0; G->reach[nb] = 1;
	while (sp >= 0) {
		n = csucc(R, stn[sp], succ);
		if (sti[sp] < n) {
			int s = succ[sti[sp]++];
			if (!G->reach[s]) { G->reach[s] = 1; sp++; stn[sp] = s; sti[sp] = 0; }
		} else { po[npo++] = stn[sp]; sp--; }
	}
	G->nrpo = npo;
	for (i = 0; i < npo; i++) { G->rpo[i] = po[npo - 1 - i]; G->rn[G->rpo[i]] = i; }
	free(po); free(stn); free(sti);
	/* predecessors (the root is not one) */
	for (b = 0; b < nb; b++) if (G->reach[b]) { n = csucc(R, b, succ); for (i = 0; i < n; i++) G->pp[succ[i] + 1]++; }
	for (b = 0; b < nb; b++) G->pp[b + 1] += G->pp[b];
	G->pl = malloc((G->pp[nb] + 1) * sizeof(int));
	{
		int *fill = calloc(nb + 1, sizeof(int));
		if (!G->pl || !fill) { free(fill); cfg_free(G); return 0; }
		for (b = 0; b < nb; b++) if (G->reach[b]) { n = csucc(R, b, succ); for (i = 0; i < n; i++) { int s = succ[i]; G->pl[G->pp[s] + fill[s]++] = b; } }
		free(fill);
	}
	/* dominators: the root dominates the entries, and the entries are roots of the graph */
	G->idom[nb] = nb;
	guard = 0; changed = 1;
	while (changed && guard++ < 64) {
		changed = 0;
		for (k = 1; k < npo; k++) {
			int nw = -1, p, j;
			b = G->rpo[k];
			if (G->isent[b]) nw = nb;
			for (j = G->pp[b]; j < G->pp[b + 1]; j++) {
				p = G->pl[j];
				if (G->idom[p] < 0) continue;
				if (nw < 0) { nw = p; continue; }
				{
					int x = p, y = nw, g2 = 0;
					while (x != y && g2++ < 2 * nb + 4) {
						while (G->rn[x] > G->rn[y] && g2++ < 2 * nb + 4) x = G->idom[x];
						while (G->rn[y] > G->rn[x] && g2++ < 2 * nb + 4) y = G->idom[y];
					}
					nw = x;
				}
			}
			if (nw >= 0 && G->idom[b] != nw) { G->idom[b] = nw; changed = 1; }
		}
	}
	if (changed) { cfg_free(G); return 0; }
	/* natural loops: a back edge is one whose target dominates its source */
	G->lp = calloc(nb + 1, sizeof *G->lp);
	if (!G->lp) { cfg_free(G); return 0; }
	for (b = 0; b < nb; b++) if (G->reach[b]) {
		n = csucc(R, b, succ);
		for (i = 0; i < n; i++) {
			int h = succ[i], li, *stk, ns = 0, x, j;
			if (!cdom(G, h, b)) continue;
			for (li = 0; li < G->nl && G->lp[li].h != h; li++);
			if (li == G->nl) {
				G->lp[li].h = h; G->lp[li].in = calloc(nb + 1, 1); G->nl++;
				if (!G->lp[li].in) { cfg_free(G); return 0; }
				G->lp[li].in[h] = 1; G->lp[li].n = 1;
			}
			stk = malloc((nb + 1) * sizeof(int));
			if (!stk) { cfg_free(G); return 0; }
			if (!G->lp[li].in[b]) { G->lp[li].in[b] = 1; G->lp[li].n++; stk[ns++] = b; }
			while (ns) {
				x = stk[--ns];
				for (j = G->pp[x]; j < G->pp[x + 1]; j++) {
					int p = G->pl[j];
					if (G->reach[p] && !G->lp[li].in[p]) { G->lp[li].in[p] = 1; G->lp[li].n++; stk[ns++] = p; }
				}
			}
			free(stk);
		}
	}
	for (i = 0; i < G->nl; i++) for (b = 0; b < nb; b++) if (G->lp[i].in[b]) {
		G->dep[b]++;
		if (G->isent[b]) G->lp[i].bad = 1;
	}
	return G;
}
static int w10(int d) { return d <= 0 ? 1 : d == 1 ? 10 : d == 2 ? 100 : d == 3 ? 1000 : 10000; }
static int opcost(int op)
{
	switch (op) {
	case U_LI: case U_VGET: case U_SLD: return 0;
	case U_LD: case U_LD8: case U_LD16: case U_LD32: case U_MUL: case U_MULHS: case U_MULHU: case U_FMUL: return 3;
	case U_SDIV: case U_UDIV: case U_SREM: case U_UREM: case U_FDIV: return 10;
	default: return 1;
	}
}
/* value numbering: a table of what each number stands for */
#define GOP_MEM (U_N + 2)
#define GOP_PHI (U_N + 3)
#define GOP_ENT (U_N + 4)
#define GOP_PHIM (U_N + 5)
#define GNMAX (1 << 17)
typedef struct { int op, a, b, c, next; Cell imm; } GN;
static VM_TLS GN *gn_; static VM_TLS int gn_n, *gh_, gn_over;
static int gn_init(void)
{
	if (!gn_) { gn_ = malloc(GNMAX * sizeof(GN)); gh_ = malloc(GNMAX * sizeof(int)); }
	if (!gn_ || !gh_) return -1;
	gn_n = 0; gn_over = 0; memset(gh_, 0, GNMAX * sizeof(int));
	return 0;
}
static int gnum(int op, int a, int b, int c, Cell imm)
{
	unsigned h; int i;
	if (op == U_ADD || op == U_MUL || op == U_AND || op == U_OR || op == U_XOR || op == U_EQ || op == U_NE ||
		op == U_MIN || op == U_MAX || op == U_MINU || op == U_MAXU || op == U_MULHS || op == U_MULHU ||
		op == U_FADD || op == U_FMUL || op == U_FEQ) if (a > b) { int t = a; a = b; b = t; }
	h = ((unsigned)op * 2654435761u) ^ ((unsigned)a * 40503u) ^ ((unsigned)b * 69069u) ^ ((unsigned)c * 1103515245u) ^ (unsigned)(imm * 0x9e3779b1u) ^ (unsigned)((UCell)imm >> 32);
	h %= GNMAX;
	for (i = gh_[h]; i; i = gn_[i - 1].next)
		if (gn_[i - 1].op == op && gn_[i - 1].a == a && gn_[i - 1].b == b && gn_[i - 1].c == c && gn_[i - 1].imm == imm) return i;
	if (gn_n >= GNMAX - 1) { gn_over = 1; return 1; }
	gn_[gn_n].op = op; gn_[gn_n].a = a; gn_[gn_n].b = b; gn_[gn_n].c = c; gn_[gn_n].imm = imm;
	gn_[gn_n].next = gh_[h]; gh_[h] = ++gn_n;
	return gn_n;
}
typedef struct { RB *src; int val, var; } Hoist;	/* write the value src computes into var, in this block */
typedef struct { int idx, var; } Export;		/* after the block is built, var := what its idx-th op computed */
/* the value val of block S built again inside the block being emitted */
static int matv(const RB *P, const int *mp, RB *S, const int *def, int *memo, int val)
{
	Uop *u = &S->u[def[val]];
	int na, a = 0, b = 0, c = 0, r;
	if (memo[val] >= 0) return memo[val];
	if (u->op == U_LI) r = K(u->imm);
	else if (u->op == U_VGET) {
		int j = (int)u->imm;
		r = j >= 0 && j < RMAXV && P->vs[j] >= 0 ? mp[P->vs[j]] : raw(U_VGET, 0, 0, 0, u->imm);
	} else {
		na = nargs(u->op);
		if (na > 0) a = matv(P, mp, S, def, memo, u->a);
		if (na > 1) b = matv(P, mp, S, def, memo, u->b);
		if (na > 2) c = matv(P, mp, S, def, memo, u->c);
		r = oe(u->op, a, b, c, u->imm);
	}
	memo[val] = r;
	return r;
}
/* the block again through the folding emitter. repl[i] (when nonzero) makes the i-th op a read of that
   variable; the hoists and exports give variables new values at the end. returns 1 if it was rebuilt */
static int rb_emit(Rg *R, RB *B, const int *repl, const Hoist *H, int nh, const Export *X, int nx)
{
	int i, j, na, n = 0, *live, *mp, bad = 0, *hv = 0, *xv = 0;
	int *sisk = isk, *somap = omap; Cell *skv = kv;
	Uop *nu;
	for (j = 0; j < B->nso; j++) if ((unsigned)B->so[2 * j + 1] >= (unsigned)B->nv) bad = 1;
	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= B->nv) bad = 1;
	if (B->cv >= B->nv) bad = 1;
	for (i = 0; i < B->n; i++) if ((unsigned)B->u[i].d >= (unsigned)B->nv && !is_store(B->u[i].op) && B->u[i].op != U_NARG) bad = 1;
	if (bad) return 0;
	mp = malloc((B->nv + 1) * sizeof(int));
	hv = malloc((nh + 1) * sizeof(int)); xv = malloc((nx + 1) * sizeof(int));
	if (!mp || !hv || !xv) { free(mp); free(hv); free(xv); return 0; }
	isk = g_isk; kv = g_kv; omap = g_omap;
	on = 0; onv = 0; uerr = 0;
	for (i = 0; i < B->n && !uerr; i++) {
		Uop *u = &B->u[i];
		na = u->op >= U_N ? 0 : nargs(u->op);
		if (repl && repl[i] && !is_store(u->op) && u->op != U_NARG && u->op != U_NCALL) mp[u->d] = raw(U_VGET, 0, 0, 0, repl[i]);
		else if (u->op == U_VGET) mp[u->d] = raw(U_VGET, 0, 0, 0, u->imm);
		else if (u->op == U_SLD) mp[u->d] = raw(U_SLD, 0, 0, 0, u->imm);
		else if (u->op == U_LI) mp[u->d] = K(u->imm);
		else if (is_store(u->op)) oe(u->op, mp[u->a], mp[u->b], 0, 0);
		else if (u->op == U_NARG) raw(U_NARG, mp[u->a], 0, 0, u->imm);
		else if (u->op == U_NCALL) mp[u->d] = raw(U_NCALL, 0, 0, 0, u->imm);
		else mp[u->d] = oe(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
	}
	for (j = 0; j < nh && !uerr; j++) {
		RB *S = H[j].src;
		int *def = malloc((S->nv + 1) * sizeof(int)), *memo = malloc((S->nv + 1) * sizeof(int));
		if (!def || !memo) { free(def); free(memo); uerr = -5; break; }
		for (i = 0; i < S->nv; i++) { def[i] = 0; memo[i] = -1; }
		for (i = 0; i < S->n; i++) if (!is_store(S->u[i].op) && S->u[i].op != U_NARG && (unsigned)S->u[i].d < (unsigned)S->nv) def[S->u[i].d] = i;
		hv[j] = matv(B, mp, S, def, memo, H[j].val);
		free(def); free(memo);
	}
	for (j = 0; j < nx; j++) xv[j] = mp[B->u[X[j].idx].d];
	if (uerr) { isk = sisk; kv = skv; omap = somap; free(mp); free(hv); free(xv); return 0; }
	live = calloc(onv + 2, sizeof *live);
	nu = malloc((on + 1) * sizeof(Uop));
	if (!live || !nu) { free(live); free(nu); isk = sisk; kv = skv; omap = somap; free(mp); free(hv); free(xv); return 0; }
	for (j = 0; j < B->nso; j++) B->so[2 * j + 1] = mp[B->so[2 * j + 1]];
	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) B->vs[j] = mp[B->vs[j]];
	if (B->cv >= 0) B->cv = mp[B->cv];
	for (j = 0; j < nh; j++) B->vs[H[j].var] = hv[j];
	for (j = 0; j < nx; j++) B->vs[X[j].var] = xv[j];
	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && ob[B->vs[j]].op == U_VGET && ob[B->vs[j]].imm == j) B->vs[j] = -1;
	for (j = 0; j < B->nso; j++) live[B->so[2 * j + 1]] = 1;
	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) live[B->vs[j]] = 1;
	if ((B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) && B->cv >= 0) live[B->cv] = 1;
	for (i = on; i--;) {
		Uop *u = &ob[i];
		if (!is_eff(u->op) && !live[u->d]) { u->op = -1; continue; }
		na = u->op >= U_N ? 0 : nargs(u->op);
		if (is_store(u->op)) na = 2;
		if (u->op == U_NARG) na = 1;
		if (na > 0) live[u->a] = 1;
		if (na > 1) live[u->b] = 1;
		if (na > 2) live[u->c] = 1;
	}
	for (i = 0; i < on; i++) if (ob[i].op >= 0) nu[n++] = ob[i];
	free(B->u); B->u = nu; B->n = n; B->nv = onv;
	if (onv > R->maxnv) R->maxnv = onv;
	free(live); free(mp); free(hv); free(xv);
	isk = sisk; kv = skv; omap = somap;
	return 1;
}
/* a new variable for a value that has to cross blocks: 0 if the region has no register left */
static int newcarry(Rg *R)
{
	int v;
	if (R->nv >= RMAXV) return 0;
	v = R->nv++;
	R->slot[v] = R->np ? R->ps[0] : 0; R->fv[v] = 0; R->sc[v] = 1;
	return v;
}
/* ---- hoisting: an expression in a loop whose operands the loop doesn't change is built once, in front of it ---- */
static int licm_loop(Rg *R, Cfg *G, int li)
{
	int nb = R->nb, b, i, j, k, W = 0, hasmem = 0, h = G->lp[li].h, nc = 0, best = -1, P = -1, nout = 0, outp[64];
	unsigned char *in = G->lp[li].in;
	struct { int kid, cost, b, d; long long save; } cand[64];
	long long bestsave = 0;
	if (G->lp[li].bad || R->nv >= RMAXV) return 0;
	for (b = 0; b < nb; b++) if (in[b]) {
		RB *B = &R->b[b];
		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) W |= 1 << j;
		for (i = 0; i < B->n; i++) if (is_store(B->u[i].op) || B->u[i].op == U_NARG || B->u[i].op == U_NCALL) hasmem = 1;
		if (B->term == T_CALL) return 0;
	}
	for (j = G->pp[h]; j < G->pp[h + 1]; j++) if (!in[G->pl[j]]) { if (nout < 64) outp[nout] = G->pl[j]; nout++; }
	if (nout < 1 || nout > 64) return 0;
	if (gn_init()) return 0;
	for (b = 0; b < nb; b++) if (in[b]) {
		RB *B = &R->b[b];
		int n = B->nv + 1, *kid = calloc(n, sizeof(int)), *cst = calloc(n, sizeof(int)), *used = calloc(n, sizeof(int));
		unsigned char *inv = calloc(n, 1), *fpv = calloc(n, 1);
		if (!kid || !cst || !used || !inv || !fpv) { free(kid); free(cst); free(used); free(inv); free(fpv); return 0; }
		class_fp(R, B, fpv);
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			int d = u->d, na;
			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
			if (u->op == U_LI) { inv[d] = 1; kid[d] = gnum(U_LI, 0, 0, 0, u->imm); }
			else if (u->op == U_VGET) {
				j = (int)u->imm;
				inv[d] = j >= 0 && j < RMAXV && !(W >> j & 1) && !fpv[d];
				kid[d] = gnum(U_VGET, 0, 0, 0, u->imm);
			} else if (u->op == U_SLD || u->op >= U_N) inv[d] = 0;
			else {
				na = nargs(u->op);
				inv[d] = (na < 1 || inv[u->a]) && (na < 2 || inv[u->b]) && (na < 3 || inv[u->c]) && !fpv[d];
				if (is_memop(u->op) && hasmem) inv[d] = 0;
				if (inv[d]) {
					kid[d] = gnum(u->op, na > 0 ? kid[u->a] : 0, na > 1 ? kid[u->b] : 0, na > 2 ? kid[u->c] : 0, u->imm);
					cst[d] = opcost(u->op) + (na > 0 ? cst[u->a] : 0) + (na > 1 ? cst[u->b] : 0) + (na > 2 ? cst[u->c] : 0);
				}
			}
		}
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			int na;
			if (is_store(u->op)) { used[u->a] = 1; used[u->b] = 1; continue; }
			if (u->op == U_NARG) { used[u->a] = 1; continue; }
			if (u->op == U_NCALL || (unsigned)u->d >= (unsigned)B->nv || inv[u->d]) continue;
			na = u->op >= U_N ? 0 : nargs(u->op);
			if (na > 0) used[u->a] = 1;
			if (na > 1) used[u->b] = 1;
			if (na > 2) used[u->c] = 1;
		}
		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) used[B->vs[j]] = 1;
		for (j = 0; j < B->nso; j++) used[B->so[2 * j + 1]] = 1;
		if (B->cv >= 0 && (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL)) used[B->cv] = 1;
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			int d = u->d;
			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
			if (!inv[d] || !used[d] || cst[d] < 2 || u->op == U_LI || u->op == U_VGET) continue;
			for (k = 0; k < nc && cand[k].kid != kid[d]; k++);
			if (k == nc) {
				if (nc >= 64) continue;
				cand[nc].kid = kid[d]; cand[nc].cost = cst[d]; cand[nc].b = b; cand[nc].d = d; cand[nc].save = 0; nc++;
			}
			cand[k].save += (long long)cst[d] * w10(G->dep[b]);
		}
		free(kid); free(cst); free(used); free(inv); free(fpv);
	}
	if (gn_over) return 0;
	for (k = 0; k < nc; k++) {
		long long net = cand[k].save - (long long)(cand[k].cost + 1) * w10(G->dep[h] - 1);
		if (net > bestsave) { bestsave = net; best = k; }
	}
	if (best < 0 || bestsave < 3) return 0;
	/* where it goes: the one block outside the loop that falls into it, or a new one */
	if (nout == 1) {
		RB *Pb = &R->b[outp[0]];
		if ((Pb->term == T_BR && Pb->s1 == h) || Pb->term == T_CBR) P = outp[0];
	}
	if (P < 0) {
		RB *grown = realloc(R->b, (nb + 1) * sizeof(RB)), *N;
		if (!grown) return 0;
		R->b = grown;
		N = &R->b[nb];
		memset(N, 0, sizeof *N);
		N->a = R->b[h].a; N->u = malloc(sizeof(Uop)); N->so = malloc(sizeof(int)); N->n = 0; N->nv = 1; N->term = T_BR; N->s1 = h; N->s2 = -1; N->cv = -1;
		for (j = 0; j < RMAXV; j++) N->vs[j] = -1;
		for (j = 0; j < nout; j++) {
			RB *Q = &R->b[outp[j]];
			if (Q->s1 == h) Q->s1 = nb;
			if (Q->term == T_CBR && Q->s2 == h) Q->s2 = nb;
		}
		R->nb = nb + 1;
		P = nb;
	}
	{
		int s = newcarry(R);
		Hoist hs;
		RB *src = &R->b[cand[best].b];
		if (!s) return 0;
		hs.src = src; hs.val = cand[best].d; hs.var = s;
		if (!rb_emit(R, &R->b[P], 0, &hs, 1, 0, 0)) { R->nv--; return 0; }
		/* every block of the loop that builds the same thing reads the new variable instead */
		for (b = 0; b < R->nb; b++) if (b < nb && in[b]) {
			RB *B = &R->b[b];
			int n = B->nv + 1, *kid = calloc(n, sizeof(int)), *rp = calloc(B->n + 1, sizeof(int)), any = 0;
			unsigned char *inv = calloc(n, 1), *fpv = calloc(n, 1);
			if (!kid || !rp || !inv || !fpv) { free(kid); free(rp); free(inv); free(fpv); continue; }
			class_fp(R, B, fpv);
			for (i = 0; i < B->n; i++) {
				Uop *u = &B->u[i];
				int d = u->d, na;
				if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || (unsigned)d >= (unsigned)B->nv) continue;
				if (u->op == U_LI) { inv[d] = 1; kid[d] = gnum(U_LI, 0, 0, 0, u->imm); }
				else if (u->op == U_VGET) { j = (int)u->imm; inv[d] = j >= 0 && j < RMAXV && !(W >> j & 1) && !fpv[d]; kid[d] = gnum(U_VGET, 0, 0, 0, u->imm); }
				else if (u->op == U_SLD || u->op >= U_N) inv[d] = 0;
				else {
					na = nargs(u->op);
					inv[d] = (na < 1 || inv[u->a]) && (na < 2 || inv[u->b]) && (na < 3 || inv[u->c]) && !fpv[d];
					if (is_memop(u->op) && hasmem) inv[d] = 0;
					if (inv[d]) {
						kid[d] = gnum(u->op, na > 0 ? kid[u->a] : 0, na > 1 ? kid[u->b] : 0, na > 2 ? kid[u->c] : 0, u->imm);
						if (kid[d] == cand[best].kid && u->op != U_LI && u->op != U_VGET) { rp[i] = s; any = 1; }
					}
				}
			}
			if (any) rb_emit(R, B, rp, 0, 0, 0, 0);
			free(kid); free(rp); free(inv); free(fpv);
		}
	}
	return 1;
}

/* ---- shared expressions: the same value built in two blocks, one of which always runs first ---- */
typedef struct { int g[RMAXV]; int mem; } GS;
static void gtrans(Rg *R, int b, const GS *in, GS *out, int *vg, int *vc)
{
	RB *B = &R->b[b];
	int i, j, mem = in->mem;
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		int op = u->op, d = u->d, na;
		if (op == U_NARG) continue;
		if (is_store(op)) { mem = gnum(GOP_MEM, b, i, 0, 0); continue; }
		if ((unsigned)d >= (unsigned)B->nv) continue;
		vc[d] = 0;
		if (op == U_LI) vg[d] = gnum(U_LI, 0, 0, 0, u->imm);
		else if (op == U_VGET) vg[d] = in->g[(int)u->imm & (RMAXV - 1)];
		else if (op == U_SLD) vg[d] = gnum(U_SLD, b, 0, 0, u->imm);
		else if (op == U_NCALL) { vg[d] = gnum(U_NCALL, b, i, 0, 0); mem = gnum(GOP_MEM, b, i, 1, 0); }
		else if (op == U_LD || op == U_LD8 || op == U_LD16 || op == U_LD32) { vg[d] = gnum(op, vg[u->a], mem, 0, 0); vc[d] = opcost(op) + vc[u->a]; }
		else {
			na = nargs(op);
			vg[d] = gnum(op, na > 0 ? vg[u->a] : 0, na > 1 ? vg[u->b] : 0, na > 2 ? vg[u->c] : 0, u->imm);
			vc[d] = opcost(op) + (na > 0 ? vc[u->a] : 0) + (na > 1 ? vc[u->b] : 0) + (na > 2 ? vc[u->c] : 0);
		}
	}
	*out = *in;
	for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && (unsigned)B->vs[j] < (unsigned)B->nv) out->g[j] = vg[B->vs[j]];
	out->mem = mem;
}
static int gjoin(GS *d, const GS *s, int blk)
{
	int v, ch = 0, p;
	for (v = 0; v < RMAXV; v++) if (d->g[v] != s->g[v]) {
		p = gnum(GOP_PHI, blk, v, 0, 0);
		if (d->g[v] != p) { d->g[v] = p; ch = 1; }
	}
	if (d->mem != s->mem) {
		p = gnum(GOP_PHIM, blk, 0, 0, 0);
		if (d->mem != p) { d->mem = p; ch = 1; }
	}
	return ch;
}
typedef struct { int g, b, i, cost; } Site;
static int sitecmp(const void *x, const void *y)
{
	const Site *a = x, *b = y;
	if (a->g != b->g) return a->g < b->g ? -1 : 1;
	if (a->b != b->b) return a->b < b->b ? -1 : 1;
	return a->i - b->i;
}
static int cse_one(Rg *R, Cfg *G)
{
	int nb = R->nb, b, i, e, k, sweeps, changed, ns = 0, capn = 0, s, did = 0;
	GS *in = 0, *out = 0;
	unsigned char *seen = 0;
	int **vgs = 0, **vcs = 0;
	Site *sites = 0;
	if (R->nv >= RMAXV || gn_init()) return 0;
	in = calloc(nb, sizeof(GS)); out = calloc(nb, sizeof(GS)); seen = calloc(nb, 1);
	vgs = calloc(nb, sizeof(int *)); vcs = calloc(nb, sizeof(int *));
	if (!in || !out || !seen || !vgs || !vcs) goto fin;
	for (b = 0; b < nb; b++) {
		vgs[b] = calloc(R->b[b].nv + 1, sizeof(int)); vcs[b] = calloc(R->b[b].nv + 1, sizeof(int));
		if (!vgs[b] || !vcs[b]) goto fin;
	}
	for (e = 0; e < R->ne; e++) {
		b = R->entb[e];
		if ((unsigned)b >= (unsigned)nb) continue;
		for (k = 0; k < RMAXV; k++) in[b].g[k] = gnum(GOP_ENT, b, k, 0, 0);
		in[b].mem = gnum(GOP_ENT, b, RMAXV, 0, 0);
		seen[b] = 1;
	}
	for (sweeps = 0, changed = 1; changed && sweeps < 80; sweeps++) {
		changed = 0;
		for (k = 0; k < G->nrpo; k++) {
			int sc[2], nsu, t;
			b = G->rpo[k];
			if (b >= nb || !seen[b]) continue;
			gtrans(R, b, &in[b], &out[b], vgs[b], vcs[b]);
			nsu = 0;
			if (R->b[b].term == T_BR || R->b[b].term == T_CBR) if ((unsigned)R->b[b].s1 < (unsigned)nb) sc[nsu++] = R->b[b].s1;
			if (R->b[b].term == T_CBR && (unsigned)R->b[b].s2 < (unsigned)nb) sc[nsu++] = R->b[b].s2;
			for (t = 0; t < nsu; t++) {
				int sx = sc[t];
				if (!seen[sx]) { seen[sx] = 1; in[sx] = out[b]; changed = 1; }
				else if (gjoin(&in[sx], &out[b], sx)) changed = 1;
			}
		}
		if (gn_over) goto fin;
	}
	if (changed) goto fin;
	for (b = 0; b < nb; b++) if (seen[b]) gtrans(R, b, &in[b], &out[b], vgs[b], vcs[b]);
	if (gn_over) goto fin;
	/* every place that builds something worth sharing */
	for (b = 0; b < nb; b++) if (seen[b]) {
		RB *B = &R->b[b];
		unsigned char *fpv = calloc(B->nv + 1, 1);
		if (!fpv) goto fin;
		class_fp(R, B, fpv);
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			int d = u->d;
			if (is_store(u->op) || u->op == U_NARG || u->op == U_NCALL || u->op == U_LI || u->op == U_VGET || u->op == U_SLD || u->op >= U_N) continue;
			if ((unsigned)d >= (unsigned)B->nv || fpv[d] || vcs[b][d] < 3) continue;
			if (ns >= capn) {
				Site *g2;
				capn = capn ? capn * 2 : 256;
				g2 = realloc(sites, capn * sizeof(Site));
				if (!g2) { free(fpv); goto fin; }
				sites = g2;
			}
			sites[ns].g = vgs[b][d]; sites[ns].b = b; sites[ns].i = i; sites[ns].cost = vcs[b][d]; ns++;
		}
		free(fpv);
	}
	if (ns > 1) qsort(sites, ns, sizeof(Site), sitecmp);	/* no sites: sites may be null, which qsort must not get */
	{
		long long bestsave = 0;
		int bs = -1, be = -1, bd = -1;
		for (s = 0; s < ns; s = e) {
			long long save = 0;
			int d0 = -1, q;
			for (e = s; e < ns && sites[e].g == sites[s].g; e++);
			if (e - s < 2) continue;
			/* the site that comes first on every path to the others */
			for (q = s; q < e && d0 < 0; q++) {
				int r2, good = 1;
				for (r2 = s; r2 < e && good; r2++) {
					if (r2 == q) continue;
					if (sites[r2].b == sites[q].b) { if (sites[r2].i < sites[q].i) good = 0; }
					else if (!cdom(G, sites[q].b, sites[r2].b)) good = 0;
				}
				if (good) d0 = q;
			}
			if (d0 < 0) continue;
			for (q = s; q < e; q++) if (q != d0 && sites[q].b != sites[d0].b) save += (long long)sites[q].cost * w10(G->dep[sites[q].b]);
			save -= (long long)(sites[d0].cost + 1) * w10(G->dep[sites[d0].b]) / 4 + w10(G->dep[sites[d0].b]);
			if (save > bestsave) { bestsave = save; bs = s; be = e; bd = d0; }
		}
		if (bs >= 0 && bestsave >= 3) {
			int v = newcarry(R), q;
			if (v) {
				Export ex;
				ex.idx = sites[bd].i; ex.var = v;
				if (!rb_emit(R, &R->b[sites[bd].b], 0, 0, 0, &ex, 1)) { R->nv--; goto fin; }
				for (q = bs; q < be; q++) {
					RB *B = &R->b[sites[q].b];
					int *rp, q2;
					if (sites[q].b == sites[bd].b) continue;
					if (q > bs && sites[q - 1].b == sites[q].b) continue;	/* a block is rebuilt once, for all its sites */
					rp = calloc(B->n + 1, sizeof(int));
					if (!rp) continue;
					for (q2 = q; q2 < be && sites[q2].b == sites[q].b; q2++) rp[sites[q2].i] = v;
					rb_emit(R, B, rp, 0, 0, 0, 0);
					free(rp);
				}
				did = 1;
			}
		}
	}
fin:
	if (vgs) for (b = 0; b < nb; b++) free(vgs[b]);
	if (vcs) for (b = 0; b < nb; b++) free(vcs[b]);
	free(vgs); free(vcs); free(in); free(out); free(seen); free(sites);
	return did;
}
/* ---- variables written and never read ---- */
static int rdead_vars(Rg *R)
{
	int nb = R->nb, b, i, j, sweeps, changed, did = 0, nreal = R->nv0 ? R->nv0 : R->nv;
	unsigned char *gen, *kill, *lin, *lout;
	unsigned allreal = 0;
	for (j = 1; j < nreal; j++) allreal |= 1u << j;
	gen = calloc(nb, 1); kill = calloc(nb, 1); lin = calloc(nb, 1); lout = calloc(nb, 1);
	if (!gen || !kill || !lin || !lout) { free(gen); free(kill); free(lin); free(lout); return 0; }
	for (b = 0; b < nb; b++) {
		RB *B = &R->b[b];
		for (i = 0; i < B->n; i++) if (B->u[i].op == U_VGET && B->u[i].imm >= 0 && B->u[i].imm < RMAXV) gen[b] |= 1u << B->u[i].imm;
		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0) kill[b] |= 1u << j;
	}
	for (sweeps = 0, changed = 1; changed && sweeps < 64; sweeps++) {
		changed = 0;
		for (b = nb; b--;) {
			RB *B = &R->b[b];
			unsigned o = 0;
			int s, x;
			if (B->term == T_BR) o = (unsigned)B->s1 < (unsigned)nb ? lin[B->s1] : allreal;
			else if (B->term == T_CBR) {
				for (s = 0; s < 2; s++) {
					int t = s ? B->s2 : B->s1, xa = s ? B->x2 : B->x1;
					if ((unsigned)t < (unsigned)nb) o |= lin[t];
					else o |= (R->fn >= 0 && xa >= R->hi) ? 0 : allreal;
				}
			} else if (B->term == T_JMP) {
				int dead = 0;
				if (R->fn >= 0 && B->cv >= 0) for (x = 0; x < B->n; x++) if (B->u[x].d == B->cv && B->u[x].op == U_LI && B->u[x].imm >= R->hi) dead = 1;
				o = dead ? 0 : allreal;
			} else if (B->term == T_CALL) {
				int ee;
				o = allreal;
				for (ee = 0; ee < R->ne; ee++) if (R->ent[ee] == B->x2 && (unsigned)R->entb[ee] < (unsigned)nb) { o = lin[R->entb[ee]]; break; }
			} else o = allreal;
			if (o != lout[b]) { lout[b] = (unsigned char)o; changed = 1; }
			x = gen[b] | (lout[b] & ~kill[b]);
			if (x != lin[b]) { lin[b] = (unsigned char)x; changed = 1; }
		}
	}
	if (changed) { free(gen); free(kill); free(lin); free(lout); return 0; }
	for (b = 0; b < nb; b++) {
		RB *B = &R->b[b];
		int any = 0;
		for (j = 1; j < RMAXV; j++) if (B->vs[j] >= 0 && !(lout[b] >> j & 1)) { B->vs[j] = -1; any = 1; }
		if (any) { rb_emit(R, B, 0, 0, 0, 0, 0); did = 1; }
	}
	free(gen); free(kill); free(lin); free(lout);
	return did;
}
static int rshare(Rg *R)
{
	int iter, did = 0;
	if (R->nb < 1 || R->nb > 300 || getenv("ZYNOSHARE")) return 0;
	for (iter = 0; iter < 8; iter++) {
		Cfg *G = cfg_build(R);
		int li, one = 0, ord[256], n, x, y;
		if (!G) break;
		n = G->nl < 256 ? G->nl : 256;
		for (x = 0; x < n; x++) ord[x] = x;
		for (x = 1; x < n; x++) for (y = x; y > 0 && G->lp[ord[y]].n < G->lp[ord[y - 1]].n; y--) { int t = ord[y]; ord[y] = ord[y - 1]; ord[y - 1] = t; }
		for (li = 0; li < n && !one; li++) one = licm_loop(R, G, ord[li]);
		if (!one) one = cse_one(R, G);
		cfg_free(G);
		if (!one) break;
		did = 1;
	}
	if (rdead_vars(R)) did = 1;
	return did;
}
static void addl(int x, int en)
{
	int e;
	if (x < R_->lo || x >= R_->hi) return;
	if (!lead_[x - R_->lo]) { lead_[x - R_->lo] = 1; wl_[nw_++] = x; }
	if (en) { for (e = 0; e < R_->ne && R_->ent[e] != x; e++); if (e == R_->ne && e < RMAXE) R_->ent[R_->ne++] = x; }
}
/* blocks, then each one lifted and optimized */
static int rlift(Rg *R)
{
	int n = R->hi - R->lo, L, i, k, t, b, e, r = 0, cnt;
	Cell w;
	if (R->lifted) rclear(R);
	if (R->nv0) R->nv = R->nv0;
	R_ = R;
	lead_ = calloc(n, 1); bidx_ = malloc(n * sizeof(int)); wl_ = malloc(n * sizeof(int)); nw_ = 0;
	R->ne = 0;
	addl(R->lo, 1);
	while (nw_) {
		L = wl_[--nw_];
		for (i = L, cnt = 0;;) {
			if (i >= R->hi || (i != L && lead_[i - R->lo])) break;
			if (cnt++ >= 1000) { addl(i, 0); break; }
			w = m[i];
			if (w & 1) {
				if (i + 1 < R->hi && !(m[i + 1] & 1) && (k = (int)(m[i + 1] >> 1)) >= 6 && k <= 8) {
					t = (int)(w >> 1);
					if (k == 6) addl(i + 2, 1);	/* a call comes back here */
					else if (k == 7) addl(t, 0);
					else { addl(t, 0); addl(i + 2, 0); }
					break;
				}
				i++;
				continue;
			}
			k = (int)(w >> 1);
			if (k == 600) { i += 2; continue; }	/* payload is data, not an opcode */
			if (k == 6) { addl(i + 1, 1); break; }
			if (k == 8) { addl(i + 1, 0); break; }
			if (k == 0 || k == 7 || k == 9) break;
			i++;
		}
	}
	for (i = 0, b = 0; i < n; i++) if (lead_[i]) bidx_[i] = b++;
	R->nb = b; R->b = calloc(b, sizeof(RB)); R->maxnv = 0;
	for (i = 0; i < n; i++) if (lead_[i]) R->b[bidx_[i]].a = R->lo + i;
	for (b = 0; b < R->nb && !r; b++) r = lift_block(b);
	for (e = 0; e < R->ne; e++) R->entb[e] = bidx_[R->ent[e] - R->lo];
	free(lead_); free(bidx_); free(wl_);
	if (r) {
		if (trace) fprintf(stderr, "o3: %d..%d doesn't lift (%d)\n", R->lo, R->hi - 1, r);
		R->lifted = 1; rclear(R); return r;
	}
	rshape(R);
	if (!getenv("ZYNOGLOBAL")) rglobal(R);
	if (rshare(R)) rshape(R);
	for (e = 0; e < R->ne; e++) { o3ent[R->ent[e]] = R->idx + 1; o3ei[R->ent[e]] = e; }
	R->lifted = 1;
	rmark(R);
	if (trace) {
		int ops = 0;
		for (b = 0; b < R->nb; b++) ops += R->b[b].n;
		fprintf(stderr, "o3: %d..%d: %d blocks, %d entries, %d vars, %d micro-ops\n", R->lo, R->hi - 1, R->nb, R->ne, R->nv - 1, ops);
		if (getenv("ZYO3DUMP"))
			for (b = 0; b < R->nb; b++) {
				RB *B = &R->b[b];
				fprintf(stderr, " b%d @%d: %d ops, sp %+d, term %d cv v%d s %d/%d x %d/%d\n", b, B->a, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2);
				for (i = 0; i < B->n; i++) {
					Uop *u = &B->u[i];
					if (u->op == U_ST) fprintf(stderr, "   st v%d v%d\n", u->a, u->b);
					else if (u->op == U_LI) fprintf(stderr, "   v%d = %lld\n", u->d, (long long)u->imm);
					else if (u->op == U_VGET) fprintf(stderr, "   v%d = var%d\n", u->d, u->imm);
					else if (u->op == U_SLD) fprintf(stderr, "   v%d = s[%d]\n", u->d, u->imm);
					else { fprintf(stderr, "   v%d = %s v%d", u->d, unames[u->op], u->a); if (nargs(u->op) > 1) fprintf(stderr, " v%d", u->b); if (nargs(u->op) > 2) fprintf(stderr, " v%d", u->c); fprintf(stderr, "\n"); }
				}
				for (i = 0; i < B->nso; i++) fprintf(stderr, "   s[%d] = v%d\n", B->so[2 * i], B->so[2 * i + 1]);
				for (i = 1; i < R->nv; i++) if (B->vs[i] >= 0) fprintf(stderr, "   var%d = v%d\n", i, B->vs[i]);
			}
	}
	return 0;
}
/* install kind 2 */
static int o3install(int d)
{
	int lo = mread(d + 2), hi = mread(d + 3), n = mread(d + 4), i, r;
	Rg *R;
	if (lo <= 0 || hi <= lo || hi > code_words || hi - lo > 1000000 || (unsigned)n > 32 || nrg >= MAXRG) return -1;
	R = calloc(1, sizeof *R);
	R->idx = nrg; rgs[nrg++] = R;
	R->lo = lo; R->hi = hi; R->nv = 1; R->np = n;
	for (i = 1; i < RMAXV; i++) R->slot[i] = -1;
	for (i = 0; i < n; i++) {	/* pairs: frame slot, register 1..7 */
		int s = mread(d + 5 + 2 * i), v = mread(d + 6 + 2 * i), fp = 0, pl = 0;
		if (v < 0) { fp = 1; v = -v; }	/* a negative register is a d-register home */
		if (v >= 16) { pl = 1; v -= 16; }
		if ((unsigned)s >= 4096 || v < 1 || v >= RMAXV) { R->dead = 1; return -1; }
		R->fv[v] = (unsigned char)fp;
		R->sc[v] = (unsigned char)((R->sc[v] || !R->seen[v]) && pl);
		R->seen[v] = 1;
		R->ps[i] = s; R->pv[i] = v;
		if (R->slot[v] < 0) R->slot[v] = s;
		if (v >= R->nv) R->nv = v + 1;
	}
	for (i = 1; i < R->nv; i++) if (R->slot[i] < 0) R->slot[i] = R->ps[0];
	R->nv0 = R->nv;
	R->fslots = mread(d + 5 + 2 * n);
	R->argc = mread(d + 6 + 2 * n);
	R->fn = mread(d + 7 + 2 * n);
	if ((unsigned)R->argc > 32 || R->fslots < 0) { R->dead = 1; return -1; }
	if ((r = rlift(R))) { R->dead = 1; return r; }
	o3_bind(R);
	rcompile(R, 1);
	return 0;
}
/* the region entered at p, lifted again if it has to be, or 0 */
static Rg *rat(int p)
{
	Rg *R = rgs[o3ent[p] - 1];
	if (R->dead) { o3ent[p] = 0; return 0; }
	if (!R->lifted && rlift(R)) { R->dead = 1; o3ent[p] = 0; return 0; }
	if (o3ei[p] >= R->ne || R->ent[o3ei[p]] != p) { o3ent[p] = 0; return 0; }
	return R;
}

/* ---- the native jits' shared state and rules ----
   both jits (arm64 and x86-64) keep the translation table, the coverage map, the
   state block and the store hook here, so a store into translated code, a flush
   and a dropped translation mean the same thing on either. the emitters and the
   instruction selection are each jit's own. */
#ifdef ZYNATIVE
static VM_TLS unsigned char *cov;	/* how many live translations cover the word (255 sticks) */
static VM_TLS void **tab;		/* vm address -> translation */
static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); }
#ifdef ZYJIT
typedef unsigned JW;		/* code is a string of 32-bit instruction words */
#else
typedef unsigned char JW;	/* code is a string of bytes */
#define rcov cov		/* a lifted o3 region marks its words in the same map */
#endif
static VM_TLS JW *code, *cp, *x_enter, *x_exit, *x_disp, *x_first;
static VM_TLS long S[15];	/* memory, stacks, hooks, limits, and the target saved on overflow */
#ifdef ZYJIT
static void o3_invoke_jit(Cell *ps, Cell *pr, Cell target, Cell retpc)
{
	o3_invoke(ps, pr, target, retpc);
	S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
}

enum { W0 = 0, W1 = 1, X9 = 9, X10, X11, M = 19, SP = 20, RP = 21, TAB = 22, ST = 23 };

static void E(unsigned x) { *cp++ = x; }
static void li(int d, Cell v)	/* movz/movk x, four halfwords; a vm immediate */
{
	UCell u = (UCell)v;
	E(0xd2800000 | ((unsigned)(u & 0xffff) << 5) | d);
	if (u >> 16) E(0xf2a00000 | ((unsigned)((u >> 16) & 0xffff) << 5) | d);
	if (u >> 32) E(0xf2c00000 | ((unsigned)((u >> 32) & 0xffff) << 5) | d);
	if (u >> 48) E(0xf2e00000 | ((unsigned)((u >> 48) & 0xffff) << 5) | d);
}
/* executable bounds are read again after a compiler grows its code storage. */
static void cmp_code(unsigned rn)
{
	int tmp = X9;
	li(tmp, (Cell)(uintptr_t)&code_words); E(0xf9400000u | ((unsigned)tmp << 5) | tmp);
	E(0xeb00001fu | ((unsigned)tmp << 16) | (rn << 5));
}
/* cmp xn, #MW through a scratch register: a data address is in range. MW can be
   far past what an immediate encodes, since the heap grows into it. */
static void cmp_mw(unsigned rn, int tmp)
{
	li(tmp, MW);
	E(0xeb00001fu | ((unsigned)tmp << 16) | (rn << 5));
}
static void ldr_post(int t, int n, int i) { E(0xf8400400 | (i & 0x1ff) << 12 | n << 5 | t); }
static void ldr_pre(int t, int n, int i) { E(0xf8400c00 | (i & 0x1ff) << 12 | n << 5 | t); }
static void str_pre(int t, int n, int i) { E(0xf8000c00 | (i & 0x1ff) << 12 | n << 5 | t); }
static void str_post(int t, int n, int i) { E(0xf8000400 | (i & 0x1ff) << 12 | n << 5 | t); }
static void ldr_u(int t, int n, int o) { E(0xf9400000 | (o / 8) << 10 | n << 5 | t); }	/* 8-byte cells */
static void str_u(int t, int n, int o) { E(0xf9000000 | (o / 8) << 10 | n << 5 | t); }
static void ldr_d(int t, int n, int o) { E(0xfd400000 | (o / 8) << 10 | n << 5 | t); }	/* ldr Dt, [Xn, #o] */
static void str_d(int t, int n, int o) { E(0xfd000000 | (o / 8) << 10 | n << 5 | t); }
static void ldr_d_rr(int t, int n, int r) { E(0xfc605800 | r << 16 | n << 5 | t); }	/* ldr Dt, [Xn, Wr, uxtw #3] */
static void str_d_rr(int t, int n, int r) { E(0xfc205800 | r << 16 | n << 5 | t); }
static void fmov_dx(int d, int n) { E(0x9e670000 | n << 5 | d); }	/* fmov Dd, Xn */
static void fmov_xd(int d, int n) { E(0x9e660000 | n << 5 | d); }	/* fmov Xd, Dn */
static void fmov_dd(int d, int n) { E(0x1e604000 | n << 5 | d); }	/* fmov Dd, Dn */
static void ldrx_u(int t, int n, int o) { E(0xf9400000 | (o / 8) << 10 | n << 5 | t); }
static void ldr_rr(int t, int n, int r, int ext) { E(0xf8600800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
static void str_rr(int t, int n, int r, int ext) { E(0xf8200800 | r << 16 | ext << 13 | 1 << 12 | n << 5 | t); }
static void ldp_post(int t, int u, int n, int i) { E(0xa8c00000 | ((i / 8) & 0x7f) << 15 | u << 10 | n << 5 | t); }
static void op3(unsigned op, int d, int n, int r) { E(op | r << 16 | n << 5 | d); }
static void addx_ext(int d, int n, int r, int ext, int sh) { E(0x8b200000 | r << 16 | ext << 13 | sh << 10 | n << 5 | d); }
static void movw(int d, int r) { E(0xaa0003e0 | r << 16 | d); }	/* mov xd, xr */
static void blr(int n) { E(0xd63f0000 | n << 5); }
static void b_to(unsigned *t) { long o = t - cp; E(0x14000000 | (o & 0x3ffffff)); }
static void cb_fix(unsigned *at, unsigned *to) { *at |= ((to - at) & 0x7ffff) << 5; }
#define SXTW 6
#define UXTW 2

static void build_stubs(void)
{
	cp = code;
	x_enter = cp;			/* int enter(long *S, void *code) */
	E(0xa9b97bfd);			/* stp x29, x30, [sp, #-112]! */
	E(0x910003fd);			/* mov x29, sp */
	E(0xa90153f3);			/* stp x19, x20, [sp, #16] */
	E(0xa9025bf5);			/* stp x21, x22, [sp, #32] */
	E(0xf9001bf7);			/* str x23, [sp, #48] */
	E(0xa90467f8);			/* stp x24, x25, [sp, #64] */
	E(0xa9056ffa);			/* stp x26, x27, [sp, #80] */
	E(0xf90033fc);			/* str x28, [sp, #96] */
	E(0xaa0003f7);			/* mov x23, x0 */
	ldrx_u(M, ST, 0);
	ldrx_u(24, ST, 88); ldrx_u(25, ST, 96);	/* x24, x25: the stack limits, as addresses */
	ldr_u(X9, ST, 8); addx_ext(SP, M, X9, UXTW, 3);	/* sp = m + s*8 */
	ldr_u(X9, ST, 16); addx_ext(RP, M, X9, UXTW, 3);
	ldrx_u(TAB, ST, 24);
	E(0xd61f0020);			/* br x1 */
	x_exit = cp;			/* x0 = next vm address */
	op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);	/* s = (x20 - x19) >> 3 */
	op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
	E(0xa94153f3);			/* ldp x19, x20, [sp, #16] */
	E(0xa9425bf5);			/* ldp x21, x22, [sp, #32] */
	E(0xf9401bf7);			/* ldr x23, [sp, #48] */
	E(0xa94467f8);			/* ldp x24, x25, [sp, #64] */
	E(0xa9456ffa);			/* ldp x26, x27, [sp, #80] */
	E(0xf94033fc);			/* ldr x28, [sp, #96] */
	E(0xa8c77bfd);			/* ldp x29, x30, [sp], #112 */
	E(0xd65f03c0);			/* ret */
	{
	unsigned *ovd = cp, *ovr;
	li(W0, PC_DATA_OVERFLOW); b_to(x_exit);	/* data stack overflow */
	ovr = cp;
	li(W0, PC_RETURN_OVERFLOW); b_to(x_exit);	/* return stack overflow */
	x_disp = cp;			/* w0 = target: check cancellation and stacks before chaining */
 if (context_current) {
  ldrx_u(X9, ST, 112); E(0x88dffc00u | (X9 << 5) | X10);
  unsigned *ready = cp; E(0x3400000a); li(W0, PC_HALT); b_to(x_exit); cb_fix(ready, cp);
 }
	str_u(0, ST, 104);
	E(0xeb18029f);			/* cmp x20, x24 */
	E(0x54000003 | ((ovd - cp) & 0x7ffff) << 5);	/* b.lo data overflow */
	E(0xeb1902bf);			/* cmp x21, x25 */
	E(0x54000002 | ((ovr - cp) & 0x7ffff) << 5);	/* b.hs return overflow */
	}
	cmp_code(0);			/* compare the current code reservation */
	E(0x54000002 | ((x_exit - cp) & 0x7ffff) << 5);	/* b.hs exit */
	ldrx_u(TAB, ST, 24);
	E(0xf8605ac9);			/* ldr x9, [x22, w0, uxtw #3] */
	E(0xb4000009 | ((x_exit - cp) & 0x7ffff) << 5);	/* cbz x9, exit */
	E(0xd61f0120);			/* br x9 */
	x_first = cp;
}
#else
/* ---- the x86-64 emitter ----
   registers: r12 holds the memory base, r13 the data stack pointer, r14 the
   return stack pointer, r15 the translation table and rbx the state block S
   (the arm64 jit's x19..x23). rax, rcx, rdx, rsi, rdi and r8..r11 are scratch.
   the callee-saved ones keep their meaning across calls into c, and rsp stays
   16-byte aligned inside translated code so those calls are plain system v
   calls. code is built in a byte buffer: a few encoders for the forms the
   translator uses (modrm with base, index and displacement, an immediate,
   or a register), then one function per vm operation. */
enum { r_ax, r_cx, r_dx, r_bx, r_sp, r_bp, r_si, r_di, r_8, r_9, r_10, r_11, r_12, r_13, r_14, r_15 };
enum { a_add = 0, a_or = 1, a_and = 4, a_sub = 5, a_xor = 6, a_cmp = 7 };	/* alu opcode groups */
enum { c_b = 2, c_ae = 3, c_e = 4, c_ne = 5, c_be = 6, c_a = 7, c_np = 11, c_l = 12, c_ge = 13, c_le = 14, c_g = 15 };
#define rMB r_12
#define rSP r_13
#define rRP r_14
#define rTAB r_15
#define rST r_bx

static void xb(int b) { *cp++ = (unsigned char)b; }
static void xd(unsigned v) { memcpy(cp, &v, 4); cp += 4; }
static void xq(uint64_t v) { memcpy(cp, &v, 8); cp += 8; }
static void rex(int w, int r, int x, int b)
{
	int v = 0x40 | (w ? 8 : 0) | ((r & 8) ? 4 : 0) | ((x & 8) ? 2 : 0) | ((b & 8) ? 1 : 0);
	if (v != 0x40) xb(v);
}
/* modrm for [base + index * scale + disp]; index < 0 means none */
static void modrm_mem(int reg, int base, int index, int scale, int disp)
{
	int mod, sib = index >= 0 || (base & 7) == 4;
	if (disp == 0 && (base & 7) != 5) mod = 0;
	else if (disp >= -128 && disp <= 127) mod = 1;
	else mod = 2;
	xb(mod << 6 | (reg & 7) << 3 | (sib ? 4 : (base & 7)));
	if (sib) xb((scale == 8 ? 3 : scale == 4 ? 2 : scale == 2 ? 1 : 0) << 6 | ((index >= 0 ? index : 4) & 7) << 3 | (base & 7));
	if (mod == 1) xb(disp); else if (mod == 2) xd((unsigned)disp);
}
/* pre: a mandatory prefix byte or 0; two: the 0f escape */
static void mem_op(int pre, int w, int two, int opc, int reg, int base, int index, int scale, int disp)
{
	if (pre) xb(pre);
	rex(w, reg, index < 0 ? 0 : index, base);
	if (two) xb(0x0f);
	xb(opc);
	modrm_mem(reg, base, index, scale, disp);
}
static void reg_op(int pre, int w, int two, int opc, int reg, int rm)
{
	if (pre) xb(pre);
	rex(w, reg, 0, rm);
	if (two) xb(0x0f);
	xb(opc);
	xb(0xc0 | (reg & 7) << 3 | (rm & 7));
}
static void mov_rr(int d, int s) { reg_op(0, 1, 0, 0x89, s, d); }
static void mov_rm(int r, int b, int disp) { mem_op(0, 1, 0, 0x8b, r, b, -1, 1, disp); }
static void mov_mr(int b, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, -1, 1, disp); }
static void mov_rx(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8b, r, b, x, sc, disp); }
static void mov_xr(int b, int x, int sc, int disp, int r) { mem_op(0, 1, 0, 0x89, r, b, x, sc, disp); }
static void mov_r32m(int r, int b, int disp) { mem_op(0, 0, 0, 0x8b, r, b, -1, 1, disp); }	/* zero extends */
static void mov_mi(int b, int disp, int imm) { mem_op(0, 1, 0, 0xc7, 0, b, -1, 1, disp); xd((unsigned)imm); }
static void mov_r32i(int r, unsigned imm) { rex(0, 0, 0, r); xb(0xb8 | (r & 7)); xd(imm); }
static void mov_ri(int r, Cell v)
{
	UCell u = (UCell)v;
	if (u <= 0xffffffffu) mov_r32i(r, (unsigned)u);
	else if (v >= -2147483647 - 1 && v < 0) { reg_op(0, 1, 0, 0xc7, 0, r); xd((unsigned)(int)v); }
	else { rex(1, 0, 0, r); xb(0xb8 | (r & 7)); xq(u); }
}
static void lea(int r, int b, int x, int sc, int disp) { mem_op(0, 1, 0, 0x8d, r, b, x, sc, disp); }
static void xor32(int r) { reg_op(0, 0, 0, 0x31, r, r); }
static void alu_rr(int op, int d, int s) { reg_op(0, 1, 0, op * 8 + 1, s, d); }
static void alu_rm(int op, int d, int b, int disp) { mem_op(0, 1, 0, op * 8 + 3, d, b, -1, 1, disp); }
static void alu_ri(int op, int d, int imm)
{
	if (imm >= -128 && imm <= 127) { reg_op(0, 1, 0, 0x83, op, d); xb(imm); }
	else { reg_op(0, 1, 0, 0x81, op, d); xd((unsigned)imm); }
}
static void alu_mi(int op, int b, int disp, int imm)
{
	if (imm >= -128 && imm <= 127) { mem_op(0, 1, 0, 0x83, op, b, -1, 1, disp); xb(imm); }
	else { mem_op(0, 1, 0, 0x81, op, b, -1, 1, disp); xd((unsigned)imm); }
}
static void test_rr(int a, int b) { reg_op(0, 1, 0, 0x85, b, a); }
static void test32(int a, int b) { reg_op(0, 0, 0, 0x85, b, a); }
static void imul_rm(int d, int b, int disp) { mem_op(0, 1, 1, 0xaf, d, b, -1, 1, disp); }
static void imul_rmi(int d, int b, int disp, int imm) { mem_op(0, 1, 0, 0x69, d, b, -1, 1, disp); xd((unsigned)imm); }
static void shift_cl(int op, int r) { reg_op(0, 1, 0, 0xd3, op, r); }	/* 4 shl, 5 shr, 7 sar */
static void shift_ri(int op, int r, int n) { reg_op(0, 1, 0, 0xc1, op, r); xb(n); }
static void shift_mi(int op, int b, int disp, int n) { mem_op(0, 1, 0, 0xc1, op, b, -1, 1, disp); xb(n); }
static void setcc(int cc, int r) { reg_op(0, 0, 1, 0x90 + cc, 0, r); }	/* al, cl or dl only */
static void movsx_rr8(int d, int s) { reg_op(0, 0, 1, 0xbe, d, s); }		/* movsx r32, r8 (al..dl) */
static void movsx64_rr8(int d, int s) { reg_op(0, 1, 1, 0xbe, d, s); }
static void movsxd_rr(int d, int s) { reg_op(0, 1, 0, 0x63, d, s); }
static void push_r(int r) { rex(0, 0, 0, r); xb(0x50 | (r & 7)); }
static void pop_r(int r) { rex(0, 0, 0, r); xb(0x58 | (r & 7)); }
static void call_m(int b, int disp) { mem_op(0, 0, 0, 0xff, 2, b, -1, 1, disp); }
static void call_r(int r) { reg_op(0, 0, 0, 0xff, 2, r); }
static void jmp_r(int r) { reg_op(0, 0, 0, 0xff, 4, r); }
static void call_abs(void *f) { mov_ri(r_ax, (Cell)(uintptr_t)f); call_r(r_ax); }
/* branches. j8/j32 return the place to patch; the *_to forms go to a known address */
static JW *j32(int cc) { JW *at; xb(0x0f); xb(0x80 + cc); at = cp; xd(0); return at; }
static JW *jm32(void) { JW *at; xb(0xe9); at = cp; xd(0); return at; }
static JW *j8(int cc) { JW *at; xb(0x70 + cc); at = cp; xb(0); return at; }
static void fix32(JW *at, JW *to) { int d = (int)(to - (at + 4)); memcpy(at, &d, 4); }
static void fix8(JW *at, JW *to) { *at = (JW)(to - (at + 1)); }
static void jcc_to(int cc, JW *to) { JW *at = j32(cc); fix32(at, to); }
static void jmp_to(JW *to) { JW *at = jm32(); fix32(at, to); }
/* sse2 on xmm0 and xmm1 only */
static void sd_m(int opc, int x, int b, int disp) { mem_op(0xf2, 0, 1, opc, x, b, -1, 1, disp); }	/* movsd 10/11, addsd 58, mulsd 59, subsd 5c, divsd 5e */

static void build_stubs(void)
{
	JW *ovd, *ovr, *at;
	cp = code;
	x_enter = cp;			/* int enter(long *S, void *code) */
	push_r(r_bp); push_r(r_bx); push_r(r_12); push_r(r_13); push_r(r_14); push_r(r_15);
	alu_ri(a_sub, r_sp, 8);		/* six pushes and the return address: 16-byte aligned again */
	mov_rr(rST, r_di);
	mov_rm(rMB, rST, 0);
	mov_rm(r_ax, rST, 8); lea(rSP, rMB, r_ax, 8, 0);		/* sp = m + s*8 */
	mov_rm(r_ax, rST, 16); lea(rRP, rMB, r_ax, 8, 0);
	mov_rm(rTAB, rST, 24);
	jmp_r(r_si);
	x_exit = cp;			/* eax = next vm address */
	mov_rr(r_cx, rSP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 8, r_cx);
	mov_rr(r_cx, rRP); alu_rr(a_sub, r_cx, rMB); shift_ri(5, r_cx, 3); mov_mr(rST, 16, r_cx);
	alu_ri(a_add, r_sp, 8);
	pop_r(r_15); pop_r(r_14); pop_r(r_13); pop_r(r_12); pop_r(r_bx); pop_r(r_bp);
	xb(0xc3);
	ovd = cp;
	mov_r32i(r_ax, PC_DATA_OVERFLOW); jmp_to(x_exit);
	ovr = cp;
	mov_r32i(r_ax, PC_RETURN_OVERFLOW); jmp_to(x_exit);
	x_disp = cp;			/* eax = target: check cancellation and stacks before chaining */
	if (context_current) {
		mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx);
		at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp);
	}
	mov_mr(rST, 104, r_ax);
	alu_rm(a_cmp, rSP, rST, 88); jcc_to(c_b, ovd);		/* sp below its limit */
	alu_rm(a_cmp, rRP, rST, 96); jcc_to(c_ae, ovr);		/* rp at its limit */
	mov_ri(r_cx, (Cell)(uintptr_t)&code_words);		/* the current code reservation */
	alu_rm(a_cmp, r_ax, r_cx, 0); jcc_to(c_ae, x_exit);
	mov_rm(rTAB, rST, 24);
	mov_rx(r_cx, rTAB, r_ax, 8, 0);
	test_rr(r_cx, r_cx); jcc_to(c_e, x_exit);
	jmp_r(r_cx);
	x_first = cp;
}
#endif

static void flush(void)
{
	memset(tab, 0, (size_t)code_words * sizeof *tab);
	rdrop_all();
	memset(cov, 0, (size_t)code_words);
	cov[SYSGATE] = 255;
	ix_reset();
	cp = x_first;
	if (trace) fprintf(stderr, "jit: flushed\n");
}

static void jitwrite(int on)
{
#ifdef __APPLE__
	pthread_jit_write_protect_np(!on);
#else
	(void)on;
#endif
}

static void syncicache(JW *a, JW *b)
{
#ifdef __APPLE__
	sys_icache_invalidate(a, (char *)b - (char *)a);
#else
	__builtin___clear_cache((char *)a, (char *)b);
#endif
}

/* store hook for words that are covered by a translation. both jits reach it with s and r
   published in S[1] and S[2] (the x86-64 jit's emit_store calls publish_stacks first), so a
   gate store sees the exact stacks: cb_r is what op 56 returns to rt.trace() */
static int hook(Cell a, Cell v)
{
	int j, hit = 0;
	if (a == SYSGATE) {
		int moving = v == 9 && (m[a + 1] == 7 || m[a + 1] == 8);
		cb_s = S[1]; cb_r = S[2]; m[a] = hostcall(v, m[a + 1], m[a + 2], m[a + 3]);
		if (moving) { S[1] = cb_s; S[2] = cb_r; }
		S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
		S[3] = (long)tab;
		return 0;
	}
	if (m[a] == v) return 0;
	m[a] = v;
	hit = ix_drop_at((int)a);
	for (j = 0; j < nrg; j++)
		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d)\n", rgs[j]->lo, rgs[j]->hi - 1, (int)a);
			rclear(rgs[j]);
			hit = 1;
		}
	if (hit) drop_epoch++;
	return hit;
}
static int inst_hook(int d) { return install(d); }
static long xrun(long sp, int slot) { return (long)xexec(&enh[slot], (Cell *)sp); }
#endif

#ifdef ZYJIT
/* float ops 601..611 and 618. 600 (flit) is handled by translate, which eats the raw bits. */
static void fjit(int k)
{
	unsigned opw = 0;
	if (k == 601) opw = 0x1e602820;		/* fadd d0, d1, d0 */
	else if (k == 602) opw = 0x1e603820;	/* fsub */
	else if (k == 603) opw = 0x1e600820;	/* fmul */
	else if (k == 604) opw = 0x1e601820;	/* fdiv */
	if (opw) {
		E(0xfd400280);	/* ldr d0, [x20] */
		E(0xfd400681);	/* ldr d1, [x20, #8] */
		E(opw);
		E(0x91002294);	/* add x20, x20, #8 */
		E(0xfd000280);	/* str d0, [x20] */
		return;
	}
	if (k >= 605 && k <= 607) {
		E(0xfd400280);
		E(0xfd400681);
		E(0x1e602020);	/* fcmp d1, d0: left ? right. mi/ls, not lt/le: unordered is not less */
		if (k == 605) E(0x9a9f17ea);		/* cset x10, eq */
		else if (k == 606) E(0x9a9f57ea);	/* cset x10, mi */
		else E(0x9a9f87ea);			/* cset x10, ls */
		E(0x91002294);
		str_u(X10, SP, 0);
		return;
	}
	if (k == 608) { ldr_u(X9, SP, 0); E(0x9e620120); E(0xfd000280); return; }	/* scvtf d0, x9; str d0 */
	if (k == 609) { E(0xfd400280); E(0x9e78000a); str_u(X10, SP, 0); return; }	/* fcvtzs x10, d0 */
	if (k == 610) { ldr_post(0, SP, 8); li(X11, (Cell)(uintptr_t)fop_print); blr(X11); return; }
	if (k == 618) {					/* (bits prec mode --) */
		ldr_u(2, SP, 0);
		ldr_u(1, SP, 8);
		ldr_u(0, SP, 16);
		E(0x91006294);				/* add x20, x20, #24 */
		li(X11, (Cell)(uintptr_t)fop_print_mode);
		blr(X11);
		return;
	}
	ldr_u(0, SP, 0); li(X11, (Cell)(uintptr_t)fop_parse_cell); blr(X11); str_u(0, SP, 0);
}

/* byte ops 612..617. the address is a byte index into m[], little-endian. */
static void bjit(int k)
{
	Cell fn;
	if (k == 612 || k == 614 || k == 616) {
		ldr_u(0, SP, 0);
		if (k == 612) fn = (Cell)(uintptr_t)bld8;
		else if (k == 614) fn = (Cell)(uintptr_t)bld16;
		else fn = (Cell)(uintptr_t)bld32;
		li(X11, fn); blr(X11); str_u(0, SP, 0);
		return;
	}
	ldp_post(0, 1, SP, 16);
	if (k == 613) fn = (Cell)(uintptr_t)bst8;
	else if (k == 615) fn = (Cell)(uintptr_t)bst16;
	else fn = (Cell)(uintptr_t)bst32;
	li(X11, fn); blr(X11);
}

static void op(int i, int k)	/* everything except control flow */
{
	unsigned *a, *b, *c, *z;
	switch (k) {
	case 1:
		ldr_u(X9, SP, 0); cmp_mw(9, X11);
		a = cp; E(0x54000002);		/* b.hs zero */
		ldr_rr(X9, M, X9, UXTW);
		b = cp; E(0x14000000);
		cb_fix(a, cp); li(X9, 0);
		*b |= (cp - b) & 0x3ffffff;
		str_u(X9, SP, 0); return;
	case 2:	/* store: inline unless the target word is translated */
		ldp_post(X9, X10, SP, 16);
		cmp_mw(9, X11);			/* cmp x9, MW */
		a = cp; E(0x54000002);		/* b.hs done */
		ldrx_u(X11, ST, 32);
		E(0x3869496b);			/* ldrb w11, [x11, w9, uxtw] */
		b = cp; E(0x3500000b);		/* cbnz w11, slow */
		str_rr(X10, M, X9, UXTW);
		c = cp; E(0x14000000);		/* b done */
		cb_fix(b, cp);			/* slow: hook(a, v) */
		movw(W0, X9); movw(W1, X10);
		/* publish s and r (as exit does) so a syscall that calls back into the vm
		   can find the stacks. the hook reads them for a gate store */
		op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);
		op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
		ldrx_u(X11, ST, 64); blr(X11);
		ldr_u(X9, ST, 8); addx_ext(SP, M, X9, UXTW, 3);
		ldr_u(X9, ST, 16); addx_ext(RP, M, X9, UXTW, 3);
		ldrx_u(24, ST, 88); ldrx_u(25, ST, 96); ldrx_u(TAB, ST, 24);
		z = cp; E(0x34000000);		/* cbz w0, done: hook's int result */
		li(W0, i + 1); b_to(x_exit);	/* a translation died: resume at i+1 in c */
		cb_fix(a, cp); cb_fix(z, cp);
		*c |= (cp - c) & 0x3ffffff;
		return;
	case 3:	/* pick uses a word address, with the same bounds as load */
		ldr_u(X9, SP, 0);
		op3(0xcb000000, X10, SP, M); E(0xd343fd4a); /* (sp - m) / 8 */
		E(0x9100054a); op3(0x8b000000, X9, X10, X9);
		cmp_mw(9, X11);
		a = cp; E(0x54000002);
		ldr_rr(X9, M, X9, UXTW);
		b = cp; E(0x14000000);
		cb_fix(a, cp); li(X9, 0);
		*b |= (cp - b) & 0x3ffffff;
		str_u(X9, SP, 0); return;
	case 4: E(0x91002294); return;	/* add x20, x20, #8 */
	case 5: E(0xa9402a89); E(0xa900268a); return;	/* ldp x9, x10, [x20]; stp x10, x9, [x20] */
	case 10: ldrx_u(X11, ST, 40); blr(X11); E(0x93401c09); str_pre(X9, SP, -8); return; /* sxtb x9, w0 */
	case 11:
		ldrx_u(X9, ST, 56);
		if (sizeof J == 1) E(0x39400129); else E(0xb9400129);	/* ldrb or ldr w: J is a host flag, not a vm word */
		str_pre(X9, SP, -8);
		return;
	case 12: ldr_post(W0, SP, 8); E(0x93401c00); ldrx_u(X11, ST, 48); blr(X11); return; /* sxtb x0, w0 */
	}
	ldr_post(X9, SP, 8);
	ldr_u(X10, SP, 0);
	switch (k) {
	case 13: op3(0x8b000000, X10, X10, X9); break;
	case 14: op3(0xcb000000, X10, X10, X9); break;
	case 15: op3(0x9b007c00, X10, X10, X9); break;
	case 16: op3(0x8a000000, X10, X10, X9); break;
	case 17: op3(0xaa000000, X10, X10, X9); break;
	case 18: op3(0xca000000, X10, X10, X9); break;
	case 19: op3(0x9ac02800, X10, X10, X9); break;
	default: E(0xeb09015f); E(0x9a9fa7ea);	/* cmp x10, x9; cset x10, lt */
	}
	str_u(X10, SP, 0);
}

/* one micro-op, operands in registers. tmp is a scratch register for the
   remainders; x16 and x17 are scratch too. */
/* a byte address is an index into the flat image. out of range loads 0 and
   stores nothing. a store into a word some translation covers calls the
   same helper as opcode 2, so the block is dropped. x16, x17 and the saved link register are scratch,
   and the address and value are parked on the machine stack across that. */
static void emit_byte(int op, int d, int a, int b)
{
	int n = bwidth(op), j, ar;
	unsigned *oob, *cov1, *nocross, *cov2, *over;
	Cell fn;
	if (!n) return;
	str_pre(30, 31, -16);	/* x18 is reserved on darwin; borrow the saved link register */
	if (!is_store(op)) {
		ar = a;
		if (d == a) { movw(17, a); ar = 17; }
		E(0xd2800000 | d);				/* mov xd, #0 */
		li(30, ((Cell)MW << 3) - (n - 1));
		E(0xeb1e001f | (ar << 5));			/* cmp xAr, x30 */
		oob = cp; E(0x54000002);			/* b.hs done */
		E(0x8b000000 | (ar << 16) | (M << 5) | 17);	/* add x17, x19, xAr */
		if (n == 1) E(0x39400000 | (17 << 5) | d);
		else if (n == 2) E(0x79400000 | (17 << 5) | d);
		else E(0xb9400000 | (17 << 5) | d);
		cb_fix(oob, cp);
		ldr_post(30, 31, 16);
		return;
	}
	str_pre(a, 31, -16);
	str_u(b, 31, 8);
	li(16, ((Cell)MW << 3) - (n - 1));
	ldr_u(17, 31, 0);
	E(0xeb10023f);					/* cmp x17, x16 */
	oob = cp; E(0x54000002);			/* b.hs pop */
	E(0xd343fe30);					/* lsr x16, x17, #3 */
	ldrx_u(30, ST, 32);
	E(0x38704bde);					/* ldrb w30, [x30, w16, uxtw] */
	cov1 = cp; E(0x35000000 | 30);			/* cbnz w30, slow */
	E(0x92400a3e);					/* and x30, x17, #7 */
	E(0x91000000 | (n << 10) | (30 << 5) | 30);	/* add x30, x30, #n */
	E(0xf10023df);					/* cmp x30, #8 */
	nocross = cp; E(0x54000009);			/* b.ls inline */
	E(0x91000610);					/* add x16, x16, #1 */
	ldrx_u(30, ST, 32);
	E(0x38704bde);					/* ldrb w30, [x30, w16, uxtw] */
	cov2 = cp; E(0x35000000 | 30);			/* cbnz w30, slow */
	cb_fix(nocross, cp);				/* inline */
	ldr_u(17, 31, 0);
	ldr_u(16, 31, 8);
	E(0x8b11027e);					/* add x30, x19, x17 */
	if (n == 1) E(0x390003d0);
	else if (n == 2) E(0x790003d0);
	else E(0xb90003d0);				/* str w16, [x30] */
	over = cp; E(0x14000000);			/* b pop */
	cb_fix(cov1, cp);
	cb_fix(cov2, cp);				/* slow */
	E(0xd10203ff);					/* sub sp, sp, #128 */
	for (j = 0; j < 16; j += 2) E(0xa9000000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
	ldr_u(0, 31, 128);
	ldr_u(1, 31, 136);
	fn = n == 1 ? (Cell)(uintptr_t)bst8 : n == 2 ? (Cell)(uintptr_t)bst16 : (Cell)(uintptr_t)bst32;
	li(16, fn); blr(16);
	for (j = 0; j < 16; j += 2) E(0xa9400000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
	E(0x910203ff);					/* add sp, sp, #128 */
	*over |= (cp - over) & 0x3ffffff;
	cb_fix(oob, cp);				/* pop */
	E(0x910043ff);					/* add sp, sp, #16 */
	ldr_post(30, 31, 16);
	(void)d;
}
static void emit_uop(int op, int d, int a, int b, int c, int tmp)
{
	switch (op) {
	case U_ADD: op3(0x8b000000, d, a, b); break;
	case U_SUB: op3(0xcb000000, d, a, b); break;
	case U_MUL: op3(0x9b007c00, d, a, b); break;
	case U_SDIV: op3(0x9ac00c00, d, a, b); break;
	case U_UDIV: op3(0x9ac00800, d, a, b); break;
	case U_SREM: op3(0x9ac00c00, tmp, a, b); E(0x9b008000 | b << 16 | a << 10 | tmp << 5 | d); break;	/* msub */
	case U_UREM: op3(0x9ac00800, tmp, a, b); E(0x9b008000 | b << 16 | a << 10 | tmp << 5 | d); break;
	case U_AND: op3(0x8a000000, d, a, b); break;
	case U_OR: op3(0xaa000000, d, a, b); break;
	case U_XOR: op3(0xca000000, d, a, b); break;
	case U_SHL: op3(0x9ac02000, d, a, b); break;
	case U_SHR: op3(0x9ac02400, d, a, b); break;
	case U_SAR: op3(0x9ac02800, d, a, b); break;
	case U_NEG: E(0xcb0003e0 | a << 16 | d); break;
	case U_NOT: E(0xaa2003e0 | a << 16 | d); break;
	case U_LT: E(0xeb00001f | b << 16 | a << 5); E(0x9a9fa7e0 | d); break;	/* cmp; cset lt */
	case U_LTU: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f27e0 | d); break;	/* cset lo */
	case U_EQ: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f17e0 | d); break;
	case U_NE: E(0xeb00001f | b << 16 | a << 5); E(0x9a9f07e0 | d); break;
	case U_MIN: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 11 << 12 | a << 5 | d); break;
	case U_MAX: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 12 << 12 | a << 5 | d); break;
	case U_MINU: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 3 << 12 | a << 5 | d); break;
	case U_MAXU: E(0xeb00001f | b << 16 | a << 5); E(0x9a800000 | b << 16 | 8 << 12 | a << 5 | d); break;
	case U_MULHS: E(0x9b407c00 | b << 16 | a << 5 | d); break;	/* smulh */
	case U_MULHU: E(0x9bc07c00 | b << 16 | a << 5 | d); break;	/* umulh */
	case U_CLZ: E(0xdac01000 | a << 5 | d); break;
	case U_CTZ: E(0xdac00000 | a << 5 | 16); E(0xdac01000 | 16 << 5 | d); break;	/* rbit; clz */
	case U_BSWAP: E(0xdac00c00 | a << 5 | d); break;				/* rev */
	case U_POPCNT:
		fmov_dx(16, a); E(0x0e205800 | 16 << 5 | 16);			/* cnt v16.8b */
		E(0x0e31b800 | 16 << 5 | 16);					/* addv b16, v16.8b */
		E(0x0e013c00 | 16 << 5 | d); break;				/* umov w, v16.b[0] */
	case U_ROTR: op3(0x9ac02c00, d, a, b); break;
	case U_ROTL: E(0xcb0003e0 | b << 16 | 16); op3(0x9ac02c00, d, a, 16); break;	/* neg x16, b; ror */
	case U_F2S: fmov_dx(16, a); E(0x1e624000 | 16 << 5 | 16); E(0x1e260000 | 16 << 5 | d); break;	/* fcvt s16, d16; fmov w, s */
	case U_S2F: E(0x1e270000 | a << 5 | 16); E(0x1e22c000 | 16 << 5 | 16); E(0x9e660000 | 16 << 5 | d); break;	/* fmov s16, w; fcvt d16, s16; fmov x, d16 */
	case U_ABS: E(0xf100001f | a << 5); E(0xda800400 | a << 16 | 10 << 12 | a << 5 | d); break;	/* cmp #0; cneg */
	case U_SEL: E(0xf100001f | a << 5); E(0x9a800000 | c << 16 | 1 << 12 | b << 5 | d); break;	/* csel ne */
	case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: {	/* operands arrive in GPRs; enhancements have no d homes */
		unsigned base = op == U_FADD ? 0x1e602800u : op == U_FSUB ? 0x1e603800u : op == U_FMUL ? 0x1e600800u : 0x1e601800u;
		fmov_dx(16, a); fmov_dx(17, b); E(base | 17u << 16 | 16u << 5 | 16u); fmov_xd(d, 16); break; }
	case U_FEQ: case U_FLT: case U_FLE:
		fmov_dx(16, a); fmov_dx(17, b); E(0x1e602000 | 17u << 16 | 16u << 5);
		E((op == U_FEQ ? 0x9a9f17e0u : op == U_FLT ? 0x9a9f57e0u : 0x9a9f87e0u) | d); break;
	case U_I2F: E(0x9e620000 | a << 5 | 16); fmov_xd(d, 16); break;	/* scvtf d16, Xa */
	case U_F2I: fmov_dx(16, a); E(0x9e780000 | 16 << 5 | d); break;	/* fcvtzs Xd, d16 */
	case U_LD:	/* out of range reads 0, like the interpreter */
		E(0xd2800011); cmp_mw(a, 16); E(0x54000042); ldr_rr(17, M, a, UXTW); movw(d, 17);
		break;
	case U_ST: cmp_mw(a, 16); E(0x54000042); str_rr(b, M, a, UXTW); break;
	case U_LD8: case U_LD16: case U_LD32: case U_ST8: case U_ST16: case U_ST32:
		emit_byte(op, d, a, b); break;
	}
}
/* an enhancement, inlined: inputs from the data stack into x0..x15, the
   optimized micro-ops, outputs back. constants are made where they're used,
   or folded into the instruction (a fixed address, a small addend). */
static VM_TLS int ekc[MAXUOP * 4 + 16]; static VM_TLS Cell ekv[MAXUOP * 4 + 16];
static void emit_enh_e(Enh *e, int s)
{
	signed char *r = e->reg;
	int i, j, delta, na;
	if (!r) {	/* didn't fit in registers: x0 = data stack, w1 = slot, call the interpreter */
		E(0xaa1403e0); li(W1, s); ldrx_u(X11, ST, 80); blr(X11); E(0xaa0003f4);
		return;
	}
	for (i = 0; i < e->nv; i++) ekc[i] = 0;
	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { ekc[e->u[i].d] = 1; ekv[e->u[i].d] = e->u[i].imm; }
#define EOP(x) (ekc[x] ? (li(r[x], ekv[x]), r[x]) : r[x])
	for (j = 0; j < e->nin; j++) if (r[j] >= 0) ldr_u(r[j], SP, 8 * (e->nin - 1 - j));
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		int d = is_store(u->op) ? 0 : r[u->d], a = 0, b = 0, c = 0;
		if (u->op == U_LI) continue;
		if (u->op == U_LD && ekc[u->a] && inmem(ekv[u->a])) {
			if (ekv[u->a] < 4096) ldr_u(d, M, (int)ekv[u->a] * 8); else ldr_rr(d, M, EOP(u->a), UXTW);
			continue;
		}
		if ((u->op == U_ADD || u->op == U_SUB) && ekc[u->b] && ekv[u->b] >= 0 && ekv[u->b] < 4096) {
			E((u->op == U_ADD ? 0x91000000 : 0xd1000000) | (unsigned)ekv[u->b] << 10 | EOP(u->a) << 5 | d);
			continue;
		}
		na = nargs(u->op);
		if (na > 0) a = EOP(u->a);
		if (na > 1) b = EOP(u->b);
		if (na > 2) c = EOP(u->c);
		emit_uop(u->op, d, a, b, c, 16);
	}
	delta = 8 * (e->nin - e->nout);
	if (delta > 0) E(0x91000294 | delta << 10);
	if (delta < 0) E(0xd1000294 | (-delta) << 10);
	for (j = 0; j < e->nout; j++) str_u(EOP(e->out[j]), SP, 8 * (e->nout - 1 - j));
#undef EOP
}
static void emit_enh(int s) { emit_enh_e(&enh[s], s); }
/* lit .. lit xop: the enhancement specialized on the literals that feed its
   last inputs, so the optimizer sees them as constants (x / 7 becomes a
   multiply, lfa(k) a load and an add). 0 if it worked. */
static int spec_enh(int s, Cell *ks, int nk, Enh *te)
{
	Enh *e = &enh[s];
	int nin = e->nin - nk, i, j, out[8], na;
	static VM_TLS int mp[MAXUOP * 4 + 16];
	if (e->nv > MAXUOP * 4) return -1;
	un = 0; unv = nin; uerr = 0;
	for (j = 0; j < nin; j++) mp[j] = j;
	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		na = nargs(u->op);
		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
		if (u->op != U_ST) mp[u->d] = j;
	}
	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
	memset(te, 0, sizeof *te);
	if (uerr || optimize(te, nin, out, e->nout)) return -1;
	regalloc(te);
	if (!te->reg) { free(te->u); return -1; }
	return 0;
}

/* ---- o3 regions on arm64 ----
   int vars live in x0..x7 (x0 is the frame pointer) and float vars in the
   d register of the same number. block values use x8..x14 and w26..w28, or
   d8..d15 when the value is a float, so a chain of float ops does not move
   through an integer register. constants have no register: they are made in
   a scratch (x16/x17 or d16/d17) where they're used. */
static VM_TLS signed char rreg[MAXUOP * 4 + 16];
static VM_TLS unsigned char rk[MAXUOP * 4 + 16], rhome[MAXUOP * 4 + 16], rfp[MAXUOP * 4 + 16];
static VM_TLS Cell rkv[MAXUOP * 4 + 16]; static VM_TLS int rlast[MAXUOP * 4 + 16];
static VM_TLS unsigned **rfa; static VM_TLS int *rfb, rnf;	/* branches to blocks, fixed up at the end */

#define LINKN 1024
static VM_TLS void *linkstk[LINKN];
static VM_TLS Cell jit_depth;
typedef struct { void *stub; int fslots; int argc; } FnNat;
static VM_TLS FnNat *fn_nat;
static void rforget(Rg *R)
{
	int e;
	if (R->nat) for (e = 0; e < R->ne; e++) if (tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0;
	if ((unsigned)R->fn < (unsigned)code_words) fn_nat[R->fn].stub = 0;
	R->nat = 0;
}
static void rmark(Rg *R) { R->marked = 1; cinc(cov, R->lo, R->hi); }
static void runmark(Rg *R) { cdec(cov, R->lo, R->hi); }
static int ralloc(Rg *R, RB *B)
{
	int i, j, k, n = B->n;
	unsigned fr = 0x7f00u | 0x1c000000u;	/* w8..w14, w26..w28 */
	unsigned ffr = 0xff00u;			/* d8..d15 */
	unsigned char *want;
	for (j = R->nv; j < 8; j++) fr |= 1u << j;
	for (j = 1; j < R->nv && j < 8; j++) if (R->fv[j]) fr |= 1u << j;	/* that x register is free */
	for (j = 1; j < 8; j++) if (!R->fv[j]) ffr |= 1u << j;
	if (B->nv > MAXUOP * 4) return -1;
	want = calloc(B->nv + 1, 1);
	class_fp(R, B, want);
	for (i = 0; i < B->nv; i++) { rlast[i] = -2; rk[i] = 0; rhome[i] = 0; rreg[i] = -1; rfp[i] = want[i]; }
	for (i = 0; i < n; i++) {
		Uop *u = &B->u[i]; int na = nargs(u->op);
		if (na > 0) rlast[u->a] = i;
		if (na > 1) rlast[u->b] = i;
		if (na > 2) rlast[u->c] = i;
	}
	for (j = 0; j < B->nso; j++) rlast[B->so[2 * j + 1]] = n;
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) rlast[B->vs[j]] = n;
	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) rlast[B->cv] = n;
	for (i = 0; i < n; i++) {
		Uop *u = &B->u[i]; int na = nargs(u->op), ops[3], d = u->d, asfp;
		ops[0] = u->a; ops[1] = u->b; ops[2] = u->c;
		for (j = 0; j < na; j++) {
			int x = ops[j];
			if (rlast[x] == i && rreg[x] >= 0 && !rhome[x]) {
				if (rfp[x]) ffr |= 1u << rreg[x]; else fr |= 1u << rreg[x];
			}
		}
		if (is_store(u->op)) continue;
		if (u->op == U_LI) { rk[d] = 1; rkv[d] = u->imm; continue; }
		if (u->op == U_VGET) {
			rreg[d] = (signed char)u->imm; rhome[d] = 1;
			rfp[d] = (unsigned)u->imm < RMAXV && R->fv[u->imm];
			continue;
		}
		asfp = rfp[d];
		if (asfp) {
			for (k = 0; k < 32 && !(ffr >> k & 1); k++);
			if (k == 32) { free(want); return -1; }
			ffr &= ~(1u << k);
		} else {
			for (k = 0; k < 32 && !(fr >> k & 1); k++);
			if (k == 32) { free(want); return -1; }
			fr &= ~(1u << k);
		}
		rreg[d] = (signed char)k;
		if (rlast[d] == -2) { if (asfp) ffr |= 1u << k; else fr |= 1u << k; }
	}
	free(want);
	return 0;
}
/* a GPR holding the bits. a float-resident value is moved across. */
static int rgpr(int x, int s)
{
	if (rk[x]) { li(s, rkv[x]); return s; }
	if (rfp[x]) { fmov_xd(s, rreg[x]); return s; }
	return rreg[x];
}
/* a d register holding the bits. scratch s is d16 or d17. */
static int rdf(int x, int s)
{
	if (rfp[x] && !rk[x]) return rreg[x];
	if (rk[x]) { li(16, rkv[x]); fmov_dx(s, 16); return s; }
	fmov_dx(s, rreg[x]);
	return s;
}
static int ropr(int x, int s) { return rgpr(x, s); }
/* w t <-> the data stack word o places from the top at block entry */
static void rsw(int t, int o, int store)
{
	int by = o * 8;
	if (by >= 0 && by < 32768) { if (store) str_u(t, SP, by); else ldr_u(t, SP, by); }
	else if (by >= -256 && by < 256) E((store ? 0xf8000000u : 0xf8400000u) | (unsigned)(by & 0x1ff) << 12 | SP << 5 | t);	/* stur, ldur */
	else { li(16, by); addx_ext(16, SP, 16, SXTW, 0); if (store) str_u(t, 16, 0); else ldr_u(t, 16, 0); }
}
static void rswd(int t, int o, int store)
{
	int by = o * 8;
	if (by >= 0 && by < 32768) { if (store) str_d(t, SP, by); else ldr_d(t, SP, by); }
	else { li(16, by); addx_ext(16, SP, 16, SXTW, 0); if (store) str_d(t, 16, 0); else ldr_d(t, 16, 0); }
}
static void addsp(int by)
{
	if (by > 0 && by < 4096) E(0x91000294 | by << 10);
	else if (by < 0 && -by < 4096) E(0xd1000294 | (-by) << 10);
	else if (by) { li(16, by); addx_ext(SP, SP, 16, SXTW, 0); }
}
static void rspill(Rg *R)
{
	int v;
	for (v = 0; v < R->np; v++) {	/* add x16, x0, #slot */
		E(0x91000000 | R->ps[v] << 10 | 16);
		if (R->fv[R->pv[v]]) str_d_rr(R->pv[v], M, 16); else str_rr(R->pv[v], M, 16, UXTW);
	}
}
static void rload(Rg *R)
{
	int v;
	li(16, ZFP); ldr_rr(0, M, 16, UXTW);
	for (v = 1; v < R->nv; v++) {
		E(0x91000000 | R->slot[v] << 10 | 16);
		if (R->fv[v]) ldr_d_rr(v, M, 16); else ldr_rr(v, M, 16, UXTW);
	}
}
/* a direct o3 call blrs the callee's entry stub. the stub saves x30 in linkstk
   when jit_depth is nonzero, and the return reloads it and rets. depth 0 leaves
   through x_exit, which is how a region entered from bytecode returns.
   x17 is the vm address being left. vars are already spilled. a fall off the
   end (pc == hi) pushes the implicit 0. the frame is given back either way. */
static void rfinish(Rg *R)
{
	unsigned *ne, *z;
	/* entered from bytecode: the epilogue is still ahead of us and it
	   pushes the implicit 0 and puts the frame back. a direct caller
	   skipped that epilogue, so do both here and ret. */
	li(16, (Cell)(uintptr_t)&jit_depth);
	ldr_u(X9, 16, 0);
	z = cp; E(0xb4000000 | X9);		/* cbz x9, leave to the epilogue */
	li(16, R->hi);
	E(0xeb10023f);				/* cmp x17, x16 */
	ne = cp; E(0x54000001);			/* b.ne value already pushed */
	E(0xd1002294);				/* sub x20, x20, #8 */
	E(0xf900029f);				/* str xzr, [x20] */
	cb_fix(ne, cp);
	if (R->fslots) {
		li(16, ZFP);
		ldr_rr(10, M, 16, UXTW);
		if (R->fslots > 0 && R->fslots < 4096) E(0x91000000 | (R->fslots << 10) | (10 << 5) | 10);
		else { li(11, R->fslots); E(0x8b0b014a); }	/* add x10, x10, x11 */
		str_rr(10, M, 16, UXTW);
	}
	li(16, (Cell)(uintptr_t)&jit_depth);
	ldr_u(X9, 16, 0);
	E(0xd1000529);				/* sub x9, x9, #1 */
	str_u(X9, 16, 0);
	li(16, (Cell)(uintptr_t)linkstk);
	E(0x8b090e10);				/* add x16, x16, x9, lsl #3 */
	ldr_u(30, 16, 0);
	E(0xd65f03c0);				/* ret */
	cb_fix(z, cp);
	movw(W0, 17);
	b_to(x_exit);
}
static void rexitk(Rg *R, int t)
{
	rspill(R);
	if (t >= R->hi) { li(17, t); rfinish(R); }
	else { li(W0, t); b_to(x_disp); }
}
/* o3-to-o3 is a blr from this region to the callee's entry, with the frame
   built here. anything else is an ordinary vm call: the return address goes
   on the vm return stack and x_disp runs the callee. */
static void rcall(Rg *R, RB *B)
{
	unsigned *to_slow, *is_null, *too_deep, *too_small, *cz;
	int e, hopped;
	rspill(R);
	li(16, (Cell)(uintptr_t)&fn_nat); ldr_u(16, 16, 0);
	cmp_code(17);
	to_slow = cp; E(0x54000002);		/* b.hs slow */
	E(0x8b11120c);				/* add x12, x16, x17, lsl #4 */
	ldr_u(X9, 12, 0);
	is_null = cp; E(0xb4000000 | X9);	/* cbz x9, slow */
	li(16, (Cell)(uintptr_t)&jit_depth);
	ldr_u(10, 16, 0);
	E(0xf110015f);				/* cmp x10, #1024 */
	too_deep = cp; E(0x54000002);		/* b.hs slow */
	E(0xb940098a);				/* ldr w10, [x12, #8] fslots */
	E(0xb9400d8b);				/* ldr w11, [x12, #12] argc */
	li(16, ZFP);
	ldr_rr(13, M, 16, UXTW);
	E(0xcb0a01ad);				/* sub x13, x13, x10 */
	li(16, (Cell)(uintptr_t)&frame_floor_word); ldr_u(16, 16, 0); ldr_rr(14, M, 16, UXTW);
	li(16, (Cell)(uintptr_t)&frame_guard_word); ldr_u(16, 16, 0); ldr_rr(15, M, 16, UXTW);
	op3(0x8b000000, 14, 14, 15); E(0xeb0e01bf); /* cmp x13, x14 */
	too_small = cp; E(0x54000003);	/* b.lo slow: bytecode can grow the frame */
	str_pre(X9, 31, -16);
	li(16, ZFP);
	str_rr(13, M, 16, UXTW);
	cz = cp; E(0x34000000 | 11);		/* cbz w11, copied */
	E(0x5100056b);				/* sub w11, w11, #1 */
	ldr_post(14, SP, 8);
	E(0x8b0b01af);				/* add x15, x13, x11 */
	str_rr(14, M, 15, UXTW);
	b_to(cz);
	cb_fix(cz, cp);
	ldr_post(X9, 31, 16);			/* stub */
	li(16, (Cell)(uintptr_t)&jit_depth);
	ldr_u(10, 16, 0);
	E(0x9100054a);				/* add x10, x10, #1 */
	str_u(10, 16, 0);
	blr(X9);
	hopped = 0;
	for (e = 0; e < R->ne; e++) if (R->ent[e] == B->x2) { b_to((unsigned *)R->stub[e]); hopped = 1; break; }
	if (!hopped) { li(W0, B->x2); b_to(x_disp); }
	cb_fix(to_slow, cp);
	cb_fix(is_null, cp);
	cb_fix(too_deep, cp);
	cb_fix(too_small, cp);
	/* nested bytecode runs with its own direct-call depth and returns here. */
	op3(0xcb000000, X9, SP, M); E(0xd343fd29); str_u(X9, ST, 8);
	op3(0xcb000000, X9, RP, M); E(0xd343fd29); str_u(X9, ST, 16);
	E(0x91000000 | (8 << 10) | (ST << 5));
	E(0x91000000 | (16 << 10) | (ST << 5) | 1);
	movw(2, 17); li(3, B->x2); li(16, (Cell)(uintptr_t)o3_invoke_jit); blr(16);
	ldr_u(X9, ST, 8); E(0x8b090e74); /* sp = m + state[1] * 8 */
	ldr_u(X9, ST, 16); E(0x8b090e75); /* rp = m + state[2] * 8 */
	ldrx_u(24, ST, 88); ldrx_u(25, ST, 96); ldrx_u(TAB, ST, 24);
	li(16, (Cell)(uintptr_t)&halt_at); ldr_u(X9, 16, 0); E(0xf100013f);
	cz = cp; E(0x5400000b); /* b.lt resumed */
	movw(W0, X9); b_to(x_exit); cb_fix(cz, cp);
	hopped = 0;
	for (e = 0; e < R->ne; e++) if (R->ent[e] == B->x2) { b_to((unsigned *)R->stub[e]); hopped = 1; break; }
	if (!hopped) { li(W0, B->x2); b_to(x_disp); }
}
static void rbranch(int b) { rfa[rnf] = cp; rfb[rnf++] = b; E(0x14000000); }
/* m[a] = v, checked like the jit's own stores: a word some translation covers goes through hook */
static void rstore(int a, int v, int fp)
{
	unsigned *p1, *p2, *p3;
	int j;
	cmp_mw(a, 15);			/* cmp xa, MW */
	p1 = cp; E(0x54000002);		/* b.hs done */
	ldrx_u(15, ST, 32);
	E(0x38604800 | a << 16 | 15 << 5 | 15);	/* ldrb w15, [x15, wa, uxtw] */
	p2 = cp; E(0x35000000 | 15);	/* cbnz w15, slow */
	if (fp) str_d_rr(v, M, a); else str_rr(v, M, a, UXTW);
	p3 = cp; E(0x14000000);
	cb_fix(p2, cp);
	E(0xd10203ff);			/* sub sp, sp, #128 */
	for (j = 0; j < 16; j += 2) E(0xa9000000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);	/* stp xj, xj+1, [sp, #8j] */
	movw(15, a); if (fp) fmov_xd(1, v); else movw(1, v); movw(0, 15);
	ldrx_u(16, ST, 64); blr(16);
	for (j = 0; j < 16; j += 2) E(0xa9400000 | (j / 2 * 2) << 15 | (j + 1) << 10 | 31 << 5 | j);
	E(0x910203ff);			/* add sp, sp, #128 */
	cb_fix(p1, cp);
	*p3 |= (cp - p3) & 0x3ffffff;
}
static void rpmove(Rg *R, RB *B)
{
	int dst[RMAXV], src[RMAXV], n = 0, j, k, x, s;
	for (j = 1; j < R->nv; j++) if (R->fv[j] && B->vs[j] >= 0) {
		s = B->vs[j];
		if (rk[s]) { li(16, rkv[s]); fmov_dx(j, 16); continue; }
		if (!rfp[s]) { fmov_dx(j, rreg[s]); continue; }
		if (rreg[s] == j) continue;
		dst[n] = j; src[n] = rreg[s]; n++;
	}
	while (n) {
		for (j = 0; j < n; j++) { for (k = 0; k < n && src[k] != dst[j]; k++); if (k == n) break; }
		if (j < n) { fmov_dd(dst[j], src[j]); n--; dst[j] = dst[n]; src[j] = src[n]; continue; }
		x = src[0]; fmov_dd(16, x);
		for (k = 0; k < n; k++) if (src[k] == x) src[k] = 16;
	}
	n = 0;
	for (j = 1; j < R->nv; j++) if (!R->fv[j] && B->vs[j] >= 0) {
		s = B->vs[j];
		if (rk[s]) { li(j, rkv[s]); continue; }
		if (rfp[s]) { fmov_xd(j, rreg[s]); continue; }
		if (rreg[s] == j) continue;
		dst[n] = j; src[n] = rreg[s]; n++;
	}
	while (n) {
		for (j = 0; j < n; j++) { for (k = 0; k < n && src[k] != dst[j]; k++); if (k == n) break; }
		if (j < n) { movw(dst[j], src[j]); n--; dst[j] = dst[n]; src[j] = src[n]; continue; }
		x = src[0]; movw(16, x);
		for (k = 0; k < n; k++) if (src[k] == x) src[k] = 16;
	}
}
static void emit_uop(int op, int d, int a, int b, int c, int tmp);
static int remit(Rg *R, int bi)
{
	RB *B = &R->b[bi];
	int i, j, d, a, b, c;
	unsigned *p;
	if (ralloc(R, B)) return -1;
 if (context_current) {
  ldrx_u(X9, ST, 112); E(0x88dffc00u | (X9 << 5) | X10);
  unsigned *ready = cp; E(0x3400000a); li(W0, PC_HALT); b_to(x_exit); cb_fix(ready, cp);
 }
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		switch (u->op) {
		case U_LI: case U_VGET: continue;
		case U_SLD:
			if (rfp[u->d] && !rk[u->d]) rswd(rreg[u->d], (int)u->imm, 0);
			else rsw(rreg[u->d], (int)u->imm, 0);
			continue;
		case U_LD:
			if (rfp[u->d] && !rk[u->d]) {
				if (rk[u->a] && inmem(rkv[u->a])) {
					if (rkv[u->a] < 4096) ldr_d(rreg[u->d], M, (int)rkv[u->a] * 8);
					else { li(16, rkv[u->a]); ldr_d_rr(rreg[u->d], M, 16); }
				} else ldr_d_rr(rreg[u->d], M, rgpr(u->a, 16));
				continue;
			}
			if (rk[u->a] && inmem(rkv[u->a])) {	/* a fixed address */
				if (rkv[u->a] < 4096) ldr_u(rreg[u->d], M, (int)rkv[u->a] * 8);
				else { li(16, rkv[u->a]); ldr_rr(rreg[u->d], M, 16, UXTW); }
				continue;
			}
			break;
		case U_ST:
			a = rgpr(u->a, 16);
			if (rfp[u->b] && !rk[u->b]) rstore(a, rreg[u->b], 1);
			else rstore(a, rgpr(u->b, 17), 0);
			continue;
		case U_LD8: case U_LD16: case U_LD32:
			emit_byte(u->op, rreg[u->d], rgpr(u->a, 16), 0);
			continue;
		case U_ST8: case U_ST16: case U_ST32:
			emit_byte(u->op, 0, rgpr(u->a, 16), rgpr(u->b, 17));
			continue;
		case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: {
			unsigned base = u->op == U_FADD ? 0x1e602800u : u->op == U_FSUB ? 0x1e603800u : u->op == U_FMUL ? 0x1e600800u : 0x1e601800u;
			int da = rdf(u->a, 16), db = rdf(u->b, 17);
			E(base | (unsigned)db << 16 | (unsigned)da << 5 | (unsigned)rreg[u->d]);
			continue; }
		case U_FEQ: case U_FLT: case U_FLE: {
			unsigned cset = u->op == U_FEQ ? 0x9a9f17e0u : u->op == U_FLT ? 0x9a9f57e0u : 0x9a9f87e0u;
			int da = rdf(u->a, 16), db = rdf(u->b, 17);
			E(0x1e602000u | (unsigned)db << 16 | (unsigned)da << 5);
			E(cset | (unsigned)rreg[u->d]);
			continue; }
		case U_I2F: E(0x9e620000u | (unsigned)rgpr(u->a, 16) << 5 | (unsigned)rreg[u->d]); continue;
		case U_F2I: E(0x9e780000u | (unsigned)rdf(u->a, 16) << 5 | (unsigned)rreg[u->d]); continue;
		case U_F2S: E(0x1e624000u | (unsigned)rdf(u->a, 16) << 5 | 16u); E(0x1e260000u | 16u << 5 | (unsigned)rreg[u->d]); continue;	/* fcvt s16, d; fmov w, s16 */
		case U_S2F: E(0x1e270000u | (unsigned)rgpr(u->a, 16) << 5 | 16u); E(0x1e22c000u | 16u << 5 | (unsigned)rreg[u->d]); continue;	/* fmov s16, w; fcvt d, s16 */
		case U_ADD: case U_SUB:
			if (rk[u->b] && rkv[u->b] >= 0 && rkv[u->b] < 4096) {
				E((u->op == U_ADD ? 0x91000000 : 0xd1000000) | (unsigned)rkv[u->b] << 10 | ropr(u->a, 16) << 5 | rreg[u->d]);
				continue;
			}
			break;
		}
		d = rreg[u->d];
		a = nargs(u->op) > 0 ? ropr(u->a, 16) : 0;
		b = nargs(u->op) > 1 ? ropr(u->b, 17) : 0;
		c = nargs(u->op) > 2 ? ropr(u->c, 15) : 0;
		emit_uop(u->op, d, a, b, c, 15);
	}
	for (j = 0; j < B->nso; j++) {
		int v = B->so[2 * j + 1];
		if (rfp[v] && !rk[v]) rswd(rreg[v], B->so[2 * j], 1);
		else rsw(rgpr(v, 17), B->so[2 * j], 1);
	}
	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) {
		if (rk[B->cv]) li(17, rkv[B->cv]);
		else if (rfp[B->cv]) fmov_xd(17, rreg[B->cv]);
		else movw(17, rreg[B->cv]);
	}
	addsp(B->delta * 8);
	rpmove(R, B);
	switch (B->term) {
	case T_BR: rbranch(B->s1); break;
	case T_CBR:
		if (B->s1 >= 0) { rfa[rnf] = cp; rfb[rnf++] = B->s1; E(0xb4000000 | 17); }	/* cbz x17, taken */
		else { p = cp; E(0xb5000000 | 17); rexitk(R, B->x1); cb_fix(p, cp); }
		if (B->s2 >= 0) rbranch(B->s2); else rexitk(R, B->x2);
		break;
	case T_CALL: rcall(R, B); break;
	case T_RET: rspill(R); ldr_pre(W0, RP, -8); b_to(x_disp); break;
	default: {
		unsigned *hs;
		rspill(R); li(16, R->hi); E(0xeb10023f);	/* cmp x17, x16 */
		hs = cp; E(0x54000002);		/* b.hs finish: the epilogue, or past the region */
		movw(W0, 17); b_to(x_disp);
		cb_fix(hs, cp);
		rfinish(R);
		break; }
	}
	return 0;
}
/* the whole region: entry stubs that load the vars, then the blocks */
static int rjit(Rg *R, int wr)
{
	unsigned *start = cp, **lab;
	int b, e, j, bad = 0;
	long need = (long)(R->hi - R->lo) * 160 + (long)R->ne * 64 + 8192;
	if (need > (code + CODEW) - cp) return -1;
	if (wr) jitwrite(1);
	lab = malloc((R->nb + 1) * sizeof *lab);
	rfa = malloc((2 * R->nb + R->ne + 4) * sizeof *rfa); rfb = malloc((2 * R->nb + R->ne + 4) * sizeof *rfb); rnf = 0;
	for (e = 0; e < R->ne; e++) {
		unsigned *skip;
		R->stub[e] = cp;
		if (e == 0) {	/* a direct caller left its return address in x30 and bumped jit_depth */
			li(16, (Cell)(uintptr_t)&jit_depth);
			ldr_u(X9, 16, 0);
			skip = cp; E(0xb4000000 | X9);		/* cbz x9, plain entry */
			E(0xd100052a);				/* sub x10, x9, #1 */
			li(16, (Cell)(uintptr_t)linkstk);
			E(0x8b0a0e10);				/* add x16, x16, x10, lsl #3 */
			str_u(30, 16, 0);
			cb_fix(skip, cp);
		}
		rload(R); rbranch(R->entb[e]);
	}
	for (b = 0; b < R->nb && !bad; b++) { lab[b] = cp; bad = remit(R, b); }
	if (!bad) {
		for (j = 0; j < rnf; j++) {
			unsigned *at = rfa[j], *to = lab[rfb[j]];
			if ((*at & 0xfc000000u) == 0x14000000u) *at |= (to - at) & 0x3ffffff; else cb_fix(at, to);
		}
		R->nat = start;
		for (e = 0; e < R->ne; e++) tab[R->ent[e]] = R->stub[e];
		if ((unsigned)R->fn < (unsigned)code_words) {
			fn_nat[R->fn].stub = R->stub[0];
			fn_nat[R->fn].fslots = R->fslots;
			fn_nat[R->fn].argc = R->argc;
		}
		syncicache(start, cp);
		if (trace) fprintf(stderr, "o3: %d..%d -> %ld bytes of arm64\n", R->lo, R->hi - 1, (long)((char *)cp - (char *)start));
	} else {
		cp = start;
		if (trace) fprintf(stderr, "o3: %d..%d: a block needs too many registers\n", R->lo, R->hi - 1);
	}
	free(lab); free(rfa); free(rfb);
	if (wr) jitwrite(0);
	return bad;
}
static void rcompile(Rg *R, int eager)
{
	if (R->nat || R->nonat) return;
	if (rjit(R, eager) && cp - code < CODEW / 2) R->nonat = 1;	/* registers, not space: give up on it */
}

static void *translate(int p)
{
	unsigned *start;
	int i = p, n = 0, k, t, used[64], nused = 0, j2, ju;
	Cell w, x;
	if (cp - code > CODEW - 65536 || !ix_room()) flush();
	if (o3ent[p]) {	/* an o3 region starts here */
		Rg *R = rat(p);
		if (R) { rcompile(R, 0); if (R->nat && tab[p]) return tab[p]; }
	}
	start = cp;
	for (;;) {
		if (i >= code_words) { E(0x92800000); b_to(x_exit); break; }	/* movn x0, #0: halt */
		if (n++ >= BLKW) { li(W0, i); b_to(x_disp); break; }
		w = m[i];
		if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
			t = (int)((w - 1) / 2);	/* literal + call/jmp/jz with a known target */
			if (k == 8) {
				unsigned *a;
				ldr_post(X10, SP, 8);
				a = cp; E(0xb500000a);	/* cbnz x10, fallthrough */
				li(W0, t); b_to(x_disp);
				cb_fix(a, cp);
				i += 2;
				continue;
			}
			li(X10, i + 2);
			if (k == 6) str_post(X10, RP, 8); else str_u(X10, RP, 0);
			li(W0, t); b_to(x_disp);
			i += 2;
			break;
		}
		if (w & 1) {
			int L = 0, s, nk, j;
			Cell ks[8];
			Enh te;
			while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++;
			x = i + L < code_words ? m[i + L] : 1;
			if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
				nk = L < enh[s].nin ? L : enh[s].nin;
				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
				if (!spec_enh(s, ks, nk, &te)) {
					for (j = 0; j < L - nk; j++) { li(X9, m[i + j] >> 1); str_pre(X9, SP, -8); }
					if (nused < 64) used[nused++] = s; else used[0] = -1;
					emit_enh_e(&te, s);
					free(te.u); free(te.reg);
					i += L + 1; n += L;
					continue;
				}
			}
			li(X9, (w - 1) / 2); str_pre(X9, SP, -8); i++; continue;
		}
		k = (int)(w / 2);
		if (k == 0) { E(0x92800000); b_to(x_exit); i++; break; }
		if (k < 0 || k == 6 || k == 7) {
			ldr_post(W0, SP, 8);
			li(X10, i + 1);
			if (k == 6) str_post(X10, RP, 8); else str_u(X10, RP, 0);
			if (k < 0) { li(X11, 7 - k); addx_ext(RP, RP, X11, SXTW, 3); }
			b_to(x_disp);
			i++;
			break;
		}
		if (k == 8) {	/* jz: taken leaves, not taken keeps going */
			ldp_post(X9, X10, SP, 16);
			E(0xb500006a);		/* cbnz x10, +3 */
			movw(W0, X9);
			b_to(x_disp);
			i++;
			continue;
		}
		if (k == 9) { ldr_pre(W0, RP, -8); b_to(x_disp); i++; break; }
		if (k == 21) {	/* install: call into c, then return to it so it can flush */
			ldr_u(W0, SP, 0); ldrx_u(X11, ST, 72); blr(X11);
			E(0x93407c00);		/* sxtw x0, w0: Wn writes zero the high half, so -4 would look positive */
			str_u(W0, SP, 0);
			li(W0, i + 1); b_to(x_exit); i++; break;
		}
		if (k == 600) {	/* flit: the next word is raw IEEE bits, even when bit 0 is set */
			if (i + 1 >= code_words) { E(0x92800000); b_to(x_exit); i++; break; }
			li(X9, m[i + 1]);
			str_pre(X9, SP, -8);
			i += 2;
			continue;
		}
		if ((k >= 601 && k <= 611) || k == 618) { fjit(k); i++; continue; }
		if (k >= 612 && k <= 617) { bjit(k); i++; continue; }
		if (k == 619) {	/* push the data-stack index: the word s points at */
			op3(0xcb000000, X9, SP, M);	/* sub x9, x20, x19 */
			E(0xd343fd29);			/* lsr x9, x9, #3 */
			str_pre(X9, SP, -8);
			i++;
			continue;
		}
		if (k == 620) {	/* a native call: the helper pops the arguments and pushes the result */
			movw(0, SP); movw(1, RP);
			li(X11, (Cell)(uintptr_t)ncall_host); blr(X11);
			movw(SP, 0);
			i++;
			continue;
		}
		if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; emit_enh(k - XOP0); i++; continue; }
		if (k >= XOP0 && k < XOP0 + NENH) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; }	/* unwired now, but an install makes this a different block */
		if (k > 20) { E(0x92800000); b_to(x_exit); i++; break; }	/* not wired: halt */
		op(i, k);
		i++;
	}
	j2 = ix_add(p, i);
	cinc(cov, p, i);
	if (nused && used[0] < 0) dep_add(NENH, j2);
	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2);
	tab[p] = start;
	syncicache(start, cp);
	if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)((char *)cp - (char *)start));
	return start;
}
static int o3_ready(Rg *C)
{
	if (!C || C->dead) return 0;
	if (!C->lifted && rlift(C)) { C->dead = 1; return 0; }
	if (!C->lifted || C->dead) return 0;
	rcompile(C, 1);
	return C->nat != 0;
}
static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr)
{
	union { unsigned *u; int (*f)(long *, void *); } enter;
	void *saved_eq = 0, *saved_ent[RMAXE];
	int i, nent = 0, flushed = 0, ep0 = drop_epoch;
	Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1;
	Cell saved_depth = jit_depth, *saved_links = 0;
	if (saved_depth) {
		saved_links = malloc((size_t)saved_depth * sizeof *saved_links);
		if (!saved_links) return PC_FRAME_OVERFLOW;
		memcpy(saved_links, linkstk, (size_t)saved_depth * sizeof *saved_links);
	}
	jit_depth = 0;
	enter.u = x_enter;
	if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; }
	if (re) {
		nent = re->ne; if (nent > RMAXE) nent = RMAXE;
		for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; }
	}
	S[1] = *ps; S[2] = *pr;
	while (!context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) {
		void *c;
		if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
			Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW;
			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
			S[1] = sp; S[2] = rp; S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
			pc = (int)S[13]; continue;
		}
		if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break;
		if (stop_ge >= 0 && pc >= stop_ge) break;
		if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break;
		c = tab[pc];
		if (!c) {
			jitwrite(1); c = translate(pc); jitwrite(0);
			if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0;
			if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0;
		}
		if (!c) break;
		pc = enter.f(S, c);
		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
	}
	*ps = S[1]; *pr = S[2];
	jit_depth = saved_depth;
	if (saved_links) { memcpy(linkstk, saved_links, (size_t)saved_depth * sizeof *saved_links); free(saved_links); }
	flushed = drop_epoch != ep0;
	if (!flushed) {
		if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq;
		if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i];
	}
	return pc;
}
static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr)
{
	union { unsigned *u; int (*f)(long *, void *); } enter;
	void *codep;
	int back;
	Cell saved_depth, *saved_links = 0;
	if (!C || C->dead || entry < 0 || entry >= C->ne) return -1;
	if (!C->lifted && rlift(C)) { C->dead = 1; return -1; }
	rcompile(C, 1);
	codep = C->nat ? C->stub[entry] : 0;
	if (!codep) return -1;
	enter.u = x_enter;
	S[1] = *ps; S[2] = *pr;
	saved_depth = jit_depth;
	if (saved_depth) {
		saved_links = malloc((size_t)saved_depth * sizeof *saved_links);
		if (!saved_links) return PC_FRAME_OVERFLOW;
		memcpy(saved_links, linkstk, (size_t)saved_depth * sizeof *saved_links);
	}
	jit_depth = 0;
	back = enter.f(S, codep);
	jit_depth = saved_depth;
	if (saved_links) { memcpy(linkstk, saved_links, (size_t)saved_depth * sizeof *saved_links); free(saved_links); }
	*ps = S[1]; *pr = S[2];
	return back;
}

#else
/* portable path for hosts without the arm64 jit: a plain interpreter with
   the same semantics (shift counts wrap at 64, stores outside memory are
   ignored, loads outside memory read 0, leaving memory halts). */
#ifdef ZYLLVM
/* ---- llvm backend ----
   a tiered jit for hosts the arm64 backend doesn't cover. the interpreter
   counts how often each block start is reached; a hot block is translated to
   llvm ir (same block rules as the arm64 jit: it ends at call, jump, ret or
   halt, and keeps going past a jz), optimized at O2 and compiled with orc.
   enhancements are inlined as ir. stores into compiled words drop the blocks
   that cover them, so self-modifying code still works. */
#include <llvm-c/Core.h>
#include <llvm/Config/llvm-config.h>
#include <llvm-c/Error.h>
#include <llvm-c/LLJIT.h>
#include <llvm-c/Orc.h>
#include <llvm-c/Target.h>
#include <llvm-c/Transforms/PassBuilder.h>
#include <llvm-c/TargetMachine.h>
static VM_TLS int HOT = 5000;		/* runs before a block gets compiled: ZYHOT */
static VM_TLS const char *lpasses = "default<O2>";	/* ZYPASSES */
#define LMAXB 65536
typedef Cell (*Blk)(Cell *mem, Cell *regs);
static VM_TLS Blk *lt;			/* address -> compiled block */
static VM_TLS unsigned short *hot;
static VM_TLS unsigned char *lcov;		/* how many compiled blocks cover the word (255 sticks) */
static VM_TLS void *lrt[MAXB];			/* each block's resource tracker: removing it frees the block's code */
static VM_TLS int llvm_ok, nblocks;
static VM_TLS LLVMOrcLLJITRef jit;
static VM_TLS LLVMTargetMachineRef tm_;	/* the host cpu, so the pass pipeline's cost models know the target */
static VM_TLS LLVMOrcThreadSafeContextRef tsc;
static VM_TLS LLVMContextRef ctx;
static VM_TLS LLVMTypeRef i8t, i32t, i64t, ptrt, i1t, f64t;
static VM_TLS LLVMBuilderRef bd;
static VM_TLS LLVMValueRef rvv[RMAXV];
static VM_TLS LLVMValueRef fn_, vm_, vregs, vs, vr;	/* function, memory, regs, s and r slots */
static VM_TLS LLVMTypeRef l_dret;	/* what a register-passing function returns: the next address and s */
static VM_TLS int l_dm;		/* the function being built is the register-passing one */
static VM_TLS Cell d_r;		/* r, left here by a register-passing function that exits abnormally */
static VM_TLS LLVMModuleRef mod_;

static VM_TLS unsigned char *rcov;	/* how many lifted o3 regions cover the word */
static int rdrop(int a);
/* code that is dropped is freed when no compiled code is running: a block can
   drop itself (it stores into its own words), and a callback can nest */
static VM_TLS int ccode;
static VM_TLS LLVMOrcResourceTrackerRef pend_rt[1024];
static VM_TLS int npend_rt;
static void rt_sweep(void)
{
	LLVMErrorRef err;
	if (ccode) return;
	while (npend_rt > 0) {
		LLVMOrcResourceTrackerRef rt = pend_rt[--npend_rt];
		if ((err = LLVMOrcResourceTrackerRemove(rt))) LLVMConsumeError(err);
		LLVMOrcReleaseResourceTracker(rt);
	}
}
static void rt_queue(void *rt)
{
	if (!rt) return;
	if (npend_rt >= 1024) { rt_sweep(); if (npend_rt >= 1024) { LLVMOrcReleaseResourceTracker((LLVMOrcResourceTrackerRef)rt); return; } }
	pend_rt[npend_rt++] = (LLVMOrcResourceTrackerRef)rt;
}
static void be_drop(int j)
{
	int a = ix_blk[j].a;
	lt[a] = 0; hot[a] = 0;
	cdec(lcov, a, ix_blk[j].b);
	rt_queue(lrt[j]); lrt[j] = 0;
	if (trace) fprintf(stderr, "llvm: dropped %d..%d\n", a, ix_blk[j].b - 1);
}
static void ldrop_all(void)
{
	int j;
	rdrop_all();
	for (j = 0; j < ix_n; j++) if (ix_blk[j].a >= 0) { rt_queue(lrt[j]); lrt[j] = 0; }
	memset(rcov, 0, (size_t)code_words);
	memset(lt, 0, (size_t)code_words * sizeof *lt); memset(lcov, 0, (size_t)code_words); memset(hot, 0, (size_t)code_words * sizeof *hot); ix_reset();
	if (trace) fprintf(stderr, "llvm: dropped every block\n");
}
/* a store to a word some block covers: write it, drop those blocks */
static int lhook(Cell a, Cell v)
{
	int hit;
	if (m[a] == v) return 0;
	m[a] = v;
	hit = ix_drop_at((int)a);
	if (rcov[a] && rdrop((int)a)) hit = 1;
	if (hit) drop_epoch++;
	return hit;
}
static Cell linstall(Cell d) { return install((int)d); }

static LLVMValueRef K32(int v) { return LLVMConstInt(i32t, (unsigned)v, 0); }
static LLVMValueRef Kw(Cell v) { return LLVMConstInt(i64t, (unsigned long long)v, 0); }
static LLVMValueRef cptr(void *p) { return LLVMConstIntToPtr(LLVMConstInt(i64t, (unsigned long long)(size_t)p, 0), ptrt); }
static LLVMValueRef mslot(LLVMValueRef idx) { return LLVMBuildInBoundsGEP2(bd, i64t, vm_, &idx, 1, ""); }
static LLVMValueRef ld(LLVMValueRef idx) { return LLVMBuildLoad2(bd, i64t, mslot(idx), ""); }
static void st(LLVMValueRef idx, LLVMValueRef v) { LLVMBuildStore(bd, v, mslot(idx)); }
static LLVMValueRef stack_ir(void) { return LLVMBuildLoad2(bd, i64t, vs, ""); }
static void set_stack_ir(LLVMValueRef v) { LLVMBuildStore(bd, v, vs); }
static LLVMValueRef add(LLVMValueRef a, int k) { return LLVMBuildAdd(bd, a, Kw(k), ""); }
/* a load that reads 0 outside memory */
static LLVMValueRef safeld(LLVMValueRef a)
{
	LLVMValueRef in = LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), "");
	LLVMValueRef v = ld(LLVMBuildSelect(bd, in, a, Kw(0), ""));
	return LLVMBuildSelect(bd, in, v, Kw(0), "");
}
static void push(LLVMValueRef v) { LLVMValueRef s = add(stack_ir(), -1); set_stack_ir(s); st(s, v); }
/* leave the block: write s and r back, return the next vm address */
static void leave(LLVMValueRef next)
{
	LLVMValueRef r0 = Kw(0), r1 = Kw(1);
	if (l_dm) {
		LLVMValueRef agg;
		LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&d_r));
		agg = LLVMBuildInsertValue(bd, LLVMGetUndef(l_dret), next, 0, "");
		agg = LLVMBuildInsertValue(bd, agg, stack_ir(), 1, "");
		LLVMBuildRet(bd, agg);
		return;
	}
	LLVMBuildStore(bd, stack_ir(), LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""));
	LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""));
	LLVMBuildRet(bd, next);
}
/* a function region returns for good: its frame is dead, so its variables are not written back */
static void leave_ok(LLVMValueRef next)
{
	if (l_dm) {
		LLVMValueRef agg = LLVMBuildInsertValue(bd, LLVMGetUndef(l_dret), next, 0, "");
		agg = LLVMBuildInsertValue(bd, agg, stack_ir(), 1, "");
		LLVMBuildRet(bd, agg);
		return;
	}
	LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, rvv[0], ""), mslot(Kw(ZFP)));
	leave(next);
}
static LLVMBasicBlockRef newbb(void) { return LLVMAppendBasicBlockInContext(ctx, fn_, ""); }
static LLVMValueRef callp(void *f, LLVMTypeRef ret, LLVMTypeRef *at, int n, LLVMValueRef *args)
{
	LLVMTypeRef ft = LLVMFunctionType(ret, at, (unsigned)n, 0);
	return LLVMBuildCall2(bd, ft, cptr(f), args, (unsigned)n, "");
}
/* one micro-op as ir: the same semantics as ev() */
static LLVMValueRef ir_op(int op, LLVMValueRef a, LLVMValueRef b, LLVMValueRef c)
{
	LLVMValueRef z = Kw(0), q, d, t, c1, c2, r = 0;
	LLVMTypeRef i128 = LLVMIntTypeInContext(ctx, 128);
		switch (op) {
	case U_ADD: r = LLVMBuildAdd(bd, a, b, ""); break;
	case U_SUB: r = LLVMBuildSub(bd, a, b, ""); break;
	case U_MUL: r = LLVMBuildMul(bd, a, b, ""); break;
	case U_SDIV: case U_SREM:	/* /0 and INT64_MIN/-1 are defined, like the cpu */
		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
		c2 = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(INT64_MIN), ""), LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(-1), ""), "");
		d = LLVMBuildSelect(bd, LLVMBuildOr(bd, c1, c2, ""), Kw(1), b, "");
		if (op == U_SDIV) { q = LLVMBuildSDiv(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, z, q, ""); }
		else { q = LLVMBuildSRem(bd, a, d, ""); r = LLVMBuildSelect(bd, c1, a, LLVMBuildSelect(bd, c2, z, q, ""), ""); }
		break;
	case U_UDIV: case U_UREM:
		c1 = LLVMBuildICmp(bd, LLVMIntEQ, b, z, "");
		d = LLVMBuildSelect(bd, c1, Kw(1), b, "");
		if (op == U_UDIV) r = LLVMBuildSelect(bd, c1, z, LLVMBuildUDiv(bd, a, d, ""), "");
		else r = LLVMBuildSelect(bd, c1, a, LLVMBuildURem(bd, a, d, ""), "");
		break;
	case U_AND: r = LLVMBuildAnd(bd, a, b, ""); break;
	case U_OR: r = LLVMBuildOr(bd, a, b, ""); break;
	case U_XOR: r = LLVMBuildXor(bd, a, b, ""); break;
	case U_SHL: r = LLVMBuildShl(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
	case U_SHR: r = LLVMBuildLShr(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
	case U_SAR: r = LLVMBuildAShr(bd, a, LLVMBuildAnd(bd, b, Kw(63), ""), ""); break;
	case U_NEG: r = LLVMBuildSub(bd, z, a, ""); break;
	case U_NOT: r = LLVMBuildNot(bd, a, ""); break;
	case U_LT: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), i64t, ""); break;
	case U_LTU: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), i64t, ""); break;
	case U_EQ: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, b, ""), i64t, ""); break;
	case U_NE: r = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntNE, a, b, ""), i64t, ""); break;
	case U_MIN: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSLT, a, b, ""), a, b, ""); break;
	case U_MAX: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntSGT, a, b, ""), a, b, ""); break;
	case U_MINU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntULT, a, b, ""), a, b, ""); break;
	case U_MAXU: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntUGT, a, b, ""), a, b, ""); break;
	case U_MULHS: case U_MULHU: {
		LLVMValueRef x = op == U_MULHS ? LLVMBuildSExt(bd, a, i128, "") : LLVMBuildZExt(bd, a, i128, "");
		LLVMValueRef y = op == U_MULHS ? LLVMBuildSExt(bd, b, i128, "") : LLVMBuildZExt(bd, b, i128, "");
		r = LLVMBuildTrunc(bd, LLVMBuildLShr(bd, LLVMBuildMul(bd, x, y, ""), LLVMConstInt(i128, 64, 0), ""), i64t, "");
		break; }
	case U_CLZ: {
		unsigned id = LLVMLookupIntrinsicID("llvm.ctlz", 9);
		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[2];
		args[0] = a; args[1] = LLVMConstInt(i1t, 0, 0);
		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, 2, "");
		break; }
	case U_CTZ: case U_POPCNT: case U_BSWAP: {
		const char *nm = op == U_CTZ ? "llvm.cttz" : op == U_POPCNT ? "llvm.ctpop" : "llvm.bswap";
		unsigned id = LLVMLookupIntrinsicID(nm, strlen(nm));
		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[2];
		args[0] = a; args[1] = LLVMConstInt(i1t, 0, 0);
		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, op == U_CTZ ? 2 : 1, "");
		break; }
	case U_ROTL: case U_ROTR: {
		const char *nm = op == U_ROTL ? "llvm.fshl" : "llvm.fshr";
		unsigned id = LLVMLookupIntrinsicID(nm, strlen(nm));
		LLVMValueRef f = LLVMGetIntrinsicDeclaration(mod_, id, &i64t, 1), args[3];
		args[0] = a; args[1] = a; args[2] = LLVMBuildAnd(bd, b, Kw(63), "");
		r = LLVMBuildCall2(bd, LLVMIntrinsicGetType(ctx, id, &i64t, 1), f, args, 3, "");
		break; }
	case U_F2S: {
		LLVMTypeRef f32t = LLVMFloatTypeInContext(ctx), i32t = LLVMInt32TypeInContext(ctx);
		LLVMValueRef s = LLVMBuildFPTrunc(bd, LLVMBuildBitCast(bd, a, f64t, ""), f32t, "");
		r = LLVMBuildZExt(bd, LLVMBuildBitCast(bd, s, i32t, ""), i64t, "");
		break; }
	case U_S2F: {
		LLVMTypeRef f32t = LLVMFloatTypeInContext(ctx), i32t = LLVMInt32TypeInContext(ctx);
		LLVMValueRef s = LLVMBuildBitCast(bd, LLVMBuildTrunc(bd, a, i32t, ""), f32t, "");
		r = LLVMBuildBitCast(bd, LLVMBuildFPExt(bd, s, f64t, ""), i64t, "");
		break; }
	case U_ABS: t = LLVMBuildICmp(bd, LLVMIntSLT, a, z, ""); r = LLVMBuildSelect(bd, t, LLVMBuildSub(bd, z, a, ""), a, ""); break;
	case U_SEL: r = LLVMBuildSelect(bd, LLVMBuildICmp(bd, LLVMIntNE, a, z, ""), b, c, ""); break;
	case U_FADD: case U_FSUB: case U_FMUL: case U_FDIV: case U_FEQ: case U_FLT: case U_FLE: case U_I2F: case U_F2I: {
		LLVMValueRef da = LLVMBuildBitCast(bd, a, f64t, ""), db = b ? LLVMBuildBitCast(bd, b, f64t, "") : 0, dv = 0;
		if (op == U_FADD) dv = LLVMBuildFAdd(bd, da, db, "");
		else if (op == U_FSUB) dv = LLVMBuildFSub(bd, da, db, "");
		else if (op == U_FMUL) dv = LLVMBuildFMul(bd, da, db, "");
		else if (op == U_FDIV) dv = LLVMBuildFDiv(bd, da, db, "");
		else if (op == U_FEQ) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, da, db, ""), i64t, "");
		else if (op == U_FLT) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, da, db, ""), i64t, "");
		else if (op == U_FLE) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, da, db, ""), i64t, "");
		else if (op == U_I2F) dv = LLVMBuildSIToFP(bd, a, f64t, "");
		else dv = LLVMBuildFPToSI(bd, da, i64t, "");
		if (op == U_FADD || op == U_FSUB || op == U_FMUL || op == U_FDIV || op == U_I2F) dv = LLVMBuildBitCast(bd, dv, i64t, "");
		r = dv;
		break; }
	}
	return r;
}
/* byte address into the flat image. an out-of-range load is 0. a store that
   misses the image does nothing, and one that touches translated code goes
   through the same helper as a word store so the translation is dropped. */
static LLVMValueRef bload_ir(LLVMValueRef addr, int n)
{
	LLVMTypeRef ty = LLVMIntTypeInContext(ctx, (unsigned)n * 8);
	LLVMValueRef lim = Kw(((Cell)MW << 3) - (n - 1));
	LLVMValueRef in = LLVMBuildICmp(bd, LLVMIntULT, addr, lim, "");
	LLVMValueRef safe = LLVMBuildSelect(bd, in, addr, Kw(0), "");
	LLVMValueRef p = LLVMBuildGEP2(bd, i8t, vm_, &safe, 1, "");
	LLVMValueRef ldv = LLVMBuildLoad2(bd, ty, p, "");
	LLVMValueRef z;
	LLVMSetAlignment(ldv, 1);
	z = LLVMBuildZExt(bd, ldv, i64t, "");
	return LLVMBuildSelect(bd, in, z, Kw(0), "");
}
static void bstore_ir(LLVMValueRef addr, LLVMValueRef val, int n)
{
	LLVMTypeRef ty = LLVMIntTypeInContext(ctx, (unsigned)n * 8);
	LLVMBasicBlockRef inb, callb, inl, out;
	LLVMValueRef lim, word, endw, c1, c2, covered, p, stv, args[2];
	LLVMTypeRef at[2];
	void *fn = n == 1 ? (void *)bst8 : n == 2 ? (void *)bst16 : (void *)bst32;
	lim = Kw(((Cell)MW << 3) - (n - 1));
	inb = newbb(); callb = newbb(); inl = newbb(); out = newbb();
	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, addr, lim, ""), inb, out);
	LLVMPositionBuilderAtEnd(bd, inb);
	word = LLVMBuildLShr(bd, addr, Kw(3), "");
	endw = LLVMBuildLShr(bd, LLVMBuildAdd(bd, addr, Kw(n - 1), ""), Kw(3), "");
	c1 = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &word, 1, ""), "");
	c2 = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &endw, 1, ""), "");
	covered = LLVMBuildOr(bd, LLVMBuildICmp(bd, LLVMIntNE, c1, LLVMConstInt(i8t, 0, 0), ""),
		LLVMBuildICmp(bd, LLVMIntNE, c2, LLVMConstInt(i8t, 0, 0), ""), "");
	LLVMBuildCondBr(bd, covered, callb, inl);
	LLVMPositionBuilderAtEnd(bd, callb);
	at[0] = i64t; at[1] = i64t; args[0] = addr; args[1] = val;
	callp(fn, LLVMVoidTypeInContext(ctx), at, 2, args);
	LLVMBuildBr(bd, out);
	LLVMPositionBuilderAtEnd(bd, inl);
	p = LLVMBuildGEP2(bd, i8t, vm_, &addr, 1, "");
	stv = LLVMBuildStore(bd, LLVMBuildTrunc(bd, val, ty, ""), p);
	LLVMSetAlignment(stv, 1);
	LLVMBuildBr(bd, out);
	LLVMPositionBuilderAtEnd(bd, out);
}
/* an enhancement as ir */
static void ir_enh(Enh *e)
{
	LLVMValueRef *v = calloc(e->nv + 1, sizeof *v), s = stack_ir();
	int i, j;
	for (j = 0; j < e->nin; j++) v[j] = ld(add(s, e->nin - 1 - j));
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		LLVMValueRef a = u->op == U_LI ? 0 : v[u->a], b = uarity[u->op] > 1 || is_store(u->op) ? v[u->b] : 0, c = uarity[u->op] > 2 ? v[u->c] : 0, r = 0;
		switch (u->op) {
		case U_LI: r = Kw(u->imm); break;
		case U_LD: r = safeld(a); break;
		case U_ST: {
			LLVMBasicBlockRef yes = newbb(), done = newbb();
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), yes, done);
			LLVMPositionBuilderAtEnd(bd, yes); st(a, b); LLVMBuildBr(bd, done);
			LLVMPositionBuilderAtEnd(bd, done);
			break; }
		case U_LD8: case U_LD16: case U_LD32: r = bload_ir(a, bwidth(u->op)); break;
		case U_ST8: case U_ST16: case U_ST32: bstore_ir(a, b, bwidth(u->op)); break;
		default: r = ir_op(u->op, a, b, c);
		}
		if (!is_store(u->op)) v[u->d] = r;
	}
	s = add(s, e->nin - e->nout); set_stack_ir(s);
	for (j = 0; j < e->nout; j++) st(add(s, e->nout - 1 - j), v[e->out[j]]);
	free(v);
}
static void llvm_init(void)
{
	LLVMErrorRef err;
	if (llvm_ok) return;	/* a nested run() must keep the jit it already built */
	if (getenv("ZYHOT")) HOT = atoi(getenv("ZYHOT"));
	if (HOT < 1) HOT = 1;
	if (HOT > 65000) HOT = 65000;
	if (getenv("ZYPASSES")) lpasses = getenv("ZYPASSES");
	LLVMInitializeNativeTarget(); LLVMInitializeNativeAsmPrinter();
	if ((err = LLVMOrcCreateLLJIT(&jit, 0))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return; }
#if LLVM_VERSION_MAJOR >= 23
	ctx = LLVMContextCreate();
	tsc = LLVMOrcCreateNewThreadSafeContextFromLLVMContext(ctx);
#else
	tsc = LLVMOrcCreateNewThreadSafeContext();
	ctx = LLVMOrcThreadSafeContextGetContext(tsc);
#endif
	{	/* a target machine for the same host the jit generates for: llvm's vectorizer,
		   unroller and inliner price instructions from it, and the passes run before
		   the module reaches the jit, so without one they would price for nothing */
		LLVMTargetRef tg; char *e = 0, *cpu = LLVMGetHostCPUName(), *feat = LLVMGetHostCPUFeatures();
		const char *tr = LLVMOrcLLJITGetTripleString(jit);
		if (!LLVMGetTargetFromTriple(tr, &tg, &e))
			tm_ = LLVMCreateTargetMachine(tg, tr, cpu, feat, LLVMCodeGenLevelAggressive, LLVMRelocDefault, LLVMCodeModelJITDefault);
		else LLVMDisposeMessage(e);
		LLVMDisposeMessage(cpu); LLVMDisposeMessage(feat);
	}
	i1t = LLVMInt1TypeInContext(ctx); i8t = LLVMInt8TypeInContext(ctx); i32t = LLVMInt32TypeInContext(ctx);
	i64t = LLVMInt64TypeInContext(ctx); f64t = LLVMDoubleTypeInContext(ctx); ptrt = LLVMPointerTypeInContext(ctx, 0);
	bd = LLVMCreateBuilderInContext(ctx);
	llvm_ok = 1;
}
/* a new module for the jit's triple and data layout, and a function with what is
   known about its two pointers: the vm memory is MW aligned words, the register
   pair is two. no noalias: the host helpers it calls read and write that memory. */
static void lfunc_attrs(LLVMValueRef f)
{
	static VM_TLS unsigned kn, ka, kd, knu;
	LLVMValueRef p0 = LLVMGetParam(f, 0);
	(void)p0;
	if (!kn) {
		kn = LLVMGetEnumAttributeKindForName("nonnull", 7);
		ka = LLVMGetEnumAttributeKindForName("align", 5);
		kd = LLVMGetEnumAttributeKindForName("dereferenceable", 15);
		knu = LLVMGetEnumAttributeKindForName("nounwind", 8);
	}
	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, kn, 0));
	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, ka, 8));
	LLVMAddAttributeAtIndex(f, 1, LLVMCreateEnumAttribute(ctx, kd, (unsigned long long)MW * 8));
	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, kn, 0));
	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, ka, 8));
	LLVMAddAttributeAtIndex(f, 2, LLVMCreateEnumAttribute(ctx, kd, 16));
	LLVMAddAttributeAtIndex(f, (LLVMAttributeIndex)-1, LLVMCreateEnumAttribute(ctx, knu, 0));
}
static void lmodule_target(void)
{
	LLVMSetTarget(mod_, LLVMOrcLLJITGetTripleString(jit));
	LLVMSetDataLayout(mod_, LLVMOrcLLJITGetDataLayoutStr(jit));
}
static void lrun_passes(const char *pipeline)
{
	LLVMErrorRef err;
	LLVMPassBuilderOptionsRef po = LLVMCreatePassBuilderOptions();
	if ((err = LLVMRunPasses(mod_, pipeline, tm_, po))) LLVMConsumeError(err);
	LLVMDisposePassBuilderOptions(po);
}
/* compile the block at p; 0 if it couldn't be */
static Blk lcompile(int p)
{
	char name[32]; LLVMTypeRef pt[2], ft; LLVMValueRef s, a, b, v, r, args[2]; LLVMTypeRef at[2];
	LLVMOrcExecutorAddress addr; LLVMErrorRef err; LLVMPassBuilderOptionsRef po;
	int i = p, n = 0, k, t, done = 0, used[64], nused = 0, jb, ju; Cell w;
	LLVMOrcResourceTrackerRef rt;
	if (!llvm_ok) return 0;
	if (!ix_room()) ldrop_all();
	sprintf(name, "b%d_%d", p, nblocks++);
	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
	lmodule_target();
	pt[0] = ptrt; pt[1] = ptrt;
	ft = LLVMFunctionType(i64t, pt, 2, 0);
	fn_ = LLVMAddFunction(mod_, name, ft);
	lfunc_attrs(fn_);
	vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1);
	LLVMPositionBuilderAtEnd(bd, newbb());
	vs = LLVMBuildAlloca(bd, i64t, "s"); vr = LLVMBuildAlloca(bd, i64t, "r");
	{ LLVMValueRef r0 = Kw(0), r1 = Kw(1);
	  set_stack_ir(LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""), ""));
	  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""), ""), vr); }
	while (!done) {
		if (i >= code_words) { leave(Kw(PC_HALT)); break; }
		if (n++ >= 1024) { leave(Kw(i)); break; }
		w = m[i];
		if (w & 1) {
			/* a literal followed by call, jump or jz: a known target */
			if (i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
				t = (int)(w >> 1);
				if (k == 8) {
					LLVMBasicBlockRef tk = newbb(), go = newbb();
					s = stack_ir(); a = ld(s); set_stack_ir(add(s, 1));
					LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(0), ""), tk, go);
					LLVMPositionBuilderAtEnd(bd, tk); leave(Kw(t));
					LLVMPositionBuilderAtEnd(bd, go);
					i += 2; continue;
				}
				if (k == 6) { r = LLVMBuildLoad2(bd, i64t, vr, ""); st(r, Kw(i + 2)); LLVMBuildStore(bd, add(r, 1), vr); }
				leave(Kw(t)); i += 2; break;
			}
			push(Kw(w >> 1)); i++; continue;
		}
		k = (int)(w / 2);
		s = stack_ir();
		switch (k) {
		case 0: leave(Kw(PC_HALT)); done = 1; break;
		case 1: st(s, safeld(ld(s))); break;
		case 2: {
			LLVMBasicBlockRef rng = newbb(), in = newbb(), cv = newbb(), plain = newbb(), out = newbb(), quit = newbb(), gate = newbb();
			LLVMTypeRef gt[4];
			LLVMValueRef gv[4];
			a = ld(s); b = ld(add(s, 1)); set_stack_ir(add(s, 2));
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, a, Kw(SYSGATE), ""), gate, rng);
			LLVMPositionBuilderAtEnd(bd, gate);
			gt[0] = gt[1] = gt[2] = gt[3] = i64t;
			gv[0] = b; gv[1] = ld(Kw(SYSGATE + 1)); gv[2] = ld(Kw(SYSGATE + 2)); gv[3] = ld(Kw(SYSGATE + 3));
			LLVMBuildStore(bd, stack_ir(), cptr(&cb_s));
			LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&cb_r));
			st(Kw(SYSGATE), callp((void *)hostcall, i64t, gt, 4, gv));
			{
				LLVMBasicBlockRef moved = newbb();
				LLVMValueRef moving = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(9), ""),
					LLVMBuildOr(bd, LLVMBuildICmp(bd, LLVMIntEQ, gv[1], Kw(7), ""), LLVMBuildICmp(bd, LLVMIntEQ, gv[1], Kw(8), ""), ""), "");
				LLVMBuildCondBr(bd, moving, moved, out);
				LLVMPositionBuilderAtEnd(bd, moved);
				set_stack_ir(LLVMBuildLoad2(bd, i64t, cptr(&cb_s), ""));
				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, cptr(&cb_r), ""), vr);
				LLVMBuildBr(bd, out);
			}
			LLVMPositionBuilderAtEnd(bd, rng);
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), in, out);
			LLVMPositionBuilderAtEnd(bd, in);
			v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
			LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
			LLVMPositionBuilderAtEnd(bd, cv);
			at[0] = i64t; at[1] = i64t; args[0] = a; args[1] = b;
			v = callp((void *)lhook, i32t, at, 2, args);
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, K32(0), ""), quit, out);
			LLVMPositionBuilderAtEnd(bd, quit); leave(Kw(i + 1));	/* a block died: back to the loop */
			LLVMPositionBuilderAtEnd(bd, out);
			break; }
		case 3: a = ld(s); st(s, safeld(LLVMBuildAdd(bd, add(s, 1), a, ""))); break;
		case 4: set_stack_ir(add(s, 1)); break;
		case 5: a = ld(s); b = ld(add(s, 1)); st(s, b); st(add(s, 1), a); break;
		case 6: case 7:
			a = ld(s); set_stack_ir(add(s, 1));
			if (k == 6) { r = LLVMBuildLoad2(bd, i64t, vr, ""); st(r, Kw(i + 1)); LLVMBuildStore(bd, add(r, 1), vr); }
			leave(a); done = 1; break;
		case 8: {
			LLVMBasicBlockRef tk = newbb(), go = newbb();
			a = ld(s); b = ld(add(s, 1)); set_stack_ir(add(s, 2));
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, b, Kw(0), ""), tk, go);
			LLVMPositionBuilderAtEnd(bd, tk); leave(a);
			LLVMPositionBuilderAtEnd(bd, go);
			break; }
		case 9:
			r = add(LLVMBuildLoad2(bd, i64t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
			leave(ld(r)); done = 1; break;
		case 10: v = callp((void *)Q, i8t, 0, 0, 0); push(LLVMBuildSExt(bd, v, i64t, "")); break;
		case 11:
			if (sizeof J == 1) push(LLVMBuildZExt(bd, LLVMBuildLoad2(bd, i8t, cptr(&J), ""), i64t, ""));
			else push(LLVMBuildSExt(bd, LLVMBuildLoad2(bd, i32t, cptr(&J), ""), i64t, ""));
			break;
		case 12: at[0] = i8t; args[0] = LLVMBuildTrunc(bd, ld(s), i8t, ""); callp((void *)G, LLVMVoidTypeInContext(ctx), at, 1, args); set_stack_ir(add(s, 1)); break;
		case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20:
			a = ld(s); b = ld(add(s, 1)); s = add(s, 1); set_stack_ir(s);
			switch (k) {
			case 13: v = LLVMBuildAdd(bd, b, a, ""); break;
			case 14: v = LLVMBuildSub(bd, b, a, ""); break;
			case 15: v = LLVMBuildMul(bd, b, a, ""); break;
			case 16: v = LLVMBuildAnd(bd, b, a, ""); break;
			case 17: v = LLVMBuildOr(bd, b, a, ""); break;
			case 18: v = LLVMBuildXor(bd, b, a, ""); break;
			case 19: v = LLVMBuildAShr(bd, b, LLVMBuildAnd(bd, a, Kw(63), ""), ""); break;
			default: v = LLVMBuildZExt(bd, LLVMBuildICmp(bd, LLVMIntSLT, b, a, ""), i64t, "");
			}
			st(s, v); break;
		case 21:	/* install: then back to the loop, which drops every block */
			at[0] = i64t; args[0] = ld(s);
			st(s, callp((void *)linstall, i64t, at, 1, args));
			leave(Kw(i + 1)); done = 1; break;
		case 600:	/* flit: next word is raw bits */
			if (i + 1 >= code_words) { leave(Kw(PC_HALT)); done = 1; break; }
			i++; push(Kw(m[i])); break;
		case 601: case 602: case 603: case 604: case 605: case 606: case 607: {
			LLVMValueRef da, db, dv;
			a = ld(s); b = ld(add(s, 1)); s = add(s, 1); set_stack_ir(s);
			da = LLVMBuildBitCast(bd, a, f64t, "");
			db = LLVMBuildBitCast(bd, b, f64t, "");
			if (k == 601) dv = LLVMBuildFAdd(bd, db, da, "");
			else if (k == 602) dv = LLVMBuildFSub(bd, db, da, "");
			else if (k == 603) dv = LLVMBuildFMul(bd, db, da, "");
			else if (k == 604) dv = LLVMBuildFDiv(bd, db, da, "");
			else if (k == 605) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, db, da, ""), i64t, "");
			else if (k == 606) dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, db, da, ""), i64t, "");
			else dv = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, db, da, ""), i64t, "");
			if (k <= 604) dv = LLVMBuildBitCast(bd, dv, i64t, "");
			st(s, dv);
			break; }
		case 608:
			st(s, LLVMBuildBitCast(bd, LLVMBuildSIToFP(bd, ld(s), f64t, ""), i64t, ""));
			break;
		case 609:
			st(s, LLVMBuildFPToSI(bd, LLVMBuildBitCast(bd, ld(s), f64t, ""), i64t, ""));
			break;
		case 610: {
			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
			callp((void *)fop_print, LLVMVoidTypeInContext(ctx), &ft, 1, &arg);
			set_stack_ir(add(s, 1));
			break; }
		case 611: {
			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
			st(s, callp((void *)fop_parse_cell, i64t, &ft, 1, &arg));
			break; }
		case 618: {					/* (bits prec mode --) */
			LLVMTypeRef ft[3];
			LLVMValueRef args3[3];
			ft[0] = ft[1] = ft[2] = i64t;
			args3[0] = ld(add(s, 2));
			args3[1] = ld(add(s, 1));
			args3[2] = ld(s);
			callp((void *)fop_print_mode, LLVMVoidTypeInContext(ctx), ft, 3, args3);
			set_stack_ir(add(s, 3));
			break; }
		case 612: case 614: case 616: {
			LLVMTypeRef ft = i64t; LLVMValueRef arg = ld(s);
			void *f = k == 612 ? (void *)bld8 : k == 614 ? (void *)bld16 : (void *)bld32;
			st(s, callp(f, i64t, &ft, 1, &arg));
			break; }
		case 619: push(s); break;
		case 620: {
			LLVMTypeRef ft = i64t; LLVMValueRef arg = s;
			LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), cptr(&cb_r));
			set_stack_ir(callp((void *)ncall_exec, i64t, &ft, 1, &arg));
			break; }
		case 613: case 615: case 617: {
			LLVMTypeRef ft[2], vt = LLVMVoidTypeInContext(ctx);
			LLVMValueRef args[2];
			void *f = k == 613 ? (void *)bst8 : k == 615 ? (void *)bst16 : (void *)bst32;
			ft[0] = i64t; ft[1] = i64t;
			args[0] = ld(s); args[1] = ld(add(s, 1));
			callp(f, vt, ft, 2, args);
			set_stack_ir(add(s, 2));
			break; }
		default:
			if (k >= XOP0 && k < XOP0 + NENH) { if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1; }
			if (k >= XOP0 && k < XOP0 + NENH && enh[k - XOP0].live) { ir_enh(&enh[k - XOP0]); break; }
			leave(Kw(PC_HALT)); done = 1;	/* not wired: halt */
		}
		i++;
	}
	jb = ix_add(p, i);
	cinc(lcov, p, i);
	if (nused && used[0] < 0) dep_add(NENH, jb);
	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], jb);
	lrun_passes(lpasses);
	rt = LLVMOrcJITDylibCreateResourceTracker(LLVMOrcLLJITGetMainJITDylib(jit));
	if ((err = LLVMOrcLLJITAddLLVMIRModuleWithRT(jit, rt, LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); LLVMOrcReleaseResourceTracker(rt); return 0;
	}
	lrt[jb] = rt;
	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
	if (trace) fprintf(stderr, "llvm: %d..%d compiled\n", p, i - 1);
	return (Blk)(size_t)addr;
}
#endif

/* ---- o3 regions without the arm64 jit: compiled by llvm, or run here ---- */
#if !defined(ZYLLVM) && !defined(ZYJIT_X64)
static VM_TLS unsigned char *rcov;	/* 1 on the words of a lifted region */
#endif
static VM_TLS Cell rvals[MAXUOP * 4 + 16];
#define RPOOL 40
static VM_TLS Cell rpool[RPOOL][MAXUOP * 4 + 16];
static VM_TLS int rpool_n;
static int rdrop(int a)		/* a store landed on region code: drop those regions */
{
	int j, hit = 0;
	for (j = 0; j < nrg; j++)
		if (rgs[j]->lifted && rgs[j]->lo <= a && a < rgs[j]->hi) {
			if (trace) fprintf(stderr, "o3: dropped %d..%d (a store to %d: %d)\n", rgs[j]->lo, rgs[j]->hi - 1, a, m[a]);
			rclear(rgs[j]); hit = 1;
		}
	return hit;
}
#ifdef ZYLLVM
static int lhook(Cell a, Cell v);
#define RSTORE(a, b) do { if (inmem(a)) { if (lcov[(int)(a)]) lhook(a, b); else m[a] = (Cell)(b); } } while (0)
#elif defined(ZYJIT_X64)
/* a region store may land on translated code too: the jit's hook drops those. the gate word is a plain store, as in the region interpreter anywhere */
#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { if (cov[a] && (a) != SYSGATE) hook(a, b); else m[a] = (b); } } while (0)
#else
#define RSTORE(a, b) do { if ((unsigned)(a) < MW) { m[a] = b; if (rcov[a]) rdrop(a); } } while (0)
#endif
/* the region interpreter: ssa values in an array, vars in locals */
static int rexec(Rg *R, int e, int *ps, int *pr)
{
	Cell var[RMAXV], nv2[RMAXV], *v, t = 0, npend[16];
	int s = *ps, r = *pr, b = R->entb[e], fp, j, i, o, pooled = 0, mine = 0;
	if (rpool_n < RPOOL) { v = rpool[rpool_n++]; pooled = 1; }
	else { v = malloc((MAXUOP * 4 + 16) * sizeof(Cell)); if (v) mine = 1; else v = rvals; }
	var[0] = fp = mread(ZFP);
	for (j = 1; j < R->nv; j++) var[j] = mread(fp + R->slot[j]);
	for (;;) {
  if (context_cancelled()) { t = PC_HALT; break; }
		RB *B = &R->b[b];
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			switch (u->op) {
			case U_LI: v[u->d] = u->imm; break;
			case U_VGET: v[u->d] = var[u->imm]; break;
			case U_SLD: v[u->d] = mread(s + u->imm); break;
			case U_LD: v[u->d] = mread(v[u->a]); break;
			case U_ST: RSTORE(v[u->a], v[u->b]); break;
			case U_NARG: npend[u->imm] = v[u->a]; break;
			case U_NCALL:
#ifdef ZYFFI
				cb_s = s; cb_r = r;
				v[u->d] = ffi_invoke(ncd[u->imm].f, ncd[u->imm].sig, npend, 0);
#endif
				break;
			default:
				if (u->op >= U_LD8 && u->op <= U_ST32) {
					if (u->op == U_LD8) v[u->d] = bld8(v[u->a]);
					else if (u->op == U_LD16) v[u->d] = bld16(v[u->a]);
					else if (u->op == U_LD32) v[u->d] = bld32(v[u->a]);
					else if (u->op == U_ST8) bst8(v[u->a], v[u->b]);
					else if (u->op == U_ST16) bst16(v[u->a], v[u->b]);
					else if (u->op == U_ST32) bst32(v[u->a], v[u->b]);
					break;
				}
				v[u->d] = ev(u->op, v[u->a], v[u->b], v[u->c]);
			}
		}
		for (j = 0; j < B->nso; j++) { o = s + B->so[2 * j]; if ((unsigned)o < MW) m[o] = v[B->so[2 * j + 1]]; }
		s += B->delta;
		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) nv2[j] = v[B->vs[j]];
		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) var[j] = nv2[j];
		if (B->term == T_BR) { b = B->s1; continue; }
		if (B->term == T_CBR) {
			if (!v[B->cv]) { if (B->s1 >= 0) { b = B->s1; continue; } t = B->x1; }
			else { if (B->s2 >= 0) { b = B->s2; continue; } t = B->x2; }
		} 		else if (B->term == T_JMP) t = v[B->cv];
		else if (B->term == T_CALL) {
			Cell cs, cr;
			for (j = 0; j < R->np; j++) if ((unsigned)(fp + R->ps[j]) < MW) m[fp + R->ps[j]] = var[R->pv[j]];
			cs = s; cr = r;
			o3_invoke(&cs, &cr, v[B->cv], B->x2);
			s = (int)cs; r = (int)cr;
			if (halt_at >= 0) { t = halt_at; break; }
			var[0] = fp = mread(ZFP);
			for (j = 1; j < R->nv; j++) var[j] = mread(fp + R->slot[j]);
			if ((unsigned)B->x2 < (unsigned)code_words && o3ent[B->x2] == R->idx + 1) { b = R->entb[o3ei[B->x2]]; continue; }
			t = B->x2;
			break;
		}
		else t = m[--r];
		break;
	}
	for (j = 0; j < R->np; j++) if ((unsigned)(fp + R->ps[j]) < MW) m[fp + R->ps[j]] = var[R->pv[j]];
	*ps = s; *pr = r;
	if (pooled) rpool_n--;
	if (mine) free(v);
	return (int)t;
}
#ifndef ZYLLVM
#ifndef ZYJIT_X64
static void be_drop(int j) { (void)j; }
#endif
#ifdef ZYJIT_X64
/* a region entry is never chained to a translation: drop any that started there.
   the jit's coverage marks the words too, so a store from translated code to
   the region's own code reaches the hook, which drops the region */
static void rmark(Rg *R)
{
	int e;
	R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(cov, R->lo, R->hi);
	for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = 0;
}
static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(cov, R->lo, R->hi); }
#else
static void rforget(Rg *R) { (void)R; }
static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); }
static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); }
static void rcompile(Rg *R, int eager) { (void)R; (void)eager; }
#endif
#endif
#ifdef ZYLLVM

/* ---- o3 regions in llvm: one function per region, vars in allocas (so
   mem2reg makes them ssa across blocks), each block's micro-ops as ir, then
   the O3 pipeline. f(mem, regs, entry) returns the next vm address. */
typedef Cell (*RFn)(Cell *mem, Cell *regs, int ent);
static VM_TLS int nregions;
static void rforget(Rg *R) { rt_queue(R->rt); R->rt = 0; R->nat = 0; R->dnat = 0; }
static void rmark(Rg *R) { R->marked = 1; cinc(rcov, R->lo, R->hi); cinc(lcov, R->lo, R->hi); }
static void runmark(Rg *R) { cdec(rcov, R->lo, R->hi); cdec(lcov, R->lo, R->hi); }
static LLVMValueRef as_f64(LLVMValueRef v, int fp) { return fp ? v : LLVMBuildBitCast(bd, v, f64t, ""); }
static LLVMValueRef as_i64(LLVMValueRef v, int fp) { return fp ? LLVMBuildBitCast(bd, v, i64t, "") : v; }
/* the variables go back to their frame slots. before a call from inside the
   region only the ones that may hold a pointer need to (the rest are never
   looked at again by anything outside: the region goes on with its own
   copies), and the frame pointer word is not touched */
static void rspill_vars(Rg *R, int ptr_only)
{
	LLVMValueRef fp = LLVMBuildLoad2(bd, i64t, rvv[0], "");
	int j;
	for (j = 0; j < R->np; j++) {
		int v = R->pv[j];
		if (ptr_only && R->sc[v]) continue;
		LLVMValueRef bits = R->fv[v] ? LLVMBuildBitCast(bd, LLVMBuildLoad2(bd, f64t, rvv[v], ""), i64t, "") : LLVMBuildLoad2(bd, i64t, rvv[v], "");
		st(add(fp, R->ps[j]), bits);
	}
}
static void rspill_ir(Rg *R)
{
	st(Kw(ZFP), LLVMBuildLoad2(bd, i64t, rvv[0], ""));
	rspill_vars(R, 0);
}
static LLVMBasicBlockRef rexit_bb(Rg *R, int t)
{
	LLVMBasicBlockRef here = LLVMGetInsertBlock(bd), x = newbb();
	LLVMPositionBuilderAtEnd(bd, x);
	if (R->fn >= 0 && t >= R->hi) leave_ok(Kw(t));
	else { rspill_ir(R); leave(Kw(t)); }
	LLVMPositionBuilderAtEnd(bd, here);
	return x;
}
/* m[a] = v, through lhook if some compiled code covers the word */
static void rstore_ir(LLVMValueRef a, LLVMValueRef b)
{
	LLVMBasicBlockRef in = newbb(), cv = newbb(), plain = newbb(), out = newbb();
	LLVMValueRef v, args[2]; LLVMTypeRef at[2];
	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntULT, a, Kw(MW), ""), in, out);
	LLVMPositionBuilderAtEnd(bd, in);
	v = LLVMBuildLoad2(bd, i8t, LLVMBuildGEP2(bd, i8t, cptr(lcov), &a, 1, ""), "");
	LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntNE, v, LLVMConstInt(i8t, 0, 0), ""), cv, plain);
	LLVMPositionBuilderAtEnd(bd, plain); st(a, b); LLVMBuildBr(bd, out);
	LLVMPositionBuilderAtEnd(bd, cv);
	at[0] = i64t; at[1] = i64t; args[0] = a; args[1] = b;
	callp((void *)lhook, i32t, at, 2, args);
	LLVMBuildBr(bd, out);
	LLVMPositionBuilderAtEnd(bd, out);
}
#ifdef ZYFFI
/* a native call as ir. the callee and its signature are known, so llvm sees a
   call with the real argument types and lays out registers and stack by the
   platform's rules itself (no limit on the number of arguments). pointer
   kinds are converted from vm addresses to host addresses inline. the
   common libm functions on doubles become llvm's own intrinsics, which the
   backend can turn into one instruction (sqrt) or fold. ai and af are each
   argument as an i64 and as a double. s0 and rcur are the stacks as of the
   start of the block, for callbacks. */
static LLVMValueRef lptr_in(LLVMValueRef v, int kind)
{
	LLVMValueRef mb = Kw((Cell)(uintptr_t)m), host, inr, nz = LLVMBuildICmp(bd, LLVMIntNE, v, Kw(0), "");
	if (kind == 3) {
		inr = LLVMBuildICmp(bd, LLVMIntULT, v, Kw((Cell)MW * 8), "");
		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, v, ""), v, "");
	} else if (kind == 6) {
		inr = LLVMBuildICmp(bd, LLVMIntULT, v, Kw(MW), "");
		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, LLVMBuildShl(bd, v, Kw(3), ""), ""), v, "");
	} else {
		LLVMValueRef v1 = LLVMBuildAdd(bd, v, Kw(1), "");
		inr = LLVMBuildICmp(bd, LLVMIntULT, v1, Kw(MW), "");
		host = LLVMBuildSelect(bd, inr, LLVMBuildAdd(bd, mb, LLVMBuildShl(bd, v1, Kw(3), ""), ""), v, "");
	}
	return LLVMBuildSelect(bd, nz, host, Kw(0), "");
}
static LLVMValueRef lptr_out(LLVMValueRef p, int kind)
{
	LLVMValueRef mb = Kw((Cell)(uintptr_t)m), off = LLVMBuildSub(bd, p, mb, "");
	LLVMValueRef inr = LLVMBuildICmp(bd, LLVMIntULT, off, Kw((Cell)MW * 8), "");
	if (kind == 6) off = LLVMBuildLShr(bd, off, Kw(3), "");
	return LLVMBuildSelect(bd, inr, off, p, "");
}
static void lext_attr(LLVMValueRef call, unsigned idx, int sign)
{
	const char *nm = sign ? "signext" : "zeroext";
	unsigned kid = LLVMGetEnumAttributeKindForName(nm, strlen(nm));
	LLVMAddCallSiteAttribute(call, idx, LLVMCreateEnumAttribute(ctx, kid, 0));
}
static LLVMValueRef lncall(int di, LLVMValueRef *ai, LLVMValueRef *af, LLVMValueRef s0, LLVMValueRef rcur)
{
	Ncd *d = &ncd[di];
	LLVMTypeRef pt[15], rt, ft, f32t = LLVMFloatTypeInContext(ctx), i16t = LLVMInt16TypeInContext(ctx);
	LLVMValueRef av[15], call, res;
	int i, k, rk = (int)(d->sig & 15), allf = rk == 2, na = d->n;
	LLVMBuildStore(bd, s0, cptr(&cb_s));
	LLVMBuildStore(bd, rcur, cptr(&cb_r));
	for (i = 0; i < na; i++) {
		k = (int)((d->sig >> (4 * (i + 1))) & 15);
		if (k != 2) allf = 0;
		switch (k) {
		case 2: pt[i] = f64t; av[i] = af[i]; break;
		case 5: pt[i] = f32t; av[i] = LLVMBuildFPTrunc(bd, af[i], f32t, ""); break;
		case 4: case 7: pt[i] = i32t; av[i] = LLVMBuildTrunc(bd, ai[i], i32t, ""); break;
		case 8: case 9: pt[i] = i16t; av[i] = LLVMBuildTrunc(bd, ai[i], i16t, ""); break;
		case 10: case 11: pt[i] = i8t; av[i] = LLVMBuildTrunc(bd, ai[i], i8t, ""); break;
		case 3: case 6: case 12: pt[i] = i64t; av[i] = lptr_in(ai[i], k); break;
		default: pt[i] = i64t; av[i] = ai[i]; break;
		}
	}
	rt = rk == 0 ? LLVMVoidTypeInContext(ctx) : rk == 2 ? f64t : rk == 5 ? f32t : (rk == 4 || rk == 7) ? i32t : (rk == 8 || rk == 9) ? i16t : (rk == 10 || rk == 11) ? i8t : i64t;
	if (allf && na >= 1 && na <= 3) {	/* libm on doubles: the intrinsic */
		static const struct { const char *c, *ll; int n; } lm[] = {
			{ "sqrt", "llvm.sqrt.f64", 1 }, { "fabs", "llvm.fabs.f64", 1 }, { "floor", "llvm.floor.f64", 1 },
			{ "ceil", "llvm.ceil.f64", 1 }, { "trunc", "llvm.trunc.f64", 1 }, { "round", "llvm.round.f64", 1 },
			{ "sin", "llvm.sin.f64", 1 }, { "cos", "llvm.cos.f64", 1 }, { "exp", "llvm.exp.f64", 1 },
			{ "exp2", "llvm.exp2.f64", 1 }, { "log", "llvm.log.f64", 1 }, { "log2", "llvm.log2.f64", 1 },
			{ "log10", "llvm.log10.f64", 1 }, { "pow", "llvm.pow.f64", 2 }, { "copysign", "llvm.copysign.f64", 2 },
			{ "fma", "llvm.fma.f64", 3 } };
		for (i = 0; i < (int)(sizeof lm / sizeof lm[0]); i++)
			if (lm[i].n == na && !strcmp(lm[i].c, d->name)) {
				LLVMTypeRef it = LLVMFunctionType(f64t, pt, (unsigned)na, 0);
				LLVMValueRef fn = LLVMGetNamedFunction(mod_, lm[i].ll);
				if (!fn) fn = LLVMAddFunction(mod_, lm[i].ll, it);
				return LLVMBuildBitCast(bd, LLVMBuildCall2(bd, it, fn, av, (unsigned)na, ""), i64t, "");
			}
	}
#if defined(__x86_64__)
	ft = LLVMFunctionType(rt, pt, (unsigned)na, 1);	/* a vararg type makes llvm set al, which a variadic callee reads */
#else
	{
		int fixed = ffi_varargs(d->f);
		if (fixed >= 0 && fixed <= na) {
			for (i = fixed; i < na; i++) {
				if (pt[i] == f32t) { av[i] = LLVMBuildFPExt(bd, av[i], f64t, ""); pt[i] = f64t; }
				else if (pt[i] == i8t || pt[i] == i16t) {
					int kind = (int)((d->sig >> (4 * (i + 1))) & 15);
					av[i] = kind == 9 || kind == 11 ? LLVMBuildZExt(bd, av[i], i32t, "") : LLVMBuildSExt(bd, av[i], i32t, ""); pt[i] = i32t;
				}
			}
			ft = LLVMFunctionType(rt, pt, (unsigned)fixed, 1);
		} else ft = LLVMFunctionType(rt, pt, (unsigned)na, 0);
	}
#endif
	call = LLVMBuildCall2(bd, ft, cptr(d->f), av, (unsigned)na, "");
	for (i = 0; i < na; i++) {
		k = (int)((d->sig >> (4 * (i + 1))) & 15);
		if (k == 4 || k == 8 || k == 10) lext_attr(call, (unsigned)i + 1, 1);
		if (k == 7 || k == 9 || k == 11) lext_attr(call, (unsigned)i + 1, 0);
	}
	switch (rk) {
	case 0: res = Kw(0); break;
	case 2: res = LLVMBuildBitCast(bd, call, i64t, ""); break;
	case 5: res = LLVMBuildBitCast(bd, LLVMBuildFPExt(bd, call, f64t, ""), i64t, ""); break;
	case 4: res = LLVMBuildSExt(bd, call, i64t, ""); break;
	case 8: case 10: res = LLVMBuildSExt(bd, call, i64t, ""); break;
	case 7: case 9: case 11: res = LLVMBuildZExt(bd, call, i64t, ""); break;
	case 3: case 6: res = lptr_out(call, rk); break;
	default: res = call; break;
	}
	return res;
}
#endif
static VM_TLS LLVMValueRef l_dfn;	/* the register-passing function being built or declared for this region */
static int has_ret(Rg *R)
{
	int b;
	for (b = 0; b < R->nb; b++) if (R->b[b].term == T_RET) return 1;
	return 0;
}
/* an o3 call to a register-passing callee is inlined: the callee's body is built again into
   the caller's module as an internal function, so llvm can inline it there. the copy is good
   for one generation of the callee (rclear bumps gen whenever the callee's code is dropped), and
   the caller checks that generation before the fast path, so a stale copy is never entered. */
#define INL_MAX 8		/* copies in one module */
#define INL_BLOCKS 24		/* largest callee copied */
typedef struct { Rg *C; unsigned gen; LLVMValueRef fn; } InlEnt;
static VM_TLS InlEnt inl_tab[INL_MAX];
static VM_TLS int ninl, inl_depth;
static void lbody(Rg *R, int direct, LLVMValueRef fnv);
static LLVMValueRef inline_fn(Rg *C, unsigned *gen)
{
	static VM_TLS int off = -1, inl_n;
	LLVMValueRef sfn, sdfn, svm, svregs, svs, svr, nf, srv[RMAXV];
	LLVMBasicBlockRef at;
	LLVMTypeRef dpt[6];
	char nm[24];
	int i, j, sdm;
	if (off < 0) off = getenv("ZYNOINLINE") != 0;
	if (off || inl_depth || !C->lifted || !C->dnat || C->nb > INL_BLOCKS || has_ret(C)) return 0;
	for (i = 0; i < ninl; i++) if (inl_tab[i].C == C && inl_tab[i].gen == C->gen) { *gen = C->gen; return inl_tab[i].fn; }
	if (ninl >= INL_MAX) return 0;
	at = LLVMGetInsertBlock(bd); sfn = fn_; sdfn = l_dfn; sdm = l_dm;
	svm = vm_; svregs = vregs; svs = vs; svr = vr;
	memcpy(srv, rvv, sizeof rvv);
	for (j = 0; j < 6; j++) dpt[j] = i64t;
	sprintf(nm, "inl_%d", inl_n++);
	nf = LLVMAddFunction(mod_, nm, LLVMFunctionType(l_dret, dpt, 6, 0));
	LLVMSetLinkage(nf, LLVMInternalLinkage);
	l_dfn = nf;
	inl_depth++;
	lbody(C, 1, nf);
	inl_depth--;
	fn_ = sfn; l_dfn = sdfn; l_dm = sdm;
	vm_ = svm; vregs = svregs; vs = svs; vr = svr;
	memcpy(rvv, srv, sizeof rvv);
	LLVMPositionBuilderAtEnd(bd, at);
	inl_tab[ninl].C = C; inl_tab[ninl].gen = C->gen; inl_tab[ninl].fn = nf; ninl++;
	*gen = C->gen;
	return nf;
}
/* one function for the region. the plain one is f(mem, regs, entry) and takes
   the vm's own protocol: s and r in regs, variables from the frame. the
   register-passing one is fd(s, r, fp, a0, a1, a2), entered only at the top of
   a function by another region: the arguments are the variables, the frame
   pointer is a value, and it returns the next address and s. */
/* the callee of a call to a known address, if it is a function region that takes its
   arguments in registers */
static Rg *dcallee(LLVMValueRef tgt)
{
	Rg *C;
	if (!LLVMIsAConstantInt(tgt)) return 0;
	C = fnlookup((int)LLVMConstIntGetZExtValue(tgt));
	if (!C || C->dead || C->fn < 0 || C->argc > 3 || C->fslots < 0) return 0;
	return C;
}
static void lbody(Rg *R, int direct, LLVMValueRef fnv)
{
	LLVMTypeRef pt[3], ft; LLVMValueRef sw = 0, *val, s0, fp, r, ent = 0, rg2;
	LLVMBasicBlockRef *bb, *ebs, eb, dflt = 0;
	int b, i, j, e;
	LLVMValueRef npi[16], npf[16], dev[4];
	int deo[4], nde = 0;
	Rg *Cp = 0;
	(void)pt; (void)ft;
	fn_ = fnv;
	l_dm = direct;
	LLVMPositionBuilderAtEnd(bd, newbb());
	vs = LLVMBuildAlloca(bd, i64t, "s"); vr = LLVMBuildAlloca(bd, i64t, "r");
	rg2 = LLVMBuildAlloca(bd, LLVMArrayType(i64t, 2), "regs");
	for (j = 0; j < R->nv; j++) rvv[j] = LLVMBuildAlloca(bd, j && R->fv[j] ? f64t : i64t, "");
	bb = malloc(R->nb * sizeof *bb);
	ebs = calloc(R->ne + 1, sizeof *ebs);
	for (b = 0; b < R->nb; b++) bb[b] = newbb();
	if (direct) {
		vm_ = cptr(m); vregs = 0;
		LLVMBuildStore(bd, LLVMGetParam(fn_, 0), vs);
		LLVMBuildStore(bd, LLVMGetParam(fn_, 1), vr);
		LLVMBuildStore(bd, LLVMGetParam(fn_, 2), rvv[0]);
		for (j = 1; j < R->nv; j++) {
			LLVMValueRef bits = Kw(0);
			if (R->slot[j] >= 0 && R->slot[j] < R->argc && R->slot[j] < 3) bits = LLVMGetParam(fn_, 3 + R->slot[j]);
			if (R->fv[j]) bits = LLVMBuildBitCast(bd, bits, f64t, "");
			LLVMBuildStore(bd, bits, rvv[j]);
		}
		LLVMBuildBr(bd, bb[R->entb[0]]);
	} else {
		vm_ = LLVMGetParam(fn_, 0); vregs = LLVMGetParam(fn_, 1); ent = LLVMGetParam(fn_, 2);
		{ LLVMValueRef r0 = Kw(0), r1 = Kw(1);
		  set_stack_ir(LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r0, 1, ""), ""));
		  LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, vregs, &r1, 1, ""), ""), vr); }
		dflt = newbb();
		sw = LLVMBuildSwitch(bd, ent, dflt, (unsigned)R->ne);
	}
	for (e = direct ? 1 : 0; e < R->ne; e++) {
		eb = ebs[e] = newbb();
		if (!direct) LLVMAddCase(sw, K32(e), eb);
		LLVMPositionBuilderAtEnd(bd, eb);
		fp = ld(Kw(ZFP));
		LLVMBuildStore(bd, fp, rvv[0]);
		for (j = 1; j < R->nv; j++) {
			LLVMValueRef bits = safeld(add(fp, R->slot[j]));
			if (R->fv[j]) bits = LLVMBuildBitCast(bd, bits, f64t, "");
			LLVMBuildStore(bd, bits, rvv[j]);
		}
		LLVMBuildBr(bd, bb[R->entb[e]]);
	}
	if (!direct) { LLVMPositionBuilderAtEnd(bd, dflt); leave(Kw(-1)); }
	val = calloc(R->maxnv + 1, sizeof *val);
	for (b = 0; b < R->nb; b++) {
		RB *B = &R->b[b];
		unsigned char *fpv = calloc(B->nv + 1, 1);
		class_fp(R, B, fpv);
		LLVMPositionBuilderAtEnd(bd, bb[b]);
		s0 = stack_ir();
		for (i = 0; i < B->n; i++) {
			Uop *u = &B->u[i];
			int na = nargs(u->op);
			LLVMValueRef a = na > 0 ? val[u->a] : 0, x = na > 1 ? val[u->b] : 0, c = na > 2 ? val[u->c] : 0;
			LLVMValueRef ai = na > 0 ? as_i64(a, fpv[u->a]) : 0, af = na > 0 ? as_f64(a, fpv[u->a]) : 0;
			LLVMValueRef xi = na > 1 ? as_i64(x, fpv[u->b]) : 0, xf = na > 1 ? as_f64(x, fpv[u->b]) : 0;
			LLVMValueRef ci = na > 2 ? as_i64(c, fpv[u->c]) : 0;
			switch (u->op) {
			case U_LI: val[u->d] = Kw(u->imm); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
			case U_VGET:
				val[u->d] = LLVMBuildLoad2(bd, (unsigned)u->imm < RMAXV && R->fv[u->imm] ? f64t : i64t, rvv[u->imm], "");
				break;
			case U_SLD: val[u->d] = ld(add(s0, (int)u->imm)); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
			case U_LD: val[u->d] = safeld(ai); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
			case U_ST: rstore_ir(ai, xi); break;
			case U_NARG: npi[u->imm] = ai; npf[u->imm] = af; break;
			case U_NCALL:
#ifdef ZYFFI
				val[u->d] = lncall((int)u->imm, npi, npf, s0, LLVMBuildLoad2(bd, i64t, vr, ""));
				if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, "");
#endif
				break;
			case U_LD8: case U_LD16: case U_LD32: val[u->d] = bload_ir(ai, bwidth(u->op)); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
			case U_ST8: case U_ST16: case U_ST32: bstore_ir(ai, xi, bwidth(u->op)); break;
			case U_FADD: val[u->d] = LLVMBuildFAdd(bd, af, xf, ""); break;
			case U_FSUB: val[u->d] = LLVMBuildFSub(bd, af, xf, ""); break;
			case U_FMUL: val[u->d] = LLVMBuildFMul(bd, af, xf, ""); break;
			case U_FDIV: val[u->d] = LLVMBuildFDiv(bd, af, xf, ""); break;
			case U_FEQ: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOEQ, af, xf, ""), i64t, ""); break;
			case U_FLT: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLT, af, xf, ""), i64t, ""); break;
			case U_FLE: val[u->d] = LLVMBuildZExt(bd, LLVMBuildFCmp(bd, LLVMRealOLE, af, xf, ""), i64t, ""); break;
			case U_I2F: val[u->d] = LLVMBuildSIToFP(bd, ai, f64t, ""); break;
			case U_F2I: val[u->d] = LLVMBuildFPToSI(bd, af, i64t, ""); break;
			default: val[u->d] = ir_op(u->op, ai, xi, ci); if (fpv[u->d]) val[u->d] = LLVMBuildBitCast(bd, val[u->d], f64t, ""); break;
			}
		}
		Cp = B->term == T_CALL && B->cv >= 0 ? dcallee(as_i64(val[B->cv], fpv[B->cv])) : 0;
		nde = 0;
		for (j = 0; j < B->nso; j++) {
			int off = B->so[2 * j];
			LLVMValueRef sv_ = as_i64(val[B->so[2 * j + 1]], fpv[B->so[2 * j + 1]]);
			if (Cp && off >= B->delta && off < B->delta + Cp->argc) { deo[nde] = off; dev[nde++] = sv_; continue; }	/* the register call does not need them in memory */
			st(add(s0, off), sv_);
		}
		set_stack_ir(add(s0, B->delta));
		for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0) {
			LLVMValueRef bits = R->fv[j] ? as_f64(val[B->vs[j]], fpv[B->vs[j]]) : as_i64(val[B->vs[j]], fpv[B->vs[j]]);
			LLVMBuildStore(bd, bits, rvv[j]);
		}
		switch (B->term) {
		case T_BR: LLVMBuildBr(bd, bb[B->s1]); break;
		case T_CBR:
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, as_i64(val[B->cv], fpv[B->cv]), Kw(0), ""),
				B->s1 >= 0 ? bb[B->s1] : rexit_bb(R, B->x1), B->s2 >= 0 ? bb[B->s2] : rexit_bb(R, B->x2));
			break;
		case T_CALL: {
			LLVMTypeRef at[4]; LLVMValueRef args[4]; LLVMBasicBlockRef bad, okb, slowb, fastb, helpb, finb, eqb, doneb, chk; int ee;
			LLVMValueRef tgt = as_i64(val[B->cv], fpv[B->cv]), natp = 0, fpn = 0, sv = 0, ret = 0;
			Rg *C = 0;
			rspill_vars(R, 1);
			for (ee = 0; ee < R->ne; ee++) if (R->ent[ee] == B->x2) break;
			if (LLVMIsAConstantInt(tgt)) {
				C = fnlookup((int)LLVMConstIntGetZExtValue(tgt));
				if (C && (C->dead || C->argc > 12 || C->fslots < 0)) C = 0;
			}
			slowb = newbb(); okb = newbb(); chk = newbb();
			if (C && C->fn >= 0 && C->argc <= 3) {
				/* a callee that is a function region: its arguments go in registers, its frame is a value,
				   and it comes back with the next address and s. the callee may not be compiled yet, or the
				   frame or the stack may be about to run out: the helper takes those */
				LLVMBasicBlockRef dfast = newbb(), dnorm = newbb(), dabn = newbb(), deq = newbb(), dcont = newbb();
				LLVMValueRef dn, cond, fpc, a3[3], da[6], res, rret, rs2, okf, tmp0, tmp1;
				LLVMTypeRef dt = LLVMFunctionType(l_dret, (LLVMTypeRef[]){ i64t, i64t, i64t, i64t, i64t, i64t }, 6, 0);
				int ai2;
				unsigned igen = 0;
				LLVMValueRef inl = C == R ? 0 : inline_fn(C, &igen);
				if (C == R) dn = l_dfn;
				else if (inl) dn = inl;
				else dn = LLVMBuildLoad2(bd, ptrt, cptr(&C->dnat), "");
				fpc = LLVMBuildLoad2(bd, i64t, rvv[0], "");
				fpn = LLVMBuildSub(bd, fpc, Kw(C->fslots), "");
				sv = stack_ir();
				cond = LLVMBuildICmp(bd, LLVMIntUGE, fpn,
					LLVMBuildAdd(bd, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_floor_word), "")), ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_guard_word), "")), ""), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, fpn, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_top_word), "")), ""), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntUGE, add(sv, -16), Kw(DSLIM), ""), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, sv, Kw(DSHI), ""), "");
				if (inl) cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntEQ, LLVMBuildLoad2(bd, i32t, cptr(&C->gen), ""), K32((int)igen), ""), "");
				else if (C != R) cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntNE, LLVMBuildPtrToInt(bd, dn, i64t, ""), Kw(0), ""), "");
				LLVMBuildCondBr(bd, cond, dfast, slowb);
				LLVMPositionBuilderAtEnd(bd, dfast);
				for (ai2 = 0; ai2 < 3; ai2++) {
					int off = B->delta + C->argc - 1 - ai2, q;
					a3[ai2] = Kw(0);
					if (ai2 >= C->argc) continue;
					a3[ai2] = 0;
					for (q = 0; q < nde; q++) if (deo[q] == off) a3[ai2] = dev[q];
					if (!a3[ai2]) a3[ai2] = ld(add(sv, C->argc - 1 - ai2));
				}
				da[0] = add(sv, C->argc); da[1] = LLVMBuildLoad2(bd, i64t, vr, "");
				da[2] = fpn; da[3] = a3[0]; da[4] = a3[1]; da[5] = a3[2];
				res = LLVMBuildCall2(bd, dt, dn, da, 6, "");
				rret = LLVMBuildExtractValue(bd, res, 0, "");
				rs2 = LLVMBuildExtractValue(bd, res, 1, "");
				okf = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntUGE, rret, Kw(C->hi), ""), LLVMBuildICmp(bd, LLVMIntULT, rret, Kw(MW), ""), "");
				LLVMBuildCondBr(bd, okf, dnorm, dabn);
				LLVMPositionBuilderAtEnd(bd, dnorm);
				set_stack_ir(rs2);
				LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, rret, Kw(C->hi), ""), deq, dcont);
				LLVMPositionBuilderAtEnd(bd, deq);
				{ LLVMValueRef sn = add(stack_ir(), -1); st(sn, Kw(0)); set_stack_ir(sn); }
				LLVMBuildBr(bd, dcont);
				LLVMPositionBuilderAtEnd(bd, dcont);
				if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
				LLVMPositionBuilderAtEnd(bd, dabn);
				tmp0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, "");
				tmp1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
				LLVMBuildStore(bd, rs2, tmp0);
				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, cptr(&d_r), ""), tmp1);
				at[0] = ptrt; at[1] = ptrt; at[2] = ptrt; at[3] = i64t;
				args[0] = tmp0; args[1] = tmp1; args[2] = cptr(C); args[3] = rret;
				callp((void *)o3_finish_c, LLVMVoidTypeInContext(ctx), at, 4, args);
				set_stack_ir(LLVMBuildLoad2(bd, i64t, tmp0, ""));
				LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, tmp1, ""), vr);
				LLVMBuildBr(bd, chk);
				(void)fastb; (void)helpb; (void)finb; (void)eqb; (void)doneb; (void)natp; (void)ret;
			} else if (C) {
				/* a callee that is already compiled is entered from here: a frame below this one,
				   the arguments copied across, a call through its entry. the other cases (not
				   compiled yet, a frame outside memory) take the helper */
				LLVMValueRef cond, fz;
				fastb = newbb(); helpb = newbb(); finb = newbb(); eqb = newbb(); doneb = newbb();
				natp = LLVMBuildLoad2(bd, ptrt, cptr(&C->nat), "");
				fz = LLVMBuildLoad2(bd, i64t, rvv[0], "");
				fpn = LLVMBuildSub(bd, fz, Kw(C->fslots), "");
				sv = stack_ir();
				cond = LLVMBuildICmp(bd, LLVMIntNE, LLVMBuildPtrToInt(bd, natp, i64t, ""), Kw(0), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntUGE, fpn,
					LLVMBuildAdd(bd, ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_floor_word), "")), ld(LLVMBuildLoad2(bd, i64t, cptr(&frame_guard_word), "")), ""), ""), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, fpn, Kw(MW - 64), ""), "");
				cond = LLVMBuildAnd(bd, cond, LLVMBuildICmp(bd, LLVMIntULT, add(sv, -1), Kw(MW - 64), ""), "");
				LLVMBuildCondBr(bd, cond, fastb, slowb);
				LLVMPositionBuilderAtEnd(bd, fastb);
				{
					LLVMTypeRef ct[3]; LLVMValueRef ca[3], r1, haltv, okf, sn, rv;
					int i2;
					st(Kw(ZFP), fpn);
					for (i2 = 0; i2 < C->argc; i2++) st(add(fpn, i2), ld(add(sv, C->argc - 1 - i2)));
					r1 = Kw(1);
					LLVMBuildStore(bd, add(sv, C->argc), rg2);
					LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), LLVMBuildGEP2(bd, i64t, rg2, &r1, 1, ""));
					ct[0] = ptrt; ct[1] = ptrt; ct[2] = i32t;
					ca[0] = vm_; ca[1] = rg2; ca[2] = K32(0);
					ret = LLVMBuildCall2(bd, LLVMFunctionType(i64t, ct, 3, 0), natp, ca, 3, "");
					set_stack_ir(LLVMBuildLoad2(bd, i64t, rg2, ""));
					LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, LLVMBuildGEP2(bd, i64t, rg2, &r1, 1, ""), ""), vr);
					haltv = LLVMBuildLoad2(bd, i64t, cptr(&halt_at), "");
					okf = LLVMBuildAnd(bd, LLVMBuildICmp(bd, LLVMIntUGE, ret, Kw(C->hi), ""), LLVMBuildICmp(bd, LLVMIntULT, ret, Kw(MW), ""), "");
					okf = LLVMBuildAnd(bd, okf, LLVMBuildICmp(bd, LLVMIntSLT, haltv, Kw(0), ""), "");
					LLVMBuildCondBr(bd, okf, finb, helpb);
					LLVMPositionBuilderAtEnd(bd, finb);
					LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntEQ, ret, Kw(C->hi), ""), eqb, doneb);
					LLVMPositionBuilderAtEnd(bd, eqb);
					sn = add(stack_ir(), -1); st(sn, Kw(0)); set_stack_ir(sn);
					LLVMBuildBr(bd, doneb);
					LLVMPositionBuilderAtEnd(bd, doneb);
					rv = ld(Kw(ZFP));
					st(Kw(ZFP), add(rv, C->fslots));
					if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
					LLVMPositionBuilderAtEnd(bd, helpb);
					{
						LLVMValueRef t0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, ""), t1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
						LLVMBuildStore(bd, stack_ir(), t0); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), t1);
						at[0] = ptrt; at[1] = ptrt; at[2] = ptrt; at[3] = i64t;
						args[0] = t0; args[1] = t1; args[2] = cptr(C); args[3] = ret;
						callp((void *)o3_finish_c, LLVMVoidTypeInContext(ctx), at, 4, args);
						set_stack_ir(LLVMBuildLoad2(bd, i64t, t0, "")); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, t1, ""), vr);
					}
					LLVMBuildBr(bd, chk);
				}
			} else LLVMBuildBr(bd, slowb);
			LLVMPositionBuilderAtEnd(bd, slowb);
			{ int q; for (q = 0; q < nde; q++) st(add(s0, deo[q]), dev[q]); }
			st(Kw(ZFP), LLVMBuildLoad2(bd, i64t, rvv[0], ""));
			{
				LLVMValueRef t0 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(0) }, 1, ""), t1 = LLVMBuildGEP2(bd, i64t, rg2, (LLVMValueRef[]){ Kw(1) }, 1, "");
				LLVMBuildStore(bd, stack_ir(), t0); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, vr, ""), t1);
				at[0] = ptrt; at[1] = ptrt; at[2] = i64t; at[3] = i64t;
				args[0] = t0; args[1] = t1;
				args[2] = tgt; args[3] = Kw(B->x2);
				callp((void *)o3_invoke, LLVMVoidTypeInContext(ctx), at, 4, args);
				set_stack_ir(LLVMBuildLoad2(bd, i64t, t0, "")); LLVMBuildStore(bd, LLVMBuildLoad2(bd, i64t, t1, ""), vr);
			}
			LLVMBuildBr(bd, chk);
			LLVMPositionBuilderAtEnd(bd, chk);
			bad = newbb();
			LLVMBuildCondBr(bd, LLVMBuildICmp(bd, LLVMIntSGE, LLVMBuildLoad2(bd, i64t, cptr(&halt_at), ""), Kw(0), ""), bad, okb);
			LLVMPositionBuilderAtEnd(bd, bad); leave(l_dm ? Kw(-1) : LLVMBuildLoad2(bd, i64t, cptr(&halt_at), ""));	/* a direct callee says -1: any address could look like a return */
			LLVMPositionBuilderAtEnd(bd, okb);
			if (ee < R->ne) LLVMBuildBr(bd, bb[R->entb[ee]]); else leave(Kw(B->x2));
			(void)fpn; (void)sv;
			break; }
		case T_RET:
			r = add(LLVMBuildLoad2(bd, i64t, vr, ""), -1); LLVMBuildStore(bd, r, vr);
			rspill_ir(R); leave(ld(r));
			break;
		default: {
			LLVMValueRef tv = as_i64(val[B->cv], fpv[B->cv]);
			if (R->fn >= 0 && LLVMIsAConstantInt(tv) && (long long)LLVMConstIntGetSExtValue(tv) >= R->hi) leave_ok(tv);
			else { rspill_ir(R); leave(tv); }
		}
		}
		free(fpv);
	}
	free(val); free(bb); free(ebs);
	l_dm = 0;
}
static void *lregion(Rg *R)
{
	char name[40], dname[48]; LLVMTypeRef pt[3], ft, dpt[6]; LLVMValueRef f1, f2 = 0;
	LLVMOrcExecutorAddress addr; LLVMErrorRef err;
	int dir, i;
	sprintf(name, "o3_%d_%d", R->lo, nregions++);
	sprintf(dname, "%s_d", name);
	mod_ = LLVMModuleCreateWithNameInContext(name, ctx);
	ninl = 0;
	lmodule_target();
	pt[0] = ptrt; pt[1] = ptrt; pt[2] = i32t;
	ft = LLVMFunctionType(i64t, pt, 3, 0);
	f1 = LLVMAddFunction(mod_, name, ft);
	lfunc_attrs(f1);
	dir = R->fn >= 0 && R->argc <= 3 && R->fslots >= 0 && !has_ret(R);
	if (dir) {
		static VM_TLS unsigned knu;
		LLVMTypeRef rt2[2];
		rt2[0] = i64t; rt2[1] = i64t;
		l_dret = LLVMStructTypeInContext(ctx, rt2, 2, 0);
		for (i = 0; i < 6; i++) dpt[i] = i64t;
		f2 = LLVMAddFunction(mod_, dname, LLVMFunctionType(l_dret, dpt, 6, 0));
		if (!knu) knu = LLVMGetEnumAttributeKindForName("nounwind", 8);
		LLVMAddAttributeAtIndex(f2, (LLVMAttributeIndex)-1, LLVMCreateEnumAttribute(ctx, knu, 0));
	}
	l_dfn = f2;
	lbody(R, 0, f1);
	if (dir) lbody(R, 1, f2);
	l_dfn = 0;
	lrun_passes("default<O3>");
	if (getenv("ZYLLIR")) LLVMDumpModule(mod_);
	{ LLVMOrcResourceTrackerRef rt = LLVMOrcJITDylibCreateResourceTracker(LLVMOrcLLJITGetMainJITDylib(jit));
	  if ((err = LLVMOrcLLJITAddLLVMIRModuleWithRT(jit, rt, LLVMOrcCreateNewThreadSafeModule(mod_, tsc)))) {
		char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); LLVMOrcReleaseResourceTracker(rt); return 0;
	  }
	  R->rt = rt; }
	if ((err = LLVMOrcLLJITLookup(jit, &addr, name))) { char *msg = LLVMGetErrorMessage(err); fprintf(stderr, "llvm: %s\n", msg); LLVMDisposeErrorMessage(msg); return 0; }
	if (dir) {
		LLVMOrcExecutorAddress da;
		if (!(err = LLVMOrcLLJITLookup(jit, &da, dname))) R->dnat = (void *)(size_t)da; else LLVMConsumeError(err);
	}
	if (trace) fprintf(stderr, "o3: %d..%d compiled by llvm%s\n", R->lo, R->hi - 1, R->dnat ? " (with a register entry)" : "");
	return (void *)(size_t)addr;
}
static void rcompile(Rg *R, int eager)
{
	(void)eager;
	if (R->nat || R->nonat || !llvm_ok) return;
	if (!(R->nat = lregion(R))) R->nonat = 1;
}
#endif
#ifdef ZYJIT_X64
/* ---- the x86-64 jit: block translation, and the loop that runs it ----
   the rules are the arm64 jit's: a block ends at call, jump, ret or halt; a jz
   does not end it (the not-taken side keeps compiling); a literal followed by
   call, jump or jz is a constant target; every exit goes through x_disp, which
   checks both stacks against their limits, looks the target up in tab and
   chains to it when it is already translated, and otherwise returns to c. the
   instruction selection differs: within a block the data stack pointer is not
   moved for every push and pop. vd counts the words pushed (negative) or
   popped since r13 was last written, stack slot k from the top is
   [r13 + 8 * (vd + k)], and r13 is brought up to date before every call into
   c and every exit. o3 regions are not compiled here: a region entry is never
   translated (the c loop sees it first), and the region interpreter runs it. */
static VM_TLS int vd;
static VM_TLS int x64_noo3;	/* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */
static VM_TLS int x64_noenh;	/* ZYNOENH=1: call every enhancement through xrun, as phase 1 did */
#define SL(k) (8 * (vd + (k)))
static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); }
static void stv(int k, int r) { mov_mr(rSP, SL(k), r); }
static void sp_keep(void) { if (vd) lea(rSP, rSP, -1, 1, 8 * vd); }	/* r13 up to date; vd stays for the other path */
static void exit_with(int v) { sp_keep(); mov_r32i(r_ax, (unsigned)v); jmp_to(x_exit); }
static void disp_imm(Cell t) { sp_keep(); mov_r32i(r_ax, (unsigned)(int)t); jmp_to(x_disp); }	/* a constant target: the low 32 bits, like the interpreter's int pc */
static void disp_ax(void) { sp_keep(); jmp_to(x_disp); }
static void publish_stacks(void)	/* s and r into S, as x_exit does, for hooks that call back into the vm */
{
	lea(r_ax, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 8, r_ax);
	mov_rr(r_ax, rRP); alu_rr(a_sub, r_ax, rMB); shift_ri(5, r_ax, 3); mov_mr(rST, 16, r_ax);
}
static void reload_stacks(void)	/* r13, r14, r15 from S after c may have moved the stacks or the table */
{
	mov_rm(r_cx, rST, 8); lea(rSP, rMB, r_cx, 8, 0);
	mov_rm(r_cx, rST, 16); lea(rRP, rMB, r_cx, 8, 0);
	mov_rm(rTAB, rST, 24);
}
/* sp = (r13 + 8 * vd - r12) / 8, the data stack index, into r */
static void sp_index(int r) { lea(r, rSP, -1, 1, 8 * vd); alu_rr(a_sub, r, rMB); shift_ri(5, r, 3); }

/* a store of rdx to the word rax. known: the address is known to be inside memory. on a
   covered word the hook runs and, if it dropped a translation, the block resumes in c at i + 1 */
static void emit_store(int i, int known)
{
	JW *done1 = 0, *slow, *done2, *z;
	if (!known) { alu_ri(a_cmp, r_ax, MW); done1 = j32(c_ae); }
	mov_rm(r_cx, rST, 32);
	mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0);		/* cmp byte [rcx + rax], 0 */
	slow = j32(c_ne);
	mov_xr(rMB, r_ax, 8, 0, r_dx);
	done2 = jm32();
	fix32(slow, cp);
	mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
	publish_stacks();
	call_m(rST, 64);
	reload_stacks();
	test32(r_ax, r_ax);
	z = j32(c_e);
	mov_r32i(r_ax, (unsigned)(i + 1)); jmp_to(x_exit);	/* a translation died: resume at i + 1 in c; r13 is exact */
	fix32(z, cp);
	if (vd) lea(rSP, rSP, -1, 1, -8 * vd);
	if (done1) fix32(done1, cp);
	fix32(done2, cp);
}
/* the word at rax, or 0 outside memory, into rcx */
static void emit_load(void)
{
	JW *over;
	xor32(r_cx);
	alu_ri(a_cmp, r_ax, MW);
	over = j8(c_ae);
	mov_rx(r_cx, rMB, r_ax, 8, 0);
	fix8(over, cp);
}
/* byte loads: zero extended, 0 unless the whole access lies in memory */
static void emit_ld_bytes(int n)
{
	JW *over;
	ldv(r_ax, 0);
	xor32(r_cx);
	mov_ri(r_dx, ((Cell)MW << 3) - (n - 1));
	alu_rr(a_cmp, r_ax, r_dx);
	over = j8(c_ae);
	if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0);
	else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0);
	else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0);
	fix8(over, cp);
	stv(0, r_cx);
}
/* c helper for byte stores: 1 if a translation died */
static long xbst(Cell a, Cell v, long n)
{
	int e = drop_epoch;
	b_store(a, v, (int)n);
	return drop_epoch != e;
}
static void float_op(int i, int k)
{
	JW *ok;
	if (k >= 601 && k <= 604) {
		static const unsigned char opc[] = { 0x58, 0x5c, 0x59, 0x5e };
		sd_m(0x10, 0, rSP, SL(1));
		sd_m(opc[k - 601], 0, rSP, SL(0));
		vd++;
		sd_m(0x11, 0, rSP, SL(0));
	} else if (k >= 605 && k <= 607) {
		xor32(r_ax);
		sd_m(0x10, 0, rSP, SL(0));
		mem_op(0x66, 0, 1, 0x2e, 0, rSP, -1, 1, SL(1));		/* ucomisd xmm0, [b]: a against b, unordered sets cf */
		if (k == 605) { setcc(c_e, r_ax); setcc(c_np, r_cx); reg_op(0, 0, 0, 0x20, r_cx, r_ax); }	/* eq and not unordered */
		else setcc(k == 606 ? c_a : c_ae, r_ax);			/* b < a is a > b; b <= a is a >= b */
		vd++;
		stv(0, r_ax);
	} else if (k == 608) {
		mem_op(0xf2, 1, 1, 0x2a, 0, rSP, -1, 1, SL(0));		/* cvtsi2sd xmm0, qword [sp] */
		sd_m(0x11, 0, rSP, SL(0));
	} else if (k == 609) {	/* truncate toward zero; the indefinite result goes to the c helper, which saturates and maps a nan to 0 */
		mem_op(0xf2, 1, 1, 0x2c, r_ax, rSP, -1, 1, SL(0));	/* cvttsd2si rax, [sp] */
		mov_ri(r_cx, (Cell)((UCell)1 << 63));
		alu_rr(a_cmp, r_ax, r_cx);
		ok = j8(c_ne);
		ldv(r_di, 0); call_abs((void *)fop_f2i);
		fix8(ok, cp);
		stv(0, r_ax);
	} else if (k == 610) {
		ldv(r_di, 0); vd++; call_abs((void *)fop_print);
	} else if (k == 618) {	/* (bits prec mode --) */
		ldv(r_di, 2); ldv(r_si, 1); ldv(r_dx, 0); vd += 3; call_abs((void *)fop_print_mode);
	} else {		/* 611 */
		ldv(r_di, 0); call_abs((void *)fop_parse_cell); stv(0, r_ax);
	}
	(void)i;
}

/* ---- enhancements, inlined ----
   an enhancement's optimized micro-ops are lowered to x86-64 instead of being
   run by xrun: values live in rsi, rdi, r8..r11 and rbp (regalloc_n with seven
   registers), rax, rcx and rdx are the scratch that division, shifts and setcc
   need, and constants are made where they are used. like the arm64 jit, a run
   of literals in front of the opcode specializes the enhancement on them
   (x64_spec_enh), so the optimizer sees them as constants. an enhancement with
   an op this lowering lacks (floats, byte loads and stores, native calls), one
   that does not fit the registers, or one too long still goes through xrun. */
static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp };
#define XNREG 7
#define XMAXOP 400
static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16];
static VM_TLS signed char *xr;
static int x64_op_ok(int op)
{
	switch (op) {
	case U_LI: case U_MOV: case U_ADD: case U_SUB: case U_MUL: case U_SDIV: case U_UDIV: case U_SREM: case U_UREM:
	case U_AND: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR: case U_NEG: case U_NOT:
	case U_LT: case U_LTU: case U_EQ: case U_NE: case U_MIN: case U_MAX: case U_MINU: case U_MAXU:
	case U_MULHS: case U_MULHU: case U_CLZ: case U_CTZ: case U_BSWAP: case U_ROTL: case U_ROTR:
	case U_ABS: case U_SEL: case U_LD: case U_ST:
		return 1;
	}
	return 0;
}
/* the enhancement in slot s, specialized on nk literals that feed its last inputs
   (nk may be 0) and given x86-64 registers. 0 if it can be inlined */
static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te)
{
	Enh *e = &enh[s];
	int nin = e->nin - nk, i, j, out[8], na;
	static VM_TLS int mp[MAXUOP * 4 + 16];
	if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1;
	for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1;
	un = 0; unv = nin; uerr = 0;
	for (j = 0; j < nin; j++) mp[j] = j;
	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
	for (i = 0; i < e->n; i++) {
		Uop *u = &e->u[i];
		na = nargs(u->op);
		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
		if (u->op != U_ST) mp[u->d] = j;
	}
	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
	memset(te, 0, sizeof *te);
	if (uerr || optimize(te, nin, out, e->nout)) return -1;
	for (i = 0; i < te->n; i++) if (!x64_op_ok(te->u[i].op)) { free(te->u); return -1; }
	regalloc_n(te, XNREG);
	if (!te->reg) { free(te->u); return -1; }
	return 0;
}
/* the register value v lives in, or scratch with the constant v loaded into it */
static VM_TLS int xvh[MAXUOP * 4 + 16];	/* regions: 1 + the var a value reads at block entry, 0 if none */
static VM_TLS Rg *xR;
static VM_TLS Cell xsyn[RMAXV];	/* the home of a var the lifter made (a hoisted value), which has no frame slot */
static int xv(int v, int scratch)
{
	if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; }
	if (xvh[v]) {	/* a var at block entry: its home slot, or fp for var 0 */
		int j = xvh[v] - 1;
		if (j == 0) { mov_rr(scratch, r_bp); alu_rr(a_sub, scratch, rMB); shift_ri(5, scratch, 3); }
		else if (xR->nv0 && j >= xR->nv0) { mov_ri(scratch, (Cell)(uintptr_t)&xsyn[j]); mov_rm(scratch, scratch, 0); }
		else mov_rm(scratch, r_bp, 8 * xR->slot[j]);
		return scratch;
	}
	return xreg[(int)xr[v]];
}
static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; }
static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); }
static void x64_uop(Uop *u)
{
	int op = u->op, d = is_store(op) ? -1 : xreg[(int)xr[u->d]], k;
	JW *z, *m1, *e1, *e2;
	switch (op) {
	case U_MOV: mov_rr(d, xv(u->a, r_ax)); return;
	case U_ADD: case U_SUB: case U_AND: case U_OR: case U_XOR:
		k = op == U_ADD ? a_add : op == U_SUB ? a_sub : op == U_AND ? a_and : op == U_OR ? a_or : a_xor;
		mov_rr(r_ax, xv(u->a, r_ax));
		if (xsmall(u->b)) alu_ri(k, r_ax, (int)xkv[u->b]); else alu_rr(k, r_ax, xv(u->b, r_cx));
		mov_rr(d, r_ax); return;
	case U_MUL:
		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 1, 0xaf, r_ax, xv(u->b, r_cx)); mov_rr(d, r_ax); return;
	case U_SHL: case U_SHR: case U_SAR: case U_ROTL: case U_ROTR:
		k = op == U_SHL ? 4 : op == U_SHR ? 5 : op == U_SAR ? 7 : op == U_ROTL ? 0 : 1;
		if (xkc[u->b]) { mov_rr(r_ax, xv(u->a, r_ax)); if (xkv[u->b] & 63) shift_ri(k, r_ax, (int)(xkv[u->b] & 63)); }
		else { mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); shift_cl(k, r_ax); }
		mov_rr(d, r_ax); return;
	case U_NEG: case U_NOT:
		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_NEG ? 3 : 2, r_ax); mov_rr(d, r_ax); return;
	case U_LT: case U_LTU: case U_EQ: case U_NE:
		mov_rr(r_ax, xv(u->a, r_ax)); xor32(r_dx);
		if (xsmall(u->b)) alu_ri(a_cmp, r_ax, (int)xkv[u->b]); else alu_rr(a_cmp, r_ax, xv(u->b, r_cx));
		setcc(op == U_LT ? c_l : op == U_LTU ? c_b : op == U_EQ ? c_e : c_ne, r_dx); mov_rr(d, r_dx); return;
	case U_MIN: case U_MAX: case U_MINU: case U_MAXU:	/* keep a unless the other one wins */
		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); alu_rr(a_cmp, r_ax, r_dx);
		cmov(op == U_MIN ? c_ge : op == U_MAX ? c_le : op == U_MINU ? c_ae : c_be, r_ax, r_dx); mov_rr(d, r_ax); return;
	case U_SEL:
		mov_rr(r_ax, xv(u->c, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); mov_rr(r_cx, xv(u->a, r_cx));
		test_rr(r_cx, r_cx); cmov(c_ne, r_ax, r_dx); mov_rr(d, r_ax); return;
	case U_ABS:
		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, r_ax); reg_op(0, 1, 0, 0xf7, 3, r_dx);
		test_rr(r_ax, r_ax); cmov(c_l, r_ax, r_dx); mov_rr(d, r_ax); return;
	case U_MULHS: case U_MULHU:
		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_MULHS ? 5 : 4, r_cx);
		mov_rr(d, r_dx); return;
	case U_SDIV: case U_SREM:	/* by 0: 0 or a; by -1: -a or 0, which also covers the minimum; else idiv */
		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
		test_rr(r_cx, r_cx); z = j32(c_e);
		alu_ri(a_cmp, r_cx, -1); m1 = j32(c_e);
		xb(0x48); xb(0x99); reg_op(0, 1, 0, 0xf7, 7, r_cx);	/* cqo; idiv rcx */
		if (op == U_SREM) mov_rr(r_ax, r_dx);
		e1 = jm32();
		fix32(m1, cp);
		if (op == U_SDIV) reg_op(0, 1, 0, 0xf7, 3, r_ax); else xor32(r_ax);
		e2 = jm32();
		fix32(z, cp);
		if (op == U_SDIV) xor32(r_ax);
		fix32(e1, cp); fix32(e2, cp);
		mov_rr(d, r_ax); return;
	case U_UDIV: case U_UREM:	/* by 0: 0 or a */
		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
		test_rr(r_cx, r_cx); z = j32(c_e);
		xor32(r_dx); reg_op(0, 1, 0, 0xf7, 6, r_cx);		/* div rcx */
		if (op == U_UREM) mov_rr(r_ax, r_dx);
		e1 = jm32();
		fix32(z, cp);
		if (op == U_UDIV) xor32(r_ax);
		fix32(e1, cp);
		mov_rr(d, r_ax); return;
	case U_CLZ:	/* bsr leaves zf set for 0: 127 ^ 63 is 64 */
		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 127); reg_op(0, 1, 1, 0xbd, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
		alu_ri(a_xor, r_dx, 63); mov_rr(d, r_dx); return;
	case U_CTZ:
		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 64); reg_op(0, 1, 1, 0xbc, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
		mov_rr(d, r_dx); return;
	case U_BSWAP:
		mov_rr(r_ax, xv(u->a, r_ax)); xb(0x48); xb(0x0f); xb(0xc8); mov_rr(d, r_ax); return;
	case U_LD:	/* out of range reads 0, like the interpreter */
		mov_rr(r_ax, xv(u->a, r_ax)); emit_load(); mov_rr(d, r_cx); return;
	case U_ST:	/* outside memory stores nothing; like xexec, no store hook */
		mov_rr(r_ax, xv(u->a, r_ax)); alu_ri(a_cmp, r_ax, MW); z = j32(c_ae);
		mov_xr(rMB, r_ax, 8, 0, xv(u->b, r_dx)); fix32(z, cp); return;
	}
}
/* inputs from the vm stack into their registers, the ops, the outputs back */
static void x64_enh(Enh *e)
{
	int i, j;
	xr = e->reg;
	for (i = 0; i < e->nv; i++) { xkc[i] = 0; xvh[i] = 0; }
	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; }
	for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j);
	for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]);
	vd += e->nin - e->nout;
	for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax));
}

/* ---- o3 regions in x86-64. there are too few registers for the arm64
   scheme (vars in registers), so a var lives in its home frame slot,
   m[fp + slot], with rbp = m + 8 * fp loaded at every entry. a block's values
   get the six caller-saved registers rsi, rdi, r8..r11; rax, rcx, rdx are
   scratch. a var read is a load from its home, var 0 is fp itself. the vars a
   block changes are written home at its end (old values read first, so a swap
   works), then the other slots that share a var get its value at every exit.
   ops the inline lowering lacks (floats, popcount and friends) call ev.
   calls are plain vm calls (the return point is an entry, so the callee's
   return chains back in); everything else leaves through x_disp. a region
   with native calls stays on the region interpreter, as does every region
   under ZYO3INTERP=1. */
static VM_TLS int x64_o3interp;
static VM_TLS signed char rxr[MAXUOP * 4 + 16];
#define RXNREG 6
static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); }
static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); }
static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); }
/* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */
static int x64_ralloc(RB *B)
{
	static VM_TLS int last[MAXUOP * 4 + 16];
	int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n;
	if (B->nv > MAXUOP * 4) return -1;
	for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; }
	for (i = 0; i < n; i++) {
		Uop *u = &B->u[i];
		if (u->op == U_NARG || u->op == U_NCALL) return -1;
		if (u->op == U_VGET || u->op == U_SLD || u->op == U_LI) continue;
		na = nargs(u->op);
		if (na > 0) last[u->a] = i;
		if (na > 1) last[u->b] = i;
		if (na > 2) last[u->c] = i;
	}
	for (j = 0; j < B->nso; j++) last[B->so[2 * j + 1]] = n;
	for (j = 1; j < xR->nv; j++) if (B->vs[j] >= 0) last[B->vs[j]] = n;
	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) last[B->cv] = n;
	for (i = 0; i < n; i++) {
		Uop *u = &B->u[i];
		na = u->op == U_VGET || u->op == U_SLD || u->op == U_LI ? 0 : nargs(u->op);
		for (k = 0; k < na; k++) {
			int a = k == 0 ? u->a : k == 1 ? u->b : u->c;
			if (last[a] == i && rxr[a] >= 0) { fr |= 1 << rxr[a]; last[a] = -2; }
		}
		if (is_store(u->op)) continue;
		if (u->op == U_LI) { xkc[u->d] = 1; xkv[u->d] = u->imm; continue; }
		if (u->op == U_VGET) { xvh[u->d] = u->imm + 1; continue; }
		if (!fr) return -1;
		for (k = 0; !(fr >> k & 1); k++);
		fr &= ~(1 << k); rxr[u->d] = (signed char)k;
		if (last[u->d] == -1) { fr |= 1 << k; }	/* never read: its register is free again at once */
	}
	return 0;
}
/* the store of rdx to the word rax; a covered word goes through the hook */
static void rstore(void)
{
	JW *out, *slow, *done;
	alu_ri(a_cmp, r_ax, MW); out = j32(c_ae);
	mov_rm(r_cx, rST, 32);
	mem_op(0, 0, 0, 0x80, 7, r_cx, r_ax, 1, 0); xb(0);
	slow = j32(c_ne);
	mov_xr(rMB, r_ax, 8, 0, r_dx);
	done = jm32();
	fix32(slow, cp);
	rsave();
	mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
	publish_stacks();
	call_m(rST, 64);
	reload_stacks();
	rrest();
	fix32(out, cp); fix32(done, cp);
}
/* a c call with up to three values: rdi = k, rsi, rdx, rcx = a, b, c; the result into d */
static void rcallc(void *f, long k, Uop *u, int na, int d)
{
	rsave();
	if (na > 0) mov_rr(r_ax, xv(u->a, r_ax));
	if (na > 1) mov_rr(r_dx, xv(u->b, r_dx));
	if (na > 2) mov_rr(r_cx, xv(u->c, r_cx));
	mov_rr(r_si, r_ax);
	mov_ri(r_di, k);
	call_abs(f);
	rrest();
	if (d >= 0) mov_rr(d, r_ax);
}
static void rlduop(Uop *u, int d)
{
	int n = u->op == U_LD8 ? 1 : u->op == U_LD16 ? 2 : 4;
	JW *over;
	mov_rr(r_ax, xv(u->a, r_ax));
	xor32(r_cx);
	mov_ri(r_dx, ((Cell)MW << 3) - (n - 1));
	alu_rr(a_cmp, r_ax, r_dx);
	over = j8(c_ae);
	if (n == 1) mem_op(0, 0, 1, 0xb6, r_cx, rMB, r_ax, 1, 0);
	else if (n == 2) mem_op(0, 0, 1, 0xb7, r_cx, rMB, r_ax, 1, 0);
	else mem_op(0, 0, 0, 0x8b, r_cx, rMB, r_ax, 1, 0);
	fix8(over, cp);
	mov_rr(d, r_cx);
}
/* the other slots a var stands for get its value: the frame is exact again */
static void rspill(Rg *R)
{
	int k;
	for (k = 0; k < R->np; k++)
		if (R->pv[k] > 0 && R->ps[k] != R->slot[R->pv[k]]) { mov_rm(r_ax, r_bp, 8 * R->slot[R->pv[k]]); mov_mr(r_bp, 8 * R->ps[k], r_ax); }
}
static void rexit(Rg *R, int t) { rspill(R); mov_r32i(r_ax, (unsigned)t); jmp_to(x_disp); }
/* var j's home gets register r (never rcx) */
static void rvset(Rg *R, int j, int r)
{
	if (R->nv0 && j >= R->nv0) { mov_ri(r_cx, (Cell)(uintptr_t)&xsyn[j]); mov_mr(r_cx, 0, r); }
	else mov_mr(r_bp, 8 * R->slot[j], r);
}
static VM_TLS JW **rfa; static VM_TLS int *rfb, nrf;
static void rjmp(JW *at, int b) { rfa[nrf] = at; rfb[nrf++] = b; }
static void remit_x64(Rg *R, RB *B)
{
	int i, j, d;
	JW *at;
	if (context_current) {
		mov_rm(r_cx, rST, 112); mov_r32m(r_cx, r_cx, 0); test32(r_cx, r_cx);
		at = j32(c_e); mov_r32i(r_ax, PC_HALT); jmp_to(x_exit); fix32(at, cp);
	}
	xr = rxr;
	for (i = 0; i < B->n; i++) {
		Uop *u = &B->u[i];
		int op = u->op;
		if (op == U_LI || op == U_VGET) continue;
		d = is_store(op) || rxr[u->d] < 0 ? -1 : xreg[(int)rxr[u->d]];
		if (op == U_SLD) { if (d >= 0) mov_rm(d, rSP, 8 * u->imm); continue; }
		if (d < 0 && !is_store(op)) continue;
		if (op == U_ST) { mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); rstore(); continue; }
		if (op >= U_LD8 && op <= U_ST32) {
			if (op == U_LD8 || op == U_LD16 || op == U_LD32) { rlduop(u, d); continue; }
			rsave();
			mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx));
			mov_rr(r_di, r_ax); mov_rr(r_si, r_dx);
			mov_r32i(r_dx, op == U_ST8 ? 1 : op == U_ST16 ? 2 : 4);
			call_abs((void *)xbst);
			rrest();
			continue;
		}
		if (x64_op_ok(op)) { x64_uop(u); continue; }
		rcallc((void *)xev, op, u, nargs(op), d);
	}
	for (j = 0; j < B->nso; j++) mov_mr(rSP, 8 * B->so[2 * j], xv(B->so[2 * j + 1], r_ax));
	if (B->term == T_CBR || B->term == T_JMP || B->term == T_CALL) mov_rr(r_dx, xv(B->cv, r_dx));
	/* the vars home: read every var-valued one first */
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) push_r(xv(B->vs[j], r_ax));
	for (j = 1; j < R->nv; j++) if (B->vs[j] >= 0 && !xvh[B->vs[j]]) rvset(R, j, xv(B->vs[j], r_ax));
	for (j = R->nv - 1; j >= 1; j--) if (B->vs[j] >= 0 && xvh[B->vs[j]] && xvh[B->vs[j]] - 1 != j) { pop_r(r_ax); rvset(R, j, r_ax); }
	if (B->delta) lea(rSP, rSP, -1, 1, 8 * B->delta);
	switch (B->term) {
	case T_BR: rjmp(jm32(), B->s1); break;
	case T_CBR:
		test_rr(r_dx, r_dx);
		if (B->s1 >= 0) rjmp(j32(c_e), B->s1);
		else { at = j32(c_ne); rexit(R, B->x1); fix32(at, cp); }
		if (B->s2 >= 0) rjmp(jm32(), B->s2); else rexit(R, B->x2);
		break;
	case T_CALL:
		rspill(R);
		mov_mi(rRP, 0, B->x2); alu_ri(a_add, rRP, 8);
		mov_rr(r_ax, r_dx); jmp_to(x_disp);
		break;
	case T_RET:
		rspill(R);
		alu_ri(a_sub, rRP, 8); mov_rm(r_ax, rRP, 0); jmp_to(x_disp);
		break;
	default:
		rspill(R); mov_rr(r_ax, r_dx); jmp_to(x_disp);
	}
}
/* 0: compiled; -1: can't be; -2: no room now */
static int rjit_x64(Rg *R)
{
	JW *start = cp, **lab;
	int b, e, need = 64 * R->ne + 8192, rc = 0;
	for (b = 0; b < R->nb; b++) need += 160 * R->b[b].n + 64 * R->b[b].nso + 256;
	if (cp - code + need > CODEW * 4 - (1 << 19)) return -2;
	lab = malloc((size_t)R->nb * sizeof *lab);
	rfa = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfa); rfb = malloc((size_t)(2 * R->nb + R->ne + 1) * sizeof *rfb); nrf = 0;
	if (!lab || !rfa || !rfb) { free(lab); free(rfa); free(rfb); return -2; }
	xR = R;
	for (b = 0; b < R->nb && !rc; b++) if (x64_ralloc(&R->b[b])) rc = -1;
#ifdef ZYRDUMP
	for (b = 0; b < R->nb; b++) {
		RB *B = &R->b[b]; int i, j;
		fprintf(stderr, "b%d: n=%d delta=%d term=%d cv=%d s1=%d s2=%d x1=%d x2=%d\n", b, B->n, B->delta, B->term, B->cv, B->s1, B->s2, B->x1, B->x2);
		for (i = 0; i < B->n; i++) { Uop *u = &B->u[i]; fprintf(stderr, "  v%d = %s %d %d %d imm=%ld\n", u->d, u->op < U_N ? unames[u->op] : u->op == U_VGET ? "vget" : u->op == U_SLD ? "sld" : "?", u->a, u->b, u->c, (long)u->imm); }
		for (j = 0; j < B->nso; j++) fprintf(stderr, "  so %d <- v%d\n", B->so[2*j], B->so[2*j+1]);
		for (j = 1; j < R->nv; j++) fprintf(stderr, "  var%d(slot %d) <- %d\n", j, R->slot[j], B->vs[j]);
	}
	for (b = 0; b < R->np; b++) fprintf(stderr, "ps %d -> var %d\n", R->ps[b], R->pv[b]);
#endif
	for (e = 0; e < R->ne && !rc; e++) {
		R->stub[e] = cp;
		mov_rm(r_ax, rMB, 8 * ZFP); lea(r_bp, rMB, r_ax, 8, 0);
		rjmp(jm32(), R->entb[e]);
	}
	for (b = 0; b < R->nb && !rc; b++) {
		lab[b] = cp;
		x64_ralloc(&R->b[b]);
		remit_x64(R, &R->b[b]);
	}
	if (!rc && cp - start > need) { fprintf(stderr, "o3: x86-64 code overran its estimate\n"); abort(); }
	if (!rc) {
		int k;
		for (k = 0; k < nrf; k++) fix32(rfa[k], lab[rfb[k]]);
		R->nat = start;
		for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words) tab[R->ent[e]] = R->stub[e];
		if (trace) fprintf(stderr, "o3: %d..%d -> %d bytes of x86-64\n", R->lo, R->hi - 1, (int)(cp - start));
	} else cp = start;
	free(lab); free(rfa); free(rfb);
	return rc;
}
static void rforget(Rg *R)
{
	int e;
	if (R->nat) for (e = 0; e < R->ne; e++) if ((unsigned)R->ent[e] < (unsigned)code_words && tab[R->ent[e]] == R->stub[e]) tab[R->ent[e]] = 0;
	R->nat = 0;
}
static void rcompile(Rg *R, int eager)
{
	(void)eager;
	if (R->nat || R->nonat || x64_o3interp || x64_noo3 || !R->lifted) return;
	if (rjit_x64(R) == -1) { R->nonat = 1; if (trace) fprintf(stderr, "o3: %d..%d stays on the region interpreter\n", R->lo, R->hi - 1); }
}

static void *translate(int p)
{
	JW *start;
	int i = p, n = 0, k, t, used[64], nused = 0, j2, ju;
	Cell w;
	if (cp - code > CODEW * 4 - (1 << 19) || !ix_room()) flush();
	start = cp;
	vd = 0;
	for (;;) {
		if (i >= code_words) { exit_with(-1); break; }
		if (n++ >= BLKW || (i != p && o3ent[i])) { disp_imm(i); break; }	/* a region entry is entered from c, never by chaining */
		w = m[i];
		if (w & 1 && i + 1 < code_words && !(m[i + 1] & 1) && (k = (int)(m[i + 1] / 2)) >= 6 && k <= 8) {
			t = (int)((w - 1) / 2);	/* literal + call/jmp/jz with a known target */
			if (k == 8) {
				JW *skip;
				ldv(r_cx, 0); vd++;
				test_rr(r_cx, r_cx);
				skip = j32(c_ne);
				disp_imm(t);
				fix32(skip, cp);
				i += 2;
				continue;
			}
			if (k == 6) { mov_mi(rRP, 0, i + 2); alu_ri(a_add, rRP, 8); }
			disp_imm(t);
			i += 2;
			break;
		}
		if (w & 1 && cp - start < (1 << 18) && !x64_noenh) {	/* literals feeding an enhancement: specialize it on them */
			int L = 0, s, nk, j;
			Cell ks[8], x;
			Enh te;
			while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++;
			x = i + L < code_words ? m[i + L] : 1;
			if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
				nk = L < enh[s].nin ? L : enh[s].nin;
				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
				if (!x64_spec_enh(s, ks, nk, &te)) {
					for (j = 0; j < L - nk; j++) { Cell v = m[i + j] >> 1; vd--; if (v >= -2147483647 - 1 && v <= 2147483647) mov_mi(rSP, SL(0), (int)v); else { mov_ri(r_ax, v); stv(0, r_ax); } }
					if (nused < 64) used[nused++] = s; else used[0] = -1;
					x64_enh(&te);
					free(te.u); free(te.reg);
					i += L + 1; n += L;
					continue;
				}
			}
		}
		if (w & 1) {
			Cell v = (w - 1) / 2;
			int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1;
			if (i + 1 < code_words && !(m[i + 1] & 1)) ko = (int)(m[i + 1] / 2);
			if (ko == 1 && v >= 0 && v < MW && v < (1 << 27)) {	/* a fixed address: one load */
				mov_rm(r_ax, rMB, (int)(8 * v)); vd--; stv(0, r_ax);
				i += 2; continue;
			}
			if (small && ko >= 13 && ko <= 20) {	/* the literal is an immediate operand */
				int d = (int)v;
				switch (ko) {
				case 13: alu_mi(a_add, rSP, SL(0), d); break;
				case 14: alu_mi(a_sub, rSP, SL(0), d); break;
				case 15: imul_rmi(r_ax, rSP, SL(0), d); stv(0, r_ax); break;
				case 16: alu_mi(a_and, rSP, SL(0), d); break;
				case 17: alu_mi(a_or, rSP, SL(0), d); break;
				case 18: alu_mi(a_xor, rSP, SL(0), d); break;
				case 19: if (d & 63) shift_mi(7, rSP, SL(0), d & 63); break;
				default: xor32(r_cx); alu_mi(a_cmp, rSP, SL(0), d); setcc(c_l, r_cx); stv(0, r_cx);
				}
				i += 2; continue;
			}
			vd--;
			if (small) mov_mi(rSP, SL(0), (int)v);
			else { mov_ri(r_ax, v); stv(0, r_ax); }
			i++; continue;
		}
		k = (int)(w / 2);
		if (k == 0) { exit_with(-1); i++; break; }
		if (k < 0) { exit_with(-1); i++; break; }	/* reserved: the interpreter halts */
		if (k == 6 || k == 7) {
			mov_r32m(r_ax, rSP, SL(0)); vd++;
			if (k == 6) { mov_mi(rRP, 0, i + 1); alu_ri(a_add, rRP, 8); }
			disp_ax();
			i++;
			break;
		}
		if (k == 8) {	/* jz: taken leaves, not taken keeps going */
			JW *skip;
			mov_r32m(r_ax, rSP, SL(0)); ldv(r_cx, 1); vd += 2;
			test_rr(r_cx, r_cx);
			skip = j32(c_ne);
			disp_ax();
			fix32(skip, cp);
			i++;
			continue;
		}
		if (k == 9) { alu_ri(a_sub, rRP, 8); mov_r32m(r_ax, rRP, 0); disp_ax(); i++; break; }
		if (k == 21) {	/* install: call into c, then return to it so it can flush */
			mov_r32m(r_di, rSP, SL(0)); call_m(rST, 72);
			movsxd_rr(r_ax, r_ax); stv(0, r_ax);
			exit_with(i + 1); i++; break;
		}
		if (k == 600) {	/* flit: the next word is raw IEEE bits, even when bit 0 is set */
			if (i + 1 >= code_words) { exit_with(-1); i++; break; }
			mov_ri(r_ax, m[i + 1]); vd--; stv(0, r_ax);
			i += 2;
			continue;
		}
		if ((k >= 601 && k <= 611) || k == 618) { float_op(i, k); i++; continue; }
		if (k == 612 || k == 614 || k == 616) { emit_ld_bytes(k == 612 ? 1 : k == 614 ? 2 : 4); i++; continue; }
		if (k == 613 || k == 615 || k == 617) {
			JW *ok;
			ldv(r_di, 0); ldv(r_si, 1); mov_r32i(r_dx, k == 613 ? 1 : k == 615 ? 2 : 4); vd += 2;
			call_abs((void *)xbst);
			test32(r_ax, r_ax);
			ok = j32(c_e);
			exit_with(i + 1);
			fix32(ok, cp);
			i++; continue;
		}
		if (k == 619) {	/* push the data-stack index: the word s points at */
			sp_index(r_ax); vd--; stv(0, r_ax);
			i++; continue;
		}
		if (k == 620) {	/* a native call: the helper pops the arguments and pushes the result */
			lea(r_di, rSP, -1, 1, 8 * vd); mov_rr(r_si, rRP);
			call_abs((void *)ncall_host);
			mov_rr(rSP, r_ax); vd = 0;
			i++; continue;
		}
		if (k >= XOP0 && k < XOP0 + NENH) {
			if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1;	/* wired or not, an install makes this a different block */
			if (enh[k - XOP0].live && cp - start < (1 << 18) && !x64_noenh) {	/* inline */
				Enh te;
				if (!x64_spec_enh(k - XOP0, 0, 0, &te)) { x64_enh(&te); free(te.u); free(te.reg); i++; continue; }
			}
			if (enh[k - XOP0].live) {	/* the interpreter runs it: x0 = data stack, slot */
				lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0));
				call_m(rST, 80);
				mov_rr(rSP, r_ax); vd = 0;
				i++; continue;
			}
		}
		if (k > 20) { exit_with(-1); i++; break; }	/* not wired: halt */
		switch (k) {
		case 1:
			ldv(r_ax, 0); emit_load(); stv(0, r_cx); break;
		case 2:	/* store: inline unless the target word is translated */
			ldv(r_ax, 0); ldv(r_dx, 1); vd += 2;
			emit_store(i, 0); break;
		case 3: {	/* pick uses a word address, with the same bounds as load */
			ldv(r_ax, 0);
			sp_index(r_cx);
			lea(r_ax, r_ax, r_cx, 1, 1);
			emit_load(); stv(0, r_cx); break; }
		case 4: vd++; break;
		case 5: ldv(r_ax, 0); ldv(r_cx, 1); stv(0, r_cx); stv(1, r_ax); break;
		case 10:
			call_m(rST, 40); movsx64_rr8(r_ax, r_ax); vd--; stv(0, r_ax); break;
		case 11:
			mov_rm(r_ax, rST, 56);
			if (sizeof J == 1) mem_op(0, 1, 1, 0xbe, r_ax, r_ax, -1, 1, 0);	/* movsx rax, byte [rax] */
			else mem_op(0, 1, 0, 0x63, r_ax, r_ax, -1, 1, 0);		/* movsxd rax, dword [rax] */
			vd--; stv(0, r_ax); break;
		case 12:
			ldv(r_ax, 0); movsx_rr8(r_di, r_ax); vd++; call_m(rST, 48); break;
		case 13: case 14: case 16: case 17: case 18:
			ldv(r_ax, 1);
			alu_rm(k == 13 ? a_add : k == 14 ? a_sub : k == 16 ? a_and : k == 17 ? a_or : a_xor, r_ax, rSP, SL(0));
			vd++; stv(0, r_ax); break;
		case 15: ldv(r_ax, 1); imul_rm(r_ax, rSP, SL(0)); vd++; stv(0, r_ax); break;
		case 19: ldv(r_cx, 0); ldv(r_ax, 1); shift_cl(7, r_ax); vd++; stv(0, r_ax); break;
		default:	/* 20 */
			ldv(r_ax, 1); xor32(r_cx); alu_rm(a_cmp, r_ax, rSP, SL(0)); setcc(c_l, r_cx); vd++; stv(0, r_cx);
		}
		i++;
	}
	j2 = ix_add(p, i);
	cinc(cov, p, i);
	if (nused && used[0] < 0) dep_add(NENH, j2);
	else for (ju = 0; ju < nused; ju++) dep_add(used[ju], j2);
	tab[p] = start;
	if (trace) fprintf(stderr, "jit: %d..%d -> %ld bytes\n", p, i - 1, (long)(cp - start));
	return start;
}
static void x64_limits(void) { S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit()); }
static int vm_until(int pc, int stop_eq, int stop_ge, Rg *re, Cell *ps, Cell *pr)
{
	union { JW *u; int (*f)(long *, void *); } enter;
	void *saved_eq = 0, *saved_ent[RMAXE];
	int i, nent = 0, flushed = 0, ep0 = drop_epoch, term = 0;
	Cell return_depth = *pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1;
	enter.u = x_enter;
	if (stop_eq >= 0 && stop_eq < code_words) { saved_eq = tab[stop_eq]; tab[stop_eq] = 0; }
	if (re) {
		nent = re->ne; if (nent > RMAXE) nent = RMAXE;
		for (i = 0; i < nent; i++) { saved_ent[i] = tab[re->ent[i]]; tab[re->ent[i]] = 0; }
	}
	S[1] = *ps; S[2] = *pr;
	x64_limits();
	while (!term && !context_cancelled() && ((unsigned)pc < (unsigned)code_words || pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW)) {
		void *c;
		if (pc == PC_DATA_OVERFLOW || pc == PC_RETURN_OVERFLOW) {
			Cell sp = S[1], rp = S[2]; int ret = pc == PC_RETURN_OVERFLOW;
			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
			S[1] = sp; S[2] = rp;
			x64_limits();
			pc = (int)S[13]; continue;
		}
		if (stop_eq >= 0 && pc == stop_eq && S[2] - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == return_depth) break;
		if (stop_ge >= 0 && pc >= stop_ge) break;
		if (re && pc >= re->lo && pc < re->hi && o3ent[pc] == re->idx + 1) break;
		if (o3ent[pc] && !x64_noo3) {	/* an o3 region: its x86-64 code if it has some, else the region interpreter */
			Rg *R = rat(pc);
			if (R) {
				int s = (int)S[1], r = (int)S[2];
				if (s < stack_data_limit() || r > stack_return_limit()) {
					Cell sp = s, rp = r;
					int ret = r > stack_return_limit();
					if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { pc = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
					s = (int)sp; r = (int)rp;
				}
				rcompile(R, 0);
				if (R->nat && tab[pc]) { S[1] = s; S[2] = r; x64_limits(); goto native; }
				pc = rexec(R, o3ei[pc], &s, &r);
				S[1] = s; S[2] = r;
				x64_limits();
				if ((unsigned)pc >= (unsigned)code_words) term = 1;	/* a stop, not a jit exit: no saved target to resume at */
				continue;
			}
		}
	native:
		c = tab[pc];
		if (!c) {
			jitwrite(1); c = translate(pc); jitwrite(0);
			if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = 0;
			if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = 0;
		}
		if (!c) break;
		pc = enter.f(S, c);
		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
	}
	*ps = S[1]; *pr = S[2];
	flushed = drop_epoch != ep0;
	if (!flushed) {
		if (stop_eq >= 0 && stop_eq < code_words) tab[stop_eq] = saved_eq;
		if (re) for (i = 0; i < nent; i++) tab[re->ent[i]] = saved_ent[i];
	}
	return pc;
}
/* the code area, the stubs and the state block, for this thread */
static int x64_boot(Cell sp, Cell rp, void *stop)
{
	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	if (code == MAP_FAILED) { perror("mmap"); return 1; }
	x64_noo3 = getenv("ZYNOO3") != 0;
	x64_noenh = getenv("ZYNOENH") != 0;
	x64_o3interp = getenv("ZYO3INTERP") != 0;
	jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
	S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
	S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)stop;
	x64_limits();
	cov[SYSGATE] = 1;
	return 0;
}
static int run(int p)
{
	Cell s = DS0, r = RS0;
	if (x64_boot(s, r, &context_default_stop)) return 1;
	p = vm_until(p, -1, -1, 0, &s, &r);
	halt_at = -1;
	return stopped(p);
}
#else
#define LD(x) (inmem(x) ? m[x] : 0)
static VM_TLS int stop_eq = -1, stop_ge = -1, stop_return_depth = -1, run_nest;
static VM_TLS Rg *stop_rg;
static VM_TLS Cell run_s, run_r;
static int run(int p)
{
	int s = run_nest ? (int)run_s : DS0, r = run_nest ? (int)run_r : RS0, xfer = 1;
	Cell w, a, b;
	unsigned long long steps = 0;
#ifdef ZYLLVM
	Cell regs[2];
	/* worker contexts interpret regions; llvm ownership stays in its compiling context. */
	if (!context_current && !getenv("ZYNOLLVM")) llvm_init();
#endif
	while ((unsigned)p < (unsigned)code_words && !context_cancelled()) {
		if (run_nest) {
			if (stop_eq >= 0 && p == stop_eq && r - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) == stop_return_depth) break;
			if (stop_ge >= 0 && p >= stop_ge) break;
			if (stop_rg && p >= stop_rg->lo && p < stop_rg->hi && o3ent[p] == stop_rg->idx + 1) break;
		}
		if (xfer && (s < stack_data_limit() || r > stack_return_limit())) {
			Cell sp = s, rp = r;
			int ret = r > stack_return_limit();
			if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) { p = ret ? PC_RETURN_OVERFLOW : PC_DATA_OVERFLOW; break; }
			s = (int)sp; r = (int)rp;
		}
		if (xfer && o3ent[p]) {	/* an o3 region */
			Rg *R = rat(p);
			if (R) {
#ifdef ZYLLVM
				rcompile(R, 0);
				if (R->nat) { regs[0] = s; regs[1] = r; ccode++; p = (int)((RFn)R->nat)(m, regs, o3ei[p]); ccode--; rt_sweep(); s = (int)regs[0]; r = (int)regs[1]; }
				else
#endif
				p = rexec(R, o3ei[p], &s, &r);
				continue;
			}
		}
#ifdef ZYLLVM
		if (xfer && llvm_ok) {	/* at a block start: run it compiled, or count it */
			xfer = 0;
			if (!lt[p] && hot[p] < 65535 && ++hot[p] == HOT && !(lt[p] = lcompile(p))) hot[p] = 65535;
			if (lt[p]) {
				regs[0] = s; regs[1] = r;
				ccode++; p = lt[p](m, regs); ccode--;
				s = regs[0]; r = regs[1];
				rt_sweep();
				if (need_flush) { need_flush = 0; ldrop_all(); }
				xfer = 1;
				continue;
			}
		}
#endif
		w = m[p++];
		steps++;
		if (w & 1) { m[--s] = w >> 1; continue; }
		a = m[s]; b = m[s + 1];
		switch ((int)(w >> 1)) {
		case 1: m[s] = LD(a); break;
		case 2:
			if (a == SYSGATE) {
				int moving = b == 9 && (m[a + 1] == 7 || m[a + 1] == 8);
				cb_s = s + 2; cb_r = r; m[a] = hostcall(b, m[a + 1], m[a + 2], m[a + 3]);
				if (moving) { s = (int)cb_s; r = (int)cb_r; } else s += 2;
				break;
			}
			if (inmem(a)) {
#ifdef ZYLLVM
				if (lcov[a]) lhook(a, b); else
#else
				if (rcov[a]) { m[a] = b; rdrop(a); } else
#endif
				m[a] = b;
			}
			s += 2; break;
		case 3: m[s] = LD(s + 1 + a); break;
		case 4: s++; break;
		case 5: m[s] = b; m[s + 1] = a; break;
		case 6: m[r++] = p; p = a; s++; xfer = 1; break;
		case 7: p = a; s++; xfer = 1; break;
		case 8: if (!b) { p = a; xfer = 1; } s += 2; break;
		case 9: p = m[--r]; xfer = 1; break;
		case 10: m[--s] = (signed char)Q(); break;
		case 11: m[--s] = J; break;
		case 12: G((char)a); s++; break;
		case 13: m[++s] = b + a; break;
		case 14: m[++s] = b - a; break;
		case 15: m[++s] = b * a; break;
		case 16: m[++s] = b & a; break;
		case 17: m[++s] = b | a; break;
		case 18: m[++s] = b ^ a; break;
		case 19: m[++s] = b >> (a & 63); break;
		case 20: m[++s] = b < a; break;
		case 600:
			if (p >= code_words) { p = PC_HALT; break; }
			m[--s] = m[p++];
			break;
		case 601: m[++s] = (Cell)fop_add((uint64_t)b, (uint64_t)a); break;
		case 602: m[++s] = (Cell)fop_sub((uint64_t)b, (uint64_t)a); break;
		case 603: m[++s] = (Cell)fop_mul((uint64_t)b, (uint64_t)a); break;
		case 604: m[++s] = (Cell)fop_div((uint64_t)b, (uint64_t)a); break;
		case 605: m[++s] = fop_eq((uint64_t)b, (uint64_t)a); break;
		case 606: m[++s] = fop_lt((uint64_t)b, (uint64_t)a); break;
		case 607: m[++s] = fop_le((uint64_t)b, (uint64_t)a); break;
		case 608: m[s] = (Cell)fop_i2f(a); break;
		case 609: m[s] = fop_f2i((uint64_t)a); break;
		case 610: fop_print(a); s++; break;
		case 611: m[s] = fop_parse_cell(a); break;
		case 618: fop_print_mode(m[s + 2], b, a); s += 3; break;
		case 612: m[s] = bld8(a); break;
		case 613: bst8(a, b); s += 2; break;
		case 614: m[s] = bld16(a); break;
		case 615: bst16(a, b); s += 2; break;
		case 616: m[s] = bld32(a); break;
		case 617: bst32(a, b); s += 2; break;
		case 619: { Cell t = s; m[--s] = t; break; }
		case 620: cb_r = r; s = (int)ncall_exec(s); break;
		case 21:
			m[s] = install(a);
#ifdef ZYLLVM
			if (need_flush) { need_flush = 0; ldrop_all(); }
#else
			if (need_flush) { need_flush = 0; rdrop_all(); memset(rcov, 0, (size_t)code_words); }
#endif
			break;
		default:
			if ((int)(w >> 1) >= XOP0 && (int)(w >> 1) < XOP0 + NENH && enh[(int)(w >> 1) - XOP0].live) s = (int)(xexec(&enh[(int)(w >> 1) - XOP0], m + s) - m);
			else p = PC_HALT;	/* halt: unwired opcodes and the reserved negative ones */
		}
	}
	if (run_nest) { run_s = s; run_r = r; return p; }
	halt_at = -1;
	if (trace) fprintf(stderr, "interp: %llu steps\n", steps);
	return stopped(p);
}
static int vm_until(int pc, int seq, int sge, Rg *re, Cell *ps, Cell *pr)
{
	int pe = stop_eq, pg = stop_ge, pd = stop_return_depth, pn = run_nest, back;
	Rg *old = stop_rg;
	Cell ss = run_s, rr = run_r;
	stop_eq = seq; stop_ge = sge; stop_return_depth = (int)(*pr - (stack_control ? stack_get(SC_RETURN_LOW) : RS0) - 1); stop_rg = re;
	run_nest = 1; run_s = *ps; run_r = *pr;
	back = run(pc);
	*ps = run_s; *pr = run_r;
	stop_eq = pe; stop_ge = pg; stop_return_depth = pd; stop_rg = old; run_nest = pn; run_s = ss; run_r = rr;
	return back;
}
#endif
static int o3_ready(Rg *C)
{
	if (!C || C->dead) return 0;
	if (!C->lifted && rlift(C)) { C->dead = 1; return 0; }
	return C->lifted && !C->dead;
}
static int o3_exec_region(Rg *C, int entry, Cell *ps, Cell *pr)
{
	int s, r, back;
	if (!C || C->dead || entry < 0 || entry >= C->ne) return -1;
	if (!C->lifted && rlift(C)) { C->dead = 1; return -1; }
#ifdef ZYLLVM
	rcompile(C, 0);
	if (C->nat) {
		typedef Cell (*RFn2)(Cell *, Cell *, int);
		Cell regs[2];
		regs[0] = *ps; regs[1] = *pr;
		back = (int)((RFn2)C->nat)(m, regs, entry);
		*ps = regs[0]; *pr = regs[1];
		return back;
	}
#endif
	s = (int)*ps; r = (int)*pr;
	back = rexec(C, entry, &s, &r);
	*ps = s; *pr = r;
	return back;
}
#endif

/* a byte store writes whole words through the same hook as opcode 2, so a
   translation covering either word is dropped. it does not run a syscall:
   the gate is a word store at SYSGATE. */
static void b_commit(Cell w, Cell v)
{
	if ((unsigned)w >= (unsigned)MW) return;
	if (w == SYSGATE) { m[w] = v; return; }
#if defined(ZYNATIVE)
	if (cov[(int)w]) hook(w, v); else m[w] = v;
#elif defined(ZYLLVM)
	if (lcov[(int)w]) lhook(w, v); else m[w] = v;
#else
	if (rcov[(int)w]) { m[w] = v; rdrop((int)w); } else m[w] = v;
#endif
}

#ifdef ZYBIGSTACK
static VM_TLS ucontext_t uc_main, uc_vm;
static VM_TLS int vm_rc;
static void vm_entry(void) { vm_rc = run(GORG); }
#endif
static int memory_option(const char *name, int fallback, int minimum)
{
	const char *s = getenv(name);
	char *end;
	unsigned long long words;
	if (!s) return fallback;
	errno = 0; words = strtoull(s, &end, 0);
	if (errno || !*s || *end || words < (unsigned)minimum || words > 0x7fffffff) {
		fprintf(stderr, "zy: invalid %s (minimum %d words)\n", name, minimum);
		return -1;
	}
	return (int)words;
}
/* reserve all address-indexed metadata together; a denied request leaves it intact. */
static int code_reserve(Cell end)
{
	Cell cap = code_words ? code_words : PW;
	void *slots[6], *previous[6], *fresh[6]; size_t widths[6], oldn[6], newn[6]; int count = 0, i, j;
	if (end <= 0 || end > memory_words || end >= PC_HALT) return -1;
	if (end <= code_words) return (int)code_words;
	while (cap < end) { if (cap > memory_words / 2) { cap = memory_words; break; } cap *= 2; }
	if (cap >= PC_HALT) cap = PC_HALT - 1;
	if (cap < end) return -1;
#define CODE_COLUMN(name, before, after) do { slots[count] = &name; previous[count] = name; widths[count] = sizeof *name; oldn[count] = (size_t)(before); newn[count] = (size_t)(after); count++; } while (0)
	CODE_COLUMN(pg_head, code_words ? NPG : 0, (cap >> PGSH) + 1);
	CODE_COLUMN(o3ent, code_words, cap); CODE_COLUMN(o3ei, code_words, cap);
#ifdef ZYNATIVE
	CODE_COLUMN(tab, code_words, cap);
#ifdef ZYJIT
	CODE_COLUMN(fn_nat, code_words, cap);
#endif
#elif defined(ZYLLVM)
	CODE_COLUMN(lt, code_words, cap); CODE_COLUMN(hot, code_words, cap);
#endif
#undef CODE_COLUMN
	for (i = 0; i < count; i++) {
		fresh[i] = calloc(newn[i], widths[i]);
		if (!fresh[i]) { for (j = 0; j < i; j++) free(fresh[j]); return -1; }
		if (oldn[i]) memcpy(fresh[i], previous[i], oldn[i] * widths[i]);
	}
	for (i = 0; i < count; i++) { free(previous[i]); memcpy(slots[i], &fresh[i], sizeof fresh[i]); }
	code_words = cap;
#ifdef ZYNATIVE
	S[3] = (long)tab;
#endif
	return (int)cap;
}
/* bulk writes use the ordinary store path when they would rewrite live code. */
static int code_range_live(Cell start, Cell count)
{
	Cell end, i;
	if (start < 0 || count <= 0 || start >= code_words) return 0;
	end = count > code_words - start ? code_words : start + count;
	for (i = start; i < end; i++) {
#ifdef ZYNATIVE
		if (cov[i]) return 1;
#else
		if (rcov[i]) return 1;
#ifdef ZYLLVM
		if (lcov[i]) return 1;
#endif
#endif
	}
	return 0;
}

/* each worker owns a host thread, vm memory and translation state. */
#if !defined(_WIN32) && !defined(__wasm__)
typedef struct ContextChannel { Cell handle; int fd; struct ContextChannel *next; } ContextChannel;
static VM_TLS ContextChannel *context_channels;
static atomic_llong context_id_sequence;
struct VmContext {
 ContextChannel parent_channel, worker_channel;
 pthread_t thread;
 struct VmContext *next;
 atomic_int stop;
 Cell token, entry, sp, rp, stack, words, limit, growth, code_size;
 Cell frame_floor, frame_top, frame_guard;
 Cell *memory;
 int exit_code, requested_exit;
 void *input_snapshot;
 int argc, envc;
 char **argv, **envp;
#ifdef ZYFFI
 unsigned char callback_snapshot[sizeof ffi_cb], symbol_snapshot[sizeof ffi_sym], library_snapshot[sizeof ffi_lib];
 int ffi_symbols, ffi_libraries, ffi_libm;
#endif
 Enh enhancements[NENH];
 Rg *regions[MAXRG];
 int region_count;
 unsigned short *entries;
 unsigned char *entry_indices;
};
static int context_descriptor(Cell handle) {
 ContextChannel *channel;
 if (handle < CONTEXT_HANDLE_LO) return handle >= 0 && handle <= 0x7fffffff ? (int)handle : -1;
 for (channel = context_channels; channel; channel = channel->next) if (channel->handle == handle) return channel->fd;
 return -1;
}
static int context_channel_close(Cell handle) {
 ContextChannel **link = &context_channels, *channel;
 while (*link && (*link)->handle != handle) link = &(*link)->next;
 if (!*link) return -1; channel = *link; *link = channel->next;
 if (channel->fd < 0) return -1;
 int result = close(channel->fd); channel->fd = -1; return result;
}
static void context_snapshot_free(VmContext *context) {
 int i;
 for (i = 0; i < NENH; i++) { free(context->enhancements[i].u); free(context->enhancements[i].reg); }
 for (i = 0; i < context->region_count; i++) free(context->regions[i]);
 free(context->input_snapshot); free(context->entries); free(context->entry_indices); if (context->memory) { free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } context->memory = 0;
}
static int context_cancelled(void) { return context_current && atomic_load_explicit(&context_current->stop, memory_order_acquire); }
static void context_exit(int value) { context_current->exit_code = value; context_current->requested_exit = 1; atomic_store_explicit(&context_current->stop, 1, memory_order_release); }
static void context_engine_free(void) {
 int i;
 for (i = 0; i < nrg; i++) { rclear(rgs[i]); free(rgs[i]); }
 for (i = 0; i < NENH; i++) { free(enh[i].u); free(enh[i].reg); }
 free(pg_head); free(o3ent); free(o3ei); free(gn_); free(gh_);
#ifdef ZYNATIVE
#ifdef ZYJIT
 free(fn_nat);
#endif
 free(tab); free(cov); if (code && code != MAP_FAILED && munmap(code, CODEW * 4) == 0) atomic_fetch_sub(&context_code_live, 1);
#else
 free(rcov);
#ifdef ZYLLVM
 free(lcov); free(lt); free(hot);
#endif
#endif
 free(m); m = 0; atomic_fetch_sub(&context_memory_live, 1);
}
static void *context_worker(void *opaque) {
 VmContext *context = opaque;
 Cell sp = context->sp, rp = context->rp, entry = context->entry;
 int i, back;
 optimizer_storage();
 context_current = context; io_context_restore(context->input_snapshot); free(context->input_snapshot); context->input_snapshot = 0; trace = getenv("ZYTRACE") != 0;
 context->worker_channel.next = context_channels; context_channels = &context->worker_channel;
 m = context->memory; context->memory = 0;
 memory_limit = (int)context->limit; memory_words = (int)context->words; memory_growth_limit = (int)context->growth;
 stack_control = context->stack; frame_floor_word = context->frame_floor; frame_top_word = context->frame_top; frame_guard_word = context->frame_guard;
 zy_argc = context->argc; zy_argv = context->argv; zy_envc = context->envc; zy_envp = context->envp;
#ifdef ZYFFI
 memcpy(ffi_cb, context->callback_snapshot, sizeof ffi_cb); memcpy(ffi_sym, context->symbol_snapshot, sizeof ffi_sym); memcpy(ffi_lib, context->library_snapshot, sizeof ffi_lib);
 ffi_nsym = context->ffi_symbols; ffi_nlib = context->ffi_libraries; ffi_tried_libm = context->ffi_libm;
#endif
#ifdef ZYNATIVE
 cov = calloc((size_t)MW, 1);
#else
 rcov = calloc((size_t)MW, 1);
#ifdef ZYLLVM
 lcov = calloc((size_t)MW, 1);
#endif
#endif
#ifdef ZYNATIVE
 if (!cov) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
#else
 if (!rcov
#ifdef ZYLLVM
 || !lcov
#endif
 ) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
#endif
 if (code_reserve(context->code_size) < 0) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
 memcpy(enh, context->enhancements, sizeof enh); memset(context->enhancements, 0, sizeof context->enhancements);
 for (i = 0; i < context->region_count; i++) { rgs[i] = context->regions[i]; context->regions[i] = 0; }
 nrg = context->region_count; context->region_count = 0;
 memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words);
 free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0;
#ifdef ZYJIT_X64
 if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
 atomic_fetch_add(&context_code_live, 1);
#endif
#ifdef ZYJIT
#ifdef __APPLE__
 code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
#else
 code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#endif
 if (code == MAP_FAILED) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; }
 atomic_fetch_add(&context_code_live, 1);
 jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
 S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
 S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
 S[9] = (long)inst_hook; S[10] = (long)xrun; S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
 S[14] = (long)&context->stop; cov[SYSGATE] = 1;
#endif
 if (entry >= HEAP_LO && inmem(entry) && m[entry - 3] == (entry ^ 0x2B7E1516)) { m[--sp] = entry + 3; entry = m[entry]; }
 m[rp++] = PC_HALT;
 back = vm_until((int)entry, PC_HALT, -1, 0, &sp, &rp);
 if (!context->requested_exit) context->exit_code = context_cancelled() ? 0 : stopped(back) ? -1 : inmem(sp) ? (int)m[sp] : 0;
 context_channel_close(context->worker_channel.handle);
 while (context_children) context_join(context_children->token, 1);
 context_engine_free(); context_current = 0; return 0;
}
static Cell context_start(Cell entry, Cell channel) {
 VmContext *context; int fd[2], i; pthread_attr_t attributes; Cell saved;
 if (entry < 0 || (!inmem(entry)) || !inmem(channel)) return -1;
 if (entry >= HEAP_LO && (!inmem(entry - 3) || m[entry - 3] != (entry ^ 0x2B7E1516))) return -1;
 if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) < 0) return -1;
 context = calloc(1, sizeof *context); if (!context) { close(fd[0]); close(fd[1]); return -1; }
 context->entry = entry; context->words = memory_words; context->limit = memory_limit; context->growth = memory_growth_limit; context->code_size = code_words;
 context->sp = cb_s; context->rp = cb_r; context->stack = stack_control; context->frame_floor = frame_floor_word; context->frame_top = frame_top_word; context->frame_guard = frame_guard_word;
 context->argc = zy_argc; context->argv = zy_argv; context->envc = zy_envc; context->envp = zy_envp;
 context->token = atomic_fetch_add(&context_id_sequence, 1) + 1;
 context->parent_channel.handle = CONTEXT_HANDLE_LO + atomic_fetch_add(&context_id_sequence, 1) + 1;
 context->worker_channel.handle = CONTEXT_HANDLE_LO + atomic_fetch_add(&context_id_sequence, 1) + 1;
 context->parent_channel.fd = fd[0]; context->worker_channel.fd = fd[1]; atomic_init(&context->stop, 0);
 context->input_snapshot = malloc(io_context_size());
 if (!context->input_snapshot) goto failed; io_context_snapshot(context->input_snapshot);
 context->memory = calloc((size_t)memory_limit, sizeof(Cell));
 if (context->memory) atomic_fetch_add(&context_memory_live, 1);
 context->entries = malloc((size_t)code_words * sizeof *o3ent); context->entry_indices = malloc((size_t)code_words);
 if (!context->memory || !context->entries || !context->entry_indices) goto failed;
 saved = m[channel]; m[channel] = context->worker_channel.handle; memcpy(context->memory, m, (size_t)memory_words * sizeof(Cell)); m[channel] = saved;
 memcpy(context->entries, o3ent, (size_t)code_words * sizeof *o3ent); memcpy(context->entry_indices, o3ei, (size_t)code_words);
 for (i = 0; i < NENH; i++) {
  context->enhancements[i] = enh[i]; context->enhancements[i].u = 0; context->enhancements[i].reg = 0;
  if (enh[i].u) { context->enhancements[i].u = malloc((size_t)(enh[i].n + 1) * sizeof(Uop)); if (!context->enhancements[i].u) goto failed; memcpy(context->enhancements[i].u, enh[i].u, (size_t)(enh[i].n + 1) * sizeof(Uop)); }
  if (enh[i].reg) { context->enhancements[i].reg = malloc((size_t)enh[i].nv); if (!context->enhancements[i].reg) goto failed; memcpy(context->enhancements[i].reg, enh[i].reg, (size_t)enh[i].nv); }
 }
 for (i = 0; i < nrg; i++) {
  Rg *region = malloc(sizeof *region); if (!region) goto failed; *region = *rgs[i];
  region->b = 0; region->nb = 0; region->lifted = 0; region->marked = 0; region->nat = 0; region->dnat = 0; region->rt = 0; memset(region->stub, 0, sizeof region->stub);
  context->regions[context->region_count++] = region;
 }
#ifdef ZYFFI
 memcpy(context->callback_snapshot, ffi_cb, sizeof ffi_cb); memcpy(context->symbol_snapshot, ffi_sym, sizeof ffi_sym); memcpy(context->library_snapshot, ffi_lib, sizeof ffi_lib);
 context->ffi_symbols = ffi_nsym; context->ffi_libraries = ffi_nlib; context->ffi_libm = ffi_tried_libm;
#endif
 pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 16u << 20);
 i = pthread_create(&context->thread, &attributes, context_worker, context); pthread_attr_destroy(&attributes);
 if (i) goto failed;
 context->parent_channel.next = context_channels; context_channels = &context->parent_channel;
 atomic_fetch_add(&context_records_live, 1);
 m[channel] = context->parent_channel.handle; context->next = context_children; context_children = context; return context->token;
failed:
 context_snapshot_free(context); free(context); close(fd[0]); close(fd[1]); return -1;
}
static Cell context_join(Cell token, int cancel) {
 VmContext **link = &context_children, *context; int result;
 while (*link && (*link)->token != token) link = &(*link)->next;
 if (!*link) return -1; context = *link;
 if (cancel) { atomic_store_explicit(&context->stop, 1, memory_order_release); if (context->parent_channel.fd >= 0) shutdown(context->parent_channel.fd, SHUT_RDWR); }
 if (pthread_join(context->thread, 0)) return -1;
 if (context->parent_channel.fd >= 0) context_channel_close(context->parent_channel.handle); result = context->exit_code; *link = context->next;
 context_snapshot_free(context); free(context); atomic_fetch_sub(&context_records_live, 1); return result;
}
#else
static int context_descriptor(Cell handle) { return -1; }
static int context_channel_close(Cell handle) { return -1; }
static int context_cancelled(void) { return 0; }
static void context_exit(int value) { }
static Cell context_start(Cell entry, Cell channel) { return -1; }
static Cell context_join(Cell token, int cancel) { return -1; }
#endif

int main(int argc, char **argv)
{
	if (argc > 1 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h"))) {
#ifdef USE_LSP_GENOME
		fputs("usage: zclsp [--format|--annotate|--types|--help|-h] < input\n"
		      "reads language-server protocol messages from stdin and writes responses to stdout.\n"
		      "uses content-length framing; plain source on stdin runs a diagnostic check.\n",
		      stdout);
		fputs("source options:\n"
		      "  --format    print formatted source\n"
		      "  --annotate  print source with inferred types\n"
		      "  --types     list inferred type edits as line:column length text\n", stdout);
#else
		fputs("usage: zy [program arguments] < source.zc\n"
		      "compiles source from stdin and runs main; input after a backtick is program input.\n"
		      "program arguments are available through std.argv; stdin has the name <eval>.\n",
		      stdout);
#endif
		fputs("options:\n  --help, -h  show this help and exit\n"
		      "environment:\n"
		      "  TINYVM_MEMORY_WORDS      initial vm memory in words\n"
		      "  TINYVM_MAX_MEMORY_WORDS  maximum vm memory in words\n"
		      "  ZYTRACE                 trace native translation and optimization\n", stdout);
		return 0;
	}
	if (argc > 1 && argv[1][0] == '-' && argv[1][1] == '-') {
#ifdef USE_LSP_GENOME
		if (strcmp(argv[1], "--format") && strcmp(argv[1], "--annotate") && strcmp(argv[1], "--types")) {
#endif
		fprintf(stderr, "tinyvm: unknown option %s; use --help for usage\n", argv[1]);
		return 2;
#ifdef USE_LSP_GENOME
		}
#endif
	}
#ifdef _WIN32
	_setmode(_fileno(stdin), _O_BINARY);
	_setmode(_fileno(stdout), _O_BINARY);
#endif
	zy_args(argc, argv);
	memory_limit = memory_option("TINYVM_MAX_MEMORY_WORDS", DEFAULT_MEMORY_LIMIT, HEAP_HI > PW ? HEAP_HI : PW);
	if (memory_limit < 0) return 1;
	memory_growth_limit = memory_limit;
	memory_words = memory_option("TINYVM_MEMORY_WORDS", HEAP_HI > PW ? HEAP_HI : PW, HEAP_HI > PW ? HEAP_HI : PW);
	if (memory_words < 0) return 1;
	if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; }
	m = calloc((size_t)MW, sizeof *m);
#ifdef ZYNATIVE
	cov = calloc((size_t)MW, 1);
	if (!cov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
#else
	rcov = calloc((size_t)MW, 1);
#ifdef ZYLLVM
	lcov = calloc((size_t)MW, 1);
	if (!lcov) { fprintf(stderr, "zy: cannot reserve translation coverage\n"); return 1; }
#endif
	if (!rcov) { fprintf(stderr, "zy: cannot reserve region coverage\n"); return 1; }
#endif
	if (!m) { fprintf(stderr, "zy: cannot reserve vm memory\n"); return 1; }
	if (code_reserve(PW) < 0) { fprintf(stderr, "zy: cannot reserve code metadata\n"); return 1; }
	trace = getenv("ZYTRACE") != 0;
	if ((unsigned)GORG + (unsigned)GLEN > (unsigned)PW || (unsigned)GORG + (unsigned)GLEN > (unsigned)FBOT) {
		fprintf(stderr, "zy: genome does not fit\n");
		return 1;
	}
	for (int g = 0; g < GLEN; g++) m[GORG + g] = genome_word(g);
	m[ZFP + 6] = FBOT; m[ZFP + 7] = FTOP; m[ZFP + 8] = 1024;
#ifdef ZYJIT
	{
	int p = GORG;
	union { unsigned *u; int (*f)(long *, void *); } enter;
#ifdef __APPLE__
	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
#else
	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#endif
	if (code == MAP_FAILED) { perror("mmap"); return 1; }
	jitwrite(1);
	build_stubs();
	jitwrite(0);
	syncicache(code, cp);
	S[0] = (long)m; S[1] = DS0; S[2] = RS0; S[3] = (long)tab; S[4] = (long)cov;
	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
	S[9] = (long)inst_hook; S[10] = (long)xrun; S[14] = (long)&context_default_stop;
	S[11] = (long)(m + DSLIM); S[12] = (long)(m + RSLIM);
	cov[SYSGATE] = 1;
	enter.u = x_enter;
		while ((unsigned)p < (unsigned)code_words || p == PC_DATA_OVERFLOW || p == PC_RETURN_OVERFLOW) {
			if (p == PC_DATA_OVERFLOW || p == PC_RETURN_OVERFLOW) {
				Cell sp = S[1], rp = S[2]; int ret = p == PC_RETURN_OVERFLOW;
				if (!stack_control || stack_expand(ret, stack_get(ret ? SC_RETURN_TOP : SC_DATA_TOP) - stack_get(ret ? SC_RETURN_LOW : SC_DATA_LOW) + 1, &sp, &rp) < 0) break;
				S[1] = sp; S[2] = rp;
				S[11] = (long)(m + stack_data_limit()); S[12] = (long)(m + stack_return_limit());
				p = (int)S[13]; continue;
			}
		void *c = tab[p];
		if (!c) { jitwrite(1); c = translate(p); jitwrite(0); }
		p = enter.f(S, c);
		if (need_flush) { need_flush = 0; jitwrite(1); flush(); jitwrite(0); }
	}
	return stopped(p);
	}
#elif defined(ZYBIGSTACK)
	{
		/* o3 calls nest on the host stack, one native frame per call: give the machine a
		   stack as deep as its own frame stack can go (untouched pages cost nothing) */
		size_t sz = (size_t)1 << 29;
		void *stk = mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
		if (stk != MAP_FAILED) {
			getcontext(&uc_vm);
			uc_vm.uc_stack.ss_sp = stk; uc_vm.uc_stack.ss_size = sz; uc_vm.uc_link = &uc_main;
			makecontext(&uc_vm, vm_entry, 0);
			swapcontext(&uc_main, &uc_vm);
			return vm_rc;
		}
		return run(GORG);
	}
#else
	return run(GORG);
#endif
	return 0;
}