Merge branch 'main' into work/spec

3 files changed, 234 insertions, 25 deletions

diff --git a/src/core/core.c b/src/core/core.c
index 2a0c467..c4306ae 100644
--- a/src/core/core.c
+++ b/src/core/core.c
@@ -4,8 +4,8 @@
    memory. stores that land on translated words throw those translations away,
    so code that rewrites itself, including the resident compiler, just gets
    retranslated on the next visit. the arm64 jit also compiles o3 regions; on
-   x86-64 they run on the region interpreter, and enhancements are called, not
-   inlined. on other hosts (or with -DZYINTERP) a plain interpreter with the same
+   x86-64 they run on the region interpreter; enhancements are inlined on both
+   (on x86-64 those with float, byte or native-call ops are still called). on other hosts (or with -DZYINTERP) a plain interpreter with the same
    semantics runs instead. ZYTRACE=1 logs translations on stderr. */
 #include <stdio.h>
 #include <stdlib.h>
@@ -633,7 +633,7 @@ static int unew(int op, int a, int b, int c, Cell imm)
 	u = &ub[un++]; u->op = op; u->d = unv++; u->a = a; u->b = b; u->c = c; u->imm = imm;
 	return u->d;
 }
-static Cell mread(int a) { return (unsigned)a < MW ? m[a] : 0; }
+static Cell mread(Cell a) { return (UCell)a < MW ? m[a] : 0; }	/* the whole word: an address past 2^32 reads 0, as on the jits and the microcore */
 
 /* recipe text: one statement per line or per ';'
      r0 = sdiv r0 r1      dst = op operands, operands are rN, parameter names or numbers
@@ -1014,11 +1014,12 @@ static void xdump(int slot, Enh *e)
 	}
 	for (i = 0; i < e->nout; i++) fprintf(stderr, "  out%d = v%d\n", i, e->out[i]);
 }
-/* registers for the jit: values live in x0..x15, straight-line linear scan.
-   if they don't fit, reg stays 0 and the jit calls the interpreter instead. */
-static void regalloc(Enh *e)
+/* registers for the jit: values live in x0..x15 (registers 0..nr-1 of the
+   x86-64 jit's own set there), straight-line linear scan. if they don't fit,
+   reg stays 0 and the jit calls the interpreter instead. */
+static void regalloc_n(Enh *e, int nr)
 {
-	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = 0xffff;
+	int *last = malloc(e->nv * sizeof(int)), i, j, k, fr = (1 << nr) - 1;
 	signed char *r = malloc(e->nv);
 	for (i = 0; i < e->nv; i++) { last[i] = -2; r[i] = -1; }
 	for (i = 0; i < e->n; i++) {
@@ -1028,7 +1029,7 @@ static void regalloc(Enh *e)
 		if (uarity[u->op] > 2) last[u->c] = i;
 	}
 	for (i = 0; i < e->nout; i++) last[e->out[i]] = e->n;
-#define TAKE(v) do { for (k = 0; k < 16 && !(fr >> k & 1); k++); if (k == 16) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
+#define TAKE(v) do { for (k = 0; k < nr && !(fr >> k & 1); k++); if (k == nr) goto fail; fr &= ~(1 << k); r[v] = (signed char)k; } while (0)
 	for (j = 0; j < e->nin; j++) if (last[j] >= -1 && last[j] != -2) TAKE(j);
 	for (i = 0; i < e->n; i++) {
 		Uop *u = &e->u[i]; int ops[3], no = 0;
@@ -1044,6 +1045,7 @@ static void regalloc(Enh *e)
 fail:
 	free(last); free(r); e->reg = 0;
 }
+static void regalloc(Enh *e) { regalloc_n(e, 16); }
 /* the install opcode. descriptor: slot, kind (0 recipe, 1 vm code), inputs,
    outputs, payload address, payload length, parameter names (or 0). kind 2
    is an o3 region (below).
@@ -5901,6 +5903,7 @@ static void rcompile(Rg *R, int eager)
    translated (the c loop sees it first), and the region interpreter runs it. */
 static VM_TLS int vd;
 static VM_TLS int x64_noo3;	/* ZYNOO3=1: leave o3 regions alone and run their vm code through the jit (what the microcore does) */
+static VM_TLS int x64_noenh;	/* ZYNOENH=1: call every enhancement through xrun, as phase 1 did */
 #define SL(k) (8 * (vd + (k)))
 static void ldv(int r, int k) { mov_rm(r, rSP, SL(k)); }
 static void stv(int k, int r) { mov_mr(rSP, SL(k), r); }
@@ -6016,6 +6019,150 @@ static void float_op(int i, int k)
 	(void)i;
 }
 
+/* ---- enhancements, inlined ----
+   an enhancement's optimized micro-ops are lowered to x86-64 instead of being
+   run by xrun: values live in rsi, rdi, r8..r11 and rbp (regalloc_n with seven
+   registers), rax, rcx and rdx are the scratch that division, shifts and setcc
+   need, and constants are made where they are used. like the arm64 jit, a run
+   of literals in front of the opcode specializes the enhancement on them
+   (x64_spec_enh), so the optimizer sees them as constants. an enhancement with
+   an op this lowering lacks (floats, byte loads and stores, native calls), one
+   that does not fit the registers, or one too long still goes through xrun. */
+static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp };
+#define XNREG 7
+#define XMAXOP 400
+static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16];
+static VM_TLS signed char *xr;
+static int x64_op_ok(int op)
+{
+	switch (op) {
+	case U_LI: case U_MOV: case U_ADD: case U_SUB: case U_MUL: case U_SDIV: case U_UDIV: case U_SREM: case U_UREM:
+	case U_AND: case U_OR: case U_XOR: case U_SHL: case U_SHR: case U_SAR: case U_NEG: case U_NOT:
+	case U_LT: case U_LTU: case U_EQ: case U_NE: case U_MIN: case U_MAX: case U_MINU: case U_MAXU:
+	case U_MULHS: case U_MULHU: case U_CLZ: case U_CTZ: case U_BSWAP: case U_ROTL: case U_ROTR:
+	case U_ABS: case U_SEL: case U_LD: case U_ST:
+		return 1;
+	}
+	return 0;
+}
+/* the enhancement in slot s, specialized on nk literals that feed its last inputs
+   (nk may be 0) and given x86-64 registers. 0 if it can be inlined */
+static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te)
+{
+	Enh *e = &enh[s];
+	int nin = e->nin - nk, i, j, out[8], na;
+	static VM_TLS int mp[MAXUOP * 4 + 16];
+	if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1;
+	for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1;
+	un = 0; unv = nin; uerr = 0;
+	for (j = 0; j < nin; j++) mp[j] = j;
+	for (j = 0; j < nk; j++) mp[nin + j] = unew(U_LI, 0, 0, 0, ks[j]);
+	for (i = 0; i < e->n; i++) {
+		Uop *u = &e->u[i];
+		na = nargs(u->op);
+		j = unew(u->op, na > 0 ? mp[u->a] : 0, na > 1 ? mp[u->b] : 0, na > 2 ? mp[u->c] : 0, u->imm);
+		if (u->op != U_ST) mp[u->d] = j;
+	}
+	for (j = 0; j < e->nout; j++) out[j] = mp[e->out[j]];
+	memset(te, 0, sizeof *te);
+	if (uerr || optimize(te, nin, out, e->nout)) return -1;
+	for (i = 0; i < te->n; i++) if (!x64_op_ok(te->u[i].op)) { free(te->u); return -1; }
+	regalloc_n(te, XNREG);
+	if (!te->reg) { free(te->u); return -1; }
+	return 0;
+}
+/* the register value v lives in, or scratch with the constant v loaded into it */
+static int xv(int v, int scratch) { if (xkc[v]) { mov_ri(scratch, xkv[v]); return scratch; } return xreg[(int)xr[v]]; }
+static int xsmall(int v) { return xkc[v] && xkv[v] >= -2147483647 - 1 && xkv[v] <= 2147483647; }
+static void cmov(int cc, int d, int s) { reg_op(0, 1, 1, 0x40 + cc, d, s); }
+static void x64_uop(Uop *u)
+{
+	int op = u->op, d = is_store(op) ? -1 : xreg[(int)xr[u->d]], k;
+	JW *z, *m1, *e1, *e2;
+	switch (op) {
+	case U_MOV: mov_rr(d, xv(u->a, r_ax)); return;
+	case U_ADD: case U_SUB: case U_AND: case U_OR: case U_XOR:
+		k = op == U_ADD ? a_add : op == U_SUB ? a_sub : op == U_AND ? a_and : op == U_OR ? a_or : a_xor;
+		mov_rr(r_ax, xv(u->a, r_ax));
+		if (xsmall(u->b)) alu_ri(k, r_ax, (int)xkv[u->b]); else alu_rr(k, r_ax, xv(u->b, r_cx));
+		mov_rr(d, r_ax); return;
+	case U_MUL:
+		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 1, 0xaf, r_ax, xv(u->b, r_cx)); mov_rr(d, r_ax); return;
+	case U_SHL: case U_SHR: case U_SAR: case U_ROTL: case U_ROTR:
+		k = op == U_SHL ? 4 : op == U_SHR ? 5 : op == U_SAR ? 7 : op == U_ROTL ? 0 : 1;
+		if (xkc[u->b]) { mov_rr(r_ax, xv(u->a, r_ax)); if (xkv[u->b] & 63) shift_ri(k, r_ax, (int)(xkv[u->b] & 63)); }
+		else { mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); shift_cl(k, r_ax); }
+		mov_rr(d, r_ax); return;
+	case U_NEG: case U_NOT:
+		mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_NEG ? 3 : 2, r_ax); mov_rr(d, r_ax); return;
+	case U_LT: case U_LTU: case U_EQ: case U_NE:
+		mov_rr(r_ax, xv(u->a, r_ax)); xor32(r_dx);
+		if (xsmall(u->b)) alu_ri(a_cmp, r_ax, (int)xkv[u->b]); else alu_rr(a_cmp, r_ax, xv(u->b, r_cx));
+		setcc(op == U_LT ? c_l : op == U_LTU ? c_b : op == U_EQ ? c_e : c_ne, r_dx); mov_rr(d, r_dx); return;
+	case U_MIN: case U_MAX: case U_MINU: case U_MAXU:	/* keep a unless the other one wins */
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); alu_rr(a_cmp, r_ax, r_dx);
+		cmov(op == U_MIN ? c_ge : op == U_MAX ? c_le : op == U_MINU ? c_ae : c_be, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_SEL:
+		mov_rr(r_ax, xv(u->c, r_ax)); mov_rr(r_dx, xv(u->b, r_dx)); mov_rr(r_cx, xv(u->a, r_cx));
+		test_rr(r_cx, r_cx); cmov(c_ne, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_ABS:
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_rr(r_dx, r_ax); reg_op(0, 1, 0, 0xf7, 3, r_dx);
+		test_rr(r_ax, r_ax); cmov(c_l, r_ax, r_dx); mov_rr(d, r_ax); return;
+	case U_MULHS: case U_MULHU:
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax)); reg_op(0, 1, 0, 0xf7, op == U_MULHS ? 5 : 4, r_cx);
+		mov_rr(d, r_dx); return;
+	case U_SDIV: case U_SREM:	/* by 0: 0 or a; by -1: -a or 0, which also covers the minimum; else idiv */
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
+		test_rr(r_cx, r_cx); z = j32(c_e);
+		alu_ri(a_cmp, r_cx, -1); m1 = j32(c_e);
+		xb(0x48); xb(0x99); reg_op(0, 1, 0, 0xf7, 7, r_cx);	/* cqo; idiv rcx */
+		if (op == U_SREM) mov_rr(r_ax, r_dx);
+		e1 = jm32();
+		fix32(m1, cp);
+		if (op == U_SDIV) reg_op(0, 1, 0, 0xf7, 3, r_ax); else xor32(r_ax);
+		e2 = jm32();
+		fix32(z, cp);
+		if (op == U_SDIV) xor32(r_ax);
+		fix32(e1, cp); fix32(e2, cp);
+		mov_rr(d, r_ax); return;
+	case U_UDIV: case U_UREM:	/* by 0: 0 or a */
+		mov_rr(r_cx, xv(u->b, r_cx)); mov_rr(r_ax, xv(u->a, r_ax));
+		test_rr(r_cx, r_cx); z = j32(c_e);
+		xor32(r_dx); reg_op(0, 1, 0, 0xf7, 6, r_cx);		/* div rcx */
+		if (op == U_UREM) mov_rr(r_ax, r_dx);
+		e1 = jm32();
+		fix32(z, cp);
+		if (op == U_UDIV) xor32(r_ax);
+		fix32(e1, cp);
+		mov_rr(d, r_ax); return;
+	case U_CLZ:	/* bsr leaves zf set for 0: 127 ^ 63 is 64 */
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 127); reg_op(0, 1, 1, 0xbd, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
+		alu_ri(a_xor, r_dx, 63); mov_rr(d, r_dx); return;
+	case U_CTZ:
+		mov_rr(r_ax, xv(u->a, r_ax)); mov_r32i(r_cx, 64); reg_op(0, 1, 1, 0xbc, r_dx, r_ax); cmov(c_e, r_dx, r_cx);
+		mov_rr(d, r_dx); return;
+	case U_BSWAP:
+		mov_rr(r_ax, xv(u->a, r_ax)); xb(0x48); xb(0x0f); xb(0xc8); mov_rr(d, r_ax); return;
+	case U_LD:	/* out of range reads 0, like the interpreter */
+		mov_rr(r_ax, xv(u->a, r_ax)); emit_load(); mov_rr(d, r_cx); return;
+	case U_ST:	/* outside memory stores nothing; like xexec, no store hook */
+		mov_rr(r_ax, xv(u->a, r_ax)); alu_ri(a_cmp, r_ax, MW); z = j32(c_ae);
+		mov_xr(rMB, r_ax, 8, 0, xv(u->b, r_dx)); fix32(z, cp); return;
+	}
+}
+/* inputs from the vm stack into their registers, the ops, the outputs back */
+static void x64_enh(Enh *e)
+{
+	int i, j;
+	xr = e->reg;
+	for (i = 0; i < e->nv; i++) xkc[i] = 0;
+	for (i = 0; i < e->n; i++) if (e->u[i].op == U_LI) { xkc[e->u[i].d] = 1; xkv[e->u[i].d] = e->u[i].imm; }
+	for (j = 0; j < e->nin; j++) if (xr[j] >= 0) ldv(xreg[(int)xr[j]], e->nin - 1 - j);
+	for (i = 0; i < e->n; i++) if (e->u[i].op != U_LI) x64_uop(&e->u[i]);
+	vd += e->nin - e->nout;
+	for (j = 0; j < e->nout; j++) stv(e->nout - 1 - j, xv(e->out[j], r_ax));
+}
+
 static void *translate(int p)
 {
 	JW *start;
@@ -6045,6 +6192,25 @@ static void *translate(int p)
 			i += 2;
 			break;
 		}
+		if (w & 1 && cp - start < (1 << 18) && !x64_noenh) {	/* literals feeding an enhancement: specialize it on them */
+			int L = 0, s, nk, j;
+			Cell ks[8], x;
+			Enh te;
+			while (L < 8 && i + L < code_words && (m[i + L] & 1)) L++;
+			x = i + L < code_words ? m[i + L] : 1;
+			if (!(x & 1) && (s = (int)(x / 2) - XOP0) >= 0 && s < NENH && enh[s].live && enh[s].nin) {
+				nk = L < enh[s].nin ? L : enh[s].nin;
+				for (j = 0; j < nk; j++) ks[j] = m[i + L - nk + j] >> 1;
+				if (!x64_spec_enh(s, ks, nk, &te)) {
+					for (j = 0; j < L - nk; j++) { Cell v = m[i + j] >> 1; vd--; if (v >= -2147483647 - 1 && v <= 2147483647) mov_mi(rSP, SL(0), (int)v); else { mov_ri(r_ax, v); stv(0, r_ax); } }
+					if (nused < 64) used[nused++] = s; else used[0] = -1;
+					x64_enh(&te);
+					free(te.u); free(te.reg);
+					i += L + 1; n += L;
+					continue;
+				}
+			}
+		}
 		if (w & 1) {
 			Cell v = (w - 1) / 2;
 			int small = v >= -2147483647 - 1 && v <= 2147483647, ko = -1;
@@ -6128,6 +6294,10 @@ static void *translate(int p)
 		}
 		if (k >= XOP0 && k < XOP0 + NENH) {
 			if (nused < 64) used[nused++] = k - XOP0; else used[0] = -1;	/* wired or not, an install makes this a different block */
+			if (enh[k - XOP0].live && cp - start < (1 << 18) && !x64_noenh) {	/* inline */
+				Enh te;
+				if (!x64_spec_enh(k - XOP0, 0, 0, &te)) { x64_enh(&te); free(te.u); free(te.reg); i++; continue; }
+			}
 			if (enh[k - XOP0].live) {	/* the interpreter runs it: x0 = data stack, slot */
 				lea(r_di, rSP, -1, 1, 8 * vd); mov_r32i(r_si, (unsigned)(k - XOP0));
 				call_m(rST, 80);
@@ -6245,6 +6415,7 @@ static int x64_boot(Cell sp, Cell rp, void *stop)
 	code = mmap(0, CODEW * 4, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 	if (code == MAP_FAILED) { perror("mmap"); return 1; }
 	x64_noo3 = getenv("ZYNOO3") != 0;
+	x64_noenh = getenv("ZYNOENH") != 0;
 	jitwrite(1); build_stubs(); jitwrite(0); syncicache(code, cp);
 	S[0] = (long)m; S[1] = sp; S[2] = rp; S[3] = (long)tab; S[4] = (long)cov;
 	S[5] = (long)Q; S[6] = (long)G; S[7] = (long)&J; S[8] = (long)hook;
diff --git a/src/runtime/rt.zc b/src/runtime/rt.zc
index a181387..44f3570 100644
--- a/src/runtime/rt.zc
+++ b/src/runtime/rt.zc
@@ -87,6 +87,10 @@ enhance ishl(a, b) micro "a = shl a b";
 enhance lfa(k) micro "r1 = ld 589824; k = add r1 k";
 // 1 when i is outside 0..n-1 (the unsigned compare also catches negative i), else 0.
 enhance outside(i, n) micro "i = ltu i n; i = xor i 1";
+//: the block length has exactly two definitions, which must agree: the r7 lines of this
+//: recipe (the inline check on every backend; the x86-64 and arm64 jits lower it to
+//: registers, with nothing pointer-specific of their own) and paylen below (bchk's report).
+//: an exact-size check or a new header layout changes those two places and nothing else.
 // the index check of a pointer, as one instruction: 1 when p is the start of a heap
 // block (its check word p[-3] is p ^ MAGIC) and the byte or word offset i * m is not
 // inside the block's payload (q units per word: 1 for word elements, 8 for packed
diff --git a/src/tools/zc0.c b/src/tools/zc0.c
index af85ff8..664ea79 100644
--- a/src/tools/zc0.c
+++ b/src/tools/zc0.c
@@ -34,7 +34,19 @@ static char module_name[64];
 static char *src; static int pos, line = 1, tstart, hdr_start;
 /* the signature of the function being defined: its header's source text, for the language server (ZC0DOCS) */
 static char sgtext[400], sgname[80];
-static int tk, glue; static int64_t tv; static char ts[4096];	/* current token */
+static int tk, glue; static int64_t tv;	/* current token */
+/* the token's text. it grows as a long string literal or name needs; there is no fixed limit */
+static char ts0[4096]; static char *ts = ts0; static size_t tscap = sizeof ts0;
+static void tsroom(size_t n)
+{
+	size_t c = tscap; char *nb;
+	if (n < tscap) return;
+	while (c <= n) c *= 2;
+	nb = ts == ts0 ? malloc(c) : realloc(ts, c);
+	if (!nb) { fprintf(stderr, "zc0: out of memory for a token\n"); exit(1); }
+	if (ts == ts0) memcpy(nb, ts0, sizeof ts0);
+	ts = nb; tscap = c;
+}
 static int xf, xs, lvi;	/* float, byte string; the plain local it came from, or -1 */
 
 static int64_t *img; static int imgcap, here = GORG;
@@ -133,6 +145,24 @@ static void docblock(void)
 	while (dn && (dbuf[dn - 1] == ' ' || dbuf[dn - 1] == '\t' || dbuf[dn - 1] == '\r' || dbuf[dn - 1] == '\n')) dn--;
 	dbuf[dn++] = '\n'; dlast = line; docpending = 1;
 }
+/* an integer literal at pos, as the compiler reads it: decimal digits (a leading 0 is
+   not octal: 010 is 10), or 0x and hex digits (0x alone is 0). the value wraps modulo
+   2^64 instead of saturating. the token text is the literal's source. */
+static void intlit(int base)
+{
+	int p = pos, d; uint64_t u = 0;
+	if (base == 16) pos += 2;
+	for (;;) {
+		int c = (unsigned char)src[pos];
+		if (c >= '0' && c <= '9') d = c - '0';
+		else if (base == 16 && c >= 'a' && c <= 'f') d = c - 'a' + 10;
+		else if (base == 16 && c >= 'A' && c <= 'F') d = c - 'A' + 10;
+		else break;
+		u = u * (uint64_t)base + (uint64_t)d; pos++;
+	}
+	tsroom((size_t)(pos - p) + 1); memcpy(ts, src + p, (size_t)(pos - p)); ts[pos - p] = 0;
+	tv = (int64_t)u; tk = T_NUM;
+}
 static void next(void)
 {
 	int ws = 0, i;
@@ -153,7 +183,7 @@ static void next(void)
 	if (isdigit((unsigned char)src[pos]) || (src[pos] == '.' && isdigit((unsigned char)src[pos + 1]))) {
 		int p = pos, fl = 0;
 		if (src[pos] == '0' && (src[pos + 1] == 'x' || src[pos + 1] == 'X')) {
-			char *e; tv = (int64_t)strtoll(src + pos, &e, 0); memcpy(ts, src + pos, e - src - pos); ts[e - src - pos] = 0; pos = e - src; tk = T_NUM; return;
+			intlit(16); return;
 		}
 		if (src[p] == '.') {
 			int nch = (unsigned char)src[p + 1];
@@ -186,22 +216,21 @@ static void next(void)
 			}
 			b[n] = 0; memcpy(ts, b, (size_t)n + 1); tv = (int64_t)fop_parse(b); tk = T_FLT; return;
 		}
-		{ char *e; tv = (int64_t)strtoll(src + pos, &e, 0); memcpy(ts, src + pos, e - src - pos); ts[e - src - pos] = 0; pos = e - src; tk = T_NUM; return; }
+		intlit(10); return;
 	}
 	if (isalpha((unsigned char)src[pos]) || src[pos] == '_') {
-		i = 0; while (isalnum((unsigned char)src[pos]) || src[pos] == '_') ts[i++] = src[pos++];
+		i = 0; while (isalnum((unsigned char)src[pos]) || src[pos] == '_') { tsroom((size_t)i + 2); ts[i++] = src[pos++]; }
 		ts[i] = 0;
 		if (i == 1 && ts[0] == 'b' && src[pos] == '"') {
 			pos++; i = 0;
 			while (src[pos] && src[pos] != '"') {
-				if (i >= 4095) { die("a byte string literal is at most 4095 bytes; embed larger data"); break; }
-				ts[i++] = (char)esc();
+				tsroom((size_t)i + 2); ts[i++] = (char)esc();
 			}
 			ts[i] = 0; if (src[pos] == '"') pos++; tk = T_BSTR; return;
 		}
         if (src[pos] == '.' && (!strcmp(ts,"rt") || !strcmp(ts,"run") || !strcmp(ts,"core") || !strcmp(ts,"lsp") || !strcmp(ts,"std") || !strcmp(ts,"icon"))) {
-            ts[i++] = src[pos++];
-            while (isalnum((unsigned char)src[pos]) || src[pos] == '_') ts[i++] = src[pos++];
+            tsroom((size_t)i + 2); ts[i++] = src[pos++];
+            while (isalnum((unsigned char)src[pos]) || src[pos] == '_') { tsroom((size_t)i + 2); ts[i++] = src[pos++]; }
             ts[i] = 0;
         }
 		tk = T_ID; return;
@@ -210,8 +239,7 @@ static void next(void)
 	if (src[pos] == '"') {
 		pos++; i = 0;
 		while (src[pos] && src[pos] != '"') {
-			if (i >= 4095) { die("a string literal is at most 4095 bytes"); break; }
-			ts[i++] = (char)esc();
+			tsroom((size_t)i + 2); ts[i++] = (char)esc();
 		}
 		ts[i] = 0; pos++; tk = T_STR; return;
 	}
@@ -220,9 +248,10 @@ static void next(void)
 	for (i = 0; ops2[i]; i++) if (!strncmp(src + pos, ops2[i], 2)) { strcpy(ts, ops2[i]); pos += 2; return; }
 	ts[0] = src[pos++]; ts[1] = 0;
 }
-typedef struct { char *src; int pos, line, tk, glue; int64_t tv; char ts[4096]; int xf, xs; } Lex;
-static void lexsave(Lex *L) { L->src = src; L->pos = pos; L->line = line; L->tk = tk; L->glue = glue; L->tv = tv; memcpy(L->ts, ts, sizeof ts); L->xf = xf; L->xs = xs; }
-static void lexrest(Lex *L) { src = L->src; pos = L->pos; line = L->line; tk = L->tk; glue = L->glue; tv = L->tv; memcpy(ts, L->ts, sizeof ts); xf = L->xf; xs = L->xs; }
+/* a saved lexer position. ts is a copy of the token text (malloc), released by lexrest */
+typedef struct { char *src; int pos, line, tk, glue; int64_t tv; char *ts; size_t tn; int xf, xs; } Lex;
+static void lexsave(Lex *L) { L->src = src; L->pos = pos; L->line = line; L->tk = tk; L->glue = glue; L->tv = tv; L->tn = strlen(ts) + 1; L->ts = malloc(L->tn); if (!L->ts) { fprintf(stderr, "zc0: out of memory\n"); exit(1); } memcpy(L->ts, ts, L->tn); L->xf = xf; L->xs = xs; }
+static void lexrest(Lex *L) { src = L->src; pos = L->pos; line = L->line; tk = L->tk; glue = L->glue; tv = L->tv; tsroom(L->tn); memcpy(ts, L->ts, L->tn); free(L->ts); L->ts = 0; xf = L->xf; xs = L->xs; }
 static int is(const char *s) { return (tk == T_OP || tk == T_ID) && !strcmp(ts, s); }
 static int accept(const char *s) { if (is(s)) { next(); return 1; } return 0; }
 static void expect(const char *s) { if (!accept(s)) { char b[64]; sprintf(b, "expected '%s'", s); die(b); } }
@@ -626,7 +655,7 @@ static void postfix(void)
 		else if (is("(")) { rval(); n = args(); lit(n); op(O_PICK); op(O_CALL); op(O_SWAP); op(O_DROP); xf = 0; xs = 0; lvi = -1; }
 		else if (accept("++")) incdec(1, 1);
 		else if (accept("--")) incdec(1, -1);
-		else if (accept("as")) ident();
+		else if (accept("as")) ptype0();	/* the whole type, as the compiler reads it: x as int*, x as char** */
 		else return;
 	}
 }
@@ -820,7 +849,10 @@ static void fndef(int ret, int rstr)	/* rstr: the return type is bstr */
 	if (!accept(")")) {
 		do {
 			t = 0; ptag = 0; pstr[np & 15] = 0;
-			if (optype() >= 0) { t = ptag == 1 ? 1 : ptag == 2 ? 2 : 0; pstr[np & 15] = ptstr; }	/* the type is read and dropped */
+			if (optype() >= 0) {	/* the type is read and dropped */
+				if (ptag == 3) die("a parameter tag is .m or .d");	/* the compiler's message: .u is only a return tag */
+				t = ptag == 1 ? 1 : ptag == 2 ? 2 : 0; pstr[np & 15] = ptstr;
+			}
 			else if (accept(".")) {	/* .m name, .d name */
 				if (is("m")) t = 1; else if (is("d")) t = 2; else die("a parameter tag is .m or .d");
 				next();
@@ -980,12 +1012,13 @@ static void embeddecl(int local)
 		}
 	} else {
 		int count; w = embed_width(ty, star);
-		if (local) die("zc0 embeds an array at the top level, not in a function");
 		if (!w) die("embed declares a bstr, a char*, a number or an array of numbers");
 		if (len % w) die("the embedded file's size is not a whole number of elements");
 		count = (int)(len / w);
 		if (arr >= 0 && count > arr) die("the embedded file is larger than the array it is declared as");
 		if (arr >= 0) count = arr;
+		/* in a function the elements sit in the code, jumped over, and the local holds their address */
+		if (local) { j = here; lit(0); op(O_JMP); }
 		a = here;
 		for (i = 0; i < count; i++) {
 			uint64_t u = 0; int k;
@@ -995,7 +1028,8 @@ static void embeddecl(int local)
 			emit((int64_t)u);
 		}
 		if (!count) emit(0);
-		add(n, K_GARR, a);
+		if (local) { setlit(j, here); l = newloc(n); lit(a); laddr(l->slot); op(O_ST); }
+		else add(n, K_GARR, a);
 	}
 	free(buf);
 }