embed: tests, editor support and docs; js vm file access; test fixes

embed declarations get accept and reject fixtures, a run-only image check
through zc0, an lsp keyword hover and the grammar keyword. errors point at
the declaration and still declare the name, so its uses are not reported
again. zc0 tracks byte-string globals and lays embedded arrays out one word
per element, the way its own indexing reads them.

the command-line javascript vm reaches node's file system for the file
syscalls, so include and embed work there; other embedders opt in.

the web build is self-contained in build/web with its examples. README,
BORROW and BOOTSTRAP-OWNERSHIP describe the new build and ownership rules,
and ONBOARD.md is a reading and verification order for the code.

the suite passes the map header to mklayout, the wasm fuzz program uses
the include format, and lsp drivers scale their deadlines on hosts
without the arm64 jit (ZC_TEST_TIMEOUT_SCALE). LLVM_CONFIG selects an
llvm with headers for the llvm section.

6 files changed, 64 insertions, 26 deletions

diff --git a/src/compiler/compiler.zc b/src/compiler/compiler.zc
index 244efc5..c6e1ca4 100644
--- a/src/compiler/compiler.zc
+++ b/src/compiler/compiler.zc
@@ -4737,7 +4737,7 @@ int kw_embed() {
 	off = toff; n = name();
 	expect(s_semi);
 	data = m*embed_open(path);
-	if (!data) { int sv = tpack(); tunpack(pp); serr("can't read the file named by embed"); tunpack(sv); data = chk_resize(0, 0, 1); }
+	if (!data) { embed_fail(pp, "can't read the file named by embed"); data = chk_resize(0, 0, 1); }
 	sz = data[0];
 	if (!t) t = tbstr;
 	// in a function the data sits in the code stream, so the code jumps over it
@@ -4750,10 +4750,10 @@ int kw_embed() {
 	}
 	if (t[0] == TY_ARR) {
 		w = embed_width(t[4]);
-		if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); if (skip >= 0) setlit(skip, here); return 0; }
-		if (sz % w) { serr("the embedded file's size is not a whole number of array elements"); if (skip >= 0) setlit(skip, here); return 0; }
+		if (!w) { embed_fail(pp, "embed declares a bstr, a char*, a number, or an array of numbers"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); }
+		if (sz % w) { embed_fail(pp, "the embedded file's size is not a whole number of array elements"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); }
 		count = sz / w;
-		if (t[3] >= 0 && count > t[3]) { serr("the embedded file is larger than the array it is declared as"); if (skip >= 0) setlit(skip, here); return 0; }
+		if (t[3] >= 0 && count > t[3]) { embed_fail(pp, "the embedded file is larger than the array it is declared as"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); }
 		if (t[3] < 0) t = tarr(t[4], count);
 		k = tsz(t); reserve_code(k); a = here; i = 0;
 		if (t[4][0] == TY_CHAR) { while (i < k) { if (i < sz) emit(data[1 + i]); else emit(0); i++; } }
@@ -4768,15 +4768,22 @@ int kw_embed() {
 		return embed_bind(n, off, t, a, 1);
 	}
 	w = embed_width(t);
-	if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); return 0; }
-	if (sz > w) { serr("the embedded file is larger than the type it is declared as"); return 0; }
-	if (isfl(t) && sz != w) { serr("an embedded float takes exactly as many bytes as its type"); return 0; }
+	if (!w) { embed_fail(pp, "embed declares a bstr, a char*, a number, or an array of numbers"); return embed_bind(n, off, tword, 0, 2); }
+	if (sz > w) { embed_fail(pp, "the embedded file is larger than the type it is declared as"); return embed_bind(n, off, tword, 0, 2); }
+	if (isfl(t) && sz != w) { embed_fail(pp, "an embedded float takes exactly as many bytes as its type"); return embed_bind(n, off, tword, 0, 2); }
 	v = 0; i = 0;
 	while (i < sz) { v = v | (data[1 + i] << (i * 8)); i++; }
 	if (isfl(t) && w == 4) v = rt.s2f(v);
 	else v = cnorm(t, v);
 	return embed_bind(n, off, t, v, 2);
 }
+// report msg at the embed declaration that starts at the token packed in pp. the
+// caller still declares the name, as a plain 0, so its uses are not reported too.
+int embed_fail(int pp, char* msg) {
+	int sv = tpack();
+	tunpack(pp); serr(msg); tunpack(sv);
+	return 0;
+}
 // the bytes one element of type t takes in an embedded file: the packed size of a
 // sized type or f32, 1 for char, 8 for the other number types, 0 when t is not a number.
 int embed_width(word t) {
diff --git a/src/js/mklayout.js b/src/js/mklayout.js
index cf95091..8e41f41 100644
--- a/src/js/mklayout.js
+++ b/src/js/mklayout.js
@@ -38,8 +38,8 @@ for (let i = 0; i < args.length; i += 2) {
 
 const mapPath = opt('--map') || path.join(root, 'build', 'core', 'map', 'memmap.h');
 const outDir = opt('--out') || path.join(root, 'build', 'js');
-if (!fs.existsSync(mapPath)) {
-    console.error('mklayout: no memory map at ' + mapPath + '; run `make` first');
+if (!fs.existsSync(mapPath) || !fs.statSync(mapPath).isFile()) {
+    console.error('mklayout: no memory map header at ' + mapPath + '; run `make` first');
     process.exit(1);
 }
 fs.mkdirSync(outDir, { recursive: true });
diff --git a/src/js/tinyvm.ts b/src/js/tinyvm.ts
index 56fab70..f779487 100644
--- a/src/js/tinyvm.ts
+++ b/src/js/tinyvm.ts
@@ -135,6 +135,10 @@ export interface VMOptions {
         buffered suspends the program instead of reading the end. resume() goes
         on from there. eof() is true only once close() has been called. */
     streaming?: boolean;
+    /** files: the file syscalls (open, read, write, close) reach the host's file
+        system through node's fs. off by default, so a vm inside the editor cannot
+        touch files; the command line turns it on, like the native vm. */
+    files?: boolean;
     input?: string | Uint8Array;
     onOutput?: (byte: number) => void;
     onMessage?: (msg: string) => void;
@@ -181,6 +185,7 @@ export class TinyVM {
     private enh: Enhancement[] = [];
     public L: Layout;
     private streaming: boolean = false;
+    private files: boolean = false;
     private closed: boolean = false;
     private blocked: boolean = false;
     private rp: number = 0;
@@ -201,6 +206,7 @@ export class TinyVM {
             throw new RangeError('invalid initial memory or growth limit');
         }
         this.streaming = !!options.streaming;
+        this.files = !!options.files && !!fs;
         const buffer = new ArrayBuffer(this.mw * 8);
         this.mem = new BigInt64Array(buffer);
         this.bytes = new Uint8Array(buffer);
@@ -396,15 +402,33 @@ export class TinyVM {
     private syscall(op: bigint, a: bigint, b: bigint, c: bigint): bigint {
         const opNum = Number(op);
         switch (opNum) {
-            case 1: // open
-                return -1n;
+            case 1: { // open(path, 0 read or 1 create/truncate for write): path is one character per word
+                if (!this.files) return -1n;
+                let path = '';
+                for (let at = Number(a); at >= 0 && at < this.mw && this.mem[at] !== 0n && path.length < 4096; at++) path += String.fromCharCode(Number(this.mem[at] & 0xFFn));
+                try { return BigInt(fs.openSync(path, b === 1n ? 'w' : 'r')); } catch { return -1n; }
+            }
             case 2: // close
+                if (this.files && a > 2n) { try { fs.closeSync(Number(a)); } catch { return -1n; } }
                 return 0n;
-            case 3: // read
-                return 0n;
+            case 3: { // read(fd, buf, n): bytes stored one per word
+                if (!this.files || a <= 2n) return 0n;
+                const n = Number(c), at = Number(b);
+                if (n <= 0 || at < 0 || at > this.mw - n) return -1n;
+                const chunk = Buffer.alloc(n);
+                let got: number;
+                try { got = fs.readSync(Number(a), chunk, 0, n, null); } catch { return -1n; }
+                for (let i = 0; i < got; i++) this.mem[at + i] = BigInt(chunk[i]);
+                return BigInt(got);
+            }
             case 4: { // write(fd, buf, n)
                 const n = Number(c);
                 const bufAddr = Number(b);
+                if (this.files && a > 2n) {
+                    const chunk = Buffer.alloc(n);
+                    for (let i = 0; i < n; i++) chunk[i] = Number(this.mem[bufAddr + i] & 0xFFn);
+                    try { return BigInt(fs.writeSync(Number(a), chunk)); } catch { return -1n; }
+                }
                 for (let i = 0; i < n; i++) {
                     const ch = Number(this.mem[bufAddr + i] & 0xFFn);
                     this.put_c(ch);
@@ -974,6 +998,7 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma
         return Number(value);
     };
     const vm = new TinyVM({
+        files: true,
         memoryWords: environmentWords('TINYVM_MEMORY_WORDS'),
         maxMemoryWords: environmentWords('TINYVM_MAX_MEMORY_WORDS'),
         onOutput: (byte: number) => process.stdout.write(Buffer.from([byte])),
diff --git a/src/lsp/server.zc b/src/lsp/server.zc
index 48fdf90..b578bd1 100644
--- a/src/lsp/server.zc
+++ b/src/lsp/server.zc
@@ -1676,6 +1676,7 @@ char* kwtext() {
 	if (hname_is("struct")) return "struct name { fields };~a record type. struct name<T> { } makes it generic, one copy for each set of type arguments.";
 	if (hname_is("type")) return "type name = T;~an alias for a type, such as type row = int[3]; for a pointer to an array.";
 	if (hname_is("include")) return "include \"path\"; / include std;~compiles a file or the embedded standard library at that point, its signatures first.";
+	if (hname_is("embed")) return "embed \"path\" as [type] name;~declares name with the bytes of the file at path, read when the unit is compiled. with no type, or bstr, name is a byte string; char* holds one character per word; a number type reads the bytes little-endian; T[] and T[n] lay them out as array elements.";
 	if (hname_is("memdebug")) return "memdebug on; / memdebug off; / memdebug scrub;~header checks, allocation sites and leak reports for the rest of the file. scrub also fills a freed payload with 0xDEAD.";
 	if (hname_is("typecheck")) return "typecheck on; / typecheck off;~turns on type checks for the code that follows, enums included.";
 	if (hname_is("hints")) return "hints on; / hints small; / hints off;~controls subsequent hint diagnostics: informational ranges by default, compact markers with small, or no hints with off.";
diff --git a/src/tools/zc0.c b/src/tools/zc0.c
index 5f5b97d..b0ab29f 100644
--- a/src/tools/zc0.c
+++ b/src/tools/zc0.c
@@ -202,7 +202,7 @@ static void expect(const char *s) { if (!accept(s)) { char b[64]; sprintf(b, "ex
 static char *ident(void) { static char b[8][256]; static int k; char *r = b[k++ & 7]; if (tk != T_ID) die("expected a name"); strcpy(r, ts); next(); return r; }
 
 /* ---- symbols ---- */
-typedef struct { char name[64]; int kind, sig, defd; int64_t val; } Sym;
+typedef struct { char name[64]; int kind, sig, defd, str; int64_t val; } Sym;	/* str: a global holding a byte string */
 static Sym sy[16384]; static int nsy;
 typedef struct { int at, sym; } Fix;
 static Fix fx[65536]; static int nfx;
@@ -225,7 +225,7 @@ static Sym *add(const char *n, int kind, int64_t val)
 	char full[256]; qualify(full, sizeof full, n); n = full;
 	Sym *s = find_raw(n);
 	if (s && s->kind == K_FN && kind == K_FN && !s->defd) { s->val = val; return s; }	/* prototype or forward use */
-	s = &sy[nsy++]; strncpy(s->name, n, 63); s->kind = kind; s->val = val; s->defd = 1; s->sig = 0; return s;
+	s = &sy[nsy++]; strncpy(s->name, n, 63); s->kind = kind; s->val = val; s->defd = 1; s->sig = 0; s->str = 0; return s;
 }
 static void fnref(const char *n)	/* push a function address, fixing it up later if needed */
 {
@@ -488,7 +488,7 @@ static void primary(void)
 	if ((l = findl(n))) { laddr(l->slot); lv = !l->arr; if (!l->arr) { xf = l->flt; xs = l->str; lvi = (int)(l - lo); } return; }
 	s = find(n);
 	if (s && s->kind == K_CONST) { lit(s->val); return; }
-	if (s && s->kind == K_GVAR) { lit(s->val); lv = 1; return; }
+	if (s && s->kind == K_GVAR) { lit(s->val); lv = 1; xs = s->str; return; }
 	if (s && s->kind == K_GARR) { lit(s->val); return; }
 	if (is("(")) {
 		int na = args();
@@ -799,8 +799,11 @@ static void enhdef(void)
 /* embed "path" as [type] name; at the top level: a global holding the file's bytes,
    read when the image is built. the path is relative to the working directory.
    bstr (the default) packs the bytes; char* keeps one character to a word; a
-   number takes them little-endian; T[] and T[n] lay them out as the array's
-   elements. the same rules as the resident compiler's kw_embed. */
+   number takes them little-endian; T[] and T[n] have one element per file chunk
+   of T's size. the checks are the resident compiler's (kw_embed). zc0 drops
+   types, so its arrays are one word per element whatever T is: an element is
+   stored as the value it reads as (sign extended for i8, i16 and i32, widened
+   for f32), and zc0 code indexing the array sees the same values. */
 static int embed_width(const char *t, int star)
 {
 	if (star) return 0;
@@ -832,7 +835,7 @@ static void embeddecl(void)
 	if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) die("can't read the file named by embed");
 	fclose(f);
 	if (arr == -2 && (!ty[0] || (!strcmp(ty, "bstr") && !star))) {
-		a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; emit(len);
+		a = here; add(n, K_GVAR, a)->str = 1; emit(0); img[a - GORG] = here; emit(len);
 		for (i = 0; i < len; i += 8) { uint64_t u = 0; for (w = 0; w < 8 && i + w < len; w++) u |= (uint64_t)buf[i + w] << (w * 8); emit((int64_t)u); }
 		if (!(len & 7)) emit(0);
 	} else if (arr == -2 && !strcmp(ty, "char") && star == 1) {
@@ -857,12 +860,14 @@ static void embeddecl(void)
 		if (arr >= 0 && count > arr) die("the embedded file is larger than the array it is declared as");
 		if (arr >= 0) count = arr;
 		a = here;
-		if (!strcmp(ty, "char")) { for (i = 0; i < count; i++) emit(i < len ? buf[i] : 0); }
-		else {
-			long bytes = (long)count * w;
-			for (i = 0; i < bytes; i += 8) { uint64_t u = 0; for (int k = 0; k < 8 && i + k < bytes; k++) u |= (uint64_t)(i + k < len ? buf[i + k] : 0) << (k * 8); emit((int64_t)u); }
-			if (!bytes) emit(0);
+		for (i = 0; i < count; i++) {
+			uint64_t u = 0; int k;
+			for (k = 0; k < w && (long)i * w + k < len; k++) u |= (uint64_t)buf[(long)i * w + k] << (k * 8);
+			if (ty[0] == 'i' && w < 8 && (u >> (w * 8 - 1) & 1)) u |= ~(uint64_t)0 << (w * 8);
+			if (!strcmp(ty, "f32")) u = fop_s2f(u);
+			emit((int64_t)u);
 		}
+		if (!count) emit(0);
 		add(n, K_GARR, a);
 	}
 	free(buf);
diff --git a/src/web/index.html b/src/web/index.html
index 4cf7476..fbc203d 100644
--- a/src/web/index.html
+++ b/src/web/index.html
@@ -41,7 +41,7 @@
 	let machine = null;
 	for (const n of examples) { const o = document.createElement('option'); o.value = n; o.textContent = n + '.zc'; $('ex').appendChild(o); }
 	$('ex').onchange = async () => {
-		try { $('src').value = await (await fetch('../examples/' + $('ex').value + '.zc')).text(); }
+		try { $('src').value = await (await fetch('examples/' + $('ex').value + '.zc')).text(); }
 		catch (e) { $('out').textContent = 'the examples folder is not served next to this page'; }
 	};
 	$('run').onclick = async () => {
@@ -55,6 +55,6 @@
 		} catch (e) { $('out').textContent = String(e); $('status').textContent = 'failed'; }
 	};
 	ZC.load('zc.wasm').then(m => { machine = m; $('status').textContent = 'ready'; })
-		.catch(e => { $('status').textContent = 'could not load zc.wasm; run src/web/build.sh and serve this folder over http'; });
+		.catch(e => { $('status').textContent = 'could not load zc.wasm; run make web and serve build/web over http'; });
 </script>
 </html>