embed: tests, editor support and docs; js vm file access; test fixes

embed declarations get accept and reject fixtures, a run-only image check
through zc0, an lsp keyword hover and the grammar keyword. errors point at
the declaration and still declare the name, so its uses are not reported
again. zc0 tracks byte-string globals and lays embedded arrays out one word
per element, the way its own indexing reads them.

the command-line javascript vm reaches node's file system for the file
syscalls, so include and embed work there; other embedders opt in.

the web build is self-contained in build/web with its examples. README,
BORROW and BOOTSTRAP-OWNERSHIP describe the new build and ownership rules,
and ONBOARD.md is a reading and verification order for the code.

the suite passes the map header to mklayout, the wasm fuzz program uses
the include format, and lsp drivers scale their deadlines on hosts
without the arm64 jit (ZC_TEST_TIMEOUT_SCALE). LLVM_CONFIG selects an
llvm with headers for the llvm section.

17 files changed, 111 insertions, 10 deletions

diff --git a/tests/data/embed/answer.bin b/tests/data/embed/answer.bin
new file mode 100644
index 0000000..447f371
--- /dev/null
+++ b/tests/data/embed/answer.bin
diff --git a/tests/data/embed/bytes.bin b/tests/data/embed/bytes.bin
new file mode 100644
index 0000000..8207d4f
--- /dev/null
+++ b/tests/data/embed/bytes.bin
@@ -0,0 +1 @@
+
ÿ
\ No newline at end of file
diff --git a/tests/data/embed/nul.bin b/tests/data/embed/nul.bin
new file mode 100644
index 0000000..20b5be9
--- /dev/null
+++ b/tests/data/embed/nul.bin
diff --git a/tests/data/embed/onehalf.f32 b/tests/data/embed/onehalf.f32
new file mode 100644
index 0000000..dfbda32
--- /dev/null
+++ b/tests/data/embed/onehalf.f32
diff --git a/tests/data/embed/shorts.bin b/tests/data/embed/shorts.bin
new file mode 100644
index 0000000..6007a59
--- /dev/null
+++ b/tests/data/embed/shorts.bin
diff --git a/tests/data/embed/text.txt b/tests/data/embed/text.txt
new file mode 100644
index 0000000..9db7df0
--- /dev/null
+++ b/tests/data/embed/text.txt
@@ -0,0 +1,2 @@
+hello
+world
\ No newline at end of file
diff --git a/tests/drivers/fuzz_wasm.js b/tests/drivers/fuzz_wasm.js
index 22fa7d5..1d256fe 100644
--- a/tests/drivers/fuzz_wasm.js
+++ b/tests/drivers/fuzz_wasm.js
@@ -13,22 +13,22 @@ function expr(depth) {
 function quote(s) { return '"' + s + '"'; }
 
 (async () => {
-	const lines = ['include std;', 'fn churn(n) { int i = 0; while (i < n) { { int* p = new(int); *p = i; } i++; } return 0; }', 'fn stable() { return 73; }', 'fn main() {', ' let base = live_objects();'];
+	const lines = ['namespace;', 'include rt, std;', 'fn churn(n) { int i = 0; while (i < n) { { int* p = rt.new(int); *p = i; } i++; } return 0; }', 'fn stable() { return 73; }', 'fn main() {', ' let base = rt.live_objects();'];
 	const expected = [];
 	for (let i = 0; i < 128; i++) {
 		const [source, value] = expr(4);
-		lines.push(` put_n(${source}); put_c(32);`); expected.push(value);
+		lines.push(` rt.put_n(${source}); rt.put_c(32);`); expected.push(value);
 	}
 	for (let i = 0; i < 32; i++) {
 		const a = Array.from({ length: next() % 11 }, () => 'abc'[next() % 3]).join('');
 		const b = Array.from({ length: next() % 6 }, () => 'abc'[next() % 3]).join('');
-		lines.push(` put_n(Text.len(${quote(a)})); put_c(32); put_n(Text.eq(${quote(a)}, ${quote(b)})); put_c(32);`);
-		lines.push(` put_n(Text.starts(${quote(a)}, ${quote(b)})); put_c(32); put_n(Text.ends(${quote(a)}, ${quote(b)})); put_c(32); put_n(Text.find(${quote(a)}, ${quote(b)})); put_c(32);`);
-		lines.push(` put_n(Bytes.len(b${quote(a)})); put_c(32); put_n(Bytes.eq(b${quote(a)}, b${quote(b)})); put_c(32);`);
-		lines.push(` put_n(Bytes.starts(b${quote(a)}, b${quote(b)})); put_c(32); put_n(Bytes.ends(b${quote(a)}, b${quote(b)})); put_c(32); put_n(Bytes.find(b${quote(a)}, b${quote(b)})); put_c(32);`);
+		lines.push(` rt.put_n(std.Text.len(${quote(a)})); rt.put_c(32); rt.put_n(std.Text.eq(${quote(a)}, ${quote(b)})); rt.put_c(32);`);
+		lines.push(` rt.put_n(std.Text.starts(${quote(a)}, ${quote(b)})); rt.put_c(32); rt.put_n(std.Text.ends(${quote(a)}, ${quote(b)})); rt.put_c(32); rt.put_n(std.Text.find(${quote(a)}, ${quote(b)})); rt.put_c(32);`);
+		lines.push(` rt.put_n(std.Bytes.len(b${quote(a)})); rt.put_c(32); rt.put_n(std.Bytes.eq(b${quote(a)}, b${quote(b)})); rt.put_c(32);`);
+		lines.push(` rt.put_n(std.Bytes.starts(b${quote(a)}, b${quote(b)})); rt.put_c(32); rt.put_n(std.Bytes.ends(b${quote(a)}, b${quote(b)})); rt.put_c(32); rt.put_n(std.Bytes.find(b${quote(a)}, b${quote(b)})); rt.put_c(32);`);
 		expected.push(a.length, Number(a === b), Number(a.startsWith(b)), Number(a.endsWith(b)), a.indexOf(b), a.length, Number(a === b), Number(a.startsWith(b)), Number(a.endsWith(b)), a.indexOf(b));
 	}
-	lines.push(' churn(300); gc(); put_n(live_objects() - base); put_c(32);', ' put_n(stable()); put_c(10);', ' return 0;', '}');
+	lines.push(' churn(300); rt.gc(); rt.put_n(rt.live_objects() - base); rt.put_c(32);', ' rt.put_n(stable()); rt.put_c(10);', ' return 0;', '}');
 	expected.push(0, 73);
 	const source = lines.join('\n') + '\n', want = expected.join(' ') + '\n';
 	const zc = await ZC.load(fs.readFileSync(process.argv[2] || 'build/web/zc.wasm'));
diff --git a/tests/langerrors/embed_array_overflow.zc b/tests/langerrors/embed_array_overflow.zc
new file mode 100644
index 0000000..eebfe1a
--- /dev/null
+++ b/tests/langerrors/embed_array_overflow.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: the embedded file is larger than the array it is declared as
+embed "tests/data/embed/bytes.bin" as u8[2] x;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_missing.zc b/tests/langerrors/embed_missing.zc
new file mode 100644
index 0000000..6144878
--- /dev/null
+++ b/tests/langerrors/embed_missing.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: can't read the file named by embed
+embed "tests/data/embed/missing.bin" as int x;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_missing_as.zc b/tests/langerrors/embed_missing_as.zc
new file mode 100644
index 0000000..785d941
--- /dev/null
+++ b/tests/langerrors/embed_missing_as.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: expected as: embed "file" as [type] name;
+embed "tests/data/embed/text.txt" x;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_partial_element.zc b/tests/langerrors/embed_partial_element.zc
new file mode 100644
index 0000000..663ca24
--- /dev/null
+++ b/tests/langerrors/embed_partial_element.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: the embedded file's size is not a whole number of array elements
+embed "tests/data/embed/bytes.bin" as u16[] x;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_struct.zc b/tests/langerrors/embed_struct.zc
new file mode 100644
index 0000000..1fad589
--- /dev/null
+++ b/tests/langerrors/embed_struct.zc
@@ -0,0 +1,7 @@
+namespace;
+include rt;
+
+// expect: embed declares a bstr, a char*, a number, or an array of numbers
+struct point { int x, y; };
+embed "tests/data/embed/answer.bin" as point p;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_syntax.zc b/tests/langerrors/embed_syntax.zc
new file mode 100644
index 0000000..0cd8b9e
--- /dev/null
+++ b/tests/langerrors/embed_syntax.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: embed takes a path in quotes: embed "file" as name;
+embed text as x;
+fn main() { return 0; }
diff --git a/tests/langerrors/embed_too_large.zc b/tests/langerrors/embed_too_large.zc
new file mode 100644
index 0000000..c8f3864
--- /dev/null
+++ b/tests/langerrors/embed_too_large.zc
@@ -0,0 +1,6 @@
+namespace;
+include rt;
+
+// expect: the embedded file is larger than the type it is declared as
+embed "tests/data/embed/text.txt" as int x;
+fn main() { return 0; }
diff --git a/tests/programs/embed.zc b/tests/programs/embed.zc
new file mode 100644
index 0000000..d8d0ca3
--- /dev/null
+++ b/tests/programs/embed.zc
@@ -0,0 +1,31 @@
+namespace;
+include rt;
+
+// embed declarations: file bytes become globals and locals of the declared type.
+embed "tests/data/embed/text.txt" as text;
+embed "tests/data/embed/answer.bin" as int answer;
+embed "tests/data/embed/bytes.bin" as u8[] bytes;
+embed "tests/data/embed/bytes.bin" as i8[8] signed;
+embed "tests/data/embed/text.txt" as char* chars;
+embed "tests/data/embed/onehalf.f32" as f32 half;
+embed "tests/data/embed/bytes.bin" as char[] cs;
+embed "tests/data/embed/nul.bin" as bstr nul;
+embed "tests/data/embed/shorts.bin" as u16[] shorts;
+fn main() {
+	rt.put_s(text); rt.put_c(32);
+	rt.put_n(len(text)); rt.put_c(32);
+	rt.put_n(answer); rt.put_c(32);
+	rt.put_n(len(bytes)); rt.put_c(32); rt.put_n(bytes[4]); rt.put_c(32);
+	rt.put_n(len(signed)); rt.put_c(32); rt.put_n(signed[4]); rt.put_c(32); rt.put_n(signed[7]); rt.put_c(32);
+	rt.put_n(chars[6]); rt.put_c(32);
+	rt.put_n((int)(half * 10.0)); rt.put_c(32);
+	rt.put_n(len(cs)); rt.put_c(32); rt.put_n(cs[4]); rt.put_c(32);
+	rt.put_n(len(nul)); rt.put_c(32); rt.put_n(nul[2]); rt.put_c(32);
+	rt.put_n(shorts[0] + shorts[1] * 10 + shorts[2] * 100); rt.put_c(32);
+	embed "tests/data/embed/answer.bin" as int local;
+	embed "tests/data/embed/bytes.bin" as u8[] copy;
+	embed "tests/data/embed/text.txt" as lt;
+	copy[0] = 9;
+	rt.put_n(local + copy[0] + bytes[0] + len(lt)); rt.put_c(10);
+	return 0;
+}
diff --git a/tests/programs/embed_top.zc b/tests/programs/embed_top.zc
new file mode 100644
index 0000000..732566d
--- /dev/null
+++ b/tests/programs/embed_top.zc
@@ -0,0 +1,21 @@
+namespace;
+include rt;
+
+// embed declarations at the top level, the form stage 0 compiles into run-only images.
+embed "tests/data/embed/text.txt" as text;
+embed "tests/data/embed/answer.bin" as int answer;
+embed "tests/data/embed/bytes.bin" as u8[] bytes;
+embed "tests/data/embed/bytes.bin" as i8[8] signed;
+embed "tests/data/embed/text.txt" as char* chars;
+embed "tests/data/embed/bytes.bin" as char[] cs;
+embed "tests/data/embed/shorts.bin" as u16[] shorts;
+fn main() {
+	rt.put_s(text); rt.put_c(32);
+	rt.put_n(answer); rt.put_c(32);
+	rt.put_n(bytes[4]); rt.put_c(32);
+	rt.put_n(signed[4]); rt.put_c(32); rt.put_n(signed[7]); rt.put_c(32);
+	rt.put_n(chars[6]); rt.put_c(32);
+	rt.put_n(cs[4]); rt.put_c(32);
+	rt.put_n(shorts[0] + shorts[1] * 10 + shorts[2] * 100); rt.put_c(10);
+	return 0;
+}
diff --git a/tests/test.sh b/tests/test.sh
index fd2bcd3..811a09a 100755
--- a/tests/test.sh
+++ b/tests/test.sh
@@ -131,6 +131,7 @@ run(){	# $1 = vm command, $2 = label
 	chk "$($vm < tests/programs/o3push.zc | md5sum)" "$(md5sum < tests/programs/o3push.expected)" "$l push and pop of a growable array are inline in o3"
 	chk "$($vm < tests/programs/o3copy.zc)" "520507 1005" "$l o3 copies of a loop variable read the source instead"
 	chk "$($vm < tests/programs/borrowmove.zc)" "6" "$l storing into a borrowed object moves it"
+	chk "$($vm < tests/programs/embed.zc | tr '\n' '/')" "hello/world 11 42 5 255 8 -1 0 119 15 5 255 3 98 321 63/" "$l embed declarations: strings, numbers and arrays from files, global and local"
 	chk "$($vm < tests/programs/ownmove.zc)" "7 50 0 1" "$l an owned local moves into a released owner, and a global borrows a local until it is restored"
 	chk "$($vm < tests/programs/alias.zc)" "11 22 4 8" "$l aliases of a live owner: arithmetic, field, index"
 	chk "$($vm < tests/programs/quarantine.zc)" "0 1234" "$l quarantine: stale alias reads intact data, second free is a no-op"
@@ -248,7 +249,7 @@ s_micro() {
 	python3 tests/drivers/actor_test.py $T/zym > $T/actor.out 2>&1; r=$?
 	grep -v '^ok' $T/actor.out
 	chk $r 0 "micro isolated actor acceptance, rejection and regression fixtures"
-	if command -v node >/dev/null && command -v tsc >/dev/null && node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null \
+	if command -v node >/dev/null && command -v tsc >/dev/null && node src/js/mklayout.js --map $MAP/memmap.h --out $T/jsvm >/dev/null \
 		&& cp $ROOT/s1/genome.bin $T/jsvm/ && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm; then
 		python3 tests/drivers/fuzz_compiler.py $ROOT/zy $T/zym --js $T/jsvm/tinyvm.js > $T/fuzz.out 2>&1; r=$?
 	else
@@ -350,6 +351,7 @@ s_runonly() {
 	runonly strings tests/programs/strings.zc
 	runonly strings-ex examples/strings.zc
 	runonly share tests/programs/share.zc
+	runonly embed tests/programs/embed_top.zc
 	d=$T/run-echo; GENMAP=$T/genmap bash src/tools/automap.sh run $d src/runtime/run.zc examples/echo.zc >/dev/null && cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $d/ \
 		&& $CC -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr \
 		&& chk "$(printf 'abc\n' | $T/zymr | tr '\n' '/')" "$( (cat examples/echo.zc; printf '`abc\n') | $T/zy | tr '\n' '/')" "run echo" \
@@ -590,7 +592,7 @@ s_lsp() {
 			else echo "skip lsp wasm tests (no clang with wasm32)"; fi
 			# the same server run by the javascript vm, the extension's fallback when wasm is not there.
 			# the vm starts at the image's heap and grows on the heap's request, so a long document is the test
-			node src/js/extract_lsp.js --lsp $T/lsp --out $T/vmserver >/dev/null && node src/js/mklayout.js --map $MAP --out $T/vmserver >/dev/null \
+			node src/js/extract_lsp.js --lsp $T/lsp --out $T/vmserver >/dev/null && node src/js/mklayout.js --map $MAP/memmap.h --out $T/vmserver >/dev/null \
 				&& tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/vmserver
 			node tests/drivers/hover_surface_test.js --vm $T/vmserver > $T/hoversurfacevm.out 2>&1; r=$?
 			grep -v '^ok' $T/hoversurfacevm.out
@@ -621,13 +623,14 @@ s_js() {
 	# the javascript vm, and the microcore as webassembly. both run the same genome
 	if command -v node >/dev/null && command -v tsc >/dev/null; then
 		# the vm as make js builds it, with the fixpoint genome beside it
-		node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null && cp $ROOT/s1/genome.bin $T/jsvm/genome.bin \
+		node src/js/mklayout.js --map $MAP/memmap.h --out $T/jsvm >/dev/null && cp $ROOT/s1/genome.bin $T/jsvm/genome.bin \
 			&& tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm || chk 0 1 "js vm builds"
 		chk "$(node $T/jsvm/tinyvm.js examples/hello.zc)" "hello, world!" "js vm: hello"
 		chk "$(node $T/jsvm/tinyvm.js examples/fact.zc | tail -1)" "3628800" "js vm: factorial"
 		chk "$(node $T/jsvm/tinyvm.js examples/primes.zc | tail -1)" "widest gap: 72, after 31397" "js vm: primes"
 		chk "$(node $T/jsvm/tinyvm.js tests/programs/growone.zc)" "12 1" "js vm: a block past the initial heap grows the machine"
 		chk "$(node $T/jsvm/tinyvm.js tests/programs/replace_owner.zc)" "$(cat tests/programs/replace_owner.expected)" "js vm: replacement ownership and cleanup"
+		chk "$(node $T/jsvm/tinyvm.js tests/programs/embed.zc | tr '\n' '/')" "hello/world 11 42 5 255 8 -1 0 119 15 5 255 3 98 321 63/" "js vm: embed declarations read their files through node"
 		if sh src/tools/findcc.sh --wasm >/dev/null 2>&1 && command -v wasm-ld >/dev/null; then
 			GENOME_DIR=$ROOT/s1 MAP_DIR=$MAP bash src/web/build.sh $T/zc.wasm >/dev/null && node tests/drivers/wasm_test.js $T/zc.wasm > $T/wasm.out 2>&1; r=$?
 			grep -v '^ok' $T/wasm.out