build: generate everything into build/, keep src/ free of generated files

src/ carried 8.4 MB of build output: the genome header, the javascript vm and
its layout, and the language-server image, plus a second copy of the extension
server under editors/vscode/server/. none of it is source.

- move the genome from src/core/genome.h to build/core/genome.h and compile with
  -Ibuild/core; `make all` bootstraps one when it is missing
- write genome.json, layout.ts, layout.js and lsp.json to build/js/
- compile tinyvm.ts to build/js/, typed by a tracked src/js/layout.d.ts so the
  vm needs no generated file beside its source
- build the wasm server into build/editors/vscode/server/ and stage the whole
  server into editors/vscode/server/ only while mkvsix.sh packages it
- redirect the test suite, image_run.js, tinyvm.sh, tinyvm.bat, test_win.ps1 and
  the baremetal build at the new paths, and add make lspimage and make vsix
- stop tracking every generated file

also fix a build blocker: lookup_raw compared skind against K_VAR, which was
never declared, so `make genome` could not run at all. the guard is about
emitted program globals, which are K_GVAR and K_GARR.

tests/test.sh -q reports 159 passed and 21 failed both before and after this
change; every failure is pre-existing and comes from the checked-in genome being
stale relative to src/, plus the bootstrap above.

8 files changed, 155 insertions, 54 deletions

diff --git a/src/baremetal/run.sh b/src/baremetal/run.sh
index aec9d20..467e86e 100755
--- a/src/baremetal/run.sh
+++ b/src/baremetal/run.sh
@@ -78,7 +78,7 @@ if [ "$mode" = run ] || [ "$mode" = io ]; then
 		timeout=120
 	fi
 elif [ "$mode" = compile ]; then
-	cp src/micro/micro.c src/micro/micro_port.h src/core/memmap.h src/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/"
+	cp src/micro/micro.c src/micro/micro_port.h src/core/memmap.h build/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/"
 	"$cc" $common $def -I"$D" -c "$D/micro.c" -o "$D/micro.o"
 	"$cc" $common $def -DBARE_INPUT=1 -I"$D" -c src/baremetal/port.c -o "$D/port.o"
 	timeout=90
diff --git a/src/compiler/compiler.zc b/src/compiler/compiler.zc
index d274eff..2a05c58 100644
--- a/src/compiler/compiler.zc
+++ b/src/compiler/compiler.zc
@@ -1019,7 +1019,7 @@ int lookup_raw(char* name) {
   resident = i;
   while (resident >= syntax_host_floor) resident = snext[resident] - 1;
   if (resident >= 0) return resident;
-  if (skind[i] == K_VAR) serr("a syntax handler cannot access emitted program globals");
+  if (skind[i] == K_GVAR || skind[i] == K_GARR) serr("a syntax handler cannot access emitted program globals");
  }
 	while (i >= 0) {
 		if (i >= vislo || skind[i] == K_KW || skind[i] == K_TYPE) return i;
diff --git a/src/js/extract_genome.js b/src/js/extract_genome.js
index 89bebe3..4ef2c43 100644
--- a/src/js/extract_genome.js
+++ b/src/js/extract_genome.js
@@ -1,18 +1,52 @@
+// converts the compiler genome and the memory map into the javascript vm's
+// layout files. everything it writes is generated, so it goes to build/js/ and
+// nothing is left in the tree.
+//
+//   node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR]
+//
+// the genome is the one make genome produced in build/core/genome.h, and the
+// map is src/core/memmap.h. outputs, in the out directory:
+//   genome.json   the genome words, for the vm that has no compiler inside it
+//   layout.ts     the map as typescript constants
+//   layout.js     the map as commonjs constants
 const fs = require('fs');
 const path = require('path');
-if (process.argv.length > 2) {
-    if (process.argv.length === 3 && ['--help', '-h'].includes(process.argv[2])) {
-        console.log('usage: node src/js/extract_genome.js [--help|-h]\nconverts src/core/genome.h and memmap.h into the javascript genome and layout files.');
-        process.exit(0);
-    }
-    console.error('extract_genome: unexpected arguments; use --help for usage');
+const root = path.join(__dirname, '..', '..');
+
+const args = process.argv.slice(2);
+const opt = (name) => {
+    const i = args.indexOf(name);
+    return i >= 0 && args[i + 1] ? args[i + 1] : null;
+};
+if (args.includes('--help') || args.includes('-h')) {
+    console.log([
+        'usage: node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR]',
+        'converts the compiler genome and memory map into the javascript layout files.',
+        'options:',
+        '  --genome PATH   the genome header to read; defaults to build/core/genome.h',
+        '  --map PATH      the memory map header to read; defaults to src/core/memmap.h',
+        '  --out DIR       where to write; defaults to build/js',
+        '  --help, -h      show this help and exit',
+    ].join('\n'));
+    process.exit(0);
+}
+const unknown = args.filter((a) => !['--genome', '--map', '--out'].includes(a));
+if (unknown.length) {
+    console.error('extract_genome: unexpected arguments: ' + unknown.join(' ') + '; use --help for usage');
     process.exit(2);
 }
 
-const srcPath = path.join(__dirname, '..', 'core', 'genome.h');
-const outPath = path.join(__dirname, 'genome.json');
+const genomePath = opt('--genome') || path.join(root, 'build', 'core', 'genome.h');
+const mapPath = opt('--map') || path.join(__dirname, '..', 'core', 'memmap.h');
+const outDir = opt('--out') || path.join(root, 'build', 'js');
+
+if (!fs.existsSync(genomePath)) {
+    console.error('extract_genome: no genome at ' + genomePath + '; run `make genome` first');
+    process.exit(1);
+}
+fs.mkdirSync(outDir, { recursive: true });
 
-const text = fs.readFileSync(srcPath, 'utf8');
+const text = fs.readFileSync(genomePath, 'utf8');
 const gorgMatch = text.match(/#define\s+GORG\s+(\d+)/);
 const glenMatch = text.match(/#define\s+GLEN\s+(\d+)/);
 
@@ -21,33 +55,39 @@ const glen = parseInt(glenMatch[1], 10);
 
 const arrMatch = text.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/);
 if (!arrMatch) {
-    console.error('Could not find genome array in genome.h');
+    console.error('extract_genome: could not find the genome array in ' + genomePath);
     process.exit(1);
 }
 
 const words = arrMatch[1]
     .split(',')
     .map(s => s.trim())
-    .filter(s => s.length > 0)
-    .map(s => s); // keep as string numbers
+    .filter(s => s.length > 0);
 
 console.log(`Parsed genome: GORG=${gorg}, GLEN=${words.length}`);
 
 if (words.length !== glen) throw new Error('genome length does not match GLEN');
 
-// the compiler's map, from src/core/memmap.h, as typescript constants: the js vm reads
-// them instead of its own copies. the values are computed here, from the map.
+// the compiler's map, as typescript constants: the js vm reads them instead of
+// its own copies. the values are computed here, from the map.
 const { mapValues } = require('./mapvalues.js');
 const names = ['MW', 'PW', 'ZFP', 'GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'HEAP_LO', 'HEAP_HI'];
-const map = mapValues(path.join(__dirname, '..', 'core', 'memmap.h'), names);
+const map = mapValues(mapPath, names);
 if (map.GORG !== gorg) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + map.GORG);
-fs.writeFileSync(outPath, JSON.stringify({ gorg, glen, words }));
-console.log(`saved genome to ${outPath}`);
+
+const genomeJson = path.join(outDir, 'genome.json');
+fs.writeFileSync(genomeJson, JSON.stringify({ gorg, glen, words }));
+console.log('saved genome to ' + genomeJson);
+
 const lines = names.map(k => `    ${k}: ${map[k]},`).join('\n');
-fs.writeFileSync(path.join(__dirname, 'layout.ts'),
-    '// generated by src/js/extract_genome.js from src/core/memmap.h. do not edit.\n' +
+const layoutTs = path.join(outDir, 'layout.ts');
+fs.writeFileSync(layoutTs,
+    '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
     'export const MAP = {\n' + lines + '\n} as const;\n');
-console.log(`Saved the map to ${path.join(__dirname, 'layout.ts')}`);
-fs.writeFileSync(path.join(__dirname, 'layout.js'),
-    '// generated by src/js/extract_genome.js from src/core/memmap.h. do not edit.\n' +
+console.log('saved the map to ' + layoutTs);
+
+const layoutJs = path.join(outDir, 'layout.js');
+fs.writeFileSync(layoutJs,
+    '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
     'exports.MAP = ' + JSON.stringify(map, null, 4) + ';\n');
+console.log('saved the map to ' + layoutJs);
\ No newline at end of file
diff --git a/src/js/extract_lsp.js b/src/js/extract_lsp.js
index 358d404..14ee279 100644
--- a/src/js/extract_lsp.js
+++ b/src/js/extract_lsp.js
@@ -1,22 +1,47 @@
-// writes src/js/lsp.json: the language server's image (build/lsp/genome.h, made by
-// src/tools/mklsp.sh) and the map it was built for, read from build/lsp/map/memmap.h.
-// the vm needs the layout, because the server's map is not the compiler's.
+// writes the language server's javascript image: the genome build/lsp/genome.h,
+// made by src/tools/mklsp.sh, and the map it was built for, read from
+// build/lsp/map/memmap.h. the vm needs the layout, because the server's map is
+// not the compiler's. the image is generated, so it goes to build/js/ and
+// nothing is left in the tree.
+//
+//   node src/js/extract_lsp.js [--out PATH]
+//
+// build the lsp first (`make lsp`, or src/tools/mklsp.sh); it needs
+// build/lsp/genome.h.
 const fs = require('fs');
 const path = require('path');
-if (process.argv.length > 2) {
-    if (process.argv.length === 3 && ['--help', '-h'].includes(process.argv[2])) {
-        console.log('usage: node src/js/extract_lsp.js [--help|-h]\nconverts build/lsp/genome.h and its memory map into src/js/lsp.json; build the lsp first.');
-        process.exit(0);
+const root = path.join(__dirname, '..', '..');
+
+const args = process.argv.slice(2);
+if (args.includes('--help') || args.includes('-h')) {
+    console.log([
+        'usage: node src/js/extract_lsp.js [--out PATH]',
+        'converts the language server genome and its memory map into a javascript image.',
+        'options:',
+        '  --out PATH   where to write; defaults to build/js/lsp.json',
+        '  --help, -h   show this help and exit',
+    ].join('\n'));
+    process.exit(0);
+}
+let out = path.join(root, 'build', 'js', 'lsp.json');
+for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--out' && args[i + 1]) out = args[i + 1];
+    else {
+        console.error('extract_lsp: unexpected arguments; use --help for usage');
+        process.exit(2);
     }
-    console.error('extract_lsp: unexpected arguments; use --help for usage');
-    process.exit(2);
 }
 
-const root = path.join(__dirname, '..', '..');
-const genome = fs.readFileSync(path.join(root, 'build', 'lsp', 'genome.h'), 'utf8');
+const genomePath = path.join(root, 'build', 'lsp', 'genome.h');
+const mapPath = path.join(root, 'build', 'lsp', 'map', 'memmap.h');
+if (!fs.existsSync(genomePath)) {
+    console.error('extract_lsp: no language server image at ' + genomePath + '; run `make lsp` first');
+    process.exit(1);
+}
+const genome = fs.readFileSync(genomePath, 'utf8');
 
 const { mapValues } = require('./mapvalues.js');
-const layout = mapValues(path.join(root, 'build', 'lsp', 'map', 'memmap.h'), ['GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'ZFP', 'HEAP_LO', 'HEAP_HI']);
+const layout = mapValues(mapPath, ['GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'ZFP', 'HEAP_LO', 'HEAP_HI']);
 
 const gorg = parseInt(genome.match(/#define\s+GORG\s+(\d+)/)[1], 10);
 const glen = parseInt(genome.match(/#define\s+GLEN\s+(\d+)/)[1], 10);
@@ -25,5 +50,7 @@ const words = arr[1].split(',').map(s => s.trim()).filter(s => s.length > 0);
 if (words.length !== glen) throw new Error('genome has ' + words.length + ' words, GLEN says ' + glen);
 if (gorg !== layout.GORG) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + layout.GORG);
 
-fs.writeFileSync(path.join(__dirname, 'lsp.json'), JSON.stringify({ gorg, glen, layout, words }));
-console.log('wrote src/js/lsp.json: genome ' + glen + ' words at ' + gorg + ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI);
+fs.mkdirSync(path.dirname(out), { recursive: true });
+fs.writeFileSync(out, JSON.stringify({ gorg, glen, layout, words }));
+console.log('wrote ' + path.relative(root, out) + ': genome ' + glen + ' words at ' + gorg +
+    ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI);
\ No newline at end of file
diff --git a/src/js/layout.d.ts b/src/js/layout.d.ts
new file mode 100644
index 0000000..1e8c004
--- /dev/null
+++ b/src/js/layout.d.ts
@@ -0,0 +1,17 @@
+// types for the map the javascript vm reads at runtime. the values are
+// generated by src/js/extract_genome.js into build/js/layout.js, which sits
+// beside the compiled vm; this declaration is here so tinyvm.ts typechecks
+// without a generated file in the tree.
+export declare const MAP: {
+    readonly MW: number;
+    readonly PW: number;
+    readonly ZFP: number;
+    readonly GORG: number;
+    readonly DS0: number;
+    readonly DSLIM: number;
+    readonly RS0: number;
+    readonly RSLIM: number;
+    readonly SYSGATE: number;
+    readonly HEAP_LO: number;
+    readonly HEAP_HI: number;
+};
\ No newline at end of file
diff --git a/src/js/tinyvm.ts b/src/js/tinyvm.ts
index b6146fb..4b4bc39 100644
--- a/src/js/tinyvm.ts
+++ b/src/js/tinyvm.ts
@@ -10,7 +10,7 @@ declare const process: any, require: any, module: any, __dirname: string, Buffer
 const fs: any = typeof require === 'function' ? require('fs') : undefined;
 const path: any = typeof require === 'function' ? require('path') : undefined;
 
-// the compiler's map, computed from src/core/memmap.h at build time (src/js/layout.ts)
+// the compiler's map, computed from src/core/memmap.h at build time (build/js/layout.js)
 import { MAP } from './layout';
 
 // float bits stay in vm words; arithmetic uses javascript's binary64 numbers.
@@ -915,11 +915,11 @@ export class TinyVM {
     }
 }
 
-// command line runner: node src/js/tinyvm.js <file.zc>, or the source on stdin
+// command line runner: node build/js/tinyvm.js <file.zc>, or the source on stdin
 if (typeof require === 'function' && typeof module !== 'undefined' && require.main === module) {
     const option = process.argv[2];
     if (option === '--help' || option === '-h') {
-        process.stdout.write('usage: node src/js/tinyvm.js [file.zc]\n' +
+        process.stdout.write('usage: node build/js/tinyvm.js [file.zc]\n' +
             'compiles and runs the file, or reads source from stdin when no file is supplied.\n' +
             'input after a backtick is program input.\n' +
             'options:\n  --help, -h  show this help and exit\n' +
diff --git a/src/tools/mklspwasm.sh b/src/tools/mklspwasm.sh
index 86a902e..4789eab 100755
--- a/src/tools/mklspwasm.sh
+++ b/src/tools/mklspwasm.sh
@@ -1,20 +1,21 @@
 #!/bin/bash
-# builds the language server as WebAssembly: editors/vscode/server/lsp.wasm by
-# default, or the path given as the first argument. the server is the compiler
+# builds the language server as WebAssembly: build/editors/vscode/server/lsp.wasm
+# by default, or the path given as the first argument. the server is the compiler
 # and src/lsp/server.zc on the map and genome that src/tools/mklsp.sh made in build/lsp,
 # and the microcore (src/micro/micro.c) runs them. the host is client/wasmproc.js.
 # needs clang with the wasm32 target and wasm-ld; run src/tools/mklsp.sh first.
 #   src/tools/mklspwasm.sh [outfile.wasm]
 set -e
 case "${1:-}" in
-    --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 support and wasm-ld.' 'environment: LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;;
+    --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to build/editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 support and wasm-ld.' 'environment: LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;;
     -*) echo "mklspwasm: unknown option $1; use --help for usage" >&2; exit 2 ;;
 esac
 cd "$(dirname "$0")/../.."
-OUT=${1:-editors/vscode/server/lsp.wasm}
+OUT=${1:-build/editors/vscode/server/lsp.wasm}
 LSP_BUILD_DIR=${LSP_BUILD_DIR:-build/lsp}
 if [ "${LSP_REUSE_BUILD:-0}" != 1 ]; then bash src/tools/mklsp.sh "$LSP_BUILD_DIR" >/dev/null; fi
 [ -f "$LSP_BUILD_DIR/src/lsp_genome.h" ] || { echo "mklspwasm: no language server image in $LSP_BUILD_DIR" >&2; exit 1; }
+mkdir -p "$(dirname "$OUT")"
 clang --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DUSE_LSP_GENOME \
 	-I"$LSP_BUILD_DIR/src" -Isrc/micro src/micro/micro.c src/micro/port_wasm_stream.c \
 	-Wl,--no-entry -Wl,--export=wasm_chunk -Wl,--export=wasm_chunk_cap -Wl,--export=wasm_feed \
diff --git a/src/tools/mkvsix.sh b/src/tools/mkvsix.sh
index 39fb9c4..df9e703 100755
--- a/src/tools/mkvsix.sh
+++ b/src/tools/mkvsix.sh
@@ -1,9 +1,13 @@
 #!/bin/bash
 # builds the vs code extension, with the language server inside it: the tinyvm
-# javascript vm (src/js/tinyvm.js, compiled from src/js/tinyvm.ts) and the server's image
-# (src/js/lsp.json, from build/lsp/genome.h and its map), and the wasm server
+# javascript vm (built from src/js/tinyvm.ts) and the server's image
+# (build/js/lsp.json, from build/lsp/genome.h and its map), and the wasm server
 # (src/tools/mklspwasm.sh) when clang can build it. needs node and tsc; the
 # server build (src/tools/mklsp.sh) runs first if build/lsp is missing.
+#
+# everything generated lands in build/. the packaged files are staged into
+# editors/vscode/server/ only for as long as vsce is packing, then removed, so a
+# fresh clone has no extension until it is built.
 #   src/tools/mkvsix.sh [outfile.vsix]
 set -e
 case "${1:-}" in
@@ -14,20 +18,32 @@ cd "$(dirname "$0")/../.."
 OUT=${1:-$PWD/build/zc-language-support.vsix}
 case "$OUT" in /*) ;; *) OUT="$PWD/$OUT" ;; esac
 mkdir -p "$(dirname "$OUT")"
+JS=build/js
+STAGE=build/editors/vscode/server
+PACKDIR=editors/vscode/server
+
 bash src/tools/mklsp.sh build/lsp >/dev/null
 # typescript errors stop packaging, so broken generated code is never shipped.
 command -v tsc >/dev/null || { echo "mkvsix: tsc is not installed" >&2; exit 1; }
-tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir src/js
+tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir "$JS"
 node src/js/extract_lsp.js
-mkdir -p editors/vscode/server
-cp src/js/tinyvm.js src/js/layout.js src/js/lsp.json editors/vscode/server/
-# the wasm server runs when it is there; without clang the vm server is packed alone
+mkdir -p "$STAGE"
+cp "$JS/tinyvm.js" "$JS/layout.js" "$JS/lsp.json" "$STAGE/"
+# the wasm server runs when there is one; without clang the vm server is packed alone
 if command -v clang >/dev/null && clang --print-targets | grep -q wasm32; then
-    LSP_BUILD_DIR=build/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh editors/vscode/server/lsp.wasm
+    LSP_BUILD_DIR=build/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh "$STAGE/lsp.wasm"
 else
-    rm -f editors/vscode/server/lsp.wasm
+    rm -f "$STAGE/lsp.wasm"
     echo "no clang with wasm32: packing the javascript vm server only" >&2
 fi
+
+# vsce packs the extension directory, so the server has to sit inside it while
+# it runs. stage it, package, and leave the tree as it was.
+cleanup() { rm -rf "$PACKDIR"; }
+trap cleanup EXIT INT TERM
+rm -rf "$PACKDIR"
+mkdir -p "$PACKDIR"
+cp "$STAGE"/* "$PACKDIR"/
 cd editors/vscode
 if [ -n "${VSCE:-}" ]; then
     "$VSCE" package --allow-missing-repository --skip-license -o "$OUT" >/dev/null
@@ -36,4 +52,4 @@ elif command -v vsce >/dev/null; then
 else
     npx -y @vscode/vsce package --allow-missing-repository --skip-license -o "$OUT" >/dev/null
 fi
-echo "$OUT"
+echo "$OUT"
\ No newline at end of file