build: embed images with #embed, keep everything generated in build/

the cores take their genome in with c23 #embed through src/core/genome.h,
so images are plain little-endian word files (genome.bin) instead of
generated c headers. zc0 writes that format, mkgenome checks an image
against its map and shows a compiler's diagnostics when it gets them,
and the javascript vm and the editor's vm host read the same files.
src/tools/findcc.sh picks a compiler with #embed (clang 19 or gcc 15).

zc gets an embed declaration, embed "file" as [type] name, for globals
and locals: a byte string by default, char*, numbers read little-endian,
and arrays of numbers. the compiler and language server carry their
library and source texts through it, so embed.sh and the text it
spliced into compiler.zc are gone.

make writes executables to build/bin, the default map to build/core/map,
the web page to build/web and the extension staging to build/vsix, and
the genome now rebuilds when its sources change. the test suite lays
out its own map and bootstrap and reads nothing from build/.

also: rt.put_s of a byte string in zc0 images, retained function
sources no longer count as live objects, a missing include no longer
cascades into undefined-name errors, consecutive comment lines join
into one hover paragraph, and stale test fixtures follow the include
format.

6 files changed, 103 insertions, 129 deletions

diff --git a/src/js/extract_genome.js b/src/js/extract_genome.js
deleted file mode 100644
index 4ef2c43..0000000
--- a/src/js/extract_genome.js
+++ /dev/null
@@ -1,93 +0,0 @@
-// converts the compiler genome and the memory map into the javascript vm's
-// layout files. everything it writes is generated, so it goes to build/js/ and
-// nothing is left in the tree.
-//
-//   node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR]
-//
-// the genome is the one make genome produced in build/core/genome.h, and the
-// map is src/core/memmap.h. outputs, in the out directory:
-//   genome.json   the genome words, for the vm that has no compiler inside it
-//   layout.ts     the map as typescript constants
-//   layout.js     the map as commonjs constants
-const fs = require('fs');
-const path = require('path');
-const root = path.join(__dirname, '..', '..');
-
-const args = process.argv.slice(2);
-const opt = (name) => {
-    const i = args.indexOf(name);
-    return i >= 0 && args[i + 1] ? args[i + 1] : null;
-};
-if (args.includes('--help') || args.includes('-h')) {
-    console.log([
-        'usage: node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR]',
-        'converts the compiler genome and memory map into the javascript layout files.',
-        'options:',
-        '  --genome PATH   the genome header to read; defaults to build/core/genome.h',
-        '  --map PATH      the memory map header to read; defaults to src/core/memmap.h',
-        '  --out DIR       where to write; defaults to build/js',
-        '  --help, -h      show this help and exit',
-    ].join('\n'));
-    process.exit(0);
-}
-const unknown = args.filter((a) => !['--genome', '--map', '--out'].includes(a));
-if (unknown.length) {
-    console.error('extract_genome: unexpected arguments: ' + unknown.join(' ') + '; use --help for usage');
-    process.exit(2);
-}
-
-const genomePath = opt('--genome') || path.join(root, 'build', 'core', 'genome.h');
-const mapPath = opt('--map') || path.join(__dirname, '..', 'core', 'memmap.h');
-const outDir = opt('--out') || path.join(root, 'build', 'js');
-
-if (!fs.existsSync(genomePath)) {
-    console.error('extract_genome: no genome at ' + genomePath + '; run `make genome` first');
-    process.exit(1);
-}
-fs.mkdirSync(outDir, { recursive: true });
-
-const text = fs.readFileSync(genomePath, 'utf8');
-const gorgMatch = text.match(/#define\s+GORG\s+(\d+)/);
-const glenMatch = text.match(/#define\s+GLEN\s+(\d+)/);
-
-const gorg = parseInt(gorgMatch[1], 10);
-const glen = parseInt(glenMatch[1], 10);
-
-const arrMatch = text.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/);
-if (!arrMatch) {
-    console.error('extract_genome: could not find the genome array in ' + genomePath);
-    process.exit(1);
-}
-
-const words = arrMatch[1]
-    .split(',')
-    .map(s => s.trim())
-    .filter(s => s.length > 0);
-
-console.log(`Parsed genome: GORG=${gorg}, GLEN=${words.length}`);
-
-if (words.length !== glen) throw new Error('genome length does not match GLEN');
-
-// the compiler's map, as typescript constants: the js vm reads them instead of
-// its own copies. the values are computed here, from the map.
-const { mapValues } = require('./mapvalues.js');
-const names = ['MW', 'PW', 'ZFP', 'GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'HEAP_LO', 'HEAP_HI'];
-const map = mapValues(mapPath, names);
-if (map.GORG !== gorg) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + map.GORG);
-
-const genomeJson = path.join(outDir, 'genome.json');
-fs.writeFileSync(genomeJson, JSON.stringify({ gorg, glen, words }));
-console.log('saved genome to ' + genomeJson);
-
-const lines = names.map(k => `    ${k}: ${map[k]},`).join('\n');
-const layoutTs = path.join(outDir, 'layout.ts');
-fs.writeFileSync(layoutTs,
-    '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
-    'export const MAP = {\n' + lines + '\n} as const;\n');
-console.log('saved the map to ' + layoutTs);
-
-const layoutJs = path.join(outDir, 'layout.js');
-fs.writeFileSync(layoutJs,
-    '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
-    'exports.MAP = ' + JSON.stringify(map, null, 4) + ';\n');
-console.log('saved the map to ' + layoutJs);
\ No newline at end of file
diff --git a/src/js/extract_lsp.js b/src/js/extract_lsp.js
index 14ee279..bdbcb3c 100644
--- a/src/js/extract_lsp.js
+++ b/src/js/extract_lsp.js
@@ -1,13 +1,12 @@
-// writes the language server's javascript image: the genome build/lsp/genome.h,
-// made by src/tools/mklsp.sh, and the map it was built for, read from
-// build/lsp/map/memmap.h. the vm needs the layout, because the server's map is
-// not the compiler's. the image is generated, so it goes to build/js/ and
-// nothing is left in the tree.
+// writes the language server's javascript image: build/js/lsp.bin, the server's
+// genome (build/lsp/lsp_genome.bin, made by src/tools/mklsp.sh), and
+// build/js/lsp.json, the map it was built for, read from build/lsp/memmap.h. the
+// vm needs the layout, because the server's map is not the compiler's. the image
+// is generated, so it goes to build/js/ and nothing is left in the tree.
 //
-//   node src/js/extract_lsp.js [--out PATH]
+//   node src/js/extract_lsp.js [--lsp DIR] [--out DIR]
 //
-// build the lsp first (`make lsp`, or src/tools/mklsp.sh); it needs
-// build/lsp/genome.h.
+// build the lsp first (`make lsp`, or src/tools/mklsp.sh).
 const fs = require('fs');
 const path = require('path');
 const root = path.join(__dirname, '..', '..');
@@ -15,42 +14,39 @@ const root = path.join(__dirname, '..', '..');
 const args = process.argv.slice(2);
 if (args.includes('--help') || args.includes('-h')) {
     console.log([
-        'usage: node src/js/extract_lsp.js [--out PATH]',
-        'converts the language server genome and its memory map into a javascript image.',
+        'usage: node src/js/extract_lsp.js [--lsp DIR] [--out DIR]',
+        'copies the language server genome beside a json description of its memory map.',
         'options:',
-        '  --out PATH   where to write; defaults to build/js/lsp.json',
+        '  --lsp DIR    the language server build to read; defaults to build/lsp',
+        '  --out DIR    where to write lsp.bin and lsp.json; defaults to build/js',
         '  --help, -h   show this help and exit',
     ].join('\n'));
     process.exit(0);
 }
-let out = path.join(root, 'build', 'js', 'lsp.json');
+let out = path.join(root, 'build', 'js'), lsp = path.join(root, 'build', 'lsp');
 for (let i = 0; i < args.length; i++) {
-    if (args[i] === '--out' && args[i + 1]) out = args[i + 1];
+    if (args[i] === '--out' && args[i + 1]) out = args[++i];
+    else if (args[i] === '--lsp' && args[i + 1]) lsp = args[++i];
     else {
         console.error('extract_lsp: unexpected arguments; use --help for usage');
         process.exit(2);
     }
 }
 
-const genomePath = path.join(root, 'build', 'lsp', 'genome.h');
-const mapPath = path.join(root, 'build', 'lsp', 'map', 'memmap.h');
+const genomePath = path.join(lsp, 'lsp_genome.bin');
+const mapPath = path.join(lsp, 'memmap.h');
 if (!fs.existsSync(genomePath)) {
     console.error('extract_lsp: no language server image at ' + genomePath + '; run `make lsp` first');
     process.exit(1);
 }
-const genome = fs.readFileSync(genomePath, 'utf8');
-
 const { mapValues } = require('./mapvalues.js');
 const layout = mapValues(mapPath, ['GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'ZFP', 'HEAP_LO', 'HEAP_HI']);
+const image = fs.readFileSync(genomePath);
+if (image.length % 8) throw new Error(genomePath + ' is not a whole number of words');
+const glen = image.length / 8;
 
-const gorg = parseInt(genome.match(/#define\s+GORG\s+(\d+)/)[1], 10);
-const glen = parseInt(genome.match(/#define\s+GLEN\s+(\d+)/)[1], 10);
-const arr = genome.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/);
-const words = arr[1].split(',').map(s => s.trim()).filter(s => s.length > 0);
-if (words.length !== glen) throw new Error('genome has ' + words.length + ' words, GLEN says ' + glen);
-if (gorg !== layout.GORG) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + layout.GORG);
-
-fs.mkdirSync(path.dirname(out), { recursive: true });
-fs.writeFileSync(out, JSON.stringify({ gorg, glen, layout, words }));
-console.log('wrote ' + path.relative(root, out) + ': genome ' + glen + ' words at ' + gorg +
-    ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI);
\ No newline at end of file
+fs.mkdirSync(out, { recursive: true });
+fs.copyFileSync(genomePath, path.join(out, 'lsp.bin'));
+fs.writeFileSync(path.join(out, 'lsp.json'), JSON.stringify({ gorg: layout.GORG, glen, layout }));
+console.log('wrote ' + path.relative(root, path.join(out, 'lsp.bin')) + ': genome ' + glen + ' words at ' + layout.GORG +
+    ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI);
diff --git a/src/js/layout.d.ts b/src/js/layout.d.ts
index 1e8c004..9a531eb 100644
--- a/src/js/layout.d.ts
+++ b/src/js/layout.d.ts
@@ -1,5 +1,5 @@
 // types for the map the javascript vm reads at runtime. the values are
-// generated by src/js/extract_genome.js into build/js/layout.js, which sits
+// generated by src/js/mklayout.js into build/js/layout.js, which sits
 // beside the compiled vm; this declaration is here so tinyvm.ts typechecks
 // without a generated file in the tree.
 export declare const MAP: {
diff --git a/src/js/mapvalues.js b/src/js/mapvalues.js
index f701ba2..bb6fc86 100644
--- a/src/js/mapvalues.js
+++ b/src/js/mapvalues.js
@@ -1,6 +1,6 @@
 // reads the values of a machine map from its memmap.h. the #define lines may use
 // other names and small sums (ZFP + 2, DSLO + 1100), so each value is evaluated
-// with its names substituted. shared by extract_genome.js and extract_lsp.js.
+// with its names substituted. shared by mklayout.js and extract_lsp.js.
 const fs = require('fs');
 
 function mapValues(mapPath, names) {
diff --git a/src/js/mklayout.js b/src/js/mklayout.js
new file mode 100644
index 0000000..cf95091
--- /dev/null
+++ b/src/js/mklayout.js
@@ -0,0 +1,61 @@
+// writes the javascript vm's view of a memory map: the map's constants as
+// typescript and commonjs modules. the vm reads its genome image itself, from
+// genome.bin beside tinyvm.js, so only the layout is generated here. everything
+// it writes is generated, so it goes to build/js/ and nothing is left in the tree.
+//
+//   node src/js/mklayout.js [--map PATH] [--out DIR]
+//
+// the map is the default one `make` generates in build/core/map/memmap.h.
+// outputs, in the out directory:
+//   layout.ts     the map as typescript constants
+//   layout.js     the map as commonjs constants
+const fs = require('fs');
+const path = require('path');
+const root = path.join(__dirname, '..', '..');
+
+const args = process.argv.slice(2);
+const opt = (name) => {
+    const i = args.indexOf(name);
+    return i >= 0 && args[i + 1] ? args[i + 1] : null;
+};
+if (args.includes('--help') || args.includes('-h')) {
+    console.log([
+        'usage: node src/js/mklayout.js [--map PATH] [--out DIR]',
+        'converts a memory map header into the javascript vm layout files.',
+        'options:',
+        '  --map PATH      the memory map header to read; defaults to build/core/map/memmap.h',
+        '  --out DIR       where to write; defaults to build/js',
+        '  --help, -h      show this help and exit',
+    ].join('\n'));
+    process.exit(0);
+}
+for (let i = 0; i < args.length; i += 2) {
+    if (!['--map', '--out'].includes(args[i]) || args[i + 1] === undefined) {
+        console.error('mklayout: unexpected arguments: ' + args.join(' ') + '; use --help for usage');
+        process.exit(2);
+    }
+}
+
+const mapPath = opt('--map') || path.join(root, 'build', 'core', 'map', 'memmap.h');
+const outDir = opt('--out') || path.join(root, 'build', 'js');
+if (!fs.existsSync(mapPath)) {
+    console.error('mklayout: no memory map at ' + mapPath + '; run `make` first');
+    process.exit(1);
+}
+fs.mkdirSync(outDir, { recursive: true });
+
+// the compiler's map, as constants: the js vm reads them instead of its own copies.
+const { mapValues } = require('./mapvalues.js');
+const names = ['MW', 'PW', 'ZFP', 'GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'HEAP_LO', 'HEAP_HI'];
+const map = mapValues(mapPath, names);
+
+const lines = names.map(k => `    ${k}: ${map[k]},`).join('\n');
+const layoutTs = path.join(outDir, 'layout.ts');
+fs.writeFileSync(layoutTs,
+    '// generated by src/js/mklayout.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
+    'export const MAP = {\n' + lines + '\n} as const;\n');
+const layoutJs = path.join(outDir, 'layout.js');
+fs.writeFileSync(layoutJs,
+    '// generated by src/js/mklayout.js from ' + path.relative(root, mapPath) + '. do not edit.\n' +
+    'exports.MAP = ' + JSON.stringify(map, null, 4) + ';\n');
+console.log('wrote ' + path.relative(root, layoutJs) + ' and layout.ts');
diff --git a/src/js/tinyvm.ts b/src/js/tinyvm.ts
index 4b4bc39..56fab70 100644
--- a/src/js/tinyvm.ts
+++ b/src/js/tinyvm.ts
@@ -10,7 +10,7 @@ declare const process: any, require: any, module: any, __dirname: string, Buffer
 const fs: any = typeof require === 'function' ? require('fs') : undefined;
 const path: any = typeof require === 'function' ? require('path') : undefined;
 
-// the compiler's map, computed from src/core/memmap.h at build time (build/js/layout.js)
+// the compiler's map, computed from build/core/map/memmap.h at build time (build/js/layout.js)
 import { MAP } from './layout';
 
 // float bits stay in vm words; arithmetic uses javascript's binary64 numbers.
@@ -257,6 +257,16 @@ export class TinyVM {
         }
     }
 
+    /** load an image file's bytes (little-endian 64-bit words, the format the c cores
+        #embed) at org. returns the number of words. */
+    public loadImage(bytes: Uint8Array, org: number = this.L.GORG): number {
+        if (bytes.length % 8) throw new RangeError('an image is a whole number of 64-bit words');
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);
+        const n = bytes.length / 8;
+        for (let i = 0; i < n; i++) this.mem[org + i] = view.getBigInt64(i * 8, true);
+        return n;
+    }
+
     public getOutput(): string {
         return new TextDecoder().decode(new Uint8Array(this.outputBuffer));
     }
@@ -931,13 +941,13 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma
         process.stderr.write('tinyvm.js: unknown option ' + option + '; use --help for usage\n');
         process.exit(2);
     }
-    const genomePath = path.join(__dirname, 'genome.json');
+    const genomePath = path.join(__dirname, 'genome.bin');
     if (!fs.existsSync(genomePath)) {
-        console.error('Error: genome.json not found. Run extract_genome.js first.');
+        console.error('Error: genome.bin not found beside tinyvm.js. Run make js first.');
         process.exit(1);
     }
 
-    const genomeData = JSON.parse(fs.readFileSync(genomePath, 'utf8'));
+    const genome: Uint8Array = fs.readFileSync(genomePath);
     let inputSource = '';
 
     if (process.argv.length > 2) {
@@ -970,9 +980,9 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma
         onMessage: (msg: string) => process.stderr.write(msg)
     });
 
-    vm.loadGenome(genomeData.words, genomeData.gorg);
+    vm.loadImage(genome, GORG);
     vm.setInput(inputSource);
-    const code = vm.run(genomeData.gorg);
+    const code = vm.run(GORG);
     process.exit(code);
 }