diff options
79 files changed, 1472 insertions, 1142 deletions
@@ -1,15 +1,7 @@ -# build output -bin/ +# everything built or generated lives in build/ (make clean removes it) +/build/ + +# stray objects and executables from building by hand *.o *.obj *.exe - -# built programs -/bin/ -/build/ -/src/web/zc.wasm - -# the editor's packaged server: layout, vm and language-server image, plus the -# wasm server. `make vsix` stages it into editors/vscode/server/ only while it -# packages, so nothing built is kept in the tree. -/editors/vscode/server/ @@ -1,27 +1,30 @@ # tinyvm and zc. `make` builds the tools, the interpreter, the microcore and the -# language server into bin/. nothing built is kept in the tree. +# language server into build/bin/. everything built or generated goes under +# build/; the tree holds sources only, and `make clean` (rm -rf build) returns it +# to a fresh checkout. # # make everything below that is a program # make test the whole suite (tests/test.sh) # make check the minimal suite, for iterating (tests/test.sh -q) # make test JOBS=4 the same, four sections at a time -# make genome rebuild build/core/genome.h from src/ and check the fixpoint -# make lsp bin/zclsp, the language server -# make web src/web/zc.wasm (needs clang with the wasm32 target) -# make js build/js/genome.json and the vm, for the javascript vm +# make genome build/core/genome.bin from src/, with the fixpoint check +# make lsp build/bin/zclsp, the language server +# make web build/web/: the page, its script and zc.wasm +# make js build/js/: the javascript vm, its layout and genome # make run PROG=examples/hello.zc # compile one program into a run-only image on a memory # map sized to it (src/tools/automap.sh) -> build/run/zymr # make vsix the vscode extension, server and all -# make clean remove bin/ and build/ and the web build +# make clean remove build/ # -# nothing built is kept in the tree: the genome, the javascript vm and its -# layout, and the packed editor server all land in build/ (the web build keeps -# its historical src/web/zc.wasm path because the page loads it by name). -# -# CC picks the compiler (cc by default; clang works too). +# the cores take their genome in with c23 #embed (src/core/genome.h), so CC has +# to be clang 19 or gcc 15 or later. src/tools/findcc.sh picks one when CC is +# not given on the command line or in the environment. -CC ?= cc +ifeq ($(origin CC),default) + CC := $(shell sh src/tools/findcc.sh || echo cc) +endif +export CC CFLAGS ?= -O2 -w -fwrapv ifeq ($(OS),Windows_NT) EXE := .exe @@ -35,99 +38,105 @@ else MICRO_THREADS := -DZY_THREADS endif -BIN := bin B := build +BIN := $(B)/bin JS := $(B)/js -# the hand-written core headers, then the generated genome beside them -COREINC := -Isrc/core -I$(B)/core -# the sources the compiler genome is built from +WEB := $(B)/web +# the default memory map, generated from the const blocks in the zc sources +MAP := $(B)/core/map +# hand-written core headers, the generated map, and where #embed finds the genome image +COREINC := -Isrc/core -I$(MAP) --embed-dir=$(B)/core +CORE_H := src/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h $(MAP)/memmap.h +# the sources the compiler genome is built from, and the files it embeds ZC_SRC := src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc +ZC_DEPS := $(ZC_SRC) src/lib/std.zc src/lib/icon.zc src/tools/build.zc TOOLS := $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/genmap$(EXE) PROGS := $(BIN)/zy$(EXE) $(BIN)/zym$(EXE) $(BIN)/zclsp$(EXE) all: $(TOOLS) $(PROGS) -$(BIN) $(B): +$(BIN): mkdir -p $@ -$(BIN)/zc0$(EXE): src/tools/zc0.c src/core/memmap.h src/core/fops.h src/core/bops.h | $(BIN) - $(CC) -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -o $@ -$(BIN)/mkgenome$(EXE): src/tools/mkgenome.c src/core/memmap.h | $(BIN) - $(CC) -O1 -w -fwrapv src/tools/mkgenome.c -Isrc/core -o $@ +# genmap needs no map of its own; everything else is built against the one it writes $(BIN)/genmap$(EXE): src/tools/genmap.c | $(BIN) - $(CC) $(CFLAGS) src/tools/genmap.c -Isrc/core -o $@ + $(CC) $(CFLAGS) src/tools/genmap.c -o $@ +$(MAP)/memmap.h: $(BIN)/genmap$(EXE) src/runtime/rt.zc src/compiler/compiler.zc + mkdir -p $(B)/core + $(BIN)/genmap$(EXE) --preset default --check -o $(MAP) >$(B)/core/genmap.log 2>&1 || { cat $(B)/core/genmap.log; exit 1; } +$(BIN)/zc0$(EXE): src/tools/zc0.c src/core/fops.h src/core/bops.h $(MAP)/memmap.h | $(BIN) + $(CC) -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -I$(MAP) -o $@ +$(BIN)/mkgenome$(EXE): src/tools/mkgenome.c $(MAP)/memmap.h | $(BIN) + $(CC) -O1 -w -fwrapv src/tools/mkgenome.c -I$(MAP) -o $@ + +# bootstrap: zc0 compiles the compiler to g0, g0 compiles it to g1, g1 to g2. +# g1 and g2 must be identical (the fixpoint), and g1 becomes build/core/genome.bin. +# each stage's core embeds the image beside it, so no stage reads another's. +genome: $(B)/core/genome.bin +$(B)/core/genome.bin: $(ZC_DEPS) $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) src/core/core.c $(IO) $(CORE_H) + mkdir -p $(B)/s0 $(B)/s1 $(B)/core + cat $(ZC_SRC) | $(BIN)/zc0$(EXE) > $(B)/s0/g0.bin + $(BIN)/mkgenome$(EXE) < $(B)/s0/g0.bin > $(B)/s0/genome.bin + $(CC) $(CFLAGS) -DJTYPE=int src/core/core.c $(IO) -Isrc/core -I$(MAP) --embed-dir=$(B)/s0 -o $(B)/s0/zy0$(EXE) + (cat src/tools/build.zc; printf '`'; cat $(ZC_SRC)) | $(B)/s0/zy0$(EXE) > $(B)/s1/g1.bin + $(BIN)/mkgenome$(EXE) < $(B)/s1/g1.bin > $(B)/s1/genome.bin + $(CC) $(CFLAGS) -DJTYPE=int src/core/core.c $(IO) -Isrc/core -I$(MAP) --embed-dir=$(B)/s1 -o $(B)/s1/zy1$(EXE) + (cat src/tools/build.zc; printf '`'; cat $(ZC_SRC)) | $(B)/s1/zy1$(EXE) > $(B)/s1/g2.bin + cmp $(B)/s1/g1.bin $(B)/s1/g2.bin + cp $(B)/s1/genome.bin $@ + @echo "fixpoint: $$(( $$(wc -c < $@) / 8 )) words, $@ updated" # the interpreter: the vm core with the compiler genome inside -$(BIN)/zy$(EXE): src/core/core.c $(IO) $(B)/core/genome.h src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h | $(BIN) +$(BIN)/zy$(EXE): src/core/core.c $(IO) $(B)/core/genome.bin $(CORE_H) | $(BIN) $(CC) $(CFLAGS) -DJTYPE=int src/core/core.c $(IO) $(COREINC) -o $@ # the portable microcore -$(BIN)/zym$(EXE): src/micro/micro.c src/micro/micro_port.h $(PORT) $(B)/core/genome.h src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h | $(BIN) +$(BIN)/zym$(EXE): src/micro/micro.c src/micro/micro_port.h $(PORT) $(B)/core/genome.bin $(CORE_H) | $(BIN) $(CC) $(CFLAGS) $(MICRO_THREADS) src/micro/micro.c $(PORT) $(COREINC) -Isrc/micro -o $@ # the language server is the compiler in check mode plus src/lsp/server.zc, on a map # sized to it +LSP_DEPS := src/core/core.c src/core/io_linux.c $(filter-out $(MAP)/memmap.h,$(CORE_H)) $(ZC_DEPS) src/lsp/server.zc \ + src/tools/zc0.c src/tools/genmap.c src/tools/mkgenome.c src/tools/automap.sh src/tools/mklsp.sh lsp: $(BIN)/zclsp$(EXE) -$(BIN)/zclsp$(EXE): src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/memmap.h src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc src/lsp/server.zc src/lib/std.zc src/lib/icon.zc src/tools/embed.sh src/tools/zc0.c src/tools/genmap.c src/tools/automap.sh src/tools/mklsp.sh | $(BIN) - CC="$(CC)" bash src/tools/mklsp.sh $(B)/lsp +$(BIN)/zclsp$(EXE): $(LSP_DEPS) | $(BIN) + bash src/tools/mklsp.sh $(B)/lsp cp $(B)/lsp/zclsp$(EXE) $@ -src/compiler/compiler.zc: src/lib/std.zc src/lib/icon.zc src/tools/embed.sh - bash src/tools/embed.sh all - -# bootstrap: zc0 compiles the compiler to g0, g0 compiles it to g1, g1 to g2. -# g1 and g2 must be identical (the fixpoint), and g1 becomes build/core/genome.h. -# run this by hand after editing a .zc source; `make check` reports a stale genome. -genome: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) src/compiler/compiler.zc | $(B) - mkdir -p $(B)/s0 $(B)/s1 $(B)/core - cp src/core/core.c $(IO) src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h $(B)/s0/ - cp src/core/core.c $(IO) src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h $(B)/s1/ - cat $(ZC_SRC) | $(BIN)/zc0$(EXE) > $(B)/s0/genome.h - $(CC) $(CFLAGS) -DJTYPE=int $(B)/s0/core.c $(B)/s0/$(notdir $(IO)) -o $(B)/zy0$(EXE) - (cat src/tools/build.zc; printf '`'; cat $(ZC_SRC)) | $(B)/zy0$(EXE) > $(B)/g1.bin - $(BIN)/mkgenome$(EXE) < $(B)/g1.bin > $(B)/s1/genome.h - $(CC) $(CFLAGS) -DJTYPE=int $(B)/s1/core.c $(B)/s1/$(notdir $(IO)) -o $(B)/zy1$(EXE) - (cat src/tools/build.zc; printf '`'; cat $(ZC_SRC)) | $(B)/zy1$(EXE) > $(B)/g2.bin - cmp $(B)/g1.bin $(B)/g2.bin - cp $(B)/s1/genome.h $(B)/core/genome.h - @echo "fixpoint: $$(( $$(wc -c < $(B)/g1.bin) / 8 )) words, $(B)/core/genome.h updated" - -# a fresh clone has no genome, so bootstrap one on demand. it is deliberately -# not rebuilt from source dependencies: the fixpoint run is expensive, and -# `make genome` is the explicit way to refresh it after editing a .zc file. -$(B)/core/genome.h: - $(MAKE) genome - -js: $(JS)/genome.json $(JS)/layout.js $(JS)/tinyvm.js +# the javascript vm: tinyvm.js loads genome.bin from beside itself +js: $(JS)/tinyvm.js $(JS)/layout.js $(JS)/genome.bin $(JS): mkdir -p $@ -$(JS)/genome.json: $(B)/core/genome.h src/core/memmap.h src/js/extract_genome.js src/js/mapvalues.js | $(JS) - node src/js/extract_genome.js -$(JS)/layout.ts $(JS)/layout.js: $(JS)/genome.json - @test -f $@ || node src/js/extract_genome.js +$(JS)/genome.bin: $(B)/core/genome.bin | $(JS) + cp $< $@ +$(JS)/layout.js: $(MAP)/memmap.h src/js/mklayout.js src/js/mapvalues.js | $(JS) + node src/js/mklayout.js --map $(MAP)/memmap.h --out $(JS) $(JS)/tinyvm.js: src/js/tinyvm.ts src/js/layout.d.ts | $(JS) tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $(JS) # the language server image for the javascript vm, and the wasm server -lspimage: $(JS)/lsp.json -$(JS)/lsp.json: $(BIN)/zclsp$(EXE) src/js/extract_lsp.js src/js/mapvalues.js | $(JS) +lspimage: $(JS)/lsp.bin +$(JS)/lsp.bin: $(BIN)/zclsp$(EXE) src/js/extract_lsp.js src/js/mapvalues.js | $(JS) node src/js/extract_lsp.js # the vscode extension, with the server and its image inside it -vsix: | $(B) +vsix: bash src/tools/mkvsix.sh -web: src/web/zc.wasm -src/web/zc.wasm: src/micro/micro.c src/micro/micro_port.h src/web/port_wasm.c src/web/build.sh $(B)/core/genome.h src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h +# the browser compiler: the page and its loader beside the wasm build +web: $(WEB)/zc.wasm $(WEB)/index.html $(WEB)/zc.js +$(WEB)/zc.wasm: src/micro/micro.c src/micro/micro_port.h src/web/port_wasm.c src/web/build.sh $(B)/core/genome.bin $(CORE_H) bash src/web/build.sh $@ +$(WEB)/index.html $(WEB)/zc.js: $(WEB)/%: src/web/% + mkdir -p $(WEB) + cp $< $@ # one program as a run-only image: the runtime, the program, and a map exactly # big enough for them -run: | $(B) +run: @test -n "$(PROG)" || { echo "make run PROG=examples/hello.zc"; exit 2; } bash src/tools/automap.sh run $(B)/run src/runtime/run.zc $(PROG) - cp src/micro/micro.c src/micro/micro_port.h $(PORT) src/core/fops.h src/core/bops.h src/core/stack.h $(B)/run/ - $(CC) $(CFLAGS) $(MICRO_THREADS) $(B)/run/micro.c $(B)/run/$(notdir $(PORT)) -I$(B)/run -o $(B)/run/zymr$(EXE) + $(CC) $(CFLAGS) $(MICRO_THREADS) src/micro/micro.c $(PORT) -Isrc/core -Isrc/micro -I$(B)/run --embed-dir=$(B)/run -o $(B)/run/zymr$(EXE) @echo "built $(B)/run/zymr$(EXE)" # JOBS sets how many sections run at once: make test JOBS=4 @@ -139,6 +148,6 @@ check: bash tests/test.sh -q -j $(or $(JOBS),2) clean: - rm -rf $(BIN) $(B) src/web/zc.wasm + rm -rf $(B) .PHONY: all lsp genome js lspimage vsix web run test check clean diff --git a/editors/vscode/client/extension.js b/editors/vscode/client/extension.js index 3846eca..cd497d8 100644 --- a/editors/vscode/client/extension.js +++ b/editors/vscode/client/extension.js @@ -185,7 +185,7 @@ function shortKey(key) { function serverStampFor(bin) { if (bin) return fileStamp(bin); const dir = path.join(__dirname, '..', 'server'); - return sha256(['lsp.wasm', 'lsp.json', 'tinyvm.js'].map(f => fileStamp(path.join(dir, f))).join(',')); + return sha256(['lsp.wasm', 'lsp.json', 'lsp.bin', 'tinyvm.js'].map(f => fileStamp(path.join(dir, f))).join(',')); } // the key of a document as it is now, and what it reads (for checking it again when one of those changes) diff --git a/editors/vscode/client/vmproc.js b/editors/vscode/client/vmproc.js index 8ce8c52..6b57554 100644 --- a/editors/vscode/client/vmproc.js +++ b/editors/vscode/client/vmproc.js @@ -16,6 +16,7 @@ function vmProcess(dir, opts = {}) { const log = opts.log || (() => {}); const { TinyVM } = require(path.join(dir, 'tinyvm.js')); const image = require(path.join(dir, 'lsp.json')); + const words = require('fs').readFileSync(path.join(dir, 'lsp.bin')); const proc = new EventEmitter(); const out = new EventEmitter(); const err = new EventEmitter(); @@ -33,8 +34,8 @@ function vmProcess(dir, opts = {}) { err.emit('data', Buffer.from(m, 'utf8')); }, }); - vm.loadGenome(image.words, image.gorg); - log('message', 'loaded the image: ' + image.words.length + ' words, code at ' + image.gorg); + const loaded = vm.loadImage(words, image.gorg); + log('message', 'loaded the image: ' + loaded + ' words, code at ' + image.gorg); const flush = () => { if (pending.length) { diff --git a/examples/quine.zc b/examples/quine.zc index 58a5351..3fb563b 100644 --- a/examples/quine.zc +++ b/examples/quine.zc @@ -1,4 +1 @@ -namespace; -include rt; - -let s="let s=#;void main(){let i=0;while(s[i]){if(s[i]==35){put_c(34);put_s(s);put_c(34);}else put_c(s[i]);i++;}put_c(10);return;}";void main(){let i=0;while(s[i]){if(s[i]==35){rt.put_c(34);rt.put_s(s);rt.put_c(34);}else rt.put_c(s[i]);i++;}rt.put_c(10);return;} +namespace;include rt;let s="namespace;include rt;let s=#;void main(){let i=0;while(s[i]){if(s[i]==35){rt.put_c(34);rt.put_s(s);rt.put_c(34);}else rt.put_c(s[i]);i++;}rt.put_c(10);return;}";void main(){let i=0;while(s[i]){if(s[i]==35){rt.put_c(34);rt.put_s(s);rt.put_c(34);}else rt.put_c(s[i]);i++;}rt.put_c(10);return;} diff --git a/src/baremetal/run.sh b/src/baremetal/run.sh index 467e86e..43f8a4c 100755 --- a/src/baremetal/run.sh +++ b/src/baremetal/run.sh @@ -6,11 +6,12 @@ # src/baremetal/run.sh riscv|aarch64 compile examples/hello.zc # run: the program is compiled into the image. nothing is read. # io: the same image, but stdin is the program's input and 0x04 ends it. -# compile: the default 1,048,576-word genome. the source, a backtick, and +# compile: the default compiler genome (build/core/genome.bin, from make genome). the source, a backtick, and # 0x04 are written to the UART. the backtick ends the unit; the # 0x04 is what eof() waits for after that. # Prints the program's UART output and nothing else. Exits 0 on a clean -# power-off. A missing toolchain exits 0 from `ready` and 1 otherwise. +# power-off. A missing toolchain exits 0 from `ready` and 1 otherwise. the cross +# compilers need c23 #embed (gcc 15 or later) to take in the genome. set -euo pipefail case "${1:-}" in --help|-h) printf '%s\n' 'usage: src/baremetal/run.sh ready riscv|aarch64' ' src/baremetal/run.sh riscv|aarch64 run|io|compile FILE.zc' 'builds a freestanding microcore and boots it in qemu-system.' 'run compiles the program into the image; io also forwards program input.' 'compile sends source to the resident compiler over the uart.' 'ready checks for the cross compiler and emulator.'; exit 0 ;; @@ -66,9 +67,9 @@ common="$cflags -ffreestanding -fno-pic -fno-pie -fno-builtin -fno-unwind-tables if [ "$mode" = run ] || [ "$mode" = io ]; then # the map is sized to the program, so a bigger runtime or program just gets a bigger board - gcc -O2 -w src/tools/genmap.c -o "$D/genmap" + ${CC:-$(sh src/tools/findcc.sh)} -O2 -w src/tools/genmap.c -o "$D/genmap" GENMAP="$D/genmap" bash src/tools/automap.sh run "$D/img" src/runtime/run.zc "$src" >"$D/genmap.log" 2>&1 || { cat "$D/genmap.log" >&2; exit 1; } - cp src/micro/micro.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/img/" + cp src/micro/micro.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h "$D/img/" "$cc" $common $def -I"$D/img" -c "$D/img/micro.c" -o "$D/micro.o" if [ "$mode" = io ]; then "$cc" $common $def -DBARE_INPUT=1 -I"$D/img" -c src/baremetal/port.c -o "$D/port.o" @@ -78,7 +79,8 @@ if [ "$mode" = run ] || [ "$mode" = io ]; then timeout=120 fi elif [ "$mode" = compile ]; then - cp src/micro/micro.c src/micro/micro_port.h src/core/memmap.h build/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/" + [ -f build/core/genome.bin ] || { echo "baremetal: no build/core/genome.bin; run make genome first" >&2; exit 1; } + cp src/micro/micro.c src/micro/micro_port.h build/core/map/memmap.h build/core/genome.bin src/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/" "$cc" $common $def -I"$D" -c "$D/micro.c" -o "$D/micro.o" "$cc" $common $def -DBARE_INPUT=1 -I"$D" -c src/baremetal/port.c -o "$D/port.o" timeout=90 diff --git a/src/compiler/compiler.zc b/src/compiler/compiler.zc index c33005d..244efc5 100644 --- a/src/compiler/compiler.zc +++ b/src/compiler/compiler.zc @@ -256,6 +256,7 @@ int s_star, s_slash, s_amp, s_colon, s_hash, s_minus, s_andand, s_oror; int s_dot, s_fn, s_var, s_const, s_else, s_as, s_unchecked, s_immediate, s_m, s_d, s_u; int s_q, s_case, s_default, s_fovf, s_start, s_main, s_free, s_recycle, s_own, s_qnew, s_enhance, s_micro, s_vm; int s_frame_floor, s_frame_top, s_frame_node, s_frame_busy, s_frame_guard, s_frame_grow, s_frame_leave; +int s_copy_w; int s_do, s_while, s_enum, s_include, s_memdebug, s_scrub, s_alloc, s_check, s_memset, s_static; //: ---- types: kinds, tables and parser state (see the types section) ---- @@ -1067,6 +1068,21 @@ int module_imported(char* name) { while (p) { if (rt.eq_w(p[1], name)) return 1; p = p[0]; } return 0; } +// modules reported missing in this unit, so their names are not reported again as undefined +word missing_imports; +// true when name is qualified by a module this unit was already told to include. +int missing_import_name(char* name) { + int i = 0; word p = missing_imports; + while (name[i] && name[i] != '.') i++; + if (!name[i]) return 0; + while (p) { + char* module = p[1]; int k = 0; + while (k < i && module[k] == name[k]) k++; + if (k == i && !module[k]) return 1; + p = p[0]; + } + return 0; +} // source references require their module's explicit import; compiler-generated calls use lookup directly. int source_lookup(char* name) { int i = 0, s; char* owner; @@ -1074,7 +1090,9 @@ int source_lookup(char* name) { if (name[i]) { owner = chk_resize(0, 0, i + 1); rt.copy_w(owner, name, i); owner[i] = 0; if ((module_known(owner) || module_loaded(owner)) && !module_imported(owner)) { - errat(); rt.put_s("module access requires `include "); rt.put_s(owner); rt.put_s(";`"); errend(); return -1; + errat(); rt.put_s("module access requires `include "); rt.put_s(owner); rt.put_s(";`"); errend(); + word link = chk_resize(0, 0, 2); link[0] = missing_imports; link[1] = owner; missing_imports = link; + // the unit already fails; resolving the name anyway keeps later diagnostics about the real code } } if (module_name) { s = lookup_raw(module_symbol(name)); if (s >= 0) return s; } @@ -4532,37 +4550,22 @@ int cover_warn(int e, int cov, int sl) { return 0; } //: ---- include ---- -//: the text of the embedded std library, one character to a word and ending in 0, as readfile gives a file. -//: it is built from the pieces src/tools/embed.sh put at the end of this file, the first time it is asked for +//: the text of an embedded library, one character to a word and ending in 0, as readfile gives a file. +//: it is unpacked from the byte string the first time it is asked for. +char* bstr_text(bstr p) { + int n = rt.len_s(p), i = 0; char* b = m*rt.new(n + 2); + while (i < n) { b[i] = rt.at_s(p, i); i++; } + b[n] = 0; + return b; +} +// the embedded std module. char* std_text() { - int n = 0, i = 0, k = 0, j; bstr p; char* b; - if (std_buf) return std_buf; - while (1) { p = std_part(i); if (!p) break; n = n + rt.len_s(p); i++; } - b = rt.new(n + 2); - i = 0; - while (1) { - p = std_part(i); if (!p) break; - j = 0; while (j < rt.len_s(p)) { b[k] = rt.at_s(p, j); k++; j++; } - i++; - } - b[k] = 0; - std_buf = b; + if (!std_buf) std_buf = bstr_text(std_source()); return std_buf; } -// the embedded icon module, assembled from its source pieces on first use. +// the embedded icon module. char* icon_text() { - int n = 0, i = 0, k = 0, j; bstr p; char* b; - if (icon_buf) return icon_buf; - while (1) { p = icon_part(i); if (!p) break; n = n + rt.len_s(p); i++; } - b = rt.new(n + 2); - i = 0; - while (1) { - p = icon_part(i); if (!p) break; - j = 0; while (j < rt.len_s(p)) { b[k] = rt.at_s(p, j); k++; j++; } - i++; - } - b[k] = 0; - icon_buf = b; + if (!icon_buf) icon_buf = bstr_text(icon_source()); return icon_buf; } // a path with its . and .. segments and doubled slashes taken out: /a/b/../c/./d.zc is /a/c/d.zc @@ -4714,6 +4717,120 @@ int kw_include() { inc_compile(buf, incres, inc_dir(incres)); return 0; } +// embed "path" as [type] name; declares name with the bytes of the file at path, read +// when the unit is compiled. the path is found the way an include's is. with no type, or +// bstr, name is a byte string of the file; char* gives one character to a word; a number +// type takes the bytes little-endian (a float or f32 needs exactly its size); T[] and +// T[n] lay the bytes out as the array's elements, packed for the sized types, one byte +// to an element for char[]. at the top level name is a global, in a function a local. +int kw_embed() { + int pp, off, n, t = 0, i, w, count, k, a, j, v, sz, skip = -1; char* path; word data; + stmtonly(); + next(); + pp = tpack(); + if (tk != T_STR) { die("embed takes a path in quotes: embed \"file\" as name;"); return 0; } + path = tv; next(); + if (!is_id(s_as)) { die("expected as: embed \"file\" as [type] name;"); return 0; } + next(); + peekat(0); + if (!(tk == T_ID && pk_tk == T_OP && pk_tv == s_semi)) t = ptype(); + off = toff; n = name(); + expect(s_semi); + data = m*embed_open(path); + if (!data) { int sv = tpack(); tunpack(pp); serr("can't read the file named by embed"); tunpack(sv); data = chk_resize(0, 0, 1); } + sz = data[0]; + if (!t) t = tbstr; + // in a function the data sits in the code stream, so the code jumps over it + if (infn && (t == tbstr || t == tptr(tchar) || t[0] == TY_ARR)) { reserve_code(2); skip = hole(); op(O_JMP); } + if (t == tbstr || t == tptr(tchar)) { + if (t == tbstr) { a = embed_bstr(data); } + else { reserve_code(sz + 1); a = here; i = 0; while (i < sz) { emit(data[1 + i]); i++; } emit(0); } + if (skip >= 0) setlit(skip, here); + return embed_bind(n, off, t, a, 0); + } + if (t[0] == TY_ARR) { + w = embed_width(t[4]); + if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); if (skip >= 0) setlit(skip, here); return 0; } + if (sz % w) { serr("the embedded file's size is not a whole number of array elements"); if (skip >= 0) setlit(skip, here); return 0; } + count = sz / w; + if (t[3] >= 0 && count > t[3]) { serr("the embedded file is larger than the array it is declared as"); if (skip >= 0) setlit(skip, here); return 0; } + if (t[3] < 0) t = tarr(t[4], count); + k = tsz(t); reserve_code(k); a = here; i = 0; + if (t[4][0] == TY_CHAR) { while (i < k) { if (i < sz) emit(data[1 + i]); else emit(0); i++; } } + else { + while (i < k) { + v = 0; j = 0; + while (j < 8) { if (i * 8 + j < sz) v = v | (data[1 + i * 8 + j] << (j * 8)); j++; } + emit(v); i++; + } + } + if (skip >= 0) setlit(skip, here); + return embed_bind(n, off, t, a, 1); + } + w = embed_width(t); + if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); return 0; } + if (sz > w) { serr("the embedded file is larger than the type it is declared as"); return 0; } + if (isfl(t) && sz != w) { serr("an embedded float takes exactly as many bytes as its type"); return 0; } + v = 0; i = 0; + while (i < sz) { v = v | (data[1 + i] << (i * 8)); i++; } + if (isfl(t) && w == 4) v = rt.s2f(v); + else v = cnorm(t, v); + return embed_bind(n, off, t, v, 2); +} +// the bytes one element of type t takes in an embedded file: the packed size of a +// sized type or f32, 1 for char, 8 for the other number types, 0 when t is not a number. +int embed_width(word t) { + if (tbytes(t)) return tbytes(t); + if (t[0] == TY_CHAR) return 1; + if (t[0] == TY_INT || t[0] == TY_WORD || t[0] == TY_BOOL || t[0] == TY_FLOAT) return 8; + return 0; +} +// the bytes of block data ([n, b0, b1, ...]) as byte string storage in the code stream. +int embed_bstr(word data) { + int n = data[0], a, i = 0, w, k; + reserve_code((n >> 3) + 2); a = here; + emit(n); + while (i < n) { + w = 0; k = 0; + while (k < 8 && i < n) { w = w | (data[1 + i] << (k * 8)); k++; i++; } + emit(w); + } + if (!(n & 7)) emit(0); + return a; +} +// declare name of type t. kind 0: a is the address of the data, which becomes the value; +// kind 1: a is an array's storage; kind 2: a is the value itself. a global keeps the +// value in its slot; a local is initialized from it, and a local array copies the storage. +int embed_bind(int n, int off, word t, int a, int kind) { + int s, i; + if (!infn) { + if (kind == 1) { s = addsym(n, K_GARR, a); } + else { reserve_code(1); s = addsym(n, K_GVAR, here); emit(a); } + rs_put(off, RS_SYM, rs_sym(s), s); stype[s] = t; + lt_note(off, t); + return 0; + } + i = newloc(n); lrsoff[i] = off; rs_note(RS_LOCAL, off); + ltag[i] = LDECL; lpos[i] = tpack(); lnsz[i] = 0; lleaf[i] = 0; ltype[i] = t; + if (kind == 1) { + larr[i] = 1; fslots = fslots + tsz(t); capture_cell(i, 0); + vaddr(i); lit(a); lit(tsz(t)); callname(s_copy_w); op(O_DROP); + } else { + fslots++; if (isfl(t)) setflt(i, 1); capture_cell(i, 0); + lit(a); vaddr(i); op(O_ST); + setstr(i, t == tbstr); + } + lt_note(off, t); + return 0; +} +// the bytes of an embedded file as [n, b0, b1, ...], found next to the including file, +// then in the workspace folder, then as written, like an include. 0 when none opens. +word.u embed_open(char*.m path) { + if (path[0] != '/' && incdir) { char* combined = inc_cat(incdir, path); char* c = inc_norm(combined); word b = rt.readbytes(c); if (b) return b; } + if (path[0] != '/' && increoot) { char* combined = inc_cat(increoot, path); char* c = inc_norm(combined); word b = rt.readbytes(c); if (b) return b; } + word b = rt.readbytes(path); + return b; +} // file namespace and imports are restored when an included source ends. int kw_namespace() { next(); @@ -9201,8 +9318,9 @@ int fn_remember(int s, int name) { } if (depth) return 0; close = j; - q = rt.new(1 + count * 3); q[0] = count; rt.copy_w(q + 1, gbuf + 1, count * 3); - meta = rt.new(6); meta[0] = q; meta[1] = module_name; meta[2] = module_imports; + // compiler storage, not program objects: internal, so live_objects and leak reports skip it + q = chk_resize(0, 0, 1 + count * 3); q[0] = count; rt.copy_w(q + 1, gbuf + 1, count * 3); + meta = chk_resize(0, 0, 6); meta[0] = q; meta[1] = module_name; meta[2] = module_imports; meta[3] = namepos; meta[4] = open; meta[5] = close; fnmeta[s] = meta; return 1; } @@ -9407,7 +9525,7 @@ int unit_end(int from) { while (i < nsym) { // a bodyless declaration (fn f();) promises a definition elsewhere. in check mode that // elsewhere is another file of the same build, which this document can't see - if (skind[i] == K_FN && !(sflag[i] & F_DEF) && sfix[i] && !(cmode && (sflag[i] & F_DECL))) { + if (skind[i] == K_FN && !(sflag[i] & F_DEF) && sfix[i] && !(cmode && (sflag[i] & F_DECL)) && !missing_import_name(sname[i])) { tunpack(sval[i]); best = similar(sname[i]); if (!dry) { @@ -9419,6 +9537,7 @@ int unit_end(int from) { } i++; } + missing_imports = 0; if (nerr) giveup(); return 0; } @@ -9657,7 +9776,7 @@ int zc_init() { s_start = intern("run.start"); s_main = intern("main"); s_free = intern("rt.free"); - s_recycle = intern("rt.recycle"); + s_recycle = intern("rt.recycle"); s_copy_w = intern("rt.copy_w"); s_do = intern("do"); s_while = intern("while"); s_enum = intern("enum"); @@ -9706,6 +9825,7 @@ int zc_tables() { add_kw("do", kw_do, F_STMT); add_kw("enum", kw_enum, F_STMT | F_TOP); add_kw("include", kw_include, F_TOP | F_STMT); + add_kw("embed", kw_embed, F_TOP | F_STMT); add_kw("run.method", kw_method, F_TOP | F_STMT); add_kw("run.share_value", kw_share_value, F_EXPR); add_kw("run.syntax", kw_syntax, F_TOP | F_STMT); @@ -9903,31 +10023,9 @@ int hl_end() { pq_n = 0; return 0; } -// ---- embedded std library (generated by src/tools/embed.sh from src/lib/std.zc; do not edit) ---- -bstr std_part(int i) { - if (i == 0) return b"namespace std;\ninclude rt, run;\n\n// small helpers for strings, files, the clock and byte buffers. not part of the\n// genome as declarations; its source is embedded. include it with: include std;\n// syscall numbers (SYS_OPEN and the rest) live in the runtime.\n// a character string has type char*, one character per word and a 0 at the end.\n// the Bytes group takes packed byte strings (b\"...\"), whose\n// type is bstr.\n// the helpers are grouped in structs, and each is a static function, so it is\n// called as std.Group.name(args): std.Text.itoa(n, buf), std.Host.now(),\n// std.Buf.get(b, 0).\n// the constructors arena(n) and bnew(n) and the argv helpers are plain\n// functions, because an owned result has to carry its .u tag.\n\n//: ---- the command line and the environment ----\n\n//: argc and argv, envc and envp, as passed to main by c. a zc string is\n//: one character per word, so an array of them is char*[?]. libinit() fills\n//: these and the runtime calls it before main, if the program has one (an\n//: image built by build_image, or a check-mode document, does not).\n\nint argc;\nchar*[?] argv;\nint envc;\nchar*[?] envp;\n\n// one entry of a vector syscall, as a fresh string, or 0 past the end.\n// op: the syscall that fetches an entry (SYS_ARG or SYS_ENV)\n// i: which entry\n// returns: the entry, owned by the caller; 0 when there is no entry i\nchar*.u argstr(int op, int i) {\n\tint n = rt.syscall(op, i, 0, 0);\n\tchar* s;\n\tif (n < 0) return 0;\n\ts = rt.new(n + 1);\n\trt.syscall(op, i, s, n + 1);\n\treturn s;\n}\n\n// every entry of a vector syscall, in order.\n// cntop: the syscall that counts the entries (SYS_ARGC or SYS_ENVC)\n// stop: the syscall that fetches one (SYS_ARG or SYS_ENV)\n// returns: a new array of the strings, owned by the caller\nchar*[?].u argsvec(int cntop, int stop) {\n\tint n = rt.syscall(cntop, 0, 0, 0), i = 0;\n\tchar*[?] a = rt.new(char*[?], 0);\n\twhile (i < n) { push(a, argstr(stop, i)); i++; }\n\treturn a;\n}\n\n// reads the command line and the environment into argc, argv, envc and envp.\n// it stays a plain function: the compiler looks it up by name.\n// returns: 0\nint libinit() {\n\targv = argsvec(rt.SYS_ARGC, rt.SYS_ARG);\n\tenvp = argsvec(rt.SYS_ENVC, rt.SYS_ENV);\n\targc = len(argv); envc = len(envp);\n\treturn 0;\n}\n\n//: ---- capturing output ----\n\n// capture.start() redirects everything the program writes into a string and\n// returns it. writes go to the buffer instead of the terminal until end()\n// stops it, so the buffer is a string once end() has run:\n// let buf = capture.start();\n// put_s(\"hello world!\"); // nothing on the terminal\n// capture.end(buf);\n// put_s(buf); // hello world\n// one capture at a time, and the buffer belongs to whoever holds it.\n\nstruct capture {\n\t// redirects program output into a collector-managed character buffer.\n\t// returns the buffer; call end(buffer) before reading its terminator.\n\t// returns: the buffer used for this capture\n\tstatic char* start() {\n\t\tint cap = 256;\n\t\trt.outmode = 2; rt.outn ="; - if (i == 1) return b" 0; rt.outcap = cap;\n\t\trt.outbuf = rt.new(cap);\n\t\treturn rt.outbuf;\n\t}\n\t// stops output capture, terminates the buffer, and returns the collected text.\n\t// buffer: the buffer returned by start\n\t// returns: the same buffer, now terminated\n\tstatic char* end(char* buffer) {\n\t\trt.outbuf[rt.outn] = 0;\n\t\trt.outmode = 0; rt.outbuf = 0; rt.outn = 0; rt.outcap = 0;\n\t\treturn buffer;\n\t}\n}\n\n//: ---- character strings ----\n\nstruct Text {\n\t// the number written in decimal at the start of s. a leading '-' makes it\n\t// negative and the first character that is not a digit ends it.\n\t// s: the string to read\n\t// returns: the number, 0 when s starts with no digit\n\tstatic int atoi(char* s) {\n\t\tint n = 0, i = 0, sign = 1;\n\t\tif (s[0] == '-') { sign = 0 - 1; i = 1; }\n\t\twhile (s[i] >= '0' && s[i] <= '9') { n = n * 10 + s[i] - '0'; i++; }\n\t\treturn n * sign;\n\t}\n\n\t// writes n in decimal into dst, with a 0 after it.\n\t// n: the number\n\t// dst: where the text goes; 24 words is always enough\n\t// returns: how many characters were written, not counting the 0\n\tstatic int itoa(int n, char* dst) {\n\t\tint b[24], i = 0, j = 0, neg = 0;\n\t\tif (!n) { dst[0] = '0'; dst[1] = 0; return 1; }\n\t\tif (n < 0) { neg = 1; n = 0 - n; }\n\t\tif (n < 0) { rt.copy_w(dst, \"-9223372036854775808\", 21); return 20; }\n\t\twhile (n) { b[i] = '0' + n % 10; i++; n = n / 10; }\n\t\tif (neg) { dst[0] = '-'; j = 1; }\n\t\twhile (i) { i--; dst[j] = b[i]; j++; }\n\t\tdst[j] = 0;\n\t\treturn j;\n\t}\n\n\t// joins two strings.\n\t// dst: where the result goes, with room for both and a 0; not a or b\n\t// a: the first string\n\t// b: the second string\n\t// returns: the length of the result\n\tstatic int cat(char* dst, char* a, char* b) {\n\t\tint i = 0, j = 0;\n\t\twhile (a[i]) { dst[j] = a[i]; i++; j++; }\n\t\ti = 0;\n\t\twhile (b[i]) { dst[j] = b[i]; i++; j++; }\n\t\tdst[j] = 0;\n\t\treturn j;\n\t}\n\n\t// does s begin with pfx?\n\t// s: the string to test\n\t// pfx: the beginning to look for\n\t// returns: 1 if it does (always, for an empty pfx), 0 if not\n\tstatic int starts(char* s, char* pfx) {\n\t\tint i = 0;\n\t\twhile (pfx[i]) { if (s[i] != pfx[i]) return 0; i++; }\n\t\treturn 1;\n\t}\n\n\t// does s end with suffix? an empty suffix always matches.\n\t// s: the string to test\n\t// suffix: the ending to look for\n\t// returns: 1 when it matches, 0 otherwise\n\tstatic int ends(char* s, char* suffix) {\n\t\tint n = 0, m = 0, i = 0;\n\t\twhile (s[n]) n++;\n\t\twhile (suffix[m]) m++;\n\t\tif (m > n) return 0;\n\t\twhile (i < m) { if (s[n - m + i] != suffix[i]) return 0; i++; }\n\t\treturn 1;\n\t}\n\n\t// how many characters come before the 0.\n\t// s: the string\n\t// returns: the length, not counting the 0\n\tstatic int len(char* s) {\n\t\tint n = 0;\n\t\twhile (s[n]) n++;\n\t\treturn n;\n\t}\n\n\t// are the two strings the same, character for character?\n\t// a, b: the strings to compare\n\t// returns: 1 if they are equal, 0 if not\n\tstatic int eq(char* a, char* b) {\n\t\tint i = 0;\n\t\twhile (a[i] && a[i] == b[i]) i++;\n\t\treturn a[i] == b[i];\n\t}\n\n\t// copies s into dst, with a 0 after it.\n\t// dst: where th"; - if (i == 2) return b"e copy goes, with room for s and the 0\n\t// s: the string to copy\n\t// returns: the length of the copy\n\tstatic int copy(char* dst, char* s) {\n\t\tint i = 0;\n\t\twhile (s[i]) { dst[i] = s[i]; i++; }\n\t\tdst[i] = 0;\n\t\treturn i;\n\t}\n\n\t// where pat first appears in s.\n\t// s: the string to search\n\t// pat: the string to look for; an empty pat is found at 0\n\t// returns: the index of the first match, or -1 when there is none\n\tstatic int find(char* s, char* pat) {\n\t\tint i = 0, j;\n\t\tif (!pat[0]) return 0;\n\t\twhile (s[i]) {\n\t\t\tj = 0;\n\t\t\twhile (pat[j] && s[i + j] == pat[j]) j++;\n\t\t\tif (!pat[j]) return i;\n\t\t\ti++;\n\t\t}\n\t\treturn 0 - 1;\n\t}\n\n\t// changes the ASCII letters of s to lower case, in place.\n\t// s: the string to change\n\t// returns: the length of s\n\tstatic int lower(char* s) {\n\t\tint i = 0;\n\t\twhile (s[i]) {\n\t\t\tif (s[i] >= 'A' && s[i] <= 'Z') s[i] = s[i] + 32;\n\t\t\ti++;\n\t\t}\n\t\treturn i;\n\t}\n\n\t// changes the ASCII letters of s to upper case, in place.\n\t// s: the string to change\n\t// returns: the length of s\n\tstatic int upper(char* s) {\n\t\tint i = 0;\n\t\twhile (s[i]) {\n\t\t\tif (s[i] >= 'a' && s[i] <= 'z') s[i] = s[i] - 32;\n\t\t\ti++;\n\t\t}\n\t\treturn i;\n\t}\n\n\t// removes the spaces, tabs and line breaks from both ends of s, in place.\n\t// s: the string to change\n\t// returns: the length of what is left\n\tstatic int trim(char* s) {\n\t\tint n = 0, a = 0, b, i = 0, c;\n\t\twhile (s[n]) n++;\n\t\twhile (a < n && (s[a] == ' ' || s[a] == '\\t' || s[a] == '\\n' || s[a] == '\\r')) a++;\n\t\tb = n;\n\t\twhile (b > a) {\n\t\t\tc = s[b - 1];\n\t\t\tif (!(c == ' ' || c == '\\t' || c == '\\n' || c == '\\r')) break;\n\t\t\tb--;\n\t\t}\n\t\tn = b - a;\n\t\twhile (i < n) { s[i] = s[i + a]; i++; }\n\t\ts[n] = 0;\n\t\treturn n;\n\t}\n}\n\n//: ---- numbers ----\n\n// plain integer helpers. none of them can fail: an out of range result wraps\n// the same way the arithmetic does.\n\nstruct Num {\n\t// the absolute value of n, using wrapping word arithmetic. the minimum\n\t// word remains negative because its positive value cannot fit in a word.\n\t// n: the value\n\t// returns: its absolute value, or the minimum word\n\tstatic int abs(int n) {\n\t\tif (n < 0) return 0 - n;\n\t\treturn n;\n\t}\n\n\t// the smaller of a and b.\n\t// a, b: the values to compare\n\t// returns: the smaller value (a when they are equal)\n\tstatic int min(int a, int b) {\n\t\tif (a < b) return a;\n\t\treturn b;\n\t}\n\n\t// the larger of a and b.\n\t// a, b: the values to compare\n\t// returns: the larger value (a when they are equal)\n\tstatic int max(int a, int b) {\n\t\tif (a > b) return a;\n\t\treturn b;\n\t}\n\n\t// v, kept between lo and hi.\n\t// v: the value\n\t// lo, hi: the bounds, lo first; lo must not exceed hi\n\t// returns: v clamped to the bounds\n\tstatic int clamp(int v, int lo, int hi) {\n\t\tif (v < lo) return lo;\n\t\tif (v > hi) return hi;\n\t\treturn v;\n\t}\n\n\t// the greatest common divisor of a and b, computed with word arithmetic.\n\t// a, b: the values\n\t// returns: their greatest common divisor\n\tstatic int gcd(int a, int b) {\n\t\tif (a < 0) a = 0 - a;\n\t\tif (b < 0) b = 0 - b;\n\t\tint t;\n\t\twhile (b) { t "; - if (i == 3) return b"= a % b; a = b; b = t; }\n\t\treturn a;\n\t}\n\n\t// b raised to the power e. e below 0 gives 1; the result wraps as a word.\n\t// b: the base\n\t// e: the exponent\n\t// returns: b to the power e, or 1 when e is negative\n\tstatic int pow(int b, int e) {\n\t\tint r = 1;\n\t\twhile (e > 0) { r = r * b; e--; }\n\t\treturn r;\n\t}\n\n\t// the whole square root of n, rounded down. n below 0 gives 0.\n\t// n: the value\n\t// returns: the greatest integer whose square is at most n, or 0 for n <= 0\n\tstatic int isqrt(int n) {\n\t\tint lo = 0, hi = 3037000499, mid;\n\t\tif (n <= 0) return 0;\n\t\twhile (lo < hi) {\n\t\t\tmid = lo + (hi - lo + 1) / 2;\n\t\t\tif (mid <= n / mid) lo = mid; else hi = mid - 1;\n\t\t}\n\t\treturn lo;\n\t}\n\n\t// -1, 0 or 1 according to the sign of n.\n\t// n: the value\n\t// returns: -1 when n is negative, 0 when it is zero, and 1 when it is positive\n\tstatic int sign(int n) {\n\t\tif (n < 0) return 0 - 1;\n\t\tif (n > 0) return 1;\n\t\treturn 0;\n\t}\n}\n\n//: ---- the host ----\n\n//: the argument block for Host.raw: the number, then its six arguments.\nint rawblk[7];\n\nstruct Host {\n\t// calls any linux system call on the host. this is the escape hatch below\n\t// the named helpers: the number is the kernel's, so the meaning of each\n\t// argument depends on the call. a pointer argument is an address in the\n\t// program's memory, and only the host reads it, so a string or buffer has\n\t// to be passed as its address.\n\t// it runs the call on the real machine, not inside the vm, so a program\n\t// that uses it can do whatever the process can do. only use it where that\n\t// is wanted.\n\t// nr: the syscall number (x86-64 and arm64 numbers differ, so check the host)\n\t// a..f: the six arguments, unused ones 0\n\t// returns: the kernel's result, -errno on failure, or -1 on unsupported hosts\n\tstatic int raw(int nr, int a, int b, int c, int d, int e, int f) {\n\t\trawblk[0] = nr; rawblk[1] = a; rawblk[2] = b; rawblk[3] = c;\n\t\trawblk[4] = d; rawblk[5] = e; rawblk[6] = f;\n\t\treturn rt.syscall(rt.SYS_RAW, rawblk, 0, 0);\n\t}\n\n\t// the value of an environment variable.\n\t// name: the variable's name\n\t// dst: where the value goes, a zero-terminated character string\n\t// n: how many words dst has\n\t// returns: the length of the value, -1 if the variable is not set\n\tstatic int env(char* name, char* dst, int n) { return rt.syscall(rt.SYS_GETENV, name, dst, n); }\n\n\t// the clock.\n\t// returns: the time in seconds since 1970\n\tstatic int now() { return rt.syscall(rt.SYS_TIME, 0, 0, 0); }\n\n\t// writes a string to a file, replacing what was there.\n\t// path: the file's name\n\t// s: the text to write\n\t// returns: how many bytes were written, -1 if the file could not be opened\n\tstatic int writefile(char* path, char* s) {\n\t\tint fd = rt.syscall(rt.SYS_OPEN, path, 1, 0), n;\n\t\tif (fd < 0) return -1;\n\t\tn = rt.syscall(rt.SYS_WRITE, fd, s, rt.len_w(s));\n\t\trt.syscall(rt.SYS_CLOSE, fd, 0, 0);\n\t\treturn n;\n\t}\n}\n\n//: ---- byte strings ----\n\n// the same helpers for packed byte strings. character-string helpers above are unchanged.\n// a"; - if (i == 4) return b" packed byte string has type bstr; len_s reads its stored byte length.\n\nstruct Bytes {\n\t// reads a decimal number at the start of s; a leading '-' makes it\n\t// negative, and the first non-digit ends the number.\n\t// s: the byte string\n\t// returns: the number, or 0 when s starts with no digit\n\tstatic int atoi(bstr s) { return rt.s_to_n(s); }\n\n\t// like Text.itoa, but the text is made fresh.\n\t// n: the number\n\t// returns: a new byte string\n\tstatic bstr itoa(int n) { return rt.n_to_s(n); }\n\n\t// like Text.starts, for byte strings.\n\t// s: the byte string to test\n\t// pfx: the beginning to look for\n\t// returns: 1 if s begins with pfx, 0 if not\n\tstatic int starts(bstr s, bstr pfx) {\n\t\tint i = 0, n = rt.len_s(s), m = rt.len_s(pfx);\n\t\tif (m > n) return 0;\n\t\twhile (i < m) { if (rt.at_s(s, i) != rt.at_s(pfx, i)) return 0; i++; }\n\t\treturn 1;\n\t}\n\n\t// the number of bytes in a packed byte string.\n\t// s: the byte string\n\t// returns: its stored length\n\tstatic int len(bstr s) { return rt.len_s(s); }\n\n\t// are two packed byte strings equal, byte for byte.\n\t// a, b: the byte strings to compare\n\t// returns: 1 if equal, 0 otherwise\n\tstatic int eq(bstr a, bstr b) {\n\t\tint i = 0;\n\t\tif (rt.len_s(a) != rt.len_s(b)) return 0;\n\t\twhile (i < rt.len_s(a)) { if (rt.at_s(a, i) != rt.at_s(b, i)) return 0; i++; }\n\t\treturn 1;\n\t}\n\n\t// where pat first appears in s.\n\t// s: the byte string to search\n\t// pat: the byte string to find; an empty pat is found at 0\n\t// returns: its byte index, or -1 when absent\n\tstatic int find(bstr s, bstr pat) {\n\t\tint i = 0, j, n = rt.len_s(s), m = rt.len_s(pat);\n\t\tif (!m) return 0;\n\t\twhile (i + m <= n) {\n\t\t\tj = 0;\n\t\t\twhile (j < m && rt.at_s(s, i + j) == rt.at_s(pat, j)) j++;\n\t\t\tif (j == m) return i;\n\t\t\ti++;\n\t\t}\n\t\treturn 0 - 1;\n\t}\n\n\t// does s end with suffix? an empty suffix always matches.\n\t// s: the byte string to test\n\t// suffix: the ending to look for\n\t// returns: 1 when it matches, 0 otherwise\n\tstatic int ends(bstr s, bstr suffix) {\n\t\tint n = rt.len_s(s), m = rt.len_s(suffix), i = 0;\n\t\tif (m > n) return 0;\n\t\twhile (i < m) { if (rt.at_s(s, n - m + i) != rt.at_s(suffix, i)) return 0; i++; }\n\t\treturn 1;\n\t}\n\n\t// like Host.env, for byte strings.\n\t// name: the variable's name, a byte string\n\t// dst: the packed byte string whose payload receives the value\n\t// n: the maximum number of bytes to write\n\t// returns: the length of the value, -1 if the variable is not set\n\tstatic int env(bstr name, bstr dst, int n) { return rt.bgetenv(name, dst, n); }\n\n\t// like Host.writefile, for a byte string.\n\t// path: the file's name, a byte string\n\t// s: the bytes to write\n\t// returns: how many bytes were written, -1 if the file could not be opened\n\tstatic int writefile(bstr path, bstr s) {\n\t\tint fd = rt.bopen(path, 1), n;\n\t\tif (fd < 0) return -1;\n\t\tn = rt.bwrite(fd, s, rt.len_s(s));\n\t\trt.syscall(rt.SYS_CLOSE, fd, 0, 0);\n\t\treturn n;\n\t}\n}\n\n//: ---- arenas ----\n\n// a region that is freed all at once.\n// n: its capacity in words; value"; - if (i == 5) return b"s below 1 allocate one word\n// returns: the arena, owned by the caller\nint .u arena(int n) { return rt.arena_new(n); }\n\nstruct Arena {\n\t// takes n zeroed words from an arena.\n\t// a: the arena\n\t// n: how many words\n\t// returns: the first word, or 0 when n words do not fit\n\tstatic int alloc(word a, int n) { return rt.arena_alloc(a, n); }\n\n\t// gives every block of an arena back at once. pointers returned by alloc\n\t// before the reset are no longer valid.\n\t// a: the arena\n\t// returns: 0\n\tstatic int reset(word a) { return rt.arena_reset(a); }\n}\n\n//: ---- growable byte buffers ----\n\n// a byte buffer stores its length and capacity in two words, followed by the\n// bytes, packed.\n// n: the capacity in bytes; a negative value is treated as 0\n// returns: an empty buffer with that capacity, owned by the caller\nword .u bnew(int n) {\n\tint words;\n\tword p;\n\tif (n < 0) n = 0;\n\twords = (n + 7) >> 3;\n\tp = rt.new(2 + words);\n\tp[0] = 0;\n\tp[1] = n;\n\treturn u*p;\n}\n\nstruct Buf {\n\t// how many bytes a buffer holds.\n\t// b: the buffer\n\t// returns: its length\n\tstatic int length(word b) { return b[0]; }\n\n\t// how many bytes a buffer can hold.\n\t// b: the buffer\n\t// returns: its capacity\n\tstatic int capacity(word b) { return b[1]; }\n\n\t// one byte of a buffer.\n\t// b: the buffer\n\t// i: the position\n\t// returns: the byte, 0 when i is outside the buffer\n\tstatic int get(word b, int i) { if (i < 0 || i >= b[0]) return 0; return rt.ld8(rt.addr_b(b + 2) + i); }\n\n\t// sets one byte of a buffer. setting the byte at the current length makes the\n\t// buffer one longer.\n\t// b: the buffer\n\t// i: the position, below the capacity\n\t// v: the value; only its low 8 bits are kept\n\t// returns: the byte stored, 0 when i is outside the capacity\n\tstatic int set(word b, int i, int v) {\n\t\tif (i < 0 || i >= b[1]) return 0;\n\t\trt.st8(rt.addr_b(b + 2) + i, v & 255);\n\t\tif (i >= b[0]) b[0] = i + 1;\n\t\treturn v & 255;\n\t}\n}\n\n//: array algorithms borrow their input and visit elements in index order.\n//: map and filter return independent handles. reference elements are shared\n//: with the collector, so freeing the input does not invalidate the result.\n\n// the first index equal to value, or -1 when no element matches.\nint array_find<T>(fixed T[?].m source, T value) {\n\tint length = len(source);\n\tfor (int i = 0; i < length; i++) if (source[i] == value) return i;\n\treturn -1;\n}\n// true when an element equals value.\nint array_contains<T>(fixed T[?].m source, T value) { return array_find(source, value) >= 0; }\n// the first index whose predicate is true, or -1 when none matches.\nint array_find_if<T>(fixed T[?].m source, int(T).m predicate) {\n\tint length = len(source);\n\tfor (int i = 0; i < length; i++) if (predicate(source[i])) return i;\n\treturn -1;\n}\n// true when any element satisfies predicate; false for an empty array.\nint array_any<T>(fixed T[?].m source, int(T).m predicate) { return array_find_if(source, predicate) >= 0; }\n// true when every element satisfies predicate, including an "; - if (i == 6) return b"empty array.\nint array_all<T>(fixed T[?].m source, int(T).m predicate) {\n\tint length = len(source);\n\tfor (int i = 0; i < length; i++) if (!predicate(source[i])) return 0;\n\treturn 1;\n}\n// the number of elements satisfying predicate.\nint array_count<T>(fixed T[?].m source, int(T).m predicate) {\n\tint length = len(source);\n\tint count = 0;\n\tfor (int i = 0; i < length; i++) if (predicate(source[i])) count++;\n\treturn count;\n}\n// apply transform once to each element and return the transformed array.\n// the result element type follows the callback's return type.\nU[?].u array_map<T,U>(fixed T[?].m source, U(T).m transform) {\n\tint length = len(source);\n\tU[?] result = rt.new(U[?]);\n\tfor (int i = 0; i < length; i++) push(result, run.share_value(transform(source[i])));\n\treturn result;\n}\n// copy elements satisfying predicate into a new array, preserving order.\nT[?].u array_filter<T>(fixed T[?].m source, int(T).m predicate) {\n\tint length = len(source);\n\tT[?] result = rt.new(T[?]);\n\tfor (int i = 0; i < length; i++) if (predicate(source[i])) push(result, run.share_value(source[i]));\n\treturn result;\n}\n// fold elements from left to right, starting with initial; an empty array\n// returns initial without calling combine.\nA array_reduce<T,A>(fixed T[?].m source, A initial, A(A,T).m combine) {\n\tint length = len(source);\n\tfor (int i = 0; i < length; i++) initial = combine(initial, source[i]);\n\treturn initial;\n}\n\nrun.method(\"find\", array_find);\nrun.method(\"contains\", array_contains);\nrun.method(\"find_if\", array_find_if);\nrun.method(\"any\", array_any);\nrun.method(\"all\", array_all);\nrun.method(\"count\", array_count);\nrun.method(\"map\", array_map);\nrun.method(\"filter\", array_filter);\nrun.method(\"reduce\", array_reduce);\n"; - return 0; -} -// ---- end embedded std library ---- -// ---- embedded icon library (generated by src/tools/embed.sh from src/lib/icon.zc; do not edit) ---- -bstr icon_part(int i) { - if (i == 0) return b"namespace icon;\ninclude rt, run;\n\n// structured runtime transformations of compiler token lists.\n// token storage follows run.tokens: count, then kind, value and source line.\n// edits create independent lists; publication recompiles a function only after\n// delimiter validation. captures match complete balanced token ranges.\n\nint icon_error;\n\nint icon_eq(word* a, int ai, word* b, int bi, int n) {\n\tint i = 0;\n\twhile (i < n) {\n\t\tif (a[1 + (ai + i) * 3] != b[1 + (bi + i) * 3] || a[2 + (ai + i) * 3] != b[2 + (bi + i) * 3]) return 0;\n\t\ti++;\n\t}\n\treturn 1;\n}\nint icon_value(word* q, int i) { if (i < 0 || i >= q[0]) return 0; return q[2 + i * 3]; }\nint icon_kind(word* q, int i) { if (i < 0 || i >= q[0]) return 0; return q[1 + i * 3]; }\nint icon_is(word* q, int i, char* s) {\n\tint kind = run.T_OP; if (s[0] >= 'a' && s[0] <= 'z') kind = run.T_ID;\n\treturn icon_kind(q, i) == kind && icon_value(q, i) == run.intern(s);\n}\nint icon_open(int v) { return v == run.intern(\"(\") || v == run.intern(\"[\") || v == run.intern(\"{\"); }\nint icon_close(int v) { return v == run.intern(\")\") || v == run.intern(\"]\") || v == run.intern(\"}\"); }\nint icon_pair(int a, int b) {\n\treturn (a == run.intern(\"(\") && b == run.intern(\")\")) || (a == run.intern(\"[\") && b == run.intern(\"]\")) || (a == run.intern(\"{\") && b == run.intern(\"}\"));\n}\nint icon_declaration(word* q, int start, int binding) {\n\tint first = icon_value(q, start), i = start;\n\tif (icon_kind(q, start) != run.T_ID || (first != run.intern(\"let\") && first != run.intern(\"const\") && !run.istypename(first))) return 0;\n\twhile (i < binding) { if (icon_is(q, i, \"(\") || icon_is(q, i, \"=\")) return 0; i++; }\n\treturn binding > start;\n}\n// locate a simple declaration at a statement boundary, including nested blocks.\n// result records [type start, binding position, semicolon, statement start].\nint icon_locate_declaration(word* q, int id, word* result) {\n\tint start = 0, i, depth, v, binding, comma, type_start;\n\twhile (start < q[0]) {\n\t\tif (start && !icon_is(q, start - 1, \";\") && !icon_is(q, start - 1, \"{\") && !icon_is(q, start - 1, \"}\")) { start++; continue; }\n\t\ttype_start = start; if (icon_is(q, type_start, \"const\")) type_start++;\n\t\tif (icon_kind(q, type_start) != run.T_ID || (!icon_is(q, type_start, \"let\") && !run.istypename(icon_value(q, type_start)))) { start++; continue; }\n\t\ti = type_start; depth = 0; binding = -1; comma = 0;\n\t\twhile (i < q[0]) {\n\t\t\tv = icon_value(q, i);\n\t\t\tif (!depth && v == id && icon_kind(q, i) == run.T_ID && binding < 0 && icon_declaration(q, type_start, i) && (icon_is(q, i + 1, \"=\") || icon_is(q, i + 1, \";\") || icon_is(q, i + 1, \",\") || icon_is(q, i + 1, \"(\"))) binding = i;\n\t\t\tif (icon_kind(q, i) == run.T_OP) {\n\t\t\t\tif (icon_open(v)) depth++;\n\t\t\t\telse if (icon_close(v)) { if (!depth) break; depth--; }\n\t\t\t\tif (!depth && v == run.intern(\",\")) comma = 1;\n\t\t\t\tif (!depth && v == run.intern(\";\")) {\n\t\t\t\t\tif (binding >= 0) {\n\t\t\t\t\t\tif (comma || icon_is(q, binding + 1, \"(\")) return 0;\n\t\t\t\t\t\tresult[0] = type_start; result[1] = bindin"; - if (i == 1) return b"g; result[2] = i; result[3] = start; return 1;\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t\ti++;\n\t\t}\n\t\tstart++;\n\t}\n\treturn 0;\n}\nint icon_balanced(word* q, int a, int b) {\n\tword* stack = rt.new(b - a + 1); int n = 0, i = a, v, valid = 1;\n\twhile (i < b) {\n\t\tv = icon_value(q, i);\n\t\tif (icon_kind(q, i) == run.T_OP) {\n\t\t\tif (icon_open(v)) { stack[n] = v; n++; }\n\t\t\telse if (icon_close(v)) { if (!n || !icon_pair(stack[n - 1], v)) { valid = 0; break; } n--; }\n\t\t}\n\t\ti++;\n\t}\n\tif (n) valid = 0;\n\trt.free(stack);\n\treturn valid;\n}\nword* .u icon_slice(word* q, int a, int b) {\n\tword* r = rt.new(1 + (b - a) * 3); int i = 0; r[0] = b - a;\n\twhile (i < (b - a) * 3) { r[1 + i] = q[1 + a * 3 + i]; i++; }\n\treturn r;\n}\nword* .u icon_splice(word* q, int a, int b, word* insert) {\n\tint n = q[0] - (b - a) + insert[0], i = 0, j = 0;\n\tword* r = rt.new(1 + n * 3); r[0] = n;\n\twhile (i < a * 3) { r[1 + j] = q[1 + i]; i++; j++; }\n\ti = 0; while (i < insert[0] * 3) { r[1 + j] = insert[1 + i]; i++; j++; }\n\ti = b * 3; while (i < q[0] * 3) { r[1 + j] = q[1 + i]; i++; j++; }\n\treturn r;\n}\n// capture classes use integer literals, identifiers, expressions, statements\n// or blocks. a named capture records its exact token range for substitution.\nint icon_class(int name) {\n\tif (name == run.intern(\"int\")) return 1;\n\tif (name == run.intern(\"name\")) return 2;\n\tif (name == run.intern(\"expr\")) return 3;\n\tif (name == run.intern(\"stmt\")) return 4;\n\tif (name == run.intern(\"block\")) return 5;\n\treturn 0;\n}\nint icon_pattern(word* q) {\n\tint i = 0;\n\tif (!q || q[0] <= 0 || !icon_balanced(q, 0, q[0])) return 0;\n\twhile (i < q[0]) {\n\t\tif (icon_is(q, i, \"$\")) {\n\t\t\tif (icon_kind(q, i + 1) != run.T_ID || !icon_class(icon_value(q, i + 1))) return 0;\n\t\t\ti += 2;\n\t\t\tif (icon_is(q, i, \":\")) { if (icon_kind(q, i + 1) != run.T_ID || icon_class(icon_value(q, i + 1))) return 0; i += 2; }\n\t\t} else i++;\n\t}\n\treturn 1;\n}\nint icon_precedence(int name) {\n\tif (name == run.intern(\"||\")) return 1;\n\tif (name == run.intern(\"&&\")) return 2;\n\tif (name == run.intern(\"|\")) return 3;\n\tif (name == run.intern(\"^\")) return 4;\n\tif (name == run.intern(\"&\")) return 5;\n\tif (name == run.intern(\"==\") || name == run.intern(\"!=\")) return 6;\n\tif (name == run.intern(\"<\") || name == run.intern(\"<=\") || name == run.intern(\">\") || name == run.intern(\">=\")) return 7;\n\tif (name == run.intern(\"<<\") || name == run.intern(\">>\")) return 8;\n\tif (name == run.intern(\"+\") || name == run.intern(\"-\")) return 9;\n\tif (name == run.intern(\"*\") || name == run.intern(\"/\") || name == run.intern(\"%\")) return 10;\n\treturn 0;\n}\n// recognize expression structure before matching a variable-length capture.\n// this excludes statement keywords and incomplete operator sequences.\nint icon_expression(word* q, int start, int end, int minimum) {\n\tint i = start, v, kind, priority, next;\n\tif (i >= end) return -1;\n\tv = icon_value(q, i); kind = icon_kind(q, i);\n\tif (kind == run.T_OP && (v == run.intern(\"!\") || v == run.intern(\"~\") || v == run.intern(\"+\") || v == run.intern(\"-\") || v == run.intern(\"*"; - if (i == 2) return b"\") || v == run.intern(\"&\"))) {\n\t\ti = icon_expression(q, i + 1, end, 11); if (i < 0) return -1;\n\t} else if (icon_is(q, i, \"(\")) {\n\t\ti = icon_expression(q, i + 1, end, 1); if (i < 0 || !icon_is(q, i, \")\")) return -1; i++;\n\t} else if (kind == run.T_NUM || kind == run.T_FLT || kind == run.T_STR || kind == run.T_BSTR || kind == run.T_ID) {\n\t\tif (kind == run.T_ID && (v == run.intern(\"return\") || v == run.intern(\"let\") || v == run.intern(\"if\") || v == run.intern(\"while\") || v == run.intern(\"for\") || v == run.intern(\"else\") || v == run.intern(\"break\") || v == run.intern(\"continue\") || v == run.intern(\"do\") || v == run.intern(\"switch\") || v == run.intern(\"const\"))) return -1;\n\t\ti++;\n\t} else return -1;\n\twhile (i < end) {\n\t\tif (icon_is(q, i, \"(\") || icon_is(q, i, \"[\")) {\n\t\t\tv = icon_value(q, i); i++;\n\t\t\tif (v != run.intern(\"(\") || !icon_is(q, i, \")\")) {\n\t\t\t\twhile (1) {\n\t\t\t\t\ti = icon_expression(q, i, end, 1); if (i < 0) return -1;\n\t\t\t\t\tif (v == run.intern(\"(\") && icon_is(q, i, \",\")) { i++; continue; } break;\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (i >= end || icon_kind(q, i) != run.T_OP || !icon_pair(v, icon_value(q, i))) return -1; i++; continue;\n\t\t}\n\t\tif (icon_is(q, i, \".\")) { if (i + 1 >= end || icon_kind(q, i + 1) != run.T_ID) return -1; i += 2; continue; }\n\t\tif (icon_is(q, i, \"++\") || icon_is(q, i, \"--\")) { i++; continue; }\n\t\tif (minimum == 1 && icon_is(q, i, \"?\")) {\n\t\t\ti = icon_expression(q, i + 1, end, 1); if (i < 0 || !icon_is(q, i, \":\")) return -1;\n\t\t\ti = icon_expression(q, i + 1, end, 1); if (i < 0) return -1; continue;\n\t\t}\n\t\tpriority = 0; if (icon_kind(q, i) == run.T_OP) priority = icon_precedence(icon_value(q, i)); if (!priority || priority < minimum) break;\n\t\tnext = icon_expression(q, i + 1, end, priority + 1); if (next < 0) return -1; i = next;\n\t}\n\treturn i;\n}\nint icon_range(word* q, int a, int b, int kind) {\n\tif (a < 0 || a >= b || b > q[0]) return 0;\n\tif (kind == 1) return b == a + 1 && icon_kind(q, a) == run.T_NUM;\n\tif (kind == 2) return b == a + 1 && icon_kind(q, a) == run.T_ID;\n\tif (kind == 3) return icon_expression(q, a, b, 1) == b;\n\tif (kind == 5) return icon_is(q, a, \"{\") && icon_mate(q, a) == b - 1;\n\tif (kind == 4) return icon_statement_end(q, a) == b;\n\treturn icon_expression(q, a, b, 1) == b;\n}\n// recursively match a pattern with backtracking at balanced capture boundaries.\n// captures contain count followed by [name, start, end, class] records.\nint icon_match(word* q, int at, word* p, int pi, word* caps) {\n\tint kind, name, next, end, old, i;\n\tif (pi == p[0]) return at + 1;\n\tif (at >= q[0]) return 0;\n\tif (!icon_is(p, pi, \"$\")) {\n\t\tif (!icon_eq(q, at, p, pi, 1)) return 0;\n\t\treturn icon_match(q, at + 1, p, pi + 1, caps);\n\t}\n\tif (pi + 1 >= p[0]) return 0;\n\tname = icon_value(p, pi + 1); kind = icon_class(name); next = pi + 2;\n\tif (!kind) return 0;\n\tif (icon_is(p, next, \":\")) { name = icon_value(p, next + 1); next += 2; if (!name) return 0; }\n\telse name = 0;\n\told = caps[0]; end = at + 1;\n\tif (kind == 4) end = icon_statement_end(q, at);\n\tif (kind == 5) { if (!ic"; - if (i == 3) return b"on_is(q, at, \"{\")) return 0; end = icon_mate(q, at) + 1; }\n\tif (end <= at) return 0;\n\twhile (end <= q[0]) {\n\t\tif (icon_range(q, at, end, kind)) {\n\t\t\ti = 0;\n\t\t\twhile (i < old) {\n\t\t\t\tif (name && caps[1 + i * 4] == name) {\n\t\t\t\t\tif (end - at != caps[3 + i * 4] - caps[2 + i * 4] || !icon_eq(q, at, q, caps[2 + i * 4], end - at)) break;\n\t\t\t\t}\n\t\t\t\ti++;\n\t\t\t}\n\t\t\tif (i == old) {\n\t\t\t\tcaps[0] = old + 1; caps[1 + old * 4] = name; caps[2 + old * 4] = at; caps[3 + old * 4] = end; caps[4 + old * 4] = kind;\n\t\t\t\ti = icon_match(q, end, p, next, caps);\n\t\t\t\tif (i) return i;\n\t\t\t\tcaps[0] = old;\n\t\t\t}\n\t\t}\n\t\tif (kind != 3) break;\n\t\tend++;\n\t}\n\treturn 0;\n}\nint icon_capture(word* caps, int name, int ordinal) {\n\tint c = 0, kind = icon_class(name), k = 0;\n\twhile (c < caps[0]) {\n\t\tif (!kind) { if (caps[1 + c * 4] == name) return c + 1; }\n\t\telse if (caps[4 + c * 4] == kind) { if (k == ordinal) return c + 1; k++; }\n\t\tc++;\n\t}\n\treturn 0;\n}\nword* .u icon_expand(word* q, word* replacement, word* caps) {\n\tint n = 0, i = 0, j = 0, c, name, kind, k, seen[6], a, b;\n\tk = 0; while (k < 6) { seen[k] = 0; k++; }\n\twhile (i < replacement[0]) {\n\t\tif (icon_is(replacement, i, \"$\")) {\n\t\t\tif (icon_kind(replacement, i + 1) != run.T_ID) { icon_error = 3; return 0; }\n\t\t\tname = icon_value(replacement, i + 1); kind = icon_class(name); c = icon_capture(caps, name, seen[kind]);\n\t\t\tif (!c) { icon_error = 3; return 0; } c--;\n\t\t\tn += caps[3 + c * 4] - caps[2 + c * 4]; if (kind) seen[kind]++; i += 2;\n\t\t} else { n++; i++; }\n\t}\n\tword* r = rt.new(1 + n * 3); i = 0;\n\tk = 0; while (k < 6) { seen[k] = 0; k++; }\n\twhile (i < replacement[0]) {\n\t\tif (icon_is(replacement, i, \"$\")) {\n\t\t\tname = icon_value(replacement, i + 1); kind = icon_class(name); c = icon_capture(caps, name, seen[kind]) - 1;\n\t\t\tif (kind) seen[kind]++;\n\t\t\ta = caps[2 + c * 4] * 3; b = caps[3 + c * 4] * 3;\n\t\t\twhile (a < b) { r[1 + j] = q[1 + a]; a++; j++; }\n\t\t\ti += 2;\n\t\t} else {\n\t\t\tk = 0; while (k < 3) { r[1 + j] = replacement[1 + i * 3 + k]; j++; k++; } i++;\n\t\t}\n\t}\n\tr[0] = j / 3;\n\tif (!icon_balanced(r, 0, r[0])) { rt.free(r); icon_error = 1; return 0; }\n\treturn r;\n}\n\n// resumable frames retain their own token list and explicit binding maps.\n// [tokens, position, state, result, error, locals, globals, blocks, steps].\n\n// locate a binding in a frame map by its interned name.\nint icon_binding(word map, int name) {\n\twhile (map) { if (map[0] == name) return map; map = map[2]; }\n\treturn 0;\n}\nint icon_read(word binding) { if (binding[3]) return *binding[3]; return binding[1]; }\nint icon_store(word*.m frame, int column, int name, int value) {\n\tword entry = icon_binding(frame[column], name);\n\tif (!entry) {\n\t\tword* node = rt.new(4); node[0] = name; node[1] = value; node[2] = frame[column]; node[3] = 0; frame[column] = node; return value;\n\t}\n\tentry[1] = value; if (entry[3]) *entry[3] = value; return value;\n}\nint icon_declare(word*.m frame, int name, int value) {\n\tword* node = rt.new(4); node[0] = name; node[1] = value; node[2] = frame[5]; node[3] = 0; frame[5] = no"; - if (i == 4) return b"de; return value;\n}\nint icon_fail(word frame) { frame[2] = 2; frame[4] = 5; icon_error = 5; return 0; }\nint icon_end(word* q, int start) {\n\tint depth = 0, i = start, v;\n\twhile (i < q[0]) {\n\t\tv = icon_value(q, i);\n\t\tif (icon_kind(q, i) == run.T_OP) {\n\t\t\tif (icon_open(v)) depth++;\n\t\t\telse if (icon_close(v)) { if (!depth) return i; depth--; }\n\t\t\tif (!depth && v == run.intern(\";\")) return i;\n\t\t}\n\t\ti++;\n\t}\n\treturn i;\n}\nint icon_mate(word* q, int start) {\n\tint depth = 1, i = start + 1, v;\n\twhile (i < q[0]) {\n\t\tv = icon_value(q, i);\n\t\tif (icon_kind(q, i) == run.T_OP) {\n\t\t\tif (icon_open(v)) depth++; else if (icon_close(v)) depth--;\n\t\t\tif (!depth) return i;\n\t\t}\n\t\ti++;\n\t}\n\treturn -1;\n}\n// find one statement boundary; control-flow captures require braced bodies.\nint icon_statement_end(word* q, int start) {\n\tint close, end;\n\tif (icon_is(q, start, \"{\")) { close = icon_mate(q, start); if (close < 0) return -1; return close + 1; }\n\tif (icon_is(q, start, \"if\") || icon_is(q, start, \"while\")) {\n\t\tif (!icon_is(q, start + 1, \"(\")) return -1;\n\t\tclose = icon_mate(q, start + 1);\n\t\tif (close < 0 || !icon_is(q, close + 1, \"{\")) return -1;\n\t\tend = icon_mate(q, close + 1); if (end < 0) return -1; end++;\n\t\tif (icon_is(q, start, \"if\") && icon_is(q, end, \"else\")) {\n\t\t\tif (!icon_is(q, end + 1, \"{\")) return -1;\n\t\t\tclose = icon_mate(q, end + 1); if (close < 0) return -1; return close + 1;\n\t\t}\n\t\treturn end;\n\t}\n\tend = icon_end(q, start);\n\tif (!icon_is(q, end, \";\")) return -1;\n\treturn end + 1;\n}\n// construct a compiler return body in one allocation, retaining source lines.\nword* .u icon_eval_tokens(word* q, int start, int end) {\n\tint count = end - start;\n\tword* expr = rt.new(7 + count * 3); expr[0] = count + 2;\n\texpr[1] = run.T_ID; expr[2] = run.intern(\"return\"); expr[3] = 1;\n\tif (count) rt.copy_w(expr + 4, q + 1 + start * 3, count * 3);\n\texpr[4 + count * 3] = run.T_OP; expr[5 + count * 3] = run.intern(\";\"); expr[6 + count * 3] = 1;\n\treturn expr;\n}\n// scalar frame substitution cannot preserve lvalue operations or implicit\n// program-global reads. reject these before compiling or executing a range.\nint icon_frame_expression(word frame, int start, int end) {\n\tword* q = frame[0]; int i = start, s; word binding;\n\tif (!icon_range(q, start, end, 3)) return 0;\n\twhile (i < end) {\n\t\tif (icon_is(q, i, \"++\") || icon_is(q, i, \"--\")) return 0;\n\t\tif (icon_kind(q, i) == run.T_ID && !icon_is(q, i - 1, \".\")) {\n\t\t\tbinding = icon_binding(frame[5], icon_value(q, i));\n\t\t\tif (!binding) binding = icon_binding(frame[6], icon_value(q, i));\n\t\t\tif (!binding) {\n\t\t\t\ts = run.lookups(icon_value(q, i));\n\t\t\t\tif (s < 0 || (run.skind[s] != run.K_FN && run.skind[s] != run.K_CONST)) return 0;\n\t\t\t}\n\t\t}\n\t\ti++;\n\t}\n\treturn 1;\n}\nint icon_frame_eval(word frame, int start, int end) {\n\tword* expr = icon_eval_tokens(frame[0], start, end);\n\tint i = 1; word binding;\n\twhile (i < expr[0] - 1) {\n\t\tif (icon_kind(expr, i) == run.T_ID && !icon_is(expr, i - 1, \".\")) {\n\t\t\tbinding = icon_binding(frame[5], icon_value(expr, i));\n\t\t\tif (!bi"; - if (i == 5) return b"nding) binding = icon_binding(frame[6], icon_value(expr, i));\n\t\t\tif (binding) { expr[1 + i * 3] = run.T_NUM; expr[2 + i * 3] = icon_read(binding); }\n\t\t}\n\t\ti++;\n\t}\n\tint result = run.eval(expr); rt.free(expr); return result;\n}\nint icon_block(word*.m frame, int open, int resume) {\n\tint close = icon_mate(frame[0], open);\n\tif (close < 0) return icon_fail(frame);\n\tword block = rt.new(4); block[0] = close; block[1] = resume; block[2] = frame[7]; block[3] = frame[5]; frame[7] = block; frame[1] = open + 1; return 1;\n}\n\nstruct icon {\n\t// the last transformation status: 0 success, 1 invalid delimiters,\n\t// 2 invalid range or declaration, 3 unresolved replacement capture,\n\t// 4 unavailable function metadata, 5 unsupported stepping syntax,\n\t// 6 rejected function publication.\n\tstatic int error() { return icon_error; }\n\t// check token delimiters without compiling or executing the list.\n\tstatic int validate(word* q) { icon_error = 0; if (!q || q[0] < 0 || !icon_balanced(q, 0, q[0])) { icon_error = 1; return 0; } return 1; }\n\t// return an independent token list with [start, end) replaced by insert.\n\tstatic word* .u splice(word* q, int start, int end, word* insert) {\n\t\ticon_error = 0;\n\t\tif (!q || !insert || q[0] < 0 || insert[0] < 0 || start < 0 || end < start || end > q[0]) { icon_error = 2; return 0; }\n\t\tword* r = icon_splice(q, start, end, insert);\n\t\tif (!icon_balanced(r, 0, r[0])) { rt.free(r); icon_error = 1; return 0; }\n\t\treturn r;\n\t}\n\t// replace each nonoverlapping pattern match. inserted tokens are not\n\t// searched again. the input and templates remain unchanged.\n\tstatic word* .u rewrite(word* q, word* pattern, word* replacement) {\n\t\ticon_error = 0;\n\t\tif (!q || q[0] < 0 || !replacement || replacement[0] < 0 || !icon_pattern(pattern) || !icon_balanced(q, 0, q[0])) { icon_error = 1; return 0; }\n\t\tword* result = icon_slice(q, 0, q[0]); word* caps = rt.new(1 + pattern[0] * 4); word* insert, next;\n\t\tint at = 0, end;\n\t\twhile (at < result[0]) {\n\t\t\tcaps[0] = 0; end = icon_match(result, at, pattern, 0, caps);\n\t\t\tif (!end) { at++; continue; }\n\t\t\tinsert = icon_expand(result, replacement, caps);\n\t\t\tif (!insert) { rt.free(result); rt.free(caps); return 0; }\n\t\t\tnext = icon_splice(result, at, end - 1, insert); at += insert[0];\n\t\t\trt.free(insert); rt.free(result); result = next;\n\t\t}\n\t\trt.free(caps); return result;\n\t}\n\t// transform retained function tokens and publish the compiled replacement.\n\t// references to the original entry continue to use the published version.\n\tstatic int find_replace(word function, word* pattern, word* replacement) {\n\t\ticon_error = 0;\n\t\tif (!replacement || replacement[0] < 0 || !icon_pattern(pattern)) { icon_error = 1; return 0; }\n\t\tword* q = run.function_tokens(function);\n\t\tif (!q) { icon_error = 4; return 0; }\n\t\tword* caps = rt.new(1 + pattern[0] * 4); int i = 0, end, count = 0;\n\t\twhile (i < q[0]) { caps[0] = 0; end = icon_match(q, i, pattern, 0, caps); if (end) { count++; i = end - 1; } else i++; }\n\t\trt.free(caps);\n\t\tif (!count) { rt.free(q); "; - if (i == 6) return b"return 0; }\n\t\tword* changed = icon.rewrite(q, pattern, replacement); rt.free(q);\n\t\tif (!changed) return 0;\n\t\tint published = run.replace_function(function, changed); rt.free(changed);\n\t\tif (!published) { icon_error = 6; return 0; }\n\t\treturn count;\n\t}\n\t// remove a complete declaration statement containing the requested name.\n\t// declarations containing multiple bindings are rejected to preserve them.\n\tstatic word* .u undeclare(word* q, char* name) {\n\t\ticon_error = 0;\n\t\tint declaration[4];\n\t\tif (!q || q[0] < 0 || !name || !icon_locate_declaration(q, run.intern(name), declaration)) { icon_error = 2; return 0; }\n\t\tword* empty = rt.new(1); empty[0] = 0;\n\t\tword* result = icon.splice(q, declaration[3], declaration[2] + 1, empty); rt.free(empty); return result;\n\t}\n\t// change the type of the first simple declaration of name.\n\tstatic word* .u change_type(word* q, char* name, word* type) {\n\t\ticon_error = 0;\n\t\tint declaration[4];\n\t\tif (!q || q[0] < 0 || !name || !icon_locate_declaration(q, run.intern(name), declaration)) { icon_error = 2; return 0; }\n\t\treturn icon.splice(q, declaration[0], declaration[1], type);\n\t}\n\t// insert a scoped statement block at the given token boundary.\n\tstatic word* .u insert_block(word* q, int at, word* body) {\n\t\ticon_error = 0;\n\t\tif (!q || !body || body[0] < 0 || at < 0 || at > q[0]) { icon_error = 2; return 0; }\n\t\tword* block = rt.new(7 + body[0] * 3); int i = 0; block[0] = body[0] + 2;\n\t\tblock[1] = run.T_OP; block[2] = run.intern(\"{\"); block[3] = 1;\n\t\twhile (i < body[0] * 3) { block[4 + i] = body[1 + i]; i++; }\n\t\tblock[4 + i] = run.T_OP; block[5 + i] = run.intern(\"}\"); block[6 + i] = 1;\n\t\tword* result = icon.splice(q, at, at, block); rt.free(block); return result;\n\t}\n\t// evaluate a selected expression using the compiler rather than a text\n\t// evaluator. the range must contain one complete expression.\n\tstatic int calculate(word* q, int start, int end) {\n\t\ticon_error = 0;\n\t\tif (!q || start < 0 || end > q[0] || !icon_range(q, start, end, 3)) { icon_error = 2; return 0; }\n\t\tword* expr = icon_eval_tokens(q, start, end);\n\t\tint value = run.eval(expr); rt.free(expr); return value;\n\t}\n\t// start a resumable evaluation with independent tokens and binding storage.\n\t// stepping executes statements through the compiler; no native frame is\n\t// suspended or inspected. scalar bindings are visible between statements.\n\tstatic word* .u begin(word* q) {\n\t\tif (!icon.validate(q)) { icon_error = 1; return 0; }\n\t\tword* frame = rt.new(9); int i = 0; while (i < 9) { frame[i] = 0; i++; } frame[0] = icon_slice(q, 0, q[0]); return frame;\n\t}\n\t// create an explicit frame from a function's retained body. argument values\n\t// are supplied through set_local before stepping starts.\n\tstatic word* .u begin_function(int function) {\n\t\tword* q = run.function_tokens(function);\n\t\tif (!q) { icon_error = 4; return 0; }\n\t\tword* frame = icon.begin(q); rt.free(q); return frame;\n\t}\n\t// set an isolated global binding, or update storage explicitly bound to it.\n\tstatic int set_gl"; - if (i == 7) return b"obal(word frame, char* name, int value) { icon_error = 0; return icon_store(frame, 6, run.intern(name), value); }\n\t// update the innermost local binding, creating a binding when absent.\n\tstatic int set_local(word frame, char* name, int value) { icon_error = 0; return icon_store(frame, 5, run.intern(name), value); }\n\t// bind one existing scalar global explicitly. reads observe its current\n\t// value, and subsequent frame assignments update its program storage.\n\tstatic int bind_global(word frame, char* name) {\n\t\tint s = run.lookups(name); icon_error = 0;\n\t\tif (s < 0 || run.skind[s] != run.K_GVAR) { icon_error = 2; return 0; }\n\t\tword type = run.stype[s];\n\t\tif (type && type != run.tint && type != run.tword) { icon_error = 2; return 0; }\n\t\ticon_store(frame, 6, run.intern(name), *run.sval[s]);\n\t\tword binding = icon_binding(frame[6], run.intern(name)); binding[3] = run.sval[s]; return 1;\n\t}\n\t// read the innermost local binding; a missing binding sets status 2.\n\tstatic int local(word frame, char* name) { icon_error = 0; word b = icon_binding(frame[5], run.intern(name)); if (!b) { icon_error = 2; return 0; } return icon_read(b); }\n\t// read an isolated or explicitly connected global binding.\n\tstatic int global(word frame, char* name) { icon_error = 0; word b = icon_binding(frame[6], run.intern(name)); if (!b) { icon_error = 2; return 0; } return icon_read(b); }\n\t// frame state: 0 resumable, 1 complete, 2 unsupported statement.\n\tstatic int state(word frame) { return frame[2]; }\n\t// the returned value after completion; zero before a return is executed.\n\tstatic int result(word frame) { return frame[3]; }\n\t// token index of the next statement or control-flow decision.\n\tstatic int position(word frame) { return frame[1]; }\n\t// number of executed statements and control-flow decisions.\n\tstatic int steps(word frame) { return frame[8]; }\n\t// number of retained local binding records, including active shadowed records.\n\tstatic int local_count(word frame) { word b = frame[5]; int n = 0; while (b) { n++; b = b[2]; } return n; }\n\t// interned name of a local record; zero when index is outside the map.\n\tstatic char* local_name(word frame, int index) { word b = frame[5]; while (b && index > 0) { index--; b = b[2]; } if (!b || index < 0) return 0; return b[0]; }\n\t// execute one statement or control-flow decision. returns 1 after progress,\n\t// 0 after completion or an unsupported statement. blocks, if and while\n\t// require braces; declarations contain one scalar binding.\n\tstatic int step(word frame) {\n\t\tif (!frame || frame[2]) return 0;\n\t\ticon_error = 0;\n\t\tword* q = frame[0]; int at = frame[1], end, name, value, column = 5, v, close, open, alternate;\n\t\tword block, binding;\n\t\twhile (frame[7] && at == frame[7][0]) {\n\t\t\tblock = frame[7];\n\t\t\twhile (frame[5] != block[3]) { binding = frame[5]; frame[5] = binding[2]; rt.free(binding); }\n\t\t\tat = block[1]; frame[7] = block[2]; rt.free(block); frame[1] = at;\n\t\t}\n\t\tif (at >= q[0]) { frame[2] = 1; return 0; }\n\t\tframe[8]++;\n\t\tif (icon_is(q"; - if (i == 8) return b", at, \";\")) { frame[1] = at + 1; return 1; }\n\t\tif (icon_is(q, at, \"{\")) { close = icon_mate(q, at); return icon_block(frame, at, close + 1); }\n\t\tif (icon_is(q, at, \"if\") || icon_is(q, at, \"while\")) {\n\t\t\tif (!icon_is(q, at + 1, \"(\")) return icon_fail(frame);\n\t\t\tclose = icon_mate(q, at + 1); open = close + 1;\n\t\t\tif (close < 0 || !icon_is(q, open, \"{\")) return icon_fail(frame);\n\t\t\tend = icon_mate(q, open); if (end < 0) return icon_fail(frame);\n\t\t\talternate = end + 1;\n\t\t\tif (icon_is(q, at, \"if\") && icon_is(q, alternate, \"else\")) {\n\t\t\t\tif (!icon_is(q, alternate + 1, \"{\")) return icon_fail(frame);\n\t\t\t}\n\t\t\tif (!icon_frame_expression(frame, at + 2, close)) return icon_fail(frame);\n\t\t\tvalue = icon_frame_eval(frame, at + 2, close);\n\t\t\tif (icon_is(q, at, \"while\")) { if (value) return icon_block(frame, open, at); frame[1] = end + 1; return 1; }\n\t\t\tif (icon_is(q, alternate, \"else\")) {\n\t\t\t\tclose = icon_mate(q, alternate + 1);\n\t\t\t\tif (value) return icon_block(frame, open, close + 1);\n\t\t\t\treturn icon_block(frame, alternate + 1, close + 1);\n\t\t\t}\n\t\t\tif (value) return icon_block(frame, open, end + 1);\n\t\t\tframe[1] = end + 1; return 1;\n\t\t}\n\t\tend = icon_end(q, at);\n\t\tif (end >= q[0] || !icon_is(q, end, \";\")) return icon_fail(frame);\n\t\tif (icon_is(q, at, \"return\")) {\n\t\t\tframe[3] = 0;\n\t\t\tif (end > at + 1) { if (!icon_frame_expression(frame, at + 1, end)) return icon_fail(frame); frame[3] = icon_frame_eval(frame, at + 1, end); }\n\t\t\tframe[1] = end + 1; frame[2] = 1; return 1;\n\t\t}\n\t\tif (icon_is(q, at, \"let\") || icon_is(q, at, \"int\") || icon_is(q, at, \"word\")) {\n\t\t\tif (icon_kind(q, at + 1) != run.T_ID) return icon_fail(frame);\n\t\t\tname = icon_value(q, at + 1); value = 0;\n\t\t\tif (end > at + 2) { if (!icon_is(q, at + 2, \"=\") || !icon_frame_expression(frame, at + 3, end)) return icon_fail(frame); value = icon_frame_eval(frame, at + 3, end); }\n\t\t\ticon_declare(frame, name, value); frame[1] = end + 1; return 1;\n\t\t}\n\t\tname = icon_value(q, at); binding = icon_binding(frame[5], name);\n\t\tif (!binding) { binding = icon_binding(frame[6], name); column = 6; }\n\t\tif (binding && icon_kind(q, at) == run.T_ID) {\n\t\t\tv = icon_value(q, at + 1);\n\t\t\tif (end == at + 2 && (v == run.intern(\"++\") || v == run.intern(\"--\"))) {\n\t\t\t\tvalue = icon_read(binding) + 1; if (v == run.intern(\"--\")) value = icon_read(binding) - 1;\n\t\t\t} else {\n\t\t\t\tif (v != run.intern(\"=\") && v != run.intern(\"+=\") && v != run.intern(\"-=\") && v != run.intern(\"*=\") && v != run.intern(\"/=\")) return icon_fail(frame);\n\t\t\t\tif (!icon_frame_expression(frame, at + 2, end)) return icon_fail(frame);\n\t\t\t\tvalue = icon_frame_eval(frame, at + 2, end);\n\t\t\t\tif (v == run.intern(\"+=\")) value = icon_read(binding) + value;\n\t\t\t\tif (v == run.intern(\"-=\")) value = icon_read(binding) - value;\n\t\t\t\tif (v == run.intern(\"*=\")) value = icon_read(binding) * value;\n\t\t\t\tif (v == run.intern(\"/=\")) value = icon_read(binding) / value;\n\t\t\t}\n\t\t\ticon_store(frame, column, name, value); frame[1] = end + 1; return 1;\n\t\t}\n\t\treturn icon_fail(frame);\n\t}\n\t// execute at most budg"; - if (i == 9) return b"et statements. returning 0 leaves the frame resumable;\n\t// callers can inspect bindings, edit them, then continue with another budget.\n\tstatic int resume(word frame, int budget) { while (budget > 0 && !frame[2]) { icon.step(frame); budget--; } return frame[2]; }\n\t// release a frame and all storage it owns.\n\tstatic int close(.d frame) {\n\t\tword b, next; int column = 5;\n\t\twhile (column <= 7) { b = frame[column]; while (b) { next = b[2]; rt.free(b); b = next; } column++; }\n\t\trt.free(frame[0]); rt.free(frame); return 0;\n\t}\n}\n\n// public entry points for the icon namespace.\n\n// status of the most recent transformation or binding operation; see docs/ICON.md.\nint error() { return icon.icon.error(); }\n// check delimiter structure only; does not compile or execute tokens.\nint validate(word* q) { return icon.icon.validate(q); }\n// return a new token list with [start, end) replaced; caller owns the result.\nword* .u splice(word* q, int start, int end, word* insert) { return icon.icon.splice(q, start, end, insert); }\n// return a new list with nonoverlapping matches replaced; inputs remain unchanged.\nword* .u rewrite(word* q, word* pattern, word* replacement) { return icon.icon.rewrite(q, pattern, replacement); }\n// publish a transformed function body; returns matches, or 0 on no match or failure.\nint find_replace(word function, word* pattern, word* replacement) { return icon.icon.find_replace(function, pattern, replacement); }\n// remove the first complete single-binding declaration named name; caller owns the result.\nword* .u undeclare(word* q, char* name) { return icon.icon.undeclare(q, name); }\n// replace the type prefix of the first simple declaration named name; caller owns the result.\nword* .u change_type(word* q, char* name, word* type) { return icon.icon.change_type(q, name, type); }\n// insert a scoped block containing body at token boundary at; caller owns the result.\nword* .u insert_block(word* q, int at, word* body) { return icon.icon.insert_block(q, at, body); }\n// compile and evaluate the expression in [start, end); expression calls retain their effects.\nint calculate(word* q, int start, int end) { return icon.icon.calculate(q, start, end); }\n// create a frame with copied tokens and isolated bindings; caller closes the frame.\nword* .u begin(word* q) { return icon.icon.begin(q); }\n// create a frame from retained function tokens; set arguments as locals before stepping.\nword* .u begin_function(int function) { return icon.icon.begin_function(function); }\n// set an isolated global binding, creating it when absent; returns the stored value.\nint set_global(word frame, char* name, int value) { return icon.icon.set_global(frame, name, value); }\n// set the innermost local binding, creating it when absent; returns the stored value.\nint set_local(word frame, char* name, int value) { return icon.icon.set_local(frame, name, value); }\n// connect an int or word program global to the frame; later frame writes update it.\nint bind_global(word frame, char* name) { ret"; - if (i == 10) return b"urn icon.icon.bind_global(frame, name); }\n// read a local binding; missing names return 0 and set error status 2.\nint local(word frame, char* name) { return icon.icon.local(frame, name); }\n// read a frame global binding; missing names return 0 and set error status 2.\nint global(word frame, char* name) { return icon.icon.global(frame, name); }\n// return frame state: 0 resumable, 1 complete, or 2 unsupported statement.\nint state(word frame) { return icon.icon.state(frame); }\n// return the value produced by return, or 0 before a return is executed.\nint result(word frame) { return icon.icon.result(frame); }\n// return the token index of the next statement or control-flow decision.\nint position(word frame) { return icon.icon.position(frame); }\n// return the number of statements and control-flow decisions executed.\nint steps(word frame) { return icon.icon.steps(frame); }\n// count retained local records, including shadowed records in active blocks.\nint local_count(word frame) { return icon.icon.local_count(frame); }\n// return the zero-based local record name, or 0 when index is out of range.\nchar* local_name(word frame, int index) { return icon.icon.local_name(frame, index); }\n// execute scalar declarations and assignments, standalone ++/--, return, blocks,\n// or braced if/else/while; direct program globals need bind_global first.\n// unsupported syntax sets error status 5.\nint step(word frame) { return icon.icon.step(frame); }\n// execute at most budget steps and return the frame state; frame bindings are scalar int/word values.\nint resume(word frame, int budget) { return icon.icon.resume(frame, budget); }\n// release the frame and owned storage; consumes frame and returns 0.\nint close(.d frame) { return icon.icon.close(d*frame); }\n"; - return 0; -} -// ---- end embedded icon library ---- +// the source of the std and icon libraries, embedded when the compiler is built, so +// `include std;` and `include icon;` need no file at run time. +embed "src/lib/std.zc" as bstr std_embedded; +embed "src/lib/icon.zc" as bstr icon_embedded; +bstr std_source() { return std_embedded; } +bstr icon_source() { return icon_embedded; } diff --git a/src/core/core.c b/src/core/core.c index 3a1d0ca..dc50644 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -58,13 +58,7 @@ #include <sys/mman.h> #include <ucontext.h> #endif -#ifdef USE_LSP_GENOME -#include "lsp_genome.h" -#elif defined(GENOME_H) -#include GENOME_H -#else #include "genome.h" -#endif #include "memmap.h" #include "fops.h" #include "bops.h" @@ -6208,7 +6202,7 @@ int main(int argc, char **argv) fprintf(stderr, "zy: genome does not fit\n"); return 1; } - memcpy(m + GORG, genome, sizeof genome); + for (int g = 0; g < GLEN; g++) m[GORG + g] = genome_word(g); m[ZFP + 6] = FBOT; m[ZFP + 7] = FTOP; m[ZFP + 8] = 1024; #ifdef ZYJIT { diff --git a/src/core/genome.h b/src/core/genome.h new file mode 100644 index 0000000..9d0ff96 --- /dev/null +++ b/src/core/genome.h @@ -0,0 +1,35 @@ +/* the genome: the image a core boots, embedded into the core when it is + compiled. an image is a file of little-endian 64-bit words placed at GORG, + as written by zc0 or by src/tools/build.zc and checked by mkgenome. the + build keeps every image in build/, never in the tree. + + GENOME_FILE names the image. it defaults to "genome.bin" (or + "lsp_genome.bin" with USE_LSP_GENOME) and is found next to this header, + then in the directories given with --embed-dir, so that option selects + which image a core carries. the file needs a compiler with c23 + #embed (clang 19 or gcc 15 and later). + + no libc here: the microcore includes this on bare metal. */ +#ifndef GENOME_H_INCLUDED +#define GENOME_H_INCLUDED +#include <stdint.h> +#ifndef GENOME_FILE +#ifdef USE_LSP_GENOME +#define GENOME_FILE "lsp_genome.bin" +#else +#define GENOME_FILE "genome.bin" +#endif +#endif +static const unsigned char genome_image[] = { +#embed GENOME_FILE +}; +/* the image's length in words */ +#define GLEN ((int)(sizeof genome_image / 8)) +/* word i of the image, independent of the host's byte order */ +static int64_t genome_word(int i) +{ + const unsigned char *b = genome_image + (long)i * 8; + return (int64_t)((uint64_t)b[0] | (uint64_t)b[1] << 8 | (uint64_t)b[2] << 16 | (uint64_t)b[3] << 24 + | (uint64_t)b[4] << 32 | (uint64_t)b[5] << 40 | (uint64_t)b[6] << 48 | (uint64_t)b[7] << 56); +} +#endif diff --git a/src/core/memmap.h b/src/core/memmap.h deleted file mode 100644 index cc768d4..0000000 --- a/src/core/memmap.h +++ /dev/null @@ -1,34 +0,0 @@ -/* generated by src/tools/genmap for the default profile. do not edit. */ -#ifndef MEMMAP_H -#define MEMMAP_H -#define PW 0x1D5000 -#define CODE_END 0x80000 -#define HEAP_LO 0x1D5000 -#define HEAP_HI 0x235000 -#define HEAP_GROW 1 -#ifndef MW -#define MW 0x8000000 -#endif -#if MW < HEAP_HI -#error MW is below the end of the initial heap -#endif -#define ZFP 0x90000 -#ifdef GORG -#if GORG != 0x114100 -#error genome GORG does not match this map -#endif -#else -#define GORG 0x114100 -#endif -#define FBOT 0x174100 -#define FTOP 0x1B4100 -#define RS0 0x1B4100 -#define MARK0 0x1C4100 -#define MARKN 0x8000 -#define DSLO 0x1CC100 -#define DSHI 0x1D4FF0 -#define DS0 DSHI -#define SYSGATE (ZFP + 2) -#define DSLIM (DSLO + 1100) -#define RSLIM (MARK0 - 8) -#endif diff --git a/src/js/extract_genome.js b/src/js/extract_genome.js deleted file mode 100644 index 4ef2c43..0000000 --- a/src/js/extract_genome.js +++ /dev/null @@ -1,93 +0,0 @@ -// converts the compiler genome and the memory map into the javascript vm's -// layout files. everything it writes is generated, so it goes to build/js/ and -// nothing is left in the tree. -// -// node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR] -// -// the genome is the one make genome produced in build/core/genome.h, and the -// map is src/core/memmap.h. outputs, in the out directory: -// genome.json the genome words, for the vm that has no compiler inside it -// layout.ts the map as typescript constants -// layout.js the map as commonjs constants -const fs = require('fs'); -const path = require('path'); -const root = path.join(__dirname, '..', '..'); - -const args = process.argv.slice(2); -const opt = (name) => { - const i = args.indexOf(name); - return i >= 0 && args[i + 1] ? args[i + 1] : null; -}; -if (args.includes('--help') || args.includes('-h')) { - console.log([ - 'usage: node src/js/extract_genome.js [--genome PATH] [--map PATH] [--out DIR]', - 'converts the compiler genome and memory map into the javascript layout files.', - 'options:', - ' --genome PATH the genome header to read; defaults to build/core/genome.h', - ' --map PATH the memory map header to read; defaults to src/core/memmap.h', - ' --out DIR where to write; defaults to build/js', - ' --help, -h show this help and exit', - ].join('\n')); - process.exit(0); -} -const unknown = args.filter((a) => !['--genome', '--map', '--out'].includes(a)); -if (unknown.length) { - console.error('extract_genome: unexpected arguments: ' + unknown.join(' ') + '; use --help for usage'); - process.exit(2); -} - -const genomePath = opt('--genome') || path.join(root, 'build', 'core', 'genome.h'); -const mapPath = opt('--map') || path.join(__dirname, '..', 'core', 'memmap.h'); -const outDir = opt('--out') || path.join(root, 'build', 'js'); - -if (!fs.existsSync(genomePath)) { - console.error('extract_genome: no genome at ' + genomePath + '; run `make genome` first'); - process.exit(1); -} -fs.mkdirSync(outDir, { recursive: true }); - -const text = fs.readFileSync(genomePath, 'utf8'); -const gorgMatch = text.match(/#define\s+GORG\s+(\d+)/); -const glenMatch = text.match(/#define\s+GLEN\s+(\d+)/); - -const gorg = parseInt(gorgMatch[1], 10); -const glen = parseInt(glenMatch[1], 10); - -const arrMatch = text.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/); -if (!arrMatch) { - console.error('extract_genome: could not find the genome array in ' + genomePath); - process.exit(1); -} - -const words = arrMatch[1] - .split(',') - .map(s => s.trim()) - .filter(s => s.length > 0); - -console.log(`Parsed genome: GORG=${gorg}, GLEN=${words.length}`); - -if (words.length !== glen) throw new Error('genome length does not match GLEN'); - -// the compiler's map, as typescript constants: the js vm reads them instead of -// its own copies. the values are computed here, from the map. -const { mapValues } = require('./mapvalues.js'); -const names = ['MW', 'PW', 'ZFP', 'GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'HEAP_LO', 'HEAP_HI']; -const map = mapValues(mapPath, names); -if (map.GORG !== gorg) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + map.GORG); - -const genomeJson = path.join(outDir, 'genome.json'); -fs.writeFileSync(genomeJson, JSON.stringify({ gorg, glen, words })); -console.log('saved genome to ' + genomeJson); - -const lines = names.map(k => ` ${k}: ${map[k]},`).join('\n'); -const layoutTs = path.join(outDir, 'layout.ts'); -fs.writeFileSync(layoutTs, - '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' + - 'export const MAP = {\n' + lines + '\n} as const;\n'); -console.log('saved the map to ' + layoutTs); - -const layoutJs = path.join(outDir, 'layout.js'); -fs.writeFileSync(layoutJs, - '// generated by src/js/extract_genome.js from ' + path.relative(root, mapPath) + '. do not edit.\n' + - 'exports.MAP = ' + JSON.stringify(map, null, 4) + ';\n'); -console.log('saved the map to ' + layoutJs);
\ No newline at end of file diff --git a/src/js/extract_lsp.js b/src/js/extract_lsp.js index 14ee279..bdbcb3c 100644 --- a/src/js/extract_lsp.js +++ b/src/js/extract_lsp.js @@ -1,13 +1,12 @@ -// writes the language server's javascript image: the genome build/lsp/genome.h, -// made by src/tools/mklsp.sh, and the map it was built for, read from -// build/lsp/map/memmap.h. the vm needs the layout, because the server's map is -// not the compiler's. the image is generated, so it goes to build/js/ and -// nothing is left in the tree. +// writes the language server's javascript image: build/js/lsp.bin, the server's +// genome (build/lsp/lsp_genome.bin, made by src/tools/mklsp.sh), and +// build/js/lsp.json, the map it was built for, read from build/lsp/memmap.h. the +// vm needs the layout, because the server's map is not the compiler's. the image +// is generated, so it goes to build/js/ and nothing is left in the tree. // -// node src/js/extract_lsp.js [--out PATH] +// node src/js/extract_lsp.js [--lsp DIR] [--out DIR] // -// build the lsp first (`make lsp`, or src/tools/mklsp.sh); it needs -// build/lsp/genome.h. +// build the lsp first (`make lsp`, or src/tools/mklsp.sh). const fs = require('fs'); const path = require('path'); const root = path.join(__dirname, '..', '..'); @@ -15,42 +14,39 @@ const root = path.join(__dirname, '..', '..'); const args = process.argv.slice(2); if (args.includes('--help') || args.includes('-h')) { console.log([ - 'usage: node src/js/extract_lsp.js [--out PATH]', - 'converts the language server genome and its memory map into a javascript image.', + 'usage: node src/js/extract_lsp.js [--lsp DIR] [--out DIR]', + 'copies the language server genome beside a json description of its memory map.', 'options:', - ' --out PATH where to write; defaults to build/js/lsp.json', + ' --lsp DIR the language server build to read; defaults to build/lsp', + ' --out DIR where to write lsp.bin and lsp.json; defaults to build/js', ' --help, -h show this help and exit', ].join('\n')); process.exit(0); } -let out = path.join(root, 'build', 'js', 'lsp.json'); +let out = path.join(root, 'build', 'js'), lsp = path.join(root, 'build', 'lsp'); for (let i = 0; i < args.length; i++) { - if (args[i] === '--out' && args[i + 1]) out = args[i + 1]; + if (args[i] === '--out' && args[i + 1]) out = args[++i]; + else if (args[i] === '--lsp' && args[i + 1]) lsp = args[++i]; else { console.error('extract_lsp: unexpected arguments; use --help for usage'); process.exit(2); } } -const genomePath = path.join(root, 'build', 'lsp', 'genome.h'); -const mapPath = path.join(root, 'build', 'lsp', 'map', 'memmap.h'); +const genomePath = path.join(lsp, 'lsp_genome.bin'); +const mapPath = path.join(lsp, 'memmap.h'); if (!fs.existsSync(genomePath)) { console.error('extract_lsp: no language server image at ' + genomePath + '; run `make lsp` first'); process.exit(1); } -const genome = fs.readFileSync(genomePath, 'utf8'); - const { mapValues } = require('./mapvalues.js'); const layout = mapValues(mapPath, ['GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'ZFP', 'HEAP_LO', 'HEAP_HI']); +const image = fs.readFileSync(genomePath); +if (image.length % 8) throw new Error(genomePath + ' is not a whole number of words'); +const glen = image.length / 8; -const gorg = parseInt(genome.match(/#define\s+GORG\s+(\d+)/)[1], 10); -const glen = parseInt(genome.match(/#define\s+GLEN\s+(\d+)/)[1], 10); -const arr = genome.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/); -const words = arr[1].split(',').map(s => s.trim()).filter(s => s.length > 0); -if (words.length !== glen) throw new Error('genome has ' + words.length + ' words, GLEN says ' + glen); -if (gorg !== layout.GORG) throw new Error('genome GORG ' + gorg + ' is not the map GORG ' + layout.GORG); - -fs.mkdirSync(path.dirname(out), { recursive: true }); -fs.writeFileSync(out, JSON.stringify({ gorg, glen, layout, words })); -console.log('wrote ' + path.relative(root, out) + ': genome ' + glen + ' words at ' + gorg + - ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI);
\ No newline at end of file +fs.mkdirSync(out, { recursive: true }); +fs.copyFileSync(genomePath, path.join(out, 'lsp.bin')); +fs.writeFileSync(path.join(out, 'lsp.json'), JSON.stringify({ gorg: layout.GORG, glen, layout })); +console.log('wrote ' + path.relative(root, path.join(out, 'lsp.bin')) + ': genome ' + glen + ' words at ' + layout.GORG + + ', heap ' + layout.HEAP_LO + '..' + layout.HEAP_HI); diff --git a/src/js/layout.d.ts b/src/js/layout.d.ts index 1e8c004..9a531eb 100644 --- a/src/js/layout.d.ts +++ b/src/js/layout.d.ts @@ -1,5 +1,5 @@ // types for the map the javascript vm reads at runtime. the values are -// generated by src/js/extract_genome.js into build/js/layout.js, which sits +// generated by src/js/mklayout.js into build/js/layout.js, which sits // beside the compiled vm; this declaration is here so tinyvm.ts typechecks // without a generated file in the tree. export declare const MAP: { diff --git a/src/js/mapvalues.js b/src/js/mapvalues.js index f701ba2..bb6fc86 100644 --- a/src/js/mapvalues.js +++ b/src/js/mapvalues.js @@ -1,6 +1,6 @@ // reads the values of a machine map from its memmap.h. the #define lines may use // other names and small sums (ZFP + 2, DSLO + 1100), so each value is evaluated -// with its names substituted. shared by extract_genome.js and extract_lsp.js. +// with its names substituted. shared by mklayout.js and extract_lsp.js. const fs = require('fs'); function mapValues(mapPath, names) { diff --git a/src/js/mklayout.js b/src/js/mklayout.js new file mode 100644 index 0000000..cf95091 --- /dev/null +++ b/src/js/mklayout.js @@ -0,0 +1,61 @@ +// writes the javascript vm's view of a memory map: the map's constants as +// typescript and commonjs modules. the vm reads its genome image itself, from +// genome.bin beside tinyvm.js, so only the layout is generated here. everything +// it writes is generated, so it goes to build/js/ and nothing is left in the tree. +// +// node src/js/mklayout.js [--map PATH] [--out DIR] +// +// the map is the default one `make` generates in build/core/map/memmap.h. +// outputs, in the out directory: +// layout.ts the map as typescript constants +// layout.js the map as commonjs constants +const fs = require('fs'); +const path = require('path'); +const root = path.join(__dirname, '..', '..'); + +const args = process.argv.slice(2); +const opt = (name) => { + const i = args.indexOf(name); + return i >= 0 && args[i + 1] ? args[i + 1] : null; +}; +if (args.includes('--help') || args.includes('-h')) { + console.log([ + 'usage: node src/js/mklayout.js [--map PATH] [--out DIR]', + 'converts a memory map header into the javascript vm layout files.', + 'options:', + ' --map PATH the memory map header to read; defaults to build/core/map/memmap.h', + ' --out DIR where to write; defaults to build/js', + ' --help, -h show this help and exit', + ].join('\n')); + process.exit(0); +} +for (let i = 0; i < args.length; i += 2) { + if (!['--map', '--out'].includes(args[i]) || args[i + 1] === undefined) { + console.error('mklayout: unexpected arguments: ' + args.join(' ') + '; use --help for usage'); + process.exit(2); + } +} + +const mapPath = opt('--map') || path.join(root, 'build', 'core', 'map', 'memmap.h'); +const outDir = opt('--out') || path.join(root, 'build', 'js'); +if (!fs.existsSync(mapPath)) { + console.error('mklayout: no memory map at ' + mapPath + '; run `make` first'); + process.exit(1); +} +fs.mkdirSync(outDir, { recursive: true }); + +// the compiler's map, as constants: the js vm reads them instead of its own copies. +const { mapValues } = require('./mapvalues.js'); +const names = ['MW', 'PW', 'ZFP', 'GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'HEAP_LO', 'HEAP_HI']; +const map = mapValues(mapPath, names); + +const lines = names.map(k => ` ${k}: ${map[k]},`).join('\n'); +const layoutTs = path.join(outDir, 'layout.ts'); +fs.writeFileSync(layoutTs, + '// generated by src/js/mklayout.js from ' + path.relative(root, mapPath) + '. do not edit.\n' + + 'export const MAP = {\n' + lines + '\n} as const;\n'); +const layoutJs = path.join(outDir, 'layout.js'); +fs.writeFileSync(layoutJs, + '// generated by src/js/mklayout.js from ' + path.relative(root, mapPath) + '. do not edit.\n' + + 'exports.MAP = ' + JSON.stringify(map, null, 4) + ';\n'); +console.log('wrote ' + path.relative(root, layoutJs) + ' and layout.ts'); diff --git a/src/js/tinyvm.ts b/src/js/tinyvm.ts index 4b4bc39..56fab70 100644 --- a/src/js/tinyvm.ts +++ b/src/js/tinyvm.ts @@ -10,7 +10,7 @@ declare const process: any, require: any, module: any, __dirname: string, Buffer const fs: any = typeof require === 'function' ? require('fs') : undefined; const path: any = typeof require === 'function' ? require('path') : undefined; -// the compiler's map, computed from src/core/memmap.h at build time (build/js/layout.js) +// the compiler's map, computed from build/core/map/memmap.h at build time (build/js/layout.js) import { MAP } from './layout'; // float bits stay in vm words; arithmetic uses javascript's binary64 numbers. @@ -257,6 +257,16 @@ export class TinyVM { } } + /** load an image file's bytes (little-endian 64-bit words, the format the c cores + #embed) at org. returns the number of words. */ + public loadImage(bytes: Uint8Array, org: number = this.L.GORG): number { + if (bytes.length % 8) throw new RangeError('an image is a whole number of 64-bit words'); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length); + const n = bytes.length / 8; + for (let i = 0; i < n; i++) this.mem[org + i] = view.getBigInt64(i * 8, true); + return n; + } + public getOutput(): string { return new TextDecoder().decode(new Uint8Array(this.outputBuffer)); } @@ -931,13 +941,13 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma process.stderr.write('tinyvm.js: unknown option ' + option + '; use --help for usage\n'); process.exit(2); } - const genomePath = path.join(__dirname, 'genome.json'); + const genomePath = path.join(__dirname, 'genome.bin'); if (!fs.existsSync(genomePath)) { - console.error('Error: genome.json not found. Run extract_genome.js first.'); + console.error('Error: genome.bin not found beside tinyvm.js. Run make js first.'); process.exit(1); } - const genomeData = JSON.parse(fs.readFileSync(genomePath, 'utf8')); + const genome: Uint8Array = fs.readFileSync(genomePath); let inputSource = ''; if (process.argv.length > 2) { @@ -970,9 +980,9 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma onMessage: (msg: string) => process.stderr.write(msg) }); - vm.loadGenome(genomeData.words, genomeData.gorg); + vm.loadImage(genome, GORG); vm.setInput(inputSource); - const code = vm.run(genomeData.gorg); + const code = vm.run(GORG); process.exit(code); } diff --git a/src/lsp/server.zc b/src/lsp/server.zc index a07e4d4..48fdf90 100644 --- a/src/lsp/server.zc +++ b/src/lsp/server.zc @@ -1057,8 +1057,15 @@ int doc_gap(int base, int from, int to) { return 1; } // remove comment delimiters and optional line-leading stars; retain markdown. +// the line comment emitted last: consecutive // lines are one paragraph, joined by a soft break +int hct_doc = 0 - 1, hct_end, hct_hn; +int hover_comment_adjacent(int base, int from, int to) { + int lines = 0, c; + while (from < to) { c = dc(base, from); if (c == 10) lines++; else if (c != 32 && c != 9 && c != 13) return 0; from++; } + return lines <= 1; +} int hover_comment_text(int i, int index) { - int base = doff[i], o = hc_start[index] + 2, end = hc_end[index], bol = 1, c, before = hn; + int base = doff[i], o = hc_start[index] + 2, end = hc_end[index], bol = 1, c, before = hn, joined; if (hc_kind[index] == 3) { if (end >= o + 2 && dc(base, end - 2) == '*' && dc(base, end - 1) == '/') end = end - 2; if (o < end && dc(base, o) == '*') o++; @@ -1066,7 +1073,8 @@ int hover_comment_text(int i, int index) { while (o < end && (dc(base, o) == 32 || dc(base, o) == 9 || dc(base, o) == 10 || dc(base, o) == 13)) o++; while (end > o && (dc(base, end - 1) == 32 || dc(base, end - 1) == 9 || dc(base, end - 1) == 10 || dc(base, end - 1) == 13)) end--; if (o == end) return 0; - hov_text("\n\n"); + joined = hc_kind[index] != 3 && hct_doc == i && hct_hn == hn && hn > 0 && hover_comment_adjacent(base, hct_end, hc_start[index]); + if (joined) hov_put(10); else hov_text("\n\n"); while (o < end) { if (bol && hc_kind[index] == 3) { while (o < end && (dc(base, o) == 32 || dc(base, o) == 9)) o++; @@ -1078,6 +1086,7 @@ int hover_comment_text(int i, int index) { if (c != 13) hov_put(c); if (c == 10) bol = 1; } + if (hc_kind[index] != 3) { hct_doc = i; hct_end = hc_end[index]; hct_hn = hn; } else hct_doc = 0 - 1; return hn - before; } // a trailing declaration comment belongs to its last name, not the next line. @@ -2085,18 +2094,21 @@ int do_nav(int i, int refs) { } //: ---- the sources of this image ---- -//: the image carries the text of rt.zc, compiler.zc, server.zc, std.zc, icon.zc and core.zc (the -//: pieces src/tools/embed.sh made), so the definition of a built-in has somewhere to go. each source is a document +//: the image carries the text of rt.zc, compiler.zc, server.zc, std.zc, icon.zc and core.zc, embedded +//: when the image is built, so the definition of a built-in has somewhere to go. each source is a document //: of its own at tinyvm:/NAME.zc, which the editor shows read only and can ask for with zc/source const int BS_COUNT = 6; -bstr rt_part(int i); bstr compiler_part(int i); bstr server_part(int i); bstr core_part(int i); -bstr bs_part(int k, int i) { - if (k == 0) return rt_part(i); - if (k == 1) return compiler_part(i); - if (k == 2) return server_part(i); - if (k == 3) return run.std_part(i); - if (k == 4) return run.icon_part(i); - return core_part(i); +embed "src/runtime/rt.zc" as bstr rt_embedded; +embed "src/compiler/compiler.zc" as bstr compiler_embedded; +embed "src/lsp/server.zc" as bstr server_embedded; +embed "src/core/core.zc" as bstr core_embedded; +bstr bs_text(int k) { + if (k == 0) return rt_embedded; + if (k == 1) return compiler_embedded; + if (k == 2) return server_embedded; + if (k == 3) return run.std_source(); + if (k == 4) return run.icon_source(); + return core_embedded; } char* bs_uri(int k) { if (k == 0) return "tinyvm:/rt.zc"; @@ -2108,21 +2120,15 @@ char* bs_uri(int k) { } // the document of built-in source k, loaded the first time it is asked for int bs_load(int k) { - int i, n = 0, j = 0, b; bstr p; word text; + int i, n = 0, j = 0; bstr p; word text; if (!bs_idx) bs_idx = lsp_grow(0, 0, BS_COUNT); if (bs_idx[k]) return bs_idx[k] - 1; - while (1) { p = bs_part(k, j); if (!p) break; n = n + rt.len_s(p); j++; } - text = lsp_grow(0, 0, (n >> 2) + 3); - n = 0; j = 0; - while (1) { - p = bs_part(k, j); if (!p) break; - b = 0; - while (b < rt.len_s(p)) { - if ((n & 3) == 0) text[n >> 2] = 0; - text[n >> 2] = text[n >> 2] | (rt.at_s(p, b) << ((n & 3) * 8)); - n++; b++; - } - j++; + p = bs_text(k); j = rt.len_s(p); + text = lsp_grow(0, 0, (j >> 2) + 3); + while (n < j) { + if ((n & 3) == 0) text[n >> 2] = 0; + text[n >> 2] = text[n >> 2] | (rt.at_s(p, n) << ((n & 3) * 8)); + n++; } i = docnew(bs_uri(k), rt.len_w(bs_uri(k))); doff[i] = text; dsize[i] = n; dver[i] = 1; dgen[i]++; @@ -2334,8 +2340,13 @@ int fm_symbol(int base, int o, int size) { return n; } // split document i into tokens, each with the spaces before it on its line (-1 for the first of a line) +// the document the token arrays describe: hover lookups lex the same documents again and +// again, so a lex of an unchanged document is reused +int fm_doc = 0 - 1, fm_gen, fm_size; word fm_base; int fm_lex(int i) { int base = doff[i], size = dsize[i], o = 0, ln = 0, gap = 0 - 1, c, s, q, e, el, n; + if (fm_doc == i && fm_gen == dgen[i] && fm_base == base && fm_size == size && fm_s) return 0; + fm_doc = i; fm_gen = dgen[i]; fm_base = base; fm_size = size; fm_nt = 0; if (!fm_s) { fm_tcap = 256; diff --git a/src/micro/micro.c b/src/micro/micro.c index 528382a..f598c80 100644 --- a/src/micro/micro.c +++ b/src/micro/micro.c @@ -30,13 +30,7 @@ #else #define VM_TLS #endif -#ifdef USE_LSP_GENOME -#include "lsp_genome.h" -#elif defined(GENOME_H) -#include GENOME_H -#else #include "genome.h" -#endif #include "micro_port.h" #include "fops.h" #include "bops.h" @@ -779,7 +773,7 @@ int micro_run(void) micro_msg("zy: genome does not fit\n"); return 1; } - for (i = 0; i < GLEN; i++) m[GORG + i] = genome[i]; + for (i = 0; i < GLEN; i++) m[GORG + i] = genome_word(i); jknown = 0; return run(GORG); } diff --git a/src/runtime/rt.zc b/src/runtime/rt.zc index ee3594d..a54d88c 100644 --- a/src/runtime/rt.zc +++ b/src/runtime/rt.zc @@ -760,7 +760,10 @@ word .u new_t(int n, int t) { if (dbg & 1) note_alloc(p, alloc_line); return p; } -// allocate n words without a type stamp; the caller owns the fresh object. +// new(n) allocates n zeroed words without a type stamp. new(T) and new(T, n) allocate +// a fresh heap object of type T (n of them for the second form) and stamp T in its +// header; the compiler rewrites those forms to new_t or new_k. the caller owns it, +// and the analyzer frees it when its owner's block ends. word .u new(int n) { return new_t(n, 0); } // the compiler works out the size class of a constant-size new at compile // time (k is the class of n + HDR words), so a repeated allocation of the same @@ -1380,3 +1383,28 @@ char* .u readfile(char* path) { syscall(SYS_CLOSE, fd, 0, 0); return buf; } +// reads a file's bytes into an owned block, one byte to a word after a length word: +// [n, b0, b1, ...]. unlike readfile, a 0 byte in the file is kept. returns null if +// opening fails. +word.u readbytes(char* path) { + int fd; word buf; int n = 0, got, cap; word nb; + fd = syscall(SYS_OPEN, path, 0, 0); + if (fd < 0) return 0; + cap = 256; + buf = new(cap); + while (1) { + if (n + 130 >= cap) { + nb = new(cap + cap); + copy_w(nb, buf, n + 1); + free(buf); + buf = nb; + cap = cap + cap; + } + got = syscall(SYS_READ, fd, buf + 1 + n, 128); + if (got <= 0) break; + n = n + got; + } + buf[0] = n; + syscall(SYS_CLOSE, fd, 0, 0); + return buf; +} diff --git a/src/tools/automap.sh b/src/tools/automap.sh index 69849ca..ff559ac 100755 --- a/src/tools/automap.sh +++ b/src/tools/automap.sh @@ -3,7 +3,8 @@ # src/tools/automap.sh PRESET OUTDIR FILE... # PRESET is run, small, default or lsp. DOCS=1 makes the genome with its doc comments. FILE... are appended after the map's own # rt.zc (and compiler.zc, for the presets that carry the compiler). the result -# is OUTDIR/memmap.h and OUTDIR/genome.h, and they agree with each other. +# is OUTDIR/memmap.h and OUTDIR/genome.bin, and they agree with each other. a core +# compiled with -IOUTDIR --embed-dir=OUTDIR embeds that image (src/core/genome.h). # # a genome's length doesn't depend on where it lives (every address is a one # word literal), so two passes are enough: compile once on the preset's map @@ -14,15 +15,14 @@ # GENMAP can name a genmap that is already built. set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' 'builds a genome and memory map sized to the supplied source files.' 'output: OUTDIR/genome.h and OUTDIR/memmap.h' 'environment: CC selects the c compiler; DOCS=1 retains documentation; GENMAP selects genmap.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' 'builds a genome and memory map sized to the supplied source files.' 'output: OUTDIR/genome.bin and OUTDIR/memmap.h' 'environment: CC selects the c compiler; DOCS=1 retains documentation; GENMAP selects genmap.'; exit 0 ;; -*) echo "automap: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." [[ $# -ge 2 ]] || { echo 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' >&2; exit 2; } P=$1; O=$2; shift 2 [ -n "$P" ] && [ -n "$O" ] || { echo "usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE..." >&2; exit 2; } -CC=${CC:-gcc} -bash src/tools/embed.sh all --check || bash src/tools/embed.sh all +CC=${CC:-$(sh src/tools/findcc.sh)} mkdir -p "$O" ZC0=$O/.zc0 if [ -z "$GENMAP" ]; then @@ -33,32 +33,36 @@ lay() { # $1 = genome slot, or empty for the preset's own rm -rf "$O/map" "$GENMAP" --preset "$P" --grow ${1:+--genome $1} -o "$O/map" >/dev/null 2>&1 $CC -O1 -w -fwrapv src/tools/zc0.c -I"$O/map" -Isrc/core -o "$ZC0" + $CC -O1 -w -fwrapv src/tools/mkgenome.c -I"$O/map" -o "$O/.mkgenome" } build() { - if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$ZC0" + if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$ZC0" > "$O/image.bin" + "$O/.mkgenome" < "$O/image.bin" > "$O/genome.bin" } # with DOCS=1 the genome is made again with zc0 built for docs (ZC0DOCS), so the # image carries the doc comments of the sources and the program can answer # docofs(). the image is the same code either way; the docs are stored after it. docs() { $CC -O1 -w -fwrapv -DZC0DOCS src/tools/zc0.c -I"$O/map" -Isrc/core -o "$O/.zc0docs" - if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$O/.zc0docs" + if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$O/.zc0docs" > "$O/image.bin" + "$O/.mkgenome" < "$O/image.bin" > "$O/genome.bin" } -len() { grep -m1 '#define GLEN' "$1" | awk '{print $3}'; } +len() { echo $(( $(wc -c < "$1") / 8 )); } lay "" -build "$@" > "$O/genome.h" -glen=$(len "$O/genome.h") +build "$@" +glen=$(len "$O/genome.bin") slot=$(( (glen + 4095) / 4096 * 4096 )) lay "$slot" -build "$@" > "$O/genome.h" +build "$@" if [ -n "$DOCS" ]; then lay $((slot + 65536)) # room for the image with its docs while it is being made - docs "$@" > "$O/genome.h" - glen=$(len "$O/genome.h") + docs "$@" + glen=$(len "$O/genome.bin") slot=$(( (glen + 4095) / 4096 * 4096 )) lay "$slot" - docs "$@" > "$O/genome.h" + docs "$@" fi -[ "$(len "$O/genome.h")" = "$glen" ] || { echo "automap: the genome changed length with the map" >&2; exit 1; } +rm -f "$O/image.bin" +[ "$(len "$O/genome.bin")" = "$glen" ] || { echo "automap: the genome changed length with the map" >&2; exit 1; } cp "$O/map/memmap.h" "$O/map/map.txt" "$O/" echo "automap $P: genome $glen words in a slot of $slot, $(grep -m1 '^MW' "$O/map/map.txt")" diff --git a/src/tools/bootstrap-check.sh b/src/tools/bootstrap-check.sh index a5de341..7acd74c 100755 --- a/src/tools/bootstrap-check.sh +++ b/src/tools/bootstrap-check.sh @@ -2,19 +2,23 @@ # rebuild the bootstrap one stage and report what the compiler says about itself. # zc0 (stage 0) compiles rt+core+compiler into build/s0, s0 becomes zy0, and zy0 # runs build.zc over the same sources to produce the image. ownership diagnostics -# come back in the image, so print them instead of the word dump. +# come back in place of the image; mkgenome rejects them and prints them. set -e +case "${1:-}" in + --help|-h) printf '%s\n' 'usage: src/tools/bootstrap-check.sh' 'runs the first two bootstrap stages and prints the compiler diagnostics on its own source, or the image size.'; exit 0 ;; + -*) echo "bootstrap-check: unknown option $1; use --help for usage" >&2; exit 2 ;; +esac cd "$(dirname "$0")/../.." -mkdir -p bin build/s0 build/s1 build/core -cc -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -o bin/zc0 -cc -O1 -w -fwrapv src/tools/mkgenome.c -Isrc/core -o bin/mkgenome -cp src/core/core.c src/core/io_linux.c src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h build/s0/ -cat src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc | bin/zc0 > build/s0/genome.h -cc -O2 -w -fwrapv -pthread -DJTYPE=int build/s0/core.c build/s0/io_linux.c -o build/zy0 -(cat src/tools/build.zc; printf '`'; cat src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc) | build/zy0 > build/g1.bin -if ! grep -q '^zc: ' build/g1.bin 2>/dev/null; then - echo "image ok: $(wc -c < build/g1.bin) bytes" +make -s build/bin/zc0 build/bin/mkgenome +CC=${CC:-$(sh src/tools/findcc.sh)} +SRC="src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc" +mkdir -p build/s0 build/s1 +cat $SRC | build/bin/zc0 > build/s0/g0.bin +build/bin/mkgenome < build/s0/g0.bin > build/s0/genome.bin +$CC -O2 -w -fwrapv -pthread -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/s0 -o build/s0/zy0 +(cat src/tools/build.zc; printf '`'; cat $SRC) | build/s0/zy0 > build/s1/g1.bin +if build/bin/mkgenome < build/s1/g1.bin > build/s1/check.bin; then + echo "image ok: $(wc -c < build/s1/g1.bin) bytes" else - echo "--- compiler diagnostics on its own source ---" - cat -v build/g1.bin -fi
\ No newline at end of file + exit 1 +fi diff --git a/src/tools/embed.sh b/src/tools/embed.sh deleted file mode 100755 index 59a377f..0000000 --- a/src/tools/embed.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# embeds source text in the compiler, so the editor can show a built-in's definition and `include std;` needs -# no file. a text becomes a function that returns it in pieces of at most 3000 bytes (the bootstrap -# compiler's token buffer is 4 KB), one byte string per piece: -# bstr NAME_part(int i) { if (i == 0) return b"..."; if (i == 1) return b"..."; ... return 0; } -# -# src/tools/embed.sh all embeds src/lib modules in the compiler -# src/tools/embed.sh all --check exits 1 when an embedded module is out of date -# src/tools/embed.sh lsp OUT writes OUT: rt_part, compiler_part and server_part, for the language server's image -set -e -case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/embed.sh std|icon|all [--check]' ' src/tools/embed.sh lsp OUT' 'embeds library sources in the compiler, or writes the lsp source providers to OUT.' 'options:' ' --check report stale embedded library sources without changing files' ' --help, -h show this help and exit'; exit 0 ;; - -*) echo "embed: unknown option $1; use --help for usage" >&2; exit 2 ;; -esac -cd "$(dirname "$0")/../.." -export LC_ALL=C -# parts NAME FILE: the NAME_part function for FILE -parts() { - echo "bstr $1_part(int i) {" - awk -v LIMIT=3000 ' - function flush() { if (cnt > 0) { printf("\tif (i == %d) return b\"%s\";\n", n, buf); n++; buf = ""; cnt = 0 } } - { - line = $0; sub(/\r$/, "", line); line = line "\n" - len = length(line) - for (j = 1; j <= len; j++) { - if (cnt >= LIMIT) flush() - c = substr(line, j, 1) - if (c == "\\") e = "\\\\"; else if (c == "\"") e = "\\\""; else if (c == "\t") e = "\\t"; else if (c == "\n") e = "\\n"; else e = c - buf = buf e; cnt++ - } - } - END { flush() }' "$2" - echo " return 0;" - echo "}" -} -embed() { - name=$1; file=$2; check=$3; C=src/compiler/compiler.zc; T=$(mktemp -d) - BEGIN_MARK="// ---- embedded $name library (generated by src/tools/embed.sh from $file; do not edit) ----" - END_MARK="// ---- end embedded $name library ----" - if grep -qF "$BEGIN_MARK" "$C"; then - awk -v b="$BEGIN_MARK" '$0 == b { exit } { print }' "$C" > "$T/head" - awk -v e="$END_MARK" 'found { print } $0 == e { found = 1; next }' "$C" > "$T/tail" - else - cp "$C" "$T/head" - : > "$T/tail" - fi - { - cat "$T/head" - echo "$BEGIN_MARK" - parts "$name" "$file" - echo "$END_MARK" - cat "$T/tail" - } > "$T/new" - if [ "$check" = "--check" ]; then - cmp -s "$T/new" "$C" && result=0 || result=1 - rm -rf "$T" - return "$result" - fi - cp "$T/new" "$C" - rm -rf "$T" -} -case "$1" in -std) - embed std src/lib/std.zc "$2" - ;; -icon) - embed icon src/lib/icon.zc "$2" - ;; -all) - if [ "$2" = "--check" ]; then - embed std src/lib/std.zc --check && embed icon src/lib/icon.zc --check - else - embed std src/lib/std.zc - embed icon src/lib/icon.zc - fi - ;; -lsp) - [ -n "$2" ] || { echo "usage: src/tools/embed.sh lsp OUT" >&2; exit 2; } - { - echo "// the sources of this image, embedded (generated by src/tools/embed.sh; do not edit)" - parts rt src/runtime/rt.zc - parts compiler src/compiler/compiler.zc - parts server src/lsp/server.zc - parts core src/core/core.zc - } > "$2" - ;; -*) - echo "usage: src/tools/embed.sh all [--check] | std | icon | lsp OUT" >&2; exit 2;; -esac diff --git a/src/tools/findcc.sh b/src/tools/findcc.sh new file mode 100755 index 0000000..caa3af3 --- /dev/null +++ b/src/tools/findcc.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# prints the first c compiler that accepts c23 #embed, which every core needs to +# take in its genome (src/core/genome.h). clang 19 and gcc 15 are the first +# releases with it, and both find #embed files through --embed-dir. a CC +# already in the environment is tried first. +# src/tools/findcc.sh a native compiler +# src/tools/findcc.sh --wasm a clang that also targets wasm32 +# exits 1, naming what was tried, when nothing fits. +case "${1:-}" in + --help|-h) printf '%s\n' 'usage: src/tools/findcc.sh [--wasm]' 'prints a c compiler that supports #embed; --wasm also requires the wasm32 target.' 'environment: CC is tried first.'; exit 0 ;; + --wasm|'') ;; + *) echo "findcc: unknown option $1; use --help for usage" >&2; exit 2 ;; +esac +wasm=${1:-} +dir=$(mktemp -d "${TMPDIR:-/tmp}/findcc.XXXXXX") || exit 1 +trap 'rm -rf "$dir"' EXIT +mkdir "$dir/data" +printf 'ab' > "$dir/data/probe.bin" +printf 'static const unsigned char probe[] = {\n#embed "probe.bin"\n};\nint main(void) { return sizeof probe == 2 ? 0 : 1; }\n' > "$dir/probe.c" +tried="" +for cc in ${CC:-} cc clang gcc clang-22 clang-21 clang-20 clang-19 gcc-16 gcc-15 /opt/homebrew/opt/llvm/bin/clang /usr/local/opt/llvm/bin/clang; do + command -v "$cc" >/dev/null 2>&1 || continue + tried="$tried $cc" + if [ -n "$wasm" ]; then + "$cc" --target=wasm32 -nostdlib -w --embed-dir="$dir/data" -c "$dir/probe.c" -o "$dir/probe.o" >/dev/null 2>&1 || continue + else + "$cc" -w --embed-dir="$dir/data" "$dir/probe.c" -o "$dir/probe" >/dev/null 2>&1 || continue + fi + echo "$cc" + exit 0 +done +echo "findcc: no c compiler with #embed${wasm:+ and wasm32} (tried:${tried:- nothing}); install clang 19 or gcc 15, or set CC" >&2 +exit 1 diff --git a/src/tools/genmap.c b/src/tools/genmap.c index 8a258e1..d063d36 100644 --- a/src/tools/genmap.c +++ b/src/tools/genmap.c @@ -609,9 +609,11 @@ static void expect_hex(const char *text, const char *where, const char *name, in static void check_sources(const Map *m) { char lfa_old[64]; + /* a memmap.h in the working directory is compared too, when there is one: the + default map's header is generated into build/core and never kept in the tree */ FILE *mf = fopen("memmap.h", "rb"); - const char *mm_path = mf ? (fclose(mf), "memmap.h") : "src/core/memmap.h"; - char *rt = slurp("src/runtime/rt.zc"), *cc = slurp("src/compiler/compiler.zc"), *mm = slurp(mm_path); + const char *mm_path = mf ? (fclose(mf), "memmap.h") : 0; + char *rt = slurp("src/runtime/rt.zc"), *cc = slurp("src/compiler/compiler.zc"), *mm = mm_path ? slurp(mm_path) : 0; char *rtb = between(rt, "// @map-rt", "// @end-map"); char *ccb = between(cc, "// @map-cc", "// @end-map"); static const char *rt_names[] = { @@ -644,7 +646,7 @@ static void check_sources(const Map *m) if (!strstr(rt, lfa_old)) die("src/runtime/rt.zc no longer has the default lfa immediate"); if (!strstr(cc, "ZFP - 16384")) die("src/compiler/compiler.zc lost its initial type-area formula"); for (i = 0; rt_names[i]; i++) expect_hex(rtb, "src/runtime/rt.zc", rt_names[i], rt_vals[i]); - for (i = 0; rt_names[i]; i++) { + for (i = 0; mm && rt_names[i]; i++) { if (!strcmp(rt_names[i], "MAGIC") || !strcmp(rt_names[i], "HEAP_GROW")) continue; expect_hex(mm, mm_path, !strcmp(rt_names[i], "MW") ? "PW" : rt_names[i], rt_vals[i]); } @@ -700,7 +702,7 @@ static void usage_status(int status) "--preset default has a growing heap above the stacks; fixed is the old 1,081,344-word map.\n" "--grow lets a map with a fixed size get bigger when its genome needs more room.\n" "--mode packs a compiler or a run-only image; spare words go to the data stack.\n" - "--check compares that layout to memmap.h and the checked-in const blocks.\n"); + "--check compares that layout to the checked-in const blocks (and to ./memmap.h if present).\n"); exit(status); } diff --git a/src/tools/mkgenome.c b/src/tools/mkgenome.c index 3bf1533..9720760 100644 --- a/src/tools/mkgenome.c +++ b/src/tools/mkgenome.c @@ -1,7 +1,12 @@ -/* turns a genome image (little-endian 64-bit words on stdin, as written by - src/tools/build.zc) into genome.h */ +/* checks a genome image (little-endian 64-bit words on stdin, as written by + zc0 or src/tools/build.zc) against the memory map it was built with, and + copies it to stdout unchanged. the cores #embed the result (src/core/genome.h), + so this is the gate that keeps a compiler's diagnostics out of a core: when + the input is not an image for this map, nothing is written and the input is + shown on stderr, since it is usually the compiler's error report. */ #include <stdio.h> #include <stdint.h> +#include <stdlib.h> #include <string.h> #ifdef _WIN32 #include <fcntl.h> @@ -11,8 +16,9 @@ int main(int argc, char **argv) { if (argc == 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h"))) { - fputs("usage: mkgenome [--help|-h] < image.bin > genome.h\n" - "converts little-endian 64-bit image words into a c genome header.\n" + fputs("usage: mkgenome [--help|-h] < image.bin > genome.bin\n" + "checks little-endian 64-bit image words against this memory map and copies them\n" + "to stdout. input that is not an image is reported on stderr and nothing is written.\n" "the image must match the memory layout selected when mkgenome is built.\n" "options:\n --help, -h show this help and exit\n", stdout); return 0; @@ -22,19 +28,29 @@ int main(int argc, char **argv) _setmode(_fileno(stdin), _O_BINARY); _setmode(_fileno(stdout), _O_BINARY); #endif - unsigned char b[8]; int n = 0; size_t got; - printf("#define GORG %d\nstatic const int64_t genome[] = {", GORG); - while ((got = fread(b, 1, 8, stdin)) == 8) { - uint64_t u = (uint64_t)b[0] | (uint64_t)b[1] << 8 | (uint64_t)b[2] << 16 | (uint64_t)b[3] << 24 - | (uint64_t)b[4] << 32 | (uint64_t)b[5] << 40 | (uint64_t)b[6] << 48 | (uint64_t)b[7] << 56; - if ((n == 0 && u != (uint64_t)FTOP * 2 + 1) || - (n == 1 && u != (uint64_t)ZFP * 2 + 1) || (n == 2 && u != 4)) { - fprintf(stderr, "mkgenome: input is not a genome for this map\n"); return 1; + size_t n = 0, cap = 1 << 20, got; + unsigned char *b = malloc(cap); + if (!b) { fputs("mkgenome: out of memory\n", stderr); return 1; } + while ((got = fread(b + n, 1, cap - n, stdin)) > 0) { + n += got; + if (n == cap) { + unsigned char *next = cap > ((size_t)1 << 30) ? 0 : realloc(b, cap * 2); + if (!next) { fputs("mkgenome: image too large\n", stderr); return 1; } + b = next; cap *= 2; } - printf("%s%lld", !n ? "\n" : n % 12 ? "," : ",\n", (long long)u); - n++; } - if (got || ferror(stdin) || n < 7) { fprintf(stderr, "mkgenome: incomplete genome\n"); return 1; } - printf("\n};\n#define GLEN %d\n", n); - return n ? 0 : 1; + if (ferror(stdin)) { fputs("mkgenome: read failed\n", stderr); return 1; } + int ok = n >= 8 * 7 && n % 8 == 0; + for (int i = 0; ok && i < 3; i++) { + uint64_t u = 0; + for (int k = 0; k < 8; k++) u |= (uint64_t)b[i * 8 + k] << (k * 8); + if (u != (i == 0 ? (uint64_t)FTOP * 2 + 1 : i == 1 ? (uint64_t)ZFP * 2 + 1 : 4)) ok = 0; + } + if (!ok) { + if (n >= 4 && !memcmp(b, "zc: ", 4)) fwrite(b, 1, n, stderr); + fprintf(stderr, n % 8 || n < 8 * 7 ? "mkgenome: incomplete genome\n" : "mkgenome: input is not a genome for this map\n"); + return 1; + } + if (fwrite(b, 1, n, stdout) != n || fflush(stdout)) { fputs("mkgenome: write failed\n", stderr); return 1; } + return 0; } diff --git a/src/tools/mklsp.sh b/src/tools/mklsp.sh index 9bab550..1c44998 100755 --- a/src/tools/mklsp.sh +++ b/src/tools/mklsp.sh @@ -2,21 +2,19 @@ # builds the language server: src/tools/mklsp.sh [outdir] leaves outdir/zclsp. # the server is the real compiler (src/compiler/compiler.zc) in check mode plus # src/lsp/server.zc. its memory map is sized to that program by src/tools/automap.sh, -# so the genome slot is as big as it needs to be and no bigger. +# so the genome slot is as big as it needs to be and no bigger. the image carries +# the doc comments (DOCS=1) and, through embed declarations in server.zc, its own +# sources, so hover and go to definition work on built-ins. the core takes the +# image in with #embed as lsp_genome.bin. set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/mklsp.sh [OUTDIR]' 'builds the native language server in OUTDIR/zclsp; defaults to build/lsp.' 'environment: CC selects the c compiler.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/mklsp.sh [OUTDIR]' 'builds the native language server in OUTDIR/zclsp; defaults to build/lsp.' 'environment: CC selects the c compiler (one with #embed).'; exit 0 ;; -*) echo "mklsp: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." -O=${1:-build/lsp}; CC=${CC:-gcc} -# DOCS=1: the image carries the doc comments, so hover can show what a name is for +O=${1:-build/lsp}; CC=${CC:-$(sh src/tools/findcc.sh)} +export CC mkdir -p "$O" -bash src/tools/embed.sh all --check || bash src/tools/embed.sh all -# the image carries its own sources (for go to definition into a built-in) -bash src/tools/embed.sh lsp "$O/embed.zc" -DOCS=1 bash src/tools/automap.sh lsp "$O" src/lsp/server.zc "$O/embed.zc" >/dev/null -rm -rf "$O/src"; mkdir -p "$O/src" -cp src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$O/memmap.h" "$O/src/" -cp "$O/genome.h" "$O/src/lsp_genome.h" -$CC -O2 -w -fwrapv -pthread -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME "$O/src/core.c" "$O/src/io_linux.c" -I"$O/src" -o "$O/zclsp" +DOCS=1 bash src/tools/automap.sh lsp "$O" src/lsp/server.zc >/dev/null +cp "$O/genome.bin" "$O/lsp_genome.bin" +$CC -O2 -w -fwrapv -pthread -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME src/core/core.c src/core/io_linux.c -Isrc/core -I"$O" --embed-dir="$O" -o "$O/zclsp" diff --git a/src/tools/mklspwasm.sh b/src/tools/mklspwasm.sh index 4789eab..d05a5a7 100755 --- a/src/tools/mklspwasm.sh +++ b/src/tools/mklspwasm.sh @@ -3,21 +3,22 @@ # by default, or the path given as the first argument. the server is the compiler # and src/lsp/server.zc on the map and genome that src/tools/mklsp.sh made in build/lsp, # and the microcore (src/micro/micro.c) runs them. the host is client/wasmproc.js. -# needs clang with the wasm32 target and wasm-ld; run src/tools/mklsp.sh first. +# needs clang with the wasm32 target, #embed and wasm-ld; run src/tools/mklsp.sh first. # src/tools/mklspwasm.sh [outfile.wasm] set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to build/editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 support and wasm-ld.' 'environment: LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to build/editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 and #embed support, and wasm-ld.' 'environment: WASM_CC selects the compiler; LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;; -*) echo "mklspwasm: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." OUT=${1:-build/editors/vscode/server/lsp.wasm} LSP_BUILD_DIR=${LSP_BUILD_DIR:-build/lsp} +WASM_CC=${WASM_CC:-$(sh src/tools/findcc.sh --wasm)} if [ "${LSP_REUSE_BUILD:-0}" != 1 ]; then bash src/tools/mklsp.sh "$LSP_BUILD_DIR" >/dev/null; fi -[ -f "$LSP_BUILD_DIR/src/lsp_genome.h" ] || { echo "mklspwasm: no language server image in $LSP_BUILD_DIR" >&2; exit 1; } +[ -f "$LSP_BUILD_DIR/lsp_genome.bin" ] || { echo "mklspwasm: no language server image in $LSP_BUILD_DIR" >&2; exit 1; } mkdir -p "$(dirname "$OUT")" -clang --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DUSE_LSP_GENOME \ - -I"$LSP_BUILD_DIR/src" -Isrc/micro src/micro/micro.c src/micro/port_wasm_stream.c \ +"$WASM_CC" --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DUSE_LSP_GENOME \ + -Isrc/core -I"$LSP_BUILD_DIR" --embed-dir="$LSP_BUILD_DIR" -Isrc/micro src/micro/micro.c src/micro/port_wasm_stream.c \ -Wl,--no-entry -Wl,--export=wasm_chunk -Wl,--export=wasm_chunk_cap -Wl,--export=wasm_feed \ -Wl,--export=wasm_polling -Wl,--export=wasm_close -Wl,--export=wasm_start -Wl,--export=wasm_resume \ -Wl,--export=memory -Wl,--allow-undefined -Wl,--initial-memory=41943040 -Wl,-z,stack-size=65536 \ diff --git a/src/tools/mkvsix.sh b/src/tools/mkvsix.sh index df9e703..61fe9a8 100755 --- a/src/tools/mkvsix.sh +++ b/src/tools/mkvsix.sh @@ -1,13 +1,12 @@ #!/bin/bash # builds the vs code extension, with the language server inside it: the tinyvm # javascript vm (built from src/js/tinyvm.ts) and the server's image -# (build/js/lsp.json, from build/lsp/genome.h and its map), and the wasm server +# (build/js/lsp.bin and its map in lsp.json, from build/lsp), and the wasm server # (src/tools/mklspwasm.sh) when clang can build it. needs node and tsc; the # server build (src/tools/mklsp.sh) runs first if build/lsp is missing. # -# everything generated lands in build/. the packaged files are staged into -# editors/vscode/server/ only for as long as vsce is packing, then removed, so a -# fresh clone has no extension until it is built. +# everything generated lands in build/: the server files in +# build/editors/vscode/server, the packing copy of the extension in build/vsix. # src/tools/mkvsix.sh [outfile.vsix] set -e case "${1:-}" in @@ -20,31 +19,33 @@ case "$OUT" in /*) ;; *) OUT="$PWD/$OUT" ;; esac mkdir -p "$(dirname "$OUT")" JS=build/js STAGE=build/editors/vscode/server -PACKDIR=editors/vscode/server bash src/tools/mklsp.sh build/lsp >/dev/null # typescript errors stop packaging, so broken generated code is never shipped. command -v tsc >/dev/null || { echo "mkvsix: tsc is not installed" >&2; exit 1; } +make --no-print-directory build/js/layout.js >/dev/null tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir "$JS" node src/js/extract_lsp.js mkdir -p "$STAGE" -cp "$JS/tinyvm.js" "$JS/layout.js" "$JS/lsp.json" "$STAGE/" +cp "$JS/tinyvm.js" "$JS/layout.js" "$JS/lsp.json" "$JS/lsp.bin" "$STAGE/" # the wasm server runs when there is one; without clang the vm server is packed alone -if command -v clang >/dev/null && clang --print-targets | grep -q wasm32; then +if sh src/tools/findcc.sh --wasm >/dev/null 2>&1; then LSP_BUILD_DIR=build/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh "$STAGE/lsp.wasm" else rm -f "$STAGE/lsp.wasm" - echo "no clang with wasm32: packing the javascript vm server only" >&2 + echo "no clang with wasm32 and #embed: packing the javascript vm server only" >&2 fi -# vsce packs the extension directory, so the server has to sit inside it while -# it runs. stage it, package, and leave the tree as it was. -cleanup() { rm -rf "$PACKDIR"; } -trap cleanup EXIT INT TERM -rm -rf "$PACKDIR" -mkdir -p "$PACKDIR" -cp "$STAGE"/* "$PACKDIR"/ -cd editors/vscode +# vsce packs a whole extension directory, so the extension is copied into +# build/vsix/extension with the server beside its client, and packed there. the +# tree is never written to. +PACKROOT=build/vsix/extension +rm -rf "$PACKROOT" +mkdir -p "$PACKROOT" +cp -R editors/vscode/. "$PACKROOT/" +mkdir -p "$PACKROOT/server" +cp "$STAGE"/* "$PACKROOT/server/" +cd "$PACKROOT" if [ -n "${VSCE:-}" ]; then "$VSCE" package --allow-missing-repository --skip-license -o "$OUT" >/dev/null elif command -v vsce >/dev/null; then @@ -52,4 +53,4 @@ elif command -v vsce >/dev/null; then else npx -y @vscode/vsce package --allow-missing-repository --skip-license -o "$OUT" >/dev/null fi -echo "$OUT"
\ No newline at end of file +echo "$OUT" diff --git a/src/tools/zc0.c b/src/tools/zc0.c index 446557b..5f5b97d 100644 --- a/src/tools/zc0.c +++ b/src/tools/zc0.c @@ -1,7 +1,8 @@ /* zc0: stage 0 of the bootstrap. a small c compiler for the subset of zc that compiler.zc is written in. it reads zc source on stdin and writes a genome - header (vm words placed at GORG) on stdout, in the same image layout the - resident compiler's image mode produces. no lifetime analysis, no quote, no + image (vm words placed at GORG, as little-endian 64-bit words) on stdout, in + the same image layout the resident compiler's image mode produces. the cores + embed that file with #embed (src/core/genome.h). no lifetime analysis, no quote, no parse words, no block expressions: those only exist in compiler.zc. tags (m, d, -> u) and casts (as u, as d) are parsed and recorded in signatures but not checked. */ @@ -454,12 +455,13 @@ static void strlit(void) /* inline word string: jump over it, push its address * static void bstrlit(void) { int j = here, a, i = 0, k, n = (int)strlen(ts); + const char *b = ts; uint64_t w; lit(0); op(O_JMP); a = here; emit(n); while (i < n) { w = 0; k = 0; - while (k < 8 && i < n) { w |= (uint64_t)(unsigned char)ts[i] << (k * 8); k++; i++; } + while (k < 8 && i < n) { w |= (uint64_t)(unsigned char)b[i] << (k * 8); k++; i++; } emit((int64_t)w); } if ((n & 7) == 0) emit(0); @@ -497,7 +499,7 @@ static void primary(void) else if (((s->sig >> 4) & 15) != 1) die("multiple enhancement outputs need asm"); xs = 0; xf = 0; return; } - if (!strcmp(n, "put_s") && na == 1 && xs) fnref("put_bs"); + if ((!strcmp(n, "put_s") || !strcmp(n, "rt.put_s")) && na == 1 && xs) fnref(n[0] == 'r' ? "rt.put_bs" : "put_bs"); else if (!strcmp(n, "len")) fnref("len_s"); else fnref(n); op(O_CALL); @@ -794,6 +796,77 @@ static void enhdef(void) s = add(n, K_ENH, nenh++); s->sig = np + nout * 16 + d * 256; expect(";"); } +/* embed "path" as [type] name; at the top level: a global holding the file's bytes, + read when the image is built. the path is relative to the working directory. + bstr (the default) packs the bytes; char* keeps one character to a word; a + number takes them little-endian; T[] and T[n] lay them out as the array's + elements. the same rules as the resident compiler's kw_embed. */ +static int embed_width(const char *t, int star) +{ + if (star) return 0; + if (!strcmp(t, "char") || !strcmp(t, "u8") || !strcmp(t, "i8") || !strcmp(t, "byte")) return 1; + if (!strcmp(t, "u16") || !strcmp(t, "i16")) return 2; + if (!strcmp(t, "u32") || !strcmp(t, "i32") || !strcmp(t, "f32")) return 4; + if (!strcmp(t, "int") || !strcmp(t, "word") || !strcmp(t, "bool") || !strcmp(t, "float") || !strcmp(t, "f64")) return 8; + return 0; +} +static void embeddecl(void) +{ + char path[1024], ty[64] = "", *n; int star = 0, arr = -2, w, i, a; long len; unsigned char *buf; FILE *f; + if (tk != T_STR) die("embed takes a path in quotes: embed \"file\" as name;"); + snprintf(path, sizeof path, "%s", ts); next(); + expect("as"); + if (tk == T_ID) { + Lex L; lexsave(&L); next(); + int named = is(";"); lexrest(&L); + if (!named) { + snprintf(ty, sizeof ty, "%s", ts); next(); + while (accept("*")) star++; + if (accept("[")) { arr = -1; if (tk == T_NUM) { arr = (int)tv; next(); } expect("]"); } + } + } + n = ident(); expect(";"); + if (!(f = fopen(path, "rb"))) die("can't read the file named by embed"); + fseek(f, 0, SEEK_END); len = ftell(f); fseek(f, 0, SEEK_SET); + buf = malloc((size_t)len + 1); + if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) die("can't read the file named by embed"); + fclose(f); + if (arr == -2 && (!ty[0] || (!strcmp(ty, "bstr") && !star))) { + a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; emit(len); + for (i = 0; i < len; i += 8) { uint64_t u = 0; for (w = 0; w < 8 && i + w < len; w++) u |= (uint64_t)buf[i + w] << (w * 8); emit((int64_t)u); } + if (!(len & 7)) emit(0); + } else if (arr == -2 && !strcmp(ty, "char") && star == 1) { + a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; + for (i = 0; i < len; i++) emit(buf[i]); + emit(0); + } else if (arr == -2) { + uint64_t u = 0; w = embed_width(ty, star); + if (!w) die("embed declares a bstr, a char*, a number or an array of numbers"); + if (len > w) die("the embedded file is larger than the type it is declared as"); + if (!strcmp(ty, "f32") && len != 4) die("an embedded f32 takes exactly 4 bytes"); + if ((!strcmp(ty, "float") || !strcmp(ty, "f64")) && len != 8) die("an embedded float takes exactly 8 bytes"); + for (i = 0; i < len; i++) u |= (uint64_t)buf[i] << (i * 8); + if (ty[0] == 'i' && w < 8 && (u >> (w * 8 - 1) & 1)) u |= ~(uint64_t)0 << (w * 8); + if (!strcmp(ty, "f32")) u = fop_s2f(u); + a = here; add(n, K_GVAR, a); emit((int64_t)u); + } else { + int count; w = embed_width(ty, star); + if (!w) die("embed declares a bstr, a char*, a number or an array of numbers"); + if (len % w) die("the embedded file's size is not a whole number of elements"); + count = (int)(len / w); + if (arr >= 0 && count > arr) die("the embedded file is larger than the array it is declared as"); + if (arr >= 0) count = arr; + a = here; + if (!strcmp(ty, "char")) { for (i = 0; i < count; i++) emit(i < len ? buf[i] : 0); } + else { + long bytes = (long)count * w; + for (i = 0; i < bytes; i += 8) { uint64_t u = 0; for (int k = 0; k < 8 && i + k < bytes; k++) u |= (uint64_t)(i + k < len ? buf[i + k] : 0) << (k * 8); emit((int64_t)u); } + if (!bytes) emit(0); + } + add(n, K_GARR, a); + } + free(buf); +} static void top1(void) { if (accept("namespace")) { @@ -802,6 +875,7 @@ static void top1(void) expect(";"); return; } if (accept("include")) { do { ident(); } while (accept(",")); expect(";"); return; } + if (accept("embed")) { embeddecl(); return; } { if (accept("unchecked")) { top1(); return; } if (accept("fn")) { fndef(0); return; } @@ -898,8 +972,9 @@ static void top(void) int main(int argc, char **argv) { if (argc == 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h"))) { - fputs("usage: zc0 [--help|-h] < source.zc > genome.h\n" - "compiles bootstrap source from stdin into a c genome header on stdout.\n" + fputs("usage: zc0 [--help|-h] < source.zc > genome.bin\n" + "compiles bootstrap source from stdin into a genome image on stdout:\n" + "little-endian 64-bit words, the format the cores #embed.\n" "source must define start(); memory layout is selected when zc0 is built.\n" "options:\n --help, -h show this help and exit\n", stdout); return 0; @@ -942,8 +1017,10 @@ int main(int argc, char **argv) } emit(0); img[6] = here; - printf("#define GORG %d\n#define GLEN %d\nstatic const int64_t genome[] = {", GORG, here - GORG); - for (i = 0; i < here - GORG; i++) printf("%s%lld", !i ? "\n" : i % 12 ? "," : ",\n", (long long)img[i]); - printf("\n};\n"); - return 0; + for (i = 0; i < here - GORG; i++) { + unsigned char b[8]; uint64_t u = (uint64_t)img[i]; + for (j = 0; j < 8; j++) b[j] = (unsigned char)(u >> (j * 8)); + if (fwrite(b, 1, 8, stdout) != 8) { fprintf(stderr, "zc0: write failed\n"); return 1; } + } + return fflush(stdout) ? 1 : 0; } diff --git a/src/tools/zcc b/src/tools/zcc index 3fb8f0c..ba38e97 100755 --- a/src/tools/zcc +++ b/src/tools/zcc @@ -8,7 +8,7 @@ # # -o OUT the binary to write (default: FILE without .zc) # -k keep the build directory and print where it is -# CC the C compiler (default cc). CFLAGS adds to the default -O2 +# CC the C compiler, one with c23 #embed (default: src/tools/findcc.sh). CFLAGS adds to -O2 # # the build is three steps: automap sizes the map to the program and writes the # genome (the compiled program image) for it; the micro core and its linux port @@ -16,13 +16,13 @@ # the same image without the static link. set -e here="$(cd "$(dirname "$0")/../.." && pwd)" -CC=${CC:-cc} +CC=${CC:-$(sh "$here/src/tools/findcc.sh")} out=""; keep=0 while [ $# -gt 0 ]; do case "$1" in -o) [ $# -ge 2 ] || { echo "zcc: -o needs a name" >&2; exit 2; }; out="$2"; shift 2 ;; -k) keep=1; shift ;; - -h|--help) printf '%s\n' 'usage: src/tools/zcc [-o OUT] [-k] FILE.zc' 'compiles a source file into a standalone binary; links statically where supported.' 'options:' ' -o OUT output binary path; defaults to the source basename without .zc' ' -k keep the temporary build directory' ' --help, -h show this help and exit' 'environment: CC selects the c compiler; CFLAGS adds compiler flags.'; exit 0 ;; + -h|--help) printf '%s\n' 'usage: src/tools/zcc [-o OUT] [-k] FILE.zc' 'compiles a source file into a standalone binary; links statically where supported.' 'options:' ' -o OUT output binary path; defaults to the source basename without .zc' ' -k keep the temporary build directory' ' --help, -h show this help and exit' 'environment: CC selects the c compiler (one with #embed); CFLAGS adds compiler flags.'; exit 0 ;; -*) echo "zcc: unknown option $1" >&2; exit 2 ;; *) break ;; esac @@ -41,7 +41,7 @@ trap cleanup EXIT # the map and the genome for this program only ( cd "$here" && bash src/tools/automap.sh run "$work/run" src/runtime/run.zc "$abs" ) >/dev/null cp "$here/src/micro/micro.c" "$here/src/micro/micro_port.h" "$here/src/micro/port_linux.c" \ - "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$work/run/" + "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$here/src/core/genome.h" "$work/run/" # static where the toolchain allows it; a plain link otherwise, with a note static=-static diff --git a/src/web/build.sh b/src/web/build.sh index 7443f46..1f2cfae 100755 --- a/src/web/build.sh +++ b/src/web/build.sh @@ -1,16 +1,23 @@ #!/bin/bash -# builds src/web/zc.wasm (or the path given as the first argument): the microcore, with the compiler genome inside, as -# WebAssembly. needs clang with the wasm32 target and wasm-ld. the page -# (src/web/index.html) loads it next to src/web/zc.js. +# builds the browser compiler: the microcore with the compiler genome inside, as +# WebAssembly, written to build/web/zc.wasm (or the path given as the first +# argument). the page (src/web/index.html) loads it next to zc.js; `make web` +# copies both beside it, so build/web/ is the whole site. needs clang with the +# wasm32 target, #embed and wasm-ld, and the genome from `make genome`. set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/web/build.sh [OUTFILE.wasm]' 'builds the browser compiler vm; defaults to src/web/zc.wasm.' 'requires clang with wasm32 support and wasm-ld.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/web/build.sh [OUTFILE.wasm]' 'builds the browser compiler vm; defaults to build/web/zc.wasm.' 'requires clang with wasm32 and #embed support, and wasm-ld.' 'environment: WASM_CC selects the compiler; GENOME_DIR holds genome.bin (build/core); MAP_DIR holds its memmap.h (build/core/map).'; exit 0 ;; -*) echo "web build: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." -clang --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int \ - -Isrc/core -Isrc/micro src/micro/micro.c src/web/port_wasm.c \ +OUT=${1:-build/web/zc.wasm} +WASM_CC=${WASM_CC:-$(sh src/tools/findcc.sh --wasm)} +GENOME_DIR=${GENOME_DIR:-build/core}; MAP_DIR=${MAP_DIR:-build/core/map} +[ -f "$GENOME_DIR/genome.bin" ] || { echo "web build: no $GENOME_DIR/genome.bin; run make genome first" >&2; exit 1; } +mkdir -p "$(dirname "$OUT")" +"$WASM_CC" --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int \ + -Isrc/core -I"$MAP_DIR" --embed-dir="$GENOME_DIR" -Isrc/micro src/micro/micro.c src/web/port_wasm.c \ -Wl,--no-entry -Wl,--export=wasm_input -Wl,--export=wasm_input_cap -Wl,--export=wasm_set_input -Wl,--export=wasm_run \ -Wl,--export=memory -Wl,--allow-undefined -Wl,--initial-memory=33554432 -Wl,-z,stack-size=65536 \ - -o "${1:-src/web/zc.wasm}" -ls -l "${1:-src/web/zc.wasm}" + -o "$OUT" +ls -l "$OUT" diff --git a/tests/bench/bench.sh b/tests/bench/bench.sh index ab6b07d..65f455e 100644 --- a/tests/bench/bench.sh +++ b/tests/bench/bench.sh @@ -2,13 +2,15 @@ # o3 against plain code: each kernel in tests/programs/bench.zc on each backend, with # the time to start up and compile (the 'z' run) taken off. needs bc. cd "$(dirname "$0")/../.."; T=$(mktemp -d); trap 'rm -rf $T' EXIT -gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zy +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 +make -s build/core/genome.bin || exit 1 +$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zy vms=("$T/zy"); names=("interp") if command -v llvm-config >/dev/null; then - gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } + $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyl $(llvm-config --ldflags --libs) && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then - aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); } + aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o $T/zyj && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64 (qemu)"); } fi # seconds per run of kernel $2 on vm $1: the slope between $3 runs and 1 run, # best of three each, so starting up and compiling cancel out diff --git a/tests/bench/bench_bytes.sh b/tests/bench/bench_bytes.sh index acff074..775b71d 100644 --- a/tests/bench/bench_bytes.sh +++ b/tests/bench/bench_bytes.sh @@ -4,16 +4,18 @@ set -e # in the process. the slope between 1 repetition and N drops startup. # prints: backend ms/rep checksum cd "$(dirname "$0")/../.." +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 +make -s build/core/genome.bin || exit 1 T=$(mktemp -d) trap 'rm -rf "$T"' EXIT -gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 +$CC -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 GENMAP="$T/genmap" bash src/tools/automap.sh run "$T/run" src/runtime/run.zc tests/programs/bench_bytes.zc >/dev/null || exit 1 -cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/" -gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 -gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 +cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h "$T/run/" +$CC -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 +$CC -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 vms=("$T/zy" "$T/zym"); names=("interp" "micro") if command -v llvm-config >/dev/null; then - gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ + $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then diff --git a/tests/bench/bench_float.sh b/tests/bench/bench_float.sh index 794e976..4ebfbe4 100644 --- a/tests/bench/bench_float.sh +++ b/tests/bench/bench_float.sh @@ -5,17 +5,19 @@ # this does not bootstrap the compiler. each kernel is poly() of 20000, # and (int) of one call is 59997. cd "$(dirname "$0")/../.." +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 +make -s build/core/genome.bin || exit 1 T=$(mktemp -d) trap 'rm -rf "$T"' EXIT -gcc -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zy" || exit 1 +$CC -O2 -w -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o "$T/zy" || exit 1 vms=("$T/zy"); names=("interp") -gcc -O2 -w src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -o "$T/zym" && { vms+=("$T/zym"); names+=("micro"); } +$CC -O2 -w src/micro/micro.c src/micro/port_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -Isrc/micro -o "$T/zym" && { vms+=("$T/zym"); names+=("micro"); } if command -v llvm-config >/dev/null; then - gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyl" $(llvm-config --ldflags --libs) \ + $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o "$T/zyl" $(llvm-config --ldflags --libs) \ && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then - aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -o "$T/zyj" \ + aarch64-linux-gnu-gcc -O2 -w -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o "$T/zyj" \ && { vms+=("qemu-aarch64 $T/zyj"); names+=("arm64"); } fi src=$(cat tests/programs/fbench.zc) diff --git a/tests/bench/bench_share.sh b/tests/bench/bench_share.sh index b449e09..5cb1bfd 100644 --- a/tests/bench/bench_share.sh +++ b/tests/bench/bench_share.sh @@ -3,18 +3,19 @@ # resident compiler is not in the process. the slope between 1 repetition # and N drops startup. prints: backend ms/rep checksum cd "$(dirname "$0")/../.." +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 T=$(mktemp -d) trap 'rm -rf "$T"' EXIT -gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 +$CC -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 "$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1 -cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/" -gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1 -cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_share.zc | "$T/zc0" > "$T/run/genome.h" || exit 1 -gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 -gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 +cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h "$T/run/" +$CC -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1 +cat "$T/run/rt.zc" src/core/core.zc src/runtime/run.zc tests/programs/bench_share.zc | "$T/zc0" > "$T/run/genome.bin" || exit 1 +$CC -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 +$CC -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 vms=("$T/zy" "$T/zym"); names=("interp" "micro") if command -v llvm-config >/dev/null; then - gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ + $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then diff --git a/tests/bench/bench_strings.sh b/tests/bench/bench_strings.sh index c353342..b14bf57 100644 --- a/tests/bench/bench_strings.sh +++ b/tests/bench/bench_strings.sh @@ -3,18 +3,19 @@ # resident compiler is not in the process. the slope between 1 repetition # and N drops startup. prints: backend ms/rep checksum cd "$(dirname "$0")/../.." +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 T=$(mktemp -d) trap 'rm -rf "$T"' EXIT -gcc -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 +$CC -O2 -w src/tools/genmap.c -o "$T/genmap" || exit 1 "$T/genmap" --preset run -o "$T/run" >/dev/null || exit 1 -cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$T/run/" -gcc -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1 -cat "$T/run/rt.zc" src/runtime/run.zc tests/programs/bench_strings.zc | "$T/zc0" > "$T/run/genome.h" || exit 1 -gcc -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 -gcc -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 +cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h "$T/run/" +$CC -O1 -w "$T/run/zc0.c" -o "$T/zc0" || exit 1 +cat "$T/run/rt.zc" src/core/core.zc src/runtime/run.zc tests/programs/bench_strings.zc | "$T/zc0" > "$T/run/genome.bin" || exit 1 +$CC -O2 -w -DJTYPE=int "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zy" || exit 1 +$CC -O2 -w "$T/run/micro.c" "$T/run/port_linux.c" -o "$T/zym" || exit 1 vms=("$T/zy" "$T/zym"); names=("interp" "micro") if command -v llvm-config >/dev/null; then - gcc -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ + $CC -O2 -w -DJTYPE=int -DZYLLVM $(llvm-config --cflags) "$T/run/core.c" "$T/run/io_linux.c" -o "$T/zyl" $(llvm-config --ldflags --libs) \ && { vms+=("$T/zyl"); names+=("llvm"); } fi if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then diff --git a/tests/bench/run.py b/tests/bench/run.py index cb28b92..7f7ea6f 100755 --- a/tests/bench/run.py +++ b/tests/bench/run.py @@ -79,18 +79,21 @@ def machine_lines(): def build(td): td = Path(td) log("building vms") + need(["make", "-s", "build/core/genome.bin"], cwd=ROOT) + cc = os.environ.get("CC") or subprocess.check_output(["sh", "src/tools/findcc.sh"], cwd=ROOT, text=True).strip() + core = ["-Isrc/core", "-Ibuild/core/map", "--embed-dir=build/core"] zy = td / "zy" - need(["gcc", "-O3", "-w", "-DJTYPE=int", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zy)], cwd=ROOT) + need([cc, "-O3", "-w", "-DJTYPE=int", "src/core/core.c", "src/core/io_linux.c", *core, "-o", str(zy)], cwd=ROOT) vms = {"interp": [str(zy)]} if shutil.which("llvm-config"): cflags = subprocess.check_output(["llvm-config", "--cflags"], text=True).split() ld = subprocess.check_output(["llvm-config", "--ldflags", "--libs"], text=True).split() zyl = td / "zyl" - need(["gcc", "-O3", "-w", "-DJTYPE=int", "-DZYLLVM", *cflags, "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyl), *ld], cwd=ROOT) + need([cc, "-O3", "-w", "-DJTYPE=int", "-DZYLLVM", *cflags, "src/core/core.c", "src/core/io_linux.c", *core, "-o", str(zyl), *ld], cwd=ROOT) vms["llvm"] = [str(zyl)] if shutil.which("aarch64-linux-gnu-gcc") and shutil.which("qemu-aarch64"): zyj = td / "zyj" - need(["aarch64-linux-gnu-gcc", "-O3", "-w", "-DJTYPE=int", "-static", "src/core/core.c", "src/core/io_linux.c", "-Isrc/core", "-o", str(zyj)], cwd=ROOT) + need(["aarch64-linux-gnu-gcc", "-O3", "-w", "-DJTYPE=int", "-static", "src/core/core.c", "src/core/io_linux.c", *core, "-o", str(zyj)], cwd=ROOT) vms["arm64"] = ["qemu-aarch64", str(zyj)] log("building C") bins = {} diff --git a/tests/drivers/actor_image_test.py b/tests/drivers/actor_image_test.py index 9df8828..5b815f2 100644 --- a/tests/drivers/actor_image_test.py +++ b/tests/drivers/actor_image_test.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 # source syntax callbacks execute in the host while actor code is emitted separately. from pathlib import Path +import sys import argparse import subprocess import tempfile +sys.dont_write_bytecode = True # importing imagekit must not leave __pycache__ in the tree +from imagekit import micro_image parser = argparse.ArgumentParser() -parser.add_argument('--host', default='bin/zy') +parser.add_argument('--host', default='build/bin/zy') +parser.add_argument('--mkgenome', default='build/bin/mkgenome') parser.add_argument('--case', default='') args = parser.parse_args() prefix = b''.join(Path(f).read_bytes() + b'\n' for f in ('src/runtime/rt.zc', 'src/core/core.zc', 'src/runtime/run.zc')) @@ -18,11 +22,8 @@ for name, source, expected in cases: directory = Path(work) compiled = subprocess.run([args.host], input=Path('src/tools/build.zc').read_bytes() + b'`' + prefix + source.encode(), capture_output=True, timeout=30) assert compiled.returncode == 0 and len(compiled.stdout) >= 56 and not compiled.stdout.startswith(b'zc:'), (name, compiled.stdout[:2000], compiled.stderr) - header = subprocess.run(['bin/mkgenome'], input=compiled.stdout, capture_output=True, check=True) - (directory / 'genome.h').write_bytes(header.stdout) - for f in ('src/core/memmap.h', 'src/core/fops.h', 'src/core/bops.h', 'src/core/stack.h', 'src/micro/micro.c', 'src/micro/micro_port.h', 'src/micro/port_linux.c'): - (directory / Path(f).name).write_bytes(Path(f).read_bytes()) - subprocess.run(['cc', '-O2', '-w', '-fwrapv', '-pthread', '-DZY_THREADS', str(directory / 'micro.c'), str(directory / 'port_linux.c'), '-I' + work, '-o', str(directory / 'image')], check=True, timeout=30) - result = subprocess.run([str(directory / 'image')], capture_output=True, timeout=12) + image = subprocess.run([args.mkgenome], input=compiled.stdout, capture_output=True, check=True) + binary = micro_image(directory, image.stdout) + result = subprocess.run([str(binary)], capture_output=True, timeout=12) assert result.returncode == 0 and result.stdout == expected, (name, result.returncode, result.stdout, result.stderr) print('ok actor compiler-free image: ' + name) diff --git a/tests/drivers/actor_test.py b/tests/drivers/actor_test.py index 43a110f..0cfa51f 100644 --- a/tests/drivers/actor_test.py +++ b/tests/drivers/actor_test.py @@ -6,7 +6,7 @@ import subprocess import time parser = argparse.ArgumentParser() -parser.add_argument('vm', nargs='?', default='bin/zy') +parser.add_argument('vm', nargs='?', default='build/bin/zy') parser.add_argument('--case', default='') parser.add_argument('--library', default='') args = parser.parse_args() diff --git a/tests/drivers/analysis_cache_test.js b/tests/drivers/analysis_cache_test.js index 3ffde33..88bb4bd 100644 --- a/tests/drivers/analysis_cache_test.js +++ b/tests/drivers/analysis_cache_test.js @@ -1,4 +1,6 @@ // Repeated replies use cached results; edits, include updates and reopen invalidate them. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'), cp = require('child_process'), path = require('path'); const uri = 'file:///cache/main.zc', text = 'include "dep.zc";\nfn main() { int n = 7; return n + value(); }'; const msgs = []; let id = 1; @@ -26,7 +28,7 @@ if (process.argv[2] === '--wasm' || process.argv[2] === '--vm') { const p = (wasm ? host.wasmProcess : host.vmProcess)(path.resolve(process.argv[3]), { log: () => {} }); const chunks = []; p.stdout.on('data', d => chunks.push(d)); msgs.forEach(m => p.stdin.write(frame(m))); out = Buffer.concat(chunks); } else { - const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(msgs.map(frame)), timeout: 60000, maxBuffer: 16 << 20 }); + const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(msgs.map(frame)), timeout: 60000 * TIMEOUT_SCALE, maxBuffer: 16 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); out = r.stdout; } const replies = new Map(); diff --git a/tests/drivers/array_methods_test.py b/tests/drivers/array_methods_test.py index 82fef67..bf6e670 100644 --- a/tests/drivers/array_methods_test.py +++ b/tests/drivers/array_methods_test.py @@ -5,7 +5,7 @@ import re import subprocess parser = argparse.ArgumentParser() -parser.add_argument('host', nargs='?', default='bin/zy') +parser.add_argument('host', nargs='?', default='build/bin/zy') parser.add_argument('--case', default='') args = parser.parse_args() diff --git a/tests/drivers/cli_help_test.py b/tests/drivers/cli_help_test.py index 1b32fa4..97eebea 100644 --- a/tests/drivers/cli_help_test.py +++ b/tests/drivers/cli_help_test.py @@ -7,15 +7,16 @@ from pathlib import Path parser = argparse.ArgumentParser() parser.add_argument('--group', choices=['binaries', 'scripts', 'js', 'all'], default='all') -parser.add_argument('--bin-dir', default='bin') +parser.add_argument('--bin-dir', default='build/bin') +parser.add_argument('--js-dir', default='build/js') args = parser.parse_args() commands = [] if args.group in ('binaries', 'all'): commands += [[str(Path(args.bin_dir) / name)] for name in ['zy', 'zym', 'zclsp', 'zc0', 'mkgenome', 'genmap']] if args.group in ('scripts', 'all'): - commands += [['bash', name] for name in ['tinyvm.sh', 'src/tools/zcc', 'src/tools/automap.sh', 'src/tools/embed.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/web/build.sh', 'src/baremetal/run.sh']] + commands += [['bash', name] for name in ['tinyvm.sh', 'src/tools/zcc', 'src/tools/automap.sh', 'src/tools/findcc.sh', 'src/tools/bootstrap-check.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/web/build.sh', 'src/baremetal/run.sh']] if args.group in ('js', 'all'): - commands += [['node', name] for name in ['src/js/tinyvm.js', 'src/js/extract_genome.js', 'src/js/extract_lsp.js']] + commands += [['node', name] for name in [str(Path(args.js_dir) / 'tinyvm.js'), 'src/js/mklayout.js', 'src/js/extract_lsp.js']] def invoke(command, option): # keep stdin open: help and rejected options must finish without reading it. diff --git a/tests/drivers/completion_test.js b/tests/drivers/completion_test.js index 22c5fa5..f1ee5e6 100644 --- a/tests/drivers/completion_test.js +++ b/tests/drivers/completion_test.js @@ -1,5 +1,7 @@ // compiler-backed completion, including incomplete source and position-aware lifetimes. // usage: node tests/drivers/completion_test.js path/to/zclsp [server arguments] +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const path = require('path'); @@ -25,7 +27,7 @@ function complete(source, extra = []) { for (const m of msgs) proc.stdin.write(frame(m)); output = Buffer.concat(chunks); } else { - const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(msgs.map(frame)), timeout: 30000, maxBuffer: 32 << 20 }); + const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(msgs.map(frame)), timeout: 30000 * TIMEOUT_SCALE, maxBuffer: 32 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); output = r.stdout; } let b = output; const replies = []; diff --git a/tests/drivers/core_lsp_test.js b/tests/drivers/core_lsp_test.js index 983f9aa..c3c01aa 100644 --- a/tests/drivers/core_lsp_test.js +++ b/tests/drivers/core_lsp_test.js @@ -1,4 +1,6 @@ // check maintained zc sources through the editor document protocol, one file at a time. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const fs = require('fs'); const cp = require('child_process'); @@ -23,7 +25,7 @@ for (const file of files) { { jsonrpc: '2.0', id: 3, method: 'shutdown', params: null }, { jsonrpc: '2.0', method: 'exit' }, ]; - const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 30000, maxBuffer: 32 << 20 }); + const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 30000 * TIMEOUT_SCALE, maxBuffer: 32 << 20 }); assert.strictEqual(result.status, 0, file + ': ' + result.stderr); let data = result.stdout; const replies = []; diff --git a/tests/drivers/enhancement_backends_test.py b/tests/drivers/enhancement_backends_test.py index b7d553b..ddfee1b 100644 --- a/tests/drivers/enhancement_backends_test.py +++ b/tests/drivers/enhancement_backends_test.py @@ -9,7 +9,7 @@ source = (root / 'tests/programs/enhance.zc').read_text() source = source.replace('i < 5000', 'i < 200') source += '\n' reference = None -for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: +for command in [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']]: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=25) assert result.returncode == 0 and result.stdout and 'zc:' not in result.stdout, (command, result.returncode, result.stdout, result.stderr) if reference is None: @@ -20,7 +20,7 @@ for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: source = '''o3 int increment(int value) { return value + 1; } fn main() { put_n(increment(6)); put_c(32); put_n(17 / 5); put_c(32); put_n(17 % 5); put_c(10); return 0; } ''' -for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: +for command in [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']]: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=15) assert result.returncode == 0 and result.stdout == '7 3 2\n', (command, result.stdout, result.stderr) print('ok optimizer descriptors preserve portable division slots: ' + command[0]) @@ -39,7 +39,7 @@ fn main() { put_n(installed); put_c(32); put_n(first); put_c(32); put_n(rejected); put_c(32); put_n(second); put_c(10); return 0; } ''' -for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: +for command in [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']]: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=15) assert result.returncode == 0 and result.stdout == '1 22 1 22\n', (command, result.stdout, result.stderr) print('ok replacing a built-in slot and rejecting a bad recipe: ' + command[0]) diff --git a/tests/drivers/ext_cache_test.js b/tests/drivers/ext_cache_test.js index f389fcc..93caf94 100644 --- a/tests/drivers/ext_cache_test.js +++ b/tests/drivers/ext_cache_test.js @@ -5,14 +5,25 @@ // starts; a change to an included file is checked again; putting it back is // answered from the cache. // -// node tests/drivers/ext_cache_test.js +// node tests/drivers/ext_cache_test.js [server-dir] +// +// the server directory holds what the packaged extension carries in server/: +// lsp.wasm, or tinyvm.js, layout.js, lsp.json and lsp.bin (the default is +// build/editors/vscode/server, which src/tools/mkvsix.sh fills). the extension is +// copied with that server into a scratch directory, since the tree has none. const fs = require('fs'); const os = require('os'); const path = require('path'); const Module = require('module'); -const client = path.join(__dirname, '..', '..', 'editors', 'vscode', 'client'); +const serverDir = path.resolve(process.argv[2] || path.join(__dirname, '..', '..', 'build', 'editors', 'vscode', 'server')); +if (!fs.existsSync(serverDir)) { console.log('FAIL no server directory at ' + serverDir); process.exit(1); } +const staged = fs.mkdtempSync(path.join(os.tmpdir(), 'zc-extension-')); +fs.cpSync(path.join(__dirname, '..', '..', 'editors', 'vscode'), staged, { recursive: true }); +fs.cpSync(serverDir, path.join(staged, 'server'), { recursive: true }); +process.on('exit', () => fs.rmSync(staged, { recursive: true, force: true })); +const client = path.join(staged, 'client'); // the vscode module, as far as the client uses it const world = { lines: [], sets: [], listeners: {}, watchers: [], textDocuments: [], saved: undefined, root: '' }; diff --git a/tests/drivers/float_backends_test.py b/tests/drivers/float_backends_test.py index cdcf48b..65901e2 100644 --- a/tests/drivers/float_backends_test.py +++ b/tests/drivers/float_backends_test.py @@ -4,7 +4,7 @@ import pathlib import subprocess root = pathlib.Path(__file__).resolve().parents[2] -commands = [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']] +commands = [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']] sources = [(root / 'tests/programs/float.zc').read_text(), '''include rt; fn render(bits, precision, mode) { asm { bits @ precision @ mode @ #1236 } return 0; } fn convert(bits) { return asm { bits @ #1218 }; } fn main() { diff --git a/tests/drivers/fuzz_compiler.py b/tests/drivers/fuzz_compiler.py index 512ec26..0fe0d27 100644 --- a/tests/drivers/fuzz_compiler.py +++ b/tests/drivers/fuzz_compiler.py @@ -1,17 +1,23 @@ #!/usr/bin/env python3 """Deterministic grammar fuzzing for the parser, runtime, and ownership paths. -Usage: python3 tests/drivers/fuzz_compiler.py [path/to/zy] [path/to/zym] [--js] +Usage: python3 tests/drivers/fuzz_compiler.py [path/to/zy] [path/to/zym] [--js [path/to/tinyvm.js]] +(--js alone runs build/js/tinyvm.js, from make js) """ import random import subprocess import sys -args = [arg for arg in sys.argv[1:] if arg != "--js"] -js = "--js" in sys.argv[1:] -zy = args[0] if len(args) > 0 else "bin/zy" -zym = args[1] if len(args) > 1 else "bin/zym" +argv = sys.argv[1:] +js = None +if "--js" in argv: + k = argv.index("--js") + del argv[k] + js = argv.pop(k) if k < len(argv) and argv[k].endswith(".js") else "build/js/tinyvm.js" +args = argv +zy = args[0] if len(args) > 0 else "build/bin/zy" +zym = args[1] if len(args) > 1 else "build/bin/zym" rng = random.Random(0x5A17) values = [] library_values = [] @@ -33,26 +39,26 @@ def expr(depth): return f"({left} {op} {right})", value -lines = ["include std;", "fn churn(n) {", " int i = 0;", " while (i < n) {", " { int* p = new(int); *p = i; }", " i++;", " }", " return 0;", "}", "fn stable() { return 73; }", "fn main() {", " let base = live_objects();"] +lines = ["namespace;", "include rt, run, std;", "fn churn(n) {", " int i = 0;", " while (i < n) {", " { int* p = rt.new(int); *p = i; }", " i++;", " }", " return 0;", "}", "fn stable() { return 73; }", "fn main() {", " let base = rt.live_objects();"] for _ in range(128): source, value = expr(4) - lines.append(f" put_n({source}); put_c(32);") + lines.append(f" rt.put_n({source}); rt.put_c(32);") values.append(value) for _ in range(32): a = "".join(rng.choice("abc") for _ in range(rng.randrange(11))) b = "".join(rng.choice("abc") for _ in range(rng.randrange(6))) - lines.append(f' put_n(Text.len("{a}")); put_c(32); put_n(Text.eq("{a}", "{b}")); put_c(32);') - lines.append(f' put_n(Text.starts("{a}", "{b}")); put_c(32); put_n(Text.ends("{a}", "{b}")); put_c(32); put_n(Text.find("{a}", "{b}")); put_c(32);') - lines.append(f' put_n(Bytes.len(b"{a}")); put_c(32); put_n(Bytes.eq(b"{a}", b"{b}")); put_c(32);') - lines.append(f' put_n(Bytes.starts(b"{a}", b"{b}")); put_c(32); put_n(Bytes.ends(b"{a}", b"{b}")); put_c(32); put_n(Bytes.find(b"{a}", b"{b}")); put_c(32);') + lines.append(f' rt.put_n(std.Text.len("{a}")); rt.put_c(32); rt.put_n(std.Text.eq("{a}", "{b}")); rt.put_c(32);') + lines.append(f' rt.put_n(std.Text.starts("{a}", "{b}")); rt.put_c(32); rt.put_n(std.Text.ends("{a}", "{b}")); rt.put_c(32); rt.put_n(std.Text.find("{a}", "{b}")); rt.put_c(32);') + lines.append(f' rt.put_n(std.Bytes.len(b"{a}")); rt.put_c(32); rt.put_n(std.Bytes.eq(b"{a}", b"{b}")); rt.put_c(32);') + lines.append(f' rt.put_n(std.Bytes.starts(b"{a}", b"{b}")); rt.put_c(32); rt.put_n(std.Bytes.ends(b"{a}", b"{b}")); rt.put_c(32); rt.put_n(std.Bytes.find(b"{a}", b"{b}")); rt.put_c(32);') library_values += [len(a), int(a == b), int(a.startswith(b)), int(a.endswith(b)), a.find(b), len(a.encode()), int(a == b), int(a.startswith(b)), int(a.endswith(b)), a.find(b)] lines += [ - " churn(300); gc(); put_n(live_objects() - base); put_c(32);", - ' char* source = "fn f() { return 7; }"; let parsed = tokens(source);', - ' put_n(parsed[0] > 0); put_c(32); put_n(eq_w(source, "fn f() { return 7; }")); put_c(32);', + " churn(300); rt.gc(); rt.put_n(rt.live_objects() - base); rt.put_c(32);", + ' char* source = "fn f() { return 7; }"; let parsed = run.tokens(source);', + ' rt.put_n(parsed[0] > 0); rt.put_c(32); rt.put_n(rt.eq_w(source, "fn f() { return 7; }")); rt.put_c(32);', ' char* eval_source = "return 42;"; int e = 0;', - ' while (e < 16) { let q = tokens(eval_source); put_n(eval(q)); put_c(32); e++; }', - ' put_n(stable()); put_c(32); put_n(eq_w(eval_source, "return 42;")); put_c(10);', + ' while (e < 16) { let q = run.tokens(eval_source); rt.put_n(run.eval(q)); rt.put_c(32); e++; }', + ' rt.put_n(stable()); rt.put_c(32); rt.put_n(rt.eq_w(eval_source, "return 42;")); rt.put_c(10);', " return 0;", "}", ] @@ -78,7 +84,7 @@ def run(binary): hosts = [(zy, [zy]), (zym, [zym])] if js: - hosts.append(("javascript", ["node", "src/js/tinyvm.js"])) + hosts.append(("javascript", ["node", js])) outputs = [run(command) for _, command in hosts] if any(output != outputs[0] for output in outputs[1:]): raise SystemExit("compiler hosts disagreed on deterministic fuzz output") diff --git a/tests/drivers/fuzz_lsp.js b/tests/drivers/fuzz_lsp.js index e22a44b..f8a1d3b 100644 --- a/tests/drivers/fuzz_lsp.js +++ b/tests/drivers/fuzz_lsp.js @@ -1,7 +1,9 @@ // Deterministic source/protocol fuzzing for diagnostics, caching, and recovery. // Usage: node tests/drivers/fuzz_lsp.js path/to/zclsp +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const { spawnSync } = require('child_process'); -const server = process.argv[2] || 'bin/zclsp'; +const server = process.argv[2] || 'build/bin/zclsp'; let state = 0x5a17; function next() { state = (Math.imul(state, 1664525) + 1013904223) >>> 0; return state; } function frame(m) { @@ -35,7 +37,7 @@ for (let i = 0; i < 32; i++) { msgs.push({ jsonrpc: '2.0', method: 'textDocument/didChange', params: { textDocument: { uri, version: 2 }, contentChanges: [{ text: fixed }] } }); } const stdUri = 'file:///fuzz-std.zc'; -msgs.push({ jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri: stdUri, languageId: 'zc', version: 1, text: 'include std;\nfn main() { return Text.starts("alpha", "al"); }\n' } } }); +msgs.push({ jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri: stdUri, languageId: 'zc', version: 1, text: 'include std;\nfn main() { return std.Text.starts("alpha", "al"); }\n' } } }); // Keep one server alive while documents are repeatedly opened, made malformed, // repaired, closed, and reopened at a new version. This exercises cache @@ -57,7 +59,7 @@ for (const [uri, text] of churn) { msgs.push({ jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri, languageId: 'zc', version: churnVersion++, text } } }); } msgs.push({ jsonrpc: '2.0', method: 'exit' }); -const run = spawnSync(server, [], { input: Buffer.concat(msgs.map(frame)), timeout: 60000, maxBuffer: 1 << 28 }); +const run = spawnSync(server, [], { input: Buffer.concat(msgs.map(frame)), timeout: 60000 * TIMEOUT_SCALE, maxBuffer: 1 << 28 }); if (run.error || run.status !== 0) throw run.error || new Error(`server exit ${run.status}: ${run.stderr}`); const out = parse(run.stdout); const diagnostics = new Map(); diff --git a/tests/drivers/fuzz_malformed.py b/tests/drivers/fuzz_malformed.py index 5bf97cf..42b8a0e 100644 --- a/tests/drivers/fuzz_malformed.py +++ b/tests/drivers/fuzz_malformed.py @@ -8,7 +8,7 @@ import subprocess import sys -bins = sys.argv[1:] or ["bin/zy", "bin/zym"] +bins = sys.argv[1:] or ["build/bin/zy", "build/bin/zym"] if len(bins) == 1: bins.append(bins[0]) rng = random.Random(0xC0DE) diff --git a/tests/drivers/fuzz_wasm.js b/tests/drivers/fuzz_wasm.js index b0982d4..22fa7d5 100644 --- a/tests/drivers/fuzz_wasm.js +++ b/tests/drivers/fuzz_wasm.js @@ -31,7 +31,7 @@ function quote(s) { return '"' + s + '"'; } lines.push(' churn(300); gc(); put_n(live_objects() - base); put_c(32);', ' put_n(stable()); put_c(10);', ' return 0;', '}'); expected.push(0, 73); const source = lines.join('\n') + '\n', want = expected.join(' ') + '\n'; - const zc = await ZC.load(fs.readFileSync(process.argv[2] || 'src/web/zc.wasm')); + const zc = await ZC.load(fs.readFileSync(process.argv[2] || 'build/web/zc.wasm')); const result = await zc.run(source); if (result.output !== want) throw new Error(`wasm fuzz mismatch\nexpected: ${want.slice(0, 180)}\nactual: ${result.output.slice(0, 180)}\nmessages: ${result.messages}`); console.log('ok wasm compiler/runtime/stdlib fuzz: 128 expressions, 300 scoped allocations and GC, and 32 Text/Bytes cases'); diff --git a/tests/drivers/genome_blob_test.py b/tests/drivers/genome_blob_test.py index 81533a7..fea055b 100644 --- a/tests/drivers/genome_blob_test.py +++ b/tests/drivers/genome_blob_test.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 # documentation precedes the symbol table; the boot header must skip those bytes. +# usage: python3 tests/drivers/genome_blob_test.py [build/lsp] (the documented language server image) import pathlib import re +import struct +import sys -source = pathlib.Path('src/core/genome.h').read_text() -org = int(re.search(r'#define GORG (\d+)', source)[1]) -length = int(re.search(r'#define GLEN (\d+)', source)[1]) -words = [int(word.strip()) for word in re.search(r'genome\[\]\s*=\s*\{([\s\S]*?)\};', source)[1].split(',') if word.strip()] -assert len(words) == length and words[6] == org + length +folder = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else 'build/lsp') +image = (folder / 'lsp_genome.bin').read_bytes() +org = int(re.search(r'#define GORG (?:0x)?([0-9A-Fa-f]+)', (folder / 'memmap.h').read_text())[1], 16) +words = list(struct.unpack('<%dq' % (len(image) // 8), image)) +length = len(words) +assert words[6] == org + length cursor = words[5] - org assert 7 <= cursor < length records = {} @@ -23,8 +27,8 @@ while words[cursor]: cursor += 1 records[name] = (kind, value, signature) assert cursor == length - 1 -assert len(records) > 1000 and records['start'][0] == 1 -assert records['reserve_code'][0] == 1 and records['build_image_grow'][0] == 1 +assert len(records) > 1000 and records['run.start'][0] == 1 +assert records['run.reserve_code'][0] == 1 and records['run.build_image'][0] == 1 assert any(name.startswith('doc ') for name in records), 'the fixture contains documentation bytes' for name, (kind, value, _) in records.items(): if kind == 8: diff --git a/tests/drivers/hints_test.js b/tests/drivers/hints_test.js index 8bb1934..b3598be 100644 --- a/tests/drivers/hints_test.js +++ b/tests/drivers/hints_test.js @@ -1,5 +1,7 @@ // hints are informational diagnostics (severity 3) and follow positional hints on / small / off directives. // usage: node tests/drivers/hints_test.js path/to/zclsp +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const server = process.argv[2]; @@ -10,7 +12,7 @@ function diagnostics(text) { { jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, { jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri, version: 1, languageId: 'zc', text } } }, { jsonrpc: '2.0', method: 'exit' }].map(frame)); - const r = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000, maxBuffer: 4 << 20 }); + const r = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000 * TIMEOUT_SCALE, maxBuffer: 4 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); let d = r.stdout, out = []; while (d.length) { diff --git a/tests/drivers/hover_features_test.js b/tests/drivers/hover_features_test.js index d6e1226..25f5965 100644 --- a/tests/drivers/hover_features_test.js +++ b/tests/drivers/hover_features_test.js @@ -1,5 +1,7 @@ // focused type, inference, and block-documentation hover regressions. // usage: node tests/drivers/hover_features_test.js path/to/zclsp +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const server = process.argv[2]; @@ -21,7 +23,7 @@ function hover(text, needle, nth = 0) { { jsonrpc: '2.0', id: 2, method: 'textDocument/hover', params: { textDocument: { uri }, position: { line, character } } }, { jsonrpc: '2.0', method: 'exit' }, ]; - const result = cp.spawnSync(server, process.argv.slice(3), { input: Buffer.concat(messages.map(frame)), timeout: 10000, maxBuffer: 4 << 20 }); + const result = cp.spawnSync(server, process.argv.slice(3), { input: Buffer.concat(messages.map(frame)), timeout: 10000 * TIMEOUT_SCALE, maxBuffer: 4 << 20 }); assert.strictEqual(result.status, 0, String(result.stderr)); let data = result.stdout; while (data.length) { diff --git a/tests/drivers/hover_surface_test.js b/tests/drivers/hover_surface_test.js index 13a9a3f..688aca6 100644 --- a/tests/drivers/hover_surface_test.js +++ b/tests/drivers/hover_surface_test.js @@ -1,4 +1,6 @@ // hover declarations and instructions in the source-only documents opened by f12. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const fs = require('fs'); const cp = require('child_process'); @@ -48,7 +50,7 @@ if (process.argv[2] === '--wasm' || process.argv[2] === '--vm') { for (const m of messages) p.stdin.write(frame({ jsonrpc: '2.0', ...m })); out = Buffer.concat(chunks); } else { - const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(messages.map(m => frame({ jsonrpc: '2.0', ...m }))), timeout: 60000, maxBuffer: 32 << 20 }); + const r = cp.spawnSync(process.argv[2], process.argv.slice(3), { input: Buffer.concat(messages.map(m => frame({ jsonrpc: '2.0', ...m }))), timeout: 60000 * TIMEOUT_SCALE, maxBuffer: 32 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); out = r.stdout; } const replies = new Map(); diff --git a/tests/drivers/hover_values_test.js b/tests/drivers/hover_values_test.js index badc2ed..0f09cfc 100644 --- a/tests/drivers/hover_values_test.js +++ b/tests/drivers/hover_values_test.js @@ -1,4 +1,6 @@ // value previews follow the compiler's bindings and the position of each use. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const server = process.argv[2]; @@ -17,7 +19,7 @@ function hover(text, needle, nth = 0) { const body = Buffer.from(JSON.stringify(m)); return Buffer.concat([Buffer.from('Content-Length: ' + body.length + '\r\n\r\n'), body]); })); - const result = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000, maxBuffer: 8 << 20 }); + const result = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000 * TIMEOUT_SCALE, maxBuffer: 8 << 20 }); assert.strictEqual(result.status, 0, String(result.stderr)); let data = result.stdout; while (data.length) { diff --git a/tests/drivers/icon_test.py b/tests/drivers/icon_test.py index 560c622..72cd69b 100644 --- a/tests/drivers/icon_test.py +++ b/tests/drivers/icon_test.py @@ -5,7 +5,7 @@ import argparse import subprocess parser = argparse.ArgumentParser() -parser.add_argument('vm', nargs='?', default='bin/zy') +parser.add_argument('vm', nargs='?', default='build/bin/zy') parser.add_argument('--case', default='') parser.add_argument('--js', action='store_true') parser.add_argument('--source', action='store_true', help='test the working library source instead of the embedded module') diff --git a/tests/drivers/image_run.js b/tests/drivers/image_run.js index f657916..ea05f9f 100644 --- a/tests/drivers/image_run.js +++ b/tests/drivers/image_run.js @@ -2,12 +2,11 @@ const fs = require('fs'); const { TinyVM } = require('../../build/js/tinyvm.js'); const { mapValues } = require('../../src/js/mapvalues.js'); -const header = fs.readFileSync(process.argv[2], 'utf8'); -const org = Number(header.match(/#define\s+GORG\s+(\d+)/)[1]); -const words = header.match(/genome\[\]\s*=\s*\{([\s\S]*?)\};/)[1].split(',').map(s => s.trim()).filter(Boolean); +// usage: node tests/drivers/image_run.js genome.bin memmap.h [--stdin] +const image = fs.readFileSync(process.argv[2]); const layout = mapValues(process.argv[3], ['GORG', 'DS0', 'DSLIM', 'RS0', 'RSLIM', 'SYSGATE', 'ZFP', 'HEAP_LO', 'HEAP_HI']); const machine = new TinyVM({ layout, onMessage: message => process.stderr.write(message) }); -machine.loadGenome(words, org); +machine.loadImage(image, layout.GORG); if (process.argv[4] === '--stdin') machine.setInput(fs.readFileSync(0)); const status = machine.run(); process.stdout.write(machine.getOutput()); diff --git a/tests/drivers/imagekit.py b/tests/drivers/imagekit.py new file mode 100644 index 0000000..6cc083f --- /dev/null +++ b/tests/drivers/imagekit.py @@ -0,0 +1,34 @@ +"""shared steps for the drivers that build their own images: pick a c compiler with +#embed, and link a microcore around a genome image (little-endian 64-bit words) +for a given memory map. everything is written into the caller's directory.""" +import os +import shutil +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_MAP = ROOT / 'build' / 'core' / 'map' / 'memmap.h' +MICRO_SOURCES = ['src/micro/micro.c', 'src/micro/micro_port.h', 'src/micro/port_linux.c', + 'src/core/fops.h', 'src/core/bops.h', 'src/core/stack.h', 'src/core/genome.h'] + + +def compiler(): + """the c compiler: CC, or the one src/tools/findcc.sh picks.""" + if os.environ.get('CC'): + return os.environ['CC'] + return subprocess.run(['sh', str(ROOT / 'src/tools/findcc.sh')], check=True, capture_output=True, text=True).stdout.strip() + + +def micro_image(directory, image, memmap=DEFAULT_MAP, threads=True, name='image', timeout=60): + """link a microcore in directory that boots image (bytes). memmap is the map header the + image was built for. returns the executable's path.""" + directory = Path(directory) + (directory / 'genome.bin').write_bytes(image) + shutil.copy(memmap, directory / 'memmap.h') + for f in MICRO_SOURCES: + shutil.copy(ROOT / f, directory) + flags = ['-pthread', '-DZY_THREADS'] if threads else [] + out = directory / name + subprocess.run([compiler(), '-O2', '-w', '-fwrapv', *flags, str(directory / 'micro.c'), str(directory / 'port_linux.c'), + '-I' + str(directory), '-o', str(out)], check=True, timeout=timeout) + return out diff --git a/tests/drivers/layout_config_test.py b/tests/drivers/layout_config_test.py index 469e7dd..2f0fa07 100644 --- a/tests/drivers/layout_config_test.py +++ b/tests/drivers/layout_config_test.py @@ -5,7 +5,7 @@ import subprocess import tempfile root = pathlib.Path(__file__).resolve().parents[2] -tool = root / 'bin/genmap' +tool = root / 'build/bin/genmap' result = subprocess.run([tool, '--help'], capture_output=True, text=True) assert result.returncode == 0 and '--generated-text' in result.stderr invalid = [ diff --git a/tests/drivers/lsp_growth_test.js b/tests/drivers/lsp_growth_test.js index b06230a..6d221fd 100644 --- a/tests/drivers/lsp_growth_test.js +++ b/tests/drivers/lsp_growth_test.js @@ -1,5 +1,7 @@ // focused regressions for growing document, uri, hover, and name-set storage. // usage: node tests/drivers/lsp_growth_test.js path/to/zclsp-or-lsp.wasm +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const { wasmProcess } = require('../../editors/vscode/client/wasmproc'); @@ -36,7 +38,7 @@ if (target.endsWith('.wasm')) { for (const message of messages) assert.ok(server.stdin.write(frame(message))); output = Buffer.concat(chunks); } else { - const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 30000, maxBuffer: 8 << 20 }); + const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 30000 * TIMEOUT_SCALE, maxBuffer: 8 << 20 }); assert.strictEqual(result.status, 0, String(result.stderr)); output = result.stdout; } diff --git a/tests/drivers/lsp_sandbox.js b/tests/drivers/lsp_sandbox.js index 14bc1e3..41523cc 100644 --- a/tests/drivers/lsp_sandbox.js +++ b/tests/drivers/lsp_sandbox.js @@ -2,10 +2,12 @@ // (immediate functions). the server must not let that code write files, so a file that // tries to leaves nothing behind and the server still answers. // usage: node tests/drivers/lsp_sandbox.js path/to/zclsp +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const cp = require('child_process'), fs = require('fs'), os = require('os'), path = require('path'); const target = path.join(os.tmpdir(), 'zc-sandbox-' + process.pid); try { fs.unlinkSync(target); } catch (e) {} -const src = 'immediate fn evil() { let fd = syscall(1, "' + target + '", 1, 0); syscall(4, fd, 65, 1); syscall(2, fd, 0, 0); return 0; }\nevil\nfn main() { return 0; }\n'; +const src = 'include rt, run;\nimmediate fn evil() { run.next(); let fd = rt.syscall(1, "' + target + '", 1, 0); rt.syscall(4, fd, 65, 1); rt.syscall(2, fd, 0, 0); return 0; }\nevil\nfn main() { return 0; }\n'; const frame = m => { const b = Buffer.from(JSON.stringify(m)); return Buffer.concat([Buffer.from('Content-Length: ' + b.length + '\r\n\r\n'), b]); }; const input = Buffer.concat([ frame({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { processId: 1, capabilities: {} } }), @@ -13,7 +15,7 @@ const input = Buffer.concat([ frame({ jsonrpc: '2.0', id: 3, method: 'textDocument/semanticTokens/full', params: { textDocument: { uri: 'file:///evil.zc' } } }), frame({ jsonrpc: '2.0', method: 'exit' }), ]); -const r = cp.spawnSync(process.argv[2], [], { input, timeout: 20000 }); +const r = cp.spawnSync(process.argv[2], [], { input, timeout: 20000 * TIMEOUT_SCALE }); const wrote = fs.existsSync(target); if (wrote) fs.unlinkSync(target); const answered = r.stdout && r.stdout.includes('"id":3'); diff --git a/tests/drivers/lsp_test.js b/tests/drivers/lsp_test.js index ab43c41..18d8bf7 100644 --- a/tests/drivers/lsp_test.js +++ b/tests/drivers/lsp_test.js @@ -1,5 +1,7 @@ // protocol tests for the language server (node, no packages). // usage: node tests/drivers/lsp_test.js path/to/zclsp +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const { spawnSync } = require('child_process'); const fs = require('fs'); const server = process.argv[2]; @@ -24,7 +26,7 @@ function parse(buf) { return out; } function run(msgs) { - const r = spawnSync(server, [], { input: Buffer.concat(msgs.map(frame)), timeout: 30000, maxBuffer: 1 << 28 }); + const r = spawnSync(server, [], { input: Buffer.concat(msgs.map(frame)), timeout: 30000 * TIMEOUT_SCALE, maxBuffer: 1 << 28 }); return { code: r.status, msgs: parse(r.stdout) }; } const init = id => ({ jsonrpc: '2.0', id, method: 'initialize', params: {} }); @@ -173,7 +175,7 @@ check(hv && hv.includes('leaves the function with a value'), 'a keyword shows wh const builtinSource = 'include rt; fn main() {\n\trt.put_c(32);\n\treturn 0;\n}\n'; hv = hover(builtinSource, 1, at(builtinSource, 1, 'put_c')); check(hv && hv.includes('put_c('), 'a built-in function shows its signature', hv); -const kwsrc = 'include rt; fn main() {\n\tint* p = new(int);\n\trt.free(p);\n\treturn 0;\n}\n'; +const kwsrc = 'include rt; fn main() {\n\tint* p = rt.new(int);\n\trt.free(p);\n\treturn 0;\n}\n'; hv = hover(kwsrc, 1, at(kwsrc, 1, 'new')); check(hv && hv.includes('new(T)') && hv.includes('fresh heap object') && hv.includes('owns it'), 'hover on new shows its form and ownership behavior', hv); hv = hover(kwsrc, 2, at(kwsrc, 2, 'free')); @@ -209,7 +211,7 @@ check(r.code === 0 && r.msgs.some(m => m.id === 9 && m.result === null), 'shutdo // the corpus: every valid program checks clean, every bad one is rejected with its message function cli(file) { - const r = spawnSync(server, [], { input: fs.readFileSync(file), timeout: 30000 }); + const r = spawnSync(server, [], { input: fs.readFileSync(file), timeout: 30000 * TIMEOUT_SCALE }); return r.stdout.toString(); } let clean = 0, dirty = []; diff --git a/tests/drivers/memory_limits_test.py b/tests/drivers/memory_limits_test.py index a973480..808a239 100644 --- a/tests/drivers/memory_limits_test.py +++ b/tests/drivers/memory_limits_test.py @@ -13,7 +13,7 @@ source = '''fn main() { return 0; } ''' -commands = [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']] +commands = [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']] environment = dict(os.environ, TINYVM_MAX_MEMORY_WORDS='16777216') for command in commands: result = subprocess.run(command, input=source, text=True, capture_output=True, env=environment, timeout=15) diff --git a/tests/drivers/nav_test.js b/tests/drivers/nav_test.js index 23afbf8..8f0d8fd 100644 --- a/tests/drivers/nav_test.js +++ b/tests/drivers/nav_test.js @@ -1,5 +1,7 @@ // go to definition and find references, in the document and across includes. // usage: node tests/drivers/nav_test.js path/to/zclsp (run from the repository root) +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const fs = require('fs'); @@ -18,7 +20,7 @@ function request(text, needle, method, extra = {}) { { jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri, version: 1, languageId: 'zc', text } } }, { jsonrpc: '2.0', id: 2, method, params: { textDocument: { uri }, position, ...extra } }, { jsonrpc: '2.0', method: 'exit' }].map(frame)); - const r = cp.spawnSync(server, process.argv.slice(3), { input, cwd: root, timeout: 30000, maxBuffer: 8 << 20 }); + const r = cp.spawnSync(server, process.argv.slice(3), { input, cwd: root, timeout: 30000 * TIMEOUT_SCALE, maxBuffer: 8 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); let d = r.stdout, init; while (d.length) { diff --git a/tests/drivers/protocol_growth_test.js b/tests/drivers/protocol_growth_test.js index 64015d8..d8dff6c 100644 --- a/tests/drivers/protocol_growth_test.js +++ b/tests/drivers/protocol_growth_test.js @@ -1,4 +1,6 @@ // preserve long request identifiers and ignore deeply nested unknown json fields. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const target = process.argv[2]; @@ -15,7 +17,7 @@ for (let i = 0; i < 24; i++) messages.push({ jsonrpc: '2.0', method: 'textDocume text: 'enum choice { item { int value; }, empty };\nfn main() { choice*.m p = choice.item(7); return p.item_value; }\n', } } }); messages.push({ jsonrpc: '2.0', id: 'finish', method: 'shutdown', params: null }, { jsonrpc: '2.0', method: 'exit' }); -const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 20000, maxBuffer: 8 << 20 }); +const result = cp.spawnSync(target, [], { input: Buffer.concat(messages.map(frame)), timeout: 20000 * TIMEOUT_SCALE, maxBuffer: 8 << 20 }); assert.strictEqual(result.status, 0, String(result.stderr)); const responses = []; let data = result.stdout; diff --git a/tests/drivers/runtime_image_test.py b/tests/drivers/runtime_image_test.py index c1f4b9c..d214613 100644 --- a/tests/drivers/runtime_image_test.py +++ b/tests/drivers/runtime_image_test.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 """self-hosted run-only images retain enhancements and generic method bodies.""" +import sys import argparse import subprocess import tempfile from pathlib import Path +sys.dont_write_bytecode = True # importing imagekit must not leave __pycache__ in the tree +from imagekit import micro_image parser = argparse.ArgumentParser() parser.add_argument('--case', default='') -parser.add_argument('--host', default='bin/zy') -parser.add_argument('--mkgenome', default='bin/mkgenome') +parser.add_argument('--host', default='build/bin/zy') +parser.add_argument('--mkgenome', default='build/bin/mkgenome') +parser.add_argument('--map', default='build/core/map/memmap.h') args = parser.parse_args() cases = [ ('enhancements', 'include rt; fn main() { rt.put_n(17 % 5); rt.put_c(32); rt.put_n(17 / 5); }', b'2 3'), @@ -23,12 +27,9 @@ for name, source, expected in cases: directory = Path(work) r = subprocess.run([args.host], input=Path('src/tools/build.zc').read_bytes() + b'`' + prefix + source.encode(), capture_output=True, timeout=30) assert r.returncode == 0 and len(r.stdout) >= 56 and not r.stdout.startswith(b'zc:'), (name, r.stdout[:1000], r.stderr) - header = subprocess.run([args.mkgenome], input=r.stdout, capture_output=True, timeout=5) - assert header.returncode == 0, (name, header.stderr) - (directory / 'genome.h').write_bytes(header.stdout) - for f in ['src/core/memmap.h', 'src/core/fops.h', 'src/core/bops.h', 'src/core/stack.h', 'src/micro/micro.c', 'src/micro/micro_port.h', 'src/micro/port_linux.c']: - (directory / Path(f).name).write_bytes(Path(f).read_bytes()) - subprocess.run(['cc', '-O2', '-w', '-fwrapv', '-pthread', '-DZY_THREADS', str(directory / 'micro.c'), str(directory / 'port_linux.c'), '-I' + work, '-o', str(directory / 'image')], check=True, timeout=30) - r = subprocess.run([str(directory / 'image')], capture_output=True, timeout=10) + image = subprocess.run([args.mkgenome], input=r.stdout, capture_output=True, timeout=5) + assert image.returncode == 0, (name, image.stderr) + binary = micro_image(directory, image.stdout, args.map) + r = subprocess.run([str(binary)], capture_output=True, timeout=10) assert r.returncode == 0 and r.stdout == expected, (name, r.returncode, r.stdout, r.stderr, expected) print('ok self-hosted run-only image: ' + name) diff --git a/tests/drivers/seed_growth_test.py b/tests/drivers/seed_growth_test.py index 426c970..b94445e 100644 --- a/tests/drivers/seed_growth_test.py +++ b/tests/drivers/seed_growth_test.py @@ -1,25 +1,24 @@ #!/usr/bin/env python3 # stage 0 accepts long source streams and grows a tiny initial image reservation. +import sys import pathlib import subprocess import tempfile +sys.dont_write_bytecode = True # importing imagekit must not leave __pycache__ in the tree +from imagekit import compiler, micro_image root = pathlib.Path(__file__).resolve().parents[2] +(root / 'build').mkdir(exist_ok=True) with tempfile.TemporaryDirectory(prefix='seed-growth-', dir=root / 'build') as directory: folder = pathlib.Path(directory) - subprocess.run(['bin/genmap', '--preset', 'run', '--grow', '--genome', '600000', '--heap', '1048576', '-o', directory], check=True, capture_output=True) - compiler = folder / 'zc0' - subprocess.run(['cc', '-O1', '-w', '-fwrapv', '-DMAXW=16', 'src/tools/zc0.c', '-I' + directory, '-Isrc/core', '-o', str(compiler)], check=True) + subprocess.run(['build/bin/genmap', '--preset', 'run', '--grow', '--genome', '600000', '--heap', '1048576', '-o', directory], check=True, capture_output=True) + zc0 = folder / 'zc0' + subprocess.run([compiler(), '-O1', '-w', '-fwrapv', '-DMAXW=16', 'src/tools/zc0.c', '-I' + directory, '-Isrc/core', '-o', str(zc0)], check=True) source = '/*' + 'x' * (5 << 20) + '*/\n' - source += (folder / 'rt.zc').read_text() + (root / 'src/runtime/run.zc').read_text() - source += 'let data[550000]; fn main() { data[549999] = 7; put_n(data[549999]); put_c(32); put_n(4611686018427387904); put_c(10); return 0; }\n' - with (folder / 'genome.h').open('w') as output: - subprocess.run([str(compiler)], input=source, text=True, stdout=output, check=True, timeout=30) - binary = folder / 'vm' - import shutil - for path in ['src/micro/micro.c', 'src/micro/micro_port.h', 'src/micro/port_linux.c', 'src/core/fops.h', 'src/core/bops.h', 'src/core/stack.h']: - shutil.copy(root / path, folder) - subprocess.run(['cc', '-O2', '-w', '-fwrapv', str(folder / 'micro.c'), str(folder / 'port_linux.c'), '-I' + directory, '-o', str(binary)], check=True) + source += (folder / 'rt.zc').read_text() + (root / 'src/core/core.zc').read_text() + (root / 'src/runtime/run.zc').read_text() + source += 'namespace; include rt; let data[550000]; fn main() { data[549999] = 7; rt.put_n(data[549999]); rt.put_c(32); rt.put_n(4611686018427387904); rt.put_c(10); return 0; }\n' + image = subprocess.run([str(zc0)], input=source.encode(), capture_output=True, check=True, timeout=30).stdout + binary = micro_image(folder / 'vm', image, folder / 'memmap.h', threads=False) if (folder / 'vm').mkdir() is None else None result = subprocess.run([str(binary)], text=True, capture_output=True, timeout=20) assert result.returncode == 0 and result.stdout == '7 4611686018427387904\n', (result.returncode, result.stdout, result.stderr) print('ok stage 0 reads beyond four megabytes and grows beyond its initial image capacity') diff --git a/tests/drivers/stack_growth_test.py b/tests/drivers/stack_growth_test.py index d4abe81..3c90305 100644 --- a/tests/drivers/stack_growth_test.py +++ b/tests/drivers/stack_growth_test.py @@ -4,6 +4,8 @@ import subprocess import sys import tempfile from pathlib import Path +sys.dont_write_bytecode = True # importing imagekit must not leave __pycache__ in the tree +from imagekit import compiler command = sys.argv[1:] @@ -12,47 +14,53 @@ def run(source, expected, compile_image=False): if compile_image and command[0] == 'node': # compile the execution stress cases once with stage 0, so a javascript # compiler's speed does not determine the execution-test timeout. + Path('build').mkdir(exist_ok=True) with tempfile.TemporaryDirectory(prefix='stack-image-', dir='build') as directory: target = Path(directory) - subprocess.run(['bin/genmap', '--preset', 'run', '--grow', '--heap', '1048576', '--genome', '131072', '--frame', '4096', '--return', '256', '--data', '3072', '-o', directory], check=True, capture_output=True) - compiler = target / 'zc0' - subprocess.run(['cc', '-O1', '-w', '-fwrapv', 'src/tools/zc0.c', '-I' + directory, '-Isrc/core', '-o', str(compiler)], check=True, capture_output=True) - full_source = (target / 'rt.zc').read_text() + Path('src/runtime/run.zc').read_text() + source - image = subprocess.run([str(compiler)], input=full_source, text=True, capture_output=True, check=True) - header = target / 'genome.h' - header.write_text(image.stdout) - result = subprocess.run(['node', 'tests/drivers/image_run.js', str(header), str(target / 'memmap.h')], text=True, capture_output=True, timeout=45) + subprocess.run(['build/bin/genmap', '--preset', 'run', '--grow', '--heap', '1048576', '--genome', '131072', '--frame', '4096', '--return', '256', '--data', '3072', '-o', directory], check=True, capture_output=True) + zc0 = target / 'zc0' + subprocess.run([compiler(), '-O1', '-w', '-fwrapv', 'src/tools/zc0.c', '-I' + directory, '-Isrc/core', '-o', str(zc0)], check=True, capture_output=True) + full_source = (target / 'rt.zc').read_text() + Path('src/core/core.zc').read_text() + Path('src/runtime/run.zc').read_text() + source + image = subprocess.run([str(zc0)], input=full_source.encode(), capture_output=True, check=True) + (target / 'genome.bin').write_bytes(image.stdout) + result = subprocess.run(['node', 'tests/drivers/image_run.js', str(target / 'genome.bin'), str(target / 'memmap.h')], text=True, capture_output=True, timeout=45) else: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=45) assert result.returncode == 0 and result.stdout == expected, (command, result.returncode, result.stdout, result.stderr) -run('''fn main() { - int old_data = data_top - data_low, old_return = return_top - return_low; - put_n(reserve_data(old_data + 1) > old_data); put_c(32); - put_n(reserve_returns(old_return + 1) > old_return); put_c(32); - gc(); put_n(data_node != 0 && return_node != 0); put_c(10); return 0; +run('''namespace; +include rt; +fn main() { + int old_data = rt.data_top - rt.data_low, old_return = rt.return_top - rt.return_low; + rt.put_n(rt.reserve_data(old_data + 1) > old_data); rt.put_c(32); + rt.put_n(rt.reserve_returns(old_return + 1) > old_return); rt.put_c(32); + rt.gc(); rt.put_n(rt.data_node != 0 && rt.return_node != 0); rt.put_c(10); return 0; } ''', '1 1 1\n') -run('''fn main() { - put_n(reserve_data(-1) == -1); put_c(32); put_n(reserve_returns(-1) == -1); put_c(32); - put_n(syscall(9, 6, 0, 0) == -1); put_c(32); - int current = syscall(9, 0, 0, 0); syscall(9, 5, current, 0); - put_n(reserve_data(0x10000000) == -1); put_c(32); gc(); put_n(7); put_c(10); return 0; +run('''namespace; +include rt; +fn main() { + rt.put_n(rt.reserve_data(-1) == -1); rt.put_c(32); rt.put_n(rt.reserve_returns(-1) == -1); rt.put_c(32); + rt.put_n(rt.syscall(9, 6, 0, 0) == -1); rt.put_c(32); + int current = rt.syscall(9, 0, 0, 0); rt.syscall(9, 5, current, 0); + rt.put_n(rt.reserve_data(0x10000000) == -1); rt.put_c(32); rt.gc(); rt.put_n(7); rt.put_c(10); return 0; } ''', '1 1 1 1 7\n') count = 40000 -source = 'fn main() { int total = asm { ' + ' '.join(str(i + 1) for i in range(count)) -source += ' + ' * (count - 1) + ' }; gc(); put_n(total); put_c(32); ' -source += 'put_n(data_node != 0); put_c(10); return 0; }\n' +source = 'namespace; include rt; fn main() { int total = asm { ' + ' '.join(str(i + 1) for i in range(count)) +source += ' + ' * (count - 1) + ' }; rt.gc(); rt.put_n(total); rt.put_c(32); ' +source += 'rt.put_n(rt.data_node != 0); rt.put_c(10); return 0; }\n' run(source, '800020000 1\n', compile_image=True) -run('''unchecked int descend(int n) { - if (!n) { gc(); return 7; } +run('''namespace; +include rt; +unchecked int descend(int n) { + if (!n) { rt.gc(); return 7; } return descend(n - 1) + 1; } -fn main() { put_n(descend(70000)); put_c(32); put_n(return_node != 0); put_c(10); return 0; } +fn main() { rt.put_n(descend(70000)); rt.put_c(32); rt.put_n(rt.return_node != 0); rt.put_c(10); return 0; } ''', '70007 1\n', compile_image=True) print('ok explicit reservation, 40000 operands, and 70000 return addresses: ' + command[0]) diff --git a/tests/drivers/storage_analysis_test.js b/tests/drivers/storage_analysis_test.js index 774fa89..704a844 100644 --- a/tests/drivers/storage_analysis_test.js +++ b/tests/drivers/storage_analysis_test.js @@ -1,8 +1,12 @@ // definite storage findings use the compiler's parser and the editor protocol. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const server = process.argv[2]; function check(text, needle) { + // the runtime is an explicit import; it shares the first line so line numbers stay as written + text = 'include rt; ' + text; const uri = 'file:///storage-analysis.zc'; const messages = [{ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, { jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri, version: 1, text } } }]; @@ -13,7 +17,7 @@ function check(text, needle) { } messages.push({ jsonrpc: '2.0', method: 'exit' }); const input = Buffer.concat(messages.map(m => { const b = Buffer.from(JSON.stringify(m)); return Buffer.concat([Buffer.from('Content-Length: ' + b.length + '\r\n\r\n'), b]); })); - const r = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000, maxBuffer: 8 << 20 }); + const r = cp.spawnSync(server, process.argv.slice(3), { input, timeout: 20000 * TIMEOUT_SCALE, maxBuffer: 8 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); const replies = []; let data = r.stdout; while (data.length) { @@ -27,50 +31,50 @@ function check(text, needle) { } const prefix = 'unchecked fn main() {\n'; for (const [type, length] of [['int', 3], ['u8', 3], ['fixed int[?]', 3], ['u8[?]', 3]]) { - const r = check(prefix + ` let p = new(${type}, ${length});\n let alias = p;\n return alias[${length}];\n}\n`, 'alias['); + const r = check(prefix + ` let p = rt.new(${type}, ${length});\n let alias = p;\n return alias[${length}];\n}\n`, 'alias['); assert.strictEqual(r.warnings.length, 1, JSON.stringify(r.diagnostics)); assert.match(r.warnings[0].message, /out-of-bounds.*index 3.*length 3/); assert.match(r.hover, /size: 3 elements.*lifetime: heap storage; live here/); assert.strictEqual(r.warnings[0].range.start.line, 3); } -const negative = check(prefix + ' let p = new(3); return p[-1];\n}\n'); +const negative = check(prefix + ' let p = rt.new(3); return p[-1];\n}\n'); assert.match(negative.warnings[0]?.message || '', /index -1.*length 3/); -const freed = check(prefix + ' let p = new(int, 3);\n let q = p;\n free(p);\n return q[0];\n}\n', 'q['); +const freed = check(prefix + ' let p = rt.new(int, 3);\n let q = p;\n rt.free(p);\n return q[0];\n}\n', 'q['); assert.strictEqual(freed.warnings.length, 1, JSON.stringify(freed.diagnostics)); assert.match(freed.warnings[0].message, /use after free.*line 4/); assert.match(freed.hover, /size: 3 elements.*lifetime: ended on line 4/); -const dereference = check(prefix + ' let p = new(int); free(p); return *p;\n}\n'); +const dereference = check(prefix + ' let p = rt.new(int); rt.free(p); return *p;\n}\n'); assert.match(dereference.warnings[0]?.message || '', /use after free/); -const field = check('struct item { int value; };\n' + prefix + ' let p = new(item); free(p); return p.value;\n}\n'); +const field = check('struct item { int value; };\n' + prefix + ' let p = rt.new(item); rt.free(p); return p.value;\n}\n'); assert.match(field.warnings[0]?.message || '', /use after free/); const scope = check(prefix + ' int* p;\n { let a[3]; p = a; }\n return p[0];\n}\n', 'p['); assert.match(scope.warnings[0]?.message || '', /use after free.*line 3/); assert.match(scope.hover, /lifetime: ended on line 3/); const safe = [ - prefix + ' let p = new(int, 3); let q = p as u8*; return q[23]; }', - prefix + ' let p = new(int, 3); let q = (u8*)p; return q[23]; }', + prefix + ' let p = rt.new(int, 3); let q = p as u8*; return q[23]; }', + prefix + ' let p = rt.new(int, 3); let q = (u8*)p; return q[23]; }', prefix + ' int n = 2; let p = &n; let q = n; return q[2]; }', - prefix + ' let p = new(int, 3); let ignored = 0 && free(p); return p[0]; }', - prefix + ' let p = new(int, 3); let ignored = 1 || free(p); return p[0]; }', - prefix + ' let p = new(int, 50); return p[(u8)300]; }', - prefix + ' let p = new(int, 50); return p[300 as u8]; }', - prefix + ' let p = new(int, 3); let q = p; q[2] = 7; free(p); return 0; }', - prefix + ' let p = new(3); free(p); p = new(3); return p[0]; }', - 'unchecked fn unknown(int* p) {}\n' + prefix + ' let p = new(int, 3); unknown(p); return p[9]; }', - 'fn maybe(int n) { return n; }\n' + prefix + ' let p = new(int, 3); return p[maybe(9)]; }', - prefix + ' let p = new(int, 3); if (1) free(p); return p[0]; }', - prefix + ' let p = new(int, 3); while (0) free(p); return p[0]; }', - prefix + ' let p = new(int, 3); let f = fn() { free(p); return 0; }; return p[0]; }', - prefix + ' let p = new(int[?], 0); push(p, 9); return p[0]; }', + prefix + ' let p = rt.new(int, 3); let ignored = 0 && rt.free(p); return p[0]; }', + prefix + ' let p = rt.new(int, 3); let ignored = 1 || rt.free(p); return p[0]; }', + prefix + ' let p = rt.new(int, 50); return p[(u8)300]; }', + prefix + ' let p = rt.new(int, 50); return p[300 as u8]; }', + prefix + ' let p = rt.new(int, 3); let q = p; q[2] = 7; rt.free(p); return 0; }', + prefix + ' let p = rt.new(3); rt.free(p); p = rt.new(3); return p[0]; }', + 'unchecked fn unknown(int* p) {}\n' + prefix + ' let p = rt.new(int, 3); unknown(p); return p[9]; }', + 'fn maybe(int n) { return n; }\n' + prefix + ' let p = rt.new(int, 3); return p[maybe(9)]; }', + prefix + ' let p = rt.new(int, 3); if (1) rt.free(p); return p[0]; }', + prefix + ' let p = rt.new(int, 3); while (0) rt.free(p); return p[0]; }', + prefix + ' let p = rt.new(int, 3); let f = fn() { rt.free(p); return 0; }; return p[0]; }', + prefix + ' let p = rt.new(int[?], 0); push(p, 9); return p[0]; }', ]; for (const source of safe) assert.deepStrictEqual(check(source).warnings, [], source); -const preserved = check('int read(int* p) { return p[0]; }\n' + prefix + ' let p = new(int, 3); read(p); return p[9]; }'); +const preserved = check('int read(int* p) { return p[0]; }\n' + prefix + ' let p = rt.new(int, 3); read(p); return p[9]; }'); assert.match(preserved.warnings[0]?.message || '', /index 9.*length 3/); const unknown = check('int read(int* p) { return p[0]; }\nfn main() { int n = 1; return read(&n); }\n', 'p['); assert.match(unknown.hover, /size: unknown.*lifetime: unknown/); const stack = check('fn main() { let a[3]; return a[2]; }\n', 'a['); assert.match(stack.hover, /size: 3 elements.*lifetime: stack storage/); -const replacement = check('fn main() {\n let owner = new(1), replacement = new(2);\n free(owner); owner = replacement;\n return owner[0];\n}\n', 'replacement;'); +const replacement = check('fn main() {\n let owner = rt.new(1), replacement = rt.new(2);\n rt.free(owner); owner = replacement;\n return owner[0];\n}\n', 'replacement;'); assert.ok(!replacement.diagnostics.some(d => d.severity === 1), JSON.stringify(replacement.diagnostics)); assert.match(replacement.hover, /moved: replaces a released owner in `owner` on line 3/); console.log('ok definite bounds, free and scope lifetimes, alias hovers, and conservative unknown cases'); diff --git a/tests/drivers/thread_context_test.py b/tests/drivers/thread_context_test.py index fa3f144..b237acd 100644 --- a/tests/drivers/thread_context_test.py +++ b/tests/drivers/thread_context_test.py @@ -5,7 +5,7 @@ import argparse import subprocess parser = argparse.ArgumentParser() -parser.add_argument('vm', nargs='?', default='bin/zy') +parser.add_argument('vm', nargs='?', default='build/bin/zy') parser.add_argument('--case', default='') parser.add_argument('--library', default='') parser.add_argument('--backend', choices=('native', 'micro'), default='native') diff --git a/tests/drivers/toolchain_lsp_contract_test.js b/tests/drivers/toolchain_lsp_contract_test.js index 0d1b515..2872282 100644 --- a/tests/drivers/toolchain_lsp_contract_test.js +++ b/tests/drivers/toolchain_lsp_contract_test.js @@ -1,4 +1,6 @@ // pending requirements have explicit protocol oracles, selectable in small groups. +// interpreted hosts (no jit) are far slower; ZC_TEST_TIMEOUT_SCALE stretches every deadline +const TIMEOUT_SCALE = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; const assert = require('assert'); const cp = require('child_process'); const fs = require('fs'); @@ -21,7 +23,7 @@ function run(text, requests = [], uri = 'file:///contracts.zc', files = []) { for (const message of messages) proc.stdin.write(frame(message)); output = Buffer.concat(chunks); } else { - const r = cp.spawnSync(host, process.argv.slice(3), { input: Buffer.concat(messages.map(frame)), timeout: 60000, maxBuffer: 32 << 20 }); + const r = cp.spawnSync(host, process.argv.slice(3), { input: Buffer.concat(messages.map(frame)), timeout: 60000 * TIMEOUT_SCALE, maxBuffer: 32 << 20 }); assert.strictEqual(r.status, 0, String(r.stderr)); output = r.stdout; } const replies = []; let data = output; diff --git a/tests/drivers/tools_contract_test.py b/tests/drivers/tools_contract_test.py index df02b2b..6a98678 100644 --- a/tests/drivers/tools_contract_test.py +++ b/tests/drivers/tools_contract_test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """bounded accept/reject tests for standalone build tools and generated images.""" import argparse -import re import struct import subprocess import tempfile @@ -17,32 +16,33 @@ def command(argv, data=None): return r if args.group == 'image': - source = Path('src/core/genome.h').read_text() - body = source.split('{', 1)[1].split('}', 1)[0] - words = [int(x) for x in re.findall(r'-?\d+', body)] + image = Path('build/core/genome.bin').read_bytes() + words = list(struct.unpack('<%dq' % (len(image) // 8), image)) pack = lambda xs: b''.join(struct.pack('<q', x) for x in xs) - good = command(['bin/mkgenome'], pack(words)) - assert good.returncode == 0 and f'#define GLEN {len(words)}'.encode() in good.stdout - for data in [b'', b'x', pack(words[:6]), pack(words) + b'x']: - r = command(['bin/mkgenome'], data) - assert r.returncode != 0 and b'incomplete' in r.stderr + good = command(['build/bin/mkgenome'], image) + assert good.returncode == 0 and good.stdout == image + for data in [b'', b'x', pack(words[:6]), image + b'x']: + r = command(['build/bin/mkgenome'], data) + assert r.returncode != 0 and b'incomplete' in r.stderr and not r.stdout for index in range(3): changed = words[:]; changed[index] ^= 2 - r = command(['bin/mkgenome'], pack(changed)) - assert r.returncode != 0 and b'not a genome' in r.stderr - print('ok mkgenome roundtrip, truncation, alignment and map-header mutation contracts') + r = command(['build/bin/mkgenome'], pack(changed)) + assert r.returncode != 0 and b'not a genome' in r.stderr and not r.stdout + r = command(['build/bin/mkgenome'], b'zc: line 1: something is wrong\nzc: 1 error\n') + assert r.returncode != 0 and b'something is wrong' in r.stderr and not r.stdout + print('ok mkgenome roundtrip, truncation, alignment, map-header mutation and diagnostic contracts') elif args.group == 'map': for preset in ['default', 'small', 'run', 'lsp', 'fixed']: with tempfile.TemporaryDirectory(prefix='tinyvm preset ') as directory: - r = command(['bin/genmap', '--preset', preset, '-o', directory]) + r = command(['build/bin/genmap', '--preset', preset, '-o', directory]) assert r.returncode == 0, (preset, r.stderr) assert (Path(directory) / 'memmap.h').is_file() - assert command(['bin/genmap', '--preset', 'default', '--check']).returncode == 0 + assert command(['build/bin/genmap', '--preset', 'default', '--check']).returncode == 0 for options in [['--words', '-1'], ['--heap', '0'], ['--data', '0'], ['--symbols', '-1'], ['--total', '999999999999999999999'], ['--preset', 'imaginary']]: - r = command(['bin/genmap', *options, '--check']) + r = command(['build/bin/genmap', *options, '--check']) assert r.returncode != 0, options with tempfile.TemporaryDirectory(prefix='tinyvm map ') as directory: - r = command(['bin/genmap', '--preset', 'small', '--locals', '32', '--symbols', '64', '-o', directory]) + r = command(['build/bin/genmap', '--preset', 'small', '--locals', '32', '--symbols', '64', '-o', directory]) assert r.returncode == 0, r.stderr for name in ['memmap.h', 'rt.zc', 'compiler.zc', 'map.txt']: assert (Path(directory) / name).is_file(), name @@ -50,7 +50,7 @@ elif args.group == 'map': else: r = command(['bash', '-n', 'tinyvm.sh']) assert r.returncode == 0, r.stderr - for script in ['src/tools/automap.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/tools/embed.sh']: + for script in ['src/tools/automap.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/tools/findcc.sh', 'src/tools/bootstrap-check.sh']: assert command(['bash', '-n', script]).returncode == 0 assert command(['bash', 'tinyvm.sh', 'imaginary-command']).returncode != 0 assert Path('tinyvm.bat').is_file() diff --git a/tests/drivers/wasm_test.js b/tests/drivers/wasm_test.js index e4f7ecb..56f32a2 100644 --- a/tests/drivers/wasm_test.js +++ b/tests/drivers/wasm_test.js @@ -1,9 +1,9 @@ -// runs programs on the webassembly build (src/web/zc.wasm) and compares with the expected output. +// runs programs on the webassembly build (build/web/zc.wasm, from make web) and compares with the expected output. // usage: node tests/drivers/wasm_test.js [path/to/zc.wasm] const fs = require('fs'); const ZC = require('../../src/web/zc.js'); (async () => { - const zc = await ZC.load(fs.readFileSync(process.argv[2] || __dirname + '/../../src/web/zc.wasm')); + const zc = await ZC.load(fs.readFileSync(process.argv[2] || __dirname + '/../../build/web/zc.wasm')); const cases = [ ['examples/hello.zc', 'hello, world!'], ['examples/fact.zc', '1101110101111100000000\n3628800\n'], diff --git a/tests/drivers/word_literals_test.py b/tests/drivers/word_literals_test.py index a2d230d..cfac811 100644 --- a/tests/drivers/word_literals_test.py +++ b/tests/drivers/word_literals_test.py @@ -5,7 +5,7 @@ import subprocess values = [0, 1, (1 << 62) - 1, 1 << 62, (1 << 63) - 1, -(1 << 62), -(1 << 62) - 1, -(1 << 63), -1] source = 'fn main() {\n' + ''.join(f'put_n({value}); put_c(10);\n' for value in values) + 'return 0; }\n' expected = ''.join(str(value) + '\n' for value in values) -for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: +for command in [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']]: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=20) assert result.returncode == 0 and result.stdout == expected, (command, result.stdout, result.stderr) print('ok full-width positive and negative word literals: ' + command[0]) @@ -13,7 +13,7 @@ source = '''immediate fn add_word_operator() { next(); add_binop("**", 11, "4611 add_word_operator fn main() { put_n(0 ** 0); put_c(10); return 0; } ''' -for command in [['bin/zy'], ['bin/zym'], ['node', 'src/js/tinyvm.js']]: +for command in [['build/bin/zy'], ['build/bin/zym'], ['node', 'src/js/tinyvm.js']]: result = subprocess.run(command, input=source, text=True, capture_output=True, timeout=20) assert result.returncode == 0 and result.stdout == '4611686018427387904\n', (command, result.stdout, result.stderr) print('ok full-width word in a stored operator sequence: ' + command[0]) diff --git a/tests/test.sh b/tests/test.sh index 16aa1a9..fd2bcd3 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,9 +1,14 @@ #!/bin/bash # full check: bootstraps the compiler from zc0, verifies the fixpoint, then runs # the examples and tests on the portable interpreter (host), the microcore, the -# arm64 jit under qemu-user, and the llvm backend. needs gcc. the jit half -# needs aarch64-linux-gnu-gcc and qemu-aarch64 (linux-user, not qemu-system). -# the llvm half needs llvm-config and libLLVM. src/baremetal/run.sh boots the +# arm64 jit under qemu-user, and the llvm backend. needs a c compiler with c23 +# #embed (src/tools/findcc.sh picks one; CC overrides it). the jit half needs +# aarch64-linux-gnu-gcc and qemu-aarch64 (linux-user, not qemu-system). +# +# the suite reads nothing from build/: it lays out the default map, bootstraps +# the compiler and builds every core into its own scratch directory, so it runs +# the same from a clean checkout as after `make`. +# the llvm half needs llvm-config (LLVM_CONFIG names another) and libLLVM. src/baremetal/run.sh boots the # microcore on qemu-system-riscv64 and qemu-system-aarch64 when those cross # compilers and system emulators are installed. each of these is skipped # when its tools are missing. @@ -43,26 +48,36 @@ if [ -n "$SELECT" ]; then fi ROOT=$T -# built here rather than inside a section: interp runs the program list on -# it, and the run-only and bare-metal sections read it as the reference. -gcc -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zy || exit 1 -gcc -O2 -w -fwrapv src/tools/genmap.c -Isrc/core -Ibuild/core -o $T/genmap || exit 1 +CC=${CC:-$(sh src/tools/findcc.sh)} || exit 1 +export CC +# the language server runs on the interpreter unless the host has the arm64 jit, and a +# check of a large document then takes many seconds: stretch the drivers' deadlines +if [ -z "${ZC_TEST_TIMEOUT_SCALE:-}" ]; then case "$(uname -m)" in arm64|aarch64) ZC_TEST_TIMEOUT_SCALE=1;; *) ZC_TEST_TIMEOUT_SCALE=8;; esac; fi +export ZC_TEST_TIMEOUT_SCALE +# the default map, generated as make does it, and the tools built against it +$CC -O2 -w -fwrapv src/tools/genmap.c -o $T/genmap || exit 1 +$T/genmap --preset default --check -o $T/map >$T/genmap.log 2>&1 || { cat $T/genmap.log; echo "FAIL the default map does not match the checked-in sources"; exit 1; } +MAP=$T/map +$CC -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -I$MAP -o $T/zc0 && $CC -O1 -w -fwrapv src/tools/mkgenome.c -I$MAP -o $T/mkg || exit 1 # the c helper for tests/programs/structval.zc: structs by value, loaded with cload -mkdir -p build && gcc -shared -fPIC -w tests/data/sbv.c -o build/sbv.so || exit 1 +mkdir -p build && $CC -shared -fPIC -w tests/data/sbv.c -o build/sbv.so || exit 1 selfc(){ (cat src/tools/build.zc; printf '`'; src) | $1 > $2; } -# bootstrap: zc0 (c) -> g0, g0 compiles compiler.zc -> g1, g1 compiles it -> g2 -gcc -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -Ibuild/core -o $T/zc0 && gcc -O1 -w -fwrapv src/tools/mkgenome.c -Isrc/core -Ibuild/core -o $T/mkg || exit 1 +# bootstrap: zc0 (c) -> g0, g0 compiles compiler.zc -> g1, g1 compiles it -> g2. +# each stage's core embeds the image in its own directory mkdir $T/s0 $T/s1 -cp src/core/core.c src/core/io_linux.c src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h $T/s0/; cp src/core/core.c src/core/io_linux.c src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h $T/s1/ -src | $T/zc0 > $T/s0/genome.h || exit 1 -gcc -pthread -O2 -w -fwrapv -DJTYPE=int $T/s0/core.c $T/s0/io_linux.c -o $T/zy0 -selfc $T/zy0 $T/g1.bin; $T/mkg < $T/g1.bin > $T/s1/genome.h -gcc -pthread -O2 -w -fwrapv -DJTYPE=int $T/s1/core.c $T/s1/io_linux.c -o $T/zy1 +src | $T/zc0 > $T/s0/g0.bin && $T/mkg < $T/s0/g0.bin > $T/s0/genome.bin || exit 1 +$CC -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -I$MAP --embed-dir=$T/s0 -o $T/zy0 || exit 1 +selfc $T/zy0 $T/g1.bin; $T/mkg < $T/g1.bin > $T/s1/genome.bin || exit 1 +$CC -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -I$MAP --embed-dir=$T/s1 -o $T/zy1 || exit 1 selfc $T/zy1 $T/g2.bin cmp -s $T/g1.bin $T/g2.bin && chk 1 1 "fixpoint: the compiler compiles itself to the same $(($(wc -c < $T/g1.bin) / 8)) words" || chk 0 1 "fixpoint" -cmp -s $T/s1/genome.h build/core/genome.h && chk 1 1 "genome.h is up to date" || chk 0 1 "genome.h is stale (run make genome)" +# every core below carries the fixpoint genome: GI is the include and embed path for it +GI="-Isrc/core -I$MAP --embed-dir=$T/s1" +# built here rather than inside a section: interp runs the program list on +# it, and the run-only and bare-metal sections read it as the reference. +$CC -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c $GI -o $T/zy || exit 1 chk "$($T/zc0 < tests/programs/zc0typed.zc | md5sum)" "$($T/zc0 < tests/programs/zc0plain.zc | md5sum)" "zc0 reads types and throws them away: typed and plain sources give one image" for n in $SECS; do @@ -80,7 +95,7 @@ run(){ # $1 = vm command, $2 = label chk "$( (cat tests/programs/backtick.zc; printf '`xyz') | $vm)" "3 3 96 39 xyz" "$l a backtick in a comment, string or char literal does not end the source" chk "$($vm < examples/selfpatch.zc)" "ABC" "$l self-patching code" chk "$($vm < examples/lists.zc | tr '\n' '/')" "[0 1 4 9 16 25]/live: 7/after the block: 0/" "$l ownership frees everything" - chk "$( (cat examples/repl.zc; printf '`fn sq(x) { return x * x; }`put_n(sq(7)); put_c(10);`put_s("hi\\n");') | $vm | tr '\n' '/')" "49/hi/" "$l repl on the resident compiler" + chk "$( (cat examples/repl.zc; printf '`fn sq(x) { return x * x; }`rt.put_n(sq(7)); rt.put_c(10);`rt.put_s("hi\\n");') | $vm | tr '\n' '/')" "49/hi/" "$l repl on the resident compiler" chk "$($vm < examples/typed.zc | tr '\n' '/')" "last: 4/point is 2 words: x int, y int/vec<point*> is 3 words: len int, cap int, items point**/push is bool(vec<T>*, T.d)/the heap says v is a vec<point*>, and v.items[0] is a point/" "$l typed example" chk "$($vm < examples/wiring.zc)" "16 1234 25 4/9" "$l enhancement example" chk "$($vm < tests/programs/allocfast.zc)" "0 0 499500 0" "$l constant-size allocation fast path" @@ -111,7 +126,7 @@ run(){ # $1 = vm command, $2 = label chk "$($vm < tests/programs/enumscope.zc)" "0 5 6 blue" "$l enum members are reached as Enum.MEMBER, also in a case" chk "$($vm < tests/programs/enumtype.zc)" "6 5 1" "$l typed enums: members take their enum, an enum becomes an int" chk "$($vm < tests/programs/tagenum.zc)" "27 20 0 0 1 2" "$l tagged enums: members carry fields, constructors build them, a case binds the fields" - chk "$($vm < tests/programs/tagenum_warn.zc)" "$(printf 'zc: line 5: warning: this switch doesn'"'"'t cover rect\n0')" "$l a switch on a tagged enum that misses a member warns, on the switch's line" + chk "$($vm < tests/programs/tagenum_warn.zc)" "$(printf 'zc: line 8: warning: this switch doesn'"'"'t cover rect\n0')" "$l a switch on a tagged enum that misses a member warns, on the switch's line" chk "$($vm < tests/programs/tagenum_two.zc)" "3 8 12" "$l two tagged enums with the same member name, and a case in an o3 loop" chk "$($vm < tests/programs/o3push.zc | md5sum)" "$(md5sum < tests/programs/o3push.expected)" "$l push and pop of a growable array are inline in o3" chk "$($vm < tests/programs/o3copy.zc)" "520507 1005" "$l o3 copies of a loop variable read the source instead" @@ -136,11 +151,11 @@ hello world!42 1 0" "$l capture.start() collects output and end() hands back the string" chk "$($vm < tests/programs/sys.zc)" "2 zy time" "$l syscalls: write, read, time" - chk "$($vm < tests/programs/memleak.zc)" $'leak: line 5 3 words\n1' "$l memdebug names a live allocation" - chk "$($vm < tests/programs/uaf.zc)" $'zc: line 7: warning: use after free: this storage\'s lifetime ended on line 6\nzy: use after free from line 4' "$l memdebug: use after free" - chk "$($vm < tests/programs/uaf_in.zc)" "zy: use after free from line 4" "$l memdebug: use after free through an interior pointer" - chk "$($vm < tests/programs/doublefree.zc)" $'zy: double free from line 4\nok' "$l memdebug: double free is reported and does nothing" - chk "$($vm < tests/programs/scrub.zc)" $'zc: line 6: warning: use after free: this storage\'s lifetime ended on line 5\n57005' "$l memdebug scrub poisons a freed payload" + chk "$($vm < tests/programs/memleak.zc)" $'leak: line 8 3 words\n1' "$l memdebug names a live allocation" + chk "$($vm < tests/programs/uaf.zc)" $'zc: line 10: warning: use after free: this storage\'s lifetime ended on line 9\nzy: use after free from line 7' "$l memdebug: use after free" + chk "$($vm < tests/programs/uaf_in.zc)" "zy: use after free from line 7" "$l memdebug: use after free through an interior pointer" + chk "$($vm < tests/programs/doublefree.zc)" $'zy: double free from line 7\nok' "$l memdebug: double free is reported and does nothing" + chk "$($vm < tests/programs/scrub.zc)" $'zc: line 9: warning: use after free: this storage\'s lifetime ended on line 8\n57005' "$l memdebug scrub poisons a freed payload" chk "$($vm < tests/programs/interior.zc)" "0" "$l an interior pointer keeps a quarantined block" chk "$($vm < tests/programs/wide.zc)" "1099511627776 1 1 1 -1" "$l 64-bit shifts and equality" chk "$($vm < tests/programs/float.zc)" "4.000000 4.500000 3.000000 2.500000 2.500000 0.001500 0.100000 -0.125000 inf -inf nan 1 1 1 0 0 3 -3 3.000000 -2.000000 2" "$l floating point" @@ -169,7 +184,7 @@ ABC" "$l o3 lifts byte loads and stores" chk "$($vm < examples/bf.zc 2>&1 | md5sum | cut -c1-32)" "223570983752b17f7ece32e93d5e19e4" "$l example: brainfuck compiled at runtime" chk "$($vm < examples/dump.zc 2>&1 | md5sum | cut -c1-32)" "757774b00f6b3cd0bfbcba66448a8619" "$l example: struct printer through reflection" chk "$($vm < examples/tree.zc 2>&1 | md5sum | cut -c1-32)" "b86a2e4e9e31acc294a4ddf86be863f2" "$l example: owning tree frees itself" - chk "$($vm < examples/lang.zc 2>&1 | md5sum | cut -c1-32)" "6b234dc8a0a1aa0fb02eb55a6ff7ccc0" "$l example: pipe operator, unless, assert, times" + chk "$($vm < examples/lang.zc 2>&1 | md5sum | cut -c1-32)" "$(printf '22\none isn'"'"'t bigger than two\n***\nassertion failed on line 50\n' | md5sum | cut -c1-32)" "$l example: pipe operator, unless, assert, times" chk "$($vm < examples/staged.zc 2>&1 | md5sum | cut -c1-32)" "6f790a125c48b0cc7a897cee73ee5627" "$l example: staged code compiled while running" chk "$($vm < examples/quine.zc 2>&1 | md5sum | cut -c1-32)" "$(md5sum < examples/quine.zc | cut -c1-32)" "$l example: quine" chk "$($vm < examples/primes.zc 2>&1 | md5sum | cut -c1-32)" "814faffc70dad0ae520c0671c33e73d4" "$l example: o3 prime sieve" @@ -200,7 +215,7 @@ ABC" "$l o3 lifts byte loads and stores" } s_interp() { T=$ROOT/interp - gcc -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zy + $CC -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c $GI -o $T/zy run $T/zy "interp" python3 tests/drivers/array_methods_test.py $T/zy > $T/arrays.out 2>&1; r=$? grep -v '^ok' $T/arrays.out @@ -218,14 +233,14 @@ s_interp() { chk "$($T/zy < tests/programs/infer_return.zc)" "3" "an owned return with no .u is inferred from the body" chk "$($T/zy < tests/programs/discard_m.zc)" "7" "m* discards an unnamed u result" chk "$($T/zy < tests/programs/closure.zc | tr '\n' ' ')" "31 35 42 2 7 7 109 30 7 16 33 20 101 " "closures: calls through values, captures, nesting, returns" - chk "$($T/zy < tests/programs/decl_undefined.zc 2>&1 | head -1)" "zc: line 1: helper is used but never defined" "a declared function nothing defines fails a build" + chk "$($T/zy < tests/programs/decl_undefined.zc 2>&1 | head -1)" "zc: line 3: helper is used but never defined" "a declared function nothing defines fails a build" if [ "$(uname -s)" = Linux ]; then chk "$($T/zy < tests/programs/rawsys.zc)" "-38" "Host.raw: an unknown syscall number gives -ENOSYS"; else chk "$($T/zy < tests/programs/rawsys.zc)" "-1" "Host.raw: unavailable on this host"; fi if [ "$(uname -m)" = x86_64 ]; then chk "$($T/zy < tests/programs/rawpid.zc)" "1" "Host.raw: getpid (39 on x86-64) answers"; fi } s_micro() { T=$ROOT/micro - gcc -pthread -DZY_THREADS -O2 -w -fwrapv src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -o $T/zym || exit 1 + $CC -pthread -DZY_THREADS -O2 -w -fwrapv src/micro/micro.c src/micro/port_linux.c $GI -Isrc/micro -o $T/zym || exit 1 run $T/zym "micro" python3 tests/drivers/array_methods_test.py $T/zym > $T/arrays.out 2>&1; r=$? grep -v '^ok' $T/arrays.out @@ -233,8 +248,9 @@ s_micro() { python3 tests/drivers/actor_test.py $T/zym > $T/actor.out 2>&1; r=$? grep -v '^ok' $T/actor.out chk $r 0 "micro isolated actor acceptance, rejection and regression fixtures" - if command -v node >/dev/null; then - python3 tests/drivers/fuzz_compiler.py $ROOT/zy $T/zym --js > $T/fuzz.out 2>&1; r=$? + if command -v node >/dev/null && command -v tsc >/dev/null && node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null \ + && cp $ROOT/s1/genome.bin $T/jsvm/ && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm; then + python3 tests/drivers/fuzz_compiler.py $ROOT/zy $T/zym --js $T/jsvm/tinyvm.js > $T/fuzz.out 2>&1; r=$? else python3 tests/drivers/fuzz_compiler.py $ROOT/zy $T/zym > $T/fuzz.out 2>&1; r=$? fi @@ -245,8 +261,8 @@ s_micro() { chk $r 0 "malformed compiler input fuzz and host agreement" selfc $T/zym $T/gm.bin cmp -s $T/g1.bin $T/gm.bin && chk 1 1 "microcore self-compile matches" || chk 0 1 "microcore self-compile matches" - gcc -ffreestanding -w -c src/micro/micro.c -Isrc/core -Isrc/micro -o $T/micro.o && gcc -ffreestanding -w -c src/micro/port_stub.c -Isrc/core -Isrc/micro -o $T/stub.o && chk 1 1 "microcore and the stub port compile freestanding" || chk 0 1 "microcore and the stub port compile freestanding" - gcc -pthread -O2 -w -fwrapv -DJTYPE=int -DZY_SOFTFLOAT -DZYINTERP src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zysoft || exit 1 + $CC -ffreestanding -w -c src/micro/micro.c $GI -Isrc/micro -o $T/micro.o && $CC -ffreestanding -w -c src/micro/port_stub.c $GI -Isrc/micro -o $T/stub.o && chk 1 1 "microcore and the stub port compile freestanding" || chk 0 1 "microcore and the stub port compile freestanding" + $CC -pthread -O2 -w -fwrapv -DJTYPE=int -DZY_SOFTFLOAT -DZYINTERP src/core/core.c src/core/io_linux.c $GI -o $T/zysoft || exit 1 chk "$($T/zysoft < tests/programs/float.zc)" "4.000000 4.500000 3.000000 2.500000 2.500000 0.001500 0.100000 -0.125000 inf -inf nan 1 1 1 0 0 3 -3 3.000000 -2.000000 2" "soft-float fallback matches the hardware line" chk "$($T/zysoft < tests/programs/ftruth.zc)" "00111000110107" "soft-float negative zero is false" chk "$($T/zysoft < tests/programs/fo3.zc)" "5997 5997 4.000000 nan 0 3.250000 1 0 0" "soft-float o3 float ops match" @@ -256,7 +272,7 @@ s_micro() { s_small() { T=$ROOT/small # smaller machines. addresses are literals, so each size is its own genome. - gcc -O2 -w -fwrapv src/tools/genmap.c -Isrc/core -Ibuild/core -o $T/genmap || exit 1 + $CC -O2 -w -fwrapv src/tools/genmap.c $GI -o $T/genmap || exit 1 $T/genmap --preset default --check >/dev/null && chk 1 1 "default map matches the checked-in sources" || chk 0 1 "default map matches the checked-in sources" $T/genmap --preset small -o $T/small || exit 1 $T/genmap --mode compiler --name small --heap-lo 8192 --heap 20480 --types 8192 \ @@ -265,13 +281,13 @@ s_small() { cmp -s $T/small/memmap.h $T/small-flags/memmap.h && cmp -s $T/small/rt.zc $T/small-flags/rt.zc \ && cmp -s $T/small/compiler.zc $T/small-flags/compiler.zc && cmp -s $T/small/map.txt $T/small-flags/map.txt \ && chk 1 1 "compiler flags reproduce the small map" || chk 0 1 "compiler flags reproduce the small map" - cp src/tools/zc0.c src/core/fops.h src/core/bops.h src/core/stack.h $T/small/ - gcc -O1 -w -fwrapv $T/small/zc0.c -o $T/zc0s || exit 1 - cat $T/small/rt.zc $T/small/compiler.zc | $T/zc0s > $T/small/genome.h || exit 1 - cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h $T/small/ - gcc -pthread -DZY_THREADS -O2 -w -fwrapv $T/small/micro.c $T/small/port_linux.c -o $T/zyms || exit 1 - gcc -pthread -O2 -w -fwrapv -DJTYPE=int $T/small/core.c $T/small/io_linux.c -o $T/zys || exit 1 - gcc -ffreestanding -w -c $T/small/micro.c -o $T/micro_s.o && gcc -ffreestanding -w -c $T/small/port_stub.c -o $T/stub_s.o && chk 1 1 "small microcore compiles freestanding" || chk 0 1 "small microcore compiles freestanding" + cp src/tools/zc0.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $T/small/ + $CC -O1 -w -fwrapv $T/small/zc0.c -o $T/zc0s || exit 1 + cat $T/small/rt.zc src/core/core.zc $T/small/compiler.zc | $T/zc0s > $T/small/genome.bin || exit 1 + cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $T/small/ + $CC -pthread -DZY_THREADS -O2 -w -fwrapv $T/small/micro.c $T/small/port_linux.c -o $T/zyms || exit 1 + $CC -pthread -O2 -w -fwrapv -DJTYPE=int $T/small/core.c $T/small/io_linux.c -o $T/zys || exit 1 + $CC -ffreestanding -w -c $T/small/micro.c -o $T/micro_s.o && $CC -ffreestanding -w -c $T/small/port_stub.c -o $T/stub_s.o && chk 1 1 "small microcore compiles freestanding" || chk 0 1 "small microcore compiles freestanding" sameout() { # $1 = other vm, $2 = label, $3 = source file local a b a=$($T/zy < "$3" 2>&1 | tr '\n' '/') @@ -309,15 +325,15 @@ s_runonly() { # its own run map: the run-only image is built against this one, and the # small-machine section lays out its own rather than sharing this $T/genmap --preset run -o $T/run || exit 1 - cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h $T/run/ - gcc -O1 -w -fwrapv $T/run/zc0.c -o $T/zc0r || exit 1 + cp src/tools/zc0.c src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $T/run/ + $CC -O1 -w -fwrapv $T/run/zc0.c -o $T/zc0r || exit 1 runonly() { # $1 label, $2 source. the program is already in the image, so it reads no source. # the map is sized to the program: src/tools/automap.sh lays it out for the image's own length local a b d=$T/run-$1 input=$2 if [ "$1" = gc ]; then input=$T/collector-progress.zc; sed 's/f >= 990/f > 0/' "$2" > "$input"; fi GENMAP=$T/genmap bash src/tools/automap.sh run $d src/runtime/run.zc "$input" >/dev/null || { chk 0 1 "run $1 builds"; return; } - cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h $d/ - gcc -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr || { chk 0 1 "run $1 links"; return; } + cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $d/ + $CC -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr || { chk 0 1 "run $1 links"; return; } a=$($T/zy < "$input" 2>&1 | tr '\n' '/') b=$($T/zymr </dev/null 2>&1 | tr '\n' '/') chk "$b" "$a" "run $1" @@ -334,12 +350,12 @@ s_runonly() { runonly strings tests/programs/strings.zc runonly strings-ex examples/strings.zc runonly share tests/programs/share.zc - d=$T/run-echo; GENMAP=$T/genmap bash src/tools/automap.sh run $d src/runtime/run.zc examples/echo.zc >/dev/null && cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h $d/ \ - && gcc -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr \ + d=$T/run-echo; GENMAP=$T/genmap bash src/tools/automap.sh run $d src/runtime/run.zc examples/echo.zc >/dev/null && cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $d/ \ + && $CC -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr \ && chk "$(printf 'abc\n' | $T/zymr | tr '\n' '/')" "$( (cat examples/echo.zc; printf '`abc\n') | $T/zy | tr '\n' '/')" "run echo" \ || chk 0 1 "run echo" # zcc: a program becomes a static binary that runs without stdin and with a held-open pipe - if CC=gcc src/tools/zcc -o $T/zcc-hello examples/hello.zc >/dev/null 2>&1; then + if src/tools/zcc -o $T/zcc-hello examples/hello.zc >/dev/null 2>&1; then chk "$($T/zcc-hello < /dev/null)" "hello, world!" "zcc: hello, static" chk "$(sleep 2 | timeout 5 $T/zcc-hello; echo $?)" "hello, world! 0" "zcc: a program that never reads stdin does not wait for it" @@ -348,7 +364,7 @@ s_runonly() { printf 'fn main() {\n\tput_n(1\n\treturn 0;\n}\n' > $T/zcc-bad.zc src/tools/zcc -o $T/zcc-bad $T/zcc-bad.zc </dev/null >/dev/null 2>&1; r=$? chk "$r $( [ -e $T/zcc-bad ] && echo yes || echo no)" "1 no" "zcc: a bad program exits 1 and writes no binary" - gcc -ffreestanding -w -c $T/run-echo/micro.c -o $T/micro_r.o && gcc -ffreestanding -w -c $T/run-echo/port_stub.c -o $T/stub_r.o && chk 1 1 "run-only microcore compiles freestanding" || chk 0 1 "run-only microcore compiles freestanding" + $CC -ffreestanding -w -c $T/run-echo/micro.c -o $T/micro_r.o && $CC -ffreestanding -w -c $T/run-echo/port_stub.c -o $T/stub_r.o && chk 1 1 "run-only microcore compiles freestanding" || chk 0 1 "run-only microcore compiles freestanding" } s_bare() { @@ -427,13 +443,13 @@ s_bare() { s_jit() { T=$ROOT/jit if command -v aarch64-linux-gnu-gcc >/dev/null && command -v qemu-aarch64 >/dev/null; then - aarch64-linux-gnu-gcc -pthread -O2 -w -fwrapv -DJTYPE=int -static src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zyj || exit 1 + aarch64-linux-gnu-gcc -pthread -O2 -w -fwrapv -DJTYPE=int -static src/core/core.c src/core/io_linux.c $GI -o $T/zyj || exit 1 printf '#!/bin/sh\nexec qemu-aarch64 %s "$@"\n' $T/zyj > $T/jit; chmod +x $T/jit run $T/jit "jit" chk "$(ZYTRACE=1 $T/jit < tests/programs/o3.zc 2>&1 | grep -c 'bytes of arm64')" "8" "jit compiles every o3 region to arm64" selfc $T/jit $T/gj.bin; cmp -s $T/g1.bin $T/gj.bin && chk 1 1 "jit self-compile matches" || chk 0 1 "jit self-compile matches" # native calls need a dynamic binary (a static one has no dlsym), run against the aarch64 libc - if [ -d /usr/aarch64-linux-gnu/lib ] && aarch64-linux-gnu-gcc -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zyjd -ldl 2>/dev/null; then + if [ -d /usr/aarch64-linux-gnu/lib ] && aarch64-linux-gnu-gcc -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c $GI -o $T/zyjd -ldl 2>/dev/null; then chk "$(qemu-aarch64 -L /usr/aarch64-linux-gnu $T/zyjd < tests/programs/cabi.zc | md5sum)" "$(md5sum < tests/programs/cabi.expected)" "jit native calls and callbacks (extern, cfunc)" chk "$(qemu-aarch64 -L /usr/aarch64-linux-gnu $T/zyjd < tests/programs/cabi_o3.zc | md5sum)" "$(md5sum < tests/programs/cabi_o3.expected)" "jit native calls from o3 code, stack arguments" chk "$(qemu-aarch64 -L /usr/aarch64-linux-gnu $T/zyjd < tests/programs/cabi_sized.zc | md5sum)" "$(md5sum < tests/programs/cabi_sized.expected)" "jit native calls with f32 and the sized integers" @@ -449,7 +465,9 @@ s_llvm() { T=$ROOT/llvm # llvm-config can exist without the headers and static libs (a runtime-only install), so the gate is a real build llvm_ok=0 - if command -v llvm-config >/dev/null && gcc -pthread -O2 -w -fwrapv -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zyl $(llvm-config --ldflags --libs) 2>/dev/null; then llvm_ok=1; fi + # LLVM_CONFIG picks the llvm build when the one on PATH has no headers (e.g. /usr/lib/llvm-20/bin/llvm-config) + local lc=${LLVM_CONFIG:-llvm-config} + if command -v $lc >/dev/null && $CC -pthread -O2 -w -fwrapv -DJTYPE=int -DZYLLVM $($lc --cflags) src/core/core.c src/core/io_linux.c $GI -o $T/zyl $($lc --ldflags --libs) 2>/dev/null; then llvm_ok=1; fi if [ $llvm_ok = 1 ]; then run $T/zyl "llvm" chk "$(ZYTRACE=1 $T/zyl < tests/programs/o3.zc 2>&1 | grep -c 'compiled by llvm')" "8" "llvm compiles every o3 region" @@ -558,11 +576,8 @@ s_lsp() { node tests/drivers/diagcache_test.js > $T/dcache.out 2>&1; r=$? grep -v '^ok' $T/dcache.out chk $r 0 "lsp: the diagnostics cache keys and store ($(grep -c '^ok' $T/dcache.out) checks)" - node tests/drivers/ext_cache_test.js > $T/ecache.out 2>&1; r=$? - grep -v '^ok' $T/ecache.out - chk $r 0 "lsp: the extension's cache, end to end ($(grep -c '^ok' $T/ecache.out) checks)" # the same server as WebAssembly, through the extension's host (needs clang) - if command -v clang >/dev/null && clang --print-targets 2>/dev/null | grep -q wasm32 && LSP_BUILD_DIR=$T/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh $T/lsp/lsp.wasm >/dev/null 2>&1; then + if sh src/tools/findcc.sh --wasm >/dev/null 2>&1 && LSP_BUILD_DIR=$T/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh $T/lsp/lsp.wasm >/dev/null 2>&1; then node tests/drivers/hover_surface_test.js --wasm $T/lsp/lsp.wasm > $T/hoversurfacewasm.out 2>&1; r=$? grep -v '^ok' $T/hoversurfacewasm.out chk $r 0 "lsp: embedded and asm hovers through the wasm host" @@ -575,7 +590,8 @@ s_lsp() { else echo "skip lsp wasm tests (no clang with wasm32)"; fi # the same server run by the javascript vm, the extension's fallback when wasm is not there. # the vm starts at the image's heap and grows on the heap's request, so a long document is the test - node src/js/extract_lsp.js >/dev/null && mkdir -p $T/vmserver && cp build/js/tinyvm.js build/js/layout.js build/js/lsp.json $T/vmserver/ + node src/js/extract_lsp.js --lsp $T/lsp --out $T/vmserver >/dev/null && node src/js/mklayout.js --map $MAP --out $T/vmserver >/dev/null \ + && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/vmserver node tests/drivers/hover_surface_test.js --vm $T/vmserver > $T/hoversurfacevm.out 2>&1; r=$? grep -v '^ok' $T/hoversurfacevm.out chk $r 0 "lsp: embedded and asm hovers through the javascript host" @@ -590,6 +606,12 @@ s_lsp() { grep -v '^ok' $T/streamhost.out chk $r 0 "lsp: the wasm and vm servers stream and drop stale checks" fi + # the extension's client end to end, with the server files the package would carry + mkdir -p $T/extserver && cp $T/vmserver/tinyvm.js $T/vmserver/layout.js $T/vmserver/lsp.json $T/vmserver/lsp.bin $T/extserver/ + [ -f $T/lsp/lsp.wasm ] && cp $T/lsp/lsp.wasm $T/extserver/ + node tests/drivers/ext_cache_test.js $T/extserver > $T/ecache.out 2>&1; r=$? + grep -v '^ok' $T/ecache.out + chk $r 0 "lsp: the extension's cache, end to end ($(grep -c '^ok' $T/ecache.out) checks)" else echo "skip lsp protocol tests (no node)"; fi else chk 0 1 "lsp server builds"; fi } @@ -597,36 +619,39 @@ s_lsp() { s_js() { T=$ROOT/js # the javascript vm, and the microcore as webassembly. both run the same genome - if command -v node >/dev/null; then - chk "$(node -e "console.log(require('./build/js/genome.json').glen)")" "$(grep -m1 '#define GLEN' build/core/genome.h | awk '{print $3}')" "build/js/genome.json is up to date" - chk "$(node build/js/tinyvm.js examples/hello.zc)" "hello, world!" "js vm: hello" - chk "$(node build/js/tinyvm.js examples/fact.zc | tail -1)" "3628800" "js vm: factorial" - chk "$(node build/js/tinyvm.js examples/primes.zc | tail -1)" "widest gap: 72, after 31397" "js vm: primes" - chk "$(node build/js/tinyvm.js tests/programs/growone.zc)" "12 1" "js vm: a block past the initial heap grows the machine" - chk "$(node build/js/tinyvm.js tests/programs/replace_owner.zc)" "$(cat tests/programs/replace_owner.expected)" "js vm: replacement ownership and cleanup" - if command -v clang >/dev/null && command -v wasm-ld >/dev/null && clang --print-targets 2>/dev/null | grep -q wasm32; then - bash src/web/build.sh $T/zc.wasm >/dev/null && node tests/drivers/wasm_test.js $T/zc.wasm > $T/wasm.out 2>&1; r=$? + if command -v node >/dev/null && command -v tsc >/dev/null; then + # the vm as make js builds it, with the fixpoint genome beside it + node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null && cp $ROOT/s1/genome.bin $T/jsvm/genome.bin \ + && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm || chk 0 1 "js vm builds" + chk "$(node $T/jsvm/tinyvm.js examples/hello.zc)" "hello, world!" "js vm: hello" + chk "$(node $T/jsvm/tinyvm.js examples/fact.zc | tail -1)" "3628800" "js vm: factorial" + chk "$(node $T/jsvm/tinyvm.js examples/primes.zc | tail -1)" "widest gap: 72, after 31397" "js vm: primes" + chk "$(node $T/jsvm/tinyvm.js tests/programs/growone.zc)" "12 1" "js vm: a block past the initial heap grows the machine" + chk "$(node $T/jsvm/tinyvm.js tests/programs/replace_owner.zc)" "$(cat tests/programs/replace_owner.expected)" "js vm: replacement ownership and cleanup" + if sh src/tools/findcc.sh --wasm >/dev/null 2>&1 && command -v wasm-ld >/dev/null; then + GENOME_DIR=$ROOT/s1 MAP_DIR=$MAP bash src/web/build.sh $T/zc.wasm >/dev/null && node tests/drivers/wasm_test.js $T/zc.wasm > $T/wasm.out 2>&1; r=$? grep -v '^ok' $T/wasm.out chk $r 0 "wasm: the microcore in webassembly ($(grep -c '^ok' $T/wasm.out) checks)" node tests/drivers/fuzz_wasm.js $T/zc.wasm > $T/fuzzw.out 2>&1; r=$? grep -v '^ok' $T/fuzzw.out chk $r 0 "wasm: compiler/runtime/stdlib differential fuzz" - else echo "skip wasm (no clang with the wasm32 target, or no wasm-ld)"; fi - else echo "skip js and wasm (no node)"; fi + else echo "skip wasm (no clang with the wasm32 target and #embed, or no wasm-ld)"; fi + else echo "skip js and wasm (no node or tsc)"; fi } s_make() { T=$ROOT/make # the makefile builds the programs and the run-only image tool if command -v make >/dev/null; then - make -s BIN=$T/mbin B=$T/mbuild all >/dev/null 2>&1 && chk "$($T/mbin/zy < examples/hello.zc)" "hello, world!" "make: bin/zy runs a program" || chk 0 1 "make: all builds" - python3 tests/drivers/cli_help_test.py --group binaries --bin-dir $T/mbin > $T/help.out 2>&1; r=$? + # from an empty build directory, as a fresh checkout would be + make -s B=$T/mbuild all >/dev/null 2>&1 && chk "$($T/mbuild/bin/zy < examples/hello.zc)" "hello, world!" "make: build/bin/zy runs a program" || chk 0 1 "make: all builds" + python3 tests/drivers/cli_help_test.py --group binaries --bin-dir $T/mbuild/bin > $T/help.out 2>&1; r=$? grep -v '^ok' $T/help.out chk $r 0 "command-line help, invalid options and existing lsp source modes" python3 tests/drivers/cli_help_test.py --group scripts > $T/scripthelp.out 2>&1; r=$? grep -v '^ok' $T/scripthelp.out chk $r 0 "build-script help exits without reading input or starting a build" - make -s BIN=$T/mbin B=$T/mbuild run PROG=examples/hello.zc >/dev/null 2>&1 && chk "$($T/mbuild/run/zymr </dev/null)" "hello, world!" "make: run PROG= builds a run-only image" || chk 0 1 "make: run builds" + make -s B=$T/mbuild run PROG=examples/hello.zc >/dev/null 2>&1 && chk "$($T/mbuild/run/zymr </dev/null)" "hello, world!" "make: run PROG= builds a run-only image" || chk 0 1 "make: run builds" else echo "skip make (not installed)"; fi } @@ -634,7 +659,7 @@ s_make() { # every block in it is one of those examples (tests/drivers/learn.py) s_learn() { T=$ROOT/learn - gcc -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core -o $T/zy + $CC -pthread -O2 -w -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c $GI -o $T/zy # learn.py prints its own ok and FAIL lines, which the tally below counts python3 tests/drivers/learn.py $T/zy; true } diff --git a/tests/test_win.ps1 b/tests/test_win.ps1 index 0edc86c..953e875 100644 --- a/tests/test_win.ps1 +++ b/tests/test_win.ps1 @@ -1,17 +1,29 @@ # TinyVM Windows test suite (PowerShell implementation of test.sh) $ErrorActionPreference = "Stop" -# detect compiler (prioritizing LLVM-MinGW with UCRT headers) +# the suite reads nothing from build\: it lays out the default map, bootstraps the +# compiler and builds every executable in its own temporary directory. the cores +# take their genome in with c23 #embed, so the compiler must be clang 19 or gcc 15. +$tempDir = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "tinyvm_test_" + [System.Guid]::NewGuid().ToString("N")) +New-Item -ItemType Directory -Path "$tempDir\s0", "$tempDir\s1", "$tempDir\probe" -Force | Out-Null +[IO.File]::WriteAllText("$tempDir\probe\probe.bin", "ab") +[IO.File]::WriteAllText("$tempDir\probe\probe.c", "static const unsigned char probe[] = {`n#embed `"probe.bin`"`n};`nint main(void) { return sizeof probe == 2 ? 0 : 1; }`n") +function Test-Embed($compiler) { + & $compiler -w "--embed-dir=$tempDir\probe" "$tempDir\probe\probe.c" -o "$tempDir\probe\probe.exe" 2>$null | Out-Null + return $LASTEXITCODE -eq 0 +} +# detect a compiler with #embed (CC first, then LLVM-MinGW with UCRT headers, then PATH) +$cc = $null +$candidates = @() +if ($env:CC) { $candidates += $env:CC } $llvmMinGW = Get-ChildItem "$env:LOCALAPPDATA\Microsoft\WinGet\Packages\MartinStorsjo.LLVM-MinGW*\llvm-mingw*\bin" -ErrorAction SilentlyContinue -if ($llvmMinGW) { - $env:Path = "$($llvmMinGW[0].FullName);$env:Path" - $cc = "clang" -} elseif (Get-Command clang -ErrorAction SilentlyContinue) { - $cc = "clang" -} elseif (Get-Command gcc -ErrorAction SilentlyContinue) { - $cc = "gcc" -} else { - Write-Error "No C compiler (clang or gcc) found in PATH." +if ($llvmMinGW) { $candidates += "$($llvmMinGW[0].FullName)\clang.exe" } +$candidates += @("clang", "gcc", "$env:ProgramFiles\LLVM\bin\clang.exe") +foreach ($c in $candidates) { + if ((Get-Command $c -ErrorAction SilentlyContinue) -and (Test-Embed $c)) { $cc = $c; break } +} +if (-not $cc) { + Write-Error "No C compiler with #embed support (clang 19 or gcc 15 and later) was found." exit 1 } @@ -21,9 +33,6 @@ Write-Host "==================================================================== Write-Host "Compiler: $cc" Write-Host "" -$tempDir = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "tinyvm_test_" + [System.Guid]::NewGuid().ToString("N")) -New-Item -ItemType Directory -Path "$tempDir\s0", "$tempDir\s1" -Force | Out-Null - $totalPassed = 0 $totalFailed = 0 @@ -40,41 +49,41 @@ function Report-Result($success, $name, $detail = "") { try { # --- 1. bootstrap & fixpoint --- Write-Host "=== 1. Bootstrap and Fixpoint Verification ===" -ForegroundColor Yellow - & $cc -fwrapv -O1 -w src\tools\zc0.c -Isrc/core -o "$tempDir\zc0.exe" - & $cc -fwrapv -O1 -w src\tools\mkgenome.c -Isrc/core -o "$tempDir\mkgenome.exe" + # the default map, as make writes it, and the tools built against it + & $cc -fwrapv -O2 -w src\tools\genmap.c -o "$tempDir\genmap.exe" + cmd.exe /c "`"$tempDir\genmap.exe`" --preset default --check -o `"$tempDir\map`" >nul 2>nul" + Report-Result ($LASTEXITCODE -eq 0) "genmap default preset matches the sources" + $map = "$tempDir\map" + & $cc -fwrapv -O1 -w src\tools\zc0.c -Isrc/core "-I$map" -o "$tempDir\zc0.exe" + & $cc -fwrapv -O1 -w src\tools\mkgenome.c "-I$map" -o "$tempDir\mkgenome.exe" - Copy-Item src\core\core.c, src\core\io_win.c, src\core\memmap.h, src\core\fops.h, src\core\bops.h, src\core\stack.h -Destination "$tempDir\s0" - Copy-Item src\core\core.c, src\core\io_win.c, src\core\memmap.h, src\core\fops.h, src\core\bops.h, src\core\stack.h -Destination "$tempDir\s1" - - cmd.exe /c "(type src\runtime\rt.zc & type src\compiler\compiler.zc) | $tempDir\zc0.exe > $tempDir\s0\genome.h" - & $cc -fwrapv -O2 -w -DJTYPE=int "$tempDir\s0\core.c" "$tempDir\s0\io_win.c" -o "$tempDir\zy0.exe" + cmd.exe /c "(type src\runtime\rt.zc & type src\core\core.zc & type src\compiler\compiler.zc) | `"$tempDir\zc0.exe`" > `"$tempDir\s0\g0.bin`"" + cmd.exe /c "`"$tempDir\mkgenome.exe`" < `"$tempDir\s0\g0.bin`" > `"$tempDir\s0\genome.bin`"" + & $cc -fwrapv -O2 -w -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core "-I$map" "--embed-dir=$tempDir\s0" -o "$tempDir\zy0.exe" $bzc = [IO.File]::ReadAllText("src\tools\build.zc") $rt = [IO.File]::ReadAllText("src\runtime\rt.zc") + $core = [IO.File]::ReadAllText("src\core\core.zc") $cp = [IO.File]::ReadAllText("src\compiler\compiler.zc") - $combined = $bzc + [char]96 + $rt + $cp + $combined = $bzc + [char]96 + $rt + $core + $cp [IO.File]::WriteAllBytes("$tempDir\in.tmp", [Text.Encoding]::ASCII.GetBytes($combined)) $p1 = Start-Process -FilePath "$tempDir\zy0.exe" -NoNewWindow -PassThru -RedirectStandardInput "$tempDir\in.tmp" -RedirectStandardOutput "$tempDir\g1.bin" -Wait - cmd.exe /c "$tempDir\mkgenome.exe < $tempDir\g1.bin > $tempDir\s1\genome.h" + cmd.exe /c "`"$tempDir\mkgenome.exe`" < `"$tempDir\g1.bin`" > `"$tempDir\s1\genome.bin`"" + Report-Result ($LASTEXITCODE -eq 0) "the compiler compiles its own source" - & $cc -fwrapv -O2 -w -DJTYPE=int "$tempDir\s1\core.c" "$tempDir\s1\io_win.c" -o "$tempDir\zy1.exe" + & $cc -fwrapv -O2 -w -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core "-I$map" "--embed-dir=$tempDir\s1" -o "$tempDir\zy1.exe" $p2 = Start-Process -FilePath "$tempDir\zy1.exe" -NoNewWindow -PassThru -RedirectStandardInput "$tempDir\in.tmp" -RedirectStandardOutput "$tempDir\g2.bin" -Wait $g1 = [IO.File]::ReadAllBytes("$tempDir\g1.bin") $g2 = [IO.File]::ReadAllBytes("$tempDir\g2.bin") $isFixpoint = [System.Linq.Enumerable]::SequenceEqual($g1, $g2) Report-Result $isFixpoint "Fixpoint: compiler compiles itself to identical $($g1.Length / 8) words" - - $ghRepo = (Get-FileHash "build\core\genome.h").Hash - $ghGen = (Get-FileHash "$tempDir\s1\genome.h").Hash - Report-Result ($ghRepo -eq $ghGen) "genome.h is up to date" - # --- 2. compile test executables --- + # --- 2. compile test executables, each carrying the fixpoint genome --- Write-Host "`n=== 2. Compiling Test Executables ===" -ForegroundColor Yellow - & $cc -fwrapv -O2 -w -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core -o "$tempDir\zy.exe" - & $cc -fwrapv -O2 -w src\micro\micro.c src\micro\port_win.c -Isrc/core -Isrc/micro -o "$tempDir\zym.exe" - & $cc -fwrapv -O2 -w src\tools\genmap.c -Isrc/core -o "$tempDir\genmap.exe" + & $cc -fwrapv -O2 -w -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core "-I$map" "--embed-dir=$tempDir\s1" -o "$tempDir\zy.exe" + & $cc -fwrapv -O2 -w src\micro\micro.c src\micro\port_win.c -Isrc/core "-I$map" "--embed-dir=$tempDir\s1" -Isrc/micro -o "$tempDir\zym.exe" # --- 3. feature tests on zy.exe --- Write-Host "`n=== 3. Running Feature Tests on VM Core (zy.exe) ===" -ForegroundColor Yellow @@ -158,7 +167,7 @@ try { $pass = 0 $fail = 0 foreach ($f in $files) { - $want = ((Get-Content $f.FullName -TotalCount 1) -replace '^// expect:\s*').Trim() + $want = ((Get-Content $f.FullName | Where-Object { $_ -match '^// expect:' } | Select-Object -First 1) -replace '^// expect:\s*').Trim() $got = (Get-Content $f.FullName -Raw | & "$tempDir\zy.exe" 2>&1 | Select-Object -First 1) if ($got -and $got.Contains($want)) { $pass++ @@ -172,9 +181,6 @@ try { # --- 6. tool verification --- Write-Host "`n=== 6. Memory Map Tool Verification (genmap.exe) ===" -ForegroundColor Yellow - cmd.exe /c "`"$tempDir\genmap.exe`" --preset default --check >nul 2>nul" - Report-Result ($LASTEXITCODE -eq 0) "genmap default preset check passed" - cmd.exe /c "`"$tempDir\genmap.exe`" --preset small -o `"$tempDir\small`" >nul 2>nul" Report-Result ($LASTEXITCODE -eq 0) "genmap small preset generated" @@ -182,21 +188,28 @@ try { Write-Host "`n=== 7. ZC Language Server (zclsp.exe) ===" -ForegroundColor Yellow # the server is the real compiler in check mode plus src\lsp\server.zc, on its own map $lspDir = "$tempDir\lsp" - New-Item -ItemType Directory -Force "$lspDir\src" | Out-Null - # two passes: compile once to learn the genome length, then lay the map out again for that length - $lspSrc = "(type `"$lspDir\map\rt.zc`" & type `"$lspDir\map\compiler.zc`" & type src\lsp\server.zc)" - cmd.exe /c "`"$tempDir\genmap.exe`" --preset lsp --grow -o `"$lspDir\map`" >nul 2>nul" - & $cc -fwrapv -O1 -w src\tools\zc0.c "-I$lspDir\map" -Isrc/core -o "$lspDir\zc0.exe" - cmd.exe /c "$lspSrc | `"$lspDir\zc0.exe`" > `"$lspDir\genome.h`"" - $glen = [int]((Select-String -Path "$lspDir\genome.h" -Pattern '#define\s+GLEN\s+(\d+)' | Select-Object -First 1).Matches[0].Groups[1].Value) + New-Item -ItemType Directory -Force "$lspDir" | Out-Null + # the steps of src/tools/automap.sh lsp with DOCS=1: compile once to learn the genome + # length, lay the map out for it, then again with the documentation the hovers read + function Build-LspImage($slot, $define, $out) { + if (Test-Path "$lspDir\map") { Remove-Item -Recurse -Force "$lspDir\map" } + $genomeOption = if ($slot) { "--genome $slot" } else { "" } + cmd.exe /c "`"$tempDir\genmap.exe`" --preset lsp --grow $genomeOption -o `"$lspDir\map`" >nul 2>nul" + & $cc -fwrapv -O1 -w $define src\tools\zc0.c "-I$lspDir\map" -Isrc/core -o "$lspDir\zc0.exe" + & $cc -fwrapv -O1 -w src\tools\mkgenome.c "-I$lspDir\map" -o "$lspDir\mkgenome.exe" + cmd.exe /c "(type `"$lspDir\map\rt.zc`" & type src\core\core.zc & type `"$lspDir\map\compiler.zc`" & type src\lsp\server.zc) | `"$lspDir\zc0.exe`" > `"$lspDir\raw.bin`"" + cmd.exe /c "`"$lspDir\mkgenome.exe`" < `"$lspDir\raw.bin`" > `"$lspDir\$out`"" + return [int]((Get-Item "$lspDir\$out").Length / 8) + } + # ZC0PLAIN defines nothing zc0 reads; it stands in for "no documentation" as one argument + $glen = Build-LspImage $null "-DZC0PLAIN" "image.bin" + $slot = [int]([math]::Ceiling($glen / 4096.0) * 4096) + $glen = Build-LspImage ($slot + 65536) "-DZC0DOCS" "image.bin" $slot = [int]([math]::Ceiling($glen / 4096.0) * 4096) - Remove-Item -Recurse -Force "$lspDir\map" - cmd.exe /c "`"$tempDir\genmap.exe`" --preset lsp --grow --genome $slot -o `"$lspDir\map`" >nul 2>nul" - & $cc -fwrapv -O1 -w src\tools\zc0.c "-I$lspDir\map" -Isrc/core -o "$lspDir\zc0.exe" - cmd.exe /c "$lspSrc | `"$lspDir\zc0.exe`" > `"$lspDir\src\lsp_genome.h`"" - foreach ($f in 'core.c','io_win.c','fops.h','bops.h','stack.h') { Copy-Item "src\core\$f" "$lspDir\src\" } - Copy-Item "$lspDir\map\memmap.h" "$lspDir\src\" - & $cc -fwrapv -O2 -w -DJTYPE=int -DUSE_LSP_GENOME "-I$lspDir\src" "$lspDir\src\core.c" "$lspDir\src\io_win.c" -o "$lspDir\zclsp.exe" + $final = Build-LspImage $slot "-DZC0DOCS" "lsp_genome.bin" + Report-Result ($final -eq $glen) "the LSP genome keeps its length on its sized map" + Copy-Item "$lspDir\map\memmap.h" "$lspDir\" + & $cc -fwrapv -O2 -w -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME src\core\core.c src\core\io_win.c -Isrc/core "-I$lspDir" "--embed-dir=$lspDir" -o "$lspDir\zclsp.exe" Report-Result (Test-Path "$lspDir\zclsp.exe") "zclsp.exe compiled" # CLI check: a clean file @@ -209,7 +222,7 @@ try { # CLI check: the compiler's optimization hints $lspOpt = (cmd.exe /c "`"$lspDir\zclsp.exe`" < examples\opt_test.zc 2>&1") -join "`n" - $hints = @("is never read", "is never reassigned; make it a const", "is a shift; write << 3", "the number 1234 has no name", "new inside a loop", "stack buffer", "handed off once") + $hints = @("is never read", "is never reassigned; make it a const", "is a shift; write << 3", "unannotated numeric literal 1234", "new inside a loop", "stack buffer") foreach ($h in $hints) { Report-Result ($lspOpt.Contains($h)) "zclsp hint: $h" } # methods with this, and closures, check clean @@ -226,23 +239,28 @@ try { Write-Host " [SKIP] Node.js not detected in PATH, skipping lsp protocol tests." -ForegroundColor DarkYellow } - # --- 8. JavaScript / TypeScript vm runtime (build/js/tinyvm.js) --- - Write-Host "`n=== 8. JavaScript VM Runtime (node build/js/tinyvm.js) ===" -ForegroundColor Yellow + # --- 8. JavaScript / TypeScript vm runtime, built as make js builds it --- + Write-Host "`n=== 8. JavaScript VM Runtime (node tinyvm.js) ===" -ForegroundColor Yellow $nodeCmd = Get-Command node -ErrorAction SilentlyContinue - if ($nodeCmd) { - $jsHello = (node build\js\tinyvm.js examples\hello.zc 2>&1) -join "`n" + $tscCmd = Get-Command tsc -ErrorAction SilentlyContinue + if ($nodeCmd -and $tscCmd) { + $jsDir = "$tempDir\js" + node src\js\mklayout.js --map "$map\memmap.h" --out $jsDir | Out-Null + Copy-Item "$tempDir\s1\genome.bin" "$jsDir\genome.bin" + tsc src\js\tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $jsDir + $jsHello = (node $jsDir\tinyvm.js examples\hello.zc 2>&1) -join "`n" Report-Result ($jsHello.Trim() -eq "hello, world!") "JS VM: hello.zc" - $jsFact = (node build\js\tinyvm.js examples\fact.zc 2>&1) -join "`n" + $jsFact = (node $jsDir\tinyvm.js examples\fact.zc 2>&1) -join "`n" Report-Result ($jsFact.Contains("3628800")) "JS VM: fact.zc" - $jsPrimes = (node build\js\tinyvm.js examples\primes.zc 2>&1) -join "`n" + $jsPrimes = (node $jsDir\tinyvm.js examples\primes.zc 2>&1) -join "`n" Report-Result ($jsPrimes.Contains("widest gap: 72")) "JS VM: primes.zc" - $jsMandel = (node build\js\tinyvm.js examples\mandel.zc 2>&1) -join "`n" + $jsMandel = (node $jsDir\tinyvm.js examples\mandel.zc 2>&1) -join "`n" Report-Result ($jsMandel.Contains("*") -or $jsMandel.Contains(".")) "JS VM: mandel.zc (custom fmul enhancement)" } else { - Write-Host " [SKIP] Node.js not detected in PATH, skipping JS tests." -ForegroundColor DarkYellow + Write-Host " [SKIP] Node.js or tsc not detected in PATH, skipping JS tests." -ForegroundColor DarkYellow } } finally { @@ -7,65 +7,83 @@ if /i "%~1"=="help" goto :show_help if /i "%~1"=="--help" goto :show_help if /i "%~1"=="-h" goto :show_help +REM everything built or generated goes under build\ (executables in build\bin); +REM the tree holds sources only. the cores take their genome in with c23 #embed, +REM so the compiler has to be clang 19 or gcc 15 or later. + REM ========================================================================= REM Compiler Detection & Environment Setup REM ========================================================================= set "CC_NAME=" - -REM preserve a compiler selected by the caller. +if not exist "build\probe" mkdir "build\probe" +> "build\probe\probe.bin" echo ab +> "build\probe\probe.c" echo static const unsigned char probe[] = { +>> "build\probe\probe.c" echo #embed "probe.bin" +>> "build\probe\probe.c" echo }; +>> "build\probe\probe.c" echo int main(void) { return sizeof probe ^> 0 ? 0 : 1; } + +REM preserve a compiler selected by the caller, if it can #embed. if defined CC ( - set "CC_NAME=Configured [%CC%]" - goto :has_compiler + call :probe_cc "%CC%" + if not errorlevel 1 ( + set "CC_NAME=Configured [%CC%]" + goto :has_compiler + ) + echo [ERROR] CC=%CC% does not support #embed; clang 19 or gcc 15 is needed. + exit /b 1 ) -REM 1. Search WinGet LLVM-MinGW portable packages first (self-contained with CRT headers) +REM 1. WinGet LLVM-MinGW portable packages (self-contained with CRT headers) for /d %%D in ("%LOCALAPPDATA%\Microsoft\WinGet\Packages\MartinStorsjo.LLVM-MinGW*") do ( for /d %%B in ("%%D\llvm-mingw*") do ( if exist "%%B\bin\clang.exe" ( - set "PATH=%%B\bin;!PATH!" - set "CC=clang" - set "CC_NAME=LLVM-MinGW Clang [WinGet]" - goto :compiler_found + call :probe_cc "%%B\bin\clang.exe" + if not errorlevel 1 ( + set "PATH=%%B\bin;!PATH!" + set "CC=clang" + set "CC_NAME=LLVM-MinGW Clang [WinGet]" + goto :has_compiler + ) ) ) ) -REM 2. Check if clang or gcc is already in PATH -where clang >nul 2>nul -if %errorlevel% equ 0 ( - set "CC=clang" - set "CC_NAME=Clang [in PATH]" - goto :compiler_found -) - -where gcc >nul 2>nul -if %errorlevel% equ 0 ( - set "CC=gcc" - set "CC_NAME=GCC [in PATH]" - goto :compiler_found +REM 2. clang or gcc already in PATH +for %%C in (clang gcc) do ( + where %%C >nul 2>nul + if !errorlevel! equ 0 ( + call :probe_cc %%C + if not errorlevel 1 ( + set "CC=%%C" + set "CC_NAME=%%C [in PATH]" + goto :has_compiler + ) + ) ) -REM 3. Search C:\Program Files\LLVM\bin +REM 3. C:\Program Files\LLVM\bin and the x86 variant if exist "%ProgramFiles%\LLVM\bin\clang.exe" ( - set "PATH=%ProgramFiles%\LLVM\bin;%PATH%" - set "CC=clang" - set "CC_NAME=LLVM Clang [Program Files]" - goto :compiler_found + call :probe_cc "%ProgramFiles%\LLVM\bin\clang.exe" + if not errorlevel 1 ( + set "PATH=%ProgramFiles%\LLVM\bin;%PATH%" + set "CC=clang" + set "CC_NAME=LLVM Clang [Program Files]" + goto :has_compiler + ) ) - set "PFX86=%ProgramFiles(x86)%" if defined PFX86 if exist "!PFX86!\LLVM\bin\clang.exe" ( - set "PATH=!PFX86!\LLVM\bin;%PATH%" - set "CC=clang" - set "CC_NAME=LLVM Clang [Program Files x86]" - goto :compiler_found + call :probe_cc "!PFX86!\LLVM\bin\clang.exe" + if not errorlevel 1 ( + set "PATH=!PFX86!\LLVM\bin;%PATH%" + set "CC=clang" + set "CC_NAME=LLVM Clang [Program Files x86]" + goto :has_compiler + ) ) -:compiler_found -if not "%CC%"=="" goto :has_compiler - -echo [ERROR] No suitable C compiler [clang or gcc] found on this system. +echo [ERROR] No C compiler with #embed support [clang 19 or gcc 15 and later] was found. echo. echo Please install LLVM / Clang: echo winget install MartinStorsjo.LLVM-MinGW.UCRT @@ -74,7 +92,14 @@ echo. pause exit /b 1 +REM :probe_cc compiler - errorlevel 0 when it compiles a file that uses #embed +:probe_cc +%1 -w --embed-dir=build\probe build\probe\probe.c -o build\probe\probe.exe >nul 2>nul +exit /b %errorlevel% + :has_compiler +REM the include and embed paths of a core that carries the compiler genome +set "COREINC=-Isrc/core -Ibuild/core/map --embed-dir=build/core" REM Handle CLI Arguments if provided if not "%~1"=="" goto :handle_cli @@ -92,29 +117,29 @@ echo Compiler : %CC_NAME% echo Working : %CD% echo ------------------------------------------------------------------------------- echo executable status: -if exist "bin\zy.exe" ( echo [X] bin\zy.exe - VM core interpreter ) else ( echo [ ] bin\zy.exe - not built ) -if exist "bin\zym.exe" ( echo [X] bin\zym.exe - portable microcore ) else ( echo [ ] bin\zym.exe - not built ) -if exist "bin\zc0.exe" ( echo [X] bin\zc0.exe - stage 0 bootstrap compiler ) else ( echo [ ] bin\zc0.exe - not built ) -if exist "bin\mkgenome.exe" ( echo [X] bin\mkgenome.exe - genome header generator ) else ( echo [ ] bin\mkgenome.exe - not built ) -if exist "bin\genmap.exe" ( echo [X] bin\genmap.exe - memory map generator ) else ( echo [ ] bin\genmap.exe - not built ) -if exist "bin\zclsp.exe" ( echo [X] bin\zclsp.exe - language server protocol ^& highlighter ) else ( echo [ ] bin\zclsp.exe - not built ) +if exist "build\bin\zy.exe" ( echo [X] build\bin\zy.exe - VM core interpreter ) else ( echo [ ] build\bin\zy.exe - not built ) +if exist "build\bin\zym.exe" ( echo [X] build\bin\zym.exe - portable microcore ) else ( echo [ ] build\bin\zym.exe - not built ) +if exist "build\bin\zc0.exe" ( echo [X] build\bin\zc0.exe - stage 0 bootstrap compiler ) else ( echo [ ] build\bin\zc0.exe - not built ) +if exist "build\bin\mkgenome.exe" ( echo [X] build\bin\mkgenome.exe - genome image checker ) else ( echo [ ] build\bin\mkgenome.exe - not built ) +if exist "build\bin\genmap.exe" ( echo [X] build\bin\genmap.exe - memory map generator ) else ( echo [ ] build\bin\genmap.exe - not built ) +if exist "build\bin\zclsp.exe" ( echo [X] build\bin\zclsp.exe - language server protocol ^& highlighter ) else ( echo [ ] build\bin\zclsp.exe - not built ) echo =============================================================================== -echo [1] build VM core (bin\zy.exe) -echo [2] build microcore (bin\zym.exe) -echo [3] build bootstrap tools (bin\zc0.exe, bin\mkgenome.exe, bin\genmap.exe) -echo [4] build LSP server ^& highlighter (bin\zclsp.exe) +echo [1] build VM core (build\bin\zy.exe) +echo [2] build microcore (build\bin\zym.exe) +echo [3] build bootstrap tools (zc0.exe, mkgenome.exe, genmap.exe) +echo [4] build LSP server ^& highlighter (build\bin\zclsp.exe) echo [5] build everything echo [6] run Windows test suite (native, LSP, JS, bootstrap) -echo [7] run an example on the native VM (bin\zy.exe) -echo [8] run an example in the JavaScript VM (node build/js/tinyvm.js) +echo [7] run an example on the native VM (build\bin\zy.exe) +echo [8] run an example in the JavaScript VM (node build\js\tinyvm.js) echo [9] run a custom .zc source file -echo [L] check/highlight .zc with LSP (bin\zclsp.exe) +echo [L] check/highlight .zc with LSP (build\bin\zclsp.exe) echo [R] launch the interactive REPL (resident compiler) -echo [B] bootstrap and run the full test suite -echo [C] clean generated build directories (bin, build) +echo [G] rebuild the compiler genome and check the fixpoint +echo [C] clean generated output (build) echo [Q] quit echo =============================================================================== -set /p "CHOICE=select an option [1-9, L, R, B, C, Q]: " +set /p "CHOICE=select an option [1-9, L, R, G, C, Q]: " if /i "%CHOICE%"=="1" goto :menu_build_core if /i "%CHOICE%"=="2" goto :menu_build_micro @@ -127,7 +152,7 @@ if /i "%CHOICE%"=="8" goto :menu_examples_js if /i "%CHOICE%"=="9" goto :menu_run_custom if /i "%CHOICE%"=="l" goto :menu_lsp_check if /i "%CHOICE%"=="r" goto :menu_repl -if /i "%CHOICE%"=="b" goto :menu_bootstrap +if /i "%CHOICE%"=="g" goto :menu_genome if /i "%CHOICE%"=="c" goto :menu_clean if /i "%CHOICE%"=="q" exit /b 0 @@ -141,7 +166,6 @@ REM ========================================================================= :menu_build_core echo. -echo [Building zy.exe]... call :build_core echo. pause @@ -149,7 +173,6 @@ goto :main_menu :menu_build_micro echo. -echo [Building zym.exe]... call :build_micro echo. pause @@ -157,7 +180,6 @@ goto :main_menu :menu_build_tools echo. -echo [Building tools: zc0.exe, mkgenome.exe, genmap.exe]... call :build_tools echo. pause @@ -165,7 +187,6 @@ goto :main_menu :menu_build_lsp echo. -echo [Building zclsp.exe (LSP Server ^& Highlighter)]... call :build_lsp echo. pause @@ -173,14 +194,7 @@ goto :main_menu :menu_build_all echo. -echo [Building all components]... -call :build_core -if errorlevel 1 goto :build_failed -call :build_micro -if errorlevel 1 goto :build_failed -call :build_tools -if errorlevel 1 goto :build_failed -call :build_lsp +call :build_all if errorlevel 1 goto :build_failed echo. echo [OK] Build complete. @@ -195,41 +209,23 @@ goto :main_menu :menu_test echo. -if not exist "bin\zy.exe" ( - call :build_core - if errorlevel 1 goto :build_failed -) -if not exist "bin\zym.exe" ( - call :build_micro - if errorlevel 1 goto :build_failed -) -echo [Running test suite]... -powershell -NoProfile -ExecutionPolicy Bypass -File "tests\test_win.ps1" +call :run_tests if errorlevel 1 echo [ERROR] Test suite failed. echo. pause goto :main_menu -:menu_bootstrap +:menu_genome echo. -echo [Running Bootstrap and Full Test Verification]... -call :build_tools +call :build_genome if errorlevel 1 goto :build_failed -if not exist "bin\zy.exe" ( - call :build_core - if errorlevel 1 goto :build_failed -) -powershell -NoProfile -ExecutionPolicy Bypass -File "tests\test_win.ps1" -if errorlevel 1 echo [ERROR] Test suite failed. echo. pause goto :main_menu :menu_clean echo. -echo [Cleaning build artifacts]... -rmdir /s /q build bin 2>nul -echo [OK] Generated bin and build directories removed. +call :clean echo. pause goto :main_menu @@ -245,18 +241,15 @@ if not exist "%CUSTOM_FILE%" ( pause goto :main_menu ) - -if not exist "bin\zy.exe" ( - echo bin\zy.exe not found, building first... +if not exist "build\bin\zy.exe" ( call :build_core if errorlevel 1 goto :build_failed ) - echo. echo =============================================================================== echo Running: %CUSTOM_FILE% echo =============================================================================== -bin\zy.exe < "%CUSTOM_FILE%" +build\bin\zy.exe < "%CUSTOM_FILE%" echo. echo =============================================================================== pause @@ -272,18 +265,15 @@ if not exist "%LSP_FILE%" ( pause goto :main_menu ) - -if not exist "bin\zclsp.exe" ( - echo bin\zclsp.exe not found, building first... +if not exist "build\bin\zclsp.exe" ( call :build_lsp if errorlevel 1 goto :build_failed ) - echo. echo =============================================================================== -echo Analyzing with bin\zclsp.exe: %LSP_FILE% +echo Analyzing with build\bin\zclsp.exe: %LSP_FILE% echo =============================================================================== -bin\zclsp.exe < "%LSP_FILE%" +build\bin\zclsp.exe < "%LSP_FILE%" echo. echo =============================================================================== pause @@ -291,8 +281,7 @@ goto :main_menu :menu_repl echo. -if not exist "bin\zy.exe" ( - echo bin\zy.exe not found, building first... +if not exist "build\bin\zy.exe" ( call :build_core if errorlevel 1 goto :build_failed ) @@ -302,7 +291,7 @@ echo Each input chunk separated by backtick (`) is compiled and run. echo Type expression or function, terminate with ` (or EOF / Ctrl+Z). echo Type 'quit;' then backtick or press Ctrl+C to exit. echo =============================================================================== -(type examples\repl.zc & echo ` & more) | bin\zy.exe +(type examples\repl.zc & echo ` & more) | build\bin\zy.exe echo. pause goto :main_menu @@ -316,47 +305,24 @@ cls echo =============================================================================== echo Run TinyVM Examples on Native VM (zy.exe) echo =============================================================================== -echo [1] hello.zc - Classic "hello, world!" -echo [2] fact.zc - Recursive & iterative factorial computation -echo [3] mandel.zc - Mandelbrot ASCII set using custom enhancement opcode -echo [4] primes.zc - Sieve of Eratosthenes with o3 region compilation -echo [5] life.zc - Conway's Game of Life on a typed torus board -echo [6] lists.zc - Dynamic array with ownership tracking (u, m, d tags) -echo [7] typed.zc - Structs, generic vectors, and reflection -echo [8] wiring.zc - Runtime custom enhancement opcodes demo -echo [9] bf.zc - Brainfuck JIT compiler running inside TinyVM -echo [B] Back to main menu -echo =============================================================================== +call :example_list set /p "EX_CHOICE=Select an example [1-9, B]: " - if /i "%EX_CHOICE%"=="b" goto :main_menu -set "EX_FILE=" -if "%EX_CHOICE%"=="1" set "EX_FILE=examples\hello.zc" -if "%EX_CHOICE%"=="2" set "EX_FILE=examples\fact.zc" -if "%EX_CHOICE%"=="3" set "EX_FILE=examples\mandel.zc" -if "%EX_CHOICE%"=="4" set "EX_FILE=examples\primes.zc" -if "%EX_CHOICE%"=="5" set "EX_FILE=examples\life.zc" -if "%EX_CHOICE%"=="6" set "EX_FILE=examples\lists.zc" -if "%EX_CHOICE%"=="7" set "EX_FILE=examples\typed.zc" -if "%EX_CHOICE%"=="8" set "EX_FILE=examples\wiring.zc" -if "%EX_CHOICE%"=="9" set "EX_FILE=examples\bf.zc" - +call :example_file "%EX_CHOICE%" if "%EX_FILE%"=="" ( echo Invalid choice. timeout /t 2 >nul goto :menu_examples ) - -if not exist "bin\zy.exe" ( +if not exist "build\bin\zy.exe" ( call :build_core if errorlevel 1 goto :build_failed ) - echo. echo =============================================================================== echo Running: %EX_FILE% echo =============================================================================== -bin\zy.exe < "%EX_FILE%" +build\bin\zy.exe < "%EX_FILE%" echo. echo =============================================================================== pause @@ -367,142 +333,216 @@ cls echo =============================================================================== echo Run TinyVM Examples in JavaScript VM (Node.js) echo =============================================================================== -echo [1] hello.zc - Classic "hello, world!" -echo [2] fact.zc - Recursive & iterative factorial computation -echo [3] mandel.zc - Mandelbrot ASCII set using custom enhancement opcode -echo [4] primes.zc - Sieve of Eratosthenes with o3 region compilation -echo [5] life.zc - Conway's Game of Life on a typed torus board -echo [6] lists.zc - Dynamic array with ownership tracking (u, m, d tags) -echo [7] typed.zc - Structs, generic vectors, and reflection -echo [8] wiring.zc - Runtime custom enhancement opcodes demo -echo [9] bf.zc - Brainfuck JIT compiler running inside TinyVM -echo [B] Back to main menu -echo =============================================================================== -set /p "JS_EX_CHOICE=Select an example [1-9, B]: " - -if /i "%JS_EX_CHOICE%"=="b" goto :main_menu -set "JS_EX_FILE=" -if "%JS_EX_CHOICE%"=="1" set "JS_EX_FILE=examples\hello.zc" -if "%JS_EX_CHOICE%"=="2" set "JS_EX_FILE=examples\fact.zc" -if "%JS_EX_CHOICE%"=="3" set "JS_EX_FILE=examples\mandel.zc" -if "%JS_EX_CHOICE%"=="4" set "JS_EX_FILE=examples\primes.zc" -if "%JS_EX_CHOICE%"=="5" set "JS_EX_FILE=examples\life.zc" -if "%JS_EX_CHOICE%"=="6" set "JS_EX_FILE=examples\lists.zc" -if "%JS_EX_CHOICE%"=="7" set "JS_EX_FILE=examples\typed.zc" -if "%JS_EX_CHOICE%"=="8" set "JS_EX_FILE=examples\wiring.zc" -if "%JS_EX_CHOICE%"=="9" set "JS_EX_FILE=examples\bf.zc" - -if "%JS_EX_FILE%"=="" ( +call :example_list +set /p "EX_CHOICE=Select an example [1-9, B]: " +if /i "%EX_CHOICE%"=="b" goto :main_menu +call :example_file "%EX_CHOICE%" +if "%EX_FILE%"=="" ( echo Invalid choice. timeout /t 2 >nul goto :menu_examples_js ) - +call :build_js +if errorlevel 1 goto :build_failed echo. echo =============================================================================== -echo Running in JS: %JS_EX_FILE% +echo Running in JS: %EX_FILE% echo =============================================================================== -node src\js\tinyvm.js "%JS_EX_FILE%" +node build\js\tinyvm.js "%EX_FILE%" echo. echo =============================================================================== pause goto :menu_examples_js +:example_list +echo [1] hello.zc - Classic "hello, world!" +echo [2] fact.zc - Recursive ^& iterative factorial computation +echo [3] mandel.zc - Mandelbrot ASCII set using custom enhancement opcode +echo [4] primes.zc - Sieve of Eratosthenes with o3 region compilation +echo [5] life.zc - Conway's Game of Life on a typed torus board +echo [6] lists.zc - Dynamic array with ownership tracking (u, m, d tags) +echo [7] typed.zc - Structs, generic vectors, and reflection +echo [8] wiring.zc - Runtime custom enhancement opcodes demo +echo [9] bf.zc - Brainfuck compiler running inside TinyVM +echo [B] Back to main menu +echo =============================================================================== +exit /b 0 + +:example_file +set "EX_FILE=" +if "%~1"=="1" set "EX_FILE=examples\hello.zc" +if "%~1"=="2" set "EX_FILE=examples\fact.zc" +if "%~1"=="3" set "EX_FILE=examples\mandel.zc" +if "%~1"=="4" set "EX_FILE=examples\primes.zc" +if "%~1"=="5" set "EX_FILE=examples\life.zc" +if "%~1"=="6" set "EX_FILE=examples\lists.zc" +if "%~1"=="7" set "EX_FILE=examples\typed.zc" +if "%~1"=="8" set "EX_FILE=examples\wiring.zc" +if "%~1"=="9" set "EX_FILE=examples\bf.zc" +exit /b 0 + REM ========================================================================= REM Build Routines REM ========================================================================= -:build_core -echo Compiling zy.exe (VM Core with JIT/Interpreter and Enhancements)... -if not exist "bin" mkdir bin -"%CC%" -O2 -w -fwrapv -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core -o bin\zy.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\zy.exe.& exit /b 1) -echo [OK] bin\zy.exe built successfully. +REM genmap needs no map; the default map comes from it and the zc sources' const blocks +:build_tools +if not exist "build\bin" mkdir build\bin +echo Compiling genmap.exe (memory map generator)... +"%CC%" -O2 -w -fwrapv src\tools\genmap.c -o build\bin\genmap.exe +if errorlevel 1 (echo [ERROR] Failed to build build\bin\genmap.exe.& exit /b 1) +if exist build\core\map rmdir /s /q build\core\map +if not exist build\core mkdir build\core +build\bin\genmap.exe --preset default --check -o build\core\map >nul +if errorlevel 1 (echo [ERROR] The default memory map does not match the sources.& exit /b 1) +echo Compiling zc0.exe (bootstrap compiler)... +"%CC%" -O1 -w -fwrapv src\tools\zc0.c -Isrc/core -Ibuild/core/map -o build\bin\zc0.exe +if errorlevel 1 (echo [ERROR] Failed to build build\bin\zc0.exe.& exit /b 1) +echo Compiling mkgenome.exe (genome image checker)... +"%CC%" -O1 -w -fwrapv src\tools\mkgenome.c -Ibuild/core/map -o build\bin\mkgenome.exe +if errorlevel 1 (echo [ERROR] Failed to build build\bin\mkgenome.exe.& exit /b 1) +echo [OK] Bootstrap tools built in build\bin. exit /b 0 -:build_micro -echo Compiling zym.exe (Portable Microcore)... -if not exist "bin" mkdir bin -"%CC%" -O2 -w -fwrapv src\micro\micro.c src\micro\port_win.c -Isrc/core -Isrc/micro -o bin\zym.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\zym.exe.& exit /b 1) -echo [OK] bin\zym.exe built successfully. +REM zc0 compiles the compiler to g0, g0 compiles it to g1, g1 to g2. g1 and g2 +REM must be identical, and g1 becomes build\core\genome.bin. +:build_genome +call :build_tools +if errorlevel 1 exit /b 1 +echo Bootstrapping the compiler genome... +if not exist build\s0 mkdir build\s0 +if not exist build\s1 mkdir build\s1 +(type src\runtime\rt.zc & type src\core\core.zc & type src\compiler\compiler.zc) | build\bin\zc0.exe > build\s0\g0.bin +if errorlevel 1 (echo [ERROR] Stage 0 failed.& exit /b 1) +build\bin\mkgenome.exe < build\s0\g0.bin > build\s0\genome.bin +if errorlevel 1 (echo [ERROR] Stage 0 did not produce a genome.& exit /b 1) +"%CC%" -O2 -w -fwrapv -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core -Ibuild/core/map --embed-dir=build/s0 -o build\s0\zy0.exe +if errorlevel 1 (echo [ERROR] Failed to build the stage 0 core.& exit /b 1) +(type src\tools\build.zc & echo ` & type src\runtime\rt.zc & type src\core\core.zc & type src\compiler\compiler.zc) | build\s0\zy0.exe > build\s1\g1.bin +build\bin\mkgenome.exe < build\s1\g1.bin > build\s1\genome.bin +if errorlevel 1 (echo [ERROR] The compiler did not compile itself; its diagnostics are above.& exit /b 1) +"%CC%" -O2 -w -fwrapv -DJTYPE=int src\core\core.c src\core\io_win.c -Isrc/core -Ibuild/core/map --embed-dir=build/s1 -o build\s1\zy1.exe +if errorlevel 1 (echo [ERROR] Failed to build the stage 1 core.& exit /b 1) +(type src\tools\build.zc & echo ` & type src\runtime\rt.zc & type src\core\core.zc & type src\compiler\compiler.zc) | build\s1\zy1.exe > build\s1\g2.bin +fc /b build\s1\g1.bin build\s1\g2.bin >nul +if errorlevel 1 (echo [ERROR] Fixpoint failed: the compiler does not compile itself to the same image.& exit /b 1) +copy /y build\s1\genome.bin build\core\genome.bin >nul +echo [OK] Fixpoint reached; build\core\genome.bin updated. exit /b 0 -:build_tools -if not exist "bin" mkdir bin -echo Compiling zc0.exe (Bootstrap compiler)... -"%CC%" -O1 -w -fwrapv src\tools\zc0.c -Isrc/core -o bin\zc0.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\zc0.exe.& exit /b 1) -echo Compiling mkgenome.exe (Genome generator)... -"%CC%" -O1 -w -fwrapv src\tools\mkgenome.c -Isrc/core -o bin\mkgenome.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\mkgenome.exe.& exit /b 1) -echo Compiling genmap.exe (Memory map generator)... -"%CC%" -O2 -w -fwrapv src\tools\genmap.c -Isrc/core -o bin\genmap.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\genmap.exe.& exit /b 1) -echo [OK] Bootstrap tools built successfully in bin\. +:need_genome +if exist build\core\genome.bin if exist build\core\map\memmap.h exit /b 0 +call :build_genome +exit /b %errorlevel% + +:build_core +call :need_genome +if errorlevel 1 exit /b 1 +echo Compiling zy.exe (VM core with the compiler genome)... +"%CC%" -O2 -w -fwrapv -DJTYPE=int src\core\core.c src\core\io_win.c %COREINC% -o build\bin\zy.exe +if errorlevel 1 (echo [ERROR] Failed to build build\bin\zy.exe.& exit /b 1) +echo [OK] build\bin\zy.exe built. +exit /b 0 + +:build_micro +call :need_genome +if errorlevel 1 exit /b 1 +echo Compiling zym.exe (portable microcore)... +"%CC%" -O2 -w -fwrapv src\micro\micro.c src\micro\port_win.c %COREINC% -Isrc/micro -o build\bin\zym.exe +if errorlevel 1 (echo [ERROR] Failed to build build\bin\zym.exe.& exit /b 1) +echo [OK] build\bin\zym.exe built. exit /b 0 +REM the language server: the compiler in check mode plus src\lsp\server.zc, on a +REM map sized to the image (the steps of src/tools/automap.sh lsp with DOCS=1) :build_lsp -echo Compiling zclsp.exe (LSP Server ^& Semantic Highlighter in zc)... -if not exist "bin" mkdir bin -if not exist "bin\zc0.exe" ( - call :build_tools - if errorlevel 1 exit /b 1 -) -if not exist "bin\genmap.exe" ( +if not exist "build\bin\genmap.exe" ( call :build_tools if errorlevel 1 exit /b 1 ) +echo Compiling zclsp.exe (language server ^& semantic highlighter)... if exist build\lsp rmdir /s /q build\lsp -mkdir build\lsp\src -if errorlevel 1 (echo [ERROR] Could not create build\lsp.& exit /b 1) -bin\genmap.exe --preset lsp --grow -o build\lsp\map >nul 2>nul -if errorlevel 1 (echo [ERROR] Could not generate the initial LSP map.& exit /b 1) -"%CC%" -O1 -w -fwrapv src\tools\zc0.c -Ibuild\lsp\map -Isrc/core -o build\lsp\zc0.exe -if errorlevel 1 (echo [ERROR] Could not build the LSP sizing compiler.& exit /b 1) -(type build\lsp\map\rt.zc & type build\lsp\map\compiler.zc & type src\lsp\server.zc) | build\lsp\zc0.exe > build\lsp\plain.h -if errorlevel 1 (echo [ERROR] Could not size the LSP genome.& exit /b 1) -set "GLEN=" -for /f "tokens=3" %%G in ('findstr /b /c:"#define GLEN" build\lsp\plain.h') do set "GLEN=%%G" -if not defined GLEN (echo [ERROR] Could not read the LSP genome size.& exit /b 1) -set /a SLOT=(GLEN+4095)/4096*4096 +mkdir build\lsp +call :lsp_pass "" "" image.bin +if errorlevel 1 exit /b 1 +call :words build\lsp\image.bin +set /a SLOT=(WORDS+4095)/4096*4096 set /a DOCS_WORK=SLOT+65536 -rmdir /s /q build\lsp\map -bin\genmap.exe --preset lsp --grow --genome %DOCS_WORK% -o build\lsp\map >nul 2>nul -if errorlevel 1 (echo [ERROR] Could not generate the documentation build map.& exit /b 1) -"%CC%" -O1 -w -fwrapv -DZC0DOCS src\tools\zc0.c -Ibuild\lsp\map -Isrc/core -o build\lsp\zc0docs.exe -if errorlevel 1 (echo [ERROR] Could not build the documentation compiler.& exit /b 1) -(type build\lsp\map\rt.zc & type build\lsp\map\compiler.zc & type src\lsp\server.zc) | build\lsp\zc0docs.exe > build\lsp\docs-first.h -if errorlevel 1 (echo [ERROR] Could not build the first documentation genome.& exit /b 1) -set "DOCS_GLEN=" -for /f "tokens=3" %%G in ('findstr /b /c:"#define GLEN" build\lsp\docs-first.h') do set "DOCS_GLEN=%%G" -if not defined DOCS_GLEN (echo [ERROR] Could not read the documentation genome size.& exit /b 1) -set /a DOCS_SLOT=(DOCS_GLEN+4095)/4096*4096 -rmdir /s /q build\lsp\map -bin\genmap.exe --preset lsp --grow --genome %DOCS_SLOT% -o build\lsp\map >nul 2>nul -if errorlevel 1 (echo [ERROR] Could not generate the sized LSP map.& exit /b 1) -"%CC%" -O1 -w -fwrapv -DZC0DOCS src\tools\zc0.c -Ibuild\lsp\map -Isrc/core -o build\lsp\zc0docs.exe -if errorlevel 1 (echo [ERROR] Could not rebuild the documentation compiler.& exit /b 1) -(type build\lsp\map\rt.zc & type build\lsp\map\compiler.zc & type src\lsp\server.zc) | build\lsp\zc0docs.exe > build\lsp\src\lsp_genome.h -if errorlevel 1 (echo [ERROR] Could not build the sized documentation genome.& exit /b 1) -set "FINAL_GLEN=" -for /f "tokens=3" %%G in ('findstr /b /c:"#define GLEN" build\lsp\src\lsp_genome.h') do set "FINAL_GLEN=%%G" -if not "!FINAL_GLEN!"=="!DOCS_GLEN!" (echo [ERROR] LSP genome changed length after map sizing.& exit /b 1) -copy /y src\core\core.c build\lsp\src\ >nul +call :lsp_pass %DOCS_WORK% -DZC0DOCS image.bin +if errorlevel 1 exit /b 1 +call :words build\lsp\image.bin +set /a DOCS_SLOT=(WORDS+4095)/4096*4096 +set "DOCS_WORDS=!WORDS!" +call :lsp_pass %DOCS_SLOT% -DZC0DOCS lsp_genome.bin +if errorlevel 1 exit /b 1 +call :words build\lsp\lsp_genome.bin +if not "!WORDS!"=="!DOCS_WORDS!" (echo [ERROR] LSP genome changed length after map sizing.& exit /b 1) +copy /y build\lsp\map\memmap.h build\lsp\memmap.h >nul +"%CC%" -O2 -w -fwrapv -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME src\core\core.c src\core\io_win.c -Isrc/core -Ibuild/lsp --embed-dir=build/lsp -o build\lsp\zclsp.exe +if errorlevel 1 (echo [ERROR] Failed to build zclsp.exe.& exit /b 1) +if not exist "build\bin" mkdir build\bin +copy /y build\lsp\zclsp.exe build\bin\zclsp.exe >nul +echo [OK] build\bin\zclsp.exe built with !WORDS! words. +exit /b 0 + +REM :lsp_pass genome-slot zc0-define output - lay out the lsp map (with that genome +REM slot, or the preset's own when it is empty) and compile the server image into it +:lsp_pass +if exist build\lsp\map rmdir /s /q build\lsp\map +if "%~1"=="" ( + build\bin\genmap.exe --preset lsp --grow -o build\lsp\map >nul 2>nul +) else ( + build\bin\genmap.exe --preset lsp --grow --genome %~1 -o build\lsp\map >nul 2>nul +) +if errorlevel 1 (echo [ERROR] Could not generate the LSP map.& exit /b 1) +"%CC%" -O1 -w -fwrapv %~2 src\tools\zc0.c -Ibuild/lsp/map -Isrc/core -o build\lsp\zc0.exe +if errorlevel 1 (echo [ERROR] Could not build the LSP stage 0 compiler.& exit /b 1) +"%CC%" -O1 -w -fwrapv src\tools\mkgenome.c -Ibuild/lsp/map -o build\lsp\mkgenome.exe +if errorlevel 1 (echo [ERROR] Could not build the LSP image checker.& exit /b 1) +(type build\lsp\map\rt.zc & type src\core\core.zc & type build\lsp\map\compiler.zc & type src\lsp\server.zc) | build\lsp\zc0.exe > build\lsp\raw.bin +if errorlevel 1 (echo [ERROR] Could not compile the LSP image.& exit /b 1) +build\lsp\mkgenome.exe < build\lsp\raw.bin > build\lsp\%~3 +if errorlevel 1 (echo [ERROR] The LSP image does not match its map.& exit /b 1) +exit /b 0 + +REM :words file - WORDS is the file's length in 64-bit words +:words +set /a WORDS=%~z1/8 +exit /b 0 + +REM the javascript vm: layout, genome and the compiled vm in build\js (needs node and tsc) +:build_js +call :need_genome if errorlevel 1 exit /b 1 -copy /y src\core\io_win.c build\lsp\src\ >nul +if not exist build\js mkdir build\js +node src\js\mklayout.js --map build\core\map\memmap.h --out build\js >nul +if errorlevel 1 (echo [ERROR] Could not write the javascript layout.& exit /b 1) +copy /y build\core\genome.bin build\js\genome.bin >nul +call tsc src\js\tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir build\js +if errorlevel 1 (echo [ERROR] Could not compile the javascript vm; tsc is needed.& exit /b 1) +exit /b 0 + +:build_all +call :build_genome if errorlevel 1 exit /b 1 -copy /y src\core\fops.h build\lsp\src\ >nul +call :build_core if errorlevel 1 exit /b 1 -copy /y src\core\bops.h build\lsp\src\ >nul +call :build_micro if errorlevel 1 exit /b 1 -copy /y src\core\stack.h build\lsp\src\ >nul +call :build_lsp +exit /b %errorlevel% + +:run_tests +call :build_core if errorlevel 1 exit /b 1 -copy /y build\lsp\map\memmap.h build\lsp\src\ >nul +call :build_micro if errorlevel 1 exit /b 1 -"%CC%" -O2 -w -fwrapv -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME -Ibuild\lsp\src build\lsp\src\core.c build\lsp\src\io_win.c -o bin\zclsp.exe -if errorlevel 1 (echo [ERROR] Failed to build bin\zclsp.exe.& exit /b 1) -echo [OK] bin\zclsp.exe built successfully with %FINAL_GLEN% words. +powershell -NoProfile -ExecutionPolicy Bypass -File "tests\test_win.ps1" +exit /b %errorlevel% + +:clean +rmdir /s /q build 2>nul +echo [OK] Generated build directory removed. exit /b 0 REM ========================================================================= @@ -514,22 +554,24 @@ set "CMD=%~1" set "ARG2=%~2" if /i "%CMD%"=="build" ( - if "%ARG2%"=="" goto :cli_build_all - if /i "%ARG2%"=="all" goto :cli_build_all + if "%ARG2%"=="" ( call :build_all & exit /b !errorlevel! ) + if /i "%ARG2%"=="all" ( call :build_all & exit /b !errorlevel! ) if /i "%ARG2%"=="core" ( call :build_core & exit /b !errorlevel! ) if /i "%ARG2%"=="micro" ( call :build_micro & exit /b !errorlevel! ) if /i "%ARG2%"=="tools" ( call :build_tools & exit /b !errorlevel! ) if /i "%ARG2%"=="lsp" ( call :build_lsp & exit /b !errorlevel! ) + if /i "%ARG2%"=="js" ( call :build_js & exit /b !errorlevel! ) echo Unknown build target: %ARG2% exit /b 1 ) +if /i "%CMD%"=="genome" ( + call :build_genome + exit /b !errorlevel! +) + if /i "%CMD%"=="test" ( - call :build_core - if errorlevel 1 exit /b 1 - call :build_micro - if errorlevel 1 exit /b 1 - powershell -NoProfile -ExecutionPolicy Bypass -File "tests\test_win.ps1" + call :run_tests exit /b !errorlevel! ) @@ -539,11 +581,11 @@ if /i "%CMD%"=="run" ( echo Usage: tinyvm.bat run path\to\program.zc exit /b 1 ) - if not exist "bin\zy.exe" ( + if not exist "build\bin\zy.exe" ( call :build_core if errorlevel 1 exit /b 1 ) - bin\zy.exe < "%SOURCE_ARG%" + build\bin\zy.exe < "%SOURCE_ARG%" exit /b !errorlevel! ) @@ -553,11 +595,11 @@ if /i "%CMD%"=="run-micro" ( echo Usage: tinyvm.bat run-micro path\to\program.zc exit /b 1 ) - if not exist "bin\zym.exe" ( + if not exist "build\bin\zym.exe" ( call :build_micro if errorlevel 1 exit /b 1 ) - bin\zym.exe < "%SOURCE_ARG%" + build\bin\zym.exe < "%SOURCE_ARG%" exit /b !errorlevel! ) @@ -567,7 +609,9 @@ if /i "%CMD%"=="js" ( echo Usage: tinyvm.bat js path\to\program.zc exit /b 1 ) - node src\js\tinyvm.js "%SOURCE_ARG%" + call :build_js + if errorlevel 1 exit /b 1 + node build\js\tinyvm.js "%SOURCE_ARG%" exit /b !errorlevel! ) @@ -577,62 +621,43 @@ if /i "%CMD%"=="lsp" ( echo Usage: tinyvm.bat lsp path\to\program.zc exit /b 1 ) - if not exist "bin\zclsp.exe" ( + if not exist "build\bin\zclsp.exe" ( call :build_lsp if errorlevel 1 exit /b 1 ) - bin\zclsp.exe < "%SOURCE_ARG%" + build\bin\zclsp.exe < "%SOURCE_ARG%" exit /b !errorlevel! ) if /i "%CMD%"=="repl" ( - if not exist "bin\zy.exe" ( + if not exist "build\bin\zy.exe" ( call :build_core if errorlevel 1 exit /b 1 ) - (type examples\repl.zc & echo ` & more) | bin\zy.exe + (type examples\repl.zc & echo ` & more) | build\bin\zy.exe exit /b !errorlevel! ) if /i "%CMD%"=="clean" ( - rmdir /s /q build bin 2>nul - echo [OK] Generated bin and build directories removed. + call :clean exit /b 0 ) -if /i "%CMD%"=="help" ( - goto :show_help -) -if /i "%CMD%"=="-h" ( - goto :show_help -) -if /i "%CMD%"=="--help" ( - goto :show_help -) - echo Unknown command: %CMD% echo. exit /b 1 :show_help echo usage: tinyvm.bat [command] echo tinyvm.bat launch interactive build ^& run menu -echo tinyvm.bat build [all^|core^|micro^|tools^|lsp] -echo compile TinyVM executables +echo tinyvm.bat build [all^|core^|micro^|tools^|lsp^|js] +echo compile TinyVM executables into build\bin +echo tinyvm.bat genome rebuild build\core\genome.bin and check the fixpoint echo tinyvm.bat run ^<file.zc^> run a .zc program on VM core (zy.exe) echo tinyvm.bat run-micro ^<file.zc^> run a .zc program on microcore (zym.exe) echo tinyvm.bat js ^<file.zc^> run a .zc program on the JavaScript VM (node) echo tinyvm.bat lsp ^<file.zc^> check/highlight a .zc program with zclsp.exe -echo tinyvm.bat test run the full Windows test suite +echo tinyvm.bat test run the Windows test suite echo tinyvm.bat repl launch the resident compiler REPL -echo tinyvm.bat clean remove generated bin and build directories +echo tinyvm.bat clean remove the generated build directory +echo the compiler must support c23 #embed: clang 19 or gcc 15 and later. exit /b 0 - -:cli_build_all -call :build_core -if errorlevel 1 exit /b 1 -call :build_micro -if errorlevel 1 exit /b 1 -call :build_tools -if errorlevel 1 exit /b 1 -call :build_lsp -exit /b !errorlevel! @@ -27,9 +27,9 @@ HELP build() { case "${1:-all}" in all) make ;; - core) make bin/zy ;; - micro) make bin/zym ;; - tools) make bin/zc0 bin/mkgenome bin/genmap ;; + core) make build/bin/zy ;; + micro) make build/bin/zym ;; + tools) make build/bin/zc0 build/bin/mkgenome build/bin/genmap ;; lsp) make lsp ;; *) printf 'Unknown build target: %s\n' "$1" >&2; return 2 ;; esac @@ -47,9 +47,9 @@ run_file() { [[ $# == 2 ]] || { printf 'Usage: tinyvm.sh %s FILE.zc\n' "$mode" >&2; return 2; } input=$(source_path "$2") || return case $mode in - run) exe=bin/zy ;; - run-micro) exe=bin/zym ;; - lsp) exe=bin/zclsp ;; + run) exe=build/bin/zy ;; + run-micro) exe=build/bin/zym ;; + lsp) exe=build/bin/zclsp ;; js) command -v node >/dev/null || { echo 'node.js is required.' >&2; return 1; }; make js || return; node build/js/tinyvm.js "$input"; return ;; esac make "$exe" || return @@ -57,9 +57,9 @@ run_file() { } repl() { - make bin/zy || return + make build/bin/zy || return echo 'Enter statements or fn definitions, ending each chunk with a backtick. Ctrl-D exits.' - { cat examples/repl.zc; printf '`'; cat; } | bin/zy + { cat examples/repl.zc; printf '`'; cat; } | build/bin/zy } dispatch() { |