diff options
35 files changed, 507 insertions, 122 deletions
@@ -123,8 +123,12 @@ $(JS)/lsp.bin: $(BIN)/zclsp$(EXE) src/js/extract_lsp.js src/js/mapvalues.js | $( vsix: bash src/tools/mkvsix.sh -# the browser compiler: the page and its loader beside the wasm build -web: $(WEB)/zc.wasm $(WEB)/index.html $(WEB)/zc.js +# the browser compiler: the page, its loader and the examples it offers beside the wasm build +WEB_EXAMPLES := $(patsubst examples/%,$(WEB)/examples/%,$(wildcard examples/*.zc)) +web: $(WEB)/zc.wasm $(WEB)/index.html $(WEB)/zc.js $(WEB_EXAMPLES) +$(WEB)/examples/%.zc: examples/%.zc + mkdir -p $(WEB)/examples + cp $< $@ $(WEB)/zc.wasm: src/micro/micro.c src/micro/micro_port.h src/web/port_wasm.c src/web/build.sh $(B)/core/genome.bin $(CORE_H) bash src/web/build.sh $@ $(WEB)/index.html $(WEB)/zc.js: $(WEB)/%: src/web/% diff --git a/ONBOARD.md b/ONBOARD.md new file mode 100644 index 0000000..7b0c045 --- /dev/null +++ b/ONBOARD.md @@ -0,0 +1,187 @@ +# onboarding + +a reading and verification order for getting from a fresh checkout to working +knowledge of tinyvm and zc. each step names what to read, what to run, and what +you should be able to answer afterwards. the steps build on each other, so follow +them in order; the whole pass takes a few hours, most of it reading. + +the order was recorded while learning the code from scratch, starting with the +markdown files (the fastest way in) and only then the sources. + +## 0. the shape of the repository + +read `README.md` from the top through "build and run", then skim its section +headings. list the tree: + +```sh +ls; ls src src/*; ls tests docs docs/design +``` + +- the root holds entry points only: `Makefile`, `tinyvm.sh`, `tinyvm.bat`, the + markdown files. +- `src/` holds every source: `core/` (the c vm), `micro/` (the portable vm), + `runtime/` and `compiler/` (zc sources that become the genome), `lib/` (std, + icon, actor), `lsp/` (the language server, in zc), `tools/` (bootstrap and build + tools), `js/`, `web/`, `baremetal/`. +- `build/` holds everything generated and is not in git. nothing outside it is + ever written by a build or a test. + +you should be able to say: what the genome is (the compiler and runtime, compiled +to vm words, embedded in every core), and which files make it +(`src/runtime/rt.zc`, `src/core/core.zc`, `src/compiler/compiler.zc`). + +## 1. build it from nothing + +the cores embed their genome with c23 `#embed`, so you need clang 19 or gcc 15 +or later; `sh src/tools/findcc.sh` prints the compiler `make` will use. + +```sh +make clean +make +build/bin/zy < examples/hello.zc +``` + +watch the output of `make`: the default memory map is generated +(`build/core/map/`), zc0 compiles the genome sources (stage 0), a core built +around that image compiles the compiler again (stage 1) and again (stage 2), the +two self-compiled images are compared, and the first becomes +`build/core/genome.bin`. then `zy`, `zym` and `zclsp` are built around it. + +you should be able to say: why the bootstrap needs three stages, what `mkgenome` +protects against (a compiler's error report taken for an image), and how a core +finds its image (`src/core/genome.h`, `--embed-dir`). + +## 2. the language, by example + +read `docs/LEARN.md` while running its examples: + +```sh +for f in examples/learn/*.zc; do echo "== $f"; build/bin/zy < $f; done +python3 tests/drivers/learn.py build/bin/zy +``` + +then read the language parts of `README.md`: "zc, the language", "memory model", +"types", "o3", "code as data". run a few of the larger examples +(`examples/lang.zc`, `examples/bf.zc`, `examples/dump.zc`, `examples/tree.zc`). + +you should be able to write a small program that includes `rt`, allocates with +`rt.new(T)`, passes an owner to a `.d` parameter, and returns one with `.u`; and +explain why `include rt;` is required before `rt.put_n`. + +## 3. ownership, the analyzer's view + +read `docs/BORROW.md`, then `docs/BOOTSTRAP-OWNERSHIP.md` (how two missing rules +once stopped the compiler compiling itself). run the rejection suites and read a +few of the fixtures, each of which starts with the message it expects: + +```sh +bash tests/runerr.sh build/bin/zy tests/errors +bash tests/runerr.sh build/bin/zy tests/typeerrors +bash tests/runerr.sh build/bin/zy tests/langerrors +``` + +in `src/compiler/compiler.zc`, find the block that starts at +`//: the rules below are the heart of the analyzer` and read `recast`, +`argcheck`, `assigncheck` and `retcheck`, then `exit_vars` and the `lend_*` +helpers. + +you should be able to say what states a local can be in (plain, m, u, d, child, +borrow, dead) and what each of `rt.free(r); r = replacement;` and `g = m*x;` +does to them. + +## 4. the compiler's structure + +`src/compiler/compiler.zc` is about ten thousand lines and begins with an +overview. read the overview, then use the section markers to navigate: + +```sh +grep -n "^//: ----" src/compiler/compiler.zc +``` + +read in this order: interning and symbols, the lexer (`lex1`), expressions +(`primary`, `unary`, `bin`, `expr`), statements and keywords (`add_kw` and the +`kw_*` functions), functions (`fndef`, `fn_begin`, `prologue`), globals +(`gdecl2`, `kw_embed`), then `chk_unit` (check mode for the language server) and +`build_image` (image mode for `src/tools/build.zc`). + +you should be able to say where a new keyword is registered, how a keyword +works at the top level and inside a function, and how the compiler emits a +byte-string literal into the code stream. + +## 5. the vm cores + +read "the instruction set", "memory map and abi", "how the core works" and "the +microcore" in `README.md`, then `docs/design/memory.md`. in `src/core/core.c`, +find `main` and follow it to `run(GORG)`; in `src/micro/micro.c`, `micro_run`. + +```sh +build/bin/genmap --preset default --check | head +ZYTRACE=1 build/bin/zy < tests/programs/o3.zc 2>&1 | head -20 +``` + +you should be able to say where the genome is loaded, where the syscall gate is, +and which backends (interpreter, arm64 jit, llvm, microcore, javascript, wasm) +run the same image. + +## 6. the toolchain around it + +read "language server", "vscode extension", "javascript & typescript runtime" +and "web" in `README.md`, then `editors/vscode/README.md`. build the rest: + +```sh +make js web lspimage +node build/js/tinyvm.js examples/hello.zc +build/bin/zclsp < examples/opt_test.zc +``` + +skim `src/lsp/server.zc` from its overview: the check (`check_doc`), hovers +(`do_hover`), navigation, the formatter (`fm_*`). + +you should be able to say how the editor gets diagnostics (the real compiler in +check mode), and why the javascript and wasm servers need no native build. + +## 7. the libraries + +read `docs/ICON.md` and `docs/ACTOR.md`, then the headers of `src/lib/std.zc`, +`src/lib/icon.zc` and `src/lib/actor.zc`. run their fixtures: + +```sh +python3 tests/drivers/icon_test.py +python3 tests/drivers/actor_test.py build/bin/zy +``` + +you should be able to say how `include actor;` adds the `actor` keyword without +any actor code in the compiler (`run.syntax`, `run.statement_hook`). + +## 8. the tests + +read the header of `tests/test.sh` and `docs/design/test-coverage.md`. run the +quick suite, then the whole one when there is time: + +```sh +make check +make test JOBS=4 +``` + +the suite builds its own map, bootstrap and cores in a scratch directory, so it +does not depend on what is in `build/`. on a host without the arm64 jit the +language server is interpreted and slow; `ZC_TEST_TIMEOUT_SCALE` stretches the +drivers' deadlines (the suite sets it). `LLVM_CONFIG` selects an llvm with +headers for the llvm section. + +## 9. history and open work + +`docs/history/README.md` is the scrapbook of earlier stages, `TODO.md` the +maintenance checklist, and `TASKLIST.md` the record of past requests. read the +open items in `TODO.md` last: by then each one should point at code you have +seen. + +## checklist + +after the pass you should be able to do each of these without looking anything up: + +- rebuild from `make clean`, and say which file each step writes under `build/` +- add a keyword to the compiler and rebuild the genome +- write a program that the analyzer rejects, and predict the message +- build a run-only image (`make run PROG=...`) and say why its source must fit zc0 +- find a built-in's declaration from a hover in the editor or with the server @@ -6,7 +6,7 @@ this folder has the latest code. `docs/LEARN.md` is a tour of the language. `doc ## files -`src/core/core.c` is the native core. it contains the arm64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; `src/compiler/compiler.zc` is the compiler. both are written in zc and begin with an overview. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by `src/compiler/compiler.zc`. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` turns that image into the generated `build/core/genome.h`. `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept: +`src/core/core.c` is the native core. it contains the arm64 jit, the llvm backend (behind `-DZYLLVM`), the portable interpreter, the enhancement unit (recipe parser, vm-code lifter, optimizer and lowering), and the o3 region compiler. `src/runtime/rt.zc` provides i/o, native division enhancements, the heap, and the collector; `src/compiler/compiler.zc` is the compiler. both are written in zc and begin with an overview. together they make up the genome. `src/tools/zc0.c` is stage 0 of the bootstrap, a c compiler for the subset of zc used by `src/compiler/compiler.zc`. `src/tools/build.zc` has the resident compiler compile its own source into a fresh image, and `src/tools/mkgenome.c` checks that image against the memory map before it becomes `build/core/genome.bin`. an image is a file of little-endian 64-bit words, and every core takes its genome in at compile time with c23 `#embed` (`src/core/genome.h`). `examples/` contains small programs (hello, factorial, echo, self-patching code, an owning list, a repl, a typed program, and an enhancement demo) and larger proofs of concept: - `mandel.zc`: the mandelbrot set in 16.16 fixed point, with the fixed-point multiply wired in as a custom opcode. - `bf.zc`: a brainfuck compiler. it turns brainfuck into zc source, has the resident compiler compile it, and runs it (native code on the jit). it takes a program after a backtick, or runs hello world. @@ -25,11 +25,11 @@ this folder has the latest code. `docs/LEARN.md` is a tour of the language. `doc ## build and run -run these commands from the repository root. `make` writes executables to `bin/` and temporary builds to `build/`: +run these commands from the repository root. everything built or generated goes under `build/`: executables in `build/bin/`, the default memory map in `build/core/map/`, the compiler genome in `build/core/genome.bin`, and the javascript, web and editor builds beside them. the tree holds sources only, `make clean` removes `build/`, and every target builds from an empty `build/`: ```sh make -bin/zy < examples/hello.zc +build/bin/zy < examples/hello.zc make test JOBS=4 make genome make web @@ -37,15 +37,15 @@ make js make run PROG=examples/hello.zc ``` -`make genome` bootstraps the compiler, compares two self-compiled images, and updates `build/core/genome.h` only after they agree. rebuild the native, javascript, and web runtimes after changing the genome. `make check` runs the smaller suite while iterating. +the cores embed their genome with c23 `#embed`, so the c compiler has to be clang 19 or gcc 15 or later. `src/tools/findcc.sh` finds one (`CC` set on the command line or in the environment is used as given, and must support it too). `make genome` bootstraps the compiler, compares two self-compiled images, and writes `build/core/genome.bin` only after they agree; `make` reruns it whenever a source of the genome changes, and the cores that embed it are rebuilt after it. `make check` runs the smaller suite while iterating. `ONBOARD.md` is a reading and verification order for getting to know the code. ### command-line help -the native vm, portable vm, language server, bootstrap compiler, image converter and map generator accept `--help` or `-h`. help exits before reading input or allocating vm memory, so it also works with stdin left open. `bin/zy` reads source from stdin and runs `main`; `bin/zym` runs the embedded image, and compiler images can read source from stdin. run-only images read program input directly. `bin/zclsp --format`, `--annotate` and `--types` remain available for source on stdin; without one of these options it reads language-server protocol messages. `bin/zc0` compiles bootstrap source from stdin to a C genome header on stdout; `bin/mkgenome` converts little-endian 64-bit image words from stdin into that header format. `bin/genmap` accepts a preset or compiler/run mode, optional output directory, and build-time map sizing options. the help for each tool lists its input/output format and memory environment variables where applicable. +the native vm, portable vm, language server, bootstrap compiler, image checker and map generator accept `--help` or `-h`. help exits before reading input or allocating vm memory, so it also works with stdin left open. `build/bin/zy` reads source from stdin and runs `main`; `build/bin/zym` runs the embedded image, and compiler images can read source from stdin. run-only images read program input directly. `build/bin/zclsp --format`, `--annotate` and `--types` remain available for source on stdin; without one of these options it reads language-server protocol messages. `build/bin/zc0` compiles bootstrap source from stdin to a genome image on stdout; `build/bin/mkgenome` checks an image from stdin against its memory map and copies it to stdout, and when the input is a compiler's error report instead of an image it prints the report. `build/bin/genmap` accepts a preset or compiler/run mode, optional output directory, and build-time map sizing options. the help for each tool lists its input/output format and memory environment variables where applicable. -the same early help option is available in `tinyvm.sh`, `tinyvm.bat`, `src/tools/zcc`, the public `src/tools/*.sh` build scripts, and the javascript vm and extraction utilities. `src/tools/zcc [-o OUT] [-k] FILE.zc` builds a standalone run-only binary. `src/tools/automap.sh run|small|default|lsp OUTDIR FILE...` generates a sized map and image; `src/tools/embed.sh std|icon|all [--check]` refreshes embedded modules, while its `lsp OUT` form writes source providers. `src/tools/mklsp.sh [OUTDIR]`, `src/tools/mklspwasm.sh [OUTFILE.wasm]` and `src/tools/mkvsix.sh [OUTFILE.vsix]` build the language server and extension. the javascript vm accepts an optional source file; without one it reads source from stdin. the extraction utilities take no input arguments and read the generated headers from their documented paths. `python3 tests/drivers/cli_help_test.py` checks the native, micro, LSP, bootstrap and image tools, shell scripts, and javascript utilities with open stdin and invalid memory settings, rejects unknown options, and confirms the language server source options still work. +the same early help option is available in `tinyvm.sh`, `tinyvm.bat`, `src/tools/zcc`, the public `src/tools/*.sh` build scripts, and the javascript vm and extraction utilities. `src/tools/zcc [-o OUT] [-k] FILE.zc` builds a standalone run-only binary. `src/tools/automap.sh run|small|default|lsp OUTDIR FILE...` generates a sized map (`OUTDIR/memmap.h`) and image (`OUTDIR/genome.bin`). `src/tools/mklsp.sh [OUTDIR]`, `src/tools/mklspwasm.sh [OUTFILE.wasm]` and `src/tools/mkvsix.sh [OUTFILE.vsix]` build the language server and extension. `src/tools/findcc.sh [--wasm]` prints a compiler with `#embed` (and the wasm32 target), and `src/tools/bootstrap-check.sh` runs the first two bootstrap stages and prints the compiler's diagnostics on its own source. the javascript vm accepts an optional source file; without one it reads source from stdin. `src/js/mklayout.js` and `src/js/extract_lsp.js` write the javascript layout and the language server's image from the maps and images under `build/`. `python3 tests/drivers/cli_help_test.py` checks the native, micro, LSP, bootstrap and image tools, shell scripts, and javascript utilities with open stdin and invalid memory settings, rejects unknown options, and confirms the language server source options still work. -`make js` regenerates the JavaScript bundle into `build/js/` and needs Node.js and the TypeScript compiler (`tsc`). running the built `build/js/tinyvm.js` directly needs Node.js only. +`make js` writes the javascript vm into `build/js/`: `tinyvm.js`, its `layout.js` and a copy of `genome.bin` that it reads at start-up. it needs node.js and the typescript compiler (`tsc`). running the built `build/js/tinyvm.js` directly needs node.js only. ### interactive runners @@ -74,12 +74,14 @@ windows needs clang or gcc and powershell for the tests. the batch file is the s on arm64, `src/core/core.c` uses its jit by default. other hosts use the interpreter. `-DZYINTERP` forces interpretation; `-DZYLLVM` enables the llvm backend when its headers and libraries are available. direct builds from the root are: ```sh -mkdir -p bin -cc -O2 -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -o bin/zy -cc -O2 -fwrapv src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -o bin/zym -cc -O2 -fwrapv -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -o bin/zy-llvm $(llvm-config --ldflags --libs) +make genome +clang -O2 -fwrapv -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o build/bin/zy +clang -O2 -fwrapv src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -Ibuild/core/map --embed-dir=build/core -o build/bin/zym +clang -O2 -fwrapv -DJTYPE=int -DZYLLVM $(llvm-config --cflags) src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/core -o build/bin/zy-llvm $(llvm-config --ldflags --libs) ``` +`-I` finds the generated `memmap.h` and `--embed-dir` finds `genome.bin`; pointing them at another map and image (a `src/tools/automap.sh` output directory) builds a core for that machine. + `J` must match the host port: the supplied ports use `int`. a source unit ends at eof or a backtick outside a comment or literal; bytes after that backtick remain available to the program. `ZYTRACE=1` logs translations and installations to stderr. with tracing, `ZYO3DUMP=1` also prints region micro-ops. on llvm builds, `ZYHOT=n` sets the compilation threshold, `ZYPASSES` selects the pass pipeline, and `ZYNOLLVM=1` disables llvm compilation. @@ -104,7 +106,7 @@ check mode records errors instead of printing them and continues after an error, the server runs on its own memory map (`genmap --preset lsp`), which has a bigger genome slot and a smaller frame stack than the default, and it keeps its documents in a store of its own, 4 chars to a word. ```cmd -tinyvm.bat build lsp # bin\zclsp.exe +tinyvm.bat build lsp # build\bin\zclsp.exe src/tools/mklsp.sh build/lsp # elsewhere: build/lsp/zclsp tinyvm.bat lsp examples\opt_test.zc ``` @@ -119,14 +121,15 @@ the `editors/vscode` directory contains the complete VS Code extension for ZC: - **language configuration**: bracket matching, indentation rules, auto-closing pairs, and comments (`//`, `/* */`). - **language client**: a small node client (`client/extension.js`) with no packages. it speaks the protocol properly (byte lengths, `textDocument` wrappers, `didClose`), restarts the server if it exits and sends the open documents again. -build a package with `bash src/tools/mkvsix.sh`, then use "extensions: install from vsix" in vscode. the bundled wasm server needs no native executable. `zc.lsp.serverPath` optionally selects a native server. see [the extension guide](editors/vscode/README.md) for settings and logs. +build a package with `make vsix` (or `bash src/tools/mkvsix.sh`), which writes `build/zc-language-support.vsix`, then use "extensions: install from vsix" in vscode. the server files are staged in `build/editors/vscode/server/` and the extension is packed from a copy in `build/vsix/`, so the tree is never written to; to run the extension from source, open that copy in an extension development host. the bundled wasm server needs no native executable. `zc.lsp.serverPath` optionally selects a native server. see [the extension guide](editors/vscode/README.md) for settings and logs. ### javascript & typescript runtime a complete, self-contained 64-bit TinyVM core implemented in JavaScript and TypeScript is provided in `src/js/`: - `src/js/tinyvm.ts` (compiled to `build/js/tinyvm.js`): Implements the 64-bit word machine on `BigInt64Array` with byte addressing (`Uint8Array` / `DataView`), dual stacks, all 22 base opcodes, byte/float opcodes (612–619), system calls, and custom virtual enhancement execution (`xrun`). -- `build/js/genome.json`: the compiler genome loaded into VM memory, generated. `node src/js/extract_genome.js` rebuilds it from `build/core/genome.h`, and `tests/test.sh` checks it is current. +- `build/js/genome.bin`: the compiler genome, the same image the native cores embed, which `tinyvm.js` reads from beside itself. `build/js/layout.js` holds the memory map's constants, written by `node src/js/mklayout.js` from `build/core/map/memmap.h`. +- file syscalls (open, read, write, close) reach node's file system when the vm is run from the command line, so `include "path"` and `embed` work there; a vm created by other code (the editor's) has no file access unless it passes `files: true`. run any `.zc` program in Node.js: ```cmd @@ -139,11 +142,11 @@ tinyvm.bat js examples/primes.zc ### web (webassembly) -`src/web/` runs the compiler in a browser. `src/web/build.sh` compiles the microcore and genome to `src/web/zc.wasm` with clang's wasm32 target, without emscripten or libc. `src/web/port_wasm.c` provides the port and the 64-by-64-to-128-bit multiply used by the core. `src/web/zc.js` loads the module and exposes `machine.run(source)`, which returns the output produced by `zy < file.zc` from a fresh instance. `src/web/index.html` provides an editor, examples list, and output pane. serve the repository with `python3 -m http.server` and open `http://localhost:8000/src/web/index.html`. +`src/web/` runs the compiler in a browser. `make web` compiles the microcore and genome to `build/web/zc.wasm` with clang's wasm32 target (through `src/web/build.sh`), without emscripten or libc, and copies the page and its loader beside it. `src/web/port_wasm.c` provides the port and the 64-by-64-to-128-bit multiply used by the core. `src/web/zc.js` loads the module and exposes `machine.run(source)`, which returns the output produced by `zy < file.zc` from a fresh instance. `src/web/index.html` provides an editor, examples list, and output pane. serve `build/web` with `python3 -m http.server -d build/web` and open `http://localhost:8000/`. ``` -bash src/web/build.sh -node tests/drivers/wasm_test.js +make web +node tests/drivers/wasm_test.js build/web/zc.wasm ``` ## zc, the language @@ -176,6 +179,16 @@ the array intrinsics also accept receiver syntax: `a.len()`, `a.cap()`, `a.push( `include std;` also registers read-only array algorithms: `find` and `find_if` return a matching index, `contains`, `any`, and `all` return a boolean word, `count` counts matches, `map` transforms elements into a new handle, `filter` copies matching elements into a new handle, and `reduce` folds them from left to right. `any` is false and `all` is true for an empty input; `reduce` returns its initial value. they accept mutable fixed or growable arrays; support for const-qualified arrays is unverified. lambda parameter types follow the element type, map's result type follows the callback return, and filter preserves the input element type. algorithms visit the input's initial length in index order, so callbacks must not change its length. map/filter results are independent handles that shallow-share reference elements with the collector. aggregate-by-value elements are not supported by `push`, `pop`, `map`, or `filter`. [the runnable example](examples/learn/31-array-algorithms.zc) shows inferred callbacks and search. +`embed "path" as [type] name;` declares `name` with the bytes of a file, read when the unit is compiled; the path is found the way an `include` path is (next to the including file, then in the workspace folder, then from the working directory). with no type, or `bstr`, `name` is a byte string of the whole file. `char*` gives a word string, one character per word. a number type reads the bytes little-endian: an integer type takes at most its size and zero-extends a shorter file (sign-extends for `i8`, `i16` and `i32`), and `float` or `f32` needs exactly 8 or 4 bytes. `T[]` and `T[n]` lay the bytes out as array elements of a number type, packed for the sized types and one byte per element for `char[]`; a file that is not a whole number of elements, or that does not fit `T[n]`, is an error. at the top level `name` is a global and the bytes sit in the image; in a function it is a local, and an array is copied into the frame each time the declaration runs. the compiler uses it for the libraries it carries (`embed "src/lib/std.zc" as bstr std_embedded;`). + +``` +include rt; +embed "tests/data/embed/text.txt" as greeting; // a bstr +embed "tests/data/embed/answer.bin" as int answer; +embed "tests/data/embed/bytes.bin" as u8[] table; +fn main() { rt.put_s(greeting); rt.put_n(answer + table[0]); return 0; } +``` + a unit is read whole and scanned for function signatures before it compiles, so calls can come before definitions, tags included, and prototypes are optional. `main` runs after the whole program has compiled. use `void f()` for a function with no result and `return;` to leave it; value-returning functions use an explicit result type or the inferred `fn` form. the runtime api lives in the `rt` namespace and requires `include rt;`: `rt.put_c`, `rt.get_c`, `rt.eof`, `rt.put_s`, `rt.put_n`, `rt.halt`, `rt.len_w`, `rt.eq_w`, `rt.copy_w`, `rt.fill_w`, the packed-string helpers `rt.len_s`, `rt.at_s`, `rt.slice_s`, `rt.cat_s`, `rt.eq_s`, `rt.s_to_n`, `rt.n_to_s` and `rt.put_bs`, allocation and ownership helpers such as `rt.new`, `rt.new_t`, `rt.free`, `rt.gc`, `rt.memreport`, and host functions including `rt.syscall` and `rt.readfile`. their signatures and descriptions are in the runtime source comments and built-in hover table. `src/lib/std.zc` declares the `std` namespace; its static groups are called as `std.Group.name(args)`: `std.Text`, `std.Num`, `std.Host`, `std.Bytes`, `std.Arena`, and `std.Buf`. the constructors `std.arena(n)` and `std.bnew(n)` and globals such as `std.argc` and `std.argv` are plain declarations in that namespace. capture uses static functions: `std.capture.start()` begins output capture and `std.capture.end(buf)` ends it and returns the buffer. @@ -213,7 +226,18 @@ fn main() { **u** transfers the object to the enclosing scope, where it becomes m. transfer occurs through `return`, a block expression value, or assignment to a variable in the directly enclosing block. returning an owned local transfers it automatically; a `.u` result declaration provides the contract to callers before the function body is available. an owned u object cannot fall out of scope or have its result discarded. bind the result to an owner and call `rt.free`, or pass it to a consuming parameter. -explicit recasts use `u*x`, `d*x`, or `m*x`, with no space before the star. a space is ordinary multiplication by a variable of that name. recasts remain available for low-level code, while checked returns, `.d` calls, frees, and owner assignments infer common transfers. a declaration can carry a tag too, `.u r;`, and then whatever lands in it gets that tag. +explicit recasts use `u*x`, `d*x`, or `m*x`, with no space before the star. a space is ordinary multiplication by a variable of that name. recasts remain available for low-level code, while checked returns, `.d` calls, frees, and owner assignments infer common transfers. a declaration can carry a tag too, `.u r;`, and then whatever lands in it gets that tag. recasting a local that owns nothing is an error that names it. + +an owner can move into another local whose object was released first: `rt.free(r); r = replacement;` gives `r` the replacement's object and retires `replacement`, when `replacement` lives in the same block as `r` or a deeper one. an owner cannot be copied into a local that still owns something. + +a global can borrow a local owner for a bounded stretch: `g = m*x;` lends `x` to `g` while `x` keeps ownership and can still be used, and the lend ends when the same block assigns `g` again (usually restoring its old value). the analyzer rejects giving `x` away, returning, or leaving that block while the lend is open, so `g` never points at a released object. this is how an output stream is pointed at a local scratch buffer: + +``` +word saved = rt.outbuf; +rt.outbuf = m*buffer; +render(); +rt.outbuf = saved; +``` parameters: an untagged parameter is a lend, and the compiler infers how a checked body uses it. `.m` states a borrow contract and `.d` receives ownership. Passing an owned value to `.d` transfers it automatically, so `take(p)` can replace `take(d*p)`. A checked borrow can return an alias of its parameter, but it cannot return an alias of a local owner. A fresh result passed to a proved non-retaining function, as in `rt.put_s(message())`, is held in a compiler slot and released after the call. Unknown or retaining callees need an explicit contract or a named owner. @@ -319,7 +343,7 @@ ordinary comments immediately above a declaration provide its hover documentatio ### reflection -`typeof(e)` and `typeof(T)` give a type record and `sizeof(...)` its size. both work on any expression, which is compiled dry and never runs. at runtime: `tkind(t)` (one of the `TY_` constants), `tsize`, `tname`, `tlen`, `tnfields`, `tfname(t, i)`, `tftype`, `tfoff`, `tret`, `tnparams`, `tparam`, `tparamtag` (the u/m/d tag of a parameter), `tretu`, `tnargs` and `targ` for generic instances, and `tprint(t)`. `sigof(f)` gives the type of a function by address (untyped ones get a signature built from their arity and tags), `sigofs("name")` by name, and `rt.typeof_obj(p)` reads the type stamped on a heap object (0 when the header has no type yet: `rt.new(n)` or `rt.new_raw`). +`typeof(e)` and `typeof(T)` give a type record and `sizeof(...)` its size. both work on any expression, which is compiled dry and never runs. the readers are compiler functions, so they need `include run;`: `run.tkind(t)` (one of the `run.TY_` constants), `run.tsize`, `run.tname`, `run.tlen`, `run.tnfields`, `run.tfname(t, i)`, `run.tftype`, `run.tfoff`, `run.tret`, `run.tnparams`, `run.tparam`, `run.tparamtag` (the u/m/d tag of a parameter), `run.tretu`, `run.tnargs` and `run.targ` for generic instances, and `run.tprint(t)`. `run.sigof(f)` gives the type of a function by address (untyped ones get a signature built from their arity and tags), `run.sigofs("name")` by name, and `rt.typeof_obj(p)` reads the type stamped on a heap object (0 when the header has no type yet: `rt.new(n)` or `rt.new_raw`). ### the typechecker @@ -485,7 +509,7 @@ tables: `run.add_binop("**", 11, "ipow c")` adds or replaces a binary operator w ## the bootstrap -`src/tools/zc0.c` compiles the subset of zc used by `src/compiler/compiler.zc` (without the analyzer, types, quote, parse words, or block expressions, but with `enhance` declarations) into the first genome. that genome compiles `src/compiler/compiler.zc` again in the VM using the full compiler, analyzer, and native division code. the compiler passes its lifetime checks and compiles itself into identical words; the test suite checks this fixpoint on the jit and llvm backends when available. an image build (`build_image(org, dst)`) compiles with only the image's names, keywords, and built-in types visible, writes code for the requested address, and appends a symbol table containing its names and enhancements. +`src/tools/zc0.c` compiles the subset of zc used by `src/compiler/compiler.zc` (without the analyzer, types, quote, parse words, or block expressions, but with `enhance` and top-level `embed` declarations) into the first genome, `build/s0/genome.bin`. a core built around that image compiles `src/compiler/compiler.zc` again in the VM using the full compiler, analyzer, and native division code, and `mkgenome` accepts the result only if it is an image (a compiler that rejects its own source prints diagnostics instead, which `mkgenome` shows). a core built around the second image compiles the source a third time; the two self-compiled images must be identical, and the first becomes `build/core/genome.bin`. the test suite checks this fixpoint on the interpreter, microcore, jit and llvm backends when available. in the zc0 subset `u`, `m` and `d` are recast tags, not names. an image build (`build_image(org, dst)`) compiles with only the image's names, keywords, and built-in types visible, writes code for the requested address, and appends a symbol table containing its names and enhancements. an image build (`build_image(org, dst)`) compiles with only the image's names, keywords, and built-in types visible, writes code for the requested address, and appends a symbol table containing its names and enhancements. ## the instruction set @@ -515,7 +539,7 @@ stack effects: `!` is `(v addr --)`, `$` is `(n -- copy of the nth item)` so `0 see [memory configuration](docs/design/memory.md) for build controls, runtime reservations, and host limits. -the initial layout reserves code, compiler tables, the genome, and stacks below `PW`. the heap starts at `PW` on hosted growing maps. tables and execution stacks grow when needed, and emitted code uses stable heap segments after its initial region fills. native code-address metadata grows with it. `MW` selects the default host memory limit (1 GiB, `0x8000000` words, on the default build); `TINYVM_MEMORY_WORDS` and `TINYVM_MAX_MEMORY_WORDS` configure startup memory and its growth limit. the native core reserves stable backing up to the startup maximum, while portable hosts replace their backing allocation when they grow. a build can select another default with `-DMW=...`; `src/core/memmap.h` rejects a value smaller than the initial heap. +the initial layout reserves code, compiler tables, the genome, and stacks below `PW`. the heap starts at `PW` on hosted growing maps. tables and execution stacks grow when needed, and emitted code uses stable heap segments after its initial region fills. native code-address metadata grows with it. `MW` selects the default host memory limit (1 GiB, `0x8000000` words, on the default build); `TINYVM_MEMORY_WORDS` and `TINYVM_MAX_MEMORY_WORDS` configure startup memory and its growth limit. the native core reserves stable backing up to the startup maximum, while portable hosts replace their backing allocation when they grow. a build can select another default with `-DMW=...`; the generated `memmap.h` rejects a value smaller than the initial heap. the default map's header is generated by `genmap --preset default` into `build/core/map/` from the `@map` const blocks in `src/runtime/rt.zc` and `src/compiler/compiler.zc`, which stay the source of truth. ``` 0x000000..0x07FFFF code: the program, with globals between functions (CODE_END) @@ -530,7 +554,7 @@ the initial layout reserves code, compiler tables, the genome, and stacks below 0x1C5000..0x224FFF the heap to start with (0x60000 words), then it grows up to MW ``` -the numbers describe the default build's initial reservations. compact compiler and run-only maps start with smaller reservations and also support growth. `bin/genmap` writes matching layouts (see "smaller machines"); `--preset fixed` retains the older bounded heap for compatibility. the host answers `core.syscall(9, k)` with facts about the map (0 total words, 1 heap start, 2 heap end for now, 3 bytes). argument 4 requests memory growth, and argument 5 changes the growth limit within the host's backing constraints. +the numbers describe the default build's initial reservations. compact compiler and run-only maps start with smaller reservations and also support growth. `build/bin/genmap` writes matching layouts (see "smaller machines"); `--preset fixed` retains the older bounded heap for compatibility. the host answers `core.syscall(9, k)` with facts about the map (0 total words, 1 heap start, 2 heap end for now, 3 bytes). argument 4 requests memory growth, and argument 5 changes the growth limit within the host's backing constraints. calls push the arguments left to right and the callee's prologue pops them into its frame. locals live in the frame at `*0x90000 + slot`. every function leaves exactly one word on the data stack. heap objects have a 5 word header: size, flags with the size class in the high bits, a check word, owner, type. @@ -548,7 +572,7 @@ platform: the darwin path uses `MAP_JIT`, `pthread_jit_write_protect_np` and `sy ## the microcore -`src/micro/micro.c` is a portable interpreter for the same genome, small enough to move to new targets, including bare metal. it complements `src/core/core.c` with the memory map, two stacks, base opcodes, call and return, and genome loading at `GORG`. it has no jit, llvm backend, o3 lifter, or enhancement optimizer. it uses `stdint.h` and the shared headers `src/micro/micro_port.h`, `src/core/fops.h`, `src/core/bops.h`, and `src/core/stack.h`, plus the generated genome header; `micro_port.h` includes `memmap.h`. its five host functions are declared in `src/micro/micro_port.h`. bss must be zero before `micro_run`. on a cpu without a 64-bit divider, the compiler may link its own division helpers. the microcore's initial memory comes from the generated map, and hosted ports can configure the initial size and growth limit. +`src/micro/micro.c` is a portable interpreter for the same genome, small enough to move to new targets, including bare metal. it complements `src/core/core.c` with the memory map, two stacks, base opcodes, call and return, and genome loading at `GORG`. it has no jit, llvm backend, o3 lifter, or enhancement optimizer. it uses `stdint.h` and the shared headers `src/micro/micro_port.h`, `src/core/fops.h`, `src/core/bops.h`, `src/core/stack.h` and `src/core/genome.h`, which embeds the image; `micro_port.h` includes the generated `memmap.h`. its five host functions are declared in `src/micro/micro_port.h`. bss must be zero before `micro_run`. on a cpu without a 64-bit divider, the compiler may link its own division helpers. the microcore's initial memory comes from the generated map, and hosted ports can configure the initial size and growth limit. the host interface is `src/micro/micro_port.h`. `src/micro/port_linux.c` provides stdin, stdout, syscalls, and stderr diagnostics, matching `src/core/io_linux.c` plus the syscall gate in `src/core/core.c`. syscalls 1 to 6 handle files, environment variables, and unix time; 7 exits, 8 reads a counter, and 9 reports or configures memory. `src/micro/port_stub.c` is the freestanding template: connect `uart_putc` to a transmit register, leave `micro_getc` at end of input, and have `micro_syscall` return -1 for every operation. the runtime then sees failed file opens and no clock. `src/baremetal/port.c` implements the same five functions for the virt boards, where the operations are available. link the stub instead of `port_linux.c` and call `micro_run` from reset. @@ -564,12 +588,11 @@ what it leaves out, on purpose: native code, the optimizer, o3's register interp ```sh bash src/tools/automap.sh small build/small -cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/core/fops.h src/core/bops.h src/core/stack.h build/small/ -cc -O2 -fwrapv build/small/micro.c build/small/port_linux.c -Ibuild/small -o build/small/zym +clang -O2 -fwrapv src/micro/micro.c src/micro/port_linux.c -Isrc/core -Isrc/micro -Ibuild/small --embed-dir=build/small -o build/small/zym build/small/zym < examples/hello.zc make run PROG=examples/hello.zc build/run/zymr -bin/genmap --preset default --check +build/bin/genmap --preset default --check ``` use `genmap` options to adjust heap, table, and stack sizes at build time. compile the core against the generated headers and the genome for that same layout. `--preset fixed` keeps the historical layout; `--preset default` places the heap above the fixed regions so it can grow. see [memory configuration](docs/design/memory.md) for controls and current limits. diff --git a/docs/ACTOR.md b/docs/ACTOR.md index 913c6bc..6ee5627 100644 --- a/docs/ACTOR.md +++ b/docs/ACTOR.md @@ -56,4 +56,4 @@ messages copy their typed object graphs into the receiving context. supported va ## checks -`python3 tests/drivers/actor_test.py bin/zy` and `python3 tests/drivers/actor_test.py bin/zym` exercise library-only syntax installation, declared actors, independent captures, two-instance parallelism, callback registration before send, statement delivery, fifo ordering, typed graph copies, closure messages, lifecycle behavior and rejection fixtures. the parallel fixture measures worker time inside the vm: two 300 ms tasks must complete in less than 580 ms. +`python3 tests/drivers/actor_test.py build/bin/zy` and `python3 tests/drivers/actor_test.py build/bin/zym` exercise library-only syntax installation, declared actors, independent captures, two-instance parallelism, callback registration before send, statement delivery, fifo ordering, typed graph copies, closure messages, lifecycle behavior and rejection fixtures. the parallel fixture measures worker time inside the vm: two 300 ms tasks must complete in less than 580 ms. diff --git a/docs/BOOTSTRAP-OWNERSHIP.md b/docs/BOOTSTRAP-OWNERSHIP.md index 509db2f..6cc61ce 100644 --- a/docs/BOOTSTRAP-OWNERSHIP.md +++ b/docs/BOOTSTRAP-OWNERSHIP.md @@ -1,7 +1,9 @@ -# the bootstrap does not compile: owned values cannot be moved between locals +# the bootstrap did not compile: owned values could not be moved between locals -status: open, blocks `make` -affects: `make genome`, the `fixpoint` check, the `genome.h is up to date` check +status: resolved. the analyzer gained the two rules proposed below and a clearer +recast diagnostic, and `make genome` reaches its fixpoint from a clean checkout. +see "resolution" at the end; the rest of this note is the record of the problem. +affects: `make genome` and the `fixpoint` check see also: `docs/BORROW.md` ## summary @@ -10,7 +12,7 @@ see also: `docs/BORROW.md` `src/core/core.zc` and `src/compiler/compiler.zc` into a stage-0 image, stage-0 becoming `build/zy0`, and `zy0` compiling the same three files again into the genome. the second compilation runs with lifetime checking enabled and rejects -the compiler's own source. `bin/mkgenome` then reads the diagnostics as if they +the compiler's own source. `mkgenome` then read the diagnostics as if they were an image: ``` @@ -152,4 +154,30 @@ reports. bash src/tools/bootstrap-check.sh ``` -a passing run prints `image ok: <n> bytes`.
\ No newline at end of file +a passing run prints `image ok: <n> bytes`. + +## resolution + +the analyzer (`src/compiler/compiler.zc`) now accepts the three spellings: + +1. **moves into a released owner.** `rt.free(r); r = joined;` moves `joined` into `r` + when `r` was released first and `joined` is an owner of the same block or a deeper + one. `joined` is retired by the move. a destination that still owns something is + still rejected (`still owns an object; overwriting it would leak it`), and so is a + copy of an owner into a fresh declaration. +2. **bounded lends to globals.** `g = m*x;` with `x` an owned local lends `x` to `g`: + `x` keeps ownership and stays usable, and the lend ends when the same block assigns + `g` again. giving `x` away, returning, breaking or leaving the block while the lend + is open is an error (`is lent to global g; assign the global again before ...`). the + lend list is per function and per block depth (`lend_add`, `lend_clear`, + `lend_exit`), so a restore inside a nested branch does not end a lend made outside it. +3. **recasts of plain locals.** `rt.code_blocks = m*block;` failed because `block` came + from `chk_resize`, which returns collector-managed storage, so `block` never owned + anything. the compiler now writes `rt.code_blocks = block;`, and a recast of a plain + local reports `block: isn't an owned object, so there is no ownership to recast` + instead of the old message. + +`methdef` uses the lend for its scratch buffer and the move for the joined token list. +the regressions are `tests/programs/ownmove.zc` and `tests/errors/lend_*.zc`, +`recast_plain.zc` and `move_live_owner.zc`. + diff --git a/docs/BORROW.md b/docs/BORROW.md index 10beb84..f270574 100644 --- a/docs/BORROW.md +++ b/docs/BORROW.md @@ -27,7 +27,11 @@ effects are cached for the current compiler analysis and participate in the lang aliases are tied to their owner. when a tracked alias has no later use, a subsequent move or free can proceed; uses after a definite transfer or free produce an error that points to both the transfer and the later use. independent struct fields and constant array elements can be used when their projections prove they do not overlap. unknown indices and overlapping projections are not treated as disjoint. -after an owned local is released, assignment from another owner in the same declaring scope moves the replacement into that empty slot. the source and its aliases expire, the destination receives its cleanup responsibility, and loop paths still have to agree. this supports buffer-growth code such as `rt.free(buffer); buffer = replacement;` without a cast. initial declarations still cannot copy an owned local, overwriting a live owner remains an error, and captured owners are excluded from this replacement rule. +after an owned local is released, assignment from another owner declared in the same scope or a deeper one moves the replacement into that empty slot. the source and its aliases expire, the destination receives its cleanup responsibility, and loop paths still have to agree. this supports buffer-growth code such as `rt.free(buffer); buffer = replacement;` without a cast, including a replacement built inside a branch. initial declarations still cannot copy an owned local, overwriting a live owner remains an error, and captured owners are excluded from this replacement rule. + +## lends to globals + +`g = m*x;` lends an owned local to a global for the rest of the block: `x` keeps ownership and stays usable while code reached through `g` reads it, and assigning `g` again in the same block ends the lend. giving `x` away, returning, breaking out, or reaching the end of the block while the lend is open is an error, so a global never holds a borrow that outlives its owner. a restore inside a nested branch does not end a lend made outside it; the restore has to be unconditional. this is the idiom for pointing the output stream at a scratch buffer. an element borrow records its base array and known range. a constant index outside a known length gets a bounds diagnostic. mutations such as `push`, `pop`, and `resize` advance the array generation and invalidate element borrows; a later use reports the mutation and where the borrow was made. uncertain branch and loop facts are not promoted to definite errors. runtime checks remain for dynamic indices. @@ -41,7 +45,7 @@ the compiler functions, standard library, and language server currently run with the analysis runs with the existing parser, ownership checker, and lifetime records; it does not have a separate source parser or a complete path-sensitive proof. snapshots conservatively join branches and loops. low-level and unknown boundaries can hide behavior, so a quiet diagnostic set is not a whole-program safety proof. zc is single-threaded; shared closure cells do not define concurrent mutation rules. -two completeness limits block the compiler from compiling itself, and are written up in `docs/BOOTSTRAP-OWNERSHIP.md`. an owned value already held in a local has no move into another local; every accepted move originates at a call, so the free-then-replace rule above applies only when the replacement comes from a call result. and a local cannot be lent to a global, so an output stream has to be redirected at a module-level buffer rather than a scratch local. neither limit accepts an unsafe program; both refuse correct ones. +two completeness limits once stopped the compiler from compiling itself: an owner held in a local could not move into another local, and a local could not be lent to a global. the replacement and lend rules above close them; `docs/BOOTSTRAP-OWNERSHIP.md` keeps the record. the maintained regressions cover these cases: diff --git a/docs/ICON.md b/docs/ICON.md index 9c222c8..3f6ec8b 100644 --- a/docs/ICON.md +++ b/docs/ICON.md @@ -109,4 +109,4 @@ the status applies to the most recent transformation or binding operation. succe ## focused tests -the acceptance, rejection and regression fixtures live under `tests/icon`. run them with `python3 tests/drivers/icon_test.py bin/zy`; use `--js` for the javascript vm. select one fixture or category with `--case`, for example `python3 tests/drivers/icon_test.py bin/zym --case loop_step`. add `--source` to test the working library source before rebuilding the embedded compiler module; omit it when checking the built image. +the acceptance, rejection and regression fixtures live under `tests/icon`. run them with `python3 tests/drivers/icon_test.py build/bin/zy`; use `--js` for the javascript vm. select one fixture or category with `--case`, for example `python3 tests/drivers/icon_test.py build/bin/zym --case loop_step`. add `--source` to test the working library source before rebuilding the embedded compiler module; omit it when checking the built image. diff --git a/docs/design/memory.md b/docs/design/memory.md index c282df5..8875c72 100644 --- a/docs/design/memory.md +++ b/docs/design/memory.md @@ -5,12 +5,12 @@ the compiler, runtime, and host must agree on their addresses. `genmap` writes a matching `memmap.h`, runtime source, and compiler source; `src/tools/automap.sh` builds an image against those files. rebuild the host against the generated headers. changing a header alone does not relocate an existing genome. compact compiler and run-only maps also put their heap above the fixed regions, so compiler storage can grow after startup. ```sh -bin/genmap --preset default --check +build/bin/genmap --preset default --check bash src/tools/automap.sh small build/small -bin/genmap --preset default --code 65536 --symbols 256 --locals 64 --fwd 512 --pool 8192 --hash 1024 -o build/custom +build/bin/genmap --preset default --code 65536 --symbols 256 --locals 64 --fwd 512 --pool 8192 --hash 1024 -o build/custom ``` -run `bin/genmap --help` for the table, heap, genome, and stack options. sizes are in vm words unless an option says otherwise; one word is eight bytes. `--snap-cap` selects the initial snapshot reservation and `--quote-cap` selects an initial token count, with three words per token. both regions grow after startup. +run `build/bin/genmap --help` for the table, heap, genome, and stack options. sizes are in vm words unless an option says otherwise; one word is eight bytes. `--snap-cap` selects the initial snapshot reservation and `--quote-cap` selects an initial token count, with three words per token. both regions grow after startup. ## layout inventory @@ -79,8 +79,8 @@ the stub and board ports reserve `MICRO_PORT_WORDS` cells at build time; it defa the default native map places a growing heap above the fixed regions. hosted runners accept `TINYVM_MEMORY_WORDS` for initial memory and `TINYVM_MAX_MEMORY_WORDS` for its growth limit. javascript also accepts `memoryWords` and `maxMemoryWords` when constructing a vm. invalid configurations are rejected before execution. ```sh -TINYVM_MAX_MEMORY_WORDS=16777216 bin/zy < examples/hello.zc -TINYVM_MEMORY_WORDS=4194304 bin/zym < examples/hello.zc +TINYVM_MAX_MEMORY_WORDS=16777216 build/bin/zy < examples/hello.zc +TINYVM_MEMORY_WORDS=4194304 build/bin/zym < examples/hello.zc ``` the native core reserves a stable backing allocation up to its startup maximum; increasing the available size keeps addresses and native code pointers valid. the microcore and javascript vm replace backing allocations when they grow, preserving existing contents. native bounds use the reservation; portable bounds use the current allocation. memory syscall `9` with argument `4` requests growth, and argument `5` changes the growth limit during execution. a limit cannot fall below the current size. the native limit cannot exceed its startup reservation. diff --git a/docs/design/storage-analysis.md b/docs/design/storage-analysis.md index cff713a..432139a 100644 --- a/docs/design/storage-analysis.md +++ b/docs/design/storage-analysis.md @@ -30,4 +30,4 @@ facts are copied into each reference's editor record while the compiler parses i ## focused checks -`node tests/drivers/storage_analysis_test.js bin/zclsp` checks known bounds, packed elements, direct aliases, free and scope lifetimes, hover metadata, and safe or uncertain neighboring cases. +`node tests/drivers/storage_analysis_test.js build/bin/zclsp` checks known bounds, packed elements, direct aliases, free and scope lifetimes, hover metadata, and safe or uncertain neighboring cases. diff --git a/docs/design/test-coverage.md b/docs/design/test-coverage.md index fae9356..e916df4 100644 --- a/docs/design/test-coverage.md +++ b/docs/design/test-coverage.md @@ -35,7 +35,7 @@ the `qualifiers` group in `toolchain_lsp_contract_test.js` checks both qualifier ## public standard-library documentation -`src/lib/std.zc` documents public operations for output capture, numeric helpers, packed byte strings, host access, arenas, growable byte buffers, and array algorithms, including parameter and return behavior. the array algorithms document their callback roles, result types, traversal order, and reference sharing. `src/tools/embed.sh all --check` verifies that the compiler's embedded source matches the module. `hover_surface_test.js` and `hover_features_test.js` check selected embedded library documentation; they do not assert that every public function has a description. edge-case behavior for the broader standard-library API still needs dedicated acceptance tests. +`src/lib/std.zc` documents public operations for output capture, numeric helpers, packed byte strings, host access, arenas, growable byte buffers, and array algorithms, including parameter and return behavior. the array algorithms document their callback roles, result types, traversal order, and reference sharing. the compiler carries the module through an `embed` declaration, so its embedded text is the module file as of the build. `hover_surface_test.js` and `hover_features_test.js` check selected embedded library documentation; they do not assert that every public function has a description. edge-case behavior for the broader standard-library API still needs dedicated acceptance tests. ## unused-argument hints @@ -69,53 +69,53 @@ the formatter indents case labels one level inside their switch by default (`Cas ## focused commands -run from the repository root. native binaries and the language server must already be built; the wasm command needs a built `src/web/zc.wasm`. +run from the repository root. native binaries and the language server must already be built; the wasm command needs a built `build/web/zc.wasm` (`make web`). ```sh -python3 tests/drivers/fuzz_compiler.py bin/zy bin/zym --js -python3 tests/drivers/fuzz_malformed.py bin/zy bin/zym -node tests/drivers/fuzz_wasm.js src/web/zc.wasm -python3 tests/drivers/image_growth_test.py bin/zy -python3 tests/drivers/code_emission_growth_test.py bin/zy -python3 tests/drivers/code_address_growth_test.py bin/zy +python3 tests/drivers/fuzz_compiler.py build/bin/zy build/bin/zym --js +python3 tests/drivers/fuzz_malformed.py build/bin/zy build/bin/zym +node tests/drivers/fuzz_wasm.js build/web/zc.wasm +python3 tests/drivers/image_growth_test.py build/bin/zy +python3 tests/drivers/code_emission_growth_test.py build/bin/zy +python3 tests/drivers/code_address_growth_test.py build/bin/zy python3 tests/drivers/seed_growth_test.py python3 tests/drivers/memory_limits_test.py -python3 tests/drivers/stack_growth_test.py bin/zy +python3 tests/drivers/stack_growth_test.py build/bin/zy python3 tests/drivers/layout_config_test.py -python3 tests/drivers/compiler_growth_test.py bin/zy -python3 tests/drivers/parser_growth_test.py bin/zy -node tests/drivers/fuzz_lsp.js bin/zclsp -node tests/drivers/lsp_growth_test.js bin/zclsp -node tests/drivers/protocol_growth_test.js bin/zclsp -node tests/drivers/core_lsp_test.js bin/zclsp -node tests/drivers/hover_values_test.js bin/zclsp -node tests/drivers/hover_features_test.js bin/zclsp -node tests/drivers/hover_surface_test.js bin/zclsp -CONTRACT_GROUP=inference node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=qualifiers node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -python3 tests/drivers/compiler_contract_test.py bin/zy --group qualifiers -python3 tests/drivers/compiler_contract_test.py bin/zym --group qualifiers -python3 tests/drivers/inference_growth_test.py bin/zclsp -node tests/drivers/storage_analysis_test.js bin/zclsp -node tests/drivers/hints_test.js bin/zclsp -node tests/drivers/include_test.js bin/zclsp -CONTRACT_GROUP=quickfix node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=modulehover node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=namespacecompletion node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=navigation node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp +python3 tests/drivers/compiler_growth_test.py build/bin/zy +python3 tests/drivers/parser_growth_test.py build/bin/zy +node tests/drivers/fuzz_lsp.js build/bin/zclsp +node tests/drivers/lsp_growth_test.js build/bin/zclsp +node tests/drivers/protocol_growth_test.js build/bin/zclsp +node tests/drivers/core_lsp_test.js build/bin/zclsp +node tests/drivers/hover_values_test.js build/bin/zclsp +node tests/drivers/hover_features_test.js build/bin/zclsp +node tests/drivers/hover_surface_test.js build/bin/zclsp +CONTRACT_GROUP=inference node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=qualifiers node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +python3 tests/drivers/compiler_contract_test.py build/bin/zy --group qualifiers +python3 tests/drivers/compiler_contract_test.py build/bin/zym --group qualifiers +python3 tests/drivers/inference_growth_test.py build/bin/zclsp +node tests/drivers/storage_analysis_test.js build/bin/zclsp +node tests/drivers/hints_test.js build/bin/zclsp +node tests/drivers/include_test.js build/bin/zclsp +CONTRACT_GROUP=quickfix node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=modulehover node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=namespacecompletion node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=navigation node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp node tests/drivers/codeactions_client_test.js node tests/drivers/ext_cache_test.js -python3 tests/drivers/array_methods_test.py bin/zy -python3 tests/drivers/array_methods_test.py bin/zy --case map -python3 tests/drivers/array_methods_test.py bin/zy --case filter -python3 tests/drivers/array_methods_test.py bin/zy --case search -python3 tests/drivers/array_methods_test.py bin/zy --case reduce +python3 tests/drivers/array_methods_test.py build/bin/zy +python3 tests/drivers/array_methods_test.py build/bin/zy --case map +python3 tests/drivers/array_methods_test.py build/bin/zy --case filter +python3 tests/drivers/array_methods_test.py build/bin/zy --case search +python3 tests/drivers/array_methods_test.py build/bin/zy --case reduce python3 tests/drivers/cli_help_test.py python3 tests/drivers/runtime_image_test.py -CONTRACT_GROUP=magic node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=strings node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=arguments node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -CONTRACT_GROUP=modules node tests/drivers/toolchain_lsp_contract_test.js bin/zclsp -node tests/drivers/nav_test.js bin/zclsp -tests/runerr.sh bin/zy tests/errors +CONTRACT_GROUP=magic node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=strings node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=arguments node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +CONTRACT_GROUP=modules node tests/drivers/toolchain_lsp_contract_test.js build/bin/zclsp +node tests/drivers/nav_test.js build/bin/zclsp +tests/runerr.sh build/bin/zy tests/errors ``` diff --git a/editors/vscode/README.md b/editors/vscode/README.md index 5c3d0c7..24f27bb 100644 --- a/editors/vscode/README.md +++ b/editors/vscode/README.md @@ -29,4 +29,4 @@ the extension sends included source files to both bundled servers. relative path see `tests/drivers/lsp_test.js` for protocol tests, `tests/drivers/codeactions_client_test.js` for workspace-edit version handling, `tests/drivers/ext_cache_test.js` for the real editor provider and bundled server, and `src/tools/mkvsix.sh` for packaging. -the native language server also accepts `--help` or `-h`; its `--format`, `--annotate` and `--types` source modes remain available. use `bin/zclsp --help` for the stdin protocol and source-mode details. `python3 tests/drivers/cli_help_test.py` checks these command-line options along with the bundled command-line tools. +the native language server also accepts `--help` or `-h`; its `--format`, `--annotate` and `--types` source modes remain available. use `build/bin/zclsp --help` for the stdin protocol and source-mode details. `python3 tests/drivers/cli_help_test.py` checks these command-line options along with the bundled command-line tools. diff --git a/editors/vscode/syntaxes/zc.tmLanguage.json b/editors/vscode/syntaxes/zc.tmLanguage.json index 49f7b37..36f1c63 100644 --- a/editors/vscode/syntaxes/zc.tmLanguage.json +++ b/editors/vscode/syntaxes/zc.tmLanguage.json @@ -91,7 +91,7 @@ }, { "name": "keyword.other.zc", - "match": "\\b(as|unchecked|asm|quote|sizeof|typeof|len|cap|push|pop|resize|o3|include|memdebug|fixed|typecheck)\\b" + "match": "\\b(as|unchecked|asm|quote|sizeof|typeof|len|cap|push|pop|resize|o3|include|embed|memdebug|fixed|typecheck)\\b" }, { "name": "keyword.operator.memory.zc", diff --git a/src/compiler/compiler.zc b/src/compiler/compiler.zc index 244efc5..c6e1ca4 100644 --- a/src/compiler/compiler.zc +++ b/src/compiler/compiler.zc @@ -4737,7 +4737,7 @@ int kw_embed() { off = toff; n = name(); expect(s_semi); data = m*embed_open(path); - if (!data) { int sv = tpack(); tunpack(pp); serr("can't read the file named by embed"); tunpack(sv); data = chk_resize(0, 0, 1); } + if (!data) { embed_fail(pp, "can't read the file named by embed"); data = chk_resize(0, 0, 1); } sz = data[0]; if (!t) t = tbstr; // in a function the data sits in the code stream, so the code jumps over it @@ -4750,10 +4750,10 @@ int kw_embed() { } if (t[0] == TY_ARR) { w = embed_width(t[4]); - if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); if (skip >= 0) setlit(skip, here); return 0; } - if (sz % w) { serr("the embedded file's size is not a whole number of array elements"); if (skip >= 0) setlit(skip, here); return 0; } + if (!w) { embed_fail(pp, "embed declares a bstr, a char*, a number, or an array of numbers"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); } + if (sz % w) { embed_fail(pp, "the embedded file's size is not a whole number of array elements"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); } count = sz / w; - if (t[3] >= 0 && count > t[3]) { serr("the embedded file is larger than the array it is declared as"); if (skip >= 0) setlit(skip, here); return 0; } + if (t[3] >= 0 && count > t[3]) { embed_fail(pp, "the embedded file is larger than the array it is declared as"); if (skip >= 0) setlit(skip, here); return embed_bind(n, off, tword, 0, 2); } if (t[3] < 0) t = tarr(t[4], count); k = tsz(t); reserve_code(k); a = here; i = 0; if (t[4][0] == TY_CHAR) { while (i < k) { if (i < sz) emit(data[1 + i]); else emit(0); i++; } } @@ -4768,15 +4768,22 @@ int kw_embed() { return embed_bind(n, off, t, a, 1); } w = embed_width(t); - if (!w) { serr("embed declares a bstr, a char*, a number, or an array of numbers"); return 0; } - if (sz > w) { serr("the embedded file is larger than the type it is declared as"); return 0; } - if (isfl(t) && sz != w) { serr("an embedded float takes exactly as many bytes as its type"); return 0; } + if (!w) { embed_fail(pp, "embed declares a bstr, a char*, a number, or an array of numbers"); return embed_bind(n, off, tword, 0, 2); } + if (sz > w) { embed_fail(pp, "the embedded file is larger than the type it is declared as"); return embed_bind(n, off, tword, 0, 2); } + if (isfl(t) && sz != w) { embed_fail(pp, "an embedded float takes exactly as many bytes as its type"); return embed_bind(n, off, tword, 0, 2); } v = 0; i = 0; while (i < sz) { v = v | (data[1 + i] << (i * 8)); i++; } if (isfl(t) && w == 4) v = rt.s2f(v); else v = cnorm(t, v); return embed_bind(n, off, t, v, 2); } +// report msg at the embed declaration that starts at the token packed in pp. the +// caller still declares the name, as a plain 0, so its uses are not reported too. +int embed_fail(int pp, char* msg) { + int sv = tpack(); + tunpack(pp); serr(msg); tunpack(sv); + return 0; +} // the bytes one element of type t takes in an embedded file: the packed size of a // sized type or f32, 1 for char, 8 for the other number types, 0 when t is not a number. int embed_width(word t) { diff --git a/src/js/mklayout.js b/src/js/mklayout.js index cf95091..8e41f41 100644 --- a/src/js/mklayout.js +++ b/src/js/mklayout.js @@ -38,8 +38,8 @@ for (let i = 0; i < args.length; i += 2) { const mapPath = opt('--map') || path.join(root, 'build', 'core', 'map', 'memmap.h'); const outDir = opt('--out') || path.join(root, 'build', 'js'); -if (!fs.existsSync(mapPath)) { - console.error('mklayout: no memory map at ' + mapPath + '; run `make` first'); +if (!fs.existsSync(mapPath) || !fs.statSync(mapPath).isFile()) { + console.error('mklayout: no memory map header at ' + mapPath + '; run `make` first'); process.exit(1); } fs.mkdirSync(outDir, { recursive: true }); diff --git a/src/js/tinyvm.ts b/src/js/tinyvm.ts index 56fab70..f779487 100644 --- a/src/js/tinyvm.ts +++ b/src/js/tinyvm.ts @@ -135,6 +135,10 @@ export interface VMOptions { buffered suspends the program instead of reading the end. resume() goes on from there. eof() is true only once close() has been called. */ streaming?: boolean; + /** files: the file syscalls (open, read, write, close) reach the host's file + system through node's fs. off by default, so a vm inside the editor cannot + touch files; the command line turns it on, like the native vm. */ + files?: boolean; input?: string | Uint8Array; onOutput?: (byte: number) => void; onMessage?: (msg: string) => void; @@ -181,6 +185,7 @@ export class TinyVM { private enh: Enhancement[] = []; public L: Layout; private streaming: boolean = false; + private files: boolean = false; private closed: boolean = false; private blocked: boolean = false; private rp: number = 0; @@ -201,6 +206,7 @@ export class TinyVM { throw new RangeError('invalid initial memory or growth limit'); } this.streaming = !!options.streaming; + this.files = !!options.files && !!fs; const buffer = new ArrayBuffer(this.mw * 8); this.mem = new BigInt64Array(buffer); this.bytes = new Uint8Array(buffer); @@ -396,15 +402,33 @@ export class TinyVM { private syscall(op: bigint, a: bigint, b: bigint, c: bigint): bigint { const opNum = Number(op); switch (opNum) { - case 1: // open - return -1n; + case 1: { // open(path, 0 read or 1 create/truncate for write): path is one character per word + if (!this.files) return -1n; + let path = ''; + for (let at = Number(a); at >= 0 && at < this.mw && this.mem[at] !== 0n && path.length < 4096; at++) path += String.fromCharCode(Number(this.mem[at] & 0xFFn)); + try { return BigInt(fs.openSync(path, b === 1n ? 'w' : 'r')); } catch { return -1n; } + } case 2: // close + if (this.files && a > 2n) { try { fs.closeSync(Number(a)); } catch { return -1n; } } return 0n; - case 3: // read - return 0n; + case 3: { // read(fd, buf, n): bytes stored one per word + if (!this.files || a <= 2n) return 0n; + const n = Number(c), at = Number(b); + if (n <= 0 || at < 0 || at > this.mw - n) return -1n; + const chunk = Buffer.alloc(n); + let got: number; + try { got = fs.readSync(Number(a), chunk, 0, n, null); } catch { return -1n; } + for (let i = 0; i < got; i++) this.mem[at + i] = BigInt(chunk[i]); + return BigInt(got); + } case 4: { // write(fd, buf, n) const n = Number(c); const bufAddr = Number(b); + if (this.files && a > 2n) { + const chunk = Buffer.alloc(n); + for (let i = 0; i < n; i++) chunk[i] = Number(this.mem[bufAddr + i] & 0xFFn); + try { return BigInt(fs.writeSync(Number(a), chunk)); } catch { return -1n; } + } for (let i = 0; i < n; i++) { const ch = Number(this.mem[bufAddr + i] & 0xFFn); this.put_c(ch); @@ -974,6 +998,7 @@ if (typeof require === 'function' && typeof module !== 'undefined' && require.ma return Number(value); }; const vm = new TinyVM({ + files: true, memoryWords: environmentWords('TINYVM_MEMORY_WORDS'), maxMemoryWords: environmentWords('TINYVM_MAX_MEMORY_WORDS'), onOutput: (byte: number) => process.stdout.write(Buffer.from([byte])), diff --git a/src/lsp/server.zc b/src/lsp/server.zc index 48fdf90..b578bd1 100644 --- a/src/lsp/server.zc +++ b/src/lsp/server.zc @@ -1676,6 +1676,7 @@ char* kwtext() { if (hname_is("struct")) return "struct name { fields };~a record type. struct name<T> { } makes it generic, one copy for each set of type arguments."; if (hname_is("type")) return "type name = T;~an alias for a type, such as type row = int[3]; for a pointer to an array."; if (hname_is("include")) return "include \"path\"; / include std;~compiles a file or the embedded standard library at that point, its signatures first."; + if (hname_is("embed")) return "embed \"path\" as [type] name;~declares name with the bytes of the file at path, read when the unit is compiled. with no type, or bstr, name is a byte string; char* holds one character per word; a number type reads the bytes little-endian; T[] and T[n] lay them out as array elements."; if (hname_is("memdebug")) return "memdebug on; / memdebug off; / memdebug scrub;~header checks, allocation sites and leak reports for the rest of the file. scrub also fills a freed payload with 0xDEAD."; if (hname_is("typecheck")) return "typecheck on; / typecheck off;~turns on type checks for the code that follows, enums included."; if (hname_is("hints")) return "hints on; / hints small; / hints off;~controls subsequent hint diagnostics: informational ranges by default, compact markers with small, or no hints with off."; diff --git a/src/tools/zc0.c b/src/tools/zc0.c index 5f5b97d..b0ab29f 100644 --- a/src/tools/zc0.c +++ b/src/tools/zc0.c @@ -202,7 +202,7 @@ static void expect(const char *s) { if (!accept(s)) { char b[64]; sprintf(b, "ex static char *ident(void) { static char b[8][256]; static int k; char *r = b[k++ & 7]; if (tk != T_ID) die("expected a name"); strcpy(r, ts); next(); return r; } /* ---- symbols ---- */ -typedef struct { char name[64]; int kind, sig, defd; int64_t val; } Sym; +typedef struct { char name[64]; int kind, sig, defd, str; int64_t val; } Sym; /* str: a global holding a byte string */ static Sym sy[16384]; static int nsy; typedef struct { int at, sym; } Fix; static Fix fx[65536]; static int nfx; @@ -225,7 +225,7 @@ static Sym *add(const char *n, int kind, int64_t val) char full[256]; qualify(full, sizeof full, n); n = full; Sym *s = find_raw(n); if (s && s->kind == K_FN && kind == K_FN && !s->defd) { s->val = val; return s; } /* prototype or forward use */ - s = &sy[nsy++]; strncpy(s->name, n, 63); s->kind = kind; s->val = val; s->defd = 1; s->sig = 0; return s; + s = &sy[nsy++]; strncpy(s->name, n, 63); s->kind = kind; s->val = val; s->defd = 1; s->sig = 0; s->str = 0; return s; } static void fnref(const char *n) /* push a function address, fixing it up later if needed */ { @@ -488,7 +488,7 @@ static void primary(void) if ((l = findl(n))) { laddr(l->slot); lv = !l->arr; if (!l->arr) { xf = l->flt; xs = l->str; lvi = (int)(l - lo); } return; } s = find(n); if (s && s->kind == K_CONST) { lit(s->val); return; } - if (s && s->kind == K_GVAR) { lit(s->val); lv = 1; return; } + if (s && s->kind == K_GVAR) { lit(s->val); lv = 1; xs = s->str; return; } if (s && s->kind == K_GARR) { lit(s->val); return; } if (is("(")) { int na = args(); @@ -799,8 +799,11 @@ static void enhdef(void) /* embed "path" as [type] name; at the top level: a global holding the file's bytes, read when the image is built. the path is relative to the working directory. bstr (the default) packs the bytes; char* keeps one character to a word; a - number takes them little-endian; T[] and T[n] lay them out as the array's - elements. the same rules as the resident compiler's kw_embed. */ + number takes them little-endian; T[] and T[n] have one element per file chunk + of T's size. the checks are the resident compiler's (kw_embed). zc0 drops + types, so its arrays are one word per element whatever T is: an element is + stored as the value it reads as (sign extended for i8, i16 and i32, widened + for f32), and zc0 code indexing the array sees the same values. */ static int embed_width(const char *t, int star) { if (star) return 0; @@ -832,7 +835,7 @@ static void embeddecl(void) if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) die("can't read the file named by embed"); fclose(f); if (arr == -2 && (!ty[0] || (!strcmp(ty, "bstr") && !star))) { - a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; emit(len); + a = here; add(n, K_GVAR, a)->str = 1; emit(0); img[a - GORG] = here; emit(len); for (i = 0; i < len; i += 8) { uint64_t u = 0; for (w = 0; w < 8 && i + w < len; w++) u |= (uint64_t)buf[i + w] << (w * 8); emit((int64_t)u); } if (!(len & 7)) emit(0); } else if (arr == -2 && !strcmp(ty, "char") && star == 1) { @@ -857,12 +860,14 @@ static void embeddecl(void) if (arr >= 0 && count > arr) die("the embedded file is larger than the array it is declared as"); if (arr >= 0) count = arr; a = here; - if (!strcmp(ty, "char")) { for (i = 0; i < count; i++) emit(i < len ? buf[i] : 0); } - else { - long bytes = (long)count * w; - for (i = 0; i < bytes; i += 8) { uint64_t u = 0; for (int k = 0; k < 8 && i + k < bytes; k++) u |= (uint64_t)(i + k < len ? buf[i + k] : 0) << (k * 8); emit((int64_t)u); } - if (!bytes) emit(0); + for (i = 0; i < count; i++) { + uint64_t u = 0; int k; + for (k = 0; k < w && (long)i * w + k < len; k++) u |= (uint64_t)buf[(long)i * w + k] << (k * 8); + if (ty[0] == 'i' && w < 8 && (u >> (w * 8 - 1) & 1)) u |= ~(uint64_t)0 << (w * 8); + if (!strcmp(ty, "f32")) u = fop_s2f(u); + emit((int64_t)u); } + if (!count) emit(0); add(n, K_GARR, a); } free(buf); diff --git a/src/web/index.html b/src/web/index.html index 4cf7476..fbc203d 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -41,7 +41,7 @@ let machine = null; for (const n of examples) { const o = document.createElement('option'); o.value = n; o.textContent = n + '.zc'; $('ex').appendChild(o); } $('ex').onchange = async () => { - try { $('src').value = await (await fetch('../examples/' + $('ex').value + '.zc')).text(); } + try { $('src').value = await (await fetch('examples/' + $('ex').value + '.zc')).text(); } catch (e) { $('out').textContent = 'the examples folder is not served next to this page'; } }; $('run').onclick = async () => { @@ -55,6 +55,6 @@ } catch (e) { $('out').textContent = String(e); $('status').textContent = 'failed'; } }; ZC.load('zc.wasm').then(m => { machine = m; $('status').textContent = 'ready'; }) - .catch(e => { $('status').textContent = 'could not load zc.wasm; run src/web/build.sh and serve this folder over http'; }); + .catch(e => { $('status').textContent = 'could not load zc.wasm; run make web and serve build/web over http'; }); </script> </html> diff --git a/tests/data/embed/answer.bin b/tests/data/embed/answer.bin Binary files differnew file mode 100644 index 0000000..447f371 --- /dev/null +++ b/tests/data/embed/answer.bin diff --git a/tests/data/embed/bytes.bin b/tests/data/embed/bytes.bin new file mode 100644 index 0000000..8207d4f --- /dev/null +++ b/tests/data/embed/bytes.bin @@ -0,0 +1 @@ +ÿ
\ No newline at end of file diff --git a/tests/data/embed/nul.bin b/tests/data/embed/nul.bin Binary files differnew file mode 100644 index 0000000..20b5be9 --- /dev/null +++ b/tests/data/embed/nul.bin diff --git a/tests/data/embed/onehalf.f32 b/tests/data/embed/onehalf.f32 Binary files differnew file mode 100644 index 0000000..dfbda32 --- /dev/null +++ b/tests/data/embed/onehalf.f32 diff --git a/tests/data/embed/shorts.bin b/tests/data/embed/shorts.bin Binary files differnew file mode 100644 index 0000000..6007a59 --- /dev/null +++ b/tests/data/embed/shorts.bin diff --git a/tests/data/embed/text.txt b/tests/data/embed/text.txt new file mode 100644 index 0000000..9db7df0 --- /dev/null +++ b/tests/data/embed/text.txt @@ -0,0 +1,2 @@ +hello +world
\ No newline at end of file diff --git a/tests/drivers/fuzz_wasm.js b/tests/drivers/fuzz_wasm.js index 22fa7d5..1d256fe 100644 --- a/tests/drivers/fuzz_wasm.js +++ b/tests/drivers/fuzz_wasm.js @@ -13,22 +13,22 @@ function expr(depth) { function quote(s) { return '"' + s + '"'; } (async () => { - const lines = ['include std;', 'fn churn(n) { int i = 0; while (i < n) { { int* p = new(int); *p = i; } i++; } return 0; }', 'fn stable() { return 73; }', 'fn main() {', ' let base = live_objects();']; + const lines = ['namespace;', 'include rt, std;', 'fn churn(n) { int i = 0; while (i < n) { { int* p = rt.new(int); *p = i; } i++; } return 0; }', 'fn stable() { return 73; }', 'fn main() {', ' let base = rt.live_objects();']; const expected = []; for (let i = 0; i < 128; i++) { const [source, value] = expr(4); - lines.push(` put_n(${source}); put_c(32);`); expected.push(value); + lines.push(` rt.put_n(${source}); rt.put_c(32);`); expected.push(value); } for (let i = 0; i < 32; i++) { const a = Array.from({ length: next() % 11 }, () => 'abc'[next() % 3]).join(''); const b = Array.from({ length: next() % 6 }, () => 'abc'[next() % 3]).join(''); - lines.push(` put_n(Text.len(${quote(a)})); put_c(32); put_n(Text.eq(${quote(a)}, ${quote(b)})); put_c(32);`); - lines.push(` put_n(Text.starts(${quote(a)}, ${quote(b)})); put_c(32); put_n(Text.ends(${quote(a)}, ${quote(b)})); put_c(32); put_n(Text.find(${quote(a)}, ${quote(b)})); put_c(32);`); - lines.push(` put_n(Bytes.len(b${quote(a)})); put_c(32); put_n(Bytes.eq(b${quote(a)}, b${quote(b)})); put_c(32);`); - lines.push(` put_n(Bytes.starts(b${quote(a)}, b${quote(b)})); put_c(32); put_n(Bytes.ends(b${quote(a)}, b${quote(b)})); put_c(32); put_n(Bytes.find(b${quote(a)}, b${quote(b)})); put_c(32);`); + lines.push(` rt.put_n(std.Text.len(${quote(a)})); rt.put_c(32); rt.put_n(std.Text.eq(${quote(a)}, ${quote(b)})); rt.put_c(32);`); + lines.push(` rt.put_n(std.Text.starts(${quote(a)}, ${quote(b)})); rt.put_c(32); rt.put_n(std.Text.ends(${quote(a)}, ${quote(b)})); rt.put_c(32); rt.put_n(std.Text.find(${quote(a)}, ${quote(b)})); rt.put_c(32);`); + lines.push(` rt.put_n(std.Bytes.len(b${quote(a)})); rt.put_c(32); rt.put_n(std.Bytes.eq(b${quote(a)}, b${quote(b)})); rt.put_c(32);`); + lines.push(` rt.put_n(std.Bytes.starts(b${quote(a)}, b${quote(b)})); rt.put_c(32); rt.put_n(std.Bytes.ends(b${quote(a)}, b${quote(b)})); rt.put_c(32); rt.put_n(std.Bytes.find(b${quote(a)}, b${quote(b)})); rt.put_c(32);`); expected.push(a.length, Number(a === b), Number(a.startsWith(b)), Number(a.endsWith(b)), a.indexOf(b), a.length, Number(a === b), Number(a.startsWith(b)), Number(a.endsWith(b)), a.indexOf(b)); } - lines.push(' churn(300); gc(); put_n(live_objects() - base); put_c(32);', ' put_n(stable()); put_c(10);', ' return 0;', '}'); + lines.push(' churn(300); rt.gc(); rt.put_n(rt.live_objects() - base); rt.put_c(32);', ' rt.put_n(stable()); rt.put_c(10);', ' return 0;', '}'); expected.push(0, 73); const source = lines.join('\n') + '\n', want = expected.join(' ') + '\n'; const zc = await ZC.load(fs.readFileSync(process.argv[2] || 'build/web/zc.wasm')); diff --git a/tests/langerrors/embed_array_overflow.zc b/tests/langerrors/embed_array_overflow.zc new file mode 100644 index 0000000..eebfe1a --- /dev/null +++ b/tests/langerrors/embed_array_overflow.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: the embedded file is larger than the array it is declared as +embed "tests/data/embed/bytes.bin" as u8[2] x; +fn main() { return 0; } diff --git a/tests/langerrors/embed_missing.zc b/tests/langerrors/embed_missing.zc new file mode 100644 index 0000000..6144878 --- /dev/null +++ b/tests/langerrors/embed_missing.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: can't read the file named by embed +embed "tests/data/embed/missing.bin" as int x; +fn main() { return 0; } diff --git a/tests/langerrors/embed_missing_as.zc b/tests/langerrors/embed_missing_as.zc new file mode 100644 index 0000000..785d941 --- /dev/null +++ b/tests/langerrors/embed_missing_as.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: expected as: embed "file" as [type] name; +embed "tests/data/embed/text.txt" x; +fn main() { return 0; } diff --git a/tests/langerrors/embed_partial_element.zc b/tests/langerrors/embed_partial_element.zc new file mode 100644 index 0000000..663ca24 --- /dev/null +++ b/tests/langerrors/embed_partial_element.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: the embedded file's size is not a whole number of array elements +embed "tests/data/embed/bytes.bin" as u16[] x; +fn main() { return 0; } diff --git a/tests/langerrors/embed_struct.zc b/tests/langerrors/embed_struct.zc new file mode 100644 index 0000000..1fad589 --- /dev/null +++ b/tests/langerrors/embed_struct.zc @@ -0,0 +1,7 @@ +namespace; +include rt; + +// expect: embed declares a bstr, a char*, a number, or an array of numbers +struct point { int x, y; }; +embed "tests/data/embed/answer.bin" as point p; +fn main() { return 0; } diff --git a/tests/langerrors/embed_syntax.zc b/tests/langerrors/embed_syntax.zc new file mode 100644 index 0000000..0cd8b9e --- /dev/null +++ b/tests/langerrors/embed_syntax.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: embed takes a path in quotes: embed "file" as name; +embed text as x; +fn main() { return 0; } diff --git a/tests/langerrors/embed_too_large.zc b/tests/langerrors/embed_too_large.zc new file mode 100644 index 0000000..c8f3864 --- /dev/null +++ b/tests/langerrors/embed_too_large.zc @@ -0,0 +1,6 @@ +namespace; +include rt; + +// expect: the embedded file is larger than the type it is declared as +embed "tests/data/embed/text.txt" as int x; +fn main() { return 0; } diff --git a/tests/programs/embed.zc b/tests/programs/embed.zc new file mode 100644 index 0000000..d8d0ca3 --- /dev/null +++ b/tests/programs/embed.zc @@ -0,0 +1,31 @@ +namespace; +include rt; + +// embed declarations: file bytes become globals and locals of the declared type. +embed "tests/data/embed/text.txt" as text; +embed "tests/data/embed/answer.bin" as int answer; +embed "tests/data/embed/bytes.bin" as u8[] bytes; +embed "tests/data/embed/bytes.bin" as i8[8] signed; +embed "tests/data/embed/text.txt" as char* chars; +embed "tests/data/embed/onehalf.f32" as f32 half; +embed "tests/data/embed/bytes.bin" as char[] cs; +embed "tests/data/embed/nul.bin" as bstr nul; +embed "tests/data/embed/shorts.bin" as u16[] shorts; +fn main() { + rt.put_s(text); rt.put_c(32); + rt.put_n(len(text)); rt.put_c(32); + rt.put_n(answer); rt.put_c(32); + rt.put_n(len(bytes)); rt.put_c(32); rt.put_n(bytes[4]); rt.put_c(32); + rt.put_n(len(signed)); rt.put_c(32); rt.put_n(signed[4]); rt.put_c(32); rt.put_n(signed[7]); rt.put_c(32); + rt.put_n(chars[6]); rt.put_c(32); + rt.put_n((int)(half * 10.0)); rt.put_c(32); + rt.put_n(len(cs)); rt.put_c(32); rt.put_n(cs[4]); rt.put_c(32); + rt.put_n(len(nul)); rt.put_c(32); rt.put_n(nul[2]); rt.put_c(32); + rt.put_n(shorts[0] + shorts[1] * 10 + shorts[2] * 100); rt.put_c(32); + embed "tests/data/embed/answer.bin" as int local; + embed "tests/data/embed/bytes.bin" as u8[] copy; + embed "tests/data/embed/text.txt" as lt; + copy[0] = 9; + rt.put_n(local + copy[0] + bytes[0] + len(lt)); rt.put_c(10); + return 0; +} diff --git a/tests/programs/embed_top.zc b/tests/programs/embed_top.zc new file mode 100644 index 0000000..732566d --- /dev/null +++ b/tests/programs/embed_top.zc @@ -0,0 +1,21 @@ +namespace; +include rt; + +// embed declarations at the top level, the form stage 0 compiles into run-only images. +embed "tests/data/embed/text.txt" as text; +embed "tests/data/embed/answer.bin" as int answer; +embed "tests/data/embed/bytes.bin" as u8[] bytes; +embed "tests/data/embed/bytes.bin" as i8[8] signed; +embed "tests/data/embed/text.txt" as char* chars; +embed "tests/data/embed/bytes.bin" as char[] cs; +embed "tests/data/embed/shorts.bin" as u16[] shorts; +fn main() { + rt.put_s(text); rt.put_c(32); + rt.put_n(answer); rt.put_c(32); + rt.put_n(bytes[4]); rt.put_c(32); + rt.put_n(signed[4]); rt.put_c(32); rt.put_n(signed[7]); rt.put_c(32); + rt.put_n(chars[6]); rt.put_c(32); + rt.put_n(cs[4]); rt.put_c(32); + rt.put_n(shorts[0] + shorts[1] * 10 + shorts[2] * 100); rt.put_c(10); + return 0; +} diff --git a/tests/test.sh b/tests/test.sh index fd2bcd3..811a09a 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -131,6 +131,7 @@ run(){ # $1 = vm command, $2 = label chk "$($vm < tests/programs/o3push.zc | md5sum)" "$(md5sum < tests/programs/o3push.expected)" "$l push and pop of a growable array are inline in o3" chk "$($vm < tests/programs/o3copy.zc)" "520507 1005" "$l o3 copies of a loop variable read the source instead" chk "$($vm < tests/programs/borrowmove.zc)" "6" "$l storing into a borrowed object moves it" + chk "$($vm < tests/programs/embed.zc | tr '\n' '/')" "hello/world 11 42 5 255 8 -1 0 119 15 5 255 3 98 321 63/" "$l embed declarations: strings, numbers and arrays from files, global and local" chk "$($vm < tests/programs/ownmove.zc)" "7 50 0 1" "$l an owned local moves into a released owner, and a global borrows a local until it is restored" chk "$($vm < tests/programs/alias.zc)" "11 22 4 8" "$l aliases of a live owner: arithmetic, field, index" chk "$($vm < tests/programs/quarantine.zc)" "0 1234" "$l quarantine: stale alias reads intact data, second free is a no-op" @@ -248,7 +249,7 @@ s_micro() { python3 tests/drivers/actor_test.py $T/zym > $T/actor.out 2>&1; r=$? grep -v '^ok' $T/actor.out chk $r 0 "micro isolated actor acceptance, rejection and regression fixtures" - if command -v node >/dev/null && command -v tsc >/dev/null && node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null \ + if command -v node >/dev/null && command -v tsc >/dev/null && node src/js/mklayout.js --map $MAP/memmap.h --out $T/jsvm >/dev/null \ && cp $ROOT/s1/genome.bin $T/jsvm/ && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm; then python3 tests/drivers/fuzz_compiler.py $ROOT/zy $T/zym --js $T/jsvm/tinyvm.js > $T/fuzz.out 2>&1; r=$? else @@ -350,6 +351,7 @@ s_runonly() { runonly strings tests/programs/strings.zc runonly strings-ex examples/strings.zc runonly share tests/programs/share.zc + runonly embed tests/programs/embed_top.zc d=$T/run-echo; GENMAP=$T/genmap bash src/tools/automap.sh run $d src/runtime/run.zc examples/echo.zc >/dev/null && cp src/micro/micro.c src/micro/micro_port.h src/micro/port_linux.c src/micro/port_stub.c src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h $d/ \ && $CC -pthread -DZY_THREADS -O2 -w -fwrapv $d/micro.c $d/port_linux.c -o $T/zymr \ && chk "$(printf 'abc\n' | $T/zymr | tr '\n' '/')" "$( (cat examples/echo.zc; printf '`abc\n') | $T/zy | tr '\n' '/')" "run echo" \ @@ -590,7 +592,7 @@ s_lsp() { else echo "skip lsp wasm tests (no clang with wasm32)"; fi # the same server run by the javascript vm, the extension's fallback when wasm is not there. # the vm starts at the image's heap and grows on the heap's request, so a long document is the test - node src/js/extract_lsp.js --lsp $T/lsp --out $T/vmserver >/dev/null && node src/js/mklayout.js --map $MAP --out $T/vmserver >/dev/null \ + node src/js/extract_lsp.js --lsp $T/lsp --out $T/vmserver >/dev/null && node src/js/mklayout.js --map $MAP/memmap.h --out $T/vmserver >/dev/null \ && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/vmserver node tests/drivers/hover_surface_test.js --vm $T/vmserver > $T/hoversurfacevm.out 2>&1; r=$? grep -v '^ok' $T/hoversurfacevm.out @@ -621,13 +623,14 @@ s_js() { # the javascript vm, and the microcore as webassembly. both run the same genome if command -v node >/dev/null && command -v tsc >/dev/null; then # the vm as make js builds it, with the fixpoint genome beside it - node src/js/mklayout.js --map $MAP --out $T/jsvm >/dev/null && cp $ROOT/s1/genome.bin $T/jsvm/genome.bin \ + node src/js/mklayout.js --map $MAP/memmap.h --out $T/jsvm >/dev/null && cp $ROOT/s1/genome.bin $T/jsvm/genome.bin \ && tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir $T/jsvm || chk 0 1 "js vm builds" chk "$(node $T/jsvm/tinyvm.js examples/hello.zc)" "hello, world!" "js vm: hello" chk "$(node $T/jsvm/tinyvm.js examples/fact.zc | tail -1)" "3628800" "js vm: factorial" chk "$(node $T/jsvm/tinyvm.js examples/primes.zc | tail -1)" "widest gap: 72, after 31397" "js vm: primes" chk "$(node $T/jsvm/tinyvm.js tests/programs/growone.zc)" "12 1" "js vm: a block past the initial heap grows the machine" chk "$(node $T/jsvm/tinyvm.js tests/programs/replace_owner.zc)" "$(cat tests/programs/replace_owner.expected)" "js vm: replacement ownership and cleanup" + chk "$(node $T/jsvm/tinyvm.js tests/programs/embed.zc | tr '\n' '/')" "hello/world 11 42 5 255 8 -1 0 119 15 5 255 3 98 321 63/" "js vm: embed declarations read their files through node" if sh src/tools/findcc.sh --wasm >/dev/null 2>&1 && command -v wasm-ld >/dev/null; then GENOME_DIR=$ROOT/s1 MAP_DIR=$MAP bash src/web/build.sh $T/zc.wasm >/dev/null && node tests/drivers/wasm_test.js $T/zc.wasm > $T/wasm.out 2>&1; r=$? grep -v '^ok' $T/wasm.out |