diff options
| author | 2026-10-14 07:28:39 +0000 | |
|---|---|---|
| committer | 2026-10-14 07:28:39 +0000 | |
| commit | 7451eb090d890fb5ea5e4d40627a1b4df15a49de (patch) | |
| tree | 357bdf4b528e860bffa688865306f656b788d2c2 /src/tools | |
| parent | analyzer: close the ownership gaps that stopped the compiler compiling itself (diff) | |
build: embed images with #embed, keep everything generated in build/
the cores take their genome in with c23 #embed through src/core/genome.h,
so images are plain little-endian word files (genome.bin) instead of
generated c headers. zc0 writes that format, mkgenome checks an image
against its map and shows a compiler's diagnostics when it gets them,
and the javascript vm and the editor's vm host read the same files.
src/tools/findcc.sh picks a compiler with #embed (clang 19 or gcc 15).
zc gets an embed declaration, embed "file" as [type] name, for globals
and locals: a byte string by default, char*, numbers read little-endian,
and arrays of numbers. the compiler and language server carry their
library and source texts through it, so embed.sh and the text it
spliced into compiler.zc are gone.
make writes executables to build/bin, the default map to build/core/map,
the web page to build/web and the extension staging to build/vsix, and
the genome now rebuilds when its sources change. the test suite lays
out its own map and bootstrap and reads nothing from build/.
also: rt.put_s of a byte string in zc0 images, retained function
sources no longer count as live objects, a missing include no longer
cascades into undefined-name errors, consecutive comment lines join
into one hover paragraph, and stale test fixtures follow the include
format.
Diffstat (limited to 'src/tools')
| -rwxr-xr-x | src/tools/automap.sh | 32 | ||||
| -rwxr-xr-x | src/tools/bootstrap-check.sh | 30 | ||||
| -rwxr-xr-x | src/tools/embed.sh | 89 | ||||
| -rwxr-xr-x | src/tools/findcc.sh | 33 | ||||
| -rw-r--r-- | src/tools/genmap.c | 10 | ||||
| -rw-r--r-- | src/tools/mkgenome.c | 50 | ||||
| -rwxr-xr-x | src/tools/mklsp.sh | 22 | ||||
| -rwxr-xr-x | src/tools/mklspwasm.sh | 11 | ||||
| -rwxr-xr-x | src/tools/mkvsix.sh | 35 | ||||
| -rw-r--r-- | src/tools/zc0.c | 97 | ||||
| -rwxr-xr-x | src/tools/zcc | 8 |
11 files changed, 232 insertions, 185 deletions
diff --git a/src/tools/automap.sh b/src/tools/automap.sh index 69849ca..ff559ac 100755 --- a/src/tools/automap.sh +++ b/src/tools/automap.sh @@ -3,7 +3,8 @@ # src/tools/automap.sh PRESET OUTDIR FILE... # PRESET is run, small, default or lsp. DOCS=1 makes the genome with its doc comments. FILE... are appended after the map's own # rt.zc (and compiler.zc, for the presets that carry the compiler). the result -# is OUTDIR/memmap.h and OUTDIR/genome.h, and they agree with each other. +# is OUTDIR/memmap.h and OUTDIR/genome.bin, and they agree with each other. a core +# compiled with -IOUTDIR --embed-dir=OUTDIR embeds that image (src/core/genome.h). # # a genome's length doesn't depend on where it lives (every address is a one # word literal), so two passes are enough: compile once on the preset's map @@ -14,15 +15,14 @@ # GENMAP can name a genmap that is already built. set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' 'builds a genome and memory map sized to the supplied source files.' 'output: OUTDIR/genome.h and OUTDIR/memmap.h' 'environment: CC selects the c compiler; DOCS=1 retains documentation; GENMAP selects genmap.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' 'builds a genome and memory map sized to the supplied source files.' 'output: OUTDIR/genome.bin and OUTDIR/memmap.h' 'environment: CC selects the c compiler; DOCS=1 retains documentation; GENMAP selects genmap.'; exit 0 ;; -*) echo "automap: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." [[ $# -ge 2 ]] || { echo 'usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE...' >&2; exit 2; } P=$1; O=$2; shift 2 [ -n "$P" ] && [ -n "$O" ] || { echo "usage: src/tools/automap.sh run|small|default|lsp OUTDIR FILE..." >&2; exit 2; } -CC=${CC:-gcc} -bash src/tools/embed.sh all --check || bash src/tools/embed.sh all +CC=${CC:-$(sh src/tools/findcc.sh)} mkdir -p "$O" ZC0=$O/.zc0 if [ -z "$GENMAP" ]; then @@ -33,32 +33,36 @@ lay() { # $1 = genome slot, or empty for the preset's own rm -rf "$O/map" "$GENMAP" --preset "$P" --grow ${1:+--genome $1} -o "$O/map" >/dev/null 2>&1 $CC -O1 -w -fwrapv src/tools/zc0.c -I"$O/map" -Isrc/core -o "$ZC0" + $CC -O1 -w -fwrapv src/tools/mkgenome.c -I"$O/map" -o "$O/.mkgenome" } build() { - if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$ZC0" + if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$ZC0" > "$O/image.bin" + "$O/.mkgenome" < "$O/image.bin" > "$O/genome.bin" } # with DOCS=1 the genome is made again with zc0 built for docs (ZC0DOCS), so the # image carries the doc comments of the sources and the program can answer # docofs(). the image is the same code either way; the docs are stored after it. docs() { $CC -O1 -w -fwrapv -DZC0DOCS src/tools/zc0.c -I"$O/map" -Isrc/core -o "$O/.zc0docs" - if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$O/.zc0docs" + if [ -f "$O/map/compiler.zc" ]; then cat "$O/map/rt.zc" src/core/core.zc "$O/map/compiler.zc" "$@"; else cat "$O/map/rt.zc" src/core/core.zc "$@"; fi | "$O/.zc0docs" > "$O/image.bin" + "$O/.mkgenome" < "$O/image.bin" > "$O/genome.bin" } -len() { grep -m1 '#define GLEN' "$1" | awk '{print $3}'; } +len() { echo $(( $(wc -c < "$1") / 8 )); } lay "" -build "$@" > "$O/genome.h" -glen=$(len "$O/genome.h") +build "$@" +glen=$(len "$O/genome.bin") slot=$(( (glen + 4095) / 4096 * 4096 )) lay "$slot" -build "$@" > "$O/genome.h" +build "$@" if [ -n "$DOCS" ]; then lay $((slot + 65536)) # room for the image with its docs while it is being made - docs "$@" > "$O/genome.h" - glen=$(len "$O/genome.h") + docs "$@" + glen=$(len "$O/genome.bin") slot=$(( (glen + 4095) / 4096 * 4096 )) lay "$slot" - docs "$@" > "$O/genome.h" + docs "$@" fi -[ "$(len "$O/genome.h")" = "$glen" ] || { echo "automap: the genome changed length with the map" >&2; exit 1; } +rm -f "$O/image.bin" +[ "$(len "$O/genome.bin")" = "$glen" ] || { echo "automap: the genome changed length with the map" >&2; exit 1; } cp "$O/map/memmap.h" "$O/map/map.txt" "$O/" echo "automap $P: genome $glen words in a slot of $slot, $(grep -m1 '^MW' "$O/map/map.txt")" diff --git a/src/tools/bootstrap-check.sh b/src/tools/bootstrap-check.sh index a5de341..7acd74c 100755 --- a/src/tools/bootstrap-check.sh +++ b/src/tools/bootstrap-check.sh @@ -2,19 +2,23 @@ # rebuild the bootstrap one stage and report what the compiler says about itself. # zc0 (stage 0) compiles rt+core+compiler into build/s0, s0 becomes zy0, and zy0 # runs build.zc over the same sources to produce the image. ownership diagnostics -# come back in the image, so print them instead of the word dump. +# come back in place of the image; mkgenome rejects them and prints them. set -e +case "${1:-}" in + --help|-h) printf '%s\n' 'usage: src/tools/bootstrap-check.sh' 'runs the first two bootstrap stages and prints the compiler diagnostics on its own source, or the image size.'; exit 0 ;; + -*) echo "bootstrap-check: unknown option $1; use --help for usage" >&2; exit 2 ;; +esac cd "$(dirname "$0")/../.." -mkdir -p bin build/s0 build/s1 build/core -cc -O1 -w -fwrapv src/tools/zc0.c -Isrc/core -o bin/zc0 -cc -O1 -w -fwrapv src/tools/mkgenome.c -Isrc/core -o bin/mkgenome -cp src/core/core.c src/core/io_linux.c src/core/memmap.h src/core/fops.h src/core/bops.h src/core/stack.h build/s0/ -cat src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc | bin/zc0 > build/s0/genome.h -cc -O2 -w -fwrapv -pthread -DJTYPE=int build/s0/core.c build/s0/io_linux.c -o build/zy0 -(cat src/tools/build.zc; printf '`'; cat src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc) | build/zy0 > build/g1.bin -if ! grep -q '^zc: ' build/g1.bin 2>/dev/null; then - echo "image ok: $(wc -c < build/g1.bin) bytes" +make -s build/bin/zc0 build/bin/mkgenome +CC=${CC:-$(sh src/tools/findcc.sh)} +SRC="src/runtime/rt.zc src/core/core.zc src/compiler/compiler.zc" +mkdir -p build/s0 build/s1 +cat $SRC | build/bin/zc0 > build/s0/g0.bin +build/bin/mkgenome < build/s0/g0.bin > build/s0/genome.bin +$CC -O2 -w -fwrapv -pthread -DJTYPE=int src/core/core.c src/core/io_linux.c -Isrc/core -Ibuild/core/map --embed-dir=build/s0 -o build/s0/zy0 +(cat src/tools/build.zc; printf '`'; cat $SRC) | build/s0/zy0 > build/s1/g1.bin +if build/bin/mkgenome < build/s1/g1.bin > build/s1/check.bin; then + echo "image ok: $(wc -c < build/s1/g1.bin) bytes" else - echo "--- compiler diagnostics on its own source ---" - cat -v build/g1.bin -fi
\ No newline at end of file + exit 1 +fi diff --git a/src/tools/embed.sh b/src/tools/embed.sh deleted file mode 100755 index 59a377f..0000000 --- a/src/tools/embed.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# embeds source text in the compiler, so the editor can show a built-in's definition and `include std;` needs -# no file. a text becomes a function that returns it in pieces of at most 3000 bytes (the bootstrap -# compiler's token buffer is 4 KB), one byte string per piece: -# bstr NAME_part(int i) { if (i == 0) return b"..."; if (i == 1) return b"..."; ... return 0; } -# -# src/tools/embed.sh all embeds src/lib modules in the compiler -# src/tools/embed.sh all --check exits 1 when an embedded module is out of date -# src/tools/embed.sh lsp OUT writes OUT: rt_part, compiler_part and server_part, for the language server's image -set -e -case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/embed.sh std|icon|all [--check]' ' src/tools/embed.sh lsp OUT' 'embeds library sources in the compiler, or writes the lsp source providers to OUT.' 'options:' ' --check report stale embedded library sources without changing files' ' --help, -h show this help and exit'; exit 0 ;; - -*) echo "embed: unknown option $1; use --help for usage" >&2; exit 2 ;; -esac -cd "$(dirname "$0")/../.." -export LC_ALL=C -# parts NAME FILE: the NAME_part function for FILE -parts() { - echo "bstr $1_part(int i) {" - awk -v LIMIT=3000 ' - function flush() { if (cnt > 0) { printf("\tif (i == %d) return b\"%s\";\n", n, buf); n++; buf = ""; cnt = 0 } } - { - line = $0; sub(/\r$/, "", line); line = line "\n" - len = length(line) - for (j = 1; j <= len; j++) { - if (cnt >= LIMIT) flush() - c = substr(line, j, 1) - if (c == "\\") e = "\\\\"; else if (c == "\"") e = "\\\""; else if (c == "\t") e = "\\t"; else if (c == "\n") e = "\\n"; else e = c - buf = buf e; cnt++ - } - } - END { flush() }' "$2" - echo " return 0;" - echo "}" -} -embed() { - name=$1; file=$2; check=$3; C=src/compiler/compiler.zc; T=$(mktemp -d) - BEGIN_MARK="// ---- embedded $name library (generated by src/tools/embed.sh from $file; do not edit) ----" - END_MARK="// ---- end embedded $name library ----" - if grep -qF "$BEGIN_MARK" "$C"; then - awk -v b="$BEGIN_MARK" '$0 == b { exit } { print }' "$C" > "$T/head" - awk -v e="$END_MARK" 'found { print } $0 == e { found = 1; next }' "$C" > "$T/tail" - else - cp "$C" "$T/head" - : > "$T/tail" - fi - { - cat "$T/head" - echo "$BEGIN_MARK" - parts "$name" "$file" - echo "$END_MARK" - cat "$T/tail" - } > "$T/new" - if [ "$check" = "--check" ]; then - cmp -s "$T/new" "$C" && result=0 || result=1 - rm -rf "$T" - return "$result" - fi - cp "$T/new" "$C" - rm -rf "$T" -} -case "$1" in -std) - embed std src/lib/std.zc "$2" - ;; -icon) - embed icon src/lib/icon.zc "$2" - ;; -all) - if [ "$2" = "--check" ]; then - embed std src/lib/std.zc --check && embed icon src/lib/icon.zc --check - else - embed std src/lib/std.zc - embed icon src/lib/icon.zc - fi - ;; -lsp) - [ -n "$2" ] || { echo "usage: src/tools/embed.sh lsp OUT" >&2; exit 2; } - { - echo "// the sources of this image, embedded (generated by src/tools/embed.sh; do not edit)" - parts rt src/runtime/rt.zc - parts compiler src/compiler/compiler.zc - parts server src/lsp/server.zc - parts core src/core/core.zc - } > "$2" - ;; -*) - echo "usage: src/tools/embed.sh all [--check] | std | icon | lsp OUT" >&2; exit 2;; -esac diff --git a/src/tools/findcc.sh b/src/tools/findcc.sh new file mode 100755 index 0000000..caa3af3 --- /dev/null +++ b/src/tools/findcc.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# prints the first c compiler that accepts c23 #embed, which every core needs to +# take in its genome (src/core/genome.h). clang 19 and gcc 15 are the first +# releases with it, and both find #embed files through --embed-dir. a CC +# already in the environment is tried first. +# src/tools/findcc.sh a native compiler +# src/tools/findcc.sh --wasm a clang that also targets wasm32 +# exits 1, naming what was tried, when nothing fits. +case "${1:-}" in + --help|-h) printf '%s\n' 'usage: src/tools/findcc.sh [--wasm]' 'prints a c compiler that supports #embed; --wasm also requires the wasm32 target.' 'environment: CC is tried first.'; exit 0 ;; + --wasm|'') ;; + *) echo "findcc: unknown option $1; use --help for usage" >&2; exit 2 ;; +esac +wasm=${1:-} +dir=$(mktemp -d "${TMPDIR:-/tmp}/findcc.XXXXXX") || exit 1 +trap 'rm -rf "$dir"' EXIT +mkdir "$dir/data" +printf 'ab' > "$dir/data/probe.bin" +printf 'static const unsigned char probe[] = {\n#embed "probe.bin"\n};\nint main(void) { return sizeof probe == 2 ? 0 : 1; }\n' > "$dir/probe.c" +tried="" +for cc in ${CC:-} cc clang gcc clang-22 clang-21 clang-20 clang-19 gcc-16 gcc-15 /opt/homebrew/opt/llvm/bin/clang /usr/local/opt/llvm/bin/clang; do + command -v "$cc" >/dev/null 2>&1 || continue + tried="$tried $cc" + if [ -n "$wasm" ]; then + "$cc" --target=wasm32 -nostdlib -w --embed-dir="$dir/data" -c "$dir/probe.c" -o "$dir/probe.o" >/dev/null 2>&1 || continue + else + "$cc" -w --embed-dir="$dir/data" "$dir/probe.c" -o "$dir/probe" >/dev/null 2>&1 || continue + fi + echo "$cc" + exit 0 +done +echo "findcc: no c compiler with #embed${wasm:+ and wasm32} (tried:${tried:- nothing}); install clang 19 or gcc 15, or set CC" >&2 +exit 1 diff --git a/src/tools/genmap.c b/src/tools/genmap.c index 8a258e1..d063d36 100644 --- a/src/tools/genmap.c +++ b/src/tools/genmap.c @@ -609,9 +609,11 @@ static void expect_hex(const char *text, const char *where, const char *name, in static void check_sources(const Map *m) { char lfa_old[64]; + /* a memmap.h in the working directory is compared too, when there is one: the + default map's header is generated into build/core and never kept in the tree */ FILE *mf = fopen("memmap.h", "rb"); - const char *mm_path = mf ? (fclose(mf), "memmap.h") : "src/core/memmap.h"; - char *rt = slurp("src/runtime/rt.zc"), *cc = slurp("src/compiler/compiler.zc"), *mm = slurp(mm_path); + const char *mm_path = mf ? (fclose(mf), "memmap.h") : 0; + char *rt = slurp("src/runtime/rt.zc"), *cc = slurp("src/compiler/compiler.zc"), *mm = mm_path ? slurp(mm_path) : 0; char *rtb = between(rt, "// @map-rt", "// @end-map"); char *ccb = between(cc, "// @map-cc", "// @end-map"); static const char *rt_names[] = { @@ -644,7 +646,7 @@ static void check_sources(const Map *m) if (!strstr(rt, lfa_old)) die("src/runtime/rt.zc no longer has the default lfa immediate"); if (!strstr(cc, "ZFP - 16384")) die("src/compiler/compiler.zc lost its initial type-area formula"); for (i = 0; rt_names[i]; i++) expect_hex(rtb, "src/runtime/rt.zc", rt_names[i], rt_vals[i]); - for (i = 0; rt_names[i]; i++) { + for (i = 0; mm && rt_names[i]; i++) { if (!strcmp(rt_names[i], "MAGIC") || !strcmp(rt_names[i], "HEAP_GROW")) continue; expect_hex(mm, mm_path, !strcmp(rt_names[i], "MW") ? "PW" : rt_names[i], rt_vals[i]); } @@ -700,7 +702,7 @@ static void usage_status(int status) "--preset default has a growing heap above the stacks; fixed is the old 1,081,344-word map.\n" "--grow lets a map with a fixed size get bigger when its genome needs more room.\n" "--mode packs a compiler or a run-only image; spare words go to the data stack.\n" - "--check compares that layout to memmap.h and the checked-in const blocks.\n"); + "--check compares that layout to the checked-in const blocks (and to ./memmap.h if present).\n"); exit(status); } diff --git a/src/tools/mkgenome.c b/src/tools/mkgenome.c index 3bf1533..9720760 100644 --- a/src/tools/mkgenome.c +++ b/src/tools/mkgenome.c @@ -1,7 +1,12 @@ -/* turns a genome image (little-endian 64-bit words on stdin, as written by - src/tools/build.zc) into genome.h */ +/* checks a genome image (little-endian 64-bit words on stdin, as written by + zc0 or src/tools/build.zc) against the memory map it was built with, and + copies it to stdout unchanged. the cores #embed the result (src/core/genome.h), + so this is the gate that keeps a compiler's diagnostics out of a core: when + the input is not an image for this map, nothing is written and the input is + shown on stderr, since it is usually the compiler's error report. */ #include <stdio.h> #include <stdint.h> +#include <stdlib.h> #include <string.h> #ifdef _WIN32 #include <fcntl.h> @@ -11,8 +16,9 @@ int main(int argc, char **argv) { if (argc == 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h"))) { - fputs("usage: mkgenome [--help|-h] < image.bin > genome.h\n" - "converts little-endian 64-bit image words into a c genome header.\n" + fputs("usage: mkgenome [--help|-h] < image.bin > genome.bin\n" + "checks little-endian 64-bit image words against this memory map and copies them\n" + "to stdout. input that is not an image is reported on stderr and nothing is written.\n" "the image must match the memory layout selected when mkgenome is built.\n" "options:\n --help, -h show this help and exit\n", stdout); return 0; @@ -22,19 +28,29 @@ int main(int argc, char **argv) _setmode(_fileno(stdin), _O_BINARY); _setmode(_fileno(stdout), _O_BINARY); #endif - unsigned char b[8]; int n = 0; size_t got; - printf("#define GORG %d\nstatic const int64_t genome[] = {", GORG); - while ((got = fread(b, 1, 8, stdin)) == 8) { - uint64_t u = (uint64_t)b[0] | (uint64_t)b[1] << 8 | (uint64_t)b[2] << 16 | (uint64_t)b[3] << 24 - | (uint64_t)b[4] << 32 | (uint64_t)b[5] << 40 | (uint64_t)b[6] << 48 | (uint64_t)b[7] << 56; - if ((n == 0 && u != (uint64_t)FTOP * 2 + 1) || - (n == 1 && u != (uint64_t)ZFP * 2 + 1) || (n == 2 && u != 4)) { - fprintf(stderr, "mkgenome: input is not a genome for this map\n"); return 1; + size_t n = 0, cap = 1 << 20, got; + unsigned char *b = malloc(cap); + if (!b) { fputs("mkgenome: out of memory\n", stderr); return 1; } + while ((got = fread(b + n, 1, cap - n, stdin)) > 0) { + n += got; + if (n == cap) { + unsigned char *next = cap > ((size_t)1 << 30) ? 0 : realloc(b, cap * 2); + if (!next) { fputs("mkgenome: image too large\n", stderr); return 1; } + b = next; cap *= 2; } - printf("%s%lld", !n ? "\n" : n % 12 ? "," : ",\n", (long long)u); - n++; } - if (got || ferror(stdin) || n < 7) { fprintf(stderr, "mkgenome: incomplete genome\n"); return 1; } - printf("\n};\n#define GLEN %d\n", n); - return n ? 0 : 1; + if (ferror(stdin)) { fputs("mkgenome: read failed\n", stderr); return 1; } + int ok = n >= 8 * 7 && n % 8 == 0; + for (int i = 0; ok && i < 3; i++) { + uint64_t u = 0; + for (int k = 0; k < 8; k++) u |= (uint64_t)b[i * 8 + k] << (k * 8); + if (u != (i == 0 ? (uint64_t)FTOP * 2 + 1 : i == 1 ? (uint64_t)ZFP * 2 + 1 : 4)) ok = 0; + } + if (!ok) { + if (n >= 4 && !memcmp(b, "zc: ", 4)) fwrite(b, 1, n, stderr); + fprintf(stderr, n % 8 || n < 8 * 7 ? "mkgenome: incomplete genome\n" : "mkgenome: input is not a genome for this map\n"); + return 1; + } + if (fwrite(b, 1, n, stdout) != n || fflush(stdout)) { fputs("mkgenome: write failed\n", stderr); return 1; } + return 0; } diff --git a/src/tools/mklsp.sh b/src/tools/mklsp.sh index 9bab550..1c44998 100755 --- a/src/tools/mklsp.sh +++ b/src/tools/mklsp.sh @@ -2,21 +2,19 @@ # builds the language server: src/tools/mklsp.sh [outdir] leaves outdir/zclsp. # the server is the real compiler (src/compiler/compiler.zc) in check mode plus # src/lsp/server.zc. its memory map is sized to that program by src/tools/automap.sh, -# so the genome slot is as big as it needs to be and no bigger. +# so the genome slot is as big as it needs to be and no bigger. the image carries +# the doc comments (DOCS=1) and, through embed declarations in server.zc, its own +# sources, so hover and go to definition work on built-ins. the core takes the +# image in with #embed as lsp_genome.bin. set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/mklsp.sh [OUTDIR]' 'builds the native language server in OUTDIR/zclsp; defaults to build/lsp.' 'environment: CC selects the c compiler.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/mklsp.sh [OUTDIR]' 'builds the native language server in OUTDIR/zclsp; defaults to build/lsp.' 'environment: CC selects the c compiler (one with #embed).'; exit 0 ;; -*) echo "mklsp: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." -O=${1:-build/lsp}; CC=${CC:-gcc} -# DOCS=1: the image carries the doc comments, so hover can show what a name is for +O=${1:-build/lsp}; CC=${CC:-$(sh src/tools/findcc.sh)} +export CC mkdir -p "$O" -bash src/tools/embed.sh all --check || bash src/tools/embed.sh all -# the image carries its own sources (for go to definition into a built-in) -bash src/tools/embed.sh lsp "$O/embed.zc" -DOCS=1 bash src/tools/automap.sh lsp "$O" src/lsp/server.zc "$O/embed.zc" >/dev/null -rm -rf "$O/src"; mkdir -p "$O/src" -cp src/core/core.c src/core/io_linux.c src/core/fops.h src/core/bops.h src/core/stack.h "$O/memmap.h" "$O/src/" -cp "$O/genome.h" "$O/src/lsp_genome.h" -$CC -O2 -w -fwrapv -pthread -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME "$O/src/core.c" "$O/src/io_linux.c" -I"$O/src" -o "$O/zclsp" +DOCS=1 bash src/tools/automap.sh lsp "$O" src/lsp/server.zc >/dev/null +cp "$O/genome.bin" "$O/lsp_genome.bin" +$CC -O2 -w -fwrapv -pthread -DJTYPE=int -DZY_SANDBOX -DUSE_LSP_GENOME src/core/core.c src/core/io_linux.c -Isrc/core -I"$O" --embed-dir="$O" -o "$O/zclsp" diff --git a/src/tools/mklspwasm.sh b/src/tools/mklspwasm.sh index 4789eab..d05a5a7 100755 --- a/src/tools/mklspwasm.sh +++ b/src/tools/mklspwasm.sh @@ -3,21 +3,22 @@ # by default, or the path given as the first argument. the server is the compiler # and src/lsp/server.zc on the map and genome that src/tools/mklsp.sh made in build/lsp, # and the microcore (src/micro/micro.c) runs them. the host is client/wasmproc.js. -# needs clang with the wasm32 target and wasm-ld; run src/tools/mklsp.sh first. +# needs clang with the wasm32 target, #embed and wasm-ld; run src/tools/mklsp.sh first. # src/tools/mklspwasm.sh [outfile.wasm] set -e case "${1:-}" in - --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to build/editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 support and wasm-ld.' 'environment: LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;; + --help|-h) printf '%s\n' 'usage: src/tools/mklspwasm.sh [OUTFILE.wasm]' 'builds the wasm language server; defaults to build/editors/vscode/server/lsp.wasm.' 'requires clang with wasm32 and #embed support, and wasm-ld.' 'environment: WASM_CC selects the compiler; LSP_BUILD_DIR selects the native lsp build; LSP_REUSE_BUILD=1 reuses its image.'; exit 0 ;; -*) echo "mklspwasm: unknown option $1; use --help for usage" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." OUT=${1:-build/editors/vscode/server/lsp.wasm} LSP_BUILD_DIR=${LSP_BUILD_DIR:-build/lsp} +WASM_CC=${WASM_CC:-$(sh src/tools/findcc.sh --wasm)} if [ "${LSP_REUSE_BUILD:-0}" != 1 ]; then bash src/tools/mklsp.sh "$LSP_BUILD_DIR" >/dev/null; fi -[ -f "$LSP_BUILD_DIR/src/lsp_genome.h" ] || { echo "mklspwasm: no language server image in $LSP_BUILD_DIR" >&2; exit 1; } +[ -f "$LSP_BUILD_DIR/lsp_genome.bin" ] || { echo "mklspwasm: no language server image in $LSP_BUILD_DIR" >&2; exit 1; } mkdir -p "$(dirname "$OUT")" -clang --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DUSE_LSP_GENOME \ - -I"$LSP_BUILD_DIR/src" -Isrc/micro src/micro/micro.c src/micro/port_wasm_stream.c \ +"$WASM_CC" --target=wasm32 -O2 -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DUSE_LSP_GENOME \ + -Isrc/core -I"$LSP_BUILD_DIR" --embed-dir="$LSP_BUILD_DIR" -Isrc/micro src/micro/micro.c src/micro/port_wasm_stream.c \ -Wl,--no-entry -Wl,--export=wasm_chunk -Wl,--export=wasm_chunk_cap -Wl,--export=wasm_feed \ -Wl,--export=wasm_polling -Wl,--export=wasm_close -Wl,--export=wasm_start -Wl,--export=wasm_resume \ -Wl,--export=memory -Wl,--allow-undefined -Wl,--initial-memory=41943040 -Wl,-z,stack-size=65536 \ diff --git a/src/tools/mkvsix.sh b/src/tools/mkvsix.sh index df9e703..61fe9a8 100755 --- a/src/tools/mkvsix.sh +++ b/src/tools/mkvsix.sh @@ -1,13 +1,12 @@ #!/bin/bash # builds the vs code extension, with the language server inside it: the tinyvm # javascript vm (built from src/js/tinyvm.ts) and the server's image -# (build/js/lsp.json, from build/lsp/genome.h and its map), and the wasm server +# (build/js/lsp.bin and its map in lsp.json, from build/lsp), and the wasm server # (src/tools/mklspwasm.sh) when clang can build it. needs node and tsc; the # server build (src/tools/mklsp.sh) runs first if build/lsp is missing. # -# everything generated lands in build/. the packaged files are staged into -# editors/vscode/server/ only for as long as vsce is packing, then removed, so a -# fresh clone has no extension until it is built. +# everything generated lands in build/: the server files in +# build/editors/vscode/server, the packing copy of the extension in build/vsix. # src/tools/mkvsix.sh [outfile.vsix] set -e case "${1:-}" in @@ -20,31 +19,33 @@ case "$OUT" in /*) ;; *) OUT="$PWD/$OUT" ;; esac mkdir -p "$(dirname "$OUT")" JS=build/js STAGE=build/editors/vscode/server -PACKDIR=editors/vscode/server bash src/tools/mklsp.sh build/lsp >/dev/null # typescript errors stop packaging, so broken generated code is never shipped. command -v tsc >/dev/null || { echo "mkvsix: tsc is not installed" >&2; exit 1; } +make --no-print-directory build/js/layout.js >/dev/null tsc src/js/tinyvm.ts --target es2020 --lib es2020,dom --module commonjs --outDir "$JS" node src/js/extract_lsp.js mkdir -p "$STAGE" -cp "$JS/tinyvm.js" "$JS/layout.js" "$JS/lsp.json" "$STAGE/" +cp "$JS/tinyvm.js" "$JS/layout.js" "$JS/lsp.json" "$JS/lsp.bin" "$STAGE/" # the wasm server runs when there is one; without clang the vm server is packed alone -if command -v clang >/dev/null && clang --print-targets | grep -q wasm32; then +if sh src/tools/findcc.sh --wasm >/dev/null 2>&1; then LSP_BUILD_DIR=build/lsp LSP_REUSE_BUILD=1 bash src/tools/mklspwasm.sh "$STAGE/lsp.wasm" else rm -f "$STAGE/lsp.wasm" - echo "no clang with wasm32: packing the javascript vm server only" >&2 + echo "no clang with wasm32 and #embed: packing the javascript vm server only" >&2 fi -# vsce packs the extension directory, so the server has to sit inside it while -# it runs. stage it, package, and leave the tree as it was. -cleanup() { rm -rf "$PACKDIR"; } -trap cleanup EXIT INT TERM -rm -rf "$PACKDIR" -mkdir -p "$PACKDIR" -cp "$STAGE"/* "$PACKDIR"/ -cd editors/vscode +# vsce packs a whole extension directory, so the extension is copied into +# build/vsix/extension with the server beside its client, and packed there. the +# tree is never written to. +PACKROOT=build/vsix/extension +rm -rf "$PACKROOT" +mkdir -p "$PACKROOT" +cp -R editors/vscode/. "$PACKROOT/" +mkdir -p "$PACKROOT/server" +cp "$STAGE"/* "$PACKROOT/server/" +cd "$PACKROOT" if [ -n "${VSCE:-}" ]; then "$VSCE" package --allow-missing-repository --skip-license -o "$OUT" >/dev/null elif command -v vsce >/dev/null; then @@ -52,4 +53,4 @@ elif command -v vsce >/dev/null; then else npx -y @vscode/vsce package --allow-missing-repository --skip-license -o "$OUT" >/dev/null fi -echo "$OUT"
\ No newline at end of file +echo "$OUT" diff --git a/src/tools/zc0.c b/src/tools/zc0.c index 446557b..5f5b97d 100644 --- a/src/tools/zc0.c +++ b/src/tools/zc0.c @@ -1,7 +1,8 @@ /* zc0: stage 0 of the bootstrap. a small c compiler for the subset of zc that compiler.zc is written in. it reads zc source on stdin and writes a genome - header (vm words placed at GORG) on stdout, in the same image layout the - resident compiler's image mode produces. no lifetime analysis, no quote, no + image (vm words placed at GORG, as little-endian 64-bit words) on stdout, in + the same image layout the resident compiler's image mode produces. the cores + embed that file with #embed (src/core/genome.h). no lifetime analysis, no quote, no parse words, no block expressions: those only exist in compiler.zc. tags (m, d, -> u) and casts (as u, as d) are parsed and recorded in signatures but not checked. */ @@ -454,12 +455,13 @@ static void strlit(void) /* inline word string: jump over it, push its address * static void bstrlit(void) { int j = here, a, i = 0, k, n = (int)strlen(ts); + const char *b = ts; uint64_t w; lit(0); op(O_JMP); a = here; emit(n); while (i < n) { w = 0; k = 0; - while (k < 8 && i < n) { w |= (uint64_t)(unsigned char)ts[i] << (k * 8); k++; i++; } + while (k < 8 && i < n) { w |= (uint64_t)(unsigned char)b[i] << (k * 8); k++; i++; } emit((int64_t)w); } if ((n & 7) == 0) emit(0); @@ -497,7 +499,7 @@ static void primary(void) else if (((s->sig >> 4) & 15) != 1) die("multiple enhancement outputs need asm"); xs = 0; xf = 0; return; } - if (!strcmp(n, "put_s") && na == 1 && xs) fnref("put_bs"); + if ((!strcmp(n, "put_s") || !strcmp(n, "rt.put_s")) && na == 1 && xs) fnref(n[0] == 'r' ? "rt.put_bs" : "put_bs"); else if (!strcmp(n, "len")) fnref("len_s"); else fnref(n); op(O_CALL); @@ -794,6 +796,77 @@ static void enhdef(void) s = add(n, K_ENH, nenh++); s->sig = np + nout * 16 + d * 256; expect(";"); } +/* embed "path" as [type] name; at the top level: a global holding the file's bytes, + read when the image is built. the path is relative to the working directory. + bstr (the default) packs the bytes; char* keeps one character to a word; a + number takes them little-endian; T[] and T[n] lay them out as the array's + elements. the same rules as the resident compiler's kw_embed. */ +static int embed_width(const char *t, int star) +{ + if (star) return 0; + if (!strcmp(t, "char") || !strcmp(t, "u8") || !strcmp(t, "i8") || !strcmp(t, "byte")) return 1; + if (!strcmp(t, "u16") || !strcmp(t, "i16")) return 2; + if (!strcmp(t, "u32") || !strcmp(t, "i32") || !strcmp(t, "f32")) return 4; + if (!strcmp(t, "int") || !strcmp(t, "word") || !strcmp(t, "bool") || !strcmp(t, "float") || !strcmp(t, "f64")) return 8; + return 0; +} +static void embeddecl(void) +{ + char path[1024], ty[64] = "", *n; int star = 0, arr = -2, w, i, a; long len; unsigned char *buf; FILE *f; + if (tk != T_STR) die("embed takes a path in quotes: embed \"file\" as name;"); + snprintf(path, sizeof path, "%s", ts); next(); + expect("as"); + if (tk == T_ID) { + Lex L; lexsave(&L); next(); + int named = is(";"); lexrest(&L); + if (!named) { + snprintf(ty, sizeof ty, "%s", ts); next(); + while (accept("*")) star++; + if (accept("[")) { arr = -1; if (tk == T_NUM) { arr = (int)tv; next(); } expect("]"); } + } + } + n = ident(); expect(";"); + if (!(f = fopen(path, "rb"))) die("can't read the file named by embed"); + fseek(f, 0, SEEK_END); len = ftell(f); fseek(f, 0, SEEK_SET); + buf = malloc((size_t)len + 1); + if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) die("can't read the file named by embed"); + fclose(f); + if (arr == -2 && (!ty[0] || (!strcmp(ty, "bstr") && !star))) { + a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; emit(len); + for (i = 0; i < len; i += 8) { uint64_t u = 0; for (w = 0; w < 8 && i + w < len; w++) u |= (uint64_t)buf[i + w] << (w * 8); emit((int64_t)u); } + if (!(len & 7)) emit(0); + } else if (arr == -2 && !strcmp(ty, "char") && star == 1) { + a = here; add(n, K_GVAR, a); emit(0); img[a - GORG] = here; + for (i = 0; i < len; i++) emit(buf[i]); + emit(0); + } else if (arr == -2) { + uint64_t u = 0; w = embed_width(ty, star); + if (!w) die("embed declares a bstr, a char*, a number or an array of numbers"); + if (len > w) die("the embedded file is larger than the type it is declared as"); + if (!strcmp(ty, "f32") && len != 4) die("an embedded f32 takes exactly 4 bytes"); + if ((!strcmp(ty, "float") || !strcmp(ty, "f64")) && len != 8) die("an embedded float takes exactly 8 bytes"); + for (i = 0; i < len; i++) u |= (uint64_t)buf[i] << (i * 8); + if (ty[0] == 'i' && w < 8 && (u >> (w * 8 - 1) & 1)) u |= ~(uint64_t)0 << (w * 8); + if (!strcmp(ty, "f32")) u = fop_s2f(u); + a = here; add(n, K_GVAR, a); emit((int64_t)u); + } else { + int count; w = embed_width(ty, star); + if (!w) die("embed declares a bstr, a char*, a number or an array of numbers"); + if (len % w) die("the embedded file's size is not a whole number of elements"); + count = (int)(len / w); + if (arr >= 0 && count > arr) die("the embedded file is larger than the array it is declared as"); + if (arr >= 0) count = arr; + a = here; + if (!strcmp(ty, "char")) { for (i = 0; i < count; i++) emit(i < len ? buf[i] : 0); } + else { + long bytes = (long)count * w; + for (i = 0; i < bytes; i += 8) { uint64_t u = 0; for (int k = 0; k < 8 && i + k < bytes; k++) u |= (uint64_t)(i + k < len ? buf[i + k] : 0) << (k * 8); emit((int64_t)u); } + if (!bytes) emit(0); + } + add(n, K_GARR, a); + } + free(buf); +} static void top1(void) { if (accept("namespace")) { @@ -802,6 +875,7 @@ static void top1(void) expect(";"); return; } if (accept("include")) { do { ident(); } while (accept(",")); expect(";"); return; } + if (accept("embed")) { embeddecl(); return; } { if (accept("unchecked")) { top1(); return; } if (accept("fn")) { fndef(0); return; } @@ -898,8 +972,9 @@ static void top(void) int main(int argc, char **argv) { if (argc == 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h"))) { - fputs("usage: zc0 [--help|-h] < source.zc > genome.h\n" - "compiles bootstrap source from stdin into a c genome header on stdout.\n" + fputs("usage: zc0 [--help|-h] < source.zc > genome.bin\n" + "compiles bootstrap source from stdin into a genome image on stdout:\n" + "little-endian 64-bit words, the format the cores #embed.\n" "source must define start(); memory layout is selected when zc0 is built.\n" "options:\n --help, -h show this help and exit\n", stdout); return 0; @@ -942,8 +1017,10 @@ int main(int argc, char **argv) } emit(0); img[6] = here; - printf("#define GORG %d\n#define GLEN %d\nstatic const int64_t genome[] = {", GORG, here - GORG); - for (i = 0; i < here - GORG; i++) printf("%s%lld", !i ? "\n" : i % 12 ? "," : ",\n", (long long)img[i]); - printf("\n};\n"); - return 0; + for (i = 0; i < here - GORG; i++) { + unsigned char b[8]; uint64_t u = (uint64_t)img[i]; + for (j = 0; j < 8; j++) b[j] = (unsigned char)(u >> (j * 8)); + if (fwrite(b, 1, 8, stdout) != 8) { fprintf(stderr, "zc0: write failed\n"); return 1; } + } + return fflush(stdout) ? 1 : 0; } diff --git a/src/tools/zcc b/src/tools/zcc index 3fb8f0c..ba38e97 100755 --- a/src/tools/zcc +++ b/src/tools/zcc @@ -8,7 +8,7 @@ # # -o OUT the binary to write (default: FILE without .zc) # -k keep the build directory and print where it is -# CC the C compiler (default cc). CFLAGS adds to the default -O2 +# CC the C compiler, one with c23 #embed (default: src/tools/findcc.sh). CFLAGS adds to -O2 # # the build is three steps: automap sizes the map to the program and writes the # genome (the compiled program image) for it; the micro core and its linux port @@ -16,13 +16,13 @@ # the same image without the static link. set -e here="$(cd "$(dirname "$0")/../.." && pwd)" -CC=${CC:-cc} +CC=${CC:-$(sh "$here/src/tools/findcc.sh")} out=""; keep=0 while [ $# -gt 0 ]; do case "$1" in -o) [ $# -ge 2 ] || { echo "zcc: -o needs a name" >&2; exit 2; }; out="$2"; shift 2 ;; -k) keep=1; shift ;; - -h|--help) printf '%s\n' 'usage: src/tools/zcc [-o OUT] [-k] FILE.zc' 'compiles a source file into a standalone binary; links statically where supported.' 'options:' ' -o OUT output binary path; defaults to the source basename without .zc' ' -k keep the temporary build directory' ' --help, -h show this help and exit' 'environment: CC selects the c compiler; CFLAGS adds compiler flags.'; exit 0 ;; + -h|--help) printf '%s\n' 'usage: src/tools/zcc [-o OUT] [-k] FILE.zc' 'compiles a source file into a standalone binary; links statically where supported.' 'options:' ' -o OUT output binary path; defaults to the source basename without .zc' ' -k keep the temporary build directory' ' --help, -h show this help and exit' 'environment: CC selects the c compiler (one with #embed); CFLAGS adds compiler flags.'; exit 0 ;; -*) echo "zcc: unknown option $1" >&2; exit 2 ;; *) break ;; esac @@ -41,7 +41,7 @@ trap cleanup EXIT # the map and the genome for this program only ( cd "$here" && bash src/tools/automap.sh run "$work/run" src/runtime/run.zc "$abs" ) >/dev/null cp "$here/src/micro/micro.c" "$here/src/micro/micro_port.h" "$here/src/micro/port_linux.c" \ - "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$work/run/" + "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$here/src/core/genome.h" "$work/run/" # static where the toolchain allows it; a plain link otherwise, with a note static=-static |
