diff options
| author | 2026-10-15 17:03:03 +0330 | |
|---|---|---|
| committer | 2026-10-15 17:03:03 +0330 | |
| commit | 7cb983bbea98d2038b2a8015a6d1f1afd40991c9 (patch) | |
| tree | ec3d88a15801207fc6d8d3c6d142617ffec85460 /src | |
| parent | merge work/borrow: borrow returns before definition, borrows clause, borrow m... (diff) | |
| parent | bench: rss probe path fix (diff) | |
merge work/actors: wasm seed bootstrap (seed/MANIFEST), shared copy-on-write actor memory, top-level hold blocks on all backends and lsp
Diffstat (limited to 'src')
| -rwxr-xr-x | src/baremetal/run.sh | 4 | ||||
| -rwxr-xr-x | src/boot/boot.sh | 50 | ||||
| -rwxr-xr-x | src/boot/mkseed.sh | 44 | ||||
| -rw-r--r-- | src/boot/port_wasi.c | 136 | ||||
| -rw-r--r-- | src/boot/run.mjs | 46 | ||||
| -rw-r--r-- | src/compiler/base.zc | 6 | ||||
| -rw-r--r-- | src/compiler/decl.zc | 56 | ||||
| -rw-r--r-- | src/compiler/expr.zc | 10 | ||||
| -rw-r--r-- | src/core/core.c | 250 | ||||
| -rw-r--r-- | src/core/hold.h | 116 | ||||
| -rw-r--r-- | src/core/io_linux.c | 21 | ||||
| -rw-r--r-- | src/lsp/server.zc | 22 | ||||
| -rw-r--r-- | src/micro/micro.c | 50 | ||||
| -rw-r--r-- | src/micro/micro_port.h | 3 | ||||
| -rw-r--r-- | src/micro/port_linux.c | 94 | ||||
| -rwxr-xr-x | src/tools/parity.sh | 2 | ||||
| -rwxr-xr-x | src/tools/zcc | 2 |
17 files changed, 848 insertions, 64 deletions
diff --git a/src/baremetal/run.sh b/src/baremetal/run.sh index 43f8a4c..5f812fa 100755 --- a/src/baremetal/run.sh +++ b/src/baremetal/run.sh @@ -69,7 +69,7 @@ if [ "$mode" = run ] || [ "$mode" = io ]; then # the map is sized to the program, so a bigger runtime or program just gets a bigger board ${CC:-$(sh src/tools/findcc.sh)} -O2 -w src/tools/genmap.c -o "$D/genmap" GENMAP="$D/genmap" bash src/tools/automap.sh run "$D/img" src/runtime/run.zc "$src" >"$D/genmap.log" 2>&1 || { cat "$D/genmap.log" >&2; exit 1; } - cp src/micro/micro.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h "$D/img/" + cp src/micro/micro.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/hold.h src/core/genome.h "$D/img/" "$cc" $common $def -I"$D/img" -c "$D/img/micro.c" -o "$D/micro.o" if [ "$mode" = io ]; then "$cc" $common $def -DBARE_INPUT=1 -I"$D/img" -c src/baremetal/port.c -o "$D/port.o" @@ -80,7 +80,7 @@ if [ "$mode" = run ] || [ "$mode" = io ]; then fi elif [ "$mode" = compile ]; then [ -f build/core/genome.bin ] || { echo "baremetal: no build/core/genome.bin; run make genome first" >&2; exit 1; } - cp src/micro/micro.c src/micro/micro_port.h build/core/map/memmap.h build/core/genome.bin src/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h "$D/" + cp src/micro/micro.c src/micro/micro_port.h build/core/map/memmap.h build/core/genome.bin src/core/genome.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/hold.h "$D/" "$cc" $common $def -I"$D" -c "$D/micro.c" -o "$D/micro.o" "$cc" $common $def -DBARE_INPUT=1 -I"$D" -c src/baremetal/port.c -o "$D/port.o" timeout=90 diff --git a/src/boot/boot.sh b/src/boot/boot.sh new file mode 100755 index 0000000..c7eb2ed --- /dev/null +++ b/src/boot/boot.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# bootstraps the compiler from the pinned wasm seed. usage: src/boot/boot.sh OUTDIR +# checks seed/zc.wasm against seed/MANIFEST, then under a wasm runtime (wasmtime, or +# node) the seed compiles the compiler sources to g1 and g1 compiles them to g2; when +# they differ g2 compiles them to g3 and g2 = g3 is required (the fixpoint). the result is +# OUTDIR/genome.bin. the seed is proven by the fixpoint, not trusted. nothing here +# uses the c core. +set -e +cd "$(dirname "$0")/../.." +OUT=${1:?outdir}; SEED=${SEED:-seed/zc.wasm}; MAN=${MAN:-seed/MANIFEST} +mkdir -p "$OUT" +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; } +want=$(sed -n 's/^seed-sha256 //p' "$MAN"); got=$(sha "$SEED") +if [ -z "$BOOT_NOCHECK" ] && [ "$want" != "$got" ]; then + echo "boot: $SEED does not match $MAN" >&2 + echo "boot: manifest sha256 $want" >&2 + echo "boot: file sha256 $got" >&2 + echo "boot: the seed was changed or damaged; restore it from git, or run make seed to rebuild it on purpose" >&2 + exit 1 +fi +if [ -n "$WASM_RUNTIME" ]; then RT=$WASM_RUNTIME +elif command -v wasmtime >/dev/null 2>&1; then RT=wasmtime +elif command -v node >/dev/null 2>&1 && node -e 'process.exit(+process.versions.node.split(".")[0] < 18 ? 1 : 0)' 2>/dev/null; then RT=node +else + echo "boot: no WebAssembly runtime found. the compiler is built from seed/zc.wasm under one of:" >&2 + echo "boot: node 18 or later (macOS: brew install node; debian/ubuntu: apt install nodejs)" >&2 + echo "boot: wasmtime (macOS: brew install wasmtime; elsewhere: curl https://wasmtime.dev/install.sh -sSf | bash)" >&2 + exit 1 +fi +run() { case "$RT" in node) node src/boot/run.mjs "$1" ;; wasmtime) wasmtime run --dir=. "$1" ;; *) $RT "$1" ;; esac; } +le64() { n=$1 i=0; while [ $i -lt 8 ]; do printf "\\$(printf %03o $(( (n >> (8 * i)) & 255 )))"; i=$((i + 1)); done; } +SRC="src/runtime/rt.zc src/core/core.zc $(sed 's#^#src/compiler/#' src/compiler/parts | tr '\n' ' ')" +input() { cat src/tools/build.zc; printf '`'; cat $SRC; } +stage() { # stage IMAGE OUT: IMAGE empty means the seed's own genome + { if [ -n "$1" ]; then printf 'ZCIMAGE\n'; le64 $(wc -c < "$1"); cat "$1"; fi; input; } | run "$SEED" > "$2.tmp" + mv "$2.tmp" "$2" +} +echo "boot: seed $got ok, runtime $RT" +stage "" "$OUT/g1.bin"; echo "boot: g1 (seed -> sources): $(sha "$OUT/g1.bin")" +[ "${BOOT_STAGES:-}" = 1 ] && exit 0 +img=$(sed -n 's/^image-sha256 //p' "$MAN") +[ -n "$img" ] && { [ "$img" = "$(sha "$OUT/g1.bin")" ] && echo "boot: g1 is the image the manifest records" || echo "boot: g1 differs from the manifest's image (the sources changed since the seed was made)"; } +stage "$OUT/g1.bin" "$OUT/g2.bin"; echo "boot: g2 (g1 -> sources): $(sha "$OUT/g2.bin")" +if cmp -s "$OUT/g1.bin" "$OUT/g2.bin"; then + # g1 rebuilt itself unchanged, so it is the fixpoint: a third stage would only repeat it + cp "$OUT/g2.bin" "$OUT/genome.bin"; echo "boot: fixpoint (g1 = g2)"; exit 0 +fi +stage "$OUT/g2.bin" "$OUT/g3.bin"; echo "boot: g3 (g2 -> sources): $(sha "$OUT/g3.bin")" +cmp "$OUT/g2.bin" "$OUT/g3.bin" || { echo "boot: no fixpoint: g2 and g3 differ" >&2; exit 1; } +cp "$OUT/g2.bin" "$OUT/genome.bin"; echo "boot: fixpoint (g2 = g3)" diff --git a/src/boot/mkseed.sh b/src/boot/mkseed.sh new file mode 100755 index 0000000..297a853 --- /dev/null +++ b/src/boot/mkseed.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# builds the bootstrap seed: the microcore with a compiler genome inside, as a WASI +# module, sized down (-Oz, lto, gc-sections, no debug info or names, then wasm-opt -Oz +# when it is on PATH). +# src/boot/mkseed.sh GENOME_DIR OUT.wasm build only +# src/boot/mkseed.sh --manifest GENOME_DIR OUT.wasm build, prove, write MANIFEST beside OUT +# with --manifest the unoptimized and the optimized module must both compile the +# compiler sources to the same image, and the optimized one must reach the fixpoint +# from it; the manifest records both hashes and sizes. +# needs clang with the wasm32 target and wasm-ld. +set -e +cd "$(dirname "$0")/../.." +MANIFEST=; [ "$1" = --manifest ] && { MANIFEST=1; shift; } +G=${1:?genome dir}; OUT=${2:?out} +CC=${WASM_CC:-$(sh src/tools/findcc.sh --wasm)} +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; } +"$CC" --target=wasm32 -Oz -flto -w -fwrapv -nostdlib -fno-builtin -mbulk-memory -DJTYPE=int -DMICRO_PORT_IMAGE \ + -ffunction-sections -fdata-sections -g0 \ + -Isrc/core -Ibuild/core/map --embed-dir="$G" -Isrc/micro src/micro/micro.c src/boot/port_wasi.c \ + -Wl,--gc-sections -Wl,--strip-all -Wl,--initial-memory=134217728 -Wl,-z,stack-size=1048576 -o "$OUT.raw" +OPT=none +if command -v wasm-opt >/dev/null 2>&1; then + wasm-opt -Oz --strip-debug --strip-producers --enable-bulk-memory "$OUT.raw" -o "$OUT.tmp"; OPT="$(wasm-opt --version)" +else cp "$OUT.raw" "$OUT.tmp"; echo "mkseed: no wasm-opt on PATH; the seed is not run through it" >&2; fi +mv "$OUT.tmp" "$OUT" +if [ -n "$MANIFEST" ]; then + M=$(dirname "$OUT")/MANIFEST; T=$(mktemp -d) + printf 'seed-sha256 %s\n' "$(sha "$OUT")" > "$M" + SEED="$OUT" MAN="$M" sh src/boot/boot.sh "$T" + SEED="$OUT.raw" MAN=/dev/null BOOT_NOCHECK=1 BOOT_STAGES=1 sh src/boot/boot.sh "$T/raw" + cmp "$T/g1.bin" "$T/raw/g1.bin" || { echo "mkseed: the size-optimized seed compiles differently from the unoptimized one" >&2; exit 1; } + { + echo "# the pinned bootstrap seed. make checks seed-sha256 before it runs the seed." + echo "# regenerate with: make seed" + printf 'seed-sha256 %s\n' "$(sha "$OUT")" + printf 'seed-size %s\n' "$(wc -c < "$OUT" | tr -d ' ')" + printf 'seed-size-unoptimized %s\n' "$(wc -c < "$OUT.raw" | tr -d ' ')" + printf 'wasm-opt %s\n' "$OPT" + printf 'image-sha256 %s\n' "$(sha "$T/genome.bin")" + printf 'image-size %s\n' "$(wc -c < "$T/genome.bin" | tr -d ' ')" + } > "$M" + rm -rf "$T"; cat "$M" +fi +rm -f "$OUT.raw" diff --git a/src/boot/port_wasi.c b/src/boot/port_wasi.c new file mode 100644 index 0000000..ba8da8c --- /dev/null +++ b/src/boot/port_wasi.c @@ -0,0 +1,136 @@ +/* WASI port of the microcore, for the bootstrap seed (seed/zc.wasm). reads the + whole of stdin, runs the genome inside, writes stdout. needs no libc: the only + imports are wasi_snapshot_preview1 fd_read, fd_write and proc_exit, so the same + module runs under node (src/boot/run.mjs) and wasmtime. build with + src/boot/mkseed.sh. */ +#include "micro_port.h" + +#define IMPORT(n) __attribute__((import_module("wasi_snapshot_preview1"), import_name(n))) +typedef struct { void *buf; unsigned long len; } Iov; +IMPORT("fd_read") int wasi_fd_read(int fd, Iov *iov, int n, unsigned long *got); +IMPORT("fd_write") int wasi_fd_write(int fd, Iov *iov, int n, unsigned long *put); +IMPORT("path_open") int wasi_path_open(int dir, int dirflags, const char *path, unsigned long len, int oflags, unsigned long long rights, unsigned long long inherit, int fdflags, int *fd); +IMPORT("fd_close") int wasi_fd_close(int fd); +IMPORT("proc_exit") _Noreturn void wasi_proc_exit(int code); + +static unsigned char ibuf[65536], obuf[65536]; +static int in_n, in_p, in_end, on; + +static void fill(void) +{ + Iov v = { ibuf, sizeof ibuf }; unsigned long got = 0; + in_p = 0; in_n = 0; + if (in_end) return; + if (wasi_fd_read(0, &v, 1, &got) || got == 0) { in_end = 1; return; } + in_n = (int)got; +} +static void flush_fd(int fd, unsigned char *b, int n) +{ + while (n > 0) { + Iov v = { b, (unsigned long)n }; unsigned long put = 0; + if (wasi_fd_write(fd, &v, 1, &put) || put == 0) return; + b += put; n -= (int)put; + } +} + +static unsigned char *mbase; +static UCell mcap; +Cell *micro_mem_alloc(UCell words) +{ + UCell need, pages; + if (words > ((UCell)-1) / sizeof(Cell)) return 0; + if (!mbase) mbase = (unsigned char *)((UCell)__builtin_wasm_memory_size(0) * 65536u); + need = words * sizeof(Cell); + if (need <= mcap) return (Cell *)mbase; + pages = (need - mcap + 65535u) / 65536u; + /* grow in large steps: some node releases crash in a gc that many small grows provoke; the module + starts with 128 MB, more than a compiler build needs, so a build never grows */ + if (pages < 1024) pages = 1024; + if (__builtin_wasm_memory_grow(0, (int)pages) < 0) return 0; + mcap += pages * 65536u; + return (Cell *)mbase; +} + +int micro_eof(void) { if (in_p >= in_n) fill(); return in_p >= in_n; } +int micro_getc(void) { return micro_eof() ? -1 : ibuf[in_p++]; } +void micro_putc(int c) { if (on == sizeof obuf) { flush_fd(1, obuf, on); on = 0; } obuf[on++] = (unsigned char)c; } +void micro_msg(const char *s) { int n = 0; while (s[n]) n++; flush_fd(2, (unsigned char *)s, n); } + +Cell micro_syscall(Cell *mem, Cell op, Cell a, Cell b, Cell c) +{ + switch (op) { + case 9: + if (a == 0) return micro_mem_words(); + if (a == 1) return HEAP_LO; + if (a == 2) return HEAP_HI; + if (a == 3) return (Cell)micro_mem_words() * 8; + return -1; + case 1: { /* open for reading, relative to the directory preopened as fd 3 (embed reads the library this way) */ + char path[1024]; int n = 0, fd = -1; + if (b != 0) return -1; + while (n < 1023 && a + n < (Cell)micro_mem_words() && mem[a + n]) { path[n] = (char)mem[a + n]; n++; } + path[n] = 0; + if (wasi_path_open(3, 1, path, (unsigned long)n, 0, 2 /* fd_read */, 0, 0, &fd)) return -1; + return fd; + } + case 2: return a > 2 ? (wasi_fd_close((int)a) ? -1 : 0) : -1; + case 3: { + unsigned char t[512]; Cell n = 0; int i; + if (c < 0 || b < 0 || b + c > (Cell)micro_mem_words()) return -1; + while (n < c) { + Iov v = { t, (unsigned long)(c - n > 512 ? 512 : c - n) }; unsigned long got = 0; + if (a == 0) { while (n < c && !micro_eof()) mem[b + n++] = micro_getc(); break; } + if (wasi_fd_read((int)a, &v, 1, &got) || got == 0) break; + for (i = 0; i < (int)got; i++) mem[b + n + i] = t[i]; + n += (Cell)got; + } + return n; + } + case 4: { + unsigned char t[512]; Cell n = 0; int i, k; + if (c < 0 || b < 0 || b + c > (Cell)micro_mem_words() || (a != 1 && a != 2)) return -1; + if (a == 1) { flush_fd(1, obuf, on); on = 0; } + while (n < c) { k = c - n > 512 ? 512 : (int)(c - n); for (i = 0; i < k; i++) t[i] = (unsigned char)mem[b + n + i]; flush_fd((int)a, t, k); n += k; } + return n; + } + case 21: return 1; + case 23: return 0; + } + return -1; +} + +void __multi3(unsigned long long *r, unsigned long long al, unsigned long long ah, unsigned long long bl, unsigned long long bh) +{ + unsigned long long a0 = al & 0xffffffffu, a1 = al >> 32, b0 = bl & 0xffffffffu, b1 = bl >> 32; + unsigned long long p00 = a0 * b0, p01 = a0 * b1, p10 = a1 * b0, p11 = a1 * b1; + unsigned long long mid = (p00 >> 32) + (p01 & 0xffffffffu) + (p10 & 0xffffffffu); + r[0] = (p00 & 0xffffffffu) | (mid << 32); + r[1] = p11 + (p01 >> 32) + (p10 >> 32) + (mid >> 32) + al * bh + ah * bl; +} + +/* an input that starts with "ZCIMAGE\n" and a little-endian 64-bit byte count carries + the image to boot ahead of the program: the bootstrap uses it to run each stage's + compiler in this same module. any other input boots the embedded seed genome. */ +int micro_port_image(Cell *mem, UCell org, UCell end) +{ + static const char tag[] = "ZCIMAGE\n"; + UCell n = 0, i, w; int k; + if (micro_eof() || in_n - in_p < 16) return 0; + for (k = 0; k < 8; k++) if (ibuf[in_p + k] != (unsigned char)tag[k]) return 0; + in_p += 8; + for (k = 0; k < 8; k++) n |= (UCell)micro_getc() << (8 * k); + if (n % 8 || org + n / 8 > end) return -1; + for (i = 0; i < n / 8; i++) { + w = 0; + for (k = 0; k < 8; k++) { int c = micro_getc(); if (c < 0) return -1; w |= (UCell)c << (8 * k); } + mem[org + i] = (Cell)w; + } + return 1; +} + +void _start(void) +{ + int r = micro_run(); + flush_fd(1, obuf, on); on = 0; + if (r) wasi_proc_exit(r); +} diff --git a/src/boot/run.mjs b/src/boot/run.mjs new file mode 100644 index 0000000..f96c3f4 --- /dev/null +++ b/src/boot/run.mjs @@ -0,0 +1,46 @@ +// runs the bootstrap seed (a WASI preview1 module) under node: stdin in, stdout out, +// exit code back. usage: node src/boot/run.mjs MODULE.wasm < in > out +// the seed imports only fd_read, fd_write, path_open (read-only, relative to the +// current directory, preopened as fd 3), fd_close and proc_exit, so they are here +// directly, without node:wasi. +import { readFileSync, readSync, writeSync, openSync, closeSync } from 'node:fs'; +const EXIT = Symbol('exit'); +let mem; +const u8 = () => new Uint8Array(mem.buffer), dv = () => new DataView(mem.buffer); +const fds = new Map([[0, 0], [1, 1], [2, 2]]); +let nextFd = 4; +function iovs(p, n, f) { + let total = 0; + for (let i = 0; i < n; i++) { + const b = dv().getUint32(p + 8 * i, true), l = dv().getUint32(p + 8 * i + 4, true); + const k = f(b, l); total += k; if (k < l) break; + } + return total; +} +const wasi = { + fd_read(fd, p, n, out) { + if (!fds.has(fd)) return 8; + let k; + try { k = iovs(p, n, (b, l) => { try { return readSync(fds.get(fd), u8(), b, l, null); } catch (e) { if (e.code === 'EAGAIN') return 0; if (e.code === 'EOF') return 0; throw e; } }); } + catch (e) { return 29; } + dv().setUint32(out, k, true); return 0; + }, + fd_write(fd, p, n, out) { + if (!fds.has(fd)) return 8; + const k = iovs(p, n, (b, l) => { let d = 0; while (d < l) d += writeSync(fds.get(fd), u8(), b + d, l - d); return l; }); + dv().setUint32(out, k, true); return 0; + }, + path_open(dir, dirflags, path, len, oflags, rights, inherit, fdflags, out) { + if (dir !== 3 || oflags !== 0) return 76; + const name = Buffer.from(u8().subarray(path, path + len)).toString(); + if (name.startsWith('/') || name.split('/').includes('..')) return 76; + let h; try { h = openSync(name, 'r'); } catch (e) { return 44; } + const fd = nextFd++; fds.set(fd, h); dv().setUint32(out, fd, true); return 0; + }, + fd_close(fd) { if (fd < 3 || !fds.has(fd)) return 8; closeSync(fds.get(fd)); fds.delete(fd); return 0; }, + proc_exit(code) { throw { [EXIT]: code }; }, +}; +const { instance } = await WebAssembly.instantiate(readFileSync(process.argv[2]), { wasi_snapshot_preview1: wasi }); +mem = instance.exports.memory; +try { instance.exports._start(); process.exitCode = 0; } +catch (e) { if (e && EXIT in e) process.exitCode = e[EXIT]; else throw e; } diff --git a/src/compiler/base.zc b/src/compiler/base.zc index 7e4b5bc..bd27e28 100644 --- a/src/compiler/base.zc +++ b/src/compiler/base.zc @@ -323,6 +323,12 @@ int s_ld8, s_st8, s_ld16, s_st16, s_ld32, s_st32; // ---- o3 region state ---- // each promoted slot maps to a register; pending regions use 67-word records and a descriptor buffer. +// hold blocks: hold_cur is the rank of the block being compiled (0 outside one), fhold the +// rank of the function being compiled, hold_desc the word that will point at the block's +// table of shared variables, hold_count the blocks so far. a shared variable's symbol has +// F_SHARED and its block's rank above it in sflag. +const int F_SHARED = 131072, F_SHARED_RANK = 262144; +int hold_cur, fhold, hold_count, hold_desc, hold_nshared; word[128] hold_tab; int o3on, o3nxt, o3one; word[32] o3sl; word[32] o3rg; int no3sl; word[O3_REGIONS * 67] o3pend_initial; word o3pend = o3pend_initial; int o3pendcap = O3_REGIONS, no3pend; word[80] o3dsc; int o3fs, o3na, o3fn0; // o3free holds registers released by closed scopes; o3cache holds the items pointer and length slots for fixed arrays. word[8] o3free; int no3free, o3nreg; word[NL] o3cache_initial; word o3cache = o3cache_initial; int argloc, s_hitems, s_hlen, s_fatten; diff --git a/src/compiler/decl.zc b/src/compiler/decl.zc index 52d08c7..55e23a7 100644 --- a/src/compiler/decl.zc +++ b/src/compiler/decl.zc @@ -230,11 +230,61 @@ int prologue() { setlit(k, here); setlit(done, here); op(O_DROP); setlit(busy, here); return 0; } +// a function of a hold block takes the block's lock on entry (op 57 of the syscall gate) +// and gives it back at its one exit (op 58), with the return value left on the stack. the +// host keeps the shared values: entering the block from outside copies them into the +// block's variables, leaving copies them back, so inside, every read and write is a plain +// global access on every backend. the lock is reentrant and held for the whole call. +int hold_gate(int code) { + if (!fhold || nclo) return 0; + lit(fhold); lit(rt.SYSGATE + 1); op(O_ST); + lit(hold_desc); lit(rt.SYSGATE + 2); op(O_ST); + lit(code); lit(rt.SYSGATE); op(O_ST); + return 0; +} +// hold { shared let c; fn a() { ... } ... }: a monitor. the shared variables can be used +// only by the functions written in the block; each call holds the block's lock. +int kw_hold() { + int rank, d, before, s, i, w; word t; + if (hold_cur) serr("hold blocks don't nest"); + next(); expect(s_lb); + rank = hold_count + 1; hold_count = rank; + reserve_code(1); d = here; emit(0); + hold_cur = rank; hold_desc = d; hold_nshared = 0; + while (!is_op(s_rb)) { + if (tk == T_EOF) { die("missing } at the end of a hold block"); break; } + if (panic) { topsync(); continue; } + if (tk == T_ID && rt.eq_w(tv, "shared")) { + next(); before = nsym; + if (is_id(s_var)) gdecl(0); + else if (tstart()) { rettype(); gdecl2(dty, dtag, name()); } + else { die("expected let or a type after shared"); continue; } + s = before; + while (s < nsym) { + if (skind[s] == K_GVAR || skind[s] == K_GARR) { + t = stype[s]; w = 1; + if (skind[s] == K_GARR) { w = tsz(t); if (t[0] == TY_ARR) t = t[1]; } + if (t && (isptr(t) || isagg(t) || t == tbstr || t[0] == TY_DYN || t[0] == TY_FN)) serrn(sname[s], "is shared, so it holds numbers: no pointers, strings or structs (each worker has its own heap)"); + sflag[s] = sflag[s] | F_SHARED | rank * F_SHARED_RANK; + if (hold_nshared >= 64) serr("too many shared variables in one hold block (64)"); + else { hold_tab[hold_nshared * 2] = sval[s]; hold_tab[hold_nshared * 2 + 1] = w; hold_nshared++; } + } + s++; + } + } else topitem0(); + } + next(); + reserve_code(1 + hold_nshared * 2); *(d + hdelta) = here; emit(hold_nshared); + i = 0; while (i < hold_nshared * 2) { emit(hold_tab[i]); i++; } + hold_cur = 0; hold_desc = 0; + return 0; +} // emit the function epilogue, patching every return to it. int epilogue() { int k, node, done, helper = !nclo && (sname[finsym] == s_frame_grow || sname[finsym] == s_frame_leave); lit(0); fixall(epichain, here); + hold_gate(58); lit(rt.ZFP); op(O_LD); lit(fslots); op(O_ADD); lit(rt.ZFP); op(O_ST); if (!helper) { lit(sval[lookup(s_frame_node)]); op(O_LD); node = hole(); op(O_JZ); @@ -404,7 +454,9 @@ int fndef(int chk, int rty, int ret, int n) { if (typed && explicit_rty) fty = rty; no3pend = 0; if (o3f) { o3on = 1; o3reset(); sflag[s] = sflag[s] | F_O3; } - prologue(); borrow_prologue(); + fhold = hold_cur; + if (fhold && o3f) serrn(n, "is in a hold block and can't be o3"); + prologue(); borrow_prologue(); hold_gate(57); i = 0; while (i < np) { t = newloc(pn[i]); lrskind[t] = RS_PARAM; lrsoff[t] = pnoffs[i]; lpos[t] = pnplaces[i]; fslots++; lsmark(1, pn[i]); @@ -431,6 +483,7 @@ int fndef(int chk, int rty, int ret, int n) { } hi = here; epilogue(); + fhold = 0; lend = here; if (o3f) { o3on = 0; @@ -720,6 +773,7 @@ int topitem1() { if (!is_op(s_lp)) { die("expected a function"); return 0; } return fndone(fndef(chk, dty, dtag == S_U, n), imm); } + if (tk == T_ID && rt.eq_w(tv, "hold")) { peek(); if (pk_tk == T_OP && pk_tv == s_lb) return kw_hold(); } if (is_id(s_var)) return gdecl(0); if (is_id(s_struct)) { kw_struct(); return 0; } if (is_id(s_type)) return kw_type(); diff --git a/src/compiler/expr.zc b/src/compiler/expr.zc index a6cc55c..141ca16 100644 --- a/src/compiler/expr.zc +++ b/src/compiler/expr.zc @@ -657,6 +657,15 @@ word closure_layout(int closure_index, int count) { // the code that creates the closure runs, it allocates the block (code, then // captured values), writes the three code words and copies each captured // variable in. ctx_push / ctx_pop save and restore the enclosing function's +// a shared variable is reachable only from a function of its own hold block, which holds +// the block's lock for the whole call. a closure might run after the call returns. +int shared_use(int s) { + int rank = sflag[s] / F_SHARED_RANK; + if (fhold == rank && !nclo && infn) return 0; + if (fhold == rank && nclo) { serrn(sname[s], "is shared; a closure can't use it, because it may run after the hold block's lock is released"); return 0; } + serrn(sname[s], "is shared in a hold block; only the functions written inside that block can use it (they hold its lock)"); + return 0; +} // compiler state around the body. int closure() { char* xcf; @@ -1095,6 +1104,7 @@ int primary() { xcval = sval[s]; return C_PLAIN; } + if (s >= 0 && (skind[s] == K_GVAR || skind[s] == K_GARR) && (sflag[s] & F_SHARED)) shared_use(s); if (s >= 0 && skind[s] == K_GVAR) { if (cmode) { vf_symbols(); xvk = svkind[s]; xvv = svvalue[s]; vf_at(rsn - 1, xvk, xvv); } lit(sval[s]); diff --git a/src/core/core.c b/src/core/core.c index ce9e5ca..7d1b91e 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -74,6 +74,10 @@ #include "memmap.h" #include "fops.h" #include "bops.h" +#if !defined(_WIN32) && !defined(__wasm__) +#define HOLD_THREADS +#endif +#include "hold.h" #ifndef JTYPE #define JTYPE char @@ -256,6 +260,7 @@ static atomic_int context_records_live, context_code_live, context_memory_live; size_t io_context_size(void); void io_context_snapshot(void *destination); void io_context_restore(const void *source); +void io_context_release(void); #else static VM_TLS int context_default_stop; #endif @@ -294,6 +299,12 @@ static Cell hostcall(Cell op, Cell a, Cell b, Cell c) #else return -1; #endif + case 57: { + Cell r = hold_enter(m, (uint64_t)MW, a, b); + if (r == -2) { fflush(stdout); fprintf(stderr, "zy: hold block %d was entered while block %d was held; hold blocks are entered in the order they are declared\n", (int)a, hold_top); exit(2); } + return r; + } + case 58: return hold_leave(m, (uint64_t)MW, a, b); case 56: return cb_r; /* the return stack pointer as of the gate store: the word after the top return address */ case 1: if (zcstr(a, path, 1024) < 0) return -1; @@ -622,11 +633,44 @@ typedef struct { Uop *u; signed char *reg; /* physical register per value, for the jit; 0 if it didn't fit */ } Enh; -static VM_TLS Enh enh[NENH]; +/* large per-thread tables live behind a pointer and are mapped on first use, so a + worker thread that never compiles anything pays nothing for them (as static tls + they were zeroed into every thread's stack mapping: about 13 MB per actor) */ +static VM_TLS void **tls_big_slots[48]; static VM_TLS size_t tls_big_sizes[48]; static VM_TLS int tls_big_n; +static void *tls_big(size_t bytes, void **slot) +{ + void *p; +#if !defined(_WIN32) && !defined(__wasm__) + p = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); if (p == MAP_FAILED) p = 0; +#else + p = calloc(1, bytes); +#endif + if (!p) { fprintf(stderr, "zy: out of memory for compiler tables\n"); exit(2); } + if (tls_big_n < 48) { tls_big_slots[tls_big_n] = slot; tls_big_sizes[tls_big_n++] = bytes; } + *slot = p; return p; +} +static void tls_big_free(void) +{ + while (tls_big_n > 0) { + tls_big_n--; +#if !defined(_WIN32) && !defined(__wasm__) + munmap(*tls_big_slots[tls_big_n], tls_big_sizes[tls_big_n]); +#else + free(*tls_big_slots[tls_big_n]); +#endif + *tls_big_slots[tls_big_n] = 0; + } +} +#define TLS_BIG(p) (*((p) ? (p) : (__typeof__(p))tls_big(sizeof *(p), (void **)&(p)))) +static VM_TLS Enh (*enh_tls)[NENH]; +#define enh TLS_BIG(enh_tls) + static VM_TLS int need_flush; #define MAXUOP 4096 -static VM_TLS Uop ub[MAXUOP]; static VM_TLS int un, unv, uerr; /* the program being built */ +static VM_TLS Uop (*ub_tls)[MAXUOP]; +#define ub TLS_BIG(ub_tls) +static VM_TLS int un, unv, uerr; /* the program being built */ static int unew(int op, int a, int b, int c, Cell imm) { Uop *u; @@ -838,7 +882,9 @@ static void umagic(UCell d, UCell *mul, int *add, int *sh) /* hacker's delight, *mul = q2 + 1; *sh = p - 64; } -static VM_TLS Uop ob[MAXUOP]; static VM_TLS int on, onv; +static VM_TLS Uop (*ob_tls)[MAXUOP]; +#define ob TLS_BIG(ob_tls) +static VM_TLS int on, onv; static VM_TLS int *omap, *isk; static VM_TLS Cell *kv; /* old value -> new value; constness of new values */ static int pure(int op) { return !is_memop(op); } static int ispow2(UCell x) { return x && !(x & (x - 1)); } @@ -939,8 +985,8 @@ static int oe(int op, int a, int b, int c, Cell imm) return raw(op, a, b, c, 0); } static void optimizer_storage(void) { - static VM_TLS int omap_[MAXUOP + 16], isk_[MAXUOP * 4]; static VM_TLS Cell kv_[MAXUOP * 4]; - omap = omap_; isk = isk_; kv = kv_; + static VM_TLS int (*omap_)[MAXUOP + 16], (*isk_)[MAXUOP * 4]; static VM_TLS Cell (*kv_)[MAXUOP * 4]; + omap = TLS_BIG(omap_); isk = TLS_BIG(isk_); kv = TLS_BIG(kv_); } /* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */ static int optimize(Enh *e, int nin, int *out, int nout) @@ -1121,11 +1167,19 @@ typedef struct { unsigned gen; /* bumped every time the region's code is dropped: a copy made from an older one is stale */ } Rg; #define MAXRG 4096 -static VM_TLS Rg *rgs[MAXRG]; +static VM_TLS Rg * (*rgs_tls)[MAXRG]; +#define rgs TLS_BIG(rgs_tls) + static VM_TLS int nrg; -static VM_TLS int fn_pc[MAXRG]; -static VM_TLS Rg *fn_rg_[MAXRG]; -static VM_TLS Rg *fn_cache[1024]; +static VM_TLS int (*fn_pc_tls)[MAXRG]; +#define fn_pc TLS_BIG(fn_pc_tls) + +static VM_TLS Rg * (*fn_rg__tls)[MAXRG]; +#define fn_rg_ TLS_BIG(fn_rg__tls) + +static VM_TLS Rg * (*fn_cache_tls)[1024]; +#define fn_cache TLS_BIG(fn_cache_tls) + static VM_TLS int nfn; /* ---- bookkeeping for translated code, shared by the backends ---- @@ -1142,12 +1196,18 @@ static VM_TLS int nfn; #define NPG ((code_words >> PGSH) + 1) #define MAXPGN (1 << 19) #define MAXDEP (1 << 18) -static VM_TLS struct { int a, b; } ix_blk[MAXB]; +static VM_TLS struct { int a, b; } (*ix_blk_tls)[MAXB]; +#define ix_blk TLS_BIG(ix_blk_tls) + static VM_TLS int ix_n; static VM_TLS int *pg_head, pg_free, pg_top, pg_live; /* node numbers are stored plus one: 0 is none */ -static VM_TLS struct { int blk, next; } pg_node[MAXPGN]; +static VM_TLS struct { int blk, next; } (*pg_node_tls)[MAXPGN]; +#define pg_node TLS_BIG(pg_node_tls) + static VM_TLS int dep_head[NENH + 1], dep_n; /* slot NENH: depends on every install */ -static VM_TLS struct { int blk, next; } dep_node[MAXDEP]; +static VM_TLS struct { int blk, next; } (*dep_node_tls)[MAXDEP]; +#define dep_node TLS_BIG(dep_node_tls) + static VM_TLS int drop_epoch; /* changes whenever any translation dies */ static void be_drop(int j); /* the backend forgets block j's code and coverage */ static void runmark(Rg *R); /* the backend stops watching the region's words */ @@ -1770,7 +1830,9 @@ static Cell ffi_cbmake(Cell entry, Cell sig) /* a native call the o3 lifter has resolved: the micro-ops narg and ncall carry an index into this table. imm of narg is the argument's position */ typedef struct { void *f; Cell sig; int n; char name[48]; } Ncd; -static VM_TLS Ncd ncd[256]; +static VM_TLS Ncd (*ncd_tls)[256]; +#define ncd TLS_BIG(ncd_tls) + static VM_TLS int nncd; static int ncd_get(Cell blk) { @@ -1878,8 +1940,12 @@ static VM_TLS int lv_[RMAXV], lw_[RMAXV]; #define SB 1100 static VM_TLS int skv[2 * SB], stp, slo, shi; static VM_TLS unsigned char skk[2 * SB]; -static VM_TLS int afp[MAXUOP * 4 + 16]; /* value -> 1 + its offset from fp, if it is fp + k */ -static VM_TLS int emap[MAXUOP * 4 + 16]; +static VM_TLS int (*afp_tls)[MAXUOP * 4 + 16]; +#define afp TLS_BIG(afp_tls) +/* value -> 1 + its offset from fp, if it is fp + k */ +static VM_TLS int (*emap_tls)[MAXUOP * 4 + 16]; +#define emap TLS_BIG(emap_tls) + static int rvar(int v) { @@ -2360,7 +2426,13 @@ enum { C_UNSEEN, C_CONST, C_VAR, C_COPY }; /* C_COPY: the variable holds the same value as variable v (both at the same point) */ static int copy_ok(const Rg *R, int j, int k) { return j > 0 && j < RMAXV && k > 0 && k < R->nv && !R->fv[j] && !R->fv[k]; } typedef struct { unsigned char st[RMAXV]; Cell v[RMAXV]; } CIn; -static VM_TLS int g_isk[MAXUOP * 4]; static VM_TLS Cell g_kv[MAXUOP * 4]; static VM_TLS int g_omap[MAXUOP + 16]; +static VM_TLS int (*g_isk_tls)[MAXUOP * 4]; +#define g_isk TLS_BIG(g_isk_tls) +static VM_TLS Cell (*g_kv_tls)[MAXUOP * 4]; +#define g_kv TLS_BIG(g_kv_tls) +static VM_TLS int (*g_omap_tls)[MAXUOP + 16]; +#define g_omap TLS_BIG(g_omap_tls) + static int cjoin(CIn *d, const CIn *s) { int j, ch = 0; @@ -3313,7 +3385,9 @@ static VM_TLS void **tab; /* vm address -> translation */ typedef struct { void *stub; int fslots; int argc; } FnNat; static VM_TLS FnNat *fn_nat; #define LINKN 1024 -static VM_TLS void *linkstk[LINKN]; +static VM_TLS void * (*linkstk_tls)[LINKN]; +#define linkstk TLS_BIG(linkstk_tls) + static VM_TLS Cell jit_depth; #endif static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); } @@ -3963,7 +4037,7 @@ static int spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4) return -1; un = 0; unv = nin; uerr = 0; for (j = 0; j < nin; j++) mp[j] = j; @@ -5237,9 +5311,13 @@ static Blk lcompile(int p) #if !defined(ZYLLVM) && !defined(ZYJIT_X64) static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */ #endif -static VM_TLS Cell rvals[MAXUOP * 4 + 16]; +static VM_TLS Cell (*rvals_tls)[MAXUOP * 4 + 16]; +#define rvals TLS_BIG(rvals_tls) + #define RPOOL 40 -static VM_TLS Cell rpool[RPOOL][MAXUOP * 4 + 16]; +static VM_TLS Cell (*rpool_tls)[RPOOL][MAXUOP * 4 + 16]; +#define rpool TLS_BIG(rpool_tls) + static VM_TLS int rpool_n; static int rdrop(int a) /* a store landed on region code: drop those regions */ { @@ -6090,7 +6168,11 @@ static void float_op(int i, int k) static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp }; #define XNREG 7 #define XMAXOP 400 -static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16]; +static VM_TLS int (*xkc_tls)[MAXUOP * 4 + 16]; +#define xkc TLS_BIG(xkc_tls) +static VM_TLS Cell (*xkv_tls)[MAXUOP * 4 + 16]; +#define xkv TLS_BIG(xkv_tls) + static VM_TLS signed char *xr; static int x64_op_ok(int op) { @@ -6110,7 +6192,7 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1; for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1; un = 0; unv = nin; uerr = 0; @@ -6131,7 +6213,9 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS int (*xvh_tls)[MAXUOP * 4 + 16]; +#define xvh TLS_BIG(xvh_tls) +/* regions: 1 + the var a value reads at block entry, 0 if none */ static VM_TLS Rg *xR; static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ static int xv(int v, int scratch) @@ -6250,7 +6334,9 @@ static void x64_enh(Enh *e) under ZYO3INTERP=1. */ static VM_TLS int x64_o3interp; static void perfmap_note(JW *start, JW *end, const char *kind, int a, int b); -static VM_TLS signed char rxr[MAXUOP * 4 + 16]; +static VM_TLS signed char (*rxr_tls)[MAXUOP * 4 + 16]; +#define rxr TLS_BIG(rxr_tls) + #define RXNREG 6 static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); } @@ -6258,7 +6344,7 @@ static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); /* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ static int x64_ralloc(RB *B) { - static VM_TLS int last[MAXUOP * 4 + 16]; + static VM_TLS int (*last_tls)[MAXUOP * 4 + 16]; int *last = TLS_BIG(last_tls); int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; if (B->nv > MAXUOP * 4) return -1; for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } @@ -7363,6 +7449,85 @@ static int code_range_live(Cell start, Cell count) /* each worker owns a host thread, vm memory and translation state. */ #if !defined(_WIN32) && !defined(__wasm__) +#include <sys/mman.h> +#ifdef __linux__ +#include <sys/syscall.h> +#endif +/* vm memory for a worker is copy-on-write against a snapshot of its parent. + the first spawn writes the parent's live words [0, memory_words) to an anonymous + shared file and maps the parent's memory privately over it at the same address, + so the parent reads the same pages and its later writes stay its own. a child maps + the same file privately, so code, genome and everything the parent has not + touched since are one set of physical pages for all of them, and only the pages + the parent changed since the snapshot are copied in. the file never changes, so a + child never sees a later write of its parent. a parent whose memory has drifted + far from its snapshot takes a fresh one. without a file (or on failure) a spawn + falls back to a private copy. */ +typedef struct { int fd; Cell *view; size_t words; } MemSnap; +static VM_TLS MemSnap mem_snap = { -1, 0, 0 }; +static VM_TLS int memory_mapped; /* m came from vm_memory_map */ +#define SNAP_PAGE 512 /* words per 4 KB page */ +static Cell *vm_memory_map(void) +{ + void *p = mmap(0, (size_t)MW * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); + return p == MAP_FAILED ? 0 : p; +} +static void vm_memory_unmap(Cell *p) { if (p) munmap(p, (size_t)MW * sizeof(Cell)); } +static int snap_file(size_t bytes) +{ + int fd = -1; +#ifdef __linux__ +#ifdef SYS_memfd_create + fd = (int)syscall(SYS_memfd_create, "zy-snapshot", 1u /* MFD_CLOEXEC */); +#endif +#else + char name[64]; snprintf(name, sizeof name, "/zy-%d-%p", (int)getpid(), (void *)&name); + fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); if (fd >= 0) shm_unlink(name); +#endif + if (fd < 0) return -1; + if (ftruncate(fd, (off_t)bytes) < 0) { close(fd); return -1; } + return fd; +} +static void snap_drop(void) +{ + if (mem_snap.view) munmap(mem_snap.view, mem_snap.words * sizeof(Cell)); + if (mem_snap.fd >= 0) close(mem_snap.fd); + mem_snap.fd = -1; mem_snap.view = 0; mem_snap.words = 0; +} +/* pages of [0, words) where m differs from the snapshot */ +static size_t snap_dirty(size_t words) +{ + size_t p, n = 0, lim = words < mem_snap.words ? words : mem_snap.words; + for (p = 0; p < lim; p += SNAP_PAGE) if (memcmp(m + p, mem_snap.view + p, SNAP_PAGE * sizeof(Cell))) n++; + return n + (words > mem_snap.words ? (words - mem_snap.words) / SNAP_PAGE : 0); +} +static int snap_take(void) +{ + size_t words = ((size_t)memory_words + SNAP_PAGE - 1) / SNAP_PAGE * SNAP_PAGE, bytes = words * sizeof(Cell), done = 0; + int fd; void *view, *over; + if (words > (size_t)MW) return -1; + fd = snap_file(bytes); if (fd < 0) return -1; + while (done < bytes) { ssize_t k = pwrite(fd, (char *)m + done, bytes - done, (off_t)done); if (k <= 0) { close(fd); return -1; } done += (size_t)k; } + view = mmap(0, bytes, PROT_READ, MAP_SHARED, fd, 0); + if (view == MAP_FAILED) { close(fd); return -1; } + over = mmap(m, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, fd, 0); + if (over == MAP_FAILED) { munmap(view, bytes); close(fd); return -1; } /* m is untouched when the map fails */ + snap_drop(); mem_snap.fd = fd; mem_snap.view = view; mem_snap.words = words; return 0; +} +/* a child's memory: the snapshot, plus the pages the parent changed since */ +static Cell *snap_clone(void) +{ + size_t p, words = (size_t)memory_words; + Cell *c; + if (mem_snap.fd < 0 || snap_dirty(words) * 4 > mem_snap.words / SNAP_PAGE) { if (snap_take() < 0) return 0; } + c = vm_memory_map(); if (!c) return 0; + if (mmap(c, mem_snap.words * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, mem_snap.fd, 0) == MAP_FAILED) { vm_memory_unmap(c); return 0; } + for (p = 0; p < words; p += SNAP_PAGE) { + size_t k = words - p < SNAP_PAGE ? words - p : SNAP_PAGE; + if (p >= mem_snap.words || memcmp(m + p, mem_snap.view + p, k * sizeof(Cell))) memcpy(c + p, m + p, k * sizeof(Cell)); + } + return c; +} typedef struct ContextChannel { Cell handle; int fd; struct ContextChannel *next; } ContextChannel; static VM_TLS ContextChannel *context_channels; static atomic_llong context_id_sequence; @@ -7374,6 +7539,7 @@ struct VmContext { Cell token, entry, sp, rp, stack, words, limit, growth, code_size; Cell frame_floor, frame_top, frame_guard; Cell *memory; + int memory_mapped; int exit_code, requested_exit; void *input_snapshot; int argc, envc; @@ -7387,6 +7553,7 @@ struct VmContext { int region_count; unsigned short *entries; unsigned char *entry_indices; + int *entry_at, entry_count; }; static int context_descriptor(Cell handle) { ContextChannel *channel; @@ -7405,7 +7572,7 @@ static void context_snapshot_free(VmContext *context) { int i; for (i = 0; i < NENH; i++) { free(context->enhancements[i].u); free(context->enhancements[i].reg); } for (i = 0; i < context->region_count; i++) free(context->regions[i]); - free(context->input_snapshot); free(context->entries); free(context->entry_indices); if (context->memory) { free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } context->memory = 0; + free(context->input_snapshot); free(context->entries); free(context->entry_indices); free(context->entry_at); if (context->memory) { if (context->memory_mapped) vm_memory_unmap(context->memory); else free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } context->memory = 0; } static int context_cancelled(void) { return context_current && atomic_load_explicit(&context_current->stop, memory_order_acquire); } static void context_exit(int value) { context_current->exit_code = value; context_current->requested_exit = 1; atomic_store_explicit(&context_current->stop, 1, memory_order_release); } @@ -7425,7 +7592,7 @@ static void context_engine_free(void) { free(lcov); free(lt); free(hot); #endif #endif - free(m); m = 0; atomic_fetch_sub(&context_memory_live, 1); + if (memory_mapped) vm_memory_unmap(m); else free(m); m = 0; snap_drop(); tls_big_free(); io_context_release(); atomic_fetch_sub(&context_memory_live, 1); } static void *context_worker(void *opaque) { VmContext *context = opaque; @@ -7434,7 +7601,7 @@ static void *context_worker(void *opaque) { optimizer_storage(); context_current = context; io_context_restore(context->input_snapshot); free(context->input_snapshot); context->input_snapshot = 0; trace = getenv("ZYTRACE") != 0; context->worker_channel.next = context_channels; context_channels = &context->worker_channel; - m = context->memory; context->memory = 0; + m = context->memory; context->memory = 0; memory_mapped = context->memory_mapped; memory_limit = (int)context->limit; memory_words = (int)context->words; memory_growth_limit = (int)context->growth; stack_control = context->stack; frame_floor_word = context->frame_floor; frame_top_word = context->frame_top; frame_guard_word = context->frame_guard; zy_argc = context->argc; zy_argv = context->argv; zy_envc = context->envc; zy_envp = context->envp; @@ -7463,8 +7630,8 @@ static void *context_worker(void *opaque) { memcpy(enh, context->enhancements, sizeof enh); memset(context->enhancements, 0, sizeof context->enhancements); for (i = 0; i < context->region_count; i++) { rgs[i] = context->regions[i]; context->regions[i] = 0; } nrg = context->region_count; context->region_count = 0; - memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words); - free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; + for (i = 0; i < context->entry_count; i++) { o3ent[context->entry_at[i]] = context->entries[i]; o3ei[context->entry_at[i]] = context->entry_indices[i]; } + free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; free(context->entry_at); context->entry_at = 0; #ifdef ZYJIT_X64 if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } atomic_fetch_add(&context_code_live, 1); @@ -7489,7 +7656,7 @@ static void *context_worker(void *opaque) { if (!context->requested_exit) context->exit_code = context_cancelled() ? 0 : stopped(back) ? -1 : inmem(sp) ? (int)m[sp] : 0; context_channel_close(context->worker_channel.handle); while (context_children) context_join(context_children->token, 1); - context_engine_free(); context_current = 0; return 0; + hold_release_all(); context_engine_free(); context_current = 0; return 0; } static Cell context_start(Cell entry, Cell channel) { VmContext *context; int fd[2], i; pthread_attr_t attributes; Cell saved; @@ -7506,12 +7673,17 @@ static Cell context_start(Cell entry, Cell channel) { context->parent_channel.fd = fd[0]; context->worker_channel.fd = fd[1]; atomic_init(&context->stop, 0); context->input_snapshot = malloc(io_context_size()); if (!context->input_snapshot) goto failed; io_context_snapshot(context->input_snapshot); - context->memory = calloc((size_t)memory_limit, sizeof(Cell)); + context->memory = snap_clone(); context->memory_mapped = context->memory != 0; + if (!context->memory) context->memory = calloc((size_t)memory_limit, sizeof(Cell)); if (context->memory) atomic_fetch_add(&context_memory_live, 1); - context->entries = malloc((size_t)code_words * sizeof *o3ent); context->entry_indices = malloc((size_t)code_words); - if (!context->memory || !context->entries || !context->entry_indices) goto failed; - saved = m[channel]; m[channel] = context->worker_channel.handle; memcpy(context->memory, m, (size_t)memory_words * sizeof(Cell)); m[channel] = saved; - memcpy(context->entries, o3ent, (size_t)code_words * sizeof *o3ent); memcpy(context->entry_indices, o3ei, (size_t)code_words); + /* o3 entry points are few and the tables mostly zero: hand over only the set ones */ + { Cell k; context->entry_count = 0; for (k = 0; k < code_words; k++) if (o3ent[k]) context->entry_count++; } + context->entry_at = malloc((size_t)(context->entry_count + 1) * sizeof *context->entry_at); + context->entries = malloc((size_t)(context->entry_count + 1) * sizeof *o3ent); context->entry_indices = malloc((size_t)(context->entry_count + 1)); + if (!context->memory || !context->entries || !context->entry_indices || !context->entry_at) goto failed; + if (context->memory_mapped) context->memory[channel] = context->worker_channel.handle; + else { saved = m[channel]; m[channel] = context->worker_channel.handle; memcpy(context->memory, m, (size_t)memory_words * sizeof(Cell)); m[channel] = saved; } + { Cell k; int j = 0; for (k = 0; k < code_words; k++) if (o3ent[k]) { context->entry_at[j] = (int)k; context->entries[j] = o3ent[k]; context->entry_indices[j] = o3ei[k]; j++; } } for (i = 0; i < NENH; i++) { context->enhancements[i] = enh[i]; context->enhancements[i].u = 0; context->enhancements[i].reg = 0; if (enh[i].u) { context->enhancements[i].u = malloc((size_t)(enh[i].n + 1) * sizeof(Uop)); if (!context->enhancements[i].u) goto failed; memcpy(context->enhancements[i].u, enh[i].u, (size_t)(enh[i].n + 1) * sizeof(Uop)); } @@ -7526,7 +7698,7 @@ static Cell context_start(Cell entry, Cell channel) { memcpy(context->callback_snapshot, ffi_cb, sizeof ffi_cb); memcpy(context->symbol_snapshot, ffi_sym, sizeof ffi_sym); memcpy(context->library_snapshot, ffi_lib, sizeof ffi_lib); context->ffi_symbols = ffi_nsym; context->ffi_libraries = ffi_nlib; context->ffi_libm = ffi_tried_libm; #endif - pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 16u << 20); + pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 4u << 20); /* the vm keeps its own stacks in vm memory; 4 MB of address space covers the deepest native recursion (jit compile, o3) */ i = pthread_create(&context->thread, &attributes, context_worker, context); pthread_attr_destroy(&attributes); if (i) goto failed; context->parent_channel.next = context_channels; context_channels = &context->parent_channel; @@ -7603,6 +7775,10 @@ int main(int argc, char **argv) memory_words = memory_option("TINYVM_MEMORY_WORDS", HEAP_HI > PW ? HEAP_HI : PW, HEAP_HI > PW ? HEAP_HI : PW); if (memory_words < 0) return 1; if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; } +#if !defined(_WIN32) && !defined(__wasm__) + m = vm_memory_map(); memory_mapped = m != 0; + if (!m) +#endif m = calloc((size_t)MW, sizeof *m); #ifdef ZYNATIVE cov = calloc((size_t)MW, 1); diff --git a/src/core/hold.h b/src/core/hold.h new file mode 100644 index 0000000..d4ab1f0 --- /dev/null +++ b/src/core/hold.h @@ -0,0 +1,116 @@ +/* hold blocks (spec 4.16): the host side of a monitor. a function of a hold block + stores op 57 (enter) at its start and op 58 (leave) at its exit, with the block's + rank and the address of its descriptor word. the descriptor points at a table in vm + memory: n, then n pairs (address, words) of the block's shared variables. + + the shared values live here, outside every vm, so all actors (each with its own + memory) see one copy. entering a block from outside takes its lock and copies the + values into the vm's variables; leaving copies them back and unlocks. inside, they + are plain globals. the lock is per block and reentrant per thread (a nested call + only counts), and blocks are taken in rank (declaration) order: entering a block + while holding a later one is an error, which rules out lock-order deadlocks between + blocks. a worker that ends while holding blocks releases them without writing back. + + included by src/core/core.c and src/micro/micro.c; HOLD_THREADS selects pthreads, + otherwise a single machine needs no locks (wasm, bare metal). no malloc. */ +#ifndef HOLD_H +#define HOLD_H +#define HOLD_BLOCKS 256 +#define HOLD_CELLS 65536 +#ifdef HOLD_THREADS +#include <pthread.h> +static pthread_mutex_t hold_lock[HOLD_BLOCKS], hold_meta = PTHREAD_MUTEX_INITIALIZER; +static pthread_once_t hold_once = PTHREAD_ONCE_INIT; +static void hold_init(void) { int i; for (i = 0; i < HOLD_BLOCKS; i++) pthread_mutex_init(&hold_lock[i], 0); } +#define HOLD_TLS _Thread_local +#else +#define HOLD_TLS +#endif +static int64_t hold_cells[HOLD_CELLS]; +static int hold_base[HOLD_BLOCKS], hold_size[HOLD_BLOCKS], hold_ready[HOLD_BLOCKS], hold_next; +static HOLD_TLS int hold_depth[HOLD_BLOCKS], hold_top; /* this thread: nesting per block, highest rank held */ +static HOLD_TLS unsigned char hold_order[HOLD_BLOCKS]; /* ranks held, in the order taken */ +static HOLD_TLS int hold_n; + +/* the table's entries, checked against the machine. -1 when it is malformed */ +static int64_t hold_table(int64_t *mem, uint64_t words, int64_t desc, int64_t *n) +{ + int64_t t, i; + if (desc <= 0 || (uint64_t)desc >= words) return -1; + t = mem[desc]; + if (t <= 0 || (uint64_t)t >= words) return -1; + *n = mem[t]; + if (*n < 0 || *n > 64 || (uint64_t)(t + 1 + 2 * *n) > words) return -1; + for (i = 0; i < *n; i++) { + int64_t a = mem[t + 1 + 2 * i], w = mem[t + 2 + 2 * i]; + if (a <= 0 || w <= 0 || (uint64_t)(a + w) > words) return -1; + } + return t; +} +static void hold_copy(int64_t *mem, int64_t t, int64_t n, int rank, int in) +{ + int64_t i, k, c = hold_base[rank]; + for (i = 0; i < n; i++) { + int64_t a = mem[t + 1 + 2 * i], w = mem[t + 2 + 2 * i]; + for (k = 0; k < w; k++, c++) { if (in) mem[a + k] = hold_cells[c]; else hold_cells[c] = mem[a + k]; } + } +} +/* op 57: 0, or -1 (bad block), or -2 (taken out of rank order; the caller stops) */ +static int64_t hold_enter(int64_t *mem, uint64_t words, int64_t rank, int64_t desc) +{ + int64_t t, n, i, total = 0; + if (rank <= 0 || rank >= HOLD_BLOCKS) return -1; + if (hold_depth[rank]) { hold_depth[rank]++; return 0; } + if (hold_top > rank) return -2; + t = hold_table(mem, words, desc, &n); if (t < 0) return -1; + for (i = 0; i < n; i++) total += mem[t + 2 + 2 * i]; +#ifdef HOLD_THREADS + pthread_once(&hold_once, hold_init); + pthread_mutex_lock(&hold_lock[rank]); + pthread_mutex_lock(&hold_meta); +#endif + if (!hold_ready[rank]) { /* the first entry anywhere: the image's initial values become the shared ones */ + if (hold_next + total > HOLD_CELLS) { +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_meta); pthread_mutex_unlock(&hold_lock[rank]); +#endif + return -1; + } + hold_base[rank] = hold_next; hold_size[rank] = (int)total; hold_next += (int)total; hold_ready[rank] = 1; + hold_copy(mem, t, n, (int)rank, 0); + } +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_meta); +#endif + hold_copy(mem, t, n, (int)rank, 1); + hold_depth[rank] = 1; hold_order[hold_n++] = (unsigned char)rank; hold_top = (int)rank; + return 0; +} +/* op 58 */ +static int64_t hold_leave(int64_t *mem, uint64_t words, int64_t rank, int64_t desc) +{ + int64_t t, n; + if (rank <= 0 || rank >= HOLD_BLOCKS || !hold_depth[rank]) return -1; + if (--hold_depth[rank]) return 0; + t = hold_table(mem, words, desc, &n); + if (t >= 0) hold_copy(mem, t, n, (int)rank, 0); + if (hold_n > 0 && hold_order[hold_n - 1] == rank) hold_n--; + hold_top = hold_n ? hold_order[hold_n - 1] : 0; +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_lock[rank]); +#endif + return 0; +} +/* a machine that ends while inside blocks gives them up, without writing back */ +static void hold_release_all(void) +{ + while (hold_n > 0) { + int r = hold_order[--hold_n]; + hold_depth[r] = 0; +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_lock[r]); +#endif + } + hold_top = 0; +} +#endif diff --git a/src/core/io_linux.c b/src/core/io_linux.c index aaf6a18..c3e05dd 100644 --- a/src/core/io_linux.c +++ b/src/core/io_linux.c @@ -1,3 +1,4 @@ +#include <stdlib.h> #include <stdio.h> #include <string.h> #ifdef _WIN32 @@ -25,7 +26,11 @@ char Q(void){int c=getchar();J=pk();return c;} one would wait for a message that comes only after the answer. so J is 1 only when the end is known, and 0 when more may come. the output is flushed before any read that can wait, for the same reason. */ -static VM_TLS unsigned char ib[65536]; +#define IBN 65536 +/* each vm thread's buffer is allocated when it first reads, so an actor that never + reads standard input carries none */ +static VM_TLS unsigned char *ibp; +#define ib (ibp ? ibp : (ibp = malloc(IBN)) ? ibp : (fputs("zy: out of memory\n", stderr), exit(2), ibp)) static VM_TLS int ip, in, ieof; /* make a byte available. wait is 0 to look without waiting. returns 1 when one is ready */ static int fill(int wait) @@ -37,7 +42,7 @@ static int fill(int wait) struct pollfd f; f.fd = 0; f.events = POLLIN; f.revents = 0; if (poll(&f, 1, 0) <= 0) return 0; /* nothing yet: not the end, not a byte */ } else fflush(stdout); - n = (int)read(0, ib, sizeof ib); + n = (int)read(0, ib, IBN); if (n <= 0) { ieof = 1; return 0; } ip = 0; in = n; return 1; @@ -62,16 +67,20 @@ int input_pending(void) #endif } #ifndef _WIN32 -struct IoSnapshot { unsigned char bytes[65536]; int position, count, ended, flag; }; -size_t io_context_size(void) { return sizeof(struct IoSnapshot); } +/* what a new worker inherits: the bytes read ahead and not consumed yet, and the flags */ +struct IoSnapshot { int position, count, ended, flag; unsigned char bytes[]; }; +size_t io_context_size(void) { return sizeof(struct IoSnapshot) + (size_t)(ip < in ? in - ip : 0); } void io_context_snapshot(void *destination) { struct IoSnapshot *snapshot = destination; - memcpy(snapshot->bytes, ib, sizeof ib); snapshot->position = ip; snapshot->count = in; snapshot->ended = ieof; snapshot->flag = J; + if (ip < in) memcpy(snapshot->bytes, ib + ip, (size_t)(in - ip)); + snapshot->position = ip; snapshot->count = in; snapshot->ended = ieof; snapshot->flag = J; } void io_context_restore(const void *source) { const struct IoSnapshot *snapshot = source; - memcpy(ib, snapshot->bytes, sizeof ib); ip = snapshot->position; in = snapshot->count; ieof = snapshot->ended; J = snapshot->flag; + ip = snapshot->position; in = snapshot->count; ieof = snapshot->ended; J = snapshot->flag; + if (ip < in) memcpy(ib + ip, snapshot->bytes, (size_t)(in - ip)); } +void io_context_release(void) { free(ibp); ibp = 0; } #endif void G(char c){putchar(c);} diff --git a/src/lsp/server.zc b/src/lsp/server.zc index 65881a9..85f3ea5 100644 --- a/src/lsp/server.zc +++ b/src/lsp/server.zc @@ -412,6 +412,7 @@ int gap_out(int from, int to) { return 0; } // is c an operator character, which the highlighter marks as an operator. +int hov_shared; // the hold block of the global being hovered, when it is shared int isopc(int c) { char* s = "+-*/%=<>!&|^~?@#$"; int i = 0; @@ -1725,6 +1726,8 @@ char* kwtext() { if (hname_is("switch")) return "switch (e) { case 1, 2: ... default: ... }~a branch on constant labels. the cases don't fall through, and break leaves early."; if (hname_is("break")) return "break;~leaves the innermost loop or switch."; if (hname_is("continue")) return "continue;~goes on to the next pass of the innermost loop."; + if (hname_is("hold")) return "hold { shared let c; fn a() { } fn b() { } }~a monitor at the top level. its shared variables are one copy for every actor, and only the functions written in the block can use them. each call to one of those functions holds the block's lock for the whole call; a call from one to another doesn't lock again. blocks are entered in the order they are declared."; + if (hname_is("shared")) return "shared let c; / shared int[4] counts;~inside a hold block: a variable every actor sees, used only by the block's functions under its lock. it holds numbers or arrays of numbers, since each actor has its own heap."; if (hname_is("defer")) return "defer e; or defer { }~runs the statement when the enclosing block ends, on every way out of it (falling off the end, return, break, continue). several run last registered first, and what they name is read when they run. return, break and continue are not allowed inside. defer rt.free(p); is the transfer of p for the ownership analysis."; if (hname_is("return")) return "return e; or return;~leaves the function with a value, or leaves a void function. returning an owned local transfers it to the caller automatically."; if (hname_is("enum")) return "enum name { A, B = 5 };~global constants. under typecheck on they are a type of their own, and a member can carry fields to make a constructor."; @@ -1843,6 +1846,7 @@ int hover_named(int i, int o, int kind) { else if (st == run.S_U) hov_text(".u"); else if (st == run.S_D) hov_text(".d"); else if (st == run.S_DEAD) hov_text(".expired"); hov_put(' '); k = o; while (k < end) { hov_put(dc(base, k)); k++; } if (kind == 1) hov_text(" // parameter;"); + else if (kind == 3 && hov_shared > 0) { hov_text(" // shared in hold block "); hov_number(hov_shared); hov_text(": the block's functions use it under its lock;"); } else if (kind == 3) hov_text(" // global;"); else if (kind == 4) hov_text(" // field;"); else hov_text("; // local"); @@ -2004,7 +2008,9 @@ int do_hover(int i) { else if ((v & 15) == run.K_GVAR || (v & 15) == run.K_GARR) { s = run.rsrec[r * run.RS_W + 3]; q = 0; o = 0 - 1; while (q < run.rsn) { if (run.rsrec[q * run.RS_W + 3] == s && lt_find(run.rsrec[q * run.RS_W]) >= 0) { o = run.rsrec[q * run.RS_W]; break; } q++; } + hov_shared = 0; if (s >= 0 && (run.sflag[s] & run.F_SHARED)) hov_shared = run.sflag[s] / run.F_SHARED_RANK; if (o >= 0) hover_named(i, o, 3); else if (v & 16) hover_global(i, 2); else hover_global(i, 1); + hov_shared = 0; } else if (v & 16) hover_global(i, 2); else hover_global(i, 1); } else hover_global(i, 0); @@ -3094,6 +3100,18 @@ int cp_score(char* name) { if (exact) { if (n == i) return 0; return 10 + n - i; } return 100 + score + n - i; } +// a label already offered (a symbol named like a keyword wins over the keyword) +int cp_has(char* name) { + int k = 0, i; + while (k < cp_n) { + i = 0; + while (name[i] && cp_name[k][i] == name[i]) i++; + if (!name[i] && !cp_name[k][i]) return 1; + k++; + } + return 0; +} + int cp_add(char* name, int kind, int dead, int relevance) { int score = cp_score(name); if (score < 0) return 0; @@ -3123,7 +3141,7 @@ int cp_symbol(int s, char* label) { bstr sig = cp_signature(run.sname[s]), doc; int k = 0, kind = 6; hn = 0; if (sig) { while (k < rt.len_s(sig)) { hov_put(rt.at_s(sig, k)); k++; } } - else { if (run.stype[s]) { hov_type(run.stype[s]); hov_put(' '); } hov_text(label); } + else { if ((run.skind[s] == run.K_GVAR || run.skind[s] == run.K_GARR) && (run.sflag[s] & run.F_SHARED)) hov_text("shared "); if (run.stype[s]) { hov_type(run.stype[s]); hov_put(' '); } hov_text(label); } if (run.skind[s] == run.K_FN || run.skind[s] == run.K_ENH) kind = 3; else if (run.skind[s] == run.K_TYPE) kind = 22; else if (run.skind[s] == run.K_CONST) kind = 21; @@ -3264,6 +3282,8 @@ int do_completion(int i) { if (run.skind[s] != run.K_DOC && run.lookup(run.sname[s]) == s && (run.skind[s] != run.K_FN || (run.sflag[s] & (run.F_DEF | run.F_DECL)))) cp_symbol(s, run.sname[s]); s++; } + if (cp_score("hold") >= 0 && !cp_has("hold")) { hn = 0; hov_text("hold { shared let c; fn f() { } }"); if (cp_add("hold", 14, 0, 1)) cp_docs[cp_n - 1] = cp_copy("a monitor: shared variables for every actor, used under the block's lock by the functions written in it"); } + if (cp_score("shared") >= 0 && !cp_has("shared")) { hn = 0; hov_text("shared let name;"); if (cp_add("shared", 14, 0, 1)) cp_docs[cp_n - 1] = cp_copy("inside a hold block: a variable every actor sees, used only by the block's functions"); } run.probe_off = cp_cursor; old = chk_stream; chk_stream = 0; check_doc(i); chk_stream = old; run.probe_off = -1; member = 0; if (run.probe_has == 2 && run.probe_mem && run.probe_mem[0] == run.TY_STRUCT) member = run.probe_mem[2]; diff --git a/src/micro/micro.c b/src/micro/micro.c index 23994dd..e095451 100644 --- a/src/micro/micro.c +++ b/src/micro/micro.c @@ -34,6 +34,10 @@ #include "micro_port.h" #include "fops.h" #include "bops.h" +#ifdef ZY_THREADS +#define HOLD_THREADS +#endif +#include "hold.h" #define NENH 512 #define XOP0 22 #define XPOOL 4096 @@ -537,6 +541,11 @@ static int run_from(int p, int s, int r) /* poll: a streaming port suspends once, so its host can deliver input that is waiting */ result = micro_syscall(m, b, 0, 0, 0); if (result == MICRO_WAIT) { sus_p = p - 1; sus_s = s; sus_r = r; return MICRO_SUSPENDED; } + } else if (b == 57) { + result = hold_enter(m, mw, m[a + 1], m[a + 2]); + if (result == -2) { micro_msg("zy: a hold block was entered while a later one was held; hold blocks are entered in the order they are declared\n"); context_call(7, 2, 0, 0, s + 2, r); return 1; } /* a trap: exit status 2 */ + } else if (b == 58) { + result = hold_leave(m, mw, m[a + 1], m[a + 2]); } else result = context_call(b, m[a + 1], m[a + 2], m[a + 3], s + 2, r); m[a] = result; s += 2; @@ -647,6 +656,11 @@ extern size_t micro_context_port_size(void); extern void micro_context_port_save(void *); extern void micro_context_port_restore(const void *); extern void micro_context_port_free(void); +#ifndef _WIN32 +extern Cell *micro_mem_clone(void); +extern void micro_mem_adopt(Cell *mem, UCell words); +extern void micro_mem_release(Cell *mem); +#endif #define CONTEXT_HANDLE_LO 0x40000000 typedef struct ContextChannel { Cell handle; int fd; struct ContextChannel *next; } ContextChannel; typedef struct VmContext { @@ -658,6 +672,7 @@ typedef struct VmContext { UCell words, limit, initial; int sp, rp, exit_code, requested_exit, enhancement_count, input_flag, input_known; Cell *memory; + int memory_mapped; void *port; Xop operations[XPOOL]; Enh enhancements[NENH]; @@ -683,15 +698,26 @@ static int context_close(Cell handle) { } static void context_snapshot_free(VmContext *context) { free(context->port); - if (context->memory) { free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } + if (context->memory) { +#ifndef _WIN32 + if (context->memory_mapped) micro_mem_release(context->memory); else +#endif + free(context->memory); + atomic_fetch_sub(&context_memory_live, 1); + } } static void *context_worker(void *opaque) { VmContext *context = opaque; context_current = context; context->worker_channel.next = context_channels; context_channels = &context->worker_channel; micro_context_port_restore(context->port); free(context->port); context->port = 0; memory_limit = context->limit; memory_initial = context->initial; + if (context->memory_mapped) { +#ifndef _WIN32 + micro_mem_adopt(context->memory, context->words); context->memory = 0; +#endif + } if (!mem_grow_words(context->words)) { context->exit_code = -1; context_close(context->worker_channel.handle); return 0; } - memcpy(m, context->memory, (size_t)context->words * sizeof(Cell)); free(context->memory); context->memory = 0; + if (context->memory) { memcpy(m, context->memory, (size_t)context->words * sizeof(Cell)); free(context->memory); context->memory = 0; } stack_control = context->stack; jflag = context->input_flag; jknown = context->input_known; xuse = context->enhancement_count; memcpy(xops, context->operations, sizeof xops); memcpy(enh, context->enhancements, sizeof enh); int sp = context->sp, rp = context->rp; Cell entry = context->entry; @@ -701,7 +727,7 @@ static void *context_worker(void *opaque) { if (!context->requested_exit) context->exit_code = context_cancelled() ? 0 : result ? -1 : inmem(final_s) ? (int)m[final_s] : 0; context_close(context->worker_channel.handle); while (context_children) context_join(context_children->token, 1); - micro_context_port_free(); m = 0; atomic_fetch_sub(&context_memory_live, 1); context_current = 0; return 0; + hold_release_all(); micro_context_port_free(); m = 0; atomic_fetch_sub(&context_memory_live, 1); context_current = 0; return 0; } static Cell context_start(Cell entry, Cell channel, int sp, int rp) { if (!inmem(entry) || !inmem(channel) || (entry >= HEAP_LO && (!inmem(entry - 3) || m[entry - 3] != (entry ^ 0x2B7E1516)))) return -1; @@ -714,12 +740,17 @@ static Cell context_start(Cell entry, Cell channel, int sp, int rp) { context->parent_channel.handle = CONTEXT_HANDLE_LO + atomic_fetch_add(&context_sequence, 1) + 1; context->worker_channel.handle = CONTEXT_HANDLE_LO + atomic_fetch_add(&context_sequence, 1) + 1; context->parent_channel.fd = fd[0]; context->worker_channel.fd = fd[1]; atomic_init(&context->stop, 0); - context->port = malloc(micro_context_port_size()); context->memory = malloc((size_t)mw * sizeof(Cell)); + context->port = malloc(micro_context_port_size()); +#ifndef _WIN32 + context->memory = micro_mem_clone(); context->memory_mapped = context->memory != 0; + if (!context->memory) +#endif + context->memory = malloc((size_t)mw * sizeof(Cell)); if (context->memory) atomic_fetch_add(&context_memory_live, 1); if (!context->port || !context->memory) goto failed; - micro_context_port_save(context->port); memcpy(context->memory, m, (size_t)mw * sizeof(Cell)); context->memory[channel] = context->worker_channel.handle; + micro_context_port_save(context->port); if (!context->memory_mapped) memcpy(context->memory, m, (size_t)mw * sizeof(Cell)); context->memory[channel] = context->worker_channel.handle; memcpy(context->operations, xops, sizeof xops); memcpy(context->enhancements, enh, sizeof enh); - pthread_attr_t attributes; pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 16u << 20); + pthread_attr_t attributes; pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 4u << 20); /* the vm keeps its own stacks in vm memory; 4 MB of address space covers the deepest native recursion (jit compile, o3) */ int result = pthread_create(&context->thread, &attributes, context_worker, context); pthread_attr_destroy(&attributes); if (result) goto failed; context->parent_channel.next = context_channels; context_channels = &context->parent_channel; @@ -776,6 +807,13 @@ int micro_run(void) micro_msg("zy: genome does not fit\n"); return 1; } +#ifdef MICRO_PORT_IMAGE + /* the port may hand over an image of its own in place of the embedded one + (the wasm bootstrap seed does, to run a freshly built compiler) */ + i = micro_port_image(m, (UCell)GORG, (UCell)FBOT); + if (i < 0) { micro_msg("zy: bad image\n"); return 1; } + if (i == 0) +#endif for (i = 0; i < GLEN; i++) m[GORG + i] = genome_word(i); jknown = 0; return run(GORG); diff --git a/src/micro/micro_port.h b/src/micro/micro_port.h index 9f77987..1653250 100644 --- a/src/micro/micro_port.h +++ b/src/micro/micro_port.h @@ -56,6 +56,9 @@ int micro_eof(void); #define MICRO_WAIT (-2) #define MICRO_SUSPENDED 3 int micro_run(void); +/* with MICRO_PORT_IMAGE defined, the port's micro_port_image(mem, org, end) may load an + image of its own at mem[org..end): 1 when it did, 0 to use the embedded genome, -1 on error. */ +int micro_port_image(Cell *mem, UCell org, UCell end); int micro_resume(void); /* write the low 8 bits. */ diff --git a/src/micro/port_linux.c b/src/micro/port_linux.c index 6245f4f..a39e3e8 100644 --- a/src/micro/port_linux.c +++ b/src/micro/port_linux.c @@ -38,16 +38,38 @@ #define VM_TLS #endif -/* the machine's memory: a block that grows by realloc, zero in the new part. the - base can move on a grow, which micro.c takes into account (it re-reads m). */ +/* the machine's memory. with threads (actors) on a posix host it is one large + reservation that grows in place, so that a worker's memory can be a + copy-on-write map of a snapshot of its parent (micro_mem_clone); a machine that + outgrows the reservation moves to a block that grows by realloc. without threads it + is a block that grows by realloc, zero in the new part. the base can move on a + grow, which micro.c takes into account (it re-reads m). */ static VM_TLS Cell *lmem; static VM_TLS UCell lcap; +#if defined(ZY_THREADS) && !defined(_WIN32) +#include <sys/mman.h> +#ifdef __linux__ +#include <sys/syscall.h> +#endif +#define RESW ((UCell)1 << 28) /* words reserved per machine: 2 GB of address space */ +static VM_TLS int lres; /* lmem is a reservation of RESW words */ +static Cell *res_map(void) { void *p = mmap(0, RESW * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); return p == MAP_FAILED ? 0 : p; } +#endif Cell *micro_mem_alloc(UCell words) { Cell *n; if (words <= lcap && lmem) return lmem; if (words > ((UCell)-1) / sizeof(Cell)) return NULL; +#if defined(ZY_THREADS) && !defined(_WIN32) + if (!lmem && words <= RESW && (lmem = res_map())) { lres = 1; lcap = words; return lmem; } + if (lres && words <= RESW) { lcap = words; return lmem; } /* the reservation reads zero past lcap */ + if (lres) { + n = malloc(words * sizeof(Cell)); if (!n) return NULL; + memcpy(n, lmem, lcap * sizeof(Cell)); memset(n + lcap, 0, (words - lcap) * sizeof(Cell)); + munmap(lmem, RESW * sizeof(Cell)); lres = 0; lmem = n; lcap = words; return n; + } +#endif n = realloc(lmem, words * sizeof(Cell)); if (!n) return NULL; memset(n + lcap, 0, (words - lcap) * sizeof(Cell)); @@ -56,15 +78,62 @@ Cell *micro_mem_alloc(UCell words) return n; } +#if defined(ZY_THREADS) && !defined(_WIN32) +/* a worker's memory is a private map of an unchanging snapshot of its parent's, + plus the pages the parent changed since; see the same scheme in src/core/core.c */ +#define SNAP_PAGE 512 +static VM_TLS int snap_fd = -1; static VM_TLS Cell *snap_view; static VM_TLS UCell snap_words; +static void snap_drop(void) { if (snap_view) munmap(snap_view, snap_words * sizeof(Cell)); if (snap_fd >= 0) close(snap_fd); snap_fd = -1; snap_view = 0; snap_words = 0; } +static int snap_take(void) +{ + UCell words = (lcap + SNAP_PAGE - 1) / SNAP_PAGE * SNAP_PAGE; size_t bytes = words * sizeof(Cell), done = 0; int fd = -1; void *view; +#ifdef __linux__ +#ifdef SYS_memfd_create + fd = (int)syscall(SYS_memfd_create, "zym-snapshot", 1u); +#endif +#else + { char name[64]; snprintf(name, sizeof name, "/zym-%d-%p", (int)getpid(), (void *)&name); fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); if (fd >= 0) shm_unlink(name); } +#endif + if (fd < 0) return -1; + if (ftruncate(fd, (off_t)bytes) < 0) { close(fd); return -1; } + while (done < bytes) { ssize_t k = pwrite(fd, (char *)lmem + done, bytes - done, (off_t)done); if (k <= 0) { close(fd); return -1; } done += (size_t)k; } + view = mmap(0, bytes, PROT_READ, MAP_SHARED, fd, 0); + if (view == MAP_FAILED) { close(fd); return -1; } + if (mmap(lmem, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, fd, 0) == MAP_FAILED) { munmap(view, bytes); close(fd); return -1; } + snap_drop(); snap_fd = fd; snap_view = view; snap_words = words; return 0; +} +Cell *micro_mem_clone(void) +{ + UCell p, dirty = 0; Cell *c; + if (!lres || !lmem) return 0; + if (snap_fd >= 0) { + for (p = 0; p < lcap; p += SNAP_PAGE) if (p >= snap_words || memcmp(lmem + p, snap_view + p, SNAP_PAGE * sizeof(Cell))) dirty++; + if (dirty * 4 > snap_words / SNAP_PAGE) snap_drop(); + } + if (snap_fd < 0 && snap_take() < 0) return 0; + c = res_map(); if (!c) return 0; + if (mmap(c, snap_words * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, snap_fd, 0) == MAP_FAILED) { munmap(c, RESW * sizeof(Cell)); return 0; } + for (p = 0; p < lcap; p += SNAP_PAGE) { + UCell k = lcap - p < SNAP_PAGE ? lcap - p : SNAP_PAGE; + if (p >= snap_words || memcmp(lmem + p, snap_view + p, k * sizeof(Cell))) memcpy(c + p, lmem + p, k * sizeof(Cell)); + } + return c; +} +void micro_mem_adopt(Cell *mem, UCell words) { lmem = mem; lcap = words; lres = 1; } +void micro_mem_release(Cell *mem) { if (mem) munmap(mem, RESW * sizeof(Cell)); } +#endif + #if defined(ZY_THREADS) -static VM_TLS unsigned char input_bytes[65536]; +/* allocated when the machine first reads, so a worker that never reads has none */ +static VM_TLS unsigned char *input_buffer; +#define input_bytes (input_buffer ? input_buffer : (input_buffer = malloc(65536)) ? input_buffer : (fputs("zym: out of memory\n", stderr), exit(2), input_buffer)) static VM_TLS int input_position, input_count, input_ended; static int input_fill(int wait) { if (input_position < input_count) return 1; if (input_ended) return 0; if (!wait) { struct pollfd descriptor = {0, POLLIN, 0}; if (poll(&descriptor, 1, 0) <= 0) return 0; } else fflush(stdout); - int n = (int)read(0, input_bytes, sizeof input_bytes); + int n = (int)read(0, input_bytes, 65536); if (n <= 0) { input_ended = 1; return 0; } input_position = 0; input_count = n; return 1; } @@ -392,22 +461,29 @@ int main(int argc, char **argv) { #ifdef ZY_THREADS struct PortSnapshot { - unsigned char bytes[65536]; int position, count, ended, argc, envc; char **argv, **envp; + unsigned char bytes[]; /* the input read ahead and not consumed yet */ }; -size_t micro_context_port_size(void) { return sizeof(struct PortSnapshot); } +size_t micro_context_port_size(void) { return sizeof(struct PortSnapshot) + (size_t)(input_position < input_count ? input_count - input_position : 0); } void micro_context_port_save(void *destination) { struct PortSnapshot *snapshot = destination; - memcpy(snapshot->bytes, input_bytes, sizeof input_bytes); + if (input_position < input_count) memcpy(snapshot->bytes, input_bytes + input_position, (size_t)(input_count - input_position)); snapshot->position = input_position; snapshot->count = input_count; snapshot->ended = input_ended; snapshot->argc = zy_argc; snapshot->argv = zy_argv; snapshot->envc = zy_envc; snapshot->envp = zy_envp; } void micro_context_port_restore(const void *source) { const struct PortSnapshot *snapshot = source; - memcpy(input_bytes, snapshot->bytes, sizeof input_bytes); input_position = snapshot->position; input_count = snapshot->count; input_ended = snapshot->ended; + if (input_position < input_count) memcpy(input_bytes + input_position, snapshot->bytes, (size_t)(input_count - input_position)); zy_argc = snapshot->argc; zy_argv = snapshot->argv; zy_envc = snapshot->envc; zy_envp = snapshot->envp; } -void micro_context_port_free(void) { free(lmem); lmem = 0; lcap = 0; } +void micro_context_port_free(void) { + free(input_buffer); input_buffer = 0; +#ifndef _WIN32 + if (lres) { munmap(lmem, RESW * sizeof(Cell)); lres = 0; snap_drop(); } else +#endif + free(lmem); + lmem = 0; lcap = 0; +} #endif diff --git a/src/tools/parity.sh b/src/tools/parity.sh index 2d2086d..1b90885 100755 --- a/src/tools/parity.sh +++ b/src/tools/parity.sh @@ -71,7 +71,7 @@ SRCS="src/runtime/rt.zc src/core/core.zc src/runtime/run.zc" if cmp -s "$W/prefix.new" "$W/prefix.zc"; then rm "$W/prefix.new"; else mv "$W/prefix.new" "$W/prefix.zc"; fi nprefix=$(wc -l < "$W/prefix.zc") # what the compiled image depends on besides the program -DEPS="$ZC0 $MKG $W/prefix.zc src/micro/micro.c src/micro/port_linux.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/genome.h build/core/map/memmap.h" +DEPS="$ZC0 $MKG $W/prefix.zc src/micro/micro.c src/micro/port_linux.c src/micro/micro_port.h src/core/fops.h src/core/bops.h src/core/stack.h src/core/hold.h src/core/genome.h build/core/map/memmap.h" # one line of a program's output: newlines as /, control bytes as ?, cut to 60 columns short() { diff --git a/src/tools/zcc b/src/tools/zcc index ba38e97..a53fe01 100755 --- a/src/tools/zcc +++ b/src/tools/zcc @@ -41,7 +41,7 @@ trap cleanup EXIT # the map and the genome for this program only ( cd "$here" && bash src/tools/automap.sh run "$work/run" src/runtime/run.zc "$abs" ) >/dev/null cp "$here/src/micro/micro.c" "$here/src/micro/micro_port.h" "$here/src/micro/port_linux.c" \ - "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$here/src/core/genome.h" "$work/run/" + "$here/src/core/fops.h" "$here/src/core/bops.h" "$here/src/core/stack.h" "$here/src/core/hold.h" "$here/src/core/genome.h" "$work/run/" # static where the toolchain allows it; a plain link otherwise, with a note static=-static |
