diff options
Diffstat (limited to 'src/core')
| -rw-r--r-- | src/core/core.c | 250 | ||||
| -rw-r--r-- | src/core/hold.h | 116 | ||||
| -rw-r--r-- | src/core/io_linux.c | 21 |
3 files changed, 344 insertions, 43 deletions
diff --git a/src/core/core.c b/src/core/core.c index ce9e5ca..7d1b91e 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -74,6 +74,10 @@ #include "memmap.h" #include "fops.h" #include "bops.h" +#if !defined(_WIN32) && !defined(__wasm__) +#define HOLD_THREADS +#endif +#include "hold.h" #ifndef JTYPE #define JTYPE char @@ -256,6 +260,7 @@ static atomic_int context_records_live, context_code_live, context_memory_live; size_t io_context_size(void); void io_context_snapshot(void *destination); void io_context_restore(const void *source); +void io_context_release(void); #else static VM_TLS int context_default_stop; #endif @@ -294,6 +299,12 @@ static Cell hostcall(Cell op, Cell a, Cell b, Cell c) #else return -1; #endif + case 57: { + Cell r = hold_enter(m, (uint64_t)MW, a, b); + if (r == -2) { fflush(stdout); fprintf(stderr, "zy: hold block %d was entered while block %d was held; hold blocks are entered in the order they are declared\n", (int)a, hold_top); exit(2); } + return r; + } + case 58: return hold_leave(m, (uint64_t)MW, a, b); case 56: return cb_r; /* the return stack pointer as of the gate store: the word after the top return address */ case 1: if (zcstr(a, path, 1024) < 0) return -1; @@ -622,11 +633,44 @@ typedef struct { Uop *u; signed char *reg; /* physical register per value, for the jit; 0 if it didn't fit */ } Enh; -static VM_TLS Enh enh[NENH]; +/* large per-thread tables live behind a pointer and are mapped on first use, so a + worker thread that never compiles anything pays nothing for them (as static tls + they were zeroed into every thread's stack mapping: about 13 MB per actor) */ +static VM_TLS void **tls_big_slots[48]; static VM_TLS size_t tls_big_sizes[48]; static VM_TLS int tls_big_n; +static void *tls_big(size_t bytes, void **slot) +{ + void *p; +#if !defined(_WIN32) && !defined(__wasm__) + p = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); if (p == MAP_FAILED) p = 0; +#else + p = calloc(1, bytes); +#endif + if (!p) { fprintf(stderr, "zy: out of memory for compiler tables\n"); exit(2); } + if (tls_big_n < 48) { tls_big_slots[tls_big_n] = slot; tls_big_sizes[tls_big_n++] = bytes; } + *slot = p; return p; +} +static void tls_big_free(void) +{ + while (tls_big_n > 0) { + tls_big_n--; +#if !defined(_WIN32) && !defined(__wasm__) + munmap(*tls_big_slots[tls_big_n], tls_big_sizes[tls_big_n]); +#else + free(*tls_big_slots[tls_big_n]); +#endif + *tls_big_slots[tls_big_n] = 0; + } +} +#define TLS_BIG(p) (*((p) ? (p) : (__typeof__(p))tls_big(sizeof *(p), (void **)&(p)))) +static VM_TLS Enh (*enh_tls)[NENH]; +#define enh TLS_BIG(enh_tls) + static VM_TLS int need_flush; #define MAXUOP 4096 -static VM_TLS Uop ub[MAXUOP]; static VM_TLS int un, unv, uerr; /* the program being built */ +static VM_TLS Uop (*ub_tls)[MAXUOP]; +#define ub TLS_BIG(ub_tls) +static VM_TLS int un, unv, uerr; /* the program being built */ static int unew(int op, int a, int b, int c, Cell imm) { Uop *u; @@ -838,7 +882,9 @@ static void umagic(UCell d, UCell *mul, int *add, int *sh) /* hacker's delight, *mul = q2 + 1; *sh = p - 64; } -static VM_TLS Uop ob[MAXUOP]; static VM_TLS int on, onv; +static VM_TLS Uop (*ob_tls)[MAXUOP]; +#define ob TLS_BIG(ob_tls) +static VM_TLS int on, onv; static VM_TLS int *omap, *isk; static VM_TLS Cell *kv; /* old value -> new value; constness of new values */ static int pure(int op) { return !is_memop(op); } static int ispow2(UCell x) { return x && !(x & (x - 1)); } @@ -939,8 +985,8 @@ static int oe(int op, int a, int b, int c, Cell imm) return raw(op, a, b, c, 0); } static void optimizer_storage(void) { - static VM_TLS int omap_[MAXUOP + 16], isk_[MAXUOP * 4]; static VM_TLS Cell kv_[MAXUOP * 4]; - omap = omap_; isk = isk_; kv = kv_; + static VM_TLS int (*omap_)[MAXUOP + 16], (*isk_)[MAXUOP * 4]; static VM_TLS Cell (*kv_)[MAXUOP * 4]; + omap = TLS_BIG(omap_); isk = TLS_BIG(isk_); kv = TLS_BIG(kv_); } /* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */ static int optimize(Enh *e, int nin, int *out, int nout) @@ -1121,11 +1167,19 @@ typedef struct { unsigned gen; /* bumped every time the region's code is dropped: a copy made from an older one is stale */ } Rg; #define MAXRG 4096 -static VM_TLS Rg *rgs[MAXRG]; +static VM_TLS Rg * (*rgs_tls)[MAXRG]; +#define rgs TLS_BIG(rgs_tls) + static VM_TLS int nrg; -static VM_TLS int fn_pc[MAXRG]; -static VM_TLS Rg *fn_rg_[MAXRG]; -static VM_TLS Rg *fn_cache[1024]; +static VM_TLS int (*fn_pc_tls)[MAXRG]; +#define fn_pc TLS_BIG(fn_pc_tls) + +static VM_TLS Rg * (*fn_rg__tls)[MAXRG]; +#define fn_rg_ TLS_BIG(fn_rg__tls) + +static VM_TLS Rg * (*fn_cache_tls)[1024]; +#define fn_cache TLS_BIG(fn_cache_tls) + static VM_TLS int nfn; /* ---- bookkeeping for translated code, shared by the backends ---- @@ -1142,12 +1196,18 @@ static VM_TLS int nfn; #define NPG ((code_words >> PGSH) + 1) #define MAXPGN (1 << 19) #define MAXDEP (1 << 18) -static VM_TLS struct { int a, b; } ix_blk[MAXB]; +static VM_TLS struct { int a, b; } (*ix_blk_tls)[MAXB]; +#define ix_blk TLS_BIG(ix_blk_tls) + static VM_TLS int ix_n; static VM_TLS int *pg_head, pg_free, pg_top, pg_live; /* node numbers are stored plus one: 0 is none */ -static VM_TLS struct { int blk, next; } pg_node[MAXPGN]; +static VM_TLS struct { int blk, next; } (*pg_node_tls)[MAXPGN]; +#define pg_node TLS_BIG(pg_node_tls) + static VM_TLS int dep_head[NENH + 1], dep_n; /* slot NENH: depends on every install */ -static VM_TLS struct { int blk, next; } dep_node[MAXDEP]; +static VM_TLS struct { int blk, next; } (*dep_node_tls)[MAXDEP]; +#define dep_node TLS_BIG(dep_node_tls) + static VM_TLS int drop_epoch; /* changes whenever any translation dies */ static void be_drop(int j); /* the backend forgets block j's code and coverage */ static void runmark(Rg *R); /* the backend stops watching the region's words */ @@ -1770,7 +1830,9 @@ static Cell ffi_cbmake(Cell entry, Cell sig) /* a native call the o3 lifter has resolved: the micro-ops narg and ncall carry an index into this table. imm of narg is the argument's position */ typedef struct { void *f; Cell sig; int n; char name[48]; } Ncd; -static VM_TLS Ncd ncd[256]; +static VM_TLS Ncd (*ncd_tls)[256]; +#define ncd TLS_BIG(ncd_tls) + static VM_TLS int nncd; static int ncd_get(Cell blk) { @@ -1878,8 +1940,12 @@ static VM_TLS int lv_[RMAXV], lw_[RMAXV]; #define SB 1100 static VM_TLS int skv[2 * SB], stp, slo, shi; static VM_TLS unsigned char skk[2 * SB]; -static VM_TLS int afp[MAXUOP * 4 + 16]; /* value -> 1 + its offset from fp, if it is fp + k */ -static VM_TLS int emap[MAXUOP * 4 + 16]; +static VM_TLS int (*afp_tls)[MAXUOP * 4 + 16]; +#define afp TLS_BIG(afp_tls) +/* value -> 1 + its offset from fp, if it is fp + k */ +static VM_TLS int (*emap_tls)[MAXUOP * 4 + 16]; +#define emap TLS_BIG(emap_tls) + static int rvar(int v) { @@ -2360,7 +2426,13 @@ enum { C_UNSEEN, C_CONST, C_VAR, C_COPY }; /* C_COPY: the variable holds the same value as variable v (both at the same point) */ static int copy_ok(const Rg *R, int j, int k) { return j > 0 && j < RMAXV && k > 0 && k < R->nv && !R->fv[j] && !R->fv[k]; } typedef struct { unsigned char st[RMAXV]; Cell v[RMAXV]; } CIn; -static VM_TLS int g_isk[MAXUOP * 4]; static VM_TLS Cell g_kv[MAXUOP * 4]; static VM_TLS int g_omap[MAXUOP + 16]; +static VM_TLS int (*g_isk_tls)[MAXUOP * 4]; +#define g_isk TLS_BIG(g_isk_tls) +static VM_TLS Cell (*g_kv_tls)[MAXUOP * 4]; +#define g_kv TLS_BIG(g_kv_tls) +static VM_TLS int (*g_omap_tls)[MAXUOP + 16]; +#define g_omap TLS_BIG(g_omap_tls) + static int cjoin(CIn *d, const CIn *s) { int j, ch = 0; @@ -3313,7 +3385,9 @@ static VM_TLS void **tab; /* vm address -> translation */ typedef struct { void *stub; int fslots; int argc; } FnNat; static VM_TLS FnNat *fn_nat; #define LINKN 1024 -static VM_TLS void *linkstk[LINKN]; +static VM_TLS void * (*linkstk_tls)[LINKN]; +#define linkstk TLS_BIG(linkstk_tls) + static VM_TLS Cell jit_depth; #endif static void be_drop(int j) { tab[ix_blk[j].a] = 0; cdec(cov, ix_blk[j].a, ix_blk[j].b); if (trace) fprintf(stderr, "jit: dropped %d..%d\n", ix_blk[j].a, ix_blk[j].b - 1); } @@ -3963,7 +4037,7 @@ static int spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4) return -1; un = 0; unv = nin; uerr = 0; for (j = 0; j < nin; j++) mp[j] = j; @@ -5237,9 +5311,13 @@ static Blk lcompile(int p) #if !defined(ZYLLVM) && !defined(ZYJIT_X64) static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */ #endif -static VM_TLS Cell rvals[MAXUOP * 4 + 16]; +static VM_TLS Cell (*rvals_tls)[MAXUOP * 4 + 16]; +#define rvals TLS_BIG(rvals_tls) + #define RPOOL 40 -static VM_TLS Cell rpool[RPOOL][MAXUOP * 4 + 16]; +static VM_TLS Cell (*rpool_tls)[RPOOL][MAXUOP * 4 + 16]; +#define rpool TLS_BIG(rpool_tls) + static VM_TLS int rpool_n; static int rdrop(int a) /* a store landed on region code: drop those regions */ { @@ -6090,7 +6168,11 @@ static void float_op(int i, int k) static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp }; #define XNREG 7 #define XMAXOP 400 -static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16]; +static VM_TLS int (*xkc_tls)[MAXUOP * 4 + 16]; +#define xkc TLS_BIG(xkc_tls) +static VM_TLS Cell (*xkv_tls)[MAXUOP * 4 + 16]; +#define xkv TLS_BIG(xkv_tls) + static VM_TLS signed char *xr; static int x64_op_ok(int op) { @@ -6110,7 +6192,7 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1; for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1; un = 0; unv = nin; uerr = 0; @@ -6131,7 +6213,9 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS int (*xvh_tls)[MAXUOP * 4 + 16]; +#define xvh TLS_BIG(xvh_tls) +/* regions: 1 + the var a value reads at block entry, 0 if none */ static VM_TLS Rg *xR; static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ static int xv(int v, int scratch) @@ -6250,7 +6334,9 @@ static void x64_enh(Enh *e) under ZYO3INTERP=1. */ static VM_TLS int x64_o3interp; static void perfmap_note(JW *start, JW *end, const char *kind, int a, int b); -static VM_TLS signed char rxr[MAXUOP * 4 + 16]; +static VM_TLS signed char (*rxr_tls)[MAXUOP * 4 + 16]; +#define rxr TLS_BIG(rxr_tls) + #define RXNREG 6 static void rsave(void) { int j; for (j = 0; j < RXNREG; j++) push_r(xreg[j]); } static void rrest(void) { int j; for (j = RXNREG - 1; j >= 0; j--) pop_r(xreg[j]); } @@ -6258,7 +6344,7 @@ static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); /* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ static int x64_ralloc(RB *B) { - static VM_TLS int last[MAXUOP * 4 + 16]; + static VM_TLS int (*last_tls)[MAXUOP * 4 + 16]; int *last = TLS_BIG(last_tls); int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; if (B->nv > MAXUOP * 4) return -1; for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } @@ -7363,6 +7449,85 @@ static int code_range_live(Cell start, Cell count) /* each worker owns a host thread, vm memory and translation state. */ #if !defined(_WIN32) && !defined(__wasm__) +#include <sys/mman.h> +#ifdef __linux__ +#include <sys/syscall.h> +#endif +/* vm memory for a worker is copy-on-write against a snapshot of its parent. + the first spawn writes the parent's live words [0, memory_words) to an anonymous + shared file and maps the parent's memory privately over it at the same address, + so the parent reads the same pages and its later writes stay its own. a child maps + the same file privately, so code, genome and everything the parent has not + touched since are one set of physical pages for all of them, and only the pages + the parent changed since the snapshot are copied in. the file never changes, so a + child never sees a later write of its parent. a parent whose memory has drifted + far from its snapshot takes a fresh one. without a file (or on failure) a spawn + falls back to a private copy. */ +typedef struct { int fd; Cell *view; size_t words; } MemSnap; +static VM_TLS MemSnap mem_snap = { -1, 0, 0 }; +static VM_TLS int memory_mapped; /* m came from vm_memory_map */ +#define SNAP_PAGE 512 /* words per 4 KB page */ +static Cell *vm_memory_map(void) +{ + void *p = mmap(0, (size_t)MW * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); + return p == MAP_FAILED ? 0 : p; +} +static void vm_memory_unmap(Cell *p) { if (p) munmap(p, (size_t)MW * sizeof(Cell)); } +static int snap_file(size_t bytes) +{ + int fd = -1; +#ifdef __linux__ +#ifdef SYS_memfd_create + fd = (int)syscall(SYS_memfd_create, "zy-snapshot", 1u /* MFD_CLOEXEC */); +#endif +#else + char name[64]; snprintf(name, sizeof name, "/zy-%d-%p", (int)getpid(), (void *)&name); + fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); if (fd >= 0) shm_unlink(name); +#endif + if (fd < 0) return -1; + if (ftruncate(fd, (off_t)bytes) < 0) { close(fd); return -1; } + return fd; +} +static void snap_drop(void) +{ + if (mem_snap.view) munmap(mem_snap.view, mem_snap.words * sizeof(Cell)); + if (mem_snap.fd >= 0) close(mem_snap.fd); + mem_snap.fd = -1; mem_snap.view = 0; mem_snap.words = 0; +} +/* pages of [0, words) where m differs from the snapshot */ +static size_t snap_dirty(size_t words) +{ + size_t p, n = 0, lim = words < mem_snap.words ? words : mem_snap.words; + for (p = 0; p < lim; p += SNAP_PAGE) if (memcmp(m + p, mem_snap.view + p, SNAP_PAGE * sizeof(Cell))) n++; + return n + (words > mem_snap.words ? (words - mem_snap.words) / SNAP_PAGE : 0); +} +static int snap_take(void) +{ + size_t words = ((size_t)memory_words + SNAP_PAGE - 1) / SNAP_PAGE * SNAP_PAGE, bytes = words * sizeof(Cell), done = 0; + int fd; void *view, *over; + if (words > (size_t)MW) return -1; + fd = snap_file(bytes); if (fd < 0) return -1; + while (done < bytes) { ssize_t k = pwrite(fd, (char *)m + done, bytes - done, (off_t)done); if (k <= 0) { close(fd); return -1; } done += (size_t)k; } + view = mmap(0, bytes, PROT_READ, MAP_SHARED, fd, 0); + if (view == MAP_FAILED) { close(fd); return -1; } + over = mmap(m, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, fd, 0); + if (over == MAP_FAILED) { munmap(view, bytes); close(fd); return -1; } /* m is untouched when the map fails */ + snap_drop(); mem_snap.fd = fd; mem_snap.view = view; mem_snap.words = words; return 0; +} +/* a child's memory: the snapshot, plus the pages the parent changed since */ +static Cell *snap_clone(void) +{ + size_t p, words = (size_t)memory_words; + Cell *c; + if (mem_snap.fd < 0 || snap_dirty(words) * 4 > mem_snap.words / SNAP_PAGE) { if (snap_take() < 0) return 0; } + c = vm_memory_map(); if (!c) return 0; + if (mmap(c, mem_snap.words * sizeof(Cell), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE, mem_snap.fd, 0) == MAP_FAILED) { vm_memory_unmap(c); return 0; } + for (p = 0; p < words; p += SNAP_PAGE) { + size_t k = words - p < SNAP_PAGE ? words - p : SNAP_PAGE; + if (p >= mem_snap.words || memcmp(m + p, mem_snap.view + p, k * sizeof(Cell))) memcpy(c + p, m + p, k * sizeof(Cell)); + } + return c; +} typedef struct ContextChannel { Cell handle; int fd; struct ContextChannel *next; } ContextChannel; static VM_TLS ContextChannel *context_channels; static atomic_llong context_id_sequence; @@ -7374,6 +7539,7 @@ struct VmContext { Cell token, entry, sp, rp, stack, words, limit, growth, code_size; Cell frame_floor, frame_top, frame_guard; Cell *memory; + int memory_mapped; int exit_code, requested_exit; void *input_snapshot; int argc, envc; @@ -7387,6 +7553,7 @@ struct VmContext { int region_count; unsigned short *entries; unsigned char *entry_indices; + int *entry_at, entry_count; }; static int context_descriptor(Cell handle) { ContextChannel *channel; @@ -7405,7 +7572,7 @@ static void context_snapshot_free(VmContext *context) { int i; for (i = 0; i < NENH; i++) { free(context->enhancements[i].u); free(context->enhancements[i].reg); } for (i = 0; i < context->region_count; i++) free(context->regions[i]); - free(context->input_snapshot); free(context->entries); free(context->entry_indices); if (context->memory) { free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } context->memory = 0; + free(context->input_snapshot); free(context->entries); free(context->entry_indices); free(context->entry_at); if (context->memory) { if (context->memory_mapped) vm_memory_unmap(context->memory); else free(context->memory); atomic_fetch_sub(&context_memory_live, 1); } context->memory = 0; } static int context_cancelled(void) { return context_current && atomic_load_explicit(&context_current->stop, memory_order_acquire); } static void context_exit(int value) { context_current->exit_code = value; context_current->requested_exit = 1; atomic_store_explicit(&context_current->stop, 1, memory_order_release); } @@ -7425,7 +7592,7 @@ static void context_engine_free(void) { free(lcov); free(lt); free(hot); #endif #endif - free(m); m = 0; atomic_fetch_sub(&context_memory_live, 1); + if (memory_mapped) vm_memory_unmap(m); else free(m); m = 0; snap_drop(); tls_big_free(); io_context_release(); atomic_fetch_sub(&context_memory_live, 1); } static void *context_worker(void *opaque) { VmContext *context = opaque; @@ -7434,7 +7601,7 @@ static void *context_worker(void *opaque) { optimizer_storage(); context_current = context; io_context_restore(context->input_snapshot); free(context->input_snapshot); context->input_snapshot = 0; trace = getenv("ZYTRACE") != 0; context->worker_channel.next = context_channels; context_channels = &context->worker_channel; - m = context->memory; context->memory = 0; + m = context->memory; context->memory = 0; memory_mapped = context->memory_mapped; memory_limit = (int)context->limit; memory_words = (int)context->words; memory_growth_limit = (int)context->growth; stack_control = context->stack; frame_floor_word = context->frame_floor; frame_top_word = context->frame_top; frame_guard_word = context->frame_guard; zy_argc = context->argc; zy_argv = context->argv; zy_envc = context->envc; zy_envp = context->envp; @@ -7463,8 +7630,8 @@ static void *context_worker(void *opaque) { memcpy(enh, context->enhancements, sizeof enh); memset(context->enhancements, 0, sizeof context->enhancements); for (i = 0; i < context->region_count; i++) { rgs[i] = context->regions[i]; context->regions[i] = 0; } nrg = context->region_count; context->region_count = 0; - memcpy(o3ent, context->entries, (size_t)code_words * sizeof *o3ent); memcpy(o3ei, context->entry_indices, (size_t)code_words); - free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; + for (i = 0; i < context->entry_count; i++) { o3ent[context->entry_at[i]] = context->entries[i]; o3ei[context->entry_at[i]] = context->entry_indices[i]; } + free(context->entries); context->entries = 0; free(context->entry_indices); context->entry_indices = 0; free(context->entry_at); context->entry_at = 0; #ifdef ZYJIT_X64 if (x64_boot(sp, rp, &context->stop)) { context->exit_code = -1; context_channel_close(context->worker_channel.handle); context_engine_free(); return 0; } atomic_fetch_add(&context_code_live, 1); @@ -7489,7 +7656,7 @@ static void *context_worker(void *opaque) { if (!context->requested_exit) context->exit_code = context_cancelled() ? 0 : stopped(back) ? -1 : inmem(sp) ? (int)m[sp] : 0; context_channel_close(context->worker_channel.handle); while (context_children) context_join(context_children->token, 1); - context_engine_free(); context_current = 0; return 0; + hold_release_all(); context_engine_free(); context_current = 0; return 0; } static Cell context_start(Cell entry, Cell channel) { VmContext *context; int fd[2], i; pthread_attr_t attributes; Cell saved; @@ -7506,12 +7673,17 @@ static Cell context_start(Cell entry, Cell channel) { context->parent_channel.fd = fd[0]; context->worker_channel.fd = fd[1]; atomic_init(&context->stop, 0); context->input_snapshot = malloc(io_context_size()); if (!context->input_snapshot) goto failed; io_context_snapshot(context->input_snapshot); - context->memory = calloc((size_t)memory_limit, sizeof(Cell)); + context->memory = snap_clone(); context->memory_mapped = context->memory != 0; + if (!context->memory) context->memory = calloc((size_t)memory_limit, sizeof(Cell)); if (context->memory) atomic_fetch_add(&context_memory_live, 1); - context->entries = malloc((size_t)code_words * sizeof *o3ent); context->entry_indices = malloc((size_t)code_words); - if (!context->memory || !context->entries || !context->entry_indices) goto failed; - saved = m[channel]; m[channel] = context->worker_channel.handle; memcpy(context->memory, m, (size_t)memory_words * sizeof(Cell)); m[channel] = saved; - memcpy(context->entries, o3ent, (size_t)code_words * sizeof *o3ent); memcpy(context->entry_indices, o3ei, (size_t)code_words); + /* o3 entry points are few and the tables mostly zero: hand over only the set ones */ + { Cell k; context->entry_count = 0; for (k = 0; k < code_words; k++) if (o3ent[k]) context->entry_count++; } + context->entry_at = malloc((size_t)(context->entry_count + 1) * sizeof *context->entry_at); + context->entries = malloc((size_t)(context->entry_count + 1) * sizeof *o3ent); context->entry_indices = malloc((size_t)(context->entry_count + 1)); + if (!context->memory || !context->entries || !context->entry_indices || !context->entry_at) goto failed; + if (context->memory_mapped) context->memory[channel] = context->worker_channel.handle; + else { saved = m[channel]; m[channel] = context->worker_channel.handle; memcpy(context->memory, m, (size_t)memory_words * sizeof(Cell)); m[channel] = saved; } + { Cell k; int j = 0; for (k = 0; k < code_words; k++) if (o3ent[k]) { context->entry_at[j] = (int)k; context->entries[j] = o3ent[k]; context->entry_indices[j] = o3ei[k]; j++; } } for (i = 0; i < NENH; i++) { context->enhancements[i] = enh[i]; context->enhancements[i].u = 0; context->enhancements[i].reg = 0; if (enh[i].u) { context->enhancements[i].u = malloc((size_t)(enh[i].n + 1) * sizeof(Uop)); if (!context->enhancements[i].u) goto failed; memcpy(context->enhancements[i].u, enh[i].u, (size_t)(enh[i].n + 1) * sizeof(Uop)); } @@ -7526,7 +7698,7 @@ static Cell context_start(Cell entry, Cell channel) { memcpy(context->callback_snapshot, ffi_cb, sizeof ffi_cb); memcpy(context->symbol_snapshot, ffi_sym, sizeof ffi_sym); memcpy(context->library_snapshot, ffi_lib, sizeof ffi_lib); context->ffi_symbols = ffi_nsym; context->ffi_libraries = ffi_nlib; context->ffi_libm = ffi_tried_libm; #endif - pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 16u << 20); + pthread_attr_init(&attributes); pthread_attr_setstacksize(&attributes, 4u << 20); /* the vm keeps its own stacks in vm memory; 4 MB of address space covers the deepest native recursion (jit compile, o3) */ i = pthread_create(&context->thread, &attributes, context_worker, context); pthread_attr_destroy(&attributes); if (i) goto failed; context->parent_channel.next = context_channels; context_channels = &context->parent_channel; @@ -7603,6 +7775,10 @@ int main(int argc, char **argv) memory_words = memory_option("TINYVM_MEMORY_WORDS", HEAP_HI > PW ? HEAP_HI : PW, HEAP_HI > PW ? HEAP_HI : PW); if (memory_words < 0) return 1; if (memory_words > memory_limit) { fprintf(stderr, "zy: initial memory exceeds its limit\n"); return 1; } +#if !defined(_WIN32) && !defined(__wasm__) + m = vm_memory_map(); memory_mapped = m != 0; + if (!m) +#endif m = calloc((size_t)MW, sizeof *m); #ifdef ZYNATIVE cov = calloc((size_t)MW, 1); diff --git a/src/core/hold.h b/src/core/hold.h new file mode 100644 index 0000000..d4ab1f0 --- /dev/null +++ b/src/core/hold.h @@ -0,0 +1,116 @@ +/* hold blocks (spec 4.16): the host side of a monitor. a function of a hold block + stores op 57 (enter) at its start and op 58 (leave) at its exit, with the block's + rank and the address of its descriptor word. the descriptor points at a table in vm + memory: n, then n pairs (address, words) of the block's shared variables. + + the shared values live here, outside every vm, so all actors (each with its own + memory) see one copy. entering a block from outside takes its lock and copies the + values into the vm's variables; leaving copies them back and unlocks. inside, they + are plain globals. the lock is per block and reentrant per thread (a nested call + only counts), and blocks are taken in rank (declaration) order: entering a block + while holding a later one is an error, which rules out lock-order deadlocks between + blocks. a worker that ends while holding blocks releases them without writing back. + + included by src/core/core.c and src/micro/micro.c; HOLD_THREADS selects pthreads, + otherwise a single machine needs no locks (wasm, bare metal). no malloc. */ +#ifndef HOLD_H +#define HOLD_H +#define HOLD_BLOCKS 256 +#define HOLD_CELLS 65536 +#ifdef HOLD_THREADS +#include <pthread.h> +static pthread_mutex_t hold_lock[HOLD_BLOCKS], hold_meta = PTHREAD_MUTEX_INITIALIZER; +static pthread_once_t hold_once = PTHREAD_ONCE_INIT; +static void hold_init(void) { int i; for (i = 0; i < HOLD_BLOCKS; i++) pthread_mutex_init(&hold_lock[i], 0); } +#define HOLD_TLS _Thread_local +#else +#define HOLD_TLS +#endif +static int64_t hold_cells[HOLD_CELLS]; +static int hold_base[HOLD_BLOCKS], hold_size[HOLD_BLOCKS], hold_ready[HOLD_BLOCKS], hold_next; +static HOLD_TLS int hold_depth[HOLD_BLOCKS], hold_top; /* this thread: nesting per block, highest rank held */ +static HOLD_TLS unsigned char hold_order[HOLD_BLOCKS]; /* ranks held, in the order taken */ +static HOLD_TLS int hold_n; + +/* the table's entries, checked against the machine. -1 when it is malformed */ +static int64_t hold_table(int64_t *mem, uint64_t words, int64_t desc, int64_t *n) +{ + int64_t t, i; + if (desc <= 0 || (uint64_t)desc >= words) return -1; + t = mem[desc]; + if (t <= 0 || (uint64_t)t >= words) return -1; + *n = mem[t]; + if (*n < 0 || *n > 64 || (uint64_t)(t + 1 + 2 * *n) > words) return -1; + for (i = 0; i < *n; i++) { + int64_t a = mem[t + 1 + 2 * i], w = mem[t + 2 + 2 * i]; + if (a <= 0 || w <= 0 || (uint64_t)(a + w) > words) return -1; + } + return t; +} +static void hold_copy(int64_t *mem, int64_t t, int64_t n, int rank, int in) +{ + int64_t i, k, c = hold_base[rank]; + for (i = 0; i < n; i++) { + int64_t a = mem[t + 1 + 2 * i], w = mem[t + 2 + 2 * i]; + for (k = 0; k < w; k++, c++) { if (in) mem[a + k] = hold_cells[c]; else hold_cells[c] = mem[a + k]; } + } +} +/* op 57: 0, or -1 (bad block), or -2 (taken out of rank order; the caller stops) */ +static int64_t hold_enter(int64_t *mem, uint64_t words, int64_t rank, int64_t desc) +{ + int64_t t, n, i, total = 0; + if (rank <= 0 || rank >= HOLD_BLOCKS) return -1; + if (hold_depth[rank]) { hold_depth[rank]++; return 0; } + if (hold_top > rank) return -2; + t = hold_table(mem, words, desc, &n); if (t < 0) return -1; + for (i = 0; i < n; i++) total += mem[t + 2 + 2 * i]; +#ifdef HOLD_THREADS + pthread_once(&hold_once, hold_init); + pthread_mutex_lock(&hold_lock[rank]); + pthread_mutex_lock(&hold_meta); +#endif + if (!hold_ready[rank]) { /* the first entry anywhere: the image's initial values become the shared ones */ + if (hold_next + total > HOLD_CELLS) { +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_meta); pthread_mutex_unlock(&hold_lock[rank]); +#endif + return -1; + } + hold_base[rank] = hold_next; hold_size[rank] = (int)total; hold_next += (int)total; hold_ready[rank] = 1; + hold_copy(mem, t, n, (int)rank, 0); + } +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_meta); +#endif + hold_copy(mem, t, n, (int)rank, 1); + hold_depth[rank] = 1; hold_order[hold_n++] = (unsigned char)rank; hold_top = (int)rank; + return 0; +} +/* op 58 */ +static int64_t hold_leave(int64_t *mem, uint64_t words, int64_t rank, int64_t desc) +{ + int64_t t, n; + if (rank <= 0 || rank >= HOLD_BLOCKS || !hold_depth[rank]) return -1; + if (--hold_depth[rank]) return 0; + t = hold_table(mem, words, desc, &n); + if (t >= 0) hold_copy(mem, t, n, (int)rank, 0); + if (hold_n > 0 && hold_order[hold_n - 1] == rank) hold_n--; + hold_top = hold_n ? hold_order[hold_n - 1] : 0; +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_lock[rank]); +#endif + return 0; +} +/* a machine that ends while inside blocks gives them up, without writing back */ +static void hold_release_all(void) +{ + while (hold_n > 0) { + int r = hold_order[--hold_n]; + hold_depth[r] = 0; +#ifdef HOLD_THREADS + pthread_mutex_unlock(&hold_lock[r]); +#endif + } + hold_top = 0; +} +#endif diff --git a/src/core/io_linux.c b/src/core/io_linux.c index aaf6a18..c3e05dd 100644 --- a/src/core/io_linux.c +++ b/src/core/io_linux.c @@ -1,3 +1,4 @@ +#include <stdlib.h> #include <stdio.h> #include <string.h> #ifdef _WIN32 @@ -25,7 +26,11 @@ char Q(void){int c=getchar();J=pk();return c;} one would wait for a message that comes only after the answer. so J is 1 only when the end is known, and 0 when more may come. the output is flushed before any read that can wait, for the same reason. */ -static VM_TLS unsigned char ib[65536]; +#define IBN 65536 +/* each vm thread's buffer is allocated when it first reads, so an actor that never + reads standard input carries none */ +static VM_TLS unsigned char *ibp; +#define ib (ibp ? ibp : (ibp = malloc(IBN)) ? ibp : (fputs("zy: out of memory\n", stderr), exit(2), ibp)) static VM_TLS int ip, in, ieof; /* make a byte available. wait is 0 to look without waiting. returns 1 when one is ready */ static int fill(int wait) @@ -37,7 +42,7 @@ static int fill(int wait) struct pollfd f; f.fd = 0; f.events = POLLIN; f.revents = 0; if (poll(&f, 1, 0) <= 0) return 0; /* nothing yet: not the end, not a byte */ } else fflush(stdout); - n = (int)read(0, ib, sizeof ib); + n = (int)read(0, ib, IBN); if (n <= 0) { ieof = 1; return 0; } ip = 0; in = n; return 1; @@ -62,16 +67,20 @@ int input_pending(void) #endif } #ifndef _WIN32 -struct IoSnapshot { unsigned char bytes[65536]; int position, count, ended, flag; }; -size_t io_context_size(void) { return sizeof(struct IoSnapshot); } +/* what a new worker inherits: the bytes read ahead and not consumed yet, and the flags */ +struct IoSnapshot { int position, count, ended, flag; unsigned char bytes[]; }; +size_t io_context_size(void) { return sizeof(struct IoSnapshot) + (size_t)(ip < in ? in - ip : 0); } void io_context_snapshot(void *destination) { struct IoSnapshot *snapshot = destination; - memcpy(snapshot->bytes, ib, sizeof ib); snapshot->position = ip; snapshot->count = in; snapshot->ended = ieof; snapshot->flag = J; + if (ip < in) memcpy(snapshot->bytes, ib + ip, (size_t)(in - ip)); + snapshot->position = ip; snapshot->count = in; snapshot->ended = ieof; snapshot->flag = J; } void io_context_restore(const void *source) { const struct IoSnapshot *snapshot = source; - memcpy(ib, snapshot->bytes, sizeof ib); ip = snapshot->position; in = snapshot->count; ieof = snapshot->ended; J = snapshot->flag; + ip = snapshot->position; in = snapshot->count; ieof = snapshot->ended; J = snapshot->flag; + if (ip < in) memcpy(ib + ip, snapshot->bytes, (size_t)(in - ip)); } +void io_context_release(void) { free(ibp); ibp = 0; } #endif void G(char c){putchar(c);} |