diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/core/core.c | 93 |
1 files changed, 75 insertions, 18 deletions
diff --git a/src/core/core.c b/src/core/core.c index af7bba7..bf48b19 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -626,7 +626,38 @@ static VM_TLS Enh enh[NENH]; static VM_TLS int need_flush; #define MAXUOP 4096 -static VM_TLS Uop ub[MAXUOP]; static VM_TLS int un, unv, uerr; /* the program being built */ +/* large per-thread tables live behind a pointer and are mapped on first use, so a + worker thread that never compiles anything pays nothing for them (as static tls + they were zeroed into every thread's stack mapping: about 13 MB per actor) */ +static VM_TLS void **tls_big_slots[48]; static VM_TLS size_t tls_big_sizes[48]; static VM_TLS int tls_big_n; +static void *tls_big(size_t bytes, void **slot) +{ + void *p; +#if !defined(_WIN32) && !defined(__wasm__) + p = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); if (p == MAP_FAILED) p = 0; +#else + p = calloc(1, bytes); +#endif + if (!p) { fprintf(stderr, "zy: out of memory for compiler tables\n"); exit(2); } + if (tls_big_n < 48) { tls_big_slots[tls_big_n] = slot; tls_big_sizes[tls_big_n++] = bytes; } + *slot = p; return p; +} +static void tls_big_free(void) +{ + while (tls_big_n > 0) { + tls_big_n--; +#if !defined(_WIN32) && !defined(__wasm__) + munmap(*tls_big_slots[tls_big_n], tls_big_sizes[tls_big_n]); +#else + free(*tls_big_slots[tls_big_n]); +#endif + *tls_big_slots[tls_big_n] = 0; + } +} +#define TLS_BIG(p) (*((p) ? (p) : (__typeof__(p))tls_big(sizeof *(p), (void **)&(p)))) +static VM_TLS Uop (*ub_tls)[MAXUOP]; +#define ub TLS_BIG(ub_tls) +static VM_TLS int un, unv, uerr; /* the program being built */ static int unew(int op, int a, int b, int c, Cell imm) { Uop *u; @@ -838,7 +869,9 @@ static void umagic(UCell d, UCell *mul, int *add, int *sh) /* hacker's delight, *mul = q2 + 1; *sh = p - 64; } -static VM_TLS Uop ob[MAXUOP]; static VM_TLS int on, onv; +static VM_TLS Uop (*ob_tls)[MAXUOP]; +#define ob TLS_BIG(ob_tls) +static VM_TLS int on, onv; static VM_TLS int *omap, *isk; static VM_TLS Cell *kv; /* old value -> new value; constness of new values */ static int pure(int op) { return !is_memop(op); } static int ispow2(UCell x) { return x && !(x & (x - 1)); } @@ -939,8 +972,8 @@ static int oe(int op, int a, int b, int c, Cell imm) return raw(op, a, b, c, 0); } static void optimizer_storage(void) { - static VM_TLS int omap_[MAXUOP + 16], isk_[MAXUOP * 4]; static VM_TLS Cell kv_[MAXUOP * 4]; - omap = omap_; isk = isk_; kv = kv_; + static VM_TLS int (*omap_)[MAXUOP + 16], (*isk_)[MAXUOP * 4]; static VM_TLS Cell (*kv_)[MAXUOP * 4]; + omap = TLS_BIG(omap_); isk = TLS_BIG(isk_); kv = TLS_BIG(kv_); } /* runs the optimizer over ub[0..un) with nin inputs; leaves the result in e */ static int optimize(Enh *e, int nin, int *out, int nout) @@ -1142,12 +1175,18 @@ static VM_TLS int nfn; #define NPG ((code_words >> PGSH) + 1) #define MAXPGN (1 << 19) #define MAXDEP (1 << 18) -static VM_TLS struct { int a, b; } ix_blk[MAXB]; +static VM_TLS struct { int a, b; } (*ix_blk_tls)[MAXB]; +#define ix_blk TLS_BIG(ix_blk_tls) + static VM_TLS int ix_n; static VM_TLS int *pg_head, pg_free, pg_top, pg_live; /* node numbers are stored plus one: 0 is none */ -static VM_TLS struct { int blk, next; } pg_node[MAXPGN]; +static VM_TLS struct { int blk, next; } (*pg_node_tls)[MAXPGN]; +#define pg_node TLS_BIG(pg_node_tls) + static VM_TLS int dep_head[NENH + 1], dep_n; /* slot NENH: depends on every install */ -static VM_TLS struct { int blk, next; } dep_node[MAXDEP]; +static VM_TLS struct { int blk, next; } (*dep_node_tls)[MAXDEP]; +#define dep_node TLS_BIG(dep_node_tls) + static VM_TLS int drop_epoch; /* changes whenever any translation dies */ static void be_drop(int j); /* the backend forgets block j's code and coverage */ static void runmark(Rg *R); /* the backend stops watching the region's words */ @@ -1878,8 +1917,12 @@ static VM_TLS int lv_[RMAXV], lw_[RMAXV]; #define SB 1100 static VM_TLS int skv[2 * SB], stp, slo, shi; static VM_TLS unsigned char skk[2 * SB]; -static VM_TLS int afp[MAXUOP * 4 + 16]; /* value -> 1 + its offset from fp, if it is fp + k */ -static VM_TLS int emap[MAXUOP * 4 + 16]; +static VM_TLS int (*afp_tls)[MAXUOP * 4 + 16]; +#define afp TLS_BIG(afp_tls) +/* value -> 1 + its offset from fp, if it is fp + k */ +static VM_TLS int (*emap_tls)[MAXUOP * 4 + 16]; +#define emap TLS_BIG(emap_tls) + static int rvar(int v) { @@ -2360,7 +2403,11 @@ enum { C_UNSEEN, C_CONST, C_VAR, C_COPY }; /* C_COPY: the variable holds the same value as variable v (both at the same point) */ static int copy_ok(const Rg *R, int j, int k) { return j > 0 && j < RMAXV && k > 0 && k < R->nv && !R->fv[j] && !R->fv[k]; } typedef struct { unsigned char st[RMAXV]; Cell v[RMAXV]; } CIn; -static VM_TLS int g_isk[MAXUOP * 4]; static VM_TLS Cell g_kv[MAXUOP * 4]; static VM_TLS int g_omap[MAXUOP + 16]; +static VM_TLS int (*g_isk_tls)[MAXUOP * 4]; +#define g_isk TLS_BIG(g_isk_tls) +static VM_TLS Cell (*g_kv_tls)[MAXUOP * 4]; +#define g_kv TLS_BIG(g_kv_tls) +static VM_TLS int g_omap[MAXUOP + 16]; static int cjoin(CIn *d, const CIn *s) { int j, ch = 0; @@ -3963,7 +4010,7 @@ static int spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4) return -1; un = 0; unv = nin; uerr = 0; for (j = 0; j < nin; j++) mp[j] = j; @@ -5237,9 +5284,13 @@ static Blk lcompile(int p) #if !defined(ZYLLVM) && !defined(ZYJIT_X64) static VM_TLS unsigned char *rcov; /* 1 on the words of a lifted region */ #endif -static VM_TLS Cell rvals[MAXUOP * 4 + 16]; +static VM_TLS Cell (*rvals_tls)[MAXUOP * 4 + 16]; +#define rvals TLS_BIG(rvals_tls) + #define RPOOL 40 -static VM_TLS Cell rpool[RPOOL][MAXUOP * 4 + 16]; +static VM_TLS Cell (*rpool_tls)[RPOOL][MAXUOP * 4 + 16]; +#define rpool TLS_BIG(rpool_tls) + static VM_TLS int rpool_n; static int rdrop(int a) /* a store landed on region code: drop those regions */ { @@ -6090,7 +6141,11 @@ static void float_op(int i, int k) static const int xreg[] = { r_si, r_di, r_8, r_9, r_10, r_11, r_bp }; #define XNREG 7 #define XMAXOP 400 -static VM_TLS int xkc[MAXUOP * 4 + 16]; static VM_TLS Cell xkv[MAXUOP * 4 + 16]; +static VM_TLS int (*xkc_tls)[MAXUOP * 4 + 16]; +#define xkc TLS_BIG(xkc_tls) +static VM_TLS Cell (*xkv_tls)[MAXUOP * 4 + 16]; +#define xkv TLS_BIG(xkv_tls) + static VM_TLS signed char *xr; static int x64_op_ok(int op) { @@ -6110,7 +6165,7 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) { Enh *e = &enh[s]; int nin = e->nin - nk, i, j, out[8], na; - static VM_TLS int mp[MAXUOP * 4 + 16]; + static VM_TLS int (*mp_tls)[MAXUOP * 4 + 16]; int *mp = TLS_BIG(mp_tls); if (e->nv > MAXUOP * 4 || e->n > XMAXOP) return -1; for (i = 0; i < e->n; i++) if (!x64_op_ok(e->u[i].op)) return -1; un = 0; unv = nin; uerr = 0; @@ -6131,7 +6186,9 @@ static int x64_spec_enh(int s, Cell *ks, int nk, Enh *te) return 0; } /* the register value v lives in, or scratch with the constant v loaded into it */ -static VM_TLS int xvh[MAXUOP * 4 + 16]; /* regions: 1 + the var a value reads at block entry, 0 if none */ +static VM_TLS int (*xvh_tls)[MAXUOP * 4 + 16]; +#define xvh TLS_BIG(xvh_tls) +/* regions: 1 + the var a value reads at block entry, 0 if none */ static VM_TLS Rg *xR; static VM_TLS Cell xsyn[RMAXV]; /* the home of a var the lifter made (a hoisted value), which has no frame slot */ static int xv(int v, int scratch) @@ -6258,7 +6315,7 @@ static Cell xev(long op, Cell a, Cell b, Cell c) { return ev((int)op, a, b, c); /* a value's last use, registers on the six, -1 if the block doesn't fit or can't be compiled */ static int x64_ralloc(RB *B) { - static VM_TLS int last[MAXUOP * 4 + 16]; + static VM_TLS int (*last_tls)[MAXUOP * 4 + 16]; int *last = TLS_BIG(last_tls); int i, j, k, na, fr = (1 << RXNREG) - 1, n = B->n; if (B->nv > MAXUOP * 4) return -1; for (i = 0; i < B->nv; i++) { last[i] = -1; xkc[i] = 0; xvh[i] = 0; rxr[i] = -1; } @@ -7506,7 +7563,7 @@ static void context_engine_free(void) { free(lcov); free(lt); free(hot); #endif #endif - if (memory_mapped) vm_memory_unmap(m); else free(m); m = 0; snap_drop(); atomic_fetch_sub(&context_memory_live, 1); + if (memory_mapped) vm_memory_unmap(m); else free(m); m = 0; snap_drop(); tls_big_free(); atomic_fetch_sub(&context_memory_live, 1); } static void *context_worker(void *opaque) { VmContext *context = opaque; |