rt: recycle, new_k and own without helper calls or || chains

perf (ZYPERFMAP) on plain trees new/free showed recycle, new_k and own
taking about 70% of the time, mostly in calls to isobj, paylen, bdrop,
bclass and fpush and in the truth values plain code builds for && and ||.
recycle writes isobj, paylen and the class-list push out, calls scan_ptr only
for 8 or more words left, and own and new_k test with nested ifs.
frame_control grows to 22 words: a tag, &dbg and the entries of new_k,
recycle_leaf and own for the x86-64 jit's inline fast paths (next commit).

1 files changed, 58 insertions, 18 deletions

diff --git a/src/runtime/rt.zc b/src/runtime/rt.zc
index 906e56a..149138b 100644
--- a/src/runtime/rt.zc
+++ b/src/runtime/rt.zc
@@ -58,7 +58,7 @@ const MAGIC = 0x2B7E1516;
 
 word frame_floor = FBOT, frame_top = FTOP, frame_node;
 int frame_chunk = FTOP - FBOT, frame_guard = (FTOP - FBOT) >> 2, frame_busy, gc_active;
-word[17] frame_control;
+word[22] frame_control;
 word data_low = DSLO, data_top = DSHI, data_node;
 word return_low = RS0, return_top = MARK0 - 8, return_node;
 const int F_STACK = 1024;
@@ -523,6 +523,9 @@ int heap_init() {
 	frame_control[9] = freel; frame_control[10] = &hstop; frame_control[11] = &hbump;
 	frame_control[12] = &hmax; frame_control[13] = &hold_lo; frame_control[14] = &hold_hi; frame_control[15] = &hcut;
 	frame_control[16] = &gc_active;
+	// for the x86-64 jit's inline new_k, recycle_leaf and own (stack.h SC_FAST)
+	frame_control[17] = 0x2B7E1517; frame_control[18] = &dbg; frame_control[19] = new_k; frame_control[20] = recycle_leaf;
+	frame_control[21] = own;
 	syscall(9, 6, frame_control, 0);
 	return 0;
 }
@@ -800,7 +803,14 @@ word .u new(int n) { return new_t(n, 0); }
 // for the compiler's type area) fall back to the general path.
 word .u new_k(int n, int t, int k) {
 	word b = freel[k], p, nx;
-	if (!b || dbg || (hold_hi && b < hold_hi && b + pow2[k] > hold_lo)) return new_t(n, t);
+	// nested tests rather than one || chain: plain code pays for every truth value it builds
+	if (b == 0) return new_t(n, t);
+	if (dbg) return new_t(n, t);
+	if (hold_hi) {
+		if (b < hold_hi) {
+			if (b + pow2[k] > hold_lo) return new_t(n, t);
+		}
+	}
 	nx = b[5];
 	freel[k] = nx;
 	if (nx) nx[6] = 0;
@@ -860,27 +870,50 @@ unchecked int free(.d p) {
 	return 0;
 }
 // the pointer is the last one. children go first, then the block returns to
-// its size-class list. no sweep, no buddy walk.
+// its size-class list. no sweep, no buddy walk. isobj, paylen and bdrop are
+// written out here: this runs once per freed object, and in a tree free the
+// calls cost more than the work.
 unchecked int recycle(.d p) {
-	word b; int n, i = 0; word w;
-	if ((dbg & 1) && quar(p)) {
-		put_s("zy: double free from line "); put_n(site_line_of(p)); put_c(10);
-		trace_at(1);
-		return 0;
+	word b, w, h; int n, i = 0, f, k;
+	if (dbg & 1) {
+		if (quar(p)) {
+			put_s("zy: double free from line "); put_n(site_line_of(p)); put_c(10);
+			trace_at(1);
+			return 0;
+		}
 	}
-	if (!isobj(p) || (p[-4] & F_SHARE)) return 0;
-	b = p - HDR;
-	if (b[1] & F_HASDROP) dropfree(p);
-	n = paylen(p);
-	b[1] = (b[1] & ~7) | 2;
+	if (p < HEAP_LO + HDR) return 0;
+	if (p >= hbump) return 0;
+	if (p[-3] != (p ^ MAGIC)) return 0;
+	b = p - HDR; f = b[1];
+	if ((f & (F_SHARE | 1)) != 1) return 0;
+	if (f & F_HASDROP) { dropfree(p); f = b[1]; }
+	n = f >> 32;
+	b[1] = (f & ~7) | 2;
 	while (i < n) {
-		i = i + scan_ptr(p + i, n - i);
-		if (i >= n) break;
-		w = p[i]; if (isobj(w) && !(w[-4] & F_SHARE) && w[-2] == p) recycle(w);
+		if (n - i >= 8) {
+			i = i + scan_ptr(p + i, n - i);
+			if (i >= n) break;
+		}
+		w = p[i];
+		// a child this block owns: a live object whose owner word is p
+		if (w >= HEAP_LO + HDR) {
+			if (w < hbump) {
+				if (w[-2] == p) {
+					if (w[-3] == (w ^ MAGIC)) {
+						if ((w[-4] & (F_SHARE | 1)) == 1) recycle(w);
+					}
+				}
+			}
+		}
 		i++;
 	}
 	if (dbg & 2) fill_w(p, 57005, n);
-	bdrop(b);
+	k = (f >> 16) & 31;
+	h = freel[k];
+	b[1] = k * pow2[16]; b[2] = 0; b[5] = h; b[6] = 0;
+	if (h) h[6] = b;
+	freel[k] = b;
 	return 0;
 }
 // the matching free for an object whose type has no pointer fields
@@ -1008,7 +1041,14 @@ int dmove(word old, word nw) {
 // make p the owner of c, so that freeing p also frees c. it returns c, so the call
 // composes: h[0] = own(b, h). a shared object has no owner and is left alone.
 word own(word c, int p) {
-	if (isobj(c) && !(c[-4] & F_SHARE)) c[-2] = p;
+	// isobj written out, as nested tests: this runs for every owned store into a field
+	if (c >= HEAP_LO + HDR) {
+		if (c < hbump) {
+			if (c[-3] == (c ^ MAGIC)) {
+				if ((c[-4] & (F_SHARE | 1)) == 1) c[-2] = p;
+			}
+		}
+	}
 	return c;
 }
 // an opt-in bump arena of n words, made as one block. the handle owns that block.