line tables and backtraces: port agent/line-tables onto main, op 56 on the x86-64 jit, docs, measurements, tests; default genome reservation 0x68000

8 files changed, 163 insertions, 9 deletions

diff --git a/src/compiler/base.zc b/src/compiler/base.zc
index d83dc9d..2f802b9 100644
--- a/src/compiler/base.zc
+++ b/src/compiler/base.zc
@@ -224,6 +224,16 @@ char[1100] tbuf_initial; char* tbuf = tbuf_initial; int tbufcap = 1100;
 // collects them in dbuf, dlast is the line of the last one, and dline the
 // line of the token that follows them. only when docs are on.
 int docs_on;			// docs on; keeps them: a symbol "doc name" holds a byte string
+// line tables: lines on; (the default) records, for each function compiled, its entry, its end and the
+// source line of each statement, as a line block among the code that rt.line_blocks chains (see
+// rt.zc). lt_buf holds the pairs of the functions being compiled, innermost last, each as a frame
+// [outer frame, entry, name, file, 0, offset << 32 | line ...]; lt_base is the innermost frame.
+// lt_head is the newest block of an image being built, whose own rt.line_blocks it will become.
+int lines_on = 1;
+// //:lines NAME (see mark_line): the markers the lexer has met, [lines before the file, NAME] each, and the
+// one in force, which a top-level item applies when it is reached
+word lt_marks; int lt_nmarks, lt_markcap, lt_next_mark; char* lt_mark_file; int lt_mark_base;
+word lt_buf, lt_head, lt_file_at; int lt_top, lt_cap, lt_base = 0 - 1, lt_file_img; char* lt_file;
 char[2100] dbuf_initial; char* dbuf = dbuf_initial; int docbufcap = 2100; int dn, dlast, dline, docpending, dsect;
 char[2100] dsnap_initial; char* dsnap = dsnap_initial; int docsnapcap = 2100; int dsn, dnm; char[300] dnmb_initial; char* dnmb = dnmb_initial; int docnamecap = 300;
 int pk_tk, pk_tv, pq_n; word[4] pq_tk; word[4] pq_tv; word[4] pq_gl; word[4] pq_ln;	// up to 4 tokens of lookahead
@@ -299,7 +309,7 @@ word tdstk; int ntd, tdcap;
 word readonly_types;
 int dty, dtag;			// result of dtype()
 int s_typedef;
-int s_lt, s_gt, s_shr, s_le, s_ge, s_eqeq, s_ne, s_plus, s_struct, s_type, s_typecheck, s_docs, s_small, s_std, s_icon, s_on, s_off, s_new, s_new_t;
+int s_lt, s_gt, s_shr, s_le, s_ge, s_eqeq, s_ne, s_plus, s_struct, s_type, s_typecheck, s_docs, s_lines, s_small, s_std, s_icon, s_on, s_off, s_new, s_new_t;
 int s_o3, s_lfa, s_outside, s_new_raw, s_new_oz, s_oob;
 int s_ld8, s_st8, s_ld16, s_st16, s_ld32, s_st32;
 // ---- o3 region state ----
@@ -1222,6 +1232,83 @@ int hole() {
 	lit(rt.MW - 1);
 	return a;
 }
+// ---- line tables ----
+
+// make room for n more words of pairs.
+int lt_room(int n) {
+	int cap = lt_cap;
+	if (lt_top + n <= cap) return 0;
+	if (cap < 256) cap = 256;
+	while (lt_top + n > cap) cap = cap * 2;
+	lt_buf = chk_resize(lt_buf, lt_top, cap); lt_cap = cap;
+	return 0;
+}
+// start recording the function whose code begins at entry, named name, with its header on line.
+// returns the frame to give lt_end, or -1 when no table is being made (lines off, a dry run,
+// check mode).
+int lt_begin(int entry, char* name, int line) {
+	int b;
+	if (!lines_on || dry || cmode) return 0 - 1;
+	lt_room(8);
+	b = lt_top;
+	lt_buf[b] = lt_base; lt_buf[b + 1] = entry; lt_buf[b + 2] = name; lt_buf[b + 3] = curinc; lt_buf[b + 4] = 0;
+	if (!curinc) { lt_buf[b + 3] = lt_mark_file; lt_buf[b + 4] = lt_mark_base; }	// the main source may be marked as a file of its own
+	lt_top = b + 5; lt_base = b;
+	lt_buf[lt_top] = line - lt_buf[b + 4]; lt_top++;	// the code before the first statement belongs to the header's line
+	return b;
+}
+// a statement begins at the current token, and its code at here.
+int lt_stmt() {
+	int off, n;
+	if (lt_base < 0 || dry) return 0;
+	off = here - lt_buf[lt_base + 1];
+	if (off < 0) return 0;
+	lt_room(2);
+	if (lt_top > lt_base + 5 && (lt_buf[lt_top - 1] >> 32) == off) lt_top--;	// same place: the later statement is the one
+	n = tline - lt_buf[lt_base + 4];
+	if (lt_top > lt_base + 5 && (lt_buf[lt_top - 1] & 4294967295) == n) return 0;
+	lt_buf[lt_top] = off * 4294967296 + n; lt_top++;
+	return 0;
+}
+// the file name in a block: the include's path, or <stdin> for the main source. one copy of the
+// bytes is shared by the blocks that follow it.
+int lt_file_data(char* f) {
+	if (!f) f = "<stdin>";
+	if (lt_file_at && lt_file_img == image_active && rt.eq_w(lt_file, f)) return lt_file_at;
+	lt_file_at = bstrdata(f); lt_file_img = image_active;
+	lt_file = chk_resize(0, 0, rt.len_w(f) + 1); rt.copy_w(lt_file, f, rt.len_w(f) + 1);
+	return lt_file_at;
+}
+// finish the function begun at frame b, whose code ends at end: put its line block among the code
+// and chain it. a block is [next, entry, end, name, file, count, pairs...].
+int lt_end(int b, int end) {
+	int entry, count, i = 0, name, file, blk, need; word head; char* nm; char* fl;
+	if (b < 0) return 0;
+	entry = lt_buf[b + 1]; nm = lt_buf[b + 2]; fl = lt_buf[b + 3]; count = lt_top - (b + 5);
+	if (!nm) nm = "<fn>";
+	if (!fl) fl = "<stdin>";
+	if (end >= entry && !dry) {
+		need = 8 + count + ((rt.len_w(nm) + 7) >> 3) + 2 + ((rt.len_w(fl) + 7) >> 3) + 2;
+		reserve_code(need);
+		name = bstrdata(nm); file = lt_file_data(fl);
+		if (image_active) head = lt_head; else head = rt.line_blocks;
+		blk = here;
+		emit(head); emit(entry); emit(end); emit(name); emit(file); emit(count);
+		while (i < count) { emit(lt_buf[b + 5 + i]); i++; }
+		if (image_active) lt_head = blk; else rt.line_blocks = blk;
+	}
+	lt_top = b; lt_base = lt_buf[b];
+	return 0;
+}
+// at the top level, a marker that comes before the item takes effect: lines are on, in its file.
+int lt_apply_marks() {
+	if (incdepth || ginst) return 0;
+	while (lt_next_mark < lt_nmarks && lt_marks[lt_next_mark * 2] < tline) {
+		lt_mark_base = lt_marks[lt_next_mark * 2]; lt_mark_file = lt_marks[lt_next_mark * 2 + 1];
+		lines_on = 1; lt_next_mark++;
+	}
+	return 0;
+}
 // forward references. calling a function that is not defined yet creates a
 // symbol with F_DEF clear whose value is the place of its first use (for the
 // "never defined" error). each call site emits a placeholder literal and adds
diff --git a/src/compiler/builtins.zc b/src/compiler/builtins.zc
index 53d8597..5d7bc9c 100644
--- a/src/compiler/builtins.zc
+++ b/src/compiler/builtins.zc
@@ -507,6 +507,15 @@ int kw_docs() {
 	next(); expect(s_semi);
 	return 0;
 }
+// lines on; records line tables for the functions that follow, lines off; stops (see lt_begin)
+int kw_lines() {
+	next();
+	if (is_id(s_on)) {
+		lines_on = 1;
+	} else if (is_id(s_off)) lines_on = 0; else die("lines on or lines off");
+	next(); expect(s_semi);
+	return 0;
+}
 // hints on; / hints small; / hints off; from here to the end of the document (or the next such line)
 int kw_hints() {
 	int level = 2, at = toff;
@@ -1167,7 +1176,7 @@ int types_init() {
 	s_le = intern("<=");
 	s_ge = intern(">=");
 	s_eqeq = intern("=="); s_ne = intern("!="); s_plus = intern("+");
-	s_struct = intern("struct"); s_type = intern("type"); s_typedef = intern("typedef"); s_typecheck = intern("typecheck"); s_docs = intern("docs"); s_std = intern("std"); s_icon = intern("icon"); s_small = intern("small");
+	s_struct = intern("struct"); s_type = intern("type"); s_typedef = intern("typedef"); s_typecheck = intern("typecheck"); s_docs = intern("docs"); s_lines = intern("lines"); s_std = intern("std"); s_icon = intern("icon"); s_small = intern("small");
 	s_on = intern("on");
 	s_off = intern("off");
 	s_new = intern("rt.new");
diff --git a/src/compiler/decl.zc b/src/compiler/decl.zc
index 19b1811..cc84603 100644
--- a/src/compiler/decl.zc
+++ b/src/compiler/decl.zc
@@ -280,7 +280,7 @@ int body() {
 // into stype, checked against any earlier prototype or prescan result, and
 // where the o3 region for an o3 function is handed to the core)
 int fndef(int chk, int rty, int ret, int n) {
-	int s, np = 0, tags = 0, t, sig, i; char*[24] pn; word[24] pt; word[24] pty; int typed = 0, ft, explicit_rty = !!rty, o3f = o3nxt, lo, hi, pk;
+	int s, np = 0, tags = 0, t, sig, i; char*[24] pn; word[24] pt; word[24] pty; int typed = 0, ft, explicit_rty = !!rty, o3f = o3nxt, lo, hi, pk, lt, lend, lfl = tline;
 	o3nxt = 0;
 	if (rty || ntp) typed = 1;
 	if (!dnm) dnm = n;
@@ -365,6 +365,7 @@ int fndef(int chk, int rty, int ret, int n) {
 	sflag[s] = sflag[s] | F_DEF;
 	fixall(sfix[s], here); sfix[s] = 0;
 	fn_begin(chk, ret);
+	lt = lt_begin(sval[s], sname[s], lfl);
 	finsym = s; finfer = !ret;
  effect_current = effect_record(s); rt.fill_w(effect_current, 0, 50); effect_current[0] = 1; if (!chk || stubgen) effect_current[0] = 2; effect_current[1] = np;
  i = 0; while (i < np) { if (pt[i] == 2) { effect_current[2 + i] = EF_CONSUME; effect_current[26 + i] = EF_CONSUME; } i++; }
@@ -397,6 +398,7 @@ int fndef(int chk, int rty, int ret, int n) {
 	}
 	hi = here;
 	epilogue();
+	lend = here;
 	if (o3f) {
 		o3on = 0;
 		o3fs = fslots;
@@ -405,6 +407,7 @@ int fndef(int chk, int rty, int ret, int n) {
 		o3region(lo, hi, o3sl, o3rg, no3sl);
 	}
 	o3flush();
+	lt_end(lt, lend);
 	ntp = 0; ifcur = 0;
 	fn_remember(s, n);
 	return s;
@@ -567,7 +570,7 @@ int topsync() {
 		if (is_op(s_lb)) depth++;
 		else if (is_op(s_rb)) { depth--; if (depth < 0) depth = 0; }
 		else if (!depth && tk == T_ID && (tv == s_fn || tv == s_var || tv == s_const || tv == s_struct || tv == s_type || tv == s_typedef
-			|| tv == s_enhance || tv == s_typecheck || tv == s_docs || tv == s_unchecked || tv == s_immediate || tv == s_o3
+			|| tv == s_enhance || tv == s_typecheck || tv == s_docs || tv == s_lines || tv == s_unchecked || tv == s_immediate || tv == s_o3
 			|| tv == s_memdebug || tv == s_enum || tv == s_include)) break;
 		next();
 	}
@@ -628,6 +631,7 @@ int fndone(int s, int imm) {
 // ( or < means function, otherwise a global.
 int topitem0() {
 	int r, i;
+	lt_apply_marks();
 	dsn = 0; dnm = 0;
 	if (docs_on && dn && dline == tline && !ginst) {
 		if (dn > docsnapcap) { docsnapcap = dn; dsnap = chk_resize(dsnap, 0, docsnapcap); }
@@ -679,6 +683,7 @@ int topitem1() {
 	if (typedef_ahead()) return kw_typedef();
 	if (is_id(s_typecheck)) return kw_typecheck();
 	if (is_id(s_docs)) return kw_docs();
+	if (is_id(s_lines)) { peek(); if (pk_tk == T_ID && (pk_tv == s_on || pk_tv == s_off)) return kw_lines(); }
 	if (is_id(s_memdebug)) return kw_memdebug();
 	if (is_id(s_enum)) return kw_enum();
 	if (is_id(s_include)) return kw_include();
@@ -1334,7 +1339,7 @@ int unit_end(int from) {
 // way is emitted behind a jump when another function is being compiled, so
 // compile() works inside a parse word.)
 int compile_list_context(int q, int top, char* module, word imports) {
-	int hop = -1, entry = 0, from = nsym;
+	int hop = -1, entry = 0, from = nsym, lt = 0 - 1;
 	ctx_push(); ifcur = 0; ifcall = -1; effect_floor = from; module_name = module; module_imports = imports;
 	if (infn) { hop = hole(); op(O_JMP); }
 	if (top) prescan(1, q);
@@ -1344,12 +1349,14 @@ int compile_list_context(int q, int top, char* module, word imports) {
 	else {
 		entry = here;
 		fn_begin(1, 0);
+		lt = lt_begin(entry, "<eval>", tline);
 		prologue(); borrow_prologue(); borrow_begin();
 		while (tk != T_EOF) stmt();
 		defer_run(1, 1);
 		unused_check(lbase);
 		if (fchk && live) exit_vars(1, 1);
 		epilogue();
+		lt_end(lt, here);
 	}
 	src_pop();
 	unit_end(from);
diff --git a/src/compiler/expr.zc b/src/compiler/expr.zc
index 40de6ef..f41109e 100644
--- a/src/compiler/expr.zc
+++ b/src/compiler/expr.zc
@@ -647,7 +647,7 @@ word closure_layout(int closure_index, int count) {
 // variable in. ctx_push / ctx_pop save and restore the enclosing function's
 // compiler state around the body.
 int closure() {
-	int np = 0; char*[20] pn; word[20] pty; int hop, entry, i, t, envl, ncap, c, result, declared = 0, tags = 0, tag; word[20] ptags;
+	int np = 0; char*[20] pn; word[20] pty; int hop, entry, i, t, envl, ncap, c, result, declared = 0, tags = 0, tag, lt, lfl = tline; word[20] ptags;
 	word expected = closure_expected; closure_expected = 0;
 	reserve_closures(nclo + 1);
 	if (o3on) serr("closures can't be made inside an o3 region");
@@ -679,6 +679,7 @@ int closure() {
 	hop = hole(); op(O_JMP);
 	entry = here;
 	fn_begin(1, 0);
+	lt = lt_begin(entry, "<closure>", lfl);
 	if (declared) fty = declared;
 	prologue(); borrow_prologue();
 	i = 0;
@@ -699,6 +700,7 @@ int closure() {
 	if (result <= 0 || live) result = tword;
 	if (declared) result = declared;
 	epilogue();
+	lt_end(lt, here);
 	nclo--;
 	c = nclo;
 	ncap = clo_n[c];
diff --git a/src/compiler/image.zc b/src/compiler/image.zc
index dfe5f77..45cd928 100644
--- a/src/compiler/image.zc
+++ b/src/compiler/image.zc
@@ -62,25 +62,48 @@ int blob_load(word p, int hide) {
 	}
 	return 0;
 }
+// the name of the function whose code holds address a, for the frames of a backtrace that have
+// no line table (rt.frame_namer is set to this when the compiler starts). the nearest function
+// at or below a, or 0.
+char* frame_symbol(int a) {
+	int i = 0, best = 0 - 1;
+	while (i < nsym) {
+		if (skind[i] == K_FN && (sflag[i] & F_DEF) && sval[i] <= a && sval[i] > 0 && (best < 0 || sval[i] > sval[best])) best = i;
+		i++;
+	}
+	if (best < 0) return 0;
+	return sname[best];
+}
+// an image built with line tables gets the head of its chain in its own rt.line_blocks, the
+// global the runtime reads, as that global's initial value.
+int lt_image_head() {
+	int s = lookup(intern("rt.line_blocks"));
+	if (lt_head && s >= 0 && skind[s] == K_GVAR) *(sval[s] + hdelta) = lt_head;
+	return 0;
+}
 // compile the rest of stdin for org into a growing output buffer. the image's
 // addresses remain logical addresses at org while its temporary storage grows.
 // image_words and image_size expose the finished image; initial is its first
 // buffer reservation. the matching host layout must still fit the finished image.
 int build_image_grow(int org, int initial) {
 	int sh = here, sd = hdelta, sl = hlimit, sub = ubase, svl = vislo, sns = nsym, sen = nenh, n;
+	word saved_head = lt_head; char* saved_file = lt_file; word saved_file_at = lt_file_at; char* saved_mark = lt_mark_file; int saved_base = lt_mark_base, saved_nm = lt_next_mark, saved_nmarks = lt_nmarks;
 	word saved_buffer = image_buffer; int saved_active = image_active, saved_org = image_org, saved_capacity = image_capacity;
 	if (org < 0 || initial < 7 || initial > 0x7fffffff - org) fatal("bad image reservation");
 	image_buffer = chk_resize(0, 0, initial); image_active = 1; image_org = org; image_capacity = initial;
 	ubase = nsym; vislo = nsym; nenh = 0;
 	here = org; hdelta = image_buffer - org; hlimit = org + initial;
+	lt_head = 0; lt_file_at = 0; lt_mark_file = 0; lt_mark_base = 0; lt_next_mark = 0; lt_nmarks = 0;
 	lit(rt.FTOP); lit(rt.ZFP); op(O_ST); fnref(fwdfn(s_start)); op(O_JMP); emit(0); emit(0);
 	unit_stdin();
+	lt_image_head();
 	n = blob_emit(ubase);
 	image_buffer[5] = n;
 	image_buffer[6] = here;
 	n = here - org; image_words = image_buffer; image_size = n;
 	popsyms(sns);
 	ubase = sub; vislo = svl; here = sh; hdelta = sd; hlimit = sl; nenh = sen;
+	lt_head = saved_head; lt_file = saved_file; lt_file_at = saved_file_at; lt_mark_file = saved_mark; lt_mark_base = saved_base; lt_next_mark = saved_nm; lt_nmarks = saved_nmarks;
 	image_buffer = saved_buffer; image_active = saved_active; image_org = saved_org; image_capacity = saved_capacity;
 	return n;
 }
diff --git a/src/compiler/lex.zc b/src/compiler/lex.zc
index 0b509e4..0348cae 100644
--- a/src/compiler/lex.zc
+++ b/src/compiler/lex.zc
@@ -366,10 +366,31 @@ int fieldscan(int n) {
 // a // comment, with ch on its second slash. one that starts a line joins the
 // doc comment being collected, if the line above was part of it, or starts a
 // new one; a comment after code on the same line is not documentation.
+// a comment //:lines NAME (ch is its colon) says that what follows is the file NAME, with its lines counted
+// from the next one, and turns line tables on from the next top-level item. src/tools/automap.sh writes one
+// before a program it joins to the runtime. the rest of the line is read here and the marker kept: the
+// source is read once to find the signatures and again to compile, and it counts only when compiling reaches it.
+int mark_line() {
+	char[300] b; int n = 0, i, base; word name;
+	while (ch >= 0 && ch != 10) { if (n < 298) { b[n] = ch; n++; } nextch(); }
+	while (n && (b[n - 1] == 13 || b[n - 1] == 32)) n--;
+	b[n] = 0;
+	if (n > 7 && b[1] == 'l' && b[2] == 'i' && b[3] == 'n' && b[4] == 'e' && b[5] == 's' && b[6] == 32) {
+		base = line;
+		if (ch == 10) base = line - 1;
+		if (!lt_nmarks || base > lt_marks[lt_nmarks * 2 - 2]) {
+			if (lt_nmarks * 2 + 2 > lt_markcap) { lt_marks = chk_resize(lt_marks, lt_nmarks * 2, lt_markcap * 2 + 16); lt_markcap = lt_markcap * 2 + 16; }
+			name = chk_resize(0, 0, n); i = 7;
+			while (i <= n) { name[i - 7] = b[i]; i++; }
+			lt_marks[lt_nmarks * 2] = base; lt_marks[lt_nmarks * 2 + 1] = name; lt_nmarks++;
+		}
+	}
+	return 0;
+}
 int doccomment(int nl) {
 	int cl = line, start;
 	nextch();
-	if (ch == ':') { while (ch >= 0 && ch != 10) nextch(); dn = 0; dsect = 1; dlast = cl; docpending = 1; return 0; }
+	if (ch == ':') { mark_line(); dn = 0; dsect = 1; dlast = cl; docpending = 1; return 0; }
 	if (!nl) { while (ch >= 0 && ch != 10) nextch(); return 0; }
 	if (dlast != cl - 1) { dn = 0; dsect = 0; }
 	if (ch == 32) nextch();
@@ -451,7 +472,11 @@ int lex1() {
 		if (ch != '/') break;
 		nextch();
 		if (ch == '/') {
-			if (docs_on) doccomment(nl); else while (ch >= 0 && ch != 10) nextch();
+			if (docs_on) doccomment(nl);
+			else {
+				nextch();
+				if (ch == ':') mark_line(); else while (ch >= 0 && ch != 10) nextch();
+			}
 			tglue = 0; continue;
 		}
 		if (ch == '*') {
diff --git a/src/compiler/setup.zc b/src/compiler/setup.zc
index 1d8ce6f..9ed938b 100644
--- a/src/compiler/setup.zc
+++ b/src/compiler/setup.zc
@@ -168,6 +168,7 @@ int start() {
 	blob_load(rt.GORG[5], 0);
 	zc_tables();
 	types_init();
+	rt.frame_namer = sval[lookup(intern("run.frame_symbol"))];
 	here = 0; hdelta = 0; hlimit = rt.CODE_END; ptop = 0;
 	ubase = nsym; vislo = 0;
 	// a genome that carries the language server runs it instead of compiling stdin
diff --git a/src/compiler/stmt.zc b/src/compiler/stmt.zc
index 21d5801..4dafe4f 100644
--- a/src/compiler/stmt.zc
+++ b/src/compiler/stmt.zc
@@ -13,7 +13,7 @@ int resync() {
 }
 // compile one statement.
 int stmt() {
- word before = borrow_slots; int result; emit_statement_hooks(); result = stmt_inner(); borrow_cleanup_since(before); return result;
+ word before = borrow_slots; int result; lt_stmt(); emit_statement_hooks(); result = stmt_inner(); borrow_cleanup_since(before); return result;
 }
 int stmt_inner() {
 	int s, c;