From 7c54e48bca711a6596be4bb17fcaa4330b9928e3 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 15:52:01 +0330 Subject: line tables and backtraces: port agent/line-tables onto main, op 56 on the x86-64 jit, docs, measurements, tests; default genome reservation 0x68000 --- src/compiler/base.zc | 89 +++++++++++++++++++++++++++++++++++++++++++++++- src/compiler/builtins.zc | 11 +++++- src/compiler/decl.zc | 13 +++++-- src/compiler/expr.zc | 4 ++- src/compiler/image.zc | 23 +++++++++++++ src/compiler/lex.zc | 29 ++++++++++++++-- src/compiler/setup.zc | 1 + src/compiler/stmt.zc | 2 +- 8 files changed, 163 insertions(+), 9 deletions(-) (limited to 'src/compiler') diff --git a/src/compiler/base.zc b/src/compiler/base.zc index d83dc9d..2f802b9 100644 --- a/src/compiler/base.zc +++ b/src/compiler/base.zc @@ -224,6 +224,16 @@ char[1100] tbuf_initial; char* tbuf = tbuf_initial; int tbufcap = 1100; // collects them in dbuf, dlast is the line of the last one, and dline the // line of the token that follows them. only when docs are on. int docs_on; // docs on; keeps them: a symbol "doc name" holds a byte string +// line tables: lines on; (the default) records, for each function compiled, its entry, its end and the +// source line of each statement, as a line block among the code that rt.line_blocks chains (see +// rt.zc). lt_buf holds the pairs of the functions being compiled, innermost last, each as a frame +// [outer frame, entry, name, file, 0, offset << 32 | line ...]; lt_base is the innermost frame. +// lt_head is the newest block of an image being built, whose own rt.line_blocks it will become. +int lines_on = 1; +// //:lines NAME (see mark_line): the markers the lexer has met, [lines before the file, NAME] each, and the +// one in force, which a top-level item applies when it is reached +word lt_marks; int lt_nmarks, lt_markcap, lt_next_mark; char* lt_mark_file; int lt_mark_base; +word lt_buf, lt_head, lt_file_at; int lt_top, lt_cap, lt_base = 0 - 1, lt_file_img; char* lt_file; char[2100] dbuf_initial; char* dbuf = dbuf_initial; int docbufcap = 2100; int dn, dlast, dline, docpending, dsect; char[2100] dsnap_initial; char* dsnap = dsnap_initial; int docsnapcap = 2100; int dsn, dnm; char[300] dnmb_initial; char* dnmb = dnmb_initial; int docnamecap = 300; int pk_tk, pk_tv, pq_n; word[4] pq_tk; word[4] pq_tv; word[4] pq_gl; word[4] pq_ln; // up to 4 tokens of lookahead @@ -299,7 +309,7 @@ word tdstk; int ntd, tdcap; word readonly_types; int dty, dtag; // result of dtype() int s_typedef; -int s_lt, s_gt, s_shr, s_le, s_ge, s_eqeq, s_ne, s_plus, s_struct, s_type, s_typecheck, s_docs, s_small, s_std, s_icon, s_on, s_off, s_new, s_new_t; +int s_lt, s_gt, s_shr, s_le, s_ge, s_eqeq, s_ne, s_plus, s_struct, s_type, s_typecheck, s_docs, s_lines, s_small, s_std, s_icon, s_on, s_off, s_new, s_new_t; int s_o3, s_lfa, s_outside, s_new_raw, s_new_oz, s_oob; int s_ld8, s_st8, s_ld16, s_st16, s_ld32, s_st32; // ---- o3 region state ---- @@ -1222,6 +1232,83 @@ int hole() { lit(rt.MW - 1); return a; } +// ---- line tables ---- + +// make room for n more words of pairs. +int lt_room(int n) { + int cap = lt_cap; + if (lt_top + n <= cap) return 0; + if (cap < 256) cap = 256; + while (lt_top + n > cap) cap = cap * 2; + lt_buf = chk_resize(lt_buf, lt_top, cap); lt_cap = cap; + return 0; +} +// start recording the function whose code begins at entry, named name, with its header on line. +// returns the frame to give lt_end, or -1 when no table is being made (lines off, a dry run, +// check mode). +int lt_begin(int entry, char* name, int line) { + int b; + if (!lines_on || dry || cmode) return 0 - 1; + lt_room(8); + b = lt_top; + lt_buf[b] = lt_base; lt_buf[b + 1] = entry; lt_buf[b + 2] = name; lt_buf[b + 3] = curinc; lt_buf[b + 4] = 0; + if (!curinc) { lt_buf[b + 3] = lt_mark_file; lt_buf[b + 4] = lt_mark_base; } // the main source may be marked as a file of its own + lt_top = b + 5; lt_base = b; + lt_buf[lt_top] = line - lt_buf[b + 4]; lt_top++; // the code before the first statement belongs to the header's line + return b; +} +// a statement begins at the current token, and its code at here. +int lt_stmt() { + int off, n; + if (lt_base < 0 || dry) return 0; + off = here - lt_buf[lt_base + 1]; + if (off < 0) return 0; + lt_room(2); + if (lt_top > lt_base + 5 && (lt_buf[lt_top - 1] >> 32) == off) lt_top--; // same place: the later statement is the one + n = tline - lt_buf[lt_base + 4]; + if (lt_top > lt_base + 5 && (lt_buf[lt_top - 1] & 4294967295) == n) return 0; + lt_buf[lt_top] = off * 4294967296 + n; lt_top++; + return 0; +} +// the file name in a block: the include's path, or for the main source. one copy of the +// bytes is shared by the blocks that follow it. +int lt_file_data(char* f) { + if (!f) f = ""; + if (lt_file_at && lt_file_img == image_active && rt.eq_w(lt_file, f)) return lt_file_at; + lt_file_at = bstrdata(f); lt_file_img = image_active; + lt_file = chk_resize(0, 0, rt.len_w(f) + 1); rt.copy_w(lt_file, f, rt.len_w(f) + 1); + return lt_file_at; +} +// finish the function begun at frame b, whose code ends at end: put its line block among the code +// and chain it. a block is [next, entry, end, name, file, count, pairs...]. +int lt_end(int b, int end) { + int entry, count, i = 0, name, file, blk, need; word head; char* nm; char* fl; + if (b < 0) return 0; + entry = lt_buf[b + 1]; nm = lt_buf[b + 2]; fl = lt_buf[b + 3]; count = lt_top - (b + 5); + if (!nm) nm = ""; + if (!fl) fl = ""; + if (end >= entry && !dry) { + need = 8 + count + ((rt.len_w(nm) + 7) >> 3) + 2 + ((rt.len_w(fl) + 7) >> 3) + 2; + reserve_code(need); + name = bstrdata(nm); file = lt_file_data(fl); + if (image_active) head = lt_head; else head = rt.line_blocks; + blk = here; + emit(head); emit(entry); emit(end); emit(name); emit(file); emit(count); + while (i < count) { emit(lt_buf[b + 5 + i]); i++; } + if (image_active) lt_head = blk; else rt.line_blocks = blk; + } + lt_top = b; lt_base = lt_buf[b]; + return 0; +} +// at the top level, a marker that comes before the item takes effect: lines are on, in its file. +int lt_apply_marks() { + if (incdepth || ginst) return 0; + while (lt_next_mark < lt_nmarks && lt_marks[lt_next_mark * 2] < tline) { + lt_mark_base = lt_marks[lt_next_mark * 2]; lt_mark_file = lt_marks[lt_next_mark * 2 + 1]; + lines_on = 1; lt_next_mark++; + } + return 0; +} // forward references. calling a function that is not defined yet creates a // symbol with F_DEF clear whose value is the place of its first use (for the // "never defined" error). each call site emits a placeholder literal and adds diff --git a/src/compiler/builtins.zc b/src/compiler/builtins.zc index 53d8597..5d7bc9c 100644 --- a/src/compiler/builtins.zc +++ b/src/compiler/builtins.zc @@ -507,6 +507,15 @@ int kw_docs() { next(); expect(s_semi); return 0; } +// lines on; records line tables for the functions that follow, lines off; stops (see lt_begin) +int kw_lines() { + next(); + if (is_id(s_on)) { + lines_on = 1; + } else if (is_id(s_off)) lines_on = 0; else die("lines on or lines off"); + next(); expect(s_semi); + return 0; +} // hints on; / hints small; / hints off; from here to the end of the document (or the next such line) int kw_hints() { int level = 2, at = toff; @@ -1167,7 +1176,7 @@ int types_init() { s_le = intern("<="); s_ge = intern(">="); s_eqeq = intern("=="); s_ne = intern("!="); s_plus = intern("+"); - s_struct = intern("struct"); s_type = intern("type"); s_typedef = intern("typedef"); s_typecheck = intern("typecheck"); s_docs = intern("docs"); s_std = intern("std"); s_icon = intern("icon"); s_small = intern("small"); + s_struct = intern("struct"); s_type = intern("type"); s_typedef = intern("typedef"); s_typecheck = intern("typecheck"); s_docs = intern("docs"); s_lines = intern("lines"); s_std = intern("std"); s_icon = intern("icon"); s_small = intern("small"); s_on = intern("on"); s_off = intern("off"); s_new = intern("rt.new"); diff --git a/src/compiler/decl.zc b/src/compiler/decl.zc index 19b1811..cc84603 100644 --- a/src/compiler/decl.zc +++ b/src/compiler/decl.zc @@ -280,7 +280,7 @@ int body() { // into stype, checked against any earlier prototype or prescan result, and // where the o3 region for an o3 function is handed to the core) int fndef(int chk, int rty, int ret, int n) { - int s, np = 0, tags = 0, t, sig, i; char*[24] pn; word[24] pt; word[24] pty; int typed = 0, ft, explicit_rty = !!rty, o3f = o3nxt, lo, hi, pk; + int s, np = 0, tags = 0, t, sig, i; char*[24] pn; word[24] pt; word[24] pty; int typed = 0, ft, explicit_rty = !!rty, o3f = o3nxt, lo, hi, pk, lt, lend, lfl = tline; o3nxt = 0; if (rty || ntp) typed = 1; if (!dnm) dnm = n; @@ -365,6 +365,7 @@ int fndef(int chk, int rty, int ret, int n) { sflag[s] = sflag[s] | F_DEF; fixall(sfix[s], here); sfix[s] = 0; fn_begin(chk, ret); + lt = lt_begin(sval[s], sname[s], lfl); finsym = s; finfer = !ret; effect_current = effect_record(s); rt.fill_w(effect_current, 0, 50); effect_current[0] = 1; if (!chk || stubgen) effect_current[0] = 2; effect_current[1] = np; i = 0; while (i < np) { if (pt[i] == 2) { effect_current[2 + i] = EF_CONSUME; effect_current[26 + i] = EF_CONSUME; } i++; } @@ -397,6 +398,7 @@ int fndef(int chk, int rty, int ret, int n) { } hi = here; epilogue(); + lend = here; if (o3f) { o3on = 0; o3fs = fslots; @@ -405,6 +407,7 @@ int fndef(int chk, int rty, int ret, int n) { o3region(lo, hi, o3sl, o3rg, no3sl); } o3flush(); + lt_end(lt, lend); ntp = 0; ifcur = 0; fn_remember(s, n); return s; @@ -567,7 +570,7 @@ int topsync() { if (is_op(s_lb)) depth++; else if (is_op(s_rb)) { depth--; if (depth < 0) depth = 0; } else if (!depth && tk == T_ID && (tv == s_fn || tv == s_var || tv == s_const || tv == s_struct || tv == s_type || tv == s_typedef - || tv == s_enhance || tv == s_typecheck || tv == s_docs || tv == s_unchecked || tv == s_immediate || tv == s_o3 + || tv == s_enhance || tv == s_typecheck || tv == s_docs || tv == s_lines || tv == s_unchecked || tv == s_immediate || tv == s_o3 || tv == s_memdebug || tv == s_enum || tv == s_include)) break; next(); } @@ -628,6 +631,7 @@ int fndone(int s, int imm) { // ( or < means function, otherwise a global. int topitem0() { int r, i; + lt_apply_marks(); dsn = 0; dnm = 0; if (docs_on && dn && dline == tline && !ginst) { if (dn > docsnapcap) { docsnapcap = dn; dsnap = chk_resize(dsnap, 0, docsnapcap); } @@ -679,6 +683,7 @@ int topitem1() { if (typedef_ahead()) return kw_typedef(); if (is_id(s_typecheck)) return kw_typecheck(); if (is_id(s_docs)) return kw_docs(); + if (is_id(s_lines)) { peek(); if (pk_tk == T_ID && (pk_tv == s_on || pk_tv == s_off)) return kw_lines(); } if (is_id(s_memdebug)) return kw_memdebug(); if (is_id(s_enum)) return kw_enum(); if (is_id(s_include)) return kw_include(); @@ -1334,7 +1339,7 @@ int unit_end(int from) { // way is emitted behind a jump when another function is being compiled, so // compile() works inside a parse word.) int compile_list_context(int q, int top, char* module, word imports) { - int hop = -1, entry = 0, from = nsym; + int hop = -1, entry = 0, from = nsym, lt = 0 - 1; ctx_push(); ifcur = 0; ifcall = -1; effect_floor = from; module_name = module; module_imports = imports; if (infn) { hop = hole(); op(O_JMP); } if (top) prescan(1, q); @@ -1344,12 +1349,14 @@ int compile_list_context(int q, int top, char* module, word imports) { else { entry = here; fn_begin(1, 0); + lt = lt_begin(entry, "", tline); prologue(); borrow_prologue(); borrow_begin(); while (tk != T_EOF) stmt(); defer_run(1, 1); unused_check(lbase); if (fchk && live) exit_vars(1, 1); epilogue(); + lt_end(lt, here); } src_pop(); unit_end(from); diff --git a/src/compiler/expr.zc b/src/compiler/expr.zc index 40de6ef..f41109e 100644 --- a/src/compiler/expr.zc +++ b/src/compiler/expr.zc @@ -647,7 +647,7 @@ word closure_layout(int closure_index, int count) { // variable in. ctx_push / ctx_pop save and restore the enclosing function's // compiler state around the body. int closure() { - int np = 0; char*[20] pn; word[20] pty; int hop, entry, i, t, envl, ncap, c, result, declared = 0, tags = 0, tag; word[20] ptags; + int np = 0; char*[20] pn; word[20] pty; int hop, entry, i, t, envl, ncap, c, result, declared = 0, tags = 0, tag, lt, lfl = tline; word[20] ptags; word expected = closure_expected; closure_expected = 0; reserve_closures(nclo + 1); if (o3on) serr("closures can't be made inside an o3 region"); @@ -679,6 +679,7 @@ int closure() { hop = hole(); op(O_JMP); entry = here; fn_begin(1, 0); + lt = lt_begin(entry, "", lfl); if (declared) fty = declared; prologue(); borrow_prologue(); i = 0; @@ -699,6 +700,7 @@ int closure() { if (result <= 0 || live) result = tword; if (declared) result = declared; epilogue(); + lt_end(lt, here); nclo--; c = nclo; ncap = clo_n[c]; diff --git a/src/compiler/image.zc b/src/compiler/image.zc index dfe5f77..45cd928 100644 --- a/src/compiler/image.zc +++ b/src/compiler/image.zc @@ -62,25 +62,48 @@ int blob_load(word p, int hide) { } return 0; } +// the name of the function whose code holds address a, for the frames of a backtrace that have +// no line table (rt.frame_namer is set to this when the compiler starts). the nearest function +// at or below a, or 0. +char* frame_symbol(int a) { + int i = 0, best = 0 - 1; + while (i < nsym) { + if (skind[i] == K_FN && (sflag[i] & F_DEF) && sval[i] <= a && sval[i] > 0 && (best < 0 || sval[i] > sval[best])) best = i; + i++; + } + if (best < 0) return 0; + return sname[best]; +} +// an image built with line tables gets the head of its chain in its own rt.line_blocks, the +// global the runtime reads, as that global's initial value. +int lt_image_head() { + int s = lookup(intern("rt.line_blocks")); + if (lt_head && s >= 0 && skind[s] == K_GVAR) *(sval[s] + hdelta) = lt_head; + return 0; +} // compile the rest of stdin for org into a growing output buffer. the image's // addresses remain logical addresses at org while its temporary storage grows. // image_words and image_size expose the finished image; initial is its first // buffer reservation. the matching host layout must still fit the finished image. int build_image_grow(int org, int initial) { int sh = here, sd = hdelta, sl = hlimit, sub = ubase, svl = vislo, sns = nsym, sen = nenh, n; + word saved_head = lt_head; char* saved_file = lt_file; word saved_file_at = lt_file_at; char* saved_mark = lt_mark_file; int saved_base = lt_mark_base, saved_nm = lt_next_mark, saved_nmarks = lt_nmarks; word saved_buffer = image_buffer; int saved_active = image_active, saved_org = image_org, saved_capacity = image_capacity; if (org < 0 || initial < 7 || initial > 0x7fffffff - org) fatal("bad image reservation"); image_buffer = chk_resize(0, 0, initial); image_active = 1; image_org = org; image_capacity = initial; ubase = nsym; vislo = nsym; nenh = 0; here = org; hdelta = image_buffer - org; hlimit = org + initial; + lt_head = 0; lt_file_at = 0; lt_mark_file = 0; lt_mark_base = 0; lt_next_mark = 0; lt_nmarks = 0; lit(rt.FTOP); lit(rt.ZFP); op(O_ST); fnref(fwdfn(s_start)); op(O_JMP); emit(0); emit(0); unit_stdin(); + lt_image_head(); n = blob_emit(ubase); image_buffer[5] = n; image_buffer[6] = here; n = here - org; image_words = image_buffer; image_size = n; popsyms(sns); ubase = sub; vislo = svl; here = sh; hdelta = sd; hlimit = sl; nenh = sen; + lt_head = saved_head; lt_file = saved_file; lt_file_at = saved_file_at; lt_mark_file = saved_mark; lt_mark_base = saved_base; lt_next_mark = saved_nm; lt_nmarks = saved_nmarks; image_buffer = saved_buffer; image_active = saved_active; image_org = saved_org; image_capacity = saved_capacity; return n; } diff --git a/src/compiler/lex.zc b/src/compiler/lex.zc index 0b509e4..0348cae 100644 --- a/src/compiler/lex.zc +++ b/src/compiler/lex.zc @@ -366,10 +366,31 @@ int fieldscan(int n) { // a // comment, with ch on its second slash. one that starts a line joins the // doc comment being collected, if the line above was part of it, or starts a // new one; a comment after code on the same line is not documentation. +// a comment //:lines NAME (ch is its colon) says that what follows is the file NAME, with its lines counted +// from the next one, and turns line tables on from the next top-level item. src/tools/automap.sh writes one +// before a program it joins to the runtime. the rest of the line is read here and the marker kept: the +// source is read once to find the signatures and again to compile, and it counts only when compiling reaches it. +int mark_line() { + char[300] b; int n = 0, i, base; word name; + while (ch >= 0 && ch != 10) { if (n < 298) { b[n] = ch; n++; } nextch(); } + while (n && (b[n - 1] == 13 || b[n - 1] == 32)) n--; + b[n] = 0; + if (n > 7 && b[1] == 'l' && b[2] == 'i' && b[3] == 'n' && b[4] == 'e' && b[5] == 's' && b[6] == 32) { + base = line; + if (ch == 10) base = line - 1; + if (!lt_nmarks || base > lt_marks[lt_nmarks * 2 - 2]) { + if (lt_nmarks * 2 + 2 > lt_markcap) { lt_marks = chk_resize(lt_marks, lt_nmarks * 2, lt_markcap * 2 + 16); lt_markcap = lt_markcap * 2 + 16; } + name = chk_resize(0, 0, n); i = 7; + while (i <= n) { name[i - 7] = b[i]; i++; } + lt_marks[lt_nmarks * 2] = base; lt_marks[lt_nmarks * 2 + 1] = name; lt_nmarks++; + } + } + return 0; +} int doccomment(int nl) { int cl = line, start; nextch(); - if (ch == ':') { while (ch >= 0 && ch != 10) nextch(); dn = 0; dsect = 1; dlast = cl; docpending = 1; return 0; } + if (ch == ':') { mark_line(); dn = 0; dsect = 1; dlast = cl; docpending = 1; return 0; } if (!nl) { while (ch >= 0 && ch != 10) nextch(); return 0; } if (dlast != cl - 1) { dn = 0; dsect = 0; } if (ch == 32) nextch(); @@ -451,7 +472,11 @@ int lex1() { if (ch != '/') break; nextch(); if (ch == '/') { - if (docs_on) doccomment(nl); else while (ch >= 0 && ch != 10) nextch(); + if (docs_on) doccomment(nl); + else { + nextch(); + if (ch == ':') mark_line(); else while (ch >= 0 && ch != 10) nextch(); + } tglue = 0; continue; } if (ch == '*') { diff --git a/src/compiler/setup.zc b/src/compiler/setup.zc index 1d8ce6f..9ed938b 100644 --- a/src/compiler/setup.zc +++ b/src/compiler/setup.zc @@ -168,6 +168,7 @@ int start() { blob_load(rt.GORG[5], 0); zc_tables(); types_init(); + rt.frame_namer = sval[lookup(intern("run.frame_symbol"))]; here = 0; hdelta = 0; hlimit = rt.CODE_END; ptop = 0; ubase = nsym; vislo = 0; // a genome that carries the language server runs it instead of compiling stdin diff --git a/src/compiler/stmt.zc b/src/compiler/stmt.zc index 21d5801..4dafe4f 100644 --- a/src/compiler/stmt.zc +++ b/src/compiler/stmt.zc @@ -13,7 +13,7 @@ int resync() { } // compile one statement. int stmt() { - word before = borrow_slots; int result; emit_statement_hooks(); result = stmt_inner(); borrow_cleanup_since(before); return result; + word before = borrow_slots; int result; lt_stmt(); emit_statement_hooks(); result = stmt_inner(); borrow_cleanup_since(before); return result; } int stmt_inner() { int s, c; -- cgit v1.2.3