From 397b4f5ca283175411677c9e9ee667f647263884 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 15:37:28 +0330 Subject: agent/xref-report: unmerged work from previous session --- src/tools/xref.js | 446 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/tools/xref.sh | 17 +++ 2 files changed, 463 insertions(+) create mode 100644 src/tools/xref.js create mode 100755 src/tools/xref.sh (limited to 'src') diff --git a/src/tools/xref.js b/src/tools/xref.js new file mode 100644 index 0000000..8d99c25 --- /dev/null +++ b/src/tools/xref.js @@ -0,0 +1,446 @@ +#!/usr/bin/env node +// cross-reference of the compiler's global state: which function writes a global and which only read it. +// +// the compiler is the files of src/compiler/parts read as one unit. its state is global arrays and scalars +// that functions far apart read and write. this tool lists, for every top-level variable of the compiler, +// the functions that write it and the functions that only read it, and can fail ci when a global gains a +// writer that tests/xref.baseline does not list. +// +// how it works: +// 1. the parts are joined (the same text the language server serves as tinyvm:/compiler.zc) and lexed +// here; the top-level declarations are found by parsing the lexed text, and so are the brace ranges +// of the top-level functions. +// 2. exact scoping is not guessed: the joined text is opened as a document in build/bin/zclsp and one +// textDocument/references request per global is sent, all in one server run. the server answers with +// the uses the compiler itself resolved to that symbol, so a local that shadows a global is not +// counted. +// 3. each reference is classified from its neighbouring tokens (see classify) and attributed to the +// function whose range holds it. +const fs = require('fs'); +const path = require('path'); +const cp = require('child_process'); +const os = require('os'); + +const ROOT = path.resolve(__dirname, '..', '..'); + +// ---- lexing ---- + +const OPS = ['<<=', '>>=', '...', '->', '++', '--', '+=', '-=', '*=', '/=', '%=', '&=', '^=', '|=', '==', '!=', '<=', '>=', '&&', '||', '<<', '>>', '::']; +const ASSIGN_OPS = new Set(['=', '+=', '-=', '*=', '/=', '%=', '&=', '^=', '|=', '<<=', '>>=']); +const IDSTART = /[A-Za-z_]/, IDCHAR = /[A-Za-z0-9_]/; + +// tokens: {k: 'id'|'num'|'str'|'chr'|'op', s, off, line, col}; comments are skipped, and a backtick outside +// a comment or literal ends the source, as it does for the compiler. +function lex(text) { + const toks = []; + let i = 0, line = 0, lineStart = 0; + const n = text.length; + const push = (k, s, off) => toks.push({ k, s, off, line, col: off - lineStart }); + while (i < n) { + const c = text[i]; + if (c === '\n') { line++; i++; lineStart = i; continue; } + if (c === ' ' || c === '\t' || c === '\r') { i++; continue; } + if (c === '/' && text[i + 1] === '/') { while (i < n && text[i] !== '\n') i++; continue; } + if (c === '/' && text[i + 1] === '*') { + i += 2; + while (i < n && !(text[i] === '*' && text[i + 1] === '/')) { if (text[i] === '\n') { line++; lineStart = i + 1; } i++; } + i += 2; continue; + } + if (c === '`') break; + if (c === '"' || c === "'") { + const start = i, startLine = line, startCol = i - lineStart; + i++; + while (i < n && text[i] !== c) { if (text[i] === '\\') i++; if (text[i] === '\n') { line++; lineStart = i + 1; } i++; } + i++; + toks.push({ k: c === '"' ? 'str' : 'chr', s: text.slice(start, i), off: start, line: startLine, col: startCol }); + continue; + } + if (IDSTART.test(c)) { const s = i; while (i < n && IDCHAR.test(text[i])) i++; push('id', text.slice(s, i), s); continue; } + if (c >= '0' && c <= '9') { + const s = i; + // digits, hex, and a fraction: a dot only when a digit follows, so "1.u" style tags are not eaten + while (i < n && (IDCHAR.test(text[i]) || (text[i] === '.' && text[i + 1] >= '0' && text[i + 1] <= '9'))) i++; + push('num', text.slice(s, i), s); continue; + } + let op = null; + for (const o of OPS) if (text.startsWith(o, i)) { op = o; break; } + if (!op) op = c; + push('op', op, i); + i += op.length; + } + return toks; +} + +// ---- top-level structure ---- + +const NOT_DECL = new Set(['const', 'struct', 'enum', 'namespace', 'include', 'embed', 'typedef', 'type', 'enhance', 'fn', 'macro', 'immediate', 'extern', 'cfunc', 'import', 'static', 'quote']); + +// finds the top-level functions (name, token range, source range) and the top-level variable declarations. +// returns {functions, globals, problems} +function structure(toks) { + const functions = [], globals = [], problems = []; + let i = 0; + const n = toks.length; + const closeOf = { '(': ')', '[': ']', '{': '}' }; + // index of the token matching the opener at i + const match = j => { + let depth = 0; + for (; j < n; j++) { + const s = toks[j].k === 'op' ? toks[j].s : ''; + if (s === '(' || s === '[' || s === '{') depth++; + else if (s === ')' || s === ']' || s === '}') { depth--; if (depth === 0) return j; } + } + return n - 1; + }; + while (i < n) { + const start = i; + // scan the statement head: up to ';' or '{' at depth 0 + let j = i, paren = -1; + while (j < n) { + const t = toks[j]; + if (t.k === 'op') { + if (t.s === ';' || t.s === '{') break; + if (t.s === '(' && paren < 0) paren = j; + if (t.s === '(' || t.s === '[') { j = match(j) + 1; continue; } + } + j++; + } + if (j >= n) break; + const first = toks[start]; + if (toks[j].s === '{') { + const end = match(j); + const isFn = paren >= 0 && !NOT_DECL.has(first.s) || (first.s === 'fn' && paren >= 0); + if (isFn) { + // name: the identifier before the first '(' (a method is Type.name) + let k = paren - 1, name = toks[k].s; + while (k >= 2 && toks[k - 1].s === '.' && toks[k - 2].k === 'id') { name = toks[k - 2].s + '.' + name; k -= 2; } + functions.push({ name, startTok: start, endTok: end, from: toks[start].off, to: toks[end].off + 1, line: toks[paren - 1].line }); + } + i = end + 1; + if (i < n && toks[i].s === ';') i++; + continue; + } + // a ';' statement + const head = toks.slice(start, j); + i = j + 1; + if (!head.length || NOT_DECL.has(first.s) || first.k !== 'id') continue; + const decl = parseDecl(head); + if (decl === null) continue; + if (decl.error) { problems.push(decl.error + ' at line ' + (first.line + 1)); continue; } + for (const d of decl.names) globals.push(d); + } + return { functions, globals, problems }; +} + +// a type (identifier, then * [..] .tag), then declarators: name [dims] [= init] separated by commas. +function parseDecl(head) { + let i = 0; + const n = head.length; + if (head[i].k !== 'id') return null; + i++; + for (;;) { + const t = head[i]; + if (!t) return null; + if (t.s === '*') { i++; continue; } + if (t.s === '.' && head[i + 1] && head[i + 1].k === 'id') { i += 2; continue; } + if (t.s === '[') { i = skipGroup(head, i) + 1; continue; } + break; + } + // a type needs a name after it; "a.b c" and "x = 1" and calls are not declarations + const names = []; + for (;;) { + const t = head[i]; + if (!t || t.k !== 'id') return names.length ? { error: 'unparsed declarator' } : null; + names.push({ name: t.s, tok: t }); + i++; + while (head[i] && head[i].s === '[') i = skipGroup(head, i) + 1; + if (head[i] && head[i].s === '=') { + // initializer: to the next ',' at depth 0 + i++; + while (i < n && head[i].s !== ',') { i = (head[i].s === '(' || head[i].s === '[' || head[i].s === '{') ? skipGroup(head, i) + 1 : i + 1; } + } + if (i >= n) return { names }; + if (head[i].s !== ',') return { error: 'unparsed declaration after ' + t.s }; + i++; + } +} +function skipGroup(toks, i) { + let depth = 0; + for (; i < toks.length; i++) { + const s = toks[i].k === 'op' ? toks[i].s : ''; + if (s === '(' || s === '[' || s === '{') depth++; + else if (s === ')' || s === ']' || s === '}') { depth--; if (depth === 0) return i; } + } + return toks.length - 1; +} + +// ---- classification ---- + +const KEYWORD_BEFORE_OPERAND = new Set(['return', 'else', 'case', 'do', 'in', 'as', 'new', 'sizeof', 'print']); + +// is the token at index i the end of an operand (so that a following & is binary)? +function endsOperand(t) { + if (!t) return false; + if (t.k === 'num' || t.k === 'str' || t.k === 'chr') return true; + if (t.k === 'id') return !KEYWORD_BEFORE_OPERAND.has(t.s); + return t.s === ')' || t.s === ']'; +} + +// 'w' when the reference at token index i is stored to: the target of = or a compound assignment, the +// operand of ++ or --, an element or field of it stored the same way, or its address is taken; else 'r'. +function classify(toks, i) { + // the lvalue chain around the name: a.b.name[..].f + let a = i; + while (a >= 2 && (toks[a - 1].s === '.' || toks[a - 1].s === '->') && toks[a - 2].k === 'id') a -= 2; + const before = toks[a - 1], before2 = toks[a - 2]; + if (before && before.k === 'op') { + if (before.s === '++' || before.s === '--') return 'w'; + if (before.s === '&' && !endsOperand(before2)) return 'w'; + } + let j = i + 1; + for (;;) { + const t = toks[j]; + if (!t || t.k !== 'op') break; + if (t.s === '[') { j = skipGroup(toks, j) + 1; continue; } + if ((t.s === '.' || t.s === '->') && toks[j + 1] && toks[j + 1].k === 'id') { j += 2; continue; } + break; + } + const t = toks[j]; + if (t && t.k === 'op' && (ASSIGN_OPS.has(t.s) || t.s === '++' || t.s === '--')) return 'w'; + return 'r'; +} + +// ---- the language server ---- + +const frame = m => { const b = Buffer.from(JSON.stringify(m)); return Buffer.concat([Buffer.from('Content-Length: ' + b.length + '\r\n\r\n'), b]); }; + +// one server run: open text, ask for the references of every position in `positions` ({line, character}), +// resolve to the array of location lists, in the same order. the server is spawned, not run synchronously, so +// several runs can share the work. +function referencesRun(server, text, positions, timeoutMs) { + return new Promise((resolve, reject) => { + const uri = 'file:///xref/compiler.zc'; + const msgs = [ + { jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, + { jsonrpc: '2.0', method: 'textDocument/didOpen', params: { textDocument: { uri, version: 1, languageId: 'zc', text } } }]; + positions.forEach((position, k) => msgs.push({ jsonrpc: '2.0', id: 2 + k, method: 'textDocument/references', params: { textDocument: { uri }, position, context: { includeDeclaration: true } } })); + msgs.push({ jsonrpc: '2.0', method: 'exit' }); + const child = cp.spawn(server, [], { stdio: ['pipe', 'pipe', 'pipe'] }); + const out = [], err = []; + const timer = setTimeout(() => { child.kill(); reject(new Error('language server timed out after ' + Math.round(timeoutMs / 1000) + 's (ZC_TEST_TIMEOUT_SCALE stretches it)')); }, timeoutMs); + child.stdout.on('data', c => out.push(c)); + child.stderr.on('data', c => err.push(c)); + child.stdin.on('error', () => {}); + child.on('error', e => { clearTimeout(timer); reject(new Error('language server ' + server + ': ' + e.message)); }); + child.on('close', status => { + clearTimeout(timer); + if (status !== 0) return reject(new Error('language server exited with status ' + status + ': ' + Buffer.concat(err).toString().slice(0, 400))); + const result = new Array(positions.length).fill(undefined); + let d = Buffer.concat(out); + while (d.length) { + const e = d.indexOf('\r\n\r\n'); + if (e < 0) break; + const len = Number(/Content-Length:\s*(\d+)/i.exec(d.subarray(0, e).toString())[1]); + const m = JSON.parse(d.subarray(e + 4, e + 4 + len)); + d = d.subarray(e + 4 + len); + if (m.id >= 2) result[m.id - 2] = m.result || []; + } + resolve(result); + }); + child.stdin.end(Buffer.concat(msgs.map(frame))); + }); +} +// the requests are dealt out to `jobs` server processes (each checks the document once, about half a +// minute interpreted); one process per core is about as fast as it gets. +async function references(server, text, positions, timeoutMs, jobs) { + jobs = Math.max(1, Math.min(jobs, positions.length)); + const slices = Array.from({ length: jobs }, () => []); + positions.forEach((p, k) => slices[k % jobs].push(k)); + const parts = await Promise.all(slices.map(ks => referencesRun(server, text, ks.map(k => positions[k]), timeoutMs))); + const out = new Array(positions.length); + slices.forEach((ks, j) => ks.forEach((k, n) => { out[k] = parts[j][n]; })); + return out; +} + +// ---- the analysis ---- + +// parts: [{name, text}] in order. returns {globals: [{name, part, line, kind, writes: {fn: n}, reads: {fn: n}}], notes} +async function analyze(compilerDir, server, timeoutMs, jobs) { + const names = fs.readFileSync(path.join(compilerDir, 'parts'), 'utf8').split(/\r?\n/).filter(Boolean); + const parts = names.map(name => ({ name, text: fs.readFileSync(path.join(compilerDir, name), 'utf8') })); + const text = parts.map(p => p.text).join(''); + const toks = lex(text); + const { functions, globals, problems } = structure(toks); + // where each part starts, in lines of the joined text + const starts = []; let acc = 0; + for (const p of parts) { starts.push(acc); acc += p.text.split('\n').length - 1; } + const partOf = line => { let k = starts.length - 1; while (k > 0 && starts[k] > line) k--; return k; }; + const byPos = new Map(toks.map((t, k) => [t.line + ':' + t.col, k])); + const notes = [...problems.map(p => 'declaration not parsed: ' + p)]; + + // duplicates would mean one name is declared twice; the second is ignored with a note + const seen = new Map(), decls = []; + for (const g of globals) { + if (seen.has(g.name)) { notes.push('declared twice, the first is used: ' + g.name); continue; } + seen.set(g.name, g); decls.push(g); + } + const answers = await references(server, text, decls.map(g => ({ line: g.tok.line, character: g.tok.col })), timeoutMs, jobs); + + // a function of the token index, by binary search over the sorted, non-overlapping ranges + const fnAt = k => { + let lo = 0, hi = functions.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1, f = functions[mid]; + if (k < f.startTok) hi = mid - 1; else if (k > f.endTok) lo = mid + 1; else return f.name; + } + return '(top level)'; + }; + const result = []; + let unmapped = 0, noDecl = []; + decls.forEach((g, gi) => { + const locs = answers[gi]; + const entry = { name: g.name, part: parts[partOf(g.tok.line)].name, line: g.tok.line - starts[partOf(g.tok.line)] + 1, writes: {}, reads: {} }; + const declKey = g.tok.line + ':' + g.tok.col; + let sawDecl = false; + for (const loc of locs || []) { + const key = loc.range.start.line + ':' + loc.range.start.character; + if (key === declKey) { sawDecl = true; continue; } + const k = byPos.get(key); + if (k === undefined || toks[k].s !== g.name) { unmapped++; continue; } + const into = classify(toks, k) === 'w' ? entry.writes : entry.reads; + const f = fnAt(k); + into[f] = (into[f] || 0) + 1; + } + if (!sawDecl) noDecl.push(g.name); + // a function that writes a global is a writer; it is a reader only when it never writes + for (const f of Object.keys(entry.writes)) delete entry.reads[f]; + result.push(entry); + }); + if (noDecl.length) notes.push('the server did not return the declaration for ' + noDecl.length + ' names (not symbols it knows?): ' + noDecl.join(' ')); + if (unmapped) notes.push(unmapped + ' references did not land on an identifier of that name and were ignored'); + result.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0); + return { globals: result, notes, functions: functions.length, tokens: toks.length }; +} + +// ---- output ---- + +const sortedKeys = o => Object.keys(o).sort(); +function reportText(res) { + const lines = []; + for (const g of res.globals) { + lines.push(g.name + ' (' + g.part + ':' + g.line + ')'); + lines.push(' writes: ' + (sortedKeys(g.writes).map(f => f + ' (' + g.writes[f] + ')').join(', ') || '-')); + lines.push(' reads: ' + (sortedKeys(g.reads).map(f => f + ' (' + g.reads[f] + ')').join(', ') || '-')); + lines.push(''); + } + return lines.join('\n'); +} +function reportJson(res) { + const globals = {}; + for (const g of res.globals) { + const w = {}, r = {}; + for (const f of sortedKeys(g.writes)) w[f] = g.writes[f]; + for (const f of sortedKeys(g.reads)) r[f] = g.reads[f]; + globals[g.name] = { part: g.part, line: g.line, writers: sortedKeys(g.writes), writes: w, readers: sortedKeys(g.reads), reads: r }; + } + return JSON.stringify({ generated_by: 'src/tools/xref.js', globals }, null, 1) + '\n'; +} +// the baseline: one global per line, the name then its sorted writer functions +function baselineText(res) { + return res.globals.map(g => [g.name, ...sortedKeys(g.writes)].join(' ')).join('\n') + '\n'; +} +function parseBaseline(text) { + const m = new Map(); + for (const line of text.split(/\r?\n/)) { + const w = line.trim().split(/\s+/).filter(Boolean); + if (w.length) m.set(w[0], new Set(w.slice(1))); + } + return m; +} +// {added: [[global, fn]], removed: [[global, fn]], newGlobals, goneGlobals} +function compare(res, base) { + const added = [], removed = [], newGlobals = [], goneGlobals = []; + const now = new Set(res.globals.map(g => g.name)); + for (const g of res.globals) { + const had = base.get(g.name); + if (!had) { newGlobals.push(g.name); } + for (const f of sortedKeys(g.writes)) if (!had || !had.has(f)) added.push([g.name, f]); + if (had) for (const f of [...had].sort()) if (!g.writes[f]) removed.push([g.name, f]); + } + for (const name of base.keys()) if (!now.has(name)) goneGlobals.push(name); + return { added, removed, newGlobals, goneGlobals }; +} + +// ---- command line ---- + +const USAGE = `usage: src/tools/xref.sh [options] +cross-reference of the compiler's global variables: for each top-level variable of src/compiler/parts, +the functions that write it and the functions that only read it. exact scoping comes from the language +server (textDocument/references), so a local that shadows a global is not counted. a write is the target +of = or a compound assignment, a ++ or --, an indexed or field store, or an address taken (&x). +options: + (none) write build/xref/report.txt and report.json + --check also compare the writers with tests/xref.baseline: exit 1 and print the difference when + a global gains a writer the baseline does not list (a new reader is fine; a removed + writer is only noted) + --update rewrite tests/xref.baseline from the current writers + --server PATH language server to use (default build/bin/zclsp; build it with make) + --compiler-dir DIR compiler parts directory (default src/compiler) + --baseline FILE baseline file (default tests/xref.baseline) + --out DIR directory for the reports (default build/xref) + --jobs N language server processes to share the work (default: one per core, or XREF_JOBS) + --help, -h show this help and exit +exit status: 0 ok, 1 --check found a new writer, 2 bad usage, 3 the analysis failed. +the language server is interpreted on hosts without the arm64 jit, so a run takes tens of seconds.`; + +async function main(argv) { + const opt = { check: false, update: false, server: path.join(ROOT, 'build', 'bin', 'zclsp'), dir: path.join(ROOT, 'src', 'compiler'), baseline: path.join(ROOT, 'tests', 'xref.baseline'), out: path.join(ROOT, 'build', 'xref'), jobs: Number(process.env.XREF_JOBS) || Math.min(os.cpus().length, 8) }; + const needs = (k) => { if (k + 1 >= argv.length) { process.stderr.write('xref: ' + argv[k] + ' needs an argument; use --help for usage\n'); process.exit(2); } return argv[k + 1]; }; + for (let k = 0; k < argv.length; k++) { + const a = argv[k]; + if (a === '--help' || a === '-h') { process.stdout.write(USAGE + '\n'); return 0; } + else if (a === '--check') opt.check = true; + else if (a === '--update') opt.update = true; + else if (a === '--server') opt.server = path.resolve(needs(k++)); + else if (a === '--compiler-dir') opt.dir = path.resolve(needs(k++)); + else if (a === '--baseline') opt.baseline = path.resolve(needs(k++)); + else if (a === '--out') opt.out = path.resolve(needs(k++)); + else if (a === '--jobs') { opt.jobs = Number(needs(k++)); if (!(opt.jobs >= 1)) { process.stderr.write('xref: --jobs needs a positive number\n'); return 2; } } + else { process.stderr.write('xref: unknown option ' + a + '; use --help for usage\n'); return 2; } + } + if (opt.check && opt.update) { process.stderr.write('xref: --check and --update exclude each other\n'); return 2; } + let res; + try { + if (!fs.existsSync(opt.server)) throw new Error('no language server at ' + opt.server + ' (run make)'); + const scale = Number(process.env.ZC_TEST_TIMEOUT_SCALE) || 1; + res = await analyze(opt.dir, opt.server, 3600000 * scale, opt.jobs); + } catch (e) { process.stderr.write('xref: ' + e.message + '\n'); return 3; } + fs.mkdirSync(opt.out, { recursive: true }); + fs.writeFileSync(path.join(opt.out, 'report.txt'), reportText(res)); + fs.writeFileSync(path.join(opt.out, 'report.json'), reportJson(res)); + for (const n of res.notes) process.stderr.write('xref: note: ' + n + '\n'); + const rel = p => path.relative(process.cwd(), p) || p; + process.stdout.write('xref: ' + res.globals.length + ' globals, ' + res.functions + ' functions; wrote ' + rel(path.join(opt.out, 'report.txt')) + ' and report.json\n'); + if (opt.update) { + fs.writeFileSync(opt.baseline, baselineText(res)); + process.stdout.write('xref: wrote ' + rel(opt.baseline) + '\n'); + return 0; + } + if (opt.check) { + let base; + try { base = parseBaseline(fs.readFileSync(opt.baseline, 'utf8')); } catch (e) { process.stderr.write('xref: cannot read the baseline ' + opt.baseline + ' (use --update)\n'); return 3; } + const d = compare(res, base); + for (const [g, f] of d.removed) process.stdout.write('note: ' + g + ' is no longer written by ' + f + '\n'); + for (const g of d.goneGlobals) process.stdout.write('note: global ' + g + ' is gone\n'); + if (d.added.length) { + process.stdout.write('FAIL: new writers of compiler globals (a new reader is fine; review these, then run src/tools/xref.sh --update):\n'); + for (const [g, f] of d.added) process.stdout.write(' ' + g + ' is now written by ' + f + (d.newGlobals.includes(g) ? ' (new global)' : '') + '\n'); + return 1; + } + process.stdout.write('xref: check ok, no new writers against ' + rel(opt.baseline) + '\n'); + } + return 0; +} + +if (require.main === module) main(process.argv.slice(2)).then(c => { process.exitCode = c; }); +else module.exports = { lex, structure, classify, analyze, main }; diff --git a/src/tools/xref.sh b/src/tools/xref.sh new file mode 100755 index 0000000..b014dae --- /dev/null +++ b/src/tools/xref.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# cross-reference of the compiler's global state: who writes each global, who only reads it. +# a shell wrapper around src/tools/xref.js (node). run `src/tools/xref.sh --help` for the options. +# needs build/bin/zclsp (make): exact scoping comes from the language server's references. +set -e +here=$(cd "$(dirname "$0")" && pwd) +if ! command -v node >/dev/null 2>&1; then + case "$1" in + --help|-h) + echo "usage: src/tools/xref.sh [--check|--update] [--server PATH] [--compiler-dir DIR] [--baseline FILE] [--out DIR] [--jobs N]" + echo "(node is required to run it)" + exit 0 ;; + esac + echo "xref: node is required" >&2 + exit 3 +fi +exec node "$here/xref.js" "$@" -- cgit v1.2.3 From 1660c16f4f296dee8c1efc972d4671208ed05a8d Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 15:41:56 +0330 Subject: xref: checked-in baseline, --check self-test hooked into the lsp section, todo --- README.md | 2 +- TODO.md | 47 ++-- src/tools/xref_selftest.sh | 21 ++ tests/test.sh | 3 + tests/xref.baseline | 658 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 702 insertions(+), 29 deletions(-) create mode 100755 src/tools/xref_selftest.sh create mode 100644 tests/xref.baseline (limited to 'src') diff --git a/README.md b/README.md index 0644d4e..35e94b6 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ make js make run PROG=examples/hello.zc ``` -the cores embed their genome with c23 `#embed`, so the c compiler has to be clang 19 or gcc 15 or later. `src/tools/findcc.sh` finds one (`CC` set on the command line or in the environment is used as given, and must support it too). `make genome` bootstraps the compiler, compares two self-compiled images, and writes `build/core/genome.bin` only after they agree; `make` reruns it whenever a source of the genome changes, and the cores that embed it are rebuilt after it. `make check` runs the smaller suite while iterating. `make parity` (`src/tools/parity.sh`, `--help` for options) compiles the test and example programs with zc0 and with the self-hosted compiler and compares what they print, which the fixpoint check cannot see; programs zc0 rejects are skipped and counted, intended differences are listed in `tests/parity.known`, and work files go to `build/parity/`. `make xref` (`src/tools/xref.sh`, `--help` for options) lists, for every top-level variable of the compiler, the functions that write it and the functions that only read it, in `build/xref/report.txt` and `report.json`; the exact scoping comes from the language server's `textDocument/references` on the joined compiler source, so a local that shadows a global is not counted, and a write is an `=`, a compound assignment, `++` or `--`, an indexed store, or `&x`. `src/tools/xref.sh --check` (or `make xref XREF_ARGS=--check`) exits 1 when a global gains a writer that `tests/xref.baseline` does not list, so a change that lets a new function write core compiler state shows up in review; `--update` rewrites the baseline. the language server is interpreted without the arm64 jit, so a run takes minutes. `ONBOARD.md` is a reading and verification order for getting to know the code. +the cores embed their genome with c23 `#embed`, so the c compiler has to be clang 19 or gcc 15 or later. `src/tools/findcc.sh` finds one (`CC` set on the command line or in the environment is used as given, and must support it too). `make genome` bootstraps the compiler, compares two self-compiled images, and writes `build/core/genome.bin` only after they agree; `make` reruns it whenever a source of the genome changes, and the cores that embed it are rebuilt after it. `make check` runs the smaller suite while iterating. `make parity` (`src/tools/parity.sh`, `--help` for options) compiles the test and example programs with zc0 and with the self-hosted compiler and compares what they print, which the fixpoint check cannot see; programs zc0 rejects are skipped and counted, intended differences are listed in `tests/parity.known`, and work files go to `build/parity/`. `make xref` (`src/tools/xref.sh`, `--help` for options) lists, for every top-level variable of the compiler, the functions that write it and the functions that only read it, in `build/xref/report.txt` and `report.json`; the exact scoping comes from the language server's `textDocument/references` on the joined compiler source, so a local that shadows a global is not counted, and a write is an `=`, a compound assignment, `++` or `--`, an indexed store, or `&x`. `src/tools/xref.sh --check` (or `make xref XREF_ARGS=--check`) exits 1 when a global gains a writer that `tests/xref.baseline` does not list, so a change that lets a new function write core compiler state shows up in review; `--update` rewrites the baseline. the language server is interpreted without the arm64 jit, so a run takes tens of seconds (about 10 s with 8 servers on the box). `ONBOARD.md` is a reading and verification order for getting to know the code. ### command-line help diff --git a/TODO.md b/TODO.md index 210b4a3..fcb4e88 100644 --- a/TODO.md +++ b/TODO.md @@ -416,34 +416,25 @@ of the open list. text); return; }`. open: whether the keyword handler can decline when the next token is not a string (so `test` stays usable as a name). diagnostics print `zc: line N:` with no file name. -- compiler cross-reference report (`agents/xref-report`, one wip commit - e5d5136). done: `src/tools/xref.js` and `src/tools/xref.sh` (`--help`, - unknown option exits 2). the language server has no documentSymbol, so the - globals come from the tool's own parse of the joined compiler source (652 - globals, 628 functions, nothing left unparsed); writers and readers come from - one zclsp references request per global over 4 parallel servers. one full run - finished in 11m53s (about 2.5 s per request, about 30 s per server to check - the document); its output is in the zip at `agents/xref-report/xref-out/` - (report.txt, report.json), not yet reviewed by a person. top 15 globals by - number of writer functions, read off that json by the coordinator: xty 29, - xbor 23, xfloat 22, lv 21, xaf 21, lvk 20, xvk 19, ltag 17, xzero 17, - lstate 16, sflag 16, effect_value 15, fslots 13, panic 13, chkabort 12 (these - are the first candidates for the "group the parallel arrays" item). 60 - globals show no writer and 198 no reader (many are `_cap` twins); some of the - "no writer" ones are the known limits below, so check before trusting them. - written but never run: the `make xref` target, the README and ONBOARD.md - sections, the xref.sh entry in `tests/drivers/cli_help_test.py`. not done: - `tests/xref.baseline` (derive it from report.json, one line per global with - its sorted writers, or run `xref.sh --update` in the background), `--check` - and `--update` never run, the self-test (copy src/compiler to a scratch dir, - add one write to a global in another function, `xref.sh --compiler-dir DIR - --out SCRATCH --check` must exit 1), README says "minutes" where it should - say about 12 minutes. known limits: a store through a pointer (`*p = ...`), - an array written by a callee (`copy_w`, `memset`) and a global reached only - through its `_initial` array are not counted as writes. next: baseline, - `--check` (expect 0), self-test, `python3 tests/drivers/cli_help_test.py - --group scripts`, commit. a 12 minute run is too slow for every ci job; run - it nightly or only when `src/compiler/` changes. +- compiler cross-reference report (`agent/xref-report`, cherry-picked onto + work/xref without conflicts). done: `src/tools/xref.js` and + `src/tools/xref.sh` (`--help`, unknown option exits 2), the `make xref` + target, the readme and onboard.md sections, and the xref.sh entry in + `tests/drivers/cli_help_test.py` (passes with `--group scripts`). + `tests/xref.baseline` is checked in, written by `xref.sh --update` (658 + globals, 633 functions, one line per global with its sorted writers); a run + took about 10 s with 8 servers, not 12 minutes, so the readme now says tens + of seconds. `--check` against it exits 0, and exits 1 when one writer is + dropped from one baseline line. `src/tools/xref_selftest.sh [server]` does + both cases in a scratch directory and runs in the lsp section of + `tests/test.sh` next to the other node drivers (not in -q). known limits: a + store through a pointer (`*p = ...`), an array written by a callee + (`copy_w`, `memset`) and a global reached only through its `_initial` array + are not counted as writes. the baseline is not yet reviewed by a person; + the 60 "no writer" globals still need checking against those limits. open: + a self-test that adds a real write to a copy of src/compiler and runs + `--compiler-dir`, and deciding whether ci runs it on every job or only + when `src/compiler/` changes. ### decisions - no structs in zc0 (owner's call): zc0 stays the small trusted root of the diff --git a/src/tools/xref_selftest.sh b/src/tools/xref_selftest.sh new file mode 100755 index 0000000..36425e4 --- /dev/null +++ b/src/tools/xref_selftest.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# self-test of src/tools/xref.sh --check: the checked-in baseline passes (exit 0), and a copy of it +# with one writer removed from one line fails (exit 1). works in a scratch directory; the tree, +# tests/xref.baseline and build/xref are not touched. +# usage: src/tools/xref_selftest.sh [SERVER] (default build/bin/zclsp) +set -u +cd "$(dirname "$0")/../.." +case "${1:-}" in -h|--help) echo "usage: src/tools/xref_selftest.sh [SERVER]"; exit 0;; esac +server=${1:-build/bin/zclsp} +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +fail=0 +bash src/tools/xref.sh --server "$server" --out "$tmp/out" --check > "$tmp/ok.log" 2>&1; r=$? +if [ $r = 0 ]; then echo "ok xref --check passes against tests/xref.baseline" +else echo "FAIL xref --check against tests/xref.baseline: exit $r"; cat "$tmp/ok.log"; fail=1; fi +# drop the last writer of the first global that has one: --check must report it as a new writer +line=$(awk 'NF>=2{print NR; exit}' tests/xref.baseline) +sed "${line}s/ [^ ]*\$//" tests/xref.baseline > "$tmp/baseline" +bash src/tools/xref.sh --server "$server" --out "$tmp/out" --baseline "$tmp/baseline" --check > "$tmp/bad.log" 2>&1; r=$? +if [ $r = 1 ] && grep -q 'is now written by' "$tmp/bad.log"; then echo "ok xref --check fails when the baseline misses a writer" +else echo "FAIL xref --check with a writer removed from the baseline: exit $r (want 1)"; cat "$tmp/bad.log"; fail=1; fi +exit $fail diff --git a/tests/test.sh b/tests/test.sh index 22fdf30..4ca000f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -598,6 +598,9 @@ s_lsp() { node tests/drivers/lsp_sandbox.js $lsp > $T/lspsb.out 2>&1; r=$? grep -v '^ok' $T/lspsb.out chk $r 0 "lsp: compile-time code in a checked file cannot write files" + bash src/tools/xref_selftest.sh $lsp > $T/xref.out 2>&1; r=$? + grep -v '^ok' $T/xref.out + chk $r 0 "xref: --check passes on tests/xref.baseline and fails on a missing writer" # the diagnostics cache: its key rules, then the extension's client end to end on the built-in server node tests/drivers/diagcache_test.js > $T/dcache.out 2>&1; r=$? grep -v '^ok' $T/dcache.out diff --git a/tests/xref.baseline b/tests/xref.baseline new file mode 100644 index 0000000..985f9c7 --- /dev/null +++ b/tests/xref.baseline @@ -0,0 +1,658 @@ +R_CTX reserve_R_CTX +R_CTX_cap reserve_R_CTX +R_EXIT reserve_R_EXIT +R_EXIT_cap reserve_R_EXIT +R_LOOP reserve_R_LOOP +R_LOOP_cap reserve_R_LOOP +R_QBUF defer_tokens grab reserve_R_QBUF tokens +R_QBUF_cap reserve_R_QBUF +R_SEQ reserve_R_SEQ seqasm +R_SEQ_cap reserve_R_SEQ +R_SNAP reserve_R_SNAP +R_SNAP_cap reserve_R_SNAP +R_SRC reserve_R_SRC +R_SRC_cap reserve_R_SRC +abin add_binop reserve_assignment +af_mutation_tick af_mutate +aname add_binop reserve_assignment +ar_flat array_designator +ar_target array_designator +argfact args chk_reset +argloc arrvalue chk_reset +argorigin args +argorigin_mask args +argowner args +argowner_fact args +argtemporary args +arrloc arrvalue chk_reset +asgcap reserve_assignment +asm_fixes reserve_asm_references +asm_labelcap reserve_asm_labels +asm_names reserve_asm_labels +asm_refcap reserve_asm_references +asm_refs reserve_asm_references +asm_sites reserve_asm_labels +bincap reserve_binary +blen add_binop reserve_binary +bname add_binop reserve_binary +borrow_depth call ctx_pop fn_begin +borrow_entry borrow_prologue ctx_pop fn_begin +borrow_jump borrow_prologue ctx_pop fn_begin +borrow_pending ctx_pop emit_borrow_hold fn_begin +borrow_slots ctx_pop emit_borrow_hold fn_begin +bprec add_binop reserve_binary +bseq add_binop reserve_binary +bxdepth blockexpr chk_reset ctx_pop fn_begin +callpos chk_reset primary +capture_dummy +capture_enabled capture_prepare ctx_pop fn_begin +capture_width reserve_captures +ch chk_reset nextch src_pop +chbase src_pop src_push +check_on chk_reset topitem1 +chk_heap chk_unit start +chk_hook +chkabort block body chk_begin errend fatal giveup hl_begin primary probe_member probe_stmt replace_function unit_top +chkfuel chk_begin next0 +chkind chk_reset src_pop src_push +chptr nextch src_pop src_push +ckn ckind +ckst ckind +clo_env closure reserve_closures +clo_env_initial +clo_hi closure reserve_closures +clo_hi_initial +clo_lo closure reserve_closures +clo_lo_initial +clo_loc clocap_at reserve_captures reserve_closures +clo_loc_initial +clo_n clocap_at closure reserve_closures +clo_n_initial +clo_name clocap_at reserve_captures reserve_closures +clo_name_initial +clo_ret closure kw_return reserve_closures +clo_ret_initial +clo_via clocap_at reserve_captures reserve_closures +clo_via_initial +closure_cap reserve_closures +closure_expected args closure zc_init +cmode chk_begin chk_end +code_chunk chk_reset reserve_code syntax_resident +ctxsp ctx_pop ctx_push zc_init +curinc chk_unit effect_finish gcompile inc_compile +dbuf doc_char +dbuf_initial +dfr defer_drop defer_one kw_defer +dfr_base chk_reset ctx_pop fn_begin +dfr_ctx defer_one +dfr_lsp defer_one +dfr_on chk_reset defer_one +dgbuf chk_begin dg_close dg_expand dg_open +dgcap chk_begin dg_expand dg_open +dgdrop chk_unit dg_close dg_open +dggrow dg_open +dgmode chk_begin +dgn chk_begin dg_close +dgopen chk_begin dg_close dg_open +dgpos chk_begin dg_close +dgprev chk_reset expect +dgseen dg_dup +dgseen_cap dg_dup +dgseen_initial +dgseen_n chk_begin dg_dup +dlast docblock doccomment +dline chk_reset lex1 topitem0 +dn chk_reset doc_char docblock doccomment +dnm chk_reset fndef gconst gdecl2 prescan_fn topitem0 +dnmb doc_attach doc_name_room docsym sig_attach +dnmb_initial +docbufcap doc_char +docnamecap doc_name_room +docpending chk_reset docblock doccomment lex1 +docs_on capture_scan chk_reset kw_docs +docsnapcap topitem0 +dry capture_scan chk_reset hl_begin hl_end prescan typearg +dsect chk_reset docblock doccomment +dsn chk_reset doc_attach topitem0 +dsnap topitem0 +dsnap_initial +dtag dtype +dty dtype genparams topitem1 +ebcap eb_char +ebn eb_char enhdef enum_tagged kw_extern +ebuf eb_char ebn10 ebs enum_tagged kw_extern +ebuf_initial +effect_argument_base args call methcall postfix_tail +effect_current ctx_pop effect_note fn_begin fndef kw_asm zc_init +effect_edges effect_arg zc_init +effect_floor compile_list_context ctx_pop unit_top zc_init +effect_requirements effect_finish effect_require_mode zc_init +effect_symbols effect_record reserve_symbols zc_init +effect_value array_cap array_len array_pop arrvalue bin call ctx_pop field fn_begin kw_share_value methcall postfix_tail primary ternary unary +epichain chk_reset ctx_pop fn_begin kw_return +evcap ev_put +evn chk_unit ev_put +evrec ev_put +export_list export_add export_owners +export_path export_owner +export_tail export_add export_owners +extop chk_reset exit_rec loop_end +factcap putfact +fchk chk_reset ctx_pop defer_one fn_begin kw_unchecked +fdepth block blockexpr casebody chk_reset ctx_pop fn_begin kw_for kw_switch substmt +fframe chk_reset ctx_pop prologue +fieldcap field_note +fieldn field_note types_init +fieldnames field_note +fieldoffs field_note +fieldtypes field_note +file_libs lib_files +file_libs_read lib_files +finfer ctx_pop fn_begin fndef retcheck +finsym ctx_pop fndef +fixcap reserve_fixes +fixfree fixall fixnew zc_init +fldi fieldscan +fldst fieldscan +flen fdrop putfact +flen_initial +flo fdrop putfact +flo_initial +floc fdrop putfact +floc_initial +fn_method_compile kw_struct +fn_return_type ctx_pop fn_begin kw_return +fnext fixall fixnew reserve_fixes +fnmeta addsym fn_remember replace_function reserve_symbols +fnpatchseq replace_function syntax_host_function +forstep chk_reset kw_for +fortest bin chk_reset kw_for +fret chk_reset ctx_pop fn_begin retcheck +fs_ok expr incdec kw_for +fsite fixnew reserve_fixes +fslots array_dynamic_literal array_literal_tokens chk_reset closure ctx_pop embed_bind emit_borrow_hold field fn_begin fndef kw_switch localdecl o3mkcache +ft_bd bin +ft_ir bin +ft_iv bin kw_for +ft_nops bin +fty chk_reset closure ctx_pop fn_begin fndef +gbind primary +gbuf grec_add topitem +gcap grec_add topitem +gi_ar ginit reserve_instances +gi_done gdrain ginit ginstance reserve_instances +gi_n chk_reset ginstance +gi_sym ginit ginstance reserve_instances +gi_t ginit ginstance reserve_instances +gib +gicap reserve_instances +ginsn gcompile +ginst chk_reset gcompile +gn grec_add topitem +gnew chk_reset fndef topitem +gnew_nm +gnew_np +gng gcompile +gnm +grec capture_scan chk_reset topitem +gt_imports ginit gtmake reserve_generics +gt_module ginit gtmake reserve_generics +gt_n chk_reset gtmake +gt_nm ginit reserve_generics +gt_np ginit gtmake reserve_generics +gt_source ginit gtmake reserve_generics +gt_sym ginit gtmake reserve_generics +gt_tok gattach ginit gtmake reserve_generics +gtab chk_reset ginit +gtcap reserve_generics +hashcap reserve_names +hdelta build_image_grow chk_unit reserve_code start syntax_resident +heldk shield_types unshield_types +here build_image_grow chk_unit emit reserve_code start syntax_resident +hl_cap hint_set +hl_lev hint_set +hl_n chk_unit hint_set +hl_off hint_set +hlimit build_image_grow chk_unit reserve_code start syntax_resident +ht_end ht_note +ht_n ht_note +ht_start ht_note +ht_type ht_note +htcap ht_note +icon_buf icon_text +icon_done chk_unit kw_include +icon_names lib_names +if_edges chk_reset chk_unit if_link +if_local chk_reset chk_unit fndef newloc note_store reserve_locals +if_value array_literal_tokens bin call chk_reset chk_unit ctx_pop field postfix_tail primary ternary unary +ifcall args call chk_unit closure compile_list_context ctx_pop methcall postfix_tail +ifcur body chk_unit closure compile_list_context ctx_pop fndef if_begin +ifgk chk_unit gdecl gdecl2 +ifkk chk_unit fndef methdef topitem1 +ifkp kw_var methdef topitem1 +ifon +ifr chk_unit if_get +ifrcap chk_unit if_get +ifs chk_unit if_get reserve_symbols +ifstrict +ift chk_unit fact_add +iftcap chk_unit fact_add +ifx chk_unit fact_add +ifxcap chk_unit fact_add +image_active build_image_grow syntax_resident +image_buffer build_image_grow reserve_code +image_capacity build_image_grow reserve_code +image_org build_image_grow +image_size build_image_grow +image_words build_image_grow +incdepth chk_unit gcompile inc_compile +incdir inc_compile +included_modules chk_reset kw_include kw_namespace +incplace kw_include +increoot +incres inc_open lib_file_names lib_files +infn chk_reset ctx_pop epilogue fn_begin +isp chk_reset hl_begin hl_end src_pop src_push +jps jadd +jps_initial +jpscap jadd +kopcap reserve_operators +kops addkop reserve_operators +kw_expr chk_reset ctx_pop kwcall +laaddr array_literal_tokens localdecl newloc reserve_locals unary +laaddr_initial +lafact af_forget af_store array_literal_tokens expr incdec localdecl newloc reserve_locals +lafact_initial +larr array_literal_tokens embed_bind localdecl newloc reserve_locals +lbase ctx_pop fn_begin zc_init +lendcap lend_add +lendrec lend_add +lheads defer_one local_bind local_drop +lheads_initial +lhome block_close join newloc reserve_locals snap_restore touch +line chk_reset nextch src_pop src_push +literal_doc_cap literal_doc_note +literal_doc_n chk_unit literal_doc_note +literal_docs literal_doc_note +live casebody chk_reset ctx_pop fn_begin join2 kw_break kw_continue kw_if kw_return kw_switch loop_end ternary +lkeycap local_bind +lkeyn local_bind zc_init +lkeys local_bind +lkeys_initial +lleaf assigncheck embed_bind localdecl reserve_locals +lleaf_initial +lname newloc reserve_locals +lnsz assigncheck embed_bind localdecl reserve_locals +lnsz_initial +local_captured clocap_at newloc reserve_locals +local_cell capture_cell newloc reserve_locals +local_end_info assigncheck end_owner newloc reserve_locals +local_ended assigncheck end_owner newloc reserve_locals +local_origin assigncheck newloc reserve_locals +local_origin_mask assigncheck newloc reserve_locals +loccap reserve_locals +loopbase ctx_pop fn_begin +loopfacts_cap loop_push reserve_R_LOOP +lown assigncheck closure fndef newloc reserve_locals touch +lowner assigncheck join newloc reserve_locals snap_restore +lpfact bpoison kw_for loop_push reserve_R_LOOP +lpfact_initial +lpos closure embed_bind fndef localdecl newloc reserve_locals +lpos_initial +lprev local_bind reserve_locals +lprev_initial +lptr fndef postfix_tail reserve_locals unary +lptr_initial +lreadpos newloc primary reserve_locals +lreadpos_initial +lrskind closure fndef newloc reserve_locals +lrskind_initial +lrsoff closure embed_bind fndef localdecl newloc reserve_locals +lrsoff_initial +lsets chk_unit +lslot newloc reserve_locals +lsp ctx_pop loop_end loop_push zc_init +lstate alias_end assigncheck backedge block_close closure end_owner fndef handup join newloc recast reserve_locals retcheck snap_cmp snap_restore touch +lt_at lt_note +lt_len lt_note +lt_n lt_note +lt_off lt_note +lt_on +lt_type lt_note +ltag asmblock embed_bind expr fndef incdec kw_o3 localdecl luse newloc o3mkcache o3promote reserve_locals rval setflt setstr touch unary +ltcap lt_note +ltxn lt_note +ltxt lt_note +ltxtcap lt_note +ltype array_literal_tokens closure embed_bind fndef kw_switch localdecl newloc reserve_locals +lused assigncheck luse newloc reserve_locals +lv array_dynamic_literal array_literal_tokens bin blockexpr call chk_reset closure ctx_pop enhcall expr field fn_begin incdec kwcall methcall newtyped postfix_tail primary rval ternary unary +lvescaped closure newloc reserve_locals unary +lvi chk_reset ctx_pop field postfix_tail primary +lvk array_dynamic_literal array_literal_tokens bin blockexpr call chk_reset closure ctx_pop enhcall expr field fn_begin incdec kwcall methcall newtyped postfix_tail primary ternary unary +lvkind closure expr incdec newloc reserve_locals unary vf_effect vf_store +lvvalue newloc reserve_locals vf_store +mdbg chk_reset kw_memdebug +mdbg_flags chk_reset kw_memdebug +mdg kw_struct +mdn +missing_imports primary source_lookup unit_end +mnem +module_imports chk_unit compile_list_context ctx_pop enum_tagged gcompile inc_compile kw_extern kw_include kw_namespace prescan unit_stdin +module_name chk_unit compile_list_context ctx_pop gcompile inc_compile kw_namespace prescan unit_stdin +mstruct kw_struct struct_const +namecount intern zc_init +namehash intern reserve_names +nameoff chk_unit name +names_read lib_names +nasg add_binop zc_init +nbin add_binop zc_init +nclo chk_reset closure +ndfr defer_drop kw_defer +nenh blob_load build_image_grow chk_reset enhdef +nerr chk_begin chk_reset errat prescan replace_function +newleaf array_dynamic_literal array_pop call localdecl newtyped +newsz array_dynamic_literal array_pop call chk_reset localdecl newtyped +nfact chk_reset fdrop putfact +nfixu fixnew zc_init +ngbind call chk_reset primary +nif chk_unit if_get +nift chk_unit fact_add +nifx chk_unit fact_add +njps chk_reset jadd join2 kw_if loop_end +nkop addkop zc_init +nlend lend_add lend_remove +nloc local_drop newloc zc_init +nnumops chk_reset types_init +no3free block_close chk_reset o3reset o3take +no3pend chk_reset fndef kw_o3 o3flush +no3sl chk_reset o3promote o3reset +nomagic chk_reset localdecl +npe enum_tagged zc_init +npm kw_enum zc_init +nsym addsym popsyms zc_init +ntd kw_typedef popsyms td_end zc_init +ntl tnew types_init +ntok next +ntp chk_reset ctx_pop fndef fndone genparams kw_struct methdef ptype1 topitem1 +numops types_init +nun add_unop zc_init +o3cache kw_o3 newloc o3mkcache reserve_locals +o3cache_initial +o3dsc o3region +o3fn0 chk_reset fndef o3flush +o3free block_close o3take +o3fs fndef o3flush +o3na chk_reset fndef o3flush +o3nreg chk_reset o3reset o3take +o3nxt chk_reset fndef methdef topitem1 +o3on chk_reset fndef kw_o3 +o3one kw_o3 +o3pend kw_o3 +o3pend_initial +o3pendcap kw_o3 +o3rg o3promote +o3sl o3promote +panic chk_begin chk_reset die diefound dien effect_finish expect hl_begin prescan replace_function resync topsync unit_end +pe_first enum_tagged reserve_tagged_enums +pe_first_initial +pe_n enum_tagged reserve_tagged_enums +pe_n_initial +pe_name enum_tagged reserve_tagged_enums +pe_name_initial +pe_st enum_tagged reserve_tagged_enums +pe_st_initial +pecap reserve_tagged_enums +pendot chk_reset hl_begin lex1 +pk_tk peekat +pk_tv peekat +pm_en enum_tagged kw_enum reserve_enum_members +pm_en_initial +pm_exp kw_enum reserve_enum_members +pm_exp_initial +pm_fnm kw_enum reserve_enum_members +pm_fnm_initial +pm_ft kw_enum reserve_enum_members +pm_ft_initial +pm_name kw_enum reserve_enum_members +pm_name_initial +pm_nf kw_enum reserve_enum_members +pm_nf_initial +pm_off enum_tagged reserve_enum_members +pm_off_initial +pm_tag kw_enum reserve_enum_members +pm_tag_initial +pm_val kw_enum reserve_enum_members +pm_val_initial +pmcap reserve_enum_members +pnoffs closure enhdef fndef +pnplaces closure enhdef fndef +poolblocks intern zc_init +poolchunk intern +poolend intern zc_init +pooltop intern zc_init +pq_gl next peekat src_pop +pq_ln next peekat src_pop +pq_n capture_scan chk_reset hl_begin hl_end next peekat src_pop src_push +pq_tk next peekat src_pop +pq_tv next peekat src_pop +probe_cap probe_add +probe_decl probe_add +probe_has body chk_unit primary probe_member probe_snap unit_top +probe_kind probe_add +probe_mem chk_unit probe_member +probe_n body chk_unit probe_add probe_snap unit_top +probe_name probe_add +probe_off +probe_state probe_add +probe_static chk_unit primary +probe_type probe_add +ptop chk_unit compile_list_context start syntax_resident +qdone chk_reset hl_begin nextch src_push +qesc readsrc src_push srcend +qp readsrc src_push srcend +qst readsrc src_push srcend +readonly_types tconst types_init +receiver_methods add_method zc_init +rsbase chk_unit +rscap chk_unit rs_put +rsn chk_unit rs_put +rsrec chk_unit os_at rs_fix rs_put rs_sort vf_at +rstmp chk_unit rs_put +s_alloc zc_init +s_amp zc_init +s_andand zc_init +s_arrow zc_init +s_as zc_init +s_at_s types_init +s_case zc_init +s_cat_s types_init +s_check zc_init +s_colon zc_init +s_comma zc_init +s_const zc_init +s_copy_w zc_init +s_d zc_init +s_dat types_init +s_dcap types_init +s_dec zc_init +s_default zc_init +s_dlen types_init +s_dnew types_init +s_do zc_init +s_docs types_init +s_dot zc_init +s_dpop types_init +s_dpush types_init +s_dresize types_init +s_else zc_init +s_enhance zc_init +s_enum zc_init +s_eq zc_init +s_eq_s types_init +s_eqeq types_init +s_fatten types_init +s_fixed types_init +s_fn zc_init +s_fovf zc_init +s_frame_busy zc_init +s_frame_floor zc_init +s_frame_grow zc_init +s_frame_guard zc_init +s_frame_leave zc_init +s_frame_node zc_init +s_frame_top zc_init +s_free zc_init +s_ge types_init +s_gt types_init +s_hash zc_init +s_hitems types_init +s_hlen types_init +s_icon types_init +s_immediate zc_init +s_inc zc_init +s_include zc_init +s_lb zc_init +s_ld16 types_init +s_ld32 types_init +s_ld8 types_init +s_le types_init +s_le_s types_init +s_len_s types_init +s_lfa types_init +s_lk zc_init +s_lp zc_init +s_lt types_init +s_lt_s types_init +s_m zc_init +s_main zc_init +s_memdebug zc_init +s_memset zc_init +s_micro zc_init +s_minus zc_init +s_n_to_s types_init +s_ne types_init +s_new types_init +s_new_k types_init +s_new_oz types_init +s_new_raw types_init +s_new_s types_init +s_new_t types_init +s_o3 types_init +s_off types_init +s_on types_init +s_oob types_init +s_oror zc_init +s_outside types_init +s_own zc_init +s_plus types_init +s_put_bs types_init +s_put_s types_init +s_q zc_init +s_qnew zc_init +s_rb zc_init +s_recycle zc_init +s_recycle_leaf types_init +s_rk zc_init +s_rp zc_init +s_scrub zc_init +s_semi zc_init +s_shr types_init +s_slash zc_init +s_slice_s types_init +s_small types_init +s_st16 types_init +s_st32 types_init +s_st8 types_init +s_star zc_init +s_start zc_init +s_static zc_init +s_std types_init +s_struct types_init +s_this types_init +s_type types_init +s_typecheck types_init +s_typedef types_init +s_u zc_init +s_unchecked zc_init +s_var zc_init +s_vm zc_init +s_while zc_init +sdoff addsym def_note reserve_symbols +sdsrc addsym def_note reserve_symbols +seqtop seqasm zc_init +sfix addsym enhdef fndef fnref reserve_symbols +sflag add_kw addsym blob_load call enhdef fndef fndone fwdfn_at ginstance kw_extern kw_struct methcall methdecl prescan_fn reserve_symbols retcheck +sgb sig_attach +sgb_initial +sigcap sig_attach +skind addsym reserve_symbols +sname addsym popsyms reserve_symbols +snaptop ctx_pop snap_pop snap_push zc_init +snext addsym reserve_symbols td_unlink +ssig addsym blob_load enhdef fndef ginstance prescan_fn reserve_symbols retcheck +statement_hooks add_statement_hook chk_reset kw_statement_hook +std_buf std_text +std_done chk_unit kw_include +std_names lib_names +stubgen enum_tagged kw_extern +stype addsym embed_bind enum_tagged fndef gdecl2 kw_enum reserve_symbols retcheck +sval addsym blob_emit fndef reserve_symbols +svkind expr reserve_symbols vf_symbols +svvalue expr reserve_symbols vf_symbols +symcap reserve_symbols +syntax_host_floor syntax_resident +syntax_host_from syntax_resident +syntax_host_symbols syntax_host_function syntax_resident +tbool types_init +tbstr types_init +tbuf add_binop floatok lex1 oplex seqasm token_char +tbuf_initial +tbufcap add_binop token_char +tc_on chk_reset kw_typecheck +tchar types_init +tdcap kw_typedef +tdstk kw_typedef +tf32 types_init +tfloat types_init +tglue chk_reset lex1 next peekat src_pop tunpack +tint types_init +tk chk_reset floatok lex1 next next0 oplex peekat src_pop +tline chk_reset lex1 tunpack +toff chk_reset lex1 tunpack +token_block_doc lex1 +toklen chk_reset lex tunpack +tpend chk_reset next +tpline chk_reset next +tpn kw_struct ptype1 +tv chk_reset closeangle floatok lex1 next next0 oplex peekat src_pop +tvauto chk_reset rettype +tvoid types_init +tword types_init +tyend talloc types_init +typeblocks talloc types_init +typecap tnew +typelist tnew types_init +tytop talloc types_init +ubase build_image_grow chk_unit start +ulen add_unop reserve_unary +uname add_unop reserve_unary +uncap reserve_unary +useq add_unop reserve_unary +vislo build_image_grow chk_unit start syntax_resident +xaf af_forget args array_cap array_dynamic_literal array_len array_literal_tokens array_pop bin call chk_reset expr field incdec kw_share_value kwcall methcall newtyped postfix_tail primary ternary unary +xbor array_cap array_dynamic_literal array_len array_literal_tokens array_pop assigncheck bin blockexpr call chk_reset closure discard enhcall field fn_begin kw_share_value kwcall methcall postfix_tail primary ternary typearg unary +xcls array_cap array_len array_pop array_push array_resize chk_reset ctx_pop kw_asm kwcall quote_tokens +xconst array_dynamic_literal array_literal_tokens bin call chk_reset closure kw_share_value methcall postfix_tail primary unary +xcval array_literal_tokens primary unary +xfloat array_dynamic_literal array_len array_literal_tokens array_pop array_push array_resize bin blockexpr call closure expr fcoerce femit ftest incdec kw_cfunc kw_for methcall postfix_tail primary ternary unary +xgive argcheck args chk_reset +xlend assigncheck expr unary +xlenloc array_len bin call closure enhcall methcall postfix_tail primary unary +xprojection ctx_pop field postfix_tail primary +xstamp array_dynamic_literal array_len bin call closure enhcall methcall newtyped postfix_tail primary unary +xstr array_dynamic_literal array_len array_literal_tokens bin blockexpr call closure incdec methcall postfix_tail primary unary +xty args array_cap array_dynamic_literal array_len array_literal_tokens array_pop array_push array_resize bin blockexpr call closure ctx_pop enhcall expr fcoerce field ftest kw_cfunc kw_share_value kw_sizeof kw_typeof kwcall methcall newtyped postfix_tail primary ternary unary +xvk array_dynamic_literal array_literal_tokens bin blockexpr chk_reset ctx_pop expr field fn_begin incdec kw_share_value kwcall postfix_tail primary ternary unary vf_coerce vf_effect vf_forget +xvv blockexpr primary unary vf_coerce +xzero array_dynamic_literal array_literal_tokens bin blockexpr call closure enhcall expr field incdec kwcall methcall newtyped postfix_tail primary ternary unary -- cgit v1.2.3 From 10e8a339b198b6e49e393c457cba7667a2a39ca5 Mon Sep 17 00:00:00 2001 From: zinc Date: Wed, 14 Oct 2026 15:43:59 +0330 Subject: xref self-test: a real write in a compiler copy, reject unknown options; runerr.sh runs every folder it is given --- TODO.md | 7 ++++--- src/tools/xref_selftest.sh | 22 ++++++++++++++++++---- tests/drivers/cli_help_test.py | 2 +- tests/runerr.sh | 27 +++++++++++++++++++-------- 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/TODO.md b/TODO.md index fcb4e88..bba2ff7 100644 --- a/TODO.md +++ b/TODO.md @@ -431,9 +431,10 @@ of the open list. store through a pointer (`*p = ...`), an array written by a callee (`copy_w`, `memset`) and a global reached only through its `_initial` array are not counted as writes. the baseline is not yet reviewed by a person; - the 60 "no writer" globals still need checking against those limits. open: - a self-test that adds a real write to a copy of src/compiler and runs - `--compiler-dir`, and deciding whether ci runs it on every job or only + the 60 "no writer" globals still need checking against those limits. the self-test also + copies src/compiler to a scratch dir, adds a new function writing + `namecount`, and checks that `--compiler-dir ... --check` exits 1 naming it; + it rejects unknown options and is in cli_help_test.py. open: deciding whether ci runs it on every job or only when `src/compiler/` changes. ### decisions diff --git a/src/tools/xref_selftest.sh b/src/tools/xref_selftest.sh index 36425e4..e8a888b 100755 --- a/src/tools/xref_selftest.sh +++ b/src/tools/xref_selftest.sh @@ -1,11 +1,17 @@ #!/bin/bash -# self-test of src/tools/xref.sh --check: the checked-in baseline passes (exit 0), and a copy of it -# with one writer removed from one line fails (exit 1). works in a scratch directory; the tree, -# tests/xref.baseline and build/xref are not touched. +# self-test of src/tools/xref.sh --check: the checked-in baseline passes (exit 0); a copy of it +# with one writer removed from one line fails (exit 1); and a scratch copy of src/compiler with one +# new write to a global (namecount) fails and names that global. works in a scratch directory; the +# tree, src/compiler, tests/xref.baseline and build/xref are not touched. # usage: src/tools/xref_selftest.sh [SERVER] (default build/bin/zclsp) set -u cd "$(dirname "$0")/../.." -case "${1:-}" in -h|--help) echo "usage: src/tools/xref_selftest.sh [SERVER]"; exit 0;; esac +usage="usage: src/tools/xref_selftest.sh [SERVER] (default build/bin/zclsp)" +case "${1:-}" in + -h|--help) echo "$usage"; exit 0;; + -*) echo "xref_selftest: unknown option $1; $usage" >&2; exit 2;; +esac +[ $# -le 1 ] || { echo "xref_selftest: too many arguments; $usage" >&2; exit 2; } server=${1:-build/bin/zclsp} tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT fail=0 @@ -18,4 +24,12 @@ sed "${line}s/ [^ ]*\$//" tests/xref.baseline > "$tmp/baseline" bash src/tools/xref.sh --server "$server" --out "$tmp/out" --baseline "$tmp/baseline" --check > "$tmp/bad.log" 2>&1; r=$? if [ $r = 1 ] && grep -q 'is now written by' "$tmp/bad.log"; then echo "ok xref --check fails when the baseline misses a writer" else echo "FAIL xref --check with a writer removed from the baseline: exit $r (want 1)"; cat "$tmp/bad.log"; fail=1; fi +# a real new write in a copy of the compiler: a new function that assigns namecount +g=namecount +mkdir "$tmp/compiler"; cp src/compiler/* "$tmp/compiler/" +last=$(tr -d '\r' < "$tmp/compiler/parts" | tail -1) +printf '\nint xref_selftest_probe() { %s = %s + 1; return 0; }\n' "$g" "$g" >> "$tmp/compiler/$last" +bash src/tools/xref.sh --server "$server" --out "$tmp/out" --compiler-dir "$tmp/compiler" --check > "$tmp/src.log" 2>&1; r=$? +if [ $r = 1 ] && grep -q "^ $g is now written by xref_selftest_probe" "$tmp/src.log"; then echo "ok xref --check fails and names $g when a copy of the compiler writes it in a new function" +else echo "FAIL xref --check with a new write to $g in a copy of the compiler: exit $r (want 1 naming $g)"; cat "$tmp/src.log"; fail=1; fi exit $fail diff --git a/tests/drivers/cli_help_test.py b/tests/drivers/cli_help_test.py index af3df2e..481d9c0 100644 --- a/tests/drivers/cli_help_test.py +++ b/tests/drivers/cli_help_test.py @@ -14,7 +14,7 @@ commands = [] if args.group in ('binaries', 'all'): commands += [[str(Path(args.bin_dir) / name)] for name in ['zy', 'zym', 'zclsp', 'zc0', 'mkgenome', 'genmap']] if args.group in ('scripts', 'all'): - commands += [['bash', name] for name in ['tinyvm.sh', 'src/tools/zcc', 'src/tools/automap.sh', 'src/tools/findcc.sh', 'src/tools/bootstrap-check.sh', 'src/tools/parity.sh', 'src/tools/xref.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/web/build.sh', 'src/baremetal/run.sh']] + commands += [['bash', name] for name in ['tinyvm.sh', 'src/tools/zcc', 'src/tools/automap.sh', 'src/tools/findcc.sh', 'src/tools/bootstrap-check.sh', 'src/tools/parity.sh', 'src/tools/xref.sh', 'src/tools/xref_selftest.sh', 'src/tools/mklsp.sh', 'src/tools/mklspwasm.sh', 'src/tools/mkvsix.sh', 'src/web/build.sh', 'src/baremetal/run.sh']] if args.group in ('js', 'all'): commands += [['node', name] for name in [str(Path(args.js_dir) / 'tinyvm.js'), 'src/js/mklayout.js', 'src/js/extract_lsp.js']] diff --git a/tests/runerr.sh b/tests/runerr.sh index f7e668c..b3767ae 100755 --- a/tests/runerr.sh +++ b/tests/runerr.sh @@ -1,10 +1,21 @@ #!/bin/bash -# runs every .zc in a folder (tests/errors by default) and checks the compiler -# rejects it with the message on its first `// expect: ...` line -vm=${1:-zy}; dir=${2:-$(dirname "$0")/errors}; pass=0; fail=0 -for f in "$dir"/*.zc; do - want=$(grep -m1 '^// expect: ' "$f" | sed 's#^// expect: ##') - got=$($vm < "$f" 2>&1 | head -1) - if [[ "$got" == *"$want"* ]]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL $(basename $f): got [$got]"; fi +# runs every .zc in one or more folders (tests/errors by default) and checks the +# compiler rejects it with the message on its first `// expect: ...` line. +# usage: tests/runerr.sh [VM] [DIR...]. with several folders each gets its own line +# and the last line is the total; exits 1 when any file fails. +vm=${1:-zy}; shift +[ $# -gt 0 ] || set -- "$(dirname "$0")/errors" +tpass=0; tfail=0 +for dir in "$@"; do + pass=0; fail=0 + if [ ! -d "$dir" ]; then echo "FAIL $dir: no such folder"; tfail=$((tfail+1)); continue; fi + for f in "$dir"/*.zc; do + want=$(grep -m1 '^// expect: ' "$f" | sed 's#^// expect: ##') + got=$($vm < "$f" 2>&1 | head -1) + if [[ "$got" == *"$want"* ]]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL $(basename $f): got [$got]"; fi + done + [ $# -gt 1 ] && echo "rejected in $dir: $pass passed, $fail failed" + tpass=$((tpass+pass)); tfail=$((tfail+fail)) done -echo "rejected: $pass passed, $fail failed" +echo "rejected: $tpass passed, $tfail failed" +[ $tfail = 0 ] -- cgit v1.2.3