wip: bench suite

1 files changed, 19 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index b3b7d56..6deb3a9 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,9 @@
 #   make xref            who writes and who reads each global of the compiler
 #                        (src/tools/xref.sh): build/xref/report.txt and report.json;
 #                        make xref XREF_ARGS=--check compares with tests/xref.baseline
+#   make bench           time the benchmark programs -> build/bench.txt
+#   make bench-save      ... and keep the result as tests/bench.baseline
+#   make bench-check     compare with tests/bench.baseline (or make bench CHECK=1)
 #   make genome          build/core/genome.bin from src/, with the fixpoint check
 #   make LLVM=1          on x86_64, build zy with the llvm backend instead of the x86-64 jit
 #   make NOLLVM=1        on other hosts, leave out the llvm backend (in by default when a
@@ -208,6 +211,21 @@ parity: $(BIN)/zc0$(EXE) $(BIN)/mkgenome$(EXE) $(BIN)/zy$(EXE)
 xref: $(BIN)/zclsp$(EXE)
 	bash src/tools/xref.sh $(XREF_ARGS)
 
+# benchmarks (tests/bench/suite.py) on build/bin/zy: build/bench.txt, ms per repetition per
+# program. bench-save makes it the committed baseline, tests/bench.baseline; bench-check
+# (or make bench CHECK=1) fails when a program is more than 10% slower than the baseline.
+bench: $(BIN)/zy$(EXE)
+ifeq ($(CHECK),1)
+	python3 tests/bench/suite.py check
+else
+	python3 tests/bench/suite.py run
+endif
+bench-save: bench
+	cp $(B)/bench.txt tests/bench.baseline
+	@echo "bench: wrote tests/bench.baseline"
+bench-check: $(BIN)/zy$(EXE)
+	python3 tests/bench/suite.py check
+
 # JOBS sets how many sections run at once: make test JOBS=4
 test:
 	bash tests/test.sh -j $(or $(JOBS),1)
@@ -219,4 +237,4 @@ check:
 clean:
 	rm -rf $(B)
 
-.PHONY: all lsp genome js lspimage vsix web run parity xref test check clean
+.PHONY: bench bench-save bench-check all lsp genome js lspimage vsix web run parity xref test check clean