#!/usr/bin/env bash # report-prove.sh — evidence the report generator works. Both directions. # The negative arms are the point: it must REFUSE, not degrade. set -uo pipefail cd "$(git rev-parse --show-toplevel)" python3 - <<'PY' import sys, os, shutil, tempfile sys.path.insert(0, 'tools/evidence') from report import load, extract, cite, narrative_ok, Unbacked F=0; N=0 def chk(name, cond, extra=""): global F,N; N+=1 print(f" {'ok ' if cond else 'FAIL'} {name}{'' if cond else ' '+extra}") if not cond: F+=1 EV = 'docs/experiments/evidence' print("POSITIVE — reads and verifies a real evidence directory") rows = load(EV) chk("manifest loaded and every artifact hash re-verified", len(rows) > 0, f"{len(rows)} rows") print() print("POSITIVE — a number is EXTRACTED from an artifact, not typed") val, r = extract(rows, 'compiler-code-lines', r'^code lines:\s*(\d+)') chk(f"pulled code-line count from captured stdout: {val}", val.isdigit()) print(f" source: {cite(r)}") print() print("NEGATIVE — refuses a claim whose artifact does not exist") try: extract(rows, 'no-such-artifact-anywhere', r'(\d+)'); chk("refused", False, "it did NOT refuse") except Unbacked as e: chk("a claim with no artifact raises, does not degrade to blank", True) print() print("NEGATIVE — refuses a pattern that is not present in the artifact") try: extract(rows, 'compiler-code-lines', r'^TOTAL COST IN DOLLARS:\s*(\d+)'); chk("refused", False, "it did NOT refuse") except Unbacked as e: chk("a number not present in the source raises", True) print() print("NEGATIVE — refuses an ALTERED artifact") tmp = tempfile.mkdtemp() shutil.copytree(EV, os.path.join(tmp,'ev')) target = [f for f in os.listdir(os.path.join(tmp,'ev')) if f.endswith('.out')][0] with open(os.path.join(tmp,'ev',target),'a') as f: f.write("tampered\n") try: load(os.path.join(tmp,'ev')); chk("refused", False, "it accepted a tampered artifact") except Unbacked as e: chk("a tampered artifact halts the report entirely", 'ALTERED' in str(e)) shutil.rmtree(tmp) print() print("NEGATIVE — refuses an empty manifest") tmp2 = tempfile.mkdtemp() open(os.path.join(tmp2,'MANIFEST.tsv'),'w').write("sha256\tbytes\texit\tms\tcommit\ttree\tutc\tartifact\tcommand\n") try: load(tmp2); chk("refused", False, "an empty manifest passed") except Unbacked as e: chk("an empty manifest is a failure, not a clean report", True) shutil.rmtree(tmp2) print() print("POSITIVE — narrative may not carry a quantity") ok1,_ = narrative_ok("The seam binds after the build, which is the whole claim.") ok2,bad = narrative_ok("The compiler shrank by 10.6 percent, which is a lot.") chk("prose with no digits passes", ok1) chk("prose containing a bare number is REJECTED", not ok2, f"caught {bad}") print() print(f" {'PROVEN' if F==0 else 'NOT PROVEN'} — {N} checks, {F} failed") sys.exit(F) PY