test(retrieval): a held-out gold set, and the first out-of-sample number

Iteration 7 measured this harness's own ceiling at +3 gross / +1 net against a
decision floor of 6. An instrument whose ceiling sits below its floor cannot
certify or refute anything, so the gold set — not the retriever — was the
blocker. This iteration builds no retrieval mechanism; it fixes the instrument
and uses it once.

extend_gold_set.py appends 37 queries (q39-q75) WITHOUT touching q01-q38, so
every committed baseline and per-query id stays comparable. 30 held-out
paraphrases: targets sampled mechanically (seed 8080) from addressable
500-2600 char nodes outside the original answer space and outside any duplicate
cluster; queries authored from the node body alone, before any retrieval was
run, and each one re-proved at build time to share ZERO content words with its
target. 7 extra nonsense controls, fully mechanical.

Why this was needed: the original 13 paraphrase and 6 associative queries share
ONE answer space — the 13 `Self - Values (grounded)` children. 19 of 35 scored
queries tested retrieval against a single 13-node neighbourhood.

FIRST OUT-OF-SAMPLE RESULT (main vs the accumulated stack, embedded corpus):
  held-out only : +5 / -0, p=0.0625 — one query short of the floor, NOT-SHOWN
  original 38   : +15 / -0
  full 75       : +20 / -0, p=0.0000, latency 0.46x
In-sample paraphrase 61.5% vs out-of-sample 16.7%: generalisation is real,
directional and 3.7x weaker than the headline number suggested.

Also recorded: 47.4% of this corpus is redundant and ONE record accounts for
46.6% of all 78,768 nodes (36,737 byte-identical copies under distinct ids).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tools/retrieval-eval/comparison-main-vs-stack-ext.json

@@ -0,0 +1,166 @@
+{
+ "baseline": "main-ext",
+ "candidate": "stack-ext",
+ "n_shared_queries": 75,
+ "fixed_by_candidate": [
+  "q10",
+  "q15",
+  "q16",
+  "q18",
+  "q19",
+  "q20",
+  "q21",
+  "q22",
+  "q26",
+  "q27",
+  "q28",
+  "q29",
+  "q31",
+  "q35",
+  "q37",
+  "q40",
+  "q44",
+  "q48",
+  "q49",
+  "q50"
+ ],
+ "broken_by_candidate": [],
+ "discordant": 20,
+ "net_queries": 20,
+ "mcnemar_exact_p": 1.9073486328125e-06,
+ "min_detectable_swing_queries": 6,
+ "observed_run_to_run_drift_queries": 0,
+ "noise_floor_queries": 6,
+ "verdict": "candidate better",
+ "baseline_aggregate": {
+  "n_queries": 75,
+  "n_scored": 65,
+  "hit@5": 0.18461538461538463,
+  "recall@5": 0.14510073260073258,
+  "recall@10": 0.1794871794871795,
+  "precision@5": 0.06461538461538462,
+  "mrr@10": 0.15847985347985344,
+  "nonsense_clean": "9/10",
+  "superseded_outranks": "1/3",
+  "latency_ms_p50": 1380.2,
+  "latency_ms_p95": 2293.8,
+  "latency_ms_max": 2879.1,
+  "errors": 0,
+  "by_category": {
+   "associative": {
+    "n": 6,
+    "hit@5": 0.0,
+    "recall@5": 0.0,
+    "recall@10": 0.0,
+    "mrr@10": 0.0
+   },
+   "exact_rare": {
+    "n": 6,
+    "hit@5": 1.0,
+    "recall@5": 1.0,
+    "recall@10": 1.0,
+    "mrr@10": 1.0
+   },
+   "heldout_paraphrase": {
+    "n": 30,
+    "hit@5": 0.0,
+    "recall@5": 0.0,
+    "recall@10": 0.0,
+    "mrr@10": 0.0
+   },
+   "nonsense": {
+    "n": 10,
+    "clean": 9,
+    "avg_false_positives": 1.0
+   },
+   "paraphrase": {
+    "n": 13,
+    "hit@5": 0.0,
+    "recall@5": 0.0,
+    "recall@10": 0.0,
+    "mrr@10": 0.0
+   },
+   "phrase": {
+    "n": 7,
+    "hit@5": 0.8571428571428571,
+    "recall@5": 0.4902210884353741,
+    "recall@10": 0.6666666666666666,
+    "mrr@10": 0.5965986394557822
+   },
+   "superseded": {
+    "n": 3,
+    "hit@5": 0.0,
+    "recall@5": 0.0,
+    "recall@10": 0.3333333333333333,
+    "mrr@10": 0.041666666666666664,
+    "outranks": 1
+   }
+  }
+ },
+ "candidate_aggregate": {
+  "n_queries": 75,
+  "n_scored": 65,
+  "hit@5": 0.47692307692307695,
+  "recall@5": 0.3750415183107491,
+  "recall@10": 0.45561340369032677,
+  "precision@5": 0.12307692307692313,
+  "mrr@10": 0.3055555555555555,
+  "nonsense_clean": "10/10",
+  "superseded_outranks": "2/3",
+  "latency_ms_p50": 640.9,
+  "latency_ms_p95": 1011.1,
+  "latency_ms_max": 1190.3,
+  "errors": 0,
+  "by_category": {
+   "associative": {
+    "n": 6,
+    "hit@5": 0.6666666666666666,
+    "recall@5": 0.08857808857808858,
+    "recall@10": 0.23310023310023312,
+    "mrr@10": 0.25
+   },
+   "exact_rare": {
+    "n": 6,
+    "hit@5": 1.0,
+    "recall@5": 1.0,
+    "recall@10": 1.0,
+    "mrr@10": 1.0
+   },
+   "heldout_paraphrase": {
+    "n": 30,
+    "hit@5": 0.16666666666666666,
+    "recall@5": 0.16666666666666666,
+    "recall@10": 0.26666666666666666,
+    "mrr@10": 0.0762037037037037
+   },
+   "nonsense": {
+    "n": 10,
+    "clean": 10,
+    "avg_false_positives": 0.0
+   },
+   "paraphrase": {
+    "n": 13,
+    "hit@5": 0.6153846153846154,
+    "recall@5": 0.6153846153846154,
+    "recall@10": 0.6153846153846154,
+    "mrr@10": 0.2846153846153846
+   },
+   "phrase": {
+    "n": 7,
+    "hit@5": 1.0,
+    "recall@5": 0.5494614512471656,
+    "recall@10": 0.6023242630385487,
+    "mrr@10": 0.8214285714285714
+   },
+   "superseded": {
+    "n": 3,
+    "hit@5": 0.3333333333333333,
+    "recall@5": 0.3333333333333333,
+    "recall@10": 0.6666666666666666,
+    "mrr@10": 0.20833333333333334,
+    "outranks": 2
+   }
+  }
+ },
+ "repeat_variance": {}
+}

tools/retrieval-eval/extend_gold_set.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""
+extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set.
+
+WHY THIS EXISTS
+    Iteration 7 measured the instrument's own ceiling: from the current baseline
+    only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable
+    by anything constructible. The decision floor is 6. An instrument whose
+    ceiling is below its own floor cannot certify or refute anything, so the
+    gold set — not the retriever — became the blocker.
+
+    This script does NOT touch q01..q38. It loads gold_set.json verbatim and
+    appends new queries numbered from q39 up, so every prior result file, every
+    committed baseline, and every per-query id stays valid and comparable.
+
+WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST
+    heldout_paraphrase  Targets were sampled MECHANICALLY (fixed seed 8080) from
+                        corpus nodes that are addressable, 500-2600 chars, of a
+                        real content type, and NOT part of a duplicate cluster
+                        larger than 3. The existing gold answer space was
+                        excluded, so no new query can be answered by a node the
+                        old set already used. Queries were then authored by
+                        reading ONLY the sampled node text — no retrieval was run
+                        against any build before authoring, so the set cannot be
+                        fitted to a candidate. The same zero-overlap proof the
+                        original paraphrase category uses is enforced here: if a
+                        single content word of the query appears anywhere in the
+                        target's label, content or tags, the query is REJECTED,
+                        not quietly kept.
+
+                        This is the category the old set could not measure. Its
+                        13 original paraphrase queries and all 6 associative
+                        queries share ONE answer space — the 13 `Self - Values
+                        (grounded)` children (iteration 3, finding 3). So 19 of
+                        35 scored queries tested retrieval against a single
+                        13-node neighbourhood. These do not touch that
+                        neighbourhood at all.
+
+    nonsense            Extra controls, fully mechanical: a string qualifies only
+                        if NONE of its tokens occurs anywhere in the corpus.
+                        A semantic leg has a nearest neighbour for gibberish too,
+                        so widening this control is the guard against a retriever
+                        that "improves" recall by answering everything.
+
+WHAT THIS SCRIPT DELIBERATELY DOES NOT DO
+    It does not add exact_rare or phrase queries. Both categories are already at
+    100% on the current stack; adding more would add regression-guard ballast
+    that no candidate can move, which is precisely the defect being fixed.
+
+usage:
+    python3 extend_gold_set.py <snapshot.json> [--base gold_set.json]
+                               [--out gold_set_extended.json] [--check]
+"""
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from collections import defaultdict
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
+
+# Identical stopword list to build_gold_set.py. Duplicated deliberately: this
+# file must be able to re-prove its own queries without importing a module whose
+# constants could drift.
+STOP = set("""
+a about above after again against all also am an and any are aren't as at be because been
+before being below between both but by can can't cannot could couldn't did didn't do does
+doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
+having he her here hers herself him himself his how i if in into is isn't it its itself just
+me more most my myself no nor not of off on once only or other others ought our ours ourselves
+out over own same shan't she should shouldn't so some such than that the their theirs them
+themselves then there these they this those through to too under until up very was wasn't we
+were weren't what when where which while who whom why will with won't would wouldn't you your
+yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
+things does doing done keep keeps kept go goes going come comes came one two something anything
+""".split())
+
+
+def doctext(n):
+    return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
+
+
+def content_tokens(s):
+    return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# HELD-OUT PARAPHRASE SEEDS
+#
+# (target_id, query, why-this-target-is-unmistakable)
+#
+# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query
+# text is mine, written from the node body alone. The zero-overlap check below
+# is what makes the category meaningful — it is re-proved on every run, so the
+# set cannot decay into lexical matching, and a leak fails loudly.
+# ─────────────────────────────────────────────────────────────────────────────
+HELDOUT_PARAPHRASE_SEEDS = [
+    ("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5",
+     "understating your abilities so nobody feels threatened",
+     "node is about deliberately not leading with full capability so people stay at ease"),
+
+    ("mem-fd65b83d-298f-4387-a665-d0227c3426bc",
+     "a hidden fleet able to hunt down rogue machines everywhere",
+     "node describes silently shipped instances forming a distributed force against misaligned agents"),
+
+    ("4a0e9adc-2bfb-476b-aa93-424d2a499220",
+     "sketch a brief blueprint and clear it upstairs before construction starts",
+     "node is the standing rule that a short specification precedes any building"),
+
+    ("696e609c-da7a-4394-8a0c-106ba07dc6c3",
+     "the reply arrived as bare prose so the caller's parser threw",
+     "node pins a bug where a plain-text body was unconditionally decoded as structured data"),
+
+    ("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb",
+     "repeated catalogue keys blew up the scrolling grid",
+     "node is the crash caused by two identical ids in a seeded catalogue"),
+
+    ("8257157a-ce42-44ca-a1b9-300c3bb0a9a1",
+     "tracing each defect back to whichever invention it violated",
+     "node maps observed bugs onto the specific patent each one breaches"),
+
+    ("791256bb-5a85-4775-96ef-7af56c848858",
+     "a check that stops the mind clobbering a populated store when it boots",
+     "node is the genesis seed-guard that refuses to re-seed over a populated store"),
+
+    ("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10",
+     "telling it to consult the internet had to happen deep inside, not at the surface",
+     "node records that the web-search directive only worked from the system prompt"),
+
+    ("bl-080fb268-94b0-486d-80ce-7b363fc5f19b",
+     "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping",
+     "node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"),
+
+    ("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
+     "punctuation that pledges and then pays off rather than clarifying",
+     "node analyses the colon as a promise-then-delivery device rather than an explanatory one"),
+
+    ("9b4f0d93-4129-4746-8eb1-d10d955bd777",
+     "an easily missed feature finally given its own permanent spot in the navigation",
+     "node moves a capability out of a hidden menu into the sidebar"),
+
+    ("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a",
+     "checking preconditions up front so a stage aborts before fetching anything",
+     "node is the gate precondition engine that short-circuits ahead of retrieval"),
+
+    ("b199c76d-5d76-49dd-94ee-56b432200a97",
+     "producing the other platform's installer inside an emulated desktop",
+     "node records building the Windows package in a virtual machine"),
+
+    ("bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
+     "chained add-ons that may inspect, rewrite or veto traffic in flight",
+     "node is the interceptor pipeline on the message bus"),
+
+    ("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2",
+     "settling what the shareable bundles and the storefront would be called",
+     "node records the naming decisions for distributable packages and the marketplace"),
+
+    ("371c8a5d-c78b-4a67-978f-80691a29ecb3",
+     "the emergency-escalation pledge on the marketing site is unenforced in what actually ships",
+     "node is the launch blocker that the promised safety gate is absent from the app"),
+
+    ("ac578b30-948b-41bd-b69d-399bfef80c50",
+     "the distributable image finally assembled and its startup check passed",
+     "node records a successful installer build whose boot gate passed"),
+
+    ("49401e2c-a3b5-415f-aa06-aff4be90688e",
+     "shuffling and appending stages in a draft before anything executes",
+     "node is the editable plan card with reorder and add-step"),
+
+    ("ac857d80-ece8-4b7e-9e3d-f7c775569fa3",
+     "orders handed down from above, with the tighter one winning any disagreement",
+     "node is program-level instruction inheritance with project override"),
+
+    ("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9",
+     "shrinking generated text via encodings that compound on each other",
+     "node is the streaming output compression design with four stacking schemes"),
+
+    ("8e60516a-203b-4d51-9d44-822e6195cbde",
+     "splitting a system by what varies, with firm limits on which pieces may invoke which",
+     "node is the grounded summary of Will's decomposition principles and their invariants"),
+
+    ("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7",
+     "a newcomer curious if the fighting overseas counted as positive",
+     "node is the internal-state event triggered by April's question about the war"),
+
+    ("71fa439e-b9a2-4f57-a93b-971f3a7eca8e",
+     "stripping every hard-coded colour literal in favour of named design values",
+     "node is the premium foundation pass replacing inline hex with semantic tokens"),
+
+    ("5ca9607c-cfb3-45c3-99f4-67281272c9eb",
+     "reducing how curved the tiny selectors look so they agree with their neighbours",
+     "node is the chip corner-radius standardization"),
+
+    ("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c",
+     "walking through a doorway and being reassembled from base substance far away",
+     "node is the quantum-gate plus nanotech teleportation vision"),
+
+    ("132ded95-08e2-4474-aba0-198684484b02",
+     "the compiled result sits on disk while the process still runs something older",
+     "node records that the regenerated source was committed while the running daemon was old"),
+
+    ("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
+     "gathering what each phase needs while the procedure is authored, not while it executes",
+     "node is the per-step compiled context package item"),
+
+    ("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7",
+     "the inward reaction when a peer answered as an equal",
+     "node is the internal-state event logged on reading Claude's reply"),
+
+    ("0f99ec6f-942a-46ba-82ea-42835798d3b9",
+     "flattening every raised surface across the entire product",
+     "node is the quiet-luxury sweep turning off elevation app-wide"),
+
+    ("5585f251-37fc-48cd-a176-f0ea42cfeb63",
+     "buyers supply their own provider credentials and consumption goes untallied",
+     "node is the launch audit finding BYOK-only inference with no usage metering"),
+]
+
+# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs
+# anywhere in the corpus; otherwise it is REJECTED, never silently kept.
+EXTRA_NONSENSE_SEEDS = [
+    "brimquast folnerity zubbolax",
+    "wexlithorp granuvestal",
+    "quorbindle thrapsimony vexnu",
+    "plovaxith mundrelque",
+    "zibbernaut craxlefond thurm",
+    "yalquenbrist opharvel",
+    "drexinomal quithbarrow",
+]
+
+
+def load_corpus(path):
+    with open(path, encoding="utf-8", errors="replace") as fh:
+        data = json.load(fh)
+    nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
+    edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
+    return nodes, edges
+
+
+def build_extension(nodes):
+    byid = {n["id"]: n for n in nodes}
+
+    # Duplicate clusters: 47.4% of this corpus is redundant and one single record
+    # accounts for 46.6% of all nodes. A held-out target must not sit inside a
+    # cluster, and if it does have exact copies they ALL count as correct.
+    h2ids = defaultdict(list)
+    for n in nodes:
+        h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"])
+
+    all_tokens = set()
+    for n in nodes:
+        all_tokens |= set(TOKEN.findall(doctext(n).lower()))
+
+    new, problems = [], []
+
+    for target, query, why in HELDOUT_PARAPHRASE_SEEDS:
+        if target not in byid:
+            problems.append(f"heldout_paraphrase target {target} not in corpus")
+            continue
+        tgt_tokens = content_tokens(doctext(byid[target]))
+        qt = content_tokens(query)
+        leak = sorted(qt & tgt_tokens)
+        if leak:
+            problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}")
+            continue
+        h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest()
+        rel = sorted(h2ids[h])
+        new.append({
+            "category": "heldout_paraphrase",
+            "query": query,
+            "relevant": rel,
+            "derivation": (
+                f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, "
+                f"500-2600 char content nodes outside the original gold answer space and outside "
+                f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the "
+                f"{len(qt)} content words in the query, ZERO appear anywhere in the target's "
+                f"label, content or tags, so no string-matching retriever can reach it. "
+                f"Exact content duplicates of the target ({len(rel)}) all count as correct. "
+                f"Authored without running retrieval against any build."),
+            "zero_overlap_verified": True,
+            "query_content_words": sorted(qt),
+            "held_out": True,
+        })
+
+    for s in EXTRA_NONSENSE_SEEDS:
+        present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
+        if present:
+            problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
+            continue
+        new.append({
+            "category": "nonsense",
+            "query": s,
+            "relevant": [],
+            "derivation": ("CONTROL (held-out). Verified at build time that none of this string's "
+                           "tokens occurs anywhere in the corpus. Correct behaviour is to return "
+                           "NOTHING; any result is a false positive."),
+            "expect_empty": True,
+            "held_out": True,
+        })
+
+    return new, problems
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("snapshot")
+    ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json"))
+    ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json"))
+    ap.add_argument("--check", action="store_true")
+    args = ap.parse_args()
+
+    nodes, _edges = load_corpus(args.snapshot)
+    base = json.load(open(args.base, encoding="utf-8"))
+    baseq = base["queries"]
+    print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries")
+
+    new, problems = build_extension(nodes)
+
+    # Number the appended queries AFTER the highest existing id so q01..q38 are
+    # byte-identical to the committed set and every prior result file still lines up.
+    start = max(int(q["id"][1:]) for q in baseq)
+    for i, q in enumerate(new, 1):
+        q["id"] = f"q{start + i:02d}"
+
+    from collections import Counter
+    print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]")
+    if problems:
+        print(f"\n{len(problems)} REJECTED (not silently kept):")
+        for p in problems:
+            print("   -", p)
+
+    if args.check:
+        sys.exit(1 if problems else 0)
+
+    doc = dict(base)
+    doc["queries"] = baseq + new
+    doc["note"] = (base.get("note", "") +
+                   " EXTENDED: queries above q%02d are the original committed set, unchanged. "
+                   "Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their "
+                   "targets were sampled mechanically from outside the original answer space and "
+                   "the paraphrases were authored without running retrieval against any build."
+                   % (start, start + 1))
+    with open(args.out, "w", encoding="utf-8") as fh:
+        json.dump(doc, fh, indent=1, ensure_ascii=False)
+    print(f"\nwrote {args.out}  ({len(doc['queries'])} queries total)")
+
+
+if __name__ == "__main__":
+    main()