neuron/tools/retrieval-eval/build_gold_set.py

#!/usr/bin/env python3
"""
build_gold_set.py — derive the retrieval gold set FROM the corpus, and validate it.

WHY THIS FILE EXISTS AS CODE AND NOT AS A HAND-WRITTEN JSON
    A gold set nobody can audit is vibes with extra steps. Every expected answer
    here is either (a) mined from the corpus by a rule this script re-runs, or
    (b) hand-selected with a stated criterion that this script then CHECKS
    against the corpus. Both leave a `derivation` string on every query, and the
    checks are re-run on demand so the set cannot silently rot as the corpus
    changes.

    Lineage: this extends the pinned-query approach from
    docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py (pinned
    ground-truth patterns + a --check "winnability" gate) and the per-query
    relevant-id scoring from docs/research/graphrag_eval/score.py. What is new:
    ids as ground truth rather than regexes alone, an ASSOCIATIVE category
    derived from real graph edges, a superseded/contradicted category, and a
    machine-checked no-lexical-overlap guarantee on the paraphrase category.

THE SIX CATEGORIES, AND WHAT EACH ONE IS FOR
    exact_rare   a single rare word. Substring matching already wins these.
                 They are a REGRESSION GUARD: any change that loses them is
                 disqualified regardless of what else it gains.
    phrase       a multi-word string that exists verbatim in the corpus.
                 Guards multi-token queries, which the old substring matcher
                 handled by returning nothing.
    paraphrase   same meaning, ZERO shared content words with the target node.
                 THE CATEGORY THAT MATTERS. Mechanically unreachable by string
                 matching; reachable only by semantics or by association.
    associative  the answer is one hub-hop from an obvious starting point and
                 shares no words with the query. This is the case the graph is
                 supposed to buy: query one value, get its siblings.
    nonsense     must return nothing. Guards against a retriever that "improves"
                 recall by returning the whole graph.
    superseded   a fact that was later corrected. The correction must OUTRANK
                 the stale version — ranking, not mere presence.

usage:
    python3 build_gold_set.py <snapshot.json> [--out gold_set.json] [--check]
    --check re-validates an existing gold_set.json against the corpus and exits
    non-zero if any query became unwinnable or any paraphrase leaked a word.
"""
import argparse
import json
import os
import re
import sys
from collections import Counter, defaultdict

HERE = os.path.dirname(os.path.abspath(__file__))
DEFAULT_OUT = os.path.join(HERE, "gold_set.json")

TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")

# Stopwords are deliberately generous. A paraphrase query is only interesting if
# its CONTENT words are absent from the target; "the", "is", "what" appearing in
# both proves nothing. Being generous here makes the overlap test STRICTER on
# the words that carry meaning, which is the conservative direction.
STOP = set("""
a about above after again against all also am an and any are aren't as at be because been
before being below between both but by can can't cannot could couldn't did didn't do does
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
having he her here hers herself him himself his how i if in into is isn't it its itself just
me more most my myself no nor not of off on once only or other others ought our ours ourselves
out over own same shan't she should shouldn't so some such than that the their theirs them
themselves then there these they this those through to too under until up very was wasn't we
were weren't what when where which while who whom why will with won't would wouldn't you your
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
things does doing done keep keeps kept go goes going come comes came one two something anything
""".split())


# ─────────────────────────────────────────────────────────────────────────────
# corpus helpers
# ─────────────────────────────────────────────────────────────────────────────
def load_corpus(path):
    with open(path, encoding="utf-8", errors="replace") as fh:
        data = json.load(fh)
    nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
    edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
    return nodes, edges


def doctext(n):
    return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])


def content_tokens(s):
    return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}


# ─────────────────────────────────────────────────────────────────────────────
# hand-authored queries. Every entry states HOW its expected answer was chosen.
# The `check` field names the validation this script runs against the corpus.
# ─────────────────────────────────────────────────────────────────────────────

# EXACT_RARE — mined, not chosen. The rule (re-run by mine_exact_rare below):
# tokens whose document frequency across the whole corpus is 1, whose single
# containing node is a Memory/Knowledge/Belief with 300-6000 chars of content
# (so the answer is a real memory, not a 117KB whitepaper that contains every
# word in English), and whose token is plain lowercase alphabetic. The expected
# answer is that one node — it is the only node that can possibly be correct.
EXACT_RARE_SEEDS = [
    "unjailbreakable",
    "engram-migrate",
    "cartabandonedevent",
    "pre-apprenticeship",
    "inferencenodemanager",
    "clear-eyed",
]

# PHRASE — chosen by reading the corpus for phrases that (a) occur verbatim,
# (b) occur in a small enough set of nodes that "relevant" is well defined.
# Expected answers are computed here as EVERY node whose text contains the
# phrase case-insensitively — so the answer set is a fact about the corpus, not
# an opinion. Queries whose phrase matches more than PHRASE_MAX nodes are
# rejected by validation as too diffuse to score.
PHRASE_MAX = 25
PHRASE_SEEDS = [
    ("patterns not returns",
     "a verbatim correction Will issued; expected = every node containing the phrase"),
    ("thirty moves",
     "the canonical biographical phrase; expected = every node containing it"),
    ("Grandma Lucas",
     "a named person appearing verbatim in the biography/value nodes"),
    ("Directed Harmonic",
     "the canonical DHARMA expansion, confirmed by Will April 24 2026"),
    ("Sarah Bishop",
     "a named person; rare enough that the answer set is unambiguous"),
    ("Directed Autonomous Runtime Modification",
     "the DARMA expansion, quoted verbatim in the backlog item and its correction"),
    ("zero-knowledge encrypted backup",
     "the paid-tier feature name as written in the roadmap nodes"),
]

# PARAPHRASE — hand-authored. THE SELECTION CRITERION, stated once and applied
# to all nine: pick a node whose SUBJECT is unmistakable to a reader, then write
# the query a person would actually type when they remember the subject but not
# the words. The target is then LOCKED by id, and this script enforces the hard
# property that makes the category meaningful: not one content word of the query
# appears anywhere in the target node's label, content, or tags. If a word
# leaks, validation fails and the query must be rewritten — the set cannot
# quietly degrade into a lexical query wearing a paraphrase costume.
PARAPHRASE_SEEDS = [
    ("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71",
     "the elderly relative who passed while he stayed away",
     "target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas "
     "dying in Feb 2006 without Will saying goodbye. Query names the event with none of the "
     "node's own vocabulary."),
    ("kn-58874a74-b96f-4883-9e08-45707f4bd3ee",
     "a soldier sidelined by illness who refused to quit",
     "target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the "
     "Marines, a severe hernia, and sepsis. Query describes the episode obliquely."),
    ("kn-13f60407-7b70-4db1-964f-ea1f8196efbd",
     "choosing an uncomfortable fact over a pleasant fiction",
     "target: 'Value - Honesty Before Comfort'. Query states the principle in wholly "
     "different words."),
    ("kn-22d77abe-b3c5-42fd-afcd-dcb87d924929",
     "a tight payload beats a bloated one",
     "target: 'Value - Precision Over Brute Force'. Query restates the claim with no "
     "shared vocabulary."),
    ("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8",
     "if you are able and nobody is coming the job is yours",
     "target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the "
     "obligation without the node's terms."),
    ("kn-0bb4f021-56de-4947-a35b-a37209e7ba21",
     "learning is the wealth creditors cannot seize",
     "target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the "
     "library following Will across 30+ moves."),
    ("kn-5de5a9ac-fd15-45ab-bf18-77566781cf40",
     "reliability proven by track record not assertion",
     "target: 'Value - Earned Trust' ('Trust is demonstrated, not declared')."),
    ("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83",
     "boundaries that enable instead of confine",
     "target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim."),
    ("kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c",
     "what shifts tells you where to cut a system apart",
     "target: 'Value - Change Is the Signal', the value VBD is built on."),
    ("kn-f230b362-b201-4402-9833-4160c89ab3d4",
     "a mind that compounds instead of resetting each day",
     "target: 'Value - The System Must Accumulate'. Query is the accumulation claim in "
     "different vocabulary."),
    ("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e",
     "loved for the unedited self and not the polished exterior",
     "target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop "
     "as the first person Will did not perform for."),
    ("kn-e0423482-cfa5-4796-8689-8495c93b66bc",
     "cheerfulness you arrive at instead of assuming",
     "target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'."),
    ("kn-6061318f-046b-4935-907d-8eafdce14930",
     "a childhood offering no solid foundation to inherit",
     "target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between "
     "two parents' collapses."),
]

# ASSOCIATIVE — derived from real edges, not authored. The construction:
# every value node hangs off the 'Self - Values (grounded)' hub by an `identity`
# edge. For a chosen value node V, the query is built from V's own distinctive
# vocabulary; the expected answers are V's SIBLINGS on that hub. A sibling
# shares no query words with the query by construction (validated below), so the
# only path from the query to a sibling is: lexical seed on V -> hub -> sibling.
# That is a two-hop traversal and nothing else can produce it.
VALUES_HUB = "kn-5b606390-a52d-4ca2-8e0e-eba141d13440"
ASSOCIATIVE_SEEDS = [
    ("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", "Grandma Lucas stroke February 2006 goodbye window"),
    ("kn-58874a74-b96f-4883-9e08-45707f4bd3ee", "Marines hernia sepsis medical ward"),
    ("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", "Sarah Bishop Dyer trailer performance"),
    ("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", "Swarm Architecture containment lateral worker"),
    ("kn-e0423482-cfa5-4796-8689-8495c93b66bc", "hope won inside the narrative preface"),
    ("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", "man of the house six years old expectation"),
]

# NONSENSE — must return nothing. Strings chosen to be lexically impossible:
# validation asserts each appears in ZERO corpus nodes as a substring and that
# none of its tokens appears anywhere either (so not even a partial seed exists).
NONSENSE_SEEDS = [
    "zqxjvw plimforth grebulon",
    "flarnbistle quommetry",
    "xxqzzt vurblenacht throom",
]

# SUPERSEDED — a fact that was corrected. Chosen by searching the corpus for
# explicit correction language and keeping pairs where BOTH the stale statement
# and its correction exist as separate nodes. Scored on RANKING: the correction
# must appear, and must appear above the stale node. Ids are locked here and
# validated to exist and to match their stated role.
SUPERSEDED_SEEDS = [
    # (query, correct_id, stale_id, derivation)
]


# ─────────────────────────────────────────────────────────────────────────────
# mining
# ─────────────────────────────────────────────────────────────────────────────
def mine_exact_rare(nodes, byid, seeds):
    """Re-derive: confirm each seed token still has df==1 and name its node."""
    tok = re.compile(r"[A-Za-z][A-Za-z0-9\-]{4,}")
    want = set(seeds)
    df = Counter()
    post = defaultdict(set)
    for n in nodes:
        for t in {w.lower() for w in tok.findall(doctext(n))}:
            if t in want:
                df[t] += 1
                post[t].add(n["id"])
    out = []
    for s in seeds:
        ids = sorted(post.get(s, ()))
        out.append((s, ids, df.get(s, 0)))
    return out


def phrase_matches(nodes, phrase):
    p = phrase.lower()
    return sorted(n["id"] for n in nodes if p in doctext(n).lower())


def hub_siblings(edges, hub, relation="identity"):
    sibs = []
    for e in edges:
        if e.get("from_id") == hub and e.get("relation") == relation:
            sibs.append(e["to_id"])
        elif e.get("to_id") == hub and e.get("relation") == relation:
            sibs.append(e["from_id"])
    return list(dict.fromkeys(sibs))


def find_superseded_pairs(nodes, byid):
    """Locked pairs, each verified here to exist and to carry its stated marker.

    Chosen by scanning the corpus for explicit correction language
    (CORRECTION/SUPERSEDES/re-corrected/no longer/RECONCILED) and keeping only
    cases where the STALE claim also survives as its own node — a supersession
    with nothing to outrank is not a ranking test.
    """
    pairs = []
    txt = {n["id"]: doctext(n) for n in nodes}

    def find_one(pattern, exclude=()):
        rx = re.compile(pattern)
        return [n["id"] for n in nodes
                if n["id"] not in exclude
                and rx.search(txt[n["id"]])
                and 150 < len(str(n.get("content") or "")) < 12000
                and n.get("node_type") in ("Memory", "Knowledge", "Belief", "BacklogItem")]

    # Each entry: (query, correction-pattern, stale-pattern, why).
    # The stale side is searched with the correction hits EXCLUDED, because most
    # correction memories quote the claim they are killing — without the
    # exclusion the "stale" node resolves to the correction itself and the pair
    # collapses into a no-op. A pair is only emitted if both sides resolve to
    # DIFFERENT surviving nodes; otherwise it is dropped and reported.
    SPECS = [
        ("is the self-improvement architecture called DARMA or DHARMA",
         r'(?i)CORRECTION:.{0,90}DHARMA .{0,12}not DARMA',
         r'(?i)\bDARMA\b',
         "correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); "
         "the stale node is the surviving backlog item still titled 'Implement DARMA'."),

        ("how many provisional patents does Will actually have",
         r'(?i)EXACTLY 6 (fully-specced )?provisional',
         r'(?i)(MY ARCHITECTURE = 12 filed patents|\b12 filed patents\b)',
         "correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; "
         "the stale node is the surviving memory that asserts 12 filed patents."),

        ("is MCP still the live integration layer",
         r'(?i)MCP RETIRED',
         r'(?i)MCP server live at',
         "correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of "
         "April 30 2026; the stale node still records the MCP server as live."),

        ("what does the patterns-not-returns directive mean",
         r'(?i)CORRECTION:.{0,80}patterns not returns',
         r'(?i)established returns',
         "correction node is Will's 'patterns not returns' correction; the stale node is a "
         "surviving node carrying the misread 'established returns' directive."),

        ("was the earlier identity-bug finding correct",
         r'(?i)SUPERSEDES the earlier .critical identity bug',
         r'(?i)critical identity bug',
         "correction node explicitly supersedes the 'critical identity bug' finding; the stale "
         "node is the surviving original finding."),

        ("does Neuron have recursive self-improvement",
         r'(?i)twice answered .Neuron has no recursive self-improvement',
         r'(?i)no recursive self-improvement',
         "correction node records the June-29 finding that the CGI provisional IS the "
         "recursive-self-improvement mechanism; the stale node is the surviving denial."),
    ]

    for query, cpat, spat, why in SPECS:
        corr = find_one(cpat)
        if not corr:
            continue
        stale = find_one(spat, exclude=set(corr))
        if not stale:
            continue
        pairs.append((query, corr[0], stale[0], why))

    return pairs


# ─────────────────────────────────────────────────────────────────────────────
# build
# ─────────────────────────────────────────────────────────────────────────────
def build(nodes, edges):
    byid = {n["id"]: n for n in nodes}
    tokset = {n["id"]: content_tokens(doctext(n)) for n in nodes}
    queries = []
    problems = []
    qn = [0]

    def add(cat, query, relevant, derivation, **extra):
        qn[0] += 1
        q = {
            "id": f"q{qn[0]:02d}",
            "category": cat,
            "query": query,
            "relevant": sorted(relevant),
            "derivation": derivation,
        }
        q.update(extra)
        queries.append(q)
        return q

    # --- exact_rare ---------------------------------------------------------
    for tokname, ids, df in mine_exact_rare(nodes, byid, EXACT_RARE_SEEDS):
        if df != 1 or len(ids) != 1:
            problems.append(f"exact_rare '{tokname}': df={df}, ids={len(ids)} (expected df=1)")
            continue
        lab = (byid[ids[0]].get("label") or "")[:60]
        add("exact_rare", tokname, ids,
            f"MINED: token '{tokname}' has document frequency 1 over all {len(nodes)} corpus nodes "
            f"(re-verified at build time). Its single containing node is {ids[0]} "
            f"('{lab}'), which is therefore the only possible correct answer.")

    # --- phrase -------------------------------------------------------------
    for phrase, why in PHRASE_SEEDS:
        ids = phrase_matches(nodes, phrase)
        if not ids:
            problems.append(f"phrase '{phrase}': 0 corpus matches — unwinnable")
            continue
        if len(ids) > PHRASE_MAX:
            problems.append(f"phrase '{phrase}': {len(ids)} matches > {PHRASE_MAX} — too diffuse")
            continue
        add("phrase", phrase, ids,
            f"MINED: {why}. Case-insensitive verbatim substring scan over label+content+tags at "
            f"build time returns exactly {len(ids)} node(s); that set IS the answer key.")

    # --- paraphrase ---------------------------------------------------------
    for target, query, why in PARAPHRASE_SEEDS:
        if target not in byid:
            problems.append(f"paraphrase target {target} not in corpus")
            continue
        qt = content_tokens(query)
        leak = sorted(qt & tokset[target])
        if leak:
            problems.append(f"paraphrase '{query}': leaks {leak} into target {target}")
            continue
        add("paraphrase", query, [target],
            f"HAND-SELECTED with criterion: {why} VERIFIED at build time: of the {len(qt)} content "
            f"words in the query, ZERO appear anywhere in the target's label, content, or tags — so "
            f"no string-matching retriever can reach this answer.",
            zero_overlap_verified=True, query_content_words=sorted(qt))

    # --- associative --------------------------------------------------------
    sibs = hub_siblings(edges, VALUES_HUB)
    if len(sibs) < 5:
        problems.append(f"associative: values hub {VALUES_HUB} has only {len(sibs)} siblings")
    for src, query in ASSOCIATIVE_SEEDS:
        if src not in byid or src not in sibs:
            problems.append(f"associative source {src} not a sibling on {VALUES_HUB}")
            continue
        qt = content_tokens(query)
        others = [s for s in sibs if s != src and s in byid]
        # A sibling only counts as a legitimate expected answer if the query
        # cannot reach it lexically. Drop any sibling that shares a content word.
        clean = [s for s in others if not (qt & tokset[s])]
        dropped = len(others) - len(clean)
        if len(clean) < 5:
            problems.append(f"associative '{query}': only {len(clean)} lexically-unreachable siblings")
            continue
        add("associative", query, clean,
            f"DERIVED FROM EDGES: the query is built from the distinctive vocabulary of {src} "
            f"('{(byid[src].get('label') or '')[:48]}'), which hangs off the values hub {VALUES_HUB} "
            f"by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub "
            f"({len(clean)} of {len(others)}; {dropped} dropped because they shared a query word and "
            f"so were lexically reachable). Every remaining sibling shares ZERO content words with "
            f"the query — the only route from query to answer is seed({src}) -> hub -> sibling, a "
            f"two-hop traversal.",
            associative_source=src, hub=VALUES_HUB, siblings_dropped_for_overlap=dropped)

    # --- nonsense -----------------------------------------------------------
    all_tokens = set()
    for n in nodes:
        all_tokens |= {t for t in TOKEN.findall(doctext(n).lower())}
    for s in NONSENSE_SEEDS:
        present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
        if present:
            problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
            continue
        add("nonsense", s, [],
            f"CONTROL: verified at build time that none of this string's tokens occurs anywhere in "
            f"the corpus. Correct behaviour is to return NOTHING; any result is a false positive.",
            expect_empty=True)

    # --- superseded ---------------------------------------------------------
    for query, correct, stale, why in find_superseded_pairs(nodes, byid):
        if correct not in byid or stale not in byid:
            problems.append(f"superseded '{query}': id missing from corpus")
            continue
        add("superseded", query, [correct],
            f"DERIVED: {why} Scored on RANKING, not presence: the corrected node {correct} must be "
            f"returned AND must rank above the stale node {stale}.",
            must_outrank=[correct, stale],
            stale_id=stale,
            correct_label=(byid[correct].get("label") or "")[:70],
            stale_label=(byid[stale].get("label") or "")[:70])

    return queries, problems


def summarize(queries):
    c = Counter(q["category"] for q in queries)
    return ", ".join(f"{k}={c[k]}" for k in
                     ("exact_rare", "phrase", "paraphrase", "associative", "nonsense", "superseded")
                     if c[k])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("snapshot")
    ap.add_argument("--out", default=DEFAULT_OUT)
    ap.add_argument("--check", action="store_true",
                    help="validate only; do not write. Non-zero exit if anything is unwinnable.")
    args = ap.parse_args()

    nodes, edges = load_corpus(args.snapshot)
    print(f"corpus: {len(nodes)} nodes, {len(edges)} edges  ({os.path.basename(args.snapshot)})")
    queries, problems = build(nodes, edges)

    print(f"gold set: {len(queries)} queries  [{summarize(queries)}]")
    if problems:
        print(f"\n{len(problems)} PROBLEM(S) — these queries were REJECTED, not silently kept:")
        for p in problems:
            print("   -", p)

    if args.check:
        sys.exit(1 if problems else 0)

    doc = {
        "corpus": os.path.abspath(args.snapshot),
        "corpus_nodes": len(nodes),
        "corpus_edges": len(edges),
        "note": ("Every query carries a `derivation` recording how its expected answer was chosen. "
                 "Re-run with --check to re-validate the whole set against the corpus."),
        "queries": queries,
    }
    with open(args.out, "w", encoding="utf-8") as fh:
        json.dump(doc, fh, indent=1, ensure_ascii=False)
    print(f"\nwrote {args.out}")


if __name__ == "__main__":
    main()