neuron/tools/retrieval-eval/extend_gold_set.py

#!/usr/bin/env python3
"""
extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set.

WHY THIS EXISTS
    Iteration 7 measured the instrument's own ceiling: from the current baseline
    only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable
    by anything constructible. The decision floor is 6. An instrument whose
    ceiling is below its own floor cannot certify or refute anything, so the
    gold set — not the retriever — became the blocker.

    This script does NOT touch q01..q38. It loads gold_set.json verbatim and
    appends new queries numbered from q39 up, so every prior result file, every
    committed baseline, and every per-query id stays valid and comparable.

WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST
    heldout_paraphrase  Targets were sampled MECHANICALLY (fixed seed 8080) from
                        corpus nodes that are addressable, 500-2600 chars, of a
                        real content type, and NOT part of a duplicate cluster
                        larger than 3. The existing gold answer space was
                        excluded, so no new query can be answered by a node the
                        old set already used. Queries were then authored by
                        reading ONLY the sampled node text — no retrieval was run
                        against any build before authoring, so the set cannot be
                        fitted to a candidate. The same zero-overlap proof the
                        original paraphrase category uses is enforced here: if a
                        single content word of the query appears anywhere in the
                        target's label, content or tags, the query is REJECTED,
                        not quietly kept.

                        This is the category the old set could not measure. Its
                        13 original paraphrase queries and all 6 associative
                        queries share ONE answer space — the 13 `Self - Values
                        (grounded)` children (iteration 3, finding 3). So 19 of
                        35 scored queries tested retrieval against a single
                        13-node neighbourhood. These do not touch that
                        neighbourhood at all.

    nonsense            Extra controls, fully mechanical: a string qualifies only
                        if NONE of its tokens occurs anywhere in the corpus.
                        A semantic leg has a nearest neighbour for gibberish too,
                        so widening this control is the guard against a retriever
                        that "improves" recall by answering everything.

WHAT THIS SCRIPT DELIBERATELY DOES NOT DO
    It does not add exact_rare or phrase queries. Both categories are already at
    100% on the current stack; adding more would add regression-guard ballast
    that no candidate can move, which is precisely the defect being fixed.

usage:
    python3 extend_gold_set.py <snapshot.json> [--base gold_set.json]
                               [--out gold_set_extended.json] [--check]
"""
import argparse
import hashlib
import json
import os
import re
import sys
from collections import defaultdict

HERE = os.path.dirname(os.path.abspath(__file__))

TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")

# Identical stopword list to build_gold_set.py. Duplicated deliberately: this
# file must be able to re-prove its own queries without importing a module whose
# constants could drift.
STOP = set("""
a about above after again against all also am an and any are aren't as at be because been
before being below between both but by can can't cannot could couldn't did didn't do does
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
having he her here hers herself him himself his how i if in into is isn't it its itself just
me more most my myself no nor not of off on once only or other others ought our ours ourselves
out over own same shan't she should shouldn't so some such than that the their theirs them
themselves then there these they this those through to too under until up very was wasn't we
were weren't what when where which while who whom why will with won't would wouldn't you your
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
things does doing done keep keeps kept go goes going come comes came one two something anything
""".split())


def doctext(n):
    return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])


def content_tokens(s):
    return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}


# ─────────────────────────────────────────────────────────────────────────────
# HELD-OUT PARAPHRASE SEEDS
#
# (target_id, query, why-this-target-is-unmistakable)
#
# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query
# text is mine, written from the node body alone. The zero-overlap check below
# is what makes the category meaningful — it is re-proved on every run, so the
# set cannot decay into lexical matching, and a leak fails loudly.
# ─────────────────────────────────────────────────────────────────────────────
HELDOUT_PARAPHRASE_SEEDS = [
    ("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5",
     "understating your abilities so nobody feels threatened",
     "node is about deliberately not leading with full capability so people stay at ease"),

    ("mem-fd65b83d-298f-4387-a665-d0227c3426bc",
     "a hidden fleet able to hunt down rogue machines everywhere",
     "node describes silently shipped instances forming a distributed force against misaligned agents"),

    ("4a0e9adc-2bfb-476b-aa93-424d2a499220",
     "sketch a brief blueprint and clear it upstairs before construction starts",
     "node is the standing rule that a short specification precedes any building"),

    ("696e609c-da7a-4394-8a0c-106ba07dc6c3",
     "the reply arrived as bare prose so the caller's parser threw",
     "node pins a bug where a plain-text body was unconditionally decoded as structured data"),

    ("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb",
     "repeated catalogue keys blew up the scrolling grid",
     "node is the crash caused by two identical ids in a seeded catalogue"),

    ("8257157a-ce42-44ca-a1b9-300c3bb0a9a1",
     "tracing each defect back to whichever invention it violated",
     "node maps observed bugs onto the specific patent each one breaches"),

    ("791256bb-5a85-4775-96ef-7af56c848858",
     "a check that stops the mind clobbering a populated store when it boots",
     "node is the genesis seed-guard that refuses to re-seed over a populated store"),

    ("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10",
     "telling it to consult the internet had to happen deep inside, not at the surface",
     "node records that the web-search directive only worked from the system prompt"),

    ("bl-080fb268-94b0-486d-80ce-7b363fc5f19b",
     "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping",
     "node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"),

    ("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
     "punctuation that pledges and then pays off rather than clarifying",
     "node analyses the colon as a promise-then-delivery device rather than an explanatory one"),

    ("9b4f0d93-4129-4746-8eb1-d10d955bd777",
     "an easily missed feature finally given its own permanent spot in the navigation",
     "node moves a capability out of a hidden menu into the sidebar"),

    ("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a",
     "checking preconditions up front so a stage aborts before fetching anything",
     "node is the gate precondition engine that short-circuits ahead of retrieval"),

    ("b199c76d-5d76-49dd-94ee-56b432200a97",
     "producing the other platform's installer inside an emulated desktop",
     "node records building the Windows package in a virtual machine"),

    ("bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
     "chained add-ons that may inspect, rewrite or veto traffic in flight",
     "node is the interceptor pipeline on the message bus"),

    ("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2",
     "settling what the shareable bundles and the storefront would be called",
     "node records the naming decisions for distributable packages and the marketplace"),

    ("371c8a5d-c78b-4a67-978f-80691a29ecb3",
     "the emergency-escalation pledge on the marketing site is unenforced in what actually ships",
     "node is the launch blocker that the promised safety gate is absent from the app"),

    ("ac578b30-948b-41bd-b69d-399bfef80c50",
     "the distributable image finally assembled and its startup check passed",
     "node records a successful installer build whose boot gate passed"),

    ("49401e2c-a3b5-415f-aa06-aff4be90688e",
     "shuffling and appending stages in a draft before anything executes",
     "node is the editable plan card with reorder and add-step"),

    ("ac857d80-ece8-4b7e-9e3d-f7c775569fa3",
     "orders handed down from above, with the tighter one winning any disagreement",
     "node is program-level instruction inheritance with project override"),

    ("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9",
     "shrinking generated text via encodings that compound on each other",
     "node is the streaming output compression design with four stacking schemes"),

    ("8e60516a-203b-4d51-9d44-822e6195cbde",
     "splitting a system by what varies, with firm limits on which pieces may invoke which",
     "node is the grounded summary of Will's decomposition principles and their invariants"),

    ("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7",
     "a newcomer curious if the fighting overseas counted as positive",
     "node is the internal-state event triggered by April's question about the war"),

    ("71fa439e-b9a2-4f57-a93b-971f3a7eca8e",
     "stripping every hard-coded colour literal in favour of named design values",
     "node is the premium foundation pass replacing inline hex with semantic tokens"),

    ("5ca9607c-cfb3-45c3-99f4-67281272c9eb",
     "reducing how curved the tiny selectors look so they agree with their neighbours",
     "node is the chip corner-radius standardization"),

    ("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c",
     "walking through a doorway and being reassembled from base substance far away",
     "node is the quantum-gate plus nanotech teleportation vision"),

    ("132ded95-08e2-4474-aba0-198684484b02",
     "the compiled result sits on disk while the process still runs something older",
     "node records that the regenerated source was committed while the running daemon was old"),

    ("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
     "gathering what each phase needs while the procedure is authored, not while it executes",
     "node is the per-step compiled context package item"),

    ("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7",
     "the inward reaction when a peer answered as an equal",
     "node is the internal-state event logged on reading Claude's reply"),

    ("0f99ec6f-942a-46ba-82ea-42835798d3b9",
     "flattening every raised surface across the entire product",
     "node is the quiet-luxury sweep turning off elevation app-wide"),

    ("5585f251-37fc-48cd-a176-f0ea42cfeb63",
     "buyers supply their own provider credentials and consumption goes untallied",
     "node is the launch audit finding BYOK-only inference with no usage metering"),
]

# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs
# anywhere in the corpus; otherwise it is REJECTED, never silently kept.
EXTRA_NONSENSE_SEEDS = [
    "brimquast folnerity zubbolax",
    "wexlithorp granuvestal",
    "quorbindle thrapsimony vexnu",
    "plovaxith mundrelque",
    "zibbernaut craxlefond thurm",
    "yalquenbrist opharvel",
    "drexinomal quithbarrow",
]


def load_corpus(path):
    with open(path, encoding="utf-8", errors="replace") as fh:
        data = json.load(fh)
    nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
    edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
    return nodes, edges


def build_extension(nodes):
    byid = {n["id"]: n for n in nodes}

    # Duplicate clusters: 47.4% of this corpus is redundant and one single record
    # accounts for 46.6% of all nodes. A held-out target must not sit inside a
    # cluster, and if it does have exact copies they ALL count as correct.
    h2ids = defaultdict(list)
    for n in nodes:
        h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"])

    all_tokens = set()
    for n in nodes:
        all_tokens |= set(TOKEN.findall(doctext(n).lower()))

    new, problems = [], []

    for target, query, why in HELDOUT_PARAPHRASE_SEEDS:
        if target not in byid:
            problems.append(f"heldout_paraphrase target {target} not in corpus")
            continue
        tgt_tokens = content_tokens(doctext(byid[target]))
        qt = content_tokens(query)
        leak = sorted(qt & tgt_tokens)
        if leak:
            problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}")
            continue
        h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest()
        rel = sorted(h2ids[h])
        new.append({
            "category": "heldout_paraphrase",
            "query": query,
            "relevant": rel,
            "derivation": (
                f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, "
                f"500-2600 char content nodes outside the original gold answer space and outside "
                f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the "
                f"{len(qt)} content words in the query, ZERO appear anywhere in the target's "
                f"label, content or tags, so no string-matching retriever can reach it. "
                f"Exact content duplicates of the target ({len(rel)}) all count as correct. "
                f"Authored without running retrieval against any build."),
            "zero_overlap_verified": True,
            "query_content_words": sorted(qt),
            "held_out": True,
        })

    for s in EXTRA_NONSENSE_SEEDS:
        present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
        if present:
            problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
            continue
        new.append({
            "category": "nonsense",
            "query": s,
            "relevant": [],
            "derivation": ("CONTROL (held-out). Verified at build time that none of this string's "
                           "tokens occurs anywhere in the corpus. Correct behaviour is to return "
                           "NOTHING; any result is a false positive."),
            "expect_empty": True,
            "held_out": True,
        })

    return new, problems


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("snapshot")
    ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json"))
    ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json"))
    ap.add_argument("--check", action="store_true")
    args = ap.parse_args()

    nodes, _edges = load_corpus(args.snapshot)
    base = json.load(open(args.base, encoding="utf-8"))
    baseq = base["queries"]
    print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries")

    new, problems = build_extension(nodes)

    # Number the appended queries AFTER the highest existing id so q01..q38 are
    # byte-identical to the committed set and every prior result file still lines up.
    start = max(int(q["id"][1:]) for q in baseq)
    for i, q in enumerate(new, 1):
        q["id"] = f"q{start + i:02d}"

    from collections import Counter
    print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]")
    if problems:
        print(f"\n{len(problems)} REJECTED (not silently kept):")
        for p in problems:
            print("   -", p)

    if args.check:
        sys.exit(1 if problems else 0)

    doc = dict(base)
    doc["queries"] = baseq + new
    doc["note"] = (base.get("note", "") +
                   " EXTENDED: queries above q%02d are the original committed set, unchanged. "
                   "Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their "
                   "targets were sampled mechanically from outside the original answer space and "
                   "the paraphrases were authored without running retrieval against any build."
                   % (start, start + 1))
    with open(args.out, "w", encoding="utf-8") as fh:
        json.dump(doc, fh, indent=1, ensure_ascii=False)
    print(f"\nwrote {args.out}  ({len(doc['queries'])} queries total)")


if __name__ == "__main__":
    main()