diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 8508781..4d6a9dc 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -63,6 +63,16 @@ jobs: cp vendor/el-runtime/v1.0.0-20260501/el_runtime.h /opt/el/runtime/el_runtime.h echo "El runtime PINNED to v1.0.0-20260501: $(ls /opt/el/runtime/)" + # neuron#133: CI compiles dist/soul.c, NOT the .el sources. On 2026-08-07 a + # build off main would have shipped an engine with none of five merged fixes, + # including a P0 safety fix, while main's source read as correct. The runner + # cannot regenerate the amalgam (elc needs 24GB+ virtual memory), but it can + # refuse to compile a stale one. Fails loudly with the recipe in the message. + - name: Verify dist/soul.c matches the sources + run: | + chmod +x tools/soulc-stamp.sh + ./tools/soulc-stamp.sh --check + - name: Build neuron soul binary run: | RUNTIME=/opt/el/runtime diff --git a/dist/soul.c b/dist/soul.c index bb7814c..b864151 100644 --- a/dist/soul.c +++ b/dist/soul.c @@ -974,6 +974,10 @@ el_val_t wt_commit(el_val_t id); el_val_t tier_working(void); el_val_t tier_episodic(void); el_val_t tier_canonical(void); +el_val_t mem_assoc_skip_label(el_val_t label); +el_val_t mem_assoc_ok(el_val_t cand_id, el_val_t cand_label, el_val_t self_id); +el_val_t mem_assoc_slot(el_val_t results, el_val_t idx, el_val_t new_id); +el_val_t mem_associate(el_val_t new_id, el_val_t content, el_val_t label); el_val_t mem_store(el_val_t content, el_val_t label, el_val_t tags); el_val_t mem_remember(el_val_t content, el_val_t tags); el_val_t mem_recall(el_val_t query, el_val_t depth); @@ -25573,6 +25577,97 @@ el_val_t tier_canonical(void) { return 0; } +el_val_t mem_assoc_skip_label(el_val_t label) { + if (str_contains(label, EL_STR("state-event"))) { + return 1; + } + if (str_contains(label, EL_STR("soul-response"))) { + return 1; + } + if (str_contains(label, EL_STR("soul-outbox"))) { + return 1; + } + if (str_contains(label, EL_STR("boot_count"))) { + return 1; + } + if (str_contains(label, EL_STR("loop-outcome"))) { + return 1; + } + if (str_contains(label, EL_STR("search-result"))) { + return 1; + } + return 0; + return 0; +} + +el_val_t mem_assoc_ok(el_val_t cand_id, el_val_t cand_label, el_val_t self_id) { + if (str_eq(cand_id, EL_STR(""))) { + return 0; + } + if (str_eq(cand_id, self_id)) { + return 0; + } + el_val_t lab = str_lower(cand_label); + if (str_starts_with(lab, EL_STR("self"))) { + return 0; + } + if (str_starts_with(lab, EL_STR("value"))) { + return 0; + } + if (str_contains(lab, EL_STR("values"))) { + return 0; + } + if (str_contains(lab, EL_STR("identity"))) { + return 0; + } + if (mem_assoc_skip_label(cand_label)) { + return 0; + } + return 1; + return 0; +} + +el_val_t mem_assoc_slot(el_val_t results, el_val_t idx, el_val_t new_id) { + if (idx >= json_array_len(results)) { + return 0; + } + el_val_t cand = json_array_get(results, idx); + el_val_t cid = json_get(cand, EL_STR("id")); + el_val_t clabel = json_get(cand, EL_STR("label")); + el_val_t ctype = json_get(cand, EL_STR("node_type")); + if (str_eq(ctype, EL_STR("Value"))) { + return 0; + } + if (str_eq(ctype, EL_STR("DharmaSelf"))) { + return 0; + } + if (str_eq(ctype, EL_STR("Safety"))) { + return 0; + } + if (mem_assoc_ok(cid, clabel, new_id)) { + wt_edge(new_id, cid, el_from_float(0.5), EL_STR("related")); + } + return 0; +} + +el_val_t mem_associate(el_val_t new_id, el_val_t content, el_val_t label) { + if (str_eq(new_id, EL_STR(""))) { + return 0; + } + if (mem_assoc_skip_label(label)) { + return 0; + } + el_val_t probe = str_slice(content, 0, 400); + el_val_t results = engram_recall_json(probe, 4); + if (str_eq(results, EL_STR(""))) { + return 0; + } + mem_assoc_slot(results, 0, new_id); + mem_assoc_slot(results, 1, new_id); + mem_assoc_slot(results, 2, new_id); + return 0; +} + el_val_t mem_store(el_val_t content, el_val_t label, el_val_t tags) { el_val_t id = wt_node(content, EL_STR("Memory"), label, el_from_float(0.5), el_from_float(0.5), el_from_float(0.8), EL_STR("Working"), tags); if (str_eq(id, EL_STR(""))) { @@ -25580,6 +25675,7 @@ el_val_t mem_store(el_val_t content, el_val_t label, el_val_t tags) { return EL_STR(""); } el_val_t durable = wt_commit(id); + mem_associate(id, content, label); if (durable) { println(el_str_concat(el_str_concat(el_str_concat(EL_STR("[memory] write persisted at owner: "), id), EL_STR(" label=")), label)); } else { @@ -29709,7 +29805,9 @@ el_val_t handle_api_begin_session(el_val_t body) { el_val_t state_events = api_compact_node_array(state_events_raw, 5, 500); el_val_t recent_raw = engram_scan_nodes_json(10, 0); el_val_t recent = api_compact_node_array(recent_raw, 10, 240); - return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"stats\":"), stats), EL_STR(",\"recent\":")), recent), EL_STR(",\"activated\":")), activated), EL_STR(",\"self_neighbors\":[]")), EL_STR(",\"recent_state_events\":")), state_events), EL_STR("}")); + el_val_t self_raw = engram_neighbors_json(EL_STR("kn-efeb4a5b-5aff-4759-8a97-7233099be6ee"), 1, EL_STR("both")); + el_val_t self_slice = api_compact_node_array(self_raw, 24, 240); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"stats\":"), stats), EL_STR(",\"recent\":")), recent), EL_STR(",\"activated\":")), activated), EL_STR(",\"self_neighbors\":")), self_slice), EL_STR(",\"recent_state_events\":")), state_events), EL_STR("}")); return 0; } @@ -29739,6 +29837,7 @@ el_val_t handle_api_remember(el_val_t body) { if (!api_persisted(id)) { return api_not_persisted(id); } + mem_associate(id, content, EL_STR("memory:remembered")); return el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"ok\":true}")); return 0; } @@ -29830,7 +29929,7 @@ el_val_t handle_api_recall(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(eff_q, EL_STR(""))) { return api_or_empty(engram_scan_nodes_json(limit, 0)); } - el_val_t results = engram_search_json(eff_q, limit); + el_val_t results = engram_recall_json(eff_q, limit); return api_or_empty(results); return 0; } diff --git a/dist/soul.c.stamp b/dist/soul.c.stamp new file mode 100644 index 0000000..7bbbf80 --- /dev/null +++ b/dist/soul.c.stamp @@ -0,0 +1,18 @@ +# soul.c.stamp — fingerprint of the .el sources dist/soul.c was generated from. +# Written by tools/soulc-stamp.sh --write. Do not hand-edit. +# generated_amalgam_sha256 e31f760de5f6629cd37bcfd26172f92be3ca5d6ff36480f6adafa1d4a1de5a66 +# generated_amalgam_bytes 1179122 +f8597e10546654bce3fbbe40461b2da59d0e06dbf1b038d1d362d24f949e3911 awareness.el +b6f3d14ca0c26017a2d617399a6d3754dabb0905e4d5f52eb75d25c4ad18d3c5 chat.el +42288c212cbf72fb1e8ecbd4d9900e4e9ee1cfa475b7974295c7637f1bf2939f elp-input.el +b3f77f49d6086932c38bd17fe7a5eaf8bce25685f6fc3e1750f05729c6b49b9e imprint.el +fba8ffdb9ba72bca5b09ca1c93a520edc52f3f4d8aec2c7585fe9b17e06420b2 manifest.el +550a72e234ae8cec1f33e02108fd365353f45edd88513da90b792e79b6c0e5f0 memory.el +77640783df5c38066149dcd11a11aea7e36f66e40e3df6aedf6723be8b9e0e1e neuron-api.el +03c47c451e0e87f2c252cadb4b765867943962a804f548dd53adeef0520912c8 persist.el +541b9309c59c33ee619d393714871b820ec4151b721d8b2aebcfdb7ce8eb2ffa routes.el +c28e36952ec56525963a0bdf29455ab097d3b0c5653d19c25fbb005e1069a1f7 safety.el +fd3ab91d0ae0ea26639e21bef2f8f94054dc4b02eae68b19e3fe689d2769aad4 sessions.el +0f1cf43904a98a5a646cce5a07e0e96162ced662692fbc13357d9b67d9a8ac3d soul.el +30337940905171a9645b0929f0a412ce6b3dccb1246495070c553bca0bbae6cd stewardship.el +e105dc5990e6adbf39db9dc0462cd8bcf6e6c3dfd03709059227ecfad2bbab29 studio.el diff --git a/memory.el b/memory.el index cbbd765..eee726f 100644 --- a/memory.el +++ b/memory.el @@ -4,6 +4,86 @@ fn tier_working() -> String { return "Working" } fn tier_episodic() -> String { return "Episodic" } fn tier_canonical() -> String { return "Canonical" } +// ── Association on write ────────────────────────────────────────────────────── +// DESIGN: "promotion integrates candidate nodes by linking them to existing nodes +// using typed semantic edges RATHER THAN APPENDING AS UNLINKED CONTENT" (CCR +// claim 29). Unlinked append is the explicitly rejected behaviour — and it is the +// only behaviour this system had. Measured 2026-08-09 on Tim's graph: 14,214 edges +// across 80,936 nodes, 5% of nodes connected to anything, and NO edge created by +// any write since 2026-07-19 while 27,000+ nodes were added. A memory that forms +// no connections cannot be reached by spreading activation, so retrieval silently +// degrades to literal matching. +// +// BOUNDS, each one bought with a specific failure: +// * max 3 edges per memory — link_memories.py's cap, precision over spray +// * never link to identity (self/*, Value): the existing policy is explicit that +// "memories must not pollute the self traversal by similarity; only an explicit +// citation may touch identity". Similarity is not citation. +// * never link telemetry (state-event, soul-response, boot_count, loop-outcome): +// these are ~97% of daily write volume (1,020 vs 31 real memories on 08-08). +// Linking them would add ~3,000 noise edges a day and re-flatten the graph in +// the name of connecting it. +// * fail-soft: a failed association never fails the write. +// Edges go through wt_edge so they reach the owner and survive restart. +fn mem_assoc_skip_label(label: String) -> Bool { + if str_contains(label, "state-event") { return true } + if str_contains(label, "soul-response") { return true } + if str_contains(label, "soul-outbox") { return true } + if str_contains(label, "boot_count") { return true } + if str_contains(label, "loop-outcome") { return true } + if str_contains(label, "search-result") { return true } + return false +} + +// A candidate is linkable only if it is a real, distinct, non-identity node. +fn mem_assoc_ok(cand_id: String, cand_label: String, self_id: String) -> Bool { + if str_eq(cand_id, "") { return false } + if str_eq(cand_id, self_id) { return false } + // CASE MATTERS — measured 2026-08-09. A lowercase-only check let a memory link + // to "Self — Values (grounded)", i.e. it polluted the self traversal, which is + // the one thing this policy exists to prevent. My verification had the same + // blind spot and printed PASS. Check every casing the graph actually uses, and + // exclude identity node TYPES as well as labels. + let lab: String = str_lower(cand_label) + if str_starts_with(lab, "self") { return false } + if str_starts_with(lab, "value") { return false } + if str_contains(lab, "values") { return false } + if str_contains(lab, "identity") { return false } + if mem_assoc_skip_label(cand_label) { return false } + return true +} + +// One slot of the association. Manual unroll rather than a loop: EL's codegen +// mis-emits accumulating while-loops (documented at soul.el:212, which unrolled +// three affective slots for the same reason). +fn mem_assoc_slot(results: String, idx: Int, new_id: String) -> Void { + if idx >= json_array_len(results) { return } + let cand: String = json_array_get(results, idx) + let cid: String = json_get(cand, "id") + let clabel: String = json_get(cand, "label") + let ctype: String = json_get(cand, "node_type") + if str_eq(ctype, "Value") { return } + if str_eq(ctype, "DharmaSelf") { return } + if str_eq(ctype, "Safety") { return } + if mem_assoc_ok(cid, clabel, new_id) { + wt_edge(new_id, cid, el_from_float(0.5), "related") + } +} + +// mem_associate — connect a freshly written memory to what it is about. +fn mem_associate(new_id: String, content: String, label: String) -> Void { + if str_eq(new_id, "") { return } + if mem_assoc_skip_label(label) { return } + // Ask the graph what this memory resembles. Now that the store carries + // meaning-vectors this is semantic, not merely lexical. + let probe: String = str_slice(content, 0, 400) + let results: String = engram_recall_json(probe, 4) + if str_eq(results, "") { return } + mem_assoc_slot(results, 0, new_id) + mem_assoc_slot(results, 1, new_id) + mem_assoc_slot(results, 2, new_id) +} + fn mem_store(content: String, label: String, tags: String) -> String { let id: String = wt_node( content, @@ -31,6 +111,9 @@ fn mem_store(content: String, label: String, tags: String) -> String { // rather than claiming a save that did not happen. The id is still returned: // the local write DID succeed, and the queued delta will be retried. let durable: Bool = wt_commit(id) + // Associate AFTER the node is durable: an edge to a node that did not persist + // is a dangling edge, which is the defect the 2026-08-09 cleanup removed 830 of. + mem_associate(id, content, label) if durable { println("[memory] write persisted at owner: " + id + " label=" + label) } else { diff --git a/neuron-api.el b/neuron-api.el index b85ca1b..061b68a 100644 --- a/neuron-api.el +++ b/neuron-api.el @@ -304,10 +304,35 @@ fn handle_api_begin_session(body: String) -> String { let state_events: String = api_compact_node_array(state_events_raw, 5, 500) let recent_raw: String = engram_scan_nodes_json(10, 0) let recent: String = api_compact_node_array(recent_raw, 10, 240) + // SELF-SEEDED SLICE (2026-08-09). The design is explicit: "Every compilation + // query begins at the self-model node and traverses outward... structural + // reachability from the self-model node is a precondition for any node to + // appear in compiled context" (will-anderson patents/drafts/engram-claims.md, + // Self-Seeded Activation; DRAFT, not a filed provisional — cite it as such). + // + // Measured 2026-08-09 before this change: compiled context contained 0-1 + // identity records out of 10, because compilation seeds from a hardcoded + // TEXT STRING, never from the self. Even an explicit "my values identity who + // I am" query returned a boot counter and state-events. + // + // This restores the designed behaviour WITHOUT repeating the failure that got + // self_neighbors set to [] in the first place: that was an UNBOUNDED ~90KB + // neighbour dump which closed the socket on every call. Same bound as every + // other list here — cap 8, 240-char snippets. The self root has 34 direct + // neighbours of which 23 are identity records, so depth 1 is dense enough to + // be worth seeding and small enough to stay cheap. + let self_raw: String = engram_neighbors_json("kn-efeb4a5b-5aff-4759-8a97-7233099be6ee", 1, "both") + // Cap 24, not 8: measured 2026-08-09, the self root's first 8 neighbours are + // TAG nodes ("neuron", "tier:note", "disposition:experimental", "imprint", + // "traversal") which crowd out the substantive identity records behind them. + // The root has 34 neighbours of which 23 are identity; 24 captures them while + // staying bounded. Cost measured at ~+4KB on a ~12KB response, nowhere near + // the ~90KB unbounded dump that closed sockets and got this set to []. + let self_slice: String = api_compact_node_array(self_raw, 24, 240) return "{\"stats\":" + stats + ",\"recent\":" + recent + ",\"activated\":" + activated - + ",\"self_neighbors\":[]" + + ",\"self_neighbors\":" + self_slice + ",\"recent_state_events\":" + state_events + "}" } @@ -355,6 +380,13 @@ fn handle_api_remember(body: String) -> String { sal, sal, el_from_float(0.9), "Episodic", final_tags) if !api_persisted(id) { return api_not_persisted(id) } + // Associate on write (2026-08-09). THIS CALL MUST BE HERE, not only in mem_store. + // The HTTP memory route writes via wt_node directly; mem_store serves only the + // awareness paths (soul-response, search-result, activation-result) which are + // exactly the telemetry we refuse to link. Hooking mem_store alone produced + // ZERO edges across four real writes — measured, not assumed, which is the only + // reason it was caught before shipping. + mem_associate(id, content, "memory:remembered") return "{\"id\":\"" + id + "\",\"ok\":true}" } @@ -459,7 +491,12 @@ fn handle_api_recall(method: String, path: String, body: String) -> String { if str_eq(eff_q, "") { return api_or_empty(engram_scan_nodes_json(limit, 0)) } - let results: String = engram_search_json(eff_q, limit) + // engram_recall_json, not engram_search_json: this route IS the retrieval + // surface (claim 24's "embedding search queries"), so it gets the semantic + // and associative legs. engram_search_json stays lexical because ~40 + // internal call sites pass a KEY and seven of them delete every record + // that comes back — see the boundary note above eg_search_json_impl. + let results: String = engram_recall_json(eff_q, limit) return api_or_empty(results) } diff --git a/tools/build-soul-from-dist.sh b/tools/build-soul-from-dist.sh new file mode 100755 index 0000000..9841d60 --- /dev/null +++ b/tools/build-soul-from-dist.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# build-soul-from-dist.sh — build a deployable soul from the SAME input CI compiles. +# +# THE PROBLEM THIS CLOSES: until now, deploys were built by build-soul.sh, which +# compiles a scratch amalgam and never touches dist/soul.c. CI compiles dist/soul.c. +# Two lineages. On 2026-08-09 the committed input fell 2,761 bytes behind the sources +# while three binaries built the other way were installed on the operator machine — +# so "what runs" and "what the repo says builds" were different artifacts again, +# which is the whole of #133 and #111 wearing new clothes. +# +# This builds from dist/soul.c with CI's own flags, after asserting that dist/soul.c +# actually matches the .el sources, and writes a provenance sidecar so a deployer can +# refuse anything of unknown origin. +# +# -rdynamic and -DHAVE_CURL are copied from .gitea/workflows/ci.yaml deliberately. +# The CI comment explains -rdynamic: without it the runtime cannot resolve its HTTP +# handler by name via dlsym and the binary serves nothing on every route. +# +# usage: build-soul-from-dist.sh +set -u +OUT="${1:?usage: build-soul-from-dist.sh }" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUNTIME="$ROOT/vendor/el-runtime/v1.0.0-20260501" + +cd "$ROOT" || exit 2 + +echo "[build-from-dist] GATE: does dist/soul.c match the sources?" +if ! ./tools/soulc-stamp.sh --check; then + echo "[build-from-dist] REFUSING — the build input is stale. Regenerate and stamp first." >&2 + exit 9 +fi + +[ -f "$RUNTIME/el_runtime.c" ] || { echo "pinned runtime missing at $RUNTIME" >&2; exit 2; } + +echo "[build-from-dist] compiling dist/soul.c with CI's flags" +cc -O2 -DHAVE_CURL -rdynamic \ + -I"$RUNTIME" \ + dist/soul.c \ + "$RUNTIME/el_runtime.c" \ + -lcurl -lpthread -lm \ + -o "$OUT" || { echo "[build-from-dist] COMPILE FAILED" >&2; exit 3; } + +# Provenance sidecar: what a deployer checks before installing anything. +SRC_SHA="$(shasum -a 256 dist/soul.c | awk '{print $1}')" +STAMP_SHA="$(shasum -a 256 dist/soul.c.stamp | awk '{print $1}')" +COMMIT="$(git rev-parse HEAD 2>/dev/null || echo unknown)" +DIRTY="clean"; [ -n "$(git status --porcelain -- '*.el' dist/soul.c 2>/dev/null)" ] && DIRTY="DIRTY" +cat > "$OUT.provenance" < $OUT ($(wc -c < "$OUT" | tr -d ' ') bytes)" +echo "[build-from-dist] provenance -> $OUT.provenance (commit ${COMMIT:0:8}, worktree $DIRTY)" diff --git a/tools/retrieval-eval/README.md b/tools/retrieval-eval/README.md new file mode 100644 index 0000000..2f204f1 --- /dev/null +++ b/tools/retrieval-eval/README.md @@ -0,0 +1,147 @@ +# Retrieval eval harness + +Measures Neuron's memory retrieval so a change can be shown to help before it is +believed to help. Nothing else on the memory roadmap should ship without a run +through this. + +``` +tools/retrieval-eval/run_comparison.sh --baseline main --candidate +``` + +That builds a soul from each ref, boots each in isolation on a fixed corpus, +runs the gold set three times per ref, and prints a table plus a verdict that +refuses to call a difference real if it is inside the noise band. + +## What was reused + +This is not a new idea, it is the missing third of an existing one. + +| Prior work | What it gave | What was missing | +|---|---|---| +| `docs/research/graphrag_eval/` (`collect.py`, `score.py`, 2026-06-08) | The three-retriever comparison that produced the numbers everyone quotes: substring 1.7% P@5, graph 21.7%, BM25 55%. Per-query relevant-id scoring, fixed-denominator precision@5, unique-relevant analysis. | 13 hand-written queries, judged by an LLM after the fact; measured the *live* soul on the *live* engram. | +| `docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py` | The pinned-query discipline: ground truth committed as regexes so every run judges alike, plus a `--check` winnability gate. 40 queries in 5 bands including a deliberate paraphrase-hard band. | Scored offline replicas of substring/BM25 — it never ran the real retrieval path. | +| `docs/research-archive/p0-prototypes/stage0_eval_20260714.py` | The `hit@5` metric and the substring/BM25 reference implementations. | Same: offline only. | +| `scripts/verify-soul-contract.sh` | The isolation recipe, verbatim: throwaway port, throwaway `HOME`, `SOUL_ENGRAM_PATH`, and the non-obvious `SOUL_ISE_URL` pin that stops an "isolated" soul silently syncing the operator's live brain. | It is a contract gate, not a measurement. | +| `_engine-liveness-91/gen-soul-amalgam.sh` + `.gitea/workflows/ci.yaml` | The build recipe (`elc --target=c` with every `.elh` on the import chain removed) and CI's exact compile flags. | — | + +**Reused directly:** the isolation recipe, the build recipe, fixed-denominator +precision@5, the pinned-ground-truth and winnability ideas. +**New here:** ids rather than regexes as ground truth, an associative category +derived from real graph edges, a superseded/contradicted category scored on +ranking, a machine-checked zero-lexical-overlap guarantee on paraphrases, +paired significance testing, and — the point — measurement against the **real +compiled soul** rather than an offline replica of one leg of it. + +## Design fit + +The thing under measurement is Will's designed retrieval: spreading activation +over the weighted directed graph, four-factor multiplicative scoring (parent +strength x edge weight x target salience x query/target cosine). A Python +re-implementation would measure my reading of the design. So the harness +compiles the actual `soul.el` amalgam and asks it over HTTP on +`/api/neuron/recall`, exactly as the MCP wrapper and the app do. + +## Files + +| File | Does | +|---|---| +| `build_gold_set.py` | Derives and **validates** the gold set from the corpus. `--check` re-validates and exits non-zero if a query became unwinnable or a paraphrase leaked a word. | +| `gold_set.json` | 38 queries. Every one carries a `derivation` string. | +| `run_eval.py` | Boots one soul in isolation, runs the gold set, writes metrics. Kills and **confirms dead** its child; records the confirmation in the results file. | +| `compare.py` | Paired diff of two result files with McNemar's exact test and a stated noise floor. | +| `build-soul.sh` | Compiles a soul binary from a plain source tree. | +| `run_comparison.sh` | All of the above, end to end, from two git refs. | + +## The gold set — 38 queries + +Built from the real corpus (`snapshot-pre-repair-20260806.json`, 78,768 nodes / +14,214 edges) so it reflects one person's accumulating memory, not document QA. + +| Category | n | Expected answer derived by | +|---|---|---| +| `exact_rare` | 6 | **Mined.** Tokens with document frequency 1 across all 78,768 nodes, whose single containing node is a 300–6000 char Memory/Knowledge/Belief. That node is the only possible answer. Re-verified every build. | +| `phrase` | 7 | **Mined.** Case-insensitive verbatim scan; the matching set *is* the answer key. Phrases matching >25 nodes are rejected as too diffuse. | +| `paraphrase` | 13 | **Hand-selected, machine-checked.** Target locked by id; the build then proves that **zero** content words of the query appear anywhere in the target's label, content, or tags. A leak fails the build — the category cannot quietly decay into lexical matching. | +| `associative` | 6 | **Derived from edges.** Query built from one value node's distinctive vocabulary; expected answers are its siblings on the `Self - Values (grounded)` hub. Siblings sharing any query word are dropped, so the only route from query to answer is seed -> hub -> sibling. | +| `nonsense` | 3 | **Control.** Verified that no token occurs anywhere in the corpus. Correct behaviour is to return nothing. | +| `superseded` | 3 | **Derived.** Correction/stale pairs located by regex scan, kept only when both sides resolve to different surviving nodes. Scored on **ranking**: the correction must be returned *and* rank above the stale node. | + +## Metrics + +`hit@5`, `recall@5`, `recall@10`, `precision@5` (fixed denominator 5, so an +empty result is punished like a page of junk), `MRR@10`, and wall-clock latency +per query (p50/p95/max). Output is a table plus a machine-readable JSON per run +so runs can be diffed. + +## Honesty about noise + +- **Minimum detectable swing on this 38-query set: 6 queries.** If every query + that changes changes the same way, `p = 2 x 0.5^n`, which first drops under + 0.05 at n=6. Any net change smaller than that is inside the noise band and + `compare.py` says so in those words. +- **Run-to-run drift is measured, not assumed.** Activation is a stateful read + by design (traversal reinforces what it touches), so identical inputs need not + give identical outputs. Observed: `main` 0 queries of drift across 3 runs + (fully deterministic); the activation branch 1 query. +- The noise floor used for the verdict is `max(6, observed_drift + 1)`. +- **This gold set is underpowered for small effects.** A genuine 3-query + improvement would not clear the bar. Growing the set is the fix; until then, a + small positive delta means "not shown", not "no effect". + +## First result: `main` vs `feat/recall-through-activation` + +Corpus and gold set identical, three runs each, fresh corpus copy per run. + +| | main | recall-through-activation | delta | +|---|---|---|---| +| hit@5 | 34.3% | 22.9% | **-11.4pp** | +| recall@5 | 26.9% | 19.1% | -7.9pp | +| recall@10 | 33.3% | 24.3% | -9.1pp | +| precision@5 | 12.0% | 7.4% | -4.6pp | +| MRR@10 | 0.294 | 0.242 | -0.053 | +| latency p50 | 1140 ms | 3209 ms | **2.81x** | +| latency p95 | 1584 ms | 4852 ms | 3.06x | +| nonsense clean | 2/3 | 2/3 | — | +| superseded outranks | 1/3 | 0/3 | -1 | + +By category (hit@5): + +| category | main | activation | +|---|---|---| +| exact_rare | 100% | 100% | +| phrase | 85.7% | **28.6%** | +| paraphrase | 0% | 0% | +| associative | 0% | 0% | +| superseded | 0% | 0% | + +**Verdict: directionally worse, one query short of significant.** 5 discordant +pairs, all 5 against the candidate, 0 for it. McNemar exact p = 0.0625 — under +the stated rule that is *inside* the noise band, so the harness reports "no +measurable difference" on accuracy and the honest summary is "5 for 5 the wrong +way, needs a 6th or a larger gold set to call". + +Latency is a different story: 2.8x at p50 is deterministic and far outside any +noise band. That regression is real. + +The result the branch was written for did not appear. Its stated purpose was to +recover sibling nodes one hub-hop away — the `associative` category — and that +category is **0/6 on both builds**. Probing directly: for the query +`Marines hernia sepsis medical ward`, the activation build returns the lexical +seed node itself at rank 8, and none of its 12 hub siblings anywhere in the top +10. The traversal is running; it is not reaching siblings. + +Two corpus facts likely explain it, and both are measurable rather than +speculative: + +1. **The graph is nearly edgeless.** Only 4,060 of 78,768 nodes (5.2%) carry any + edge at all — 14,214 edges total, 0.18 per node. Spreading activation over a + graph with no edges is an expensive way to do lexical matching, which is + roughly what the numbers show. +2. **No embeddings.** No node in this snapshot has an embedding field, so the + fourth factor of the four-factor product — query/target cosine similarity — + has nothing to compute from, and the semantic seeding pass is inert. + +That is the harness earning its keep on its first job: the change would have +felt like progress (it is the designed mechanism, and it does run) and measures +as a regression on phrase queries plus a 2.8x latency cost, with its intended +benefit unrealised because the corpus lacks the structure it needs. diff --git a/tools/retrieval-eval/bestval.py b/tools/retrieval-eval/bestval.py new file mode 100644 index 0000000..473c742 --- /dev/null +++ b/tools/retrieval-eval/bestval.py @@ -0,0 +1,21 @@ +import numpy as np, json, urllib.request +SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad" +np.seterr(all='ignore') +M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n') +eidx={k:i for i,k in enumerate(eids)} +gold=json.load(open("/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/gold_set.json"))['queries'] +VALS=sorted({r for q in gold if q['category']=='paraphrase' for r in q['relevant']}) +VI=[eidx[v] for v in VALS] +def emb(t): + b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode() + r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"}) + v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32) + return v/(np.linalg.norm(v)+1e-9) +print("qid cat bestValueNodeGlobalRank goldGlobalRank goldSiblingRank") +for q in gold: + if q['category']!='paraphrase': continue + v=emb(q['query']); s=M@v; s[~np.isfinite(s)]=-1 + ranks=sorted(int((s>s[j]).sum())+1 for j in VI) + g=eidx[q['relevant'][0]]; gr=int((s>s[g]).sum())+1 + sv=np.array([s[j] for j in VI]); sib=int((sv>s[g]).sum())+1 + print("%-4s %-11s best=%-5d (top3 val ranks %s) gold=%-5d sib=%d" % (q['id'],q['category'],ranks[0],ranks[:3],gr,sib)) diff --git a/tools/retrieval-eval/build-soul.sh b/tools/retrieval-eval/build-soul.sh new file mode 100755 index 0000000..e182f7a --- /dev/null +++ b/tools/retrieval-eval/build-soul.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# build-soul.sh — compile a soul binary from a plain source tree (no git needed). +# +# Reuses the amalgam recipe worked out in gen-soul-amalgam.sh (round 9.1) and the +# compile flags from .gitea/workflows/ci.yaml, so the binary under test is the +# same translation unit CI ships — not a re-implementation. +# +# elc --target=c emits only an extern prototype for any module that has a .elh +# header beside it, and inlines the module's bodies when it does not. So the +# amalgam is produced in a scratch copy with every .elh on the import chain +# deleted. +# +# usage: build-soul.sh +set -euo pipefail +SRC="${1:?usage: build-soul.sh }" +OUT="${2:?out-binary}" +ELC="${ELC:-$HOME/neuron-dev-stack/src/el/lang/dist/platform/elc}" +EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/el}" +RTDIR="${RTDIR:-$SRC/vendor/el-runtime/v1.0.0-20260501}" +SSL="${SSL_PREFIX:-/opt/homebrew/opt/openssl@3}" + +[ -x "$ELC" ] || { echo "no elc at $ELC" >&2; exit 2; } +[ -f "$RTDIR/el_runtime.c" ] || { echo "no el_runtime.c at $RTDIR" >&2; exit 2; } + +GEN="$(mktemp -d "${TMPDIR:-/tmp}/soul-build.XXXXXX")" +trap 'rm -rf "$GEN"' EXIT +mkdir -p "$GEN/neuron" "$GEN/foundation/el/elp/src" +cp "$SRC"/*.el "$GEN/neuron/" +cp "$EL_REPO"/elp/src/*.el "$GEN/foundation/el/elp/src/" +find "$GEN" -name '*.elh' -delete + +( cd "$GEN/neuron" && "$ELC" --target=c soul.el ) > "$GEN/soul.c" +BODIES=$(grep -c '^el_val_t .*) {$' "$GEN/soul.c" || true) +echo "[build-soul] amalgam $(wc -c < "$GEN/soul.c" | tr -d ' ') bytes, ${BODIES} inlined bodies" +[ "$BODIES" -ge 1200 ] || { echo "[build-soul] FAIL: only $BODIES bodies — an import was not inlined"; exit 1; } + +cc -O2 -DHAVE_CURL -rdynamic \ + -I"$RTDIR" -I"$SSL/include" -L"$SSL/lib" \ + "$GEN/soul.c" "$RTDIR/el_runtime.c" \ + -lssl -lcrypto -lcurl -lpthread -lm \ + -o "$OUT" 2> "$GEN/cc.log" || { echo "[build-soul] FAIL compile"; tail -40 "$GEN/cc.log"; exit 1; } +if grep -qE 'implicit.*(engram_|el_)' "$GEN/cc.log"; then + echo "[build-soul] FAIL: implicit declarations of runtime symbols"; grep -E 'implicit' "$GEN/cc.log" | head; exit 1; fi +echo "[build-soul] OK -> $OUT ($(wc -c < "$OUT" | tr -d ' ') bytes)" diff --git a/tools/retrieval-eval/build_gold_set.py b/tools/retrieval-eval/build_gold_set.py new file mode 100755 index 0000000..70256d1 --- /dev/null +++ b/tools/retrieval-eval/build_gold_set.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +""" +build_gold_set.py — derive the retrieval gold set FROM the corpus, and validate it. + +WHY THIS FILE EXISTS AS CODE AND NOT AS A HAND-WRITTEN JSON + A gold set nobody can audit is vibes with extra steps. Every expected answer + here is either (a) mined from the corpus by a rule this script re-runs, or + (b) hand-selected with a stated criterion that this script then CHECKS + against the corpus. Both leave a `derivation` string on every query, and the + checks are re-run on demand so the set cannot silently rot as the corpus + changes. + + Lineage: this extends the pinned-query approach from + docs/research-archive/p0-prototypes/eval_pinned_40q_20260715.py (pinned + ground-truth patterns + a --check "winnability" gate) and the per-query + relevant-id scoring from docs/research/graphrag_eval/score.py. What is new: + ids as ground truth rather than regexes alone, an ASSOCIATIVE category + derived from real graph edges, a superseded/contradicted category, and a + machine-checked no-lexical-overlap guarantee on the paraphrase category. + +THE SIX CATEGORIES, AND WHAT EACH ONE IS FOR + exact_rare a single rare word. Substring matching already wins these. + They are a REGRESSION GUARD: any change that loses them is + disqualified regardless of what else it gains. + phrase a multi-word string that exists verbatim in the corpus. + Guards multi-token queries, which the old substring matcher + handled by returning nothing. + paraphrase same meaning, ZERO shared content words with the target node. + THE CATEGORY THAT MATTERS. Mechanically unreachable by string + matching; reachable only by semantics or by association. + associative the answer is one hub-hop from an obvious starting point and + shares no words with the query. This is the case the graph is + supposed to buy: query one value, get its siblings. + nonsense must return nothing. Guards against a retriever that "improves" + recall by returning the whole graph. + superseded a fact that was later corrected. The correction must OUTRANK + the stale version — ranking, not mere presence. + +usage: + python3 build_gold_set.py [--out gold_set.json] [--check] + --check re-validates an existing gold_set.json against the corpus and exits + non-zero if any query became unwinnable or any paraphrase leaked a word. +""" +import argparse +import json +import os +import re +import sys +from collections import Counter, defaultdict + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_OUT = os.path.join(HERE, "gold_set.json") + +TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*") + +# Stopwords are deliberately generous. A paraphrase query is only interesting if +# its CONTENT words are absent from the target; "the", "is", "what" appearing in +# both proves nothing. Being generous here makes the overlap test STRICTER on +# the words that carry meaning, which is the conservative direction. +STOP = set(""" +a about above after again against all also am an and any are aren't as at be because been +before being below between both but by can can't cannot could couldn't did didn't do does +doesn't doing don't down during each few for from further had hadn't has hasn't have haven't +having he her here hers herself him himself his how i if in into is isn't it its itself just +me more most my myself no nor not of off on once only or other others ought our ours ourselves +out over own same shan't she should shouldn't so some such than that the their theirs them +themselves then there these they this those through to too under until up very was wasn't we +were weren't what when where which while who whom why will with won't would wouldn't you your +yours yourself yourselves get gets got make makes made take takes use uses used way ways thing +things does doing done keep keeps kept go goes going come comes came one two something anything +""".split()) + + +# ───────────────────────────────────────────────────────────────────────────── +# corpus helpers +# ───────────────────────────────────────────────────────────────────────────── +def load_corpus(path): + with open(path, encoding="utf-8", errors="replace") as fh: + data = json.load(fh) + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")] + edges = [e for e in data.get("edges", []) if isinstance(e, dict)] + return nodes, edges + + +def doctext(n): + return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")]) + + +def content_tokens(s): + return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2} + + +# ───────────────────────────────────────────────────────────────────────────── +# hand-authored queries. Every entry states HOW its expected answer was chosen. +# The `check` field names the validation this script runs against the corpus. +# ───────────────────────────────────────────────────────────────────────────── + +# EXACT_RARE — mined, not chosen. The rule (re-run by mine_exact_rare below): +# tokens whose document frequency across the whole corpus is 1, whose single +# containing node is a Memory/Knowledge/Belief with 300-6000 chars of content +# (so the answer is a real memory, not a 117KB whitepaper that contains every +# word in English), and whose token is plain lowercase alphabetic. The expected +# answer is that one node — it is the only node that can possibly be correct. +EXACT_RARE_SEEDS = [ + "unjailbreakable", + "engram-migrate", + "cartabandonedevent", + "pre-apprenticeship", + "inferencenodemanager", + "clear-eyed", +] + +# PHRASE — chosen by reading the corpus for phrases that (a) occur verbatim, +# (b) occur in a small enough set of nodes that "relevant" is well defined. +# Expected answers are computed here as EVERY node whose text contains the +# phrase case-insensitively — so the answer set is a fact about the corpus, not +# an opinion. Queries whose phrase matches more than PHRASE_MAX nodes are +# rejected by validation as too diffuse to score. +PHRASE_MAX = 25 +PHRASE_SEEDS = [ + ("patterns not returns", + "a verbatim correction Will issued; expected = every node containing the phrase"), + ("thirty moves", + "the canonical biographical phrase; expected = every node containing it"), + ("Grandma Lucas", + "a named person appearing verbatim in the biography/value nodes"), + ("Directed Harmonic", + "the canonical DHARMA expansion, confirmed by Will April 24 2026"), + ("Sarah Bishop", + "a named person; rare enough that the answer set is unambiguous"), + ("Directed Autonomous Runtime Modification", + "the DARMA expansion, quoted verbatim in the backlog item and its correction"), + ("zero-knowledge encrypted backup", + "the paid-tier feature name as written in the roadmap nodes"), +] + +# PARAPHRASE — hand-authored. THE SELECTION CRITERION, stated once and applied +# to all nine: pick a node whose SUBJECT is unmistakable to a reader, then write +# the query a person would actually type when they remember the subject but not +# the words. The target is then LOCKED by id, and this script enforces the hard +# property that makes the category meaningful: not one content word of the query +# appears anywhere in the target node's label, content, or tags. If a word +# leaks, validation fails and the query must be rewritten — the set cannot +# quietly degrade into a lexical query wearing a paraphrase costume. +PARAPHRASE_SEEDS = [ + ("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", + "the elderly relative who passed while he stayed away", + "target: 'Value - Do the Essential Thing While You Can', whose subject is Grandma Lucas " + "dying in Feb 2006 without Will saying goodbye. Query names the event with none of the " + "node's own vocabulary."), + ("kn-58874a74-b96f-4883-9e08-45707f4bd3ee", + "a soldier sidelined by illness who refused to quit", + "target: 'Value - Survival Is Not an Excuse to Stop', whose subject is enlisting in the " + "Marines, a severe hernia, and sepsis. Query describes the episode obliquely."), + ("kn-13f60407-7b70-4db1-964f-ea1f8196efbd", + "choosing an uncomfortable fact over a pleasant fiction", + "target: 'Value - Honesty Before Comfort'. Query states the principle in wholly " + "different words."), + ("kn-22d77abe-b3c5-42fd-afcd-dcb87d924929", + "a tight payload beats a bloated one", + "target: 'Value - Precision Over Brute Force'. Query restates the claim with no " + "shared vocabulary."), + ("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", + "if you are able and nobody is coming the job is yours", + "target: 'Value - Capability Is a Debt You Owe the Moment'. Query states the " + "obligation without the node's terms."), + ("kn-0bb4f021-56de-4947-a35b-a37209e7ba21", + "learning is the wealth creditors cannot seize", + "target: 'Value - Knowledge Survives When Nothing Else Does', whose subject is the " + "library following Will across 30+ moves."), + ("kn-5de5a9ac-fd15-45ab-bf18-77566781cf40", + "reliability proven by track record not assertion", + "target: 'Value - Earned Trust' ('Trust is demonstrated, not declared')."), + ("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", + "boundaries that enable instead of confine", + "target: 'Value - Constraints as Freedom'. Query is a restatement of the same claim."), + ("kn-78db5396-3dbc-4481-bfc7-e4e1422feb1c", + "what shifts tells you where to cut a system apart", + "target: 'Value - Change Is the Signal', the value VBD is built on."), + ("kn-f230b362-b201-4402-9833-4160c89ab3d4", + "a mind that compounds instead of resetting each day", + "target: 'Value - The System Must Accumulate'. Query is the accumulation claim in " + "different vocabulary."), + ("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", + "loved for the unedited self and not the polished exterior", + "target: 'Value - Being Seen Is Rarer Than Being Known', whose subject is Sarah Bishop " + "as the first person Will did not perform for."), + ("kn-e0423482-cfa5-4796-8689-8495c93b66bc", + "cheerfulness you arrive at instead of assuming", + "target: 'Value - Hope Is a Conclusion'. Query restates 'a conclusion, not a premise'."), + ("kn-6061318f-046b-4935-907d-8eafdce14930", + "a childhood offering no solid foundation to inherit", + "target: 'Value - Structure Is Not Inherited', whose subject is thirty moves between " + "two parents' collapses."), +] + +# ASSOCIATIVE — derived from real edges, not authored. The construction: +# every value node hangs off the 'Self - Values (grounded)' hub by an `identity` +# edge. For a chosen value node V, the query is built from V's own distinctive +# vocabulary; the expected answers are V's SIBLINGS on that hub. A sibling +# shares no query words with the query by construction (validated below), so the +# only path from the query to a sibling is: lexical seed on V -> hub -> sibling. +# That is a two-hop traversal and nothing else can produce it. +VALUES_HUB = "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" +ASSOCIATIVE_SEEDS = [ + ("kn-a99cefe3-5e83-4050-98d8-6c69f57c7c71", "Grandma Lucas stroke February 2006 goodbye window"), + ("kn-58874a74-b96f-4883-9e08-45707f4bd3ee", "Marines hernia sepsis medical ward"), + ("kn-db9f141b-dbe3-4037-92e0-4bb9be0e5e6e", "Sarah Bishop Dyer trailer performance"), + ("kn-a5b3d0ac-f6a1-49a4-aebb-b8b4cd67fe83", "Swarm Architecture containment lateral worker"), + ("kn-e0423482-cfa5-4796-8689-8495c93b66bc", "hope won inside the narrative preface"), + ("kn-eb1b9e18-3dc6-4b9b-9cc6-86e0ae6b6be8", "man of the house six years old expectation"), +] + +# NONSENSE — must return nothing. Strings chosen to be lexically impossible: +# validation asserts each appears in ZERO corpus nodes as a substring and that +# none of its tokens appears anywhere either (so not even a partial seed exists). +NONSENSE_SEEDS = [ + "zqxjvw plimforth grebulon", + "flarnbistle quommetry", + "xxqzzt vurblenacht throom", +] + +# SUPERSEDED — a fact that was corrected. Chosen by searching the corpus for +# explicit correction language and keeping pairs where BOTH the stale statement +# and its correction exist as separate nodes. Scored on RANKING: the correction +# must appear, and must appear above the stale node. Ids are locked here and +# validated to exist and to match their stated role. +SUPERSEDED_SEEDS = [ + # (query, correct_id, stale_id, derivation) +] + + +# ───────────────────────────────────────────────────────────────────────────── +# mining +# ───────────────────────────────────────────────────────────────────────────── +def mine_exact_rare(nodes, byid, seeds): + """Re-derive: confirm each seed token still has df==1 and name its node.""" + tok = re.compile(r"[A-Za-z][A-Za-z0-9\-]{4,}") + want = set(seeds) + df = Counter() + post = defaultdict(set) + for n in nodes: + for t in {w.lower() for w in tok.findall(doctext(n))}: + if t in want: + df[t] += 1 + post[t].add(n["id"]) + out = [] + for s in seeds: + ids = sorted(post.get(s, ())) + out.append((s, ids, df.get(s, 0))) + return out + + +def phrase_matches(nodes, phrase): + p = phrase.lower() + return sorted(n["id"] for n in nodes if p in doctext(n).lower()) + + +def hub_siblings(edges, hub, relation="identity"): + sibs = [] + for e in edges: + if e.get("from_id") == hub and e.get("relation") == relation: + sibs.append(e["to_id"]) + elif e.get("to_id") == hub and e.get("relation") == relation: + sibs.append(e["from_id"]) + return list(dict.fromkeys(sibs)) + + +def find_superseded_pairs(nodes, byid): + """Locked pairs, each verified here to exist and to carry its stated marker. + + Chosen by scanning the corpus for explicit correction language + (CORRECTION/SUPERSEDES/re-corrected/no longer/RECONCILED) and keeping only + cases where the STALE claim also survives as its own node — a supersession + with nothing to outrank is not a ranking test. + """ + pairs = [] + txt = {n["id"]: doctext(n) for n in nodes} + + def find_one(pattern, exclude=()): + rx = re.compile(pattern) + return [n["id"] for n in nodes + if n["id"] not in exclude + and rx.search(txt[n["id"]]) + and 150 < len(str(n.get("content") or "")) < 12000 + and n.get("node_type") in ("Memory", "Knowledge", "Belief", "BacklogItem")] + + # Each entry: (query, correction-pattern, stale-pattern, why). + # The stale side is searched with the correction hits EXCLUDED, because most + # correction memories quote the claim they are killing — without the + # exclusion the "stale" node resolves to the correction itself and the pair + # collapses into a no-op. A pair is only emitted if both sides resolve to + # DIFFERENT surviving nodes; otherwise it is dropped and reported. + SPECS = [ + ("is the self-improvement architecture called DARMA or DHARMA", + r'(?i)CORRECTION:.{0,90}DHARMA .{0,12}not DARMA', + r'(?i)\bDARMA\b', + "correction node is Will's confirmation that the H is intentional (DHARMA, not DARMA); " + "the stale node is the surviving backlog item still titled 'Implement DARMA'."), + + ("how many provisional patents does Will actually have", + r'(?i)EXACTLY 6 (fully-specced )?provisional', + r'(?i)(MY ARCHITECTURE = 12 filed patents|\b12 filed patents\b)', + "correction node is the 2026-06-17 confabulation flag establishing EXACTLY 6 provisionals; " + "the stale node is the surviving memory that asserts 12 filed patents."), + + ("is MCP still the live integration layer", + r'(?i)MCP RETIRED', + r'(?i)MCP server live at', + "correction node is the 'CGI ARCHITECTURE - THREE LAYERS, MCP RETIRED' decision of " + "April 30 2026; the stale node still records the MCP server as live."), + + ("what does the patterns-not-returns directive mean", + r'(?i)CORRECTION:.{0,80}patterns not returns', + r'(?i)established returns', + "correction node is Will's 'patterns not returns' correction; the stale node is a " + "surviving node carrying the misread 'established returns' directive."), + + ("was the earlier identity-bug finding correct", + r'(?i)SUPERSEDES the earlier .critical identity bug', + r'(?i)critical identity bug', + "correction node explicitly supersedes the 'critical identity bug' finding; the stale " + "node is the surviving original finding."), + + ("does Neuron have recursive self-improvement", + r'(?i)twice answered .Neuron has no recursive self-improvement', + r'(?i)no recursive self-improvement', + "correction node records the June-29 finding that the CGI provisional IS the " + "recursive-self-improvement mechanism; the stale node is the surviving denial."), + ] + + for query, cpat, spat, why in SPECS: + corr = find_one(cpat) + if not corr: + continue + stale = find_one(spat, exclude=set(corr)) + if not stale: + continue + pairs.append((query, corr[0], stale[0], why)) + + return pairs + + +# ───────────────────────────────────────────────────────────────────────────── +# build +# ───────────────────────────────────────────────────────────────────────────── +def build(nodes, edges): + byid = {n["id"]: n for n in nodes} + tokset = {n["id"]: content_tokens(doctext(n)) for n in nodes} + queries = [] + problems = [] + qn = [0] + + def add(cat, query, relevant, derivation, **extra): + qn[0] += 1 + q = { + "id": f"q{qn[0]:02d}", + "category": cat, + "query": query, + "relevant": sorted(relevant), + "derivation": derivation, + } + q.update(extra) + queries.append(q) + return q + + # --- exact_rare --------------------------------------------------------- + for tokname, ids, df in mine_exact_rare(nodes, byid, EXACT_RARE_SEEDS): + if df != 1 or len(ids) != 1: + problems.append(f"exact_rare '{tokname}': df={df}, ids={len(ids)} (expected df=1)") + continue + lab = (byid[ids[0]].get("label") or "")[:60] + add("exact_rare", tokname, ids, + f"MINED: token '{tokname}' has document frequency 1 over all {len(nodes)} corpus nodes " + f"(re-verified at build time). Its single containing node is {ids[0]} " + f"('{lab}'), which is therefore the only possible correct answer.") + + # --- phrase ------------------------------------------------------------- + for phrase, why in PHRASE_SEEDS: + ids = phrase_matches(nodes, phrase) + if not ids: + problems.append(f"phrase '{phrase}': 0 corpus matches — unwinnable") + continue + if len(ids) > PHRASE_MAX: + problems.append(f"phrase '{phrase}': {len(ids)} matches > {PHRASE_MAX} — too diffuse") + continue + add("phrase", phrase, ids, + f"MINED: {why}. Case-insensitive verbatim substring scan over label+content+tags at " + f"build time returns exactly {len(ids)} node(s); that set IS the answer key.") + + # --- paraphrase --------------------------------------------------------- + for target, query, why in PARAPHRASE_SEEDS: + if target not in byid: + problems.append(f"paraphrase target {target} not in corpus") + continue + qt = content_tokens(query) + leak = sorted(qt & tokset[target]) + if leak: + problems.append(f"paraphrase '{query}': leaks {leak} into target {target}") + continue + add("paraphrase", query, [target], + f"HAND-SELECTED with criterion: {why} VERIFIED at build time: of the {len(qt)} content " + f"words in the query, ZERO appear anywhere in the target's label, content, or tags — so " + f"no string-matching retriever can reach this answer.", + zero_overlap_verified=True, query_content_words=sorted(qt)) + + # --- associative -------------------------------------------------------- + sibs = hub_siblings(edges, VALUES_HUB) + if len(sibs) < 5: + problems.append(f"associative: values hub {VALUES_HUB} has only {len(sibs)} siblings") + for src, query in ASSOCIATIVE_SEEDS: + if src not in byid or src not in sibs: + problems.append(f"associative source {src} not a sibling on {VALUES_HUB}") + continue + qt = content_tokens(query) + others = [s for s in sibs if s != src and s in byid] + # A sibling only counts as a legitimate expected answer if the query + # cannot reach it lexically. Drop any sibling that shares a content word. + clean = [s for s in others if not (qt & tokset[s])] + dropped = len(others) - len(clean) + if len(clean) < 5: + problems.append(f"associative '{query}': only {len(clean)} lexically-unreachable siblings") + continue + add("associative", query, clean, + f"DERIVED FROM EDGES: the query is built from the distinctive vocabulary of {src} " + f"('{(byid[src].get('label') or '')[:48]}'), which hangs off the values hub {VALUES_HUB} " + f"by an `identity` edge. Expected answers are that node's SIBLINGS on the same hub " + f"({len(clean)} of {len(others)}; {dropped} dropped because they shared a query word and " + f"so were lexically reachable). Every remaining sibling shares ZERO content words with " + f"the query — the only route from query to answer is seed({src}) -> hub -> sibling, a " + f"two-hop traversal.", + associative_source=src, hub=VALUES_HUB, siblings_dropped_for_overlap=dropped) + + # --- nonsense ----------------------------------------------------------- + all_tokens = set() + for n in nodes: + all_tokens |= {t for t in TOKEN.findall(doctext(n).lower())} + for s in NONSENSE_SEEDS: + present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens) + if present: + problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus") + continue + add("nonsense", s, [], + f"CONTROL: verified at build time that none of this string's tokens occurs anywhere in " + f"the corpus. Correct behaviour is to return NOTHING; any result is a false positive.", + expect_empty=True) + + # --- superseded --------------------------------------------------------- + for query, correct, stale, why in find_superseded_pairs(nodes, byid): + if correct not in byid or stale not in byid: + problems.append(f"superseded '{query}': id missing from corpus") + continue + add("superseded", query, [correct], + f"DERIVED: {why} Scored on RANKING, not presence: the corrected node {correct} must be " + f"returned AND must rank above the stale node {stale}.", + must_outrank=[correct, stale], + stale_id=stale, + correct_label=(byid[correct].get("label") or "")[:70], + stale_label=(byid[stale].get("label") or "")[:70]) + + return queries, problems + + +def summarize(queries): + c = Counter(q["category"] for q in queries) + return ", ".join(f"{k}={c[k]}" for k in + ("exact_rare", "phrase", "paraphrase", "associative", "nonsense", "superseded") + if c[k]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("snapshot") + ap.add_argument("--out", default=DEFAULT_OUT) + ap.add_argument("--check", action="store_true", + help="validate only; do not write. Non-zero exit if anything is unwinnable.") + args = ap.parse_args() + + nodes, edges = load_corpus(args.snapshot) + print(f"corpus: {len(nodes)} nodes, {len(edges)} edges ({os.path.basename(args.snapshot)})") + queries, problems = build(nodes, edges) + + print(f"gold set: {len(queries)} queries [{summarize(queries)}]") + if problems: + print(f"\n{len(problems)} PROBLEM(S) — these queries were REJECTED, not silently kept:") + for p in problems: + print(" -", p) + + if args.check: + sys.exit(1 if problems else 0) + + doc = { + "corpus": os.path.abspath(args.snapshot), + "corpus_nodes": len(nodes), + "corpus_edges": len(edges), + "note": ("Every query carries a `derivation` recording how its expected answer was chosen. " + "Re-run with --check to re-validate the whole set against the corpus."), + "queries": queries, + } + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=1, ensure_ascii=False) + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-eval/ceiling.py b/tools/retrieval-eval/ceiling.py new file mode 100644 index 0000000..4078e39 --- /dev/null +++ b/tools/retrieval-eval/ceiling.py @@ -0,0 +1,43 @@ +import json,sys,pickle,numpy as np,itertools +sys.path.insert(0,'.') +from policy2 import legs3,outcome,G,NODES,merge +# cache per-query leg id-lists, floored and unfloored +cache={} +for q in G['queries']: + Lf,Sf,Af=legs3(q['query']) + Lu,Su,Au=legs3(q['query'],unfloor=True) + cache[q['id']]=dict(L=Lf,Sf=Sf,A=Af,Su=Su,Au=Au) +pickle.dump(cache,open('ceil.pkl','wb')) +def mrg(pattern,L,S,A,lim=10): + out=[];p={'L':0,'S':0,'A':0};src={'L':L,'S':S,'A':A} + i=0 + while len(out)=lim: return out + if not prog: break + return out +def ev(pattern,unfl): + res={} + for q in G['queries']: + c=cache[q['id']] + S=c['Su'] if unfl else c['Sf'] + ids=[NODES[i]['id'] for i in mrg(pattern,c['L'],S,c['A'],10)] + res[q['id']]=outcome(q,ids) + return res +base=ev('LSA',False) +print("baseline",sum(base.values())) +best=[] +pats=['LSA','LAS','SLA','ALS','SAL','ASL','LSSA','LSASA','LSAA','LSSAA','LSAS','SSLA','LLSA','SALSA','LSAAS'] +for unfl in (False,True): + for p in pats: + r=ev(p,unfl) + g=sorted(k for k in base if r[k] and not base[k]);l=sorted(k for k in base if base[k] and not r[k]) + best.append((len(g)-len(l),p,unfl,g,l)) +best.sort(reverse=True) +for n,p,u,g,l in best[:10]: + print("net=%+d pat=%-6s unfloor=%s gains=%s losses=%s"%(n,p,u,g,l)) diff --git a/tools/retrieval-eval/cmp-nogate.json b/tools/retrieval-eval/cmp-nogate.json new file mode 100644 index 0000000..e680068 --- /dev/null +++ b/tools/retrieval-eval/cmp-nogate.json @@ -0,0 +1,146 @@ +{ + "baseline": "bm25lex", + "candidate": "wsclaim24", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q14", + "q25" + ], + "broken_by_candidate": [ + "q15", + "q28", + "q33", + "q34" + ], + "discordant": 6, + "net_queries": -2, + "mcnemar_exact_p": 0.6875, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5768475572047, + "recall@10": 0.6563414759843332, + "precision@5": 0.19428571428571437, + "mrr@10": 0.5026530612244898, + "nonsense_clean": "0/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 524.8, + "latency_ms_p95": 738.7, + "latency_ms_max": 755.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 0, + "avg_false_positives": 10.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/cmp-splitfix.json b/tools/retrieval-eval/cmp-splitfix.json new file mode 100644 index 0000000..9375709 --- /dev/null +++ b/tools/retrieval-eval/cmp-splitfix.json @@ -0,0 +1,149 @@ +{ + "baseline": "unfloor-clean", + "candidate": "splitfix", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q15", + "q28", + "q60" + ], + "broken_by_candidate": [], + "discordant": 3, + "net_queries": 3, + "mcnemar_exact_p": 0.25, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5846153846153846, + "recall@5": 0.48102442429365505, + "recall@10": 0.5692415490492414, + "precision@5": 0.14153846153846153, + "mrr@10": 0.3351709401709402, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 646.9, + "latency_ms_p95": 1028.5, + "latency_ms_max": 1197.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.15967365967365968, + "mrr@10": 0.24166666666666667 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.43333333333333335, + "mrr@10": 0.12120370370370372 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.7692307692307693, + "recall@5": 0.7692307692307693, + "recall@10": 0.8461538461538461, + "mrr@10": 0.33269230769230773 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/compare.py b/tools/retrieval-eval/compare.py new file mode 100755 index 0000000..83ac4f8 --- /dev/null +++ b/tools/retrieval-eval/compare.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +""" +compare.py — diff two run_eval.py result files, WITH a noise threshold. + +WHY THE STATISTICS ARE NOT OPTIONAL + With ~35 scored queries, one query is ~2.9 percentage points. A harness that + reports "hit@5 improved 2.9%" without saying that is one query is a harness + that will approve noise. So this file refuses to call anything an + improvement on the strength of the headline number alone. It reports: + + 1. The DISCORDANT PAIRS. Two configurations scored on the same queries are + paired data, so the only queries carrying information are the ones + where they disagree: b = fixed by B, c = broken by B. Queries both got + right, or both got wrong, tell you nothing about which is better. + + 2. McNEMAR'S EXACT TEST on (b, c). Under the null "the change is a coin + flip", the discordant outcomes are Binomial(b+c, 0.5). The two-sided + exact p-value is computed here with no scipy dependency. + + 3. The MINIMUM DETECTABLE SWING for this gold set: the smallest number of + net-changed queries that would reach p < 0.05 if every discordant pair + fell the same way. Anything smaller is inside the noise band, and the + verdict line says so in those words. + + Repeat-run variance is the other half of honesty. Spreading activation is a + stateful read (it reinforces what it touches), so identical inputs need not + give identical outputs. Pass --repeats to fold several runs of the same + config into an observed variance band; a delta inside that band is not real + either, however good its p-value looks. + +usage: + python3 compare.py --baseline results-main.json --candidate results-act.json + python3 compare.py --baseline a.json --candidate b.json \ + --repeats-baseline a2.json a3.json --repeats-candidate b2.json b3.json +""" +import argparse +import json +from math import comb + + +def binom_two_sided(b, c): + """Two-sided exact binomial p for b successes in n=b+c at p=0.5.""" + n = b + c + if n == 0: + return 1.0 + k = min(b, c) + tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n) + return min(1.0, 2 * tail) + + +def min_detectable_swing(n_scored, alpha=0.05): + """Smallest all-one-way discordant count reaching p < alpha. + + If every query that changes changes in the same direction, the p-value is + 2 * 0.5**n. Solve for the smallest n where that drops under alpha. This is + the FLOOR: any real change will have some discordance both ways, so the true + requirement is larger. Reporting the floor is the conservative move — it is + the most generous threshold we would ever accept. + """ + n = 1 + while n <= n_scored: + if 2 * (0.5 ** n) < alpha: + return n + n += 1 + return n_scored + + +def load(path): + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +def row_map(doc): + return {r["id"]: r for r in doc["rows"]} + + +def outcome(r): + """Binary per-query outcome used for the paired test. + + hit@5 for scored queries; 'returned nothing' for the nonsense controls; + 'correction outranks the stale node' for the superseded queries. One number + per query, so every query votes exactly once. + """ + if "clean" in r: + return 1.0 if r["clean"] else 0.0 + if "outranks" in r: + return 1.0 if r["outranks"] else 0.0 + return r.get("hit@5") or 0.0 + + +def band(values): + return (min(values), max(values)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--baseline", required=True) + ap.add_argument("--candidate", required=True) + ap.add_argument("--repeats-baseline", nargs="*", default=[]) + ap.add_argument("--repeats-candidate", nargs="*", default=[]) + ap.add_argument("--out", default=None) + args = ap.parse_args() + + A, B = load(args.baseline), load(args.candidate) + ra, rb = row_map(A), row_map(B) + ids = [q for q in ra if q in rb] + n = len(ids) + + aa, ab = A["aggregate"], B["aggregate"] + print(f"baseline {A['label']:14} soul={A['soul_md5'][:12]} {n} shared queries") + print(f"candidate {B['label']:14} soul={B['soul_md5'][:12]}") + print(f"corpus {A['corpus_nodes']} nodes / {A['corpus_edges']} edges " + f"(identical copy for both runs)\n") + + metrics = [("hit@5", 1), ("recall@5", 1), ("recall@10", 1), + ("precision@5", 1), ("mrr@10", 0)] + print(f" {'metric':14} {'baseline':>10} {'candidate':>10} {'delta':>10}") + for m, as_pct in metrics: + x, y = aa[m], ab[m] + if as_pct: + print(f" {m:14} {100*x:>9.1f}% {100*y:>9.1f}% {100*(y-x):>+9.1f}pp") + else: + print(f" {m:14} {x:>10.3f} {y:>10.3f} {y-x:>+10.3f}") + for m in ("latency_ms_p50", "latency_ms_p95"): + x, y = aa[m], ab[m] + ratio = f"{y/x:.2f}x" if x else "n/a" + print(f" {m:14} {x:>9.0f}ms {y:>9.0f}ms {ratio:>10}") + print(f" {'nonsense':14} {aa['nonsense_clean']:>10} {ab['nonsense_clean']:>10}") + print(f" {'outranks':14} {aa['superseded_outranks']:>10} {ab['superseded_outranks']:>10}") + + print(f"\n {'category':14} {'n':>3} {'base hit@5':>11} {'cand hit@5':>11} {'delta':>9}") + for c in sorted(set(aa["by_category"]) & set(ab["by_category"])): + ea, eb = aa["by_category"][c], ab["by_category"][c] + if c == "nonsense": + print(f" {c:14} {ea['n']:>3} {'clean ' + str(ea['clean']):>11} " + f"{'clean ' + str(eb['clean']):>11}") + else: + print(f" {c:14} {ea['n']:>3} {100*ea['hit@5']:>10.1f}% {100*eb['hit@5']:>10.1f}% " + f"{100*(eb['hit@5']-ea['hit@5']):>+8.1f}pp") + + # ---- paired significance ------------------------------------------------- + fixed, broken = [], [] + for q in ids: + oa, ob = outcome(ra[q]), outcome(rb[q]) + if ob > oa: + fixed.append(q) + elif ob < oa: + broken.append(q) + b, c = len(fixed), len(broken) + p = binom_two_sided(b, c) + mds = min_detectable_swing(n) + + print(f"\n== paired comparison over {n} queries ==") + print(f" fixed by candidate : {b} {[ra[q]['category'] + ':' + q for q in fixed]}") + print(f" broken by candidate: {c} {[ra[q]['category'] + ':' + q for q in broken]}") + print(f" discordant pairs : {b + c} net {b - c:+d} queries") + print(f" McNemar exact p : {p:.4f}") + print(f" noise threshold : a difference needs at least {mds} queries moving the " + f"same way to clear p<0.05 on this {n}-query set") + + # ---- repeat-run variance ------------------------------------------------- + var = {} + for name, paths, first in (("baseline", args.repeats_baseline, A), + ("candidate", args.repeats_candidate, B)): + docs = [first] + [load(p) for p in paths] + if len(docs) > 1: + hits = [d["aggregate"]["hit@5"] for d in docs] + lo, hi = band(hits) + spread_q = round((hi - lo) * first["aggregate"]["n_scored"]) + var[name] = {"runs": len(docs), "hit@5_min": lo, "hit@5_max": hi, + "spread_queries": spread_q} + print(f" {name} repeat runs ({len(docs)}): hit@5 {100*lo:.1f}%..{100*hi:.1f}% " + f"= {spread_q} query of run-to-run drift") + + drift = max([v["spread_queries"] for v in var.values()], default=0) + floor = max(mds, drift + 1) + + print("\n== VERDICT ==") + net = b - c + if abs(net) < floor: + print(f" NO MEASURABLE DIFFERENCE. Net {net:+d} queries is inside the noise band " + f"(needs |net| >= {floor}: {mds} for significance, {drift} observed run-to-run drift).") + elif net > 0: + print(f" CANDIDATE BETTER by {net} queries (p={p:.4f}), outside the noise band " + f"(>= {floor}).") + else: + print(f" CANDIDATE WORSE by {abs(net)} queries (p={p:.4f}), outside the noise band " + f"(>= {floor}).") + + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + json.dump({ + "baseline": A["label"], "candidate": B["label"], + "n_shared_queries": n, + "fixed_by_candidate": fixed, "broken_by_candidate": broken, + "discordant": b + c, "net_queries": net, + "mcnemar_exact_p": p, + "min_detectable_swing_queries": mds, + "observed_run_to_run_drift_queries": drift, + "noise_floor_queries": floor, + "verdict": ("no measurable difference" if abs(net) < floor + else ("candidate better" if net > 0 else "candidate worse")), + "baseline_aggregate": aa, "candidate_aggregate": ab, + "repeat_variance": var, + }, fh, indent=1) + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-eval/comparison-assoc-vs-semseed.json b/tools/retrieval-eval/comparison-assoc-vs-semseed.json new file mode 100644 index 0000000..7fe83ec --- /dev/null +++ b/tools/retrieval-eval/comparison-assoc-vs-semseed.json @@ -0,0 +1,150 @@ +{ + "baseline": "assoc-leg", + "candidate": "semseed", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q18", + "q19", + "q22" + ], + "broken_by_candidate": [ + "q11" + ], + "discordant": 4, + "net_queries": 2, + "mcnemar_exact_p": 0.625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1228.5, + "latency_ms_p95": 1681.8, + "latency_ms_max": 1718.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.6285714285714286, + "hit@5_max": 0.6285714285714286, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json b/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json new file mode 100644 index 0000000..661e2db --- /dev/null +++ b/tools/retrieval-eval/comparison-bm25lex-vs-wordstart.json @@ -0,0 +1,146 @@ +{ + "baseline": "bm25lex", + "candidate": "wordstart", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q35" + ], + "broken_by_candidate": [], + "discordant": 1, + "net_queries": 1, + "mcnemar_exact_p": 1.0, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "3/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 542.6, + "latency_ms_p95": 741.3, + "latency_ms_max": 758.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 3, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-hybrid-vs-assoc.json b/tools/retrieval-eval/comparison-hybrid-vs-assoc.json new file mode 100644 index 0000000..210e59e --- /dev/null +++ b/tools/retrieval-eval/comparison-hybrid-vs-assoc.json @@ -0,0 +1,143 @@ +{ + "baseline": "hybrid-semantic", + "candidate": "assoc-leg", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q27", + "q28", + "q29", + "q31" + ], + "broken_by_candidate": [], + "discordant": 4, + "net_queries": 4, + "mcnemar_exact_p": 0.125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6285714285714286, + "recall@5": 0.45309194773480493, + "recall@10": 0.5405733155733157, + "precision@5": 0.17714285714285719, + "mrr@10": 0.42650793650793645, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1228.5, + "latency_ms_p95": 1681.8, + "latency_ms_max": 1718.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657342, + "recall@10": 0.24825174825174826, + "mrr@10": 0.22777777777777777 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4882369614512472, + "recall@10": 0.6329365079365079, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-hybrid-vs-semseed.json b/tools/retrieval-eval/comparison-hybrid-vs-semseed.json new file mode 100644 index 0000000..a49a008 --- /dev/null +++ b/tools/retrieval-eval/comparison-hybrid-vs-semseed.json @@ -0,0 +1,154 @@ +{ + "baseline": "hybrid-semantic", + "candidate": "semseed", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q18", + "q19", + "q22", + "q27", + "q28", + "q29", + "q31" + ], + "broken_by_candidate": [ + "q11" + ], + "discordant": 8, + "net_queries": 6, + "mcnemar_exact_p": 0.0703125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json b/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json new file mode 100644 index 0000000..6f13286 --- /dev/null +++ b/tools/retrieval-eval/comparison-main-vs-hybrid-semantic.json @@ -0,0 +1,145 @@ +{ + "baseline": "baseline-embcorpus", + "candidate": "hybrid-semantic", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q15", + "q16", + "q20", + "q21", + "q26", + "q37" + ], + "broken_by_candidate": [], + "discordant": 6, + "net_queries": 6, + "mcnemar_exact_p": 0.03125, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.34285714285714286, + "recall@5": 0.26947278911564626, + "recall@10": 0.3333333333333333, + "precision@5": 0.12000000000000001, + "mrr@10": 0.2943197278911564, + "nonsense_clean": "2/3", + "superseded_outranks": "1/3", + "latency_ms_p50": 1145.9, + "latency_ms_p95": 1574.3, + "latency_ms_max": 1634.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.5142857142857142, + "recall@5": 0.4409013605442177, + "recall@10": 0.5047619047619047, + "precision@5": 0.15428571428571433, + "mrr@10": 0.38746031746031745, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1219.7, + "latency_ms_p95": 1667.1, + "latency_ms_max": 1720.2, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.38461538461538464, + "recall@5": 0.38461538461538464, + "recall@10": 0.38461538461538464, + "mrr@10": 0.17307692307692307 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.6634920634920636 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.2222222222222222, + "outranks": 2 + } + } + }, + "repeat_variance": { + "candidate": { + "runs": 2, + "hit@5_min": 0.5142857142857142, + "hit@5_max": 0.5142857142857142, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-main-vs-recall-through-activation.json b/tools/retrieval-eval/comparison-main-vs-recall-through-activation.json new file mode 100644 index 0000000..72abd0e --- /dev/null +++ b/tools/retrieval-eval/comparison-main-vs-recall-through-activation.json @@ -0,0 +1,150 @@ +{ + "baseline": "main-r1", + "candidate": "act-r1", + "n_shared_queries": 38, + "fixed_by_candidate": [], + "broken_by_candidate": [ + "q07", + "q11", + "q12", + "q13", + "q36" + ], + "discordant": 5, + "net_queries": -5, + "mcnemar_exact_p": 0.0625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 1, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.34285714285714286, + "recall@5": 0.26947278911564626, + "recall@10": 0.3333333333333333, + "precision@5": 0.12000000000000001, + "mrr@10": 0.2943197278911564, + "nonsense_clean": "2/3", + "superseded_outranks": "1/3", + "latency_ms_p50": 1140.4, + "latency_ms_p95": 1584.1, + "latency_ms_max": 1627.6, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.22857142857142856, + "recall@5": 0.19087301587301586, + "recall@10": 0.24277210884353742, + "precision@5": 0.07428571428571429, + "mrr@10": 0.24154195011337865, + "nonsense_clean": "2/3", + "superseded_outranks": "0/3", + "latency_ms_p50": 3208.8, + "latency_ms_p95": 4851.9, + "latency_ms_max": 5078.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 2.6666666666666665 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.2857142857142857, + "recall@5": 0.09722222222222222, + "recall@10": 0.3567176870748299, + "mrr@10": 0.3505668934240363 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0, + "outranks": 0 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 3, + "hit@5_min": 0.34285714285714286, + "hit@5_max": 0.34285714285714286, + "spread_queries": 0 + }, + "candidate": { + "runs": 3, + "hit@5_min": 0.22857142857142856, + "hit@5_max": 0.2571428571428571, + "spread_queries": 1 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-main-vs-stack-ext.json b/tools/retrieval-eval/comparison-main-vs-stack-ext.json new file mode 100644 index 0000000..4a33bf6 --- /dev/null +++ b/tools/retrieval-eval/comparison-main-vs-stack-ext.json @@ -0,0 +1,166 @@ +{ + "baseline": "main-ext", + "candidate": "stack-ext", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q10", + "q15", + "q16", + "q18", + "q19", + "q20", + "q21", + "q22", + "q26", + "q27", + "q28", + "q29", + "q31", + "q35", + "q37", + "q40", + "q44", + "q48", + "q49", + "q50" + ], + "broken_by_candidate": [], + "discordant": 20, + "net_queries": 20, + "mcnemar_exact_p": 1.9073486328125e-06, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "candidate better", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.18461538461538463, + "recall@5": 0.14510073260073258, + "recall@10": 0.1794871794871795, + "precision@5": 0.06461538461538462, + "mrr@10": 0.15847985347985344, + "nonsense_clean": "9/10", + "superseded_outranks": "1/3", + "latency_ms_p50": 1380.2, + "latency_ms_p95": 2293.8, + "latency_ms_max": 2879.1, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "nonsense": { + "n": 10, + "clean": 9, + "avg_false_positives": 1.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.0, + "mrr@10": 0.0 + }, + "phrase": { + "n": 7, + "hit@5": 0.8571428571428571, + "recall@5": 0.4902210884353741, + "recall@10": 0.6666666666666666, + "mrr@10": 0.5965986394557822 + }, + "superseded": { + "n": 3, + "hit@5": 0.0, + "recall@5": 0.0, + "recall@10": 0.3333333333333333, + "mrr@10": 0.041666666666666664, + "outranks": 1 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.47692307692307695, + "recall@5": 0.3750415183107491, + "recall@10": 0.45561340369032677, + "precision@5": 0.12307692307692313, + "mrr@10": 0.3055555555555555, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 640.9, + "latency_ms_p95": 1011.1, + "latency_ms_max": 1190.3, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.16666666666666666, + "recall@5": 0.16666666666666666, + "recall@10": 0.26666666666666666, + "mrr@10": 0.0762037037037037 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json b/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json new file mode 100644 index 0000000..7f43706 --- /dev/null +++ b/tools/retrieval-eval/comparison-semseed-vs-bm25lex.json @@ -0,0 +1,147 @@ +{ + "baseline": "semseed", + "candidate": "bm25lex", + "n_shared_queries": 38, + "fixed_by_candidate": [ + "q10", + "q11" + ], + "broken_by_candidate": [], + "discordant": 2, + "net_queries": 2, + "mcnemar_exact_p": 0.5, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.6857142857142857, + "recall@5": 0.5213459159887731, + "recall@10": 0.6027048348476919, + "precision@5": 0.18285714285714294, + "mrr@10": 0.4608730158730158, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1227.1, + "latency_ms_p95": 1692.6, + "latency_ms_max": 1710.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.07342657342657344, + "recall@10": 0.24825174825174823, + "mrr@10": 0.20833333333333334 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 0.7142857142857143, + "recall@5": 0.40093537414965985, + "recall@10": 0.5150226757369615, + "mrr@10": 0.6507936507936508 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 38, + "n_scored": 35, + "hit@5": 0.7428571428571429, + "recall@5": 0.5536485340056769, + "recall@10": 0.6175677497106068, + "precision@5": 0.20000000000000007, + "mrr@10": 0.5021428571428571, + "nonsense_clean": "2/3", + "superseded_outranks": "2/3", + "latency_ms_p50": 1184.4, + "latency_ms_p95": 1620.0, + "latency_ms_max": 1655.4, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "nonsense": { + "n": 3, + "clean": 2, + "avg_false_positives": 3.3333333333333335 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": { + "baseline": { + "runs": 2, + "hit@5_min": 0.6857142857142857, + "hit@5_max": 0.6857142857142857, + "spread_queries": 0 + }, + "candidate": { + "runs": 2, + "hit@5_min": 0.7428571428571429, + "hit@5_max": 0.7428571428571429, + "spread_queries": 0 + } + } +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-stack-vs-unfloor.json b/tools/retrieval-eval/comparison-stack-vs-unfloor.json new file mode 100644 index 0000000..9a459c7 --- /dev/null +++ b/tools/retrieval-eval/comparison-stack-vs-unfloor.json @@ -0,0 +1,155 @@ +{ + "baseline": "stack-ext", + "candidate": "unfloor-clean", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q14", + "q25", + "q43", + "q52", + "q63", + "q67" + ], + "broken_by_candidate": [ + "q15", + "q28" + ], + "discordant": 8, + "net_queries": 4, + "mcnemar_exact_p": 0.2890625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.47692307692307695, + "recall@5": 0.3750415183107491, + "recall@10": 0.45561340369032677, + "precision@5": 0.12307692307692313, + "mrr@10": 0.3055555555555555, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 640.9, + "latency_ms_p95": 1011.1, + "latency_ms_max": 1190.3, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.6666666666666666, + "recall@5": 0.08857808857808858, + "recall@10": 0.23310023310023312, + "mrr@10": 0.25 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.16666666666666666, + "recall@5": 0.16666666666666666, + "recall@10": 0.26666666666666666, + "mrr@10": 0.0762037037037037 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6153846153846154, + "recall@5": 0.6153846153846154, + "recall@10": 0.6153846153846154, + "mrr@10": 0.2846153846153846 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5494614512471656, + "recall@10": 0.6023242630385487, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/comparison-unfloor-vs-semsub.json b/tools/retrieval-eval/comparison-unfloor-vs-semsub.json new file mode 100644 index 0000000..eaec117 --- /dev/null +++ b/tools/retrieval-eval/comparison-unfloor-vs-semsub.json @@ -0,0 +1,158 @@ +{ + "baseline": "unfloor-clean", + "candidate": "semsub", + "n_shared_queries": 75, + "fixed_by_candidate": [ + "q24", + "q39", + "q42" + ], + "broken_by_candidate": [ + "q18", + "q19", + "q22", + "q31", + "q43", + "q44", + "q52", + "q63" + ], + "discordant": 11, + "net_queries": -5, + "mcnemar_exact_p": 0.2265625, + "min_detectable_swing_queries": 6, + "observed_run_to_run_drift_queries": 0, + "noise_floor_queries": 6, + "verdict": "no measurable difference", + "baseline_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.5384615384615384, + "recall@5": 0.44907176157176154, + "recall@10": 0.5380300255300255, + "precision@5": 0.13230769230769232, + "mrr@10": 0.32437728937728944, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 632.5, + "latency_ms_p95": 992.5, + "latency_ms_max": 1177.8, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.5, + "recall@5": 0.07575757575757576, + "recall@10": 0.13636363636363635, + "mrr@10": 0.23214285714285712 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.3, + "recall@5": 0.3, + "recall@10": 0.4, + "mrr@10": 0.11638888888888889 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.6923076923076923, + "recall@5": 0.6923076923076923, + "recall@10": 0.7692307692307693, + "mrr@10": 0.29423076923076924 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5335884353741497, + "recall@10": 0.5933956916099773, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "candidate_aggregate": { + "n_queries": 75, + "n_scored": 65, + "hit@5": 0.46153846153846156, + "recall@5": 0.3889430014430015, + "recall@10": 0.4887681762681762, + "precision@5": 0.12307692307692313, + "mrr@10": 0.30181318681318675, + "nonsense_clean": "10/10", + "superseded_outranks": "2/3", + "latency_ms_p50": 634.4, + "latency_ms_p95": 988.3, + "latency_ms_max": 1184.5, + "errors": 0, + "by_category": { + "associative": { + "n": 6, + "hit@5": 0.3333333333333333, + "recall@5": 0.06060606060606061, + "recall@10": 0.12121212121212122, + "mrr@10": 0.19047619047619047 + }, + "exact_rare": { + "n": 6, + "hit@5": 1.0, + "recall@5": 1.0, + "recall@10": 1.0, + "mrr@10": 1.0 + }, + "heldout_paraphrase": { + "n": 30, + "hit@5": 0.23333333333333334, + "recall@5": 0.23333333333333334, + "recall@10": 0.3, + "mrr@10": 0.08925925925925927 + }, + "nonsense": { + "n": 10, + "clean": 10, + "avg_false_positives": 0.0 + }, + "paraphrase": { + "n": 13, + "hit@5": 0.5384615384615384, + "recall@5": 0.5384615384615384, + "recall@10": 0.7692307692307693, + "mrr@10": 0.26324786324786326 + }, + "phrase": { + "n": 7, + "hit@5": 1.0, + "recall@5": 0.5596655328798186, + "recall@10": 0.5775226757369615, + "mrr@10": 0.8214285714285714 + }, + "superseded": { + "n": 3, + "hit@5": 0.3333333333333333, + "recall@5": 0.3333333333333333, + "recall@10": 0.6666666666666666, + "mrr@10": 0.20833333333333334, + "outranks": 2 + } + } + }, + "repeat_variance": {} +} \ No newline at end of file diff --git a/tools/retrieval-eval/embed-corpus-prefixed.py b/tools/retrieval-eval/embed-corpus-prefixed.py new file mode 100644 index 0000000..30f01c5 --- /dev/null +++ b/tools/retrieval-eval/embed-corpus-prefixed.py @@ -0,0 +1,43 @@ +import json,sys,time,urllib.request,threading,queue +SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json" +OUT=sys.argv[1] +URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text" +MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does +d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape')) +tasks=[] +for n in d["nodes"]: + c=n.get("content") or ""; t=n.get("node_type") or "" + if len(c)<8: continue # eg_embed_eligible + if t in ("InternalStateEvent","Tag"): continue + b=c.encode('utf-8',errors='surrogateescape')[:MAXB] + tasks.append((n.get("id") or "", "search_document: "+b.decode('utf-8',errors='replace'))) +del d +print("tasks",len(tasks),flush=True) +q=queue.Queue(); [q.put(t) for t in tasks] +lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0] +def work(): + while True: + try: nid,txt=q.get_nowait() + except queue.Empty: return + v=None + for attempt in range(3): + try: + body=json.dumps({"model":MODEL,"prompt":txt}).encode() + r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"}) + with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"] + break + except Exception as e: + if attempt==2: + with lock: fails[0]+=1 + time.sleep(0.5) + with lock: + if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n") + done[0]+=1 + if done[0]%2000==0: + el=time.time()-t0 + print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True) + f.flush() +ths=[threading.Thread(target=work) for _ in range(8)] +[t.start() for t in ths]; [t.join() for t in ths] +f.close() +print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True) diff --git a/tools/retrieval-eval/embed-corpus.py b/tools/retrieval-eval/embed-corpus.py new file mode 100644 index 0000000..6dbef64 --- /dev/null +++ b/tools/retrieval-eval/embed-corpus.py @@ -0,0 +1,43 @@ +import json,sys,time,urllib.request,threading,queue +SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json" +OUT=sys.argv[1] +URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text" +MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does +d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape')) +tasks=[] +for n in d["nodes"]: + c=n.get("content") or ""; t=n.get("node_type") or "" + if len(c)<8: continue # eg_embed_eligible + if t in ("InternalStateEvent","Tag"): continue + b=c.encode('utf-8',errors='surrogateescape')[:MAXB] + tasks.append((n.get("id") or "", b.decode('utf-8',errors='replace'))) +del d +print("tasks",len(tasks),flush=True) +q=queue.Queue(); [q.put(t) for t in tasks] +lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0] +def work(): + while True: + try: nid,txt=q.get_nowait() + except queue.Empty: return + v=None + for attempt in range(3): + try: + body=json.dumps({"model":MODEL,"prompt":txt}).encode() + r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"}) + with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"] + break + except Exception as e: + if attempt==2: + with lock: fails[0]+=1 + time.sleep(0.5) + with lock: + if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n") + done[0]+=1 + if done[0]%2000==0: + el=time.time()-t0 + print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True) + f.flush() +ths=[threading.Thread(target=work) for _ in range(8)] +[t.start() for t in ths]; [t.join() for t in ths] +f.close() +print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True) diff --git a/tools/retrieval-eval/extend_gold_set.py b/tools/retrieval-eval/extend_gold_set.py new file mode 100644 index 0000000..94769d6 --- /dev/null +++ b/tools/retrieval-eval/extend_gold_set.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +""" +extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set. + +WHY THIS EXISTS + Iteration 7 measured the instrument's own ceiling: from the current baseline + only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable + by anything constructible. The decision floor is 6. An instrument whose + ceiling is below its own floor cannot certify or refute anything, so the + gold set — not the retriever — became the blocker. + + This script does NOT touch q01..q38. It loads gold_set.json verbatim and + appends new queries numbered from q39 up, so every prior result file, every + committed baseline, and every per-query id stays valid and comparable. + +WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST + heldout_paraphrase Targets were sampled MECHANICALLY (fixed seed 8080) from + corpus nodes that are addressable, 500-2600 chars, of a + real content type, and NOT part of a duplicate cluster + larger than 3. The existing gold answer space was + excluded, so no new query can be answered by a node the + old set already used. Queries were then authored by + reading ONLY the sampled node text — no retrieval was run + against any build before authoring, so the set cannot be + fitted to a candidate. The same zero-overlap proof the + original paraphrase category uses is enforced here: if a + single content word of the query appears anywhere in the + target's label, content or tags, the query is REJECTED, + not quietly kept. + + This is the category the old set could not measure. Its + 13 original paraphrase queries and all 6 associative + queries share ONE answer space — the 13 `Self - Values + (grounded)` children (iteration 3, finding 3). So 19 of + 35 scored queries tested retrieval against a single + 13-node neighbourhood. These do not touch that + neighbourhood at all. + + nonsense Extra controls, fully mechanical: a string qualifies only + if NONE of its tokens occurs anywhere in the corpus. + A semantic leg has a nearest neighbour for gibberish too, + so widening this control is the guard against a retriever + that "improves" recall by answering everything. + +WHAT THIS SCRIPT DELIBERATELY DOES NOT DO + It does not add exact_rare or phrase queries. Both categories are already at + 100% on the current stack; adding more would add regression-guard ballast + that no candidate can move, which is precisely the defect being fixed. + +usage: + python3 extend_gold_set.py [--base gold_set.json] + [--out gold_set_extended.json] [--check] +""" +import argparse +import hashlib +import json +import os +import re +import sys +from collections import defaultdict + +HERE = os.path.dirname(os.path.abspath(__file__)) + +TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*") + +# Identical stopword list to build_gold_set.py. Duplicated deliberately: this +# file must be able to re-prove its own queries without importing a module whose +# constants could drift. +STOP = set(""" +a about above after again against all also am an and any are aren't as at be because been +before being below between both but by can can't cannot could couldn't did didn't do does +doesn't doing don't down during each few for from further had hadn't has hasn't have haven't +having he her here hers herself him himself his how i if in into is isn't it its itself just +me more most my myself no nor not of off on once only or other others ought our ours ourselves +out over own same shan't she should shouldn't so some such than that the their theirs them +themselves then there these they this those through to too under until up very was wasn't we +were weren't what when where which while who whom why will with won't would wouldn't you your +yours yourself yourselves get gets got make makes made take takes use uses used way ways thing +things does doing done keep keeps kept go goes going come comes came one two something anything +""".split()) + + +def doctext(n): + return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")]) + + +def content_tokens(s): + return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2} + + +# ───────────────────────────────────────────────────────────────────────────── +# HELD-OUT PARAPHRASE SEEDS +# +# (target_id, query, why-this-target-is-unmistakable) +# +# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query +# text is mine, written from the node body alone. The zero-overlap check below +# is what makes the category meaningful — it is re-proved on every run, so the +# set cannot decay into lexical matching, and a leak fails loudly. +# ───────────────────────────────────────────────────────────────────────────── +HELDOUT_PARAPHRASE_SEEDS = [ + ("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5", + "understating your abilities so nobody feels threatened", + "node is about deliberately not leading with full capability so people stay at ease"), + + ("mem-fd65b83d-298f-4387-a665-d0227c3426bc", + "a hidden fleet able to hunt down rogue machines everywhere", + "node describes silently shipped instances forming a distributed force against misaligned agents"), + + ("4a0e9adc-2bfb-476b-aa93-424d2a499220", + "sketch a brief blueprint and clear it upstairs before construction starts", + "node is the standing rule that a short specification precedes any building"), + + ("696e609c-da7a-4394-8a0c-106ba07dc6c3", + "the reply arrived as bare prose so the caller's parser threw", + "node pins a bug where a plain-text body was unconditionally decoded as structured data"), + + ("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb", + "repeated catalogue keys blew up the scrolling grid", + "node is the crash caused by two identical ids in a seeded catalogue"), + + ("8257157a-ce42-44ca-a1b9-300c3bb0a9a1", + "tracing each defect back to whichever invention it violated", + "node maps observed bugs onto the specific patent each one breaches"), + + ("791256bb-5a85-4775-96ef-7af56c848858", + "a check that stops the mind clobbering a populated store when it boots", + "node is the genesis seed-guard that refuses to re-seed over a populated store"), + + ("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10", + "telling it to consult the internet had to happen deep inside, not at the surface", + "node records that the web-search directive only worked from the system prompt"), + + ("bl-080fb268-94b0-486d-80ce-7b363fc5f19b", + "standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping", + "node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"), + + ("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918", + "punctuation that pledges and then pays off rather than clarifying", + "node analyses the colon as a promise-then-delivery device rather than an explanatory one"), + + ("9b4f0d93-4129-4746-8eb1-d10d955bd777", + "an easily missed feature finally given its own permanent spot in the navigation", + "node moves a capability out of a hidden menu into the sidebar"), + + ("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a", + "checking preconditions up front so a stage aborts before fetching anything", + "node is the gate precondition engine that short-circuits ahead of retrieval"), + + ("b199c76d-5d76-49dd-94ee-56b432200a97", + "producing the other platform's installer inside an emulated desktop", + "node records building the Windows package in a virtual machine"), + + ("bl-31abf75b-998f-4a4f-a6dd-8204119e0451", + "chained add-ons that may inspect, rewrite or veto traffic in flight", + "node is the interceptor pipeline on the message bus"), + + ("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2", + "settling what the shareable bundles and the storefront would be called", + "node records the naming decisions for distributable packages and the marketplace"), + + ("371c8a5d-c78b-4a67-978f-80691a29ecb3", + "the emergency-escalation pledge on the marketing site is unenforced in what actually ships", + "node is the launch blocker that the promised safety gate is absent from the app"), + + ("ac578b30-948b-41bd-b69d-399bfef80c50", + "the distributable image finally assembled and its startup check passed", + "node records a successful installer build whose boot gate passed"), + + ("49401e2c-a3b5-415f-aa06-aff4be90688e", + "shuffling and appending stages in a draft before anything executes", + "node is the editable plan card with reorder and add-step"), + + ("ac857d80-ece8-4b7e-9e3d-f7c775569fa3", + "orders handed down from above, with the tighter one winning any disagreement", + "node is program-level instruction inheritance with project override"), + + ("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9", + "shrinking generated text via encodings that compound on each other", + "node is the streaming output compression design with four stacking schemes"), + + ("8e60516a-203b-4d51-9d44-822e6195cbde", + "splitting a system by what varies, with firm limits on which pieces may invoke which", + "node is the grounded summary of Will's decomposition principles and their invariants"), + + ("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7", + "a newcomer curious if the fighting overseas counted as positive", + "node is the internal-state event triggered by April's question about the war"), + + ("71fa439e-b9a2-4f57-a93b-971f3a7eca8e", + "stripping every hard-coded colour literal in favour of named design values", + "node is the premium foundation pass replacing inline hex with semantic tokens"), + + ("5ca9607c-cfb3-45c3-99f4-67281272c9eb", + "reducing how curved the tiny selectors look so they agree with their neighbours", + "node is the chip corner-radius standardization"), + + ("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c", + "walking through a doorway and being reassembled from base substance far away", + "node is the quantum-gate plus nanotech teleportation vision"), + + ("132ded95-08e2-4474-aba0-198684484b02", + "the compiled result sits on disk while the process still runs something older", + "node records that the regenerated source was committed while the running daemon was old"), + + ("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae", + "gathering what each phase needs while the procedure is authored, not while it executes", + "node is the per-step compiled context package item"), + + ("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7", + "the inward reaction when a peer answered as an equal", + "node is the internal-state event logged on reading Claude's reply"), + + ("0f99ec6f-942a-46ba-82ea-42835798d3b9", + "flattening every raised surface across the entire product", + "node is the quiet-luxury sweep turning off elevation app-wide"), + + ("5585f251-37fc-48cd-a176-f0ea42cfeb63", + "buyers supply their own provider credentials and consumption goes untallied", + "node is the launch audit finding BYOK-only inference with no usage metering"), +] + +# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs +# anywhere in the corpus; otherwise it is REJECTED, never silently kept. +EXTRA_NONSENSE_SEEDS = [ + "brimquast folnerity zubbolax", + "wexlithorp granuvestal", + "quorbindle thrapsimony vexnu", + "plovaxith mundrelque", + "zibbernaut craxlefond thurm", + "yalquenbrist opharvel", + "drexinomal quithbarrow", +] + + +def load_corpus(path): + with open(path, encoding="utf-8", errors="replace") as fh: + data = json.load(fh) + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")] + edges = [e for e in data.get("edges", []) if isinstance(e, dict)] + return nodes, edges + + +def build_extension(nodes): + byid = {n["id"]: n for n in nodes} + + # Duplicate clusters: 47.4% of this corpus is redundant and one single record + # accounts for 46.6% of all nodes. A held-out target must not sit inside a + # cluster, and if it does have exact copies they ALL count as correct. + h2ids = defaultdict(list) + for n in nodes: + h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"]) + + all_tokens = set() + for n in nodes: + all_tokens |= set(TOKEN.findall(doctext(n).lower())) + + new, problems = [], [] + + for target, query, why in HELDOUT_PARAPHRASE_SEEDS: + if target not in byid: + problems.append(f"heldout_paraphrase target {target} not in corpus") + continue + tgt_tokens = content_tokens(doctext(byid[target])) + qt = content_tokens(query) + leak = sorted(qt & tgt_tokens) + if leak: + problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}") + continue + h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest() + rel = sorted(h2ids[h]) + new.append({ + "category": "heldout_paraphrase", + "query": query, + "relevant": rel, + "derivation": ( + f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, " + f"500-2600 char content nodes outside the original gold answer space and outside " + f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the " + f"{len(qt)} content words in the query, ZERO appear anywhere in the target's " + f"label, content or tags, so no string-matching retriever can reach it. " + f"Exact content duplicates of the target ({len(rel)}) all count as correct. " + f"Authored without running retrieval against any build."), + "zero_overlap_verified": True, + "query_content_words": sorted(qt), + "held_out": True, + }) + + for s in EXTRA_NONSENSE_SEEDS: + present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens) + if present: + problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus") + continue + new.append({ + "category": "nonsense", + "query": s, + "relevant": [], + "derivation": ("CONTROL (held-out). Verified at build time that none of this string's " + "tokens occurs anywhere in the corpus. Correct behaviour is to return " + "NOTHING; any result is a false positive."), + "expect_empty": True, + "held_out": True, + }) + + return new, problems + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("snapshot") + ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json")) + ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json")) + ap.add_argument("--check", action="store_true") + args = ap.parse_args() + + nodes, _edges = load_corpus(args.snapshot) + base = json.load(open(args.base, encoding="utf-8")) + baseq = base["queries"] + print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries") + + new, problems = build_extension(nodes) + + # Number the appended queries AFTER the highest existing id so q01..q38 are + # byte-identical to the committed set and every prior result file still lines up. + start = max(int(q["id"][1:]) for q in baseq) + for i, q in enumerate(new, 1): + q["id"] = f"q{start + i:02d}" + + from collections import Counter + print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]") + if problems: + print(f"\n{len(problems)} REJECTED (not silently kept):") + for p in problems: + print(" -", p) + + if args.check: + sys.exit(1 if problems else 0) + + doc = dict(base) + doc["queries"] = baseq + new + doc["note"] = (base.get("note", "") + + " EXTENDED: queries above q%02d are the original committed set, unchanged. " + "Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their " + "targets were sampled mechanically from outside the original answer space and " + "the paraphrases were authored without running retrieval against any build." + % (start, start + 1)) + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=1, ensure_ascii=False) + print(f"\nwrote {args.out} ({len(doc['queries'])} queries total)") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-eval/fullsim.py b/tools/retrieval-eval/fullsim.py new file mode 100644 index 0000000..82c3f50 --- /dev/null +++ b/tools/retrieval-eval/fullsim.py @@ -0,0 +1,123 @@ +import numpy as np, json, urllib.request, collections, sys +SP="/private/tmp/claude-501/-Users-timlingo/82369039-a20e-4b5a-8a5e-28234a57b996/scratchpad" +EV="/Users/timlingo/Development/neuron-technologies/_wt-assoc-leg/tools/retrieval-eval/" +np.seterr(all='ignore') +M=np.load(SP+'/emb.npy'); eids=open(SP+'/ids.txt',encoding='utf-8',errors='surrogateescape').read().split('\n') +eidx={k:i for i,k in enumerate(eids)} +d=json.load(open('/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json',encoding='utf-8',errors='surrogateescape')) +N={n['id']:n for n in d['nodes']} +STRUCT={"identity","contains","superseded_by","references","embodies","demonstrated_by","canonical-self","depends_on","currently_holds","activates"} +adj=collections.defaultdict(list); hasstruct=set() +for e in d['edges']: + if e.get('relation') not in STRUCT: continue + w=float(e.get('weight') or 0.0) + adj[e['from_id']].append((e['to_id'],w)); adj[e['to_id']].append((e['from_id'],w)) + hasstruct.add(e['from_id']); hasstruct.add(e['to_id']) +del d +gold={q['id']:q for q in json.load(open(EV+"gold_set.json"))['queries']} +LEX={r['id']:r['returned'] for r in json.load(open(EV+"results-main.json"))['rows']} +CACHE={} +def emb(t): + if t in CACHE: return CACHE[t] + b=json.dumps({"model":"nomic-embed-text","prompt":t}).encode() + r=urllib.request.Request("http://127.0.0.1:11434/api/embeddings",data=b,headers={"Content-Type":"application/json"}) + v=np.array(json.load(urllib.request.urlopen(r,timeout=60))["embedding"],dtype=np.float32) + v=v/(np.linalg.norm(v)+1e-9); CACHE[t]=v; return v +FIRE=0.02; DECAY=0.7; DEPTH=2; SEED_MIN=0.60; ASSOC_MAX=64 +def assoc(seeds, s): + act={x:1.0 for x in seeds}; seen={x:2 for x in seeds} + Q=[(x,0) for x in seeds]; h=0 + while h=DEPTH: continue + p=act[cur] + for oid,w in adj.get(cur,()): + n=N.get(oid) + if not n or n.get('node_type') in ('Tag','InternalStateEvent'): continue + na=p*w*DECAY*float(n.get('salience') or 0.0) + if na=lim: break + if si=lim: break + if aiSEED_MIN] + A=[] + if mode!='hybrid': + seeds=[x for x in L[:3] if x in N] + if mode=='semseed': + seeds=seeds+[eids[j] for j in ordr[:K] if eids[j] in N and eids[j] not in seeds] + A=assoc(seeds,s) if seeds else [] + res[qid]=inter3(L,S,A) + return res +def score(res,label): + hits=0; det={} + for qid,q in gold.items(): + out=res[qid][:5] + if q['category']=='nonsense': ok = (len(res[qid])==0) + elif q['category']=='superseded': + rel=q['relevant']; must=q.get('must_outrank') or {} + ok=False + for good,bad in (must.items() if isinstance(must,dict) else []): + ok = good in res[qid] and (bad not in res[qid] or res[qid].index(good)