From 3bf44dee2d3f9d865b59dc89a11875b22bdac37c Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 5 Aug 2026 08:40:08 -0500 Subject: [PATCH] self-review 2026-08-05: redundancy must not buy a scarce slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content-hash census of the live graph: 1,858 redundant copies, 44.9% of the non-ISE store, all from a June id-scheme migration that re-added nodes under fresh UUIDs instead of matching on content. Generation stopped in June; the copies did not. Being byte-identical they carry identical embeddings, so they score identically against any query. Measured over 50 real query probes against the live 3,998-vector set: 40.2% of semantic seed slots were consumed by redundant copies of content already in the seed set, 92% of retrievals affected, effective distinct seeds 4.78 of 8. Two fifths of every retrieval was spent re-reading the same page. Deleting nodes is a separate operation with its own backup discipline. This change makes the runtime immune to the condition instead: redundancy can never buy a scarce slot, whatever state the graph is in. Enforced at both scarcity points — semantic seed selection (a rejected copy does not consume one of the K slots; the loop retries for the next distinct node) and WM admission via a new Pass 3+1/2 ahead of the capacity cap, so 24 slots are contested by 24 distinct meanings rather than by however many copies of one document exist. Identity is exact content hash first, then cosine >= 0.995 for copies that differ only in insignificant characters. At 768 dimensions that admits only near-verbatim text: this suppresses redundancy, never similarity. Live after restart: ~8.8 redundant seed candidates rejected per activation. New dup_seeds/dup_wm gauges in act-stats. --- lang/releases/v1.0.0-20260501/el_runtime.c | 195 ++++++++++++++++++++- 1 file changed, 192 insertions(+), 3 deletions(-) diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index 9c05363..ebbdba2 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -6198,6 +6198,12 @@ static int64_t _eg_embed_breaker_until = 0; * rates keep the previous reading and diff. Restart legitimately resets to 0. */ static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */ static int64_t _eg_act_wm_evicted = 0; /* ALL WM evictions, cumulative (see below) */ +/* Redundancy suppression counters (2026-08-05 self-review) — see + * ENGRAM_DEDUP_COS. dup_seeds = semantic seed slots reclaimed from redundant + * copies; dup_wm = WM candidates dropped for duplicating a higher-ranked + * candidate's content. Both cumulative for the process lifetime. */ +static int64_t _eg_act_dup_seeds = 0; +static int64_t _eg_act_dup_wm = 0; /* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor * and rank paths. The two carry-over eviction paths (base-level below τ, and * decayed weight below WM_FLOOR) were silent, so the reported eviction rate @@ -6230,6 +6236,87 @@ static double eg_cosine(const float* a, const float* b, int32_t dim) { return dot / (sqrt(na) * sqrt(nb)); } +/* ── Redundancy suppression (2026-08-05 self-review) ───────────────────────── + * MEASUREMENT, not intuition. Content-hash census of the live snapshot + * (13,216 nodes / 41,213 edges) on 2026-08-05: + * + * non-ISE nodes 4,138 + * duplicate content groups 1,489 + * REDUNDANT copies 1,858 (44.9% of the non-ISE graph) + * redundant copies embedded 1,856 + * + * The copies were all created in a single June 2026 import (1,854 of 1,858; + * 2 in July) — an id-scheme migration re-added nodes under fresh UUIDs + * instead of matching on content. Generation has stopped. The copies have + * not: they are byte-identical, so they carry IDENTICAL embeddings, and + * therefore identical cosine to any query. + * + * That is the damage. Semantic seeding takes the top-K by cosine + * (ENGRAM_EMBED_SEED_K = 8, ≥ ENGRAM_EMBED_SEED_MIN). A document with six + * copies does not compete for one of those eight slots — it takes six. + * Measured over 50 real query probes against the live 3,998-vector set: + * + * seed slots filled 400 + * slots consumed by redundant copies 161 (40.2%) + * probes affected 46/50 (92%) + * effective DISTINCT seeds per scan 4.78 of 8 + * + * Two fifths of every retrieval was spent re-reading the same page. The + * same collapse hits WM: duplicates score identically, so they promote + * together and hold multiple of the 24 slots for one thought. + * + * The fix is not to delete nodes — data repair is a separate, reversible + * operation with its own backup discipline. The fix is that redundancy must + * never buy a scarce slot, whatever the graph's state. Enforced at both + * scarcity points: semantic seed selection, and WM admission (Pass 3½). + * + * Identity test, cheapest first: + * 1. type|content FNV-1a — exact; catches the import duplicates + * 2. cosine ≥ ENGRAM_DEDUP_COS — catches copies differing only in + * whitespace/punctuation, which hash differently but embed identically + * 0.995 is deliberately severe: at 768 dimensions this admits only + * near-verbatim text. Distinct-but-related nodes (the associative structure + * this system exists to traverse) sit far below it and are untouched. + * This suppresses REDUNDANCY, never similarity. */ +#define ENGRAM_DEDUP_COS 0.995 + +/* eg_content_key — FNV-1a over node_type|content. Identical prose under a + * different node_type is not a duplicate (a Knowledge note and the + * BacklogItem quoting it are different objects), so the type is folded in. */ +static uint64_t eg_content_key(const EngramNode* n) { + uint64_t h = 14695981039346656037ULL; + const char* s = n->node_type; + if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + h ^= (unsigned char)'|'; h *= 1099511628211ULL; + s = n->content; + if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +/* eg_same_content — is node `a` redundant with node `b`? + * Hash equality first (one pass, no allocation); embedding near-identity as + * the fallback for copies that differ only in insignificant characters. + * Deliberately does NOT strcmp on hash collision: a 64-bit FNV-1a collision + * across ~4k candidates is ~1e-13, and the cost of being wrong is one node + * losing one slot on one call — not corruption. */ +static int eg_same_content(const EngramNode* a, const EngramNode* b, + uint64_t ka, uint64_t kb) { + if (ka == kb) return 1; + if (a->emb && b->emb && a->emb_dim > 0 && a->emb_dim == b->emb_dim) { + if (eg_cosine(a->emb, b->emb, a->emb_dim) >= ENGRAM_DEDUP_COS) return 1; + } + return 0; +} + +/* Weight-carrying index for the Pass 3½ descending walk. */ +typedef struct { double w; int64_t idx; } EgDupCand; +static int eg_dupcand_cmp_desc(const void* a, const void* b) { + double wa = ((const EgDupCand*)a)->w, wb = ((const EgDupCand*)b)->w; + if (wa < wb) return 1; + if (wa > wb) return -1; + return 0; +} + /* eg_ctx_blend — fold one touch embedding into the context centroid: * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy) * on first touch or dim change; silently skips degenerate vectors. */ @@ -8085,14 +8172,42 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * literal substrings. Lexically-seeded nodes are skipped — the lexical * path already gave them coverage-scaled activation. */ if (cosq) { - for (int k = 0; k < ENGRAM_EMBED_SEED_K; k++) { + /* Redundancy-suppressed top-K (2026-08-05 self-review; see + * ENGRAM_DEDUP_COS for the measurement that motivated it). A rejected + * candidate does NOT consume one of the K slots — the loop retries for + * the next-best distinct node, so K distinct meanings are seeded rather + * than K copies of one. Rejects are recorded in seed_dup[] rather than + * reached[] or cosq[]: marking reached[] would suppress the node's + * propagation, and clobbering cosq[] would change the downstream + * query-aware propagation gate. Neither belongs in a seeding decision. + * `guard` bounds the retries so a pathological duplicate cluster can + * never turn seed selection into an O(K·N²) scan. */ + unsigned char* seed_dup = calloc((size_t)g->node_count, 1); + int64_t sel[ENGRAM_EMBED_SEED_K]; + uint64_t selkey[ENGRAM_EMBED_SEED_K]; + int nsel = 0; + int guard = ENGRAM_EMBED_SEED_K * 8; + while (nsel < ENGRAM_EMBED_SEED_K && guard-- > 0) { int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN; for (int64_t i = 0; i < g->node_count; i++) { if (reached[i]) continue; + if (seed_dup && seed_dup[i]) continue; if (cosq[i] > bc) { bc = cosq[i]; bi = i; } } if (bi < 0) break; EngramNode* n = &g->nodes[bi]; + uint64_t key = eg_content_key(n); + int dup = 0; + for (int s = 0; s < nsel; s++) { + if (eg_same_content(n, &g->nodes[sel[s]], key, selkey[s])) { + dup = 1; break; + } + } + if (dup) { + _eg_act_dup_seeds++; + if (seed_dup) { seed_dup[bi] = 1; continue; } + break; /* OOM on the skip map: stop rather than spin */ + } double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double act = bc * tdecay * dampen; @@ -8103,7 +8218,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { best_bg[bi] = act; best_hops[bi] = 0; reached[bi] = 1; + sel[nsel] = bi; selkey[nsel] = key; nsel++; } + free(seed_dup); } /* Compute mean seed created_at for temporal proximity bonus. * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — @@ -8414,6 +8531,76 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { n->suppression_count = 0; } + /* ── PASS 3½: redundancy suppression ──────────────────────────────────── + * (2026-08-05 self-review; see ENGRAM_DEDUP_COS for the census.) Runs + * BEFORE the capacity cap, so the 24 slots are contested by 24 distinct + * meanings rather than by however many copies of one document the graph + * happens to hold. Byte-identical nodes score byte-identically, so without + * this they promote as a block — a six-copy document could hold a quarter + * of working memory while saying one thing. + * + * Walk candidates in descending weight; the first occurrence of a content + * survives, later ones are evicted. Highest-weight copy wins, which keeps + * the survivor choice deterministic and preserves the strongest activation. + * + * Bounded work: once WM_CAP distinct survivors are held, every remaining + * candidate is weaker than the weakest survivor and Pass 4 would evict it + * anyway — so the walk stops at the first candidate STRICTLY below the + * cap-th survivor's weight. Ties keep being processed, because a tie can + * still take a slot through Pass 4's at_cutoff_slots path. Typical cost is + * a few dozen content hashes per call, not a full-graph sweep. + * + * This runs after Pass 3 deliberately: Layer 0 (safety) force-promotions + * are already in wm_weights and are ranked like anything else. If safety + * content is genuinely duplicated, one copy still holds a slot — the + * guarantee is that the content is present, not that every copy of it is. */ + { + int64_t nc = 0; + for (int64_t i = 0; i < g->node_count; i++) + if (wm_weights[i] > 0.0) nc++; + if (nc > 1) { + EgDupCand* dc = malloc((size_t)nc * sizeof(EgDupCand)); + if (dc) { + int64_t ci = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) { + dc[ci].w = wm_weights[i]; dc[ci].idx = i; ci++; + } + } + qsort(dc, (size_t)nc, sizeof(EgDupCand), eg_dupcand_cmp_desc); + int64_t keep[ENGRAM_WM_CAP]; + uint64_t kkey[ENGRAM_WM_CAP]; + int nk = 0; + double cap_w = 0.0; /* weight of the WM_CAP-th survivor */ + for (int64_t z = 0; z < nc; z++) { + if (nk >= ENGRAM_WM_CAP && dc[z].w < cap_w) break; + int64_t i = dc[z].idx; + EngramNode* n = &g->nodes[i]; + uint64_t key = eg_content_key(n); + int dup = 0; + for (int s = 0; s < nk; s++) { + if (eg_same_content(n, &g->nodes[keep[s]], key, kkey[s])) { + dup = 1; break; + } + } + if (dup) { + wm_weights[i] = 0.0; + _eg_act_wm_evicted++; + _eg_act_dup_wm++; + continue; + } + if (nk < ENGRAM_WM_CAP) { + keep[nk] = i; kkey[nk] = key; nk++; + if (nk == ENGRAM_WM_CAP) cap_w = dc[z].w; + } + } + free(dc); + } + /* malloc failure: skip suppression — duplicates may share slots + * this call, which is the pre-2026-08-05 behavior. No corruption. */ + } + } + /* ── PASS 4: WM capacity cap (per-call) ───────────────────────────────── * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this * activation call. Without this, broad curiosity seeds like "knowledge" @@ -9970,13 +10157,15 @@ el_val_t engram_act_stats_json(void) { "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d," "\"ctx_cos\":%.3f," "\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f," - "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld}", + "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld," + "\"dup_seeds\":%lld,\"dup_wm\":%lld}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, breaker_open, _eg_embed_consec_fail, _eg_act_ctx_cos, (long long)hebb_edges, hebb_max, hebb_mass, - hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed); + hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed, + (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm); return el_wrap_str(el_strdup(buf)); }