diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index 9c05363..ebbdba2 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -6198,6 +6198,12 @@ static int64_t _eg_embed_breaker_until = 0; * rates keep the previous reading and diff. Restart legitimately resets to 0. */ static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */ static int64_t _eg_act_wm_evicted = 0; /* ALL WM evictions, cumulative (see below) */ +/* Redundancy suppression counters (2026-08-05 self-review) — see + * ENGRAM_DEDUP_COS. dup_seeds = semantic seed slots reclaimed from redundant + * copies; dup_wm = WM candidates dropped for duplicating a higher-ranked + * candidate's content. Both cumulative for the process lifetime. */ +static int64_t _eg_act_dup_seeds = 0; +static int64_t _eg_act_dup_wm = 0; /* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor * and rank paths. The two carry-over eviction paths (base-level below τ, and * decayed weight below WM_FLOOR) were silent, so the reported eviction rate @@ -6230,6 +6236,87 @@ static double eg_cosine(const float* a, const float* b, int32_t dim) { return dot / (sqrt(na) * sqrt(nb)); } +/* ── Redundancy suppression (2026-08-05 self-review) ───────────────────────── + * MEASUREMENT, not intuition. Content-hash census of the live snapshot + * (13,216 nodes / 41,213 edges) on 2026-08-05: + * + * non-ISE nodes 4,138 + * duplicate content groups 1,489 + * REDUNDANT copies 1,858 (44.9% of the non-ISE graph) + * redundant copies embedded 1,856 + * + * The copies were all created in a single June 2026 import (1,854 of 1,858; + * 2 in July) — an id-scheme migration re-added nodes under fresh UUIDs + * instead of matching on content. Generation has stopped. The copies have + * not: they are byte-identical, so they carry IDENTICAL embeddings, and + * therefore identical cosine to any query. + * + * That is the damage. Semantic seeding takes the top-K by cosine + * (ENGRAM_EMBED_SEED_K = 8, ≥ ENGRAM_EMBED_SEED_MIN). A document with six + * copies does not compete for one of those eight slots — it takes six. + * Measured over 50 real query probes against the live 3,998-vector set: + * + * seed slots filled 400 + * slots consumed by redundant copies 161 (40.2%) + * probes affected 46/50 (92%) + * effective DISTINCT seeds per scan 4.78 of 8 + * + * Two fifths of every retrieval was spent re-reading the same page. The + * same collapse hits WM: duplicates score identically, so they promote + * together and hold multiple of the 24 slots for one thought. + * + * The fix is not to delete nodes — data repair is a separate, reversible + * operation with its own backup discipline. The fix is that redundancy must + * never buy a scarce slot, whatever the graph's state. Enforced at both + * scarcity points: semantic seed selection, and WM admission (Pass 3½). + * + * Identity test, cheapest first: + * 1. type|content FNV-1a — exact; catches the import duplicates + * 2. cosine ≥ ENGRAM_DEDUP_COS — catches copies differing only in + * whitespace/punctuation, which hash differently but embed identically + * 0.995 is deliberately severe: at 768 dimensions this admits only + * near-verbatim text. Distinct-but-related nodes (the associative structure + * this system exists to traverse) sit far below it and are untouched. + * This suppresses REDUNDANCY, never similarity. */ +#define ENGRAM_DEDUP_COS 0.995 + +/* eg_content_key — FNV-1a over node_type|content. Identical prose under a + * different node_type is not a duplicate (a Knowledge note and the + * BacklogItem quoting it are different objects), so the type is folded in. */ +static uint64_t eg_content_key(const EngramNode* n) { + uint64_t h = 14695981039346656037ULL; + const char* s = n->node_type; + if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + h ^= (unsigned char)'|'; h *= 1099511628211ULL; + s = n->content; + if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +/* eg_same_content — is node `a` redundant with node `b`? + * Hash equality first (one pass, no allocation); embedding near-identity as + * the fallback for copies that differ only in insignificant characters. + * Deliberately does NOT strcmp on hash collision: a 64-bit FNV-1a collision + * across ~4k candidates is ~1e-13, and the cost of being wrong is one node + * losing one slot on one call — not corruption. */ +static int eg_same_content(const EngramNode* a, const EngramNode* b, + uint64_t ka, uint64_t kb) { + if (ka == kb) return 1; + if (a->emb && b->emb && a->emb_dim > 0 && a->emb_dim == b->emb_dim) { + if (eg_cosine(a->emb, b->emb, a->emb_dim) >= ENGRAM_DEDUP_COS) return 1; + } + return 0; +} + +/* Weight-carrying index for the Pass 3½ descending walk. */ +typedef struct { double w; int64_t idx; } EgDupCand; +static int eg_dupcand_cmp_desc(const void* a, const void* b) { + double wa = ((const EgDupCand*)a)->w, wb = ((const EgDupCand*)b)->w; + if (wa < wb) return 1; + if (wa > wb) return -1; + return 0; +} + /* eg_ctx_blend — fold one touch embedding into the context centroid: * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy) * on first touch or dim change; silently skips degenerate vectors. */ @@ -8085,14 +8172,42 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * literal substrings. Lexically-seeded nodes are skipped — the lexical * path already gave them coverage-scaled activation. */ if (cosq) { - for (int k = 0; k < ENGRAM_EMBED_SEED_K; k++) { + /* Redundancy-suppressed top-K (2026-08-05 self-review; see + * ENGRAM_DEDUP_COS for the measurement that motivated it). A rejected + * candidate does NOT consume one of the K slots — the loop retries for + * the next-best distinct node, so K distinct meanings are seeded rather + * than K copies of one. Rejects are recorded in seed_dup[] rather than + * reached[] or cosq[]: marking reached[] would suppress the node's + * propagation, and clobbering cosq[] would change the downstream + * query-aware propagation gate. Neither belongs in a seeding decision. + * `guard` bounds the retries so a pathological duplicate cluster can + * never turn seed selection into an O(K·N²) scan. */ + unsigned char* seed_dup = calloc((size_t)g->node_count, 1); + int64_t sel[ENGRAM_EMBED_SEED_K]; + uint64_t selkey[ENGRAM_EMBED_SEED_K]; + int nsel = 0; + int guard = ENGRAM_EMBED_SEED_K * 8; + while (nsel < ENGRAM_EMBED_SEED_K && guard-- > 0) { int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN; for (int64_t i = 0; i < g->node_count; i++) { if (reached[i]) continue; + if (seed_dup && seed_dup[i]) continue; if (cosq[i] > bc) { bc = cosq[i]; bi = i; } } if (bi < 0) break; EngramNode* n = &g->nodes[bi]; + uint64_t key = eg_content_key(n); + int dup = 0; + for (int s = 0; s < nsel; s++) { + if (eg_same_content(n, &g->nodes[sel[s]], key, selkey[s])) { + dup = 1; break; + } + } + if (dup) { + _eg_act_dup_seeds++; + if (seed_dup) { seed_dup[bi] = 1; continue; } + break; /* OOM on the skip map: stop rather than spin */ + } double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double act = bc * tdecay * dampen; @@ -8103,7 +8218,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { best_bg[bi] = act; best_hops[bi] = 0; reached[bi] = 1; + sel[nsel] = bi; selkey[nsel] = key; nsel++; } + free(seed_dup); } /* Compute mean seed created_at for temporal proximity bonus. * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — @@ -8414,6 +8531,76 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { n->suppression_count = 0; } + /* ── PASS 3½: redundancy suppression ──────────────────────────────────── + * (2026-08-05 self-review; see ENGRAM_DEDUP_COS for the census.) Runs + * BEFORE the capacity cap, so the 24 slots are contested by 24 distinct + * meanings rather than by however many copies of one document the graph + * happens to hold. Byte-identical nodes score byte-identically, so without + * this they promote as a block — a six-copy document could hold a quarter + * of working memory while saying one thing. + * + * Walk candidates in descending weight; the first occurrence of a content + * survives, later ones are evicted. Highest-weight copy wins, which keeps + * the survivor choice deterministic and preserves the strongest activation. + * + * Bounded work: once WM_CAP distinct survivors are held, every remaining + * candidate is weaker than the weakest survivor and Pass 4 would evict it + * anyway — so the walk stops at the first candidate STRICTLY below the + * cap-th survivor's weight. Ties keep being processed, because a tie can + * still take a slot through Pass 4's at_cutoff_slots path. Typical cost is + * a few dozen content hashes per call, not a full-graph sweep. + * + * This runs after Pass 3 deliberately: Layer 0 (safety) force-promotions + * are already in wm_weights and are ranked like anything else. If safety + * content is genuinely duplicated, one copy still holds a slot — the + * guarantee is that the content is present, not that every copy of it is. */ + { + int64_t nc = 0; + for (int64_t i = 0; i < g->node_count; i++) + if (wm_weights[i] > 0.0) nc++; + if (nc > 1) { + EgDupCand* dc = malloc((size_t)nc * sizeof(EgDupCand)); + if (dc) { + int64_t ci = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) { + dc[ci].w = wm_weights[i]; dc[ci].idx = i; ci++; + } + } + qsort(dc, (size_t)nc, sizeof(EgDupCand), eg_dupcand_cmp_desc); + int64_t keep[ENGRAM_WM_CAP]; + uint64_t kkey[ENGRAM_WM_CAP]; + int nk = 0; + double cap_w = 0.0; /* weight of the WM_CAP-th survivor */ + for (int64_t z = 0; z < nc; z++) { + if (nk >= ENGRAM_WM_CAP && dc[z].w < cap_w) break; + int64_t i = dc[z].idx; + EngramNode* n = &g->nodes[i]; + uint64_t key = eg_content_key(n); + int dup = 0; + for (int s = 0; s < nk; s++) { + if (eg_same_content(n, &g->nodes[keep[s]], key, kkey[s])) { + dup = 1; break; + } + } + if (dup) { + wm_weights[i] = 0.0; + _eg_act_wm_evicted++; + _eg_act_dup_wm++; + continue; + } + if (nk < ENGRAM_WM_CAP) { + keep[nk] = i; kkey[nk] = key; nk++; + if (nk == ENGRAM_WM_CAP) cap_w = dc[z].w; + } + } + free(dc); + } + /* malloc failure: skip suppression — duplicates may share slots + * this call, which is the pre-2026-08-05 behavior. No corruption. */ + } + } + /* ── PASS 4: WM capacity cap (per-call) ───────────────────────────────── * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this * activation call. Without this, broad curiosity seeds like "knowledge" @@ -9970,13 +10157,15 @@ el_val_t engram_act_stats_json(void) { "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d," "\"ctx_cos\":%.3f," "\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f," - "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld}", + "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld," + "\"dup_seeds\":%lld,\"dup_wm\":%lld}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, breaker_open, _eg_embed_consec_fail, _eg_act_ctx_cos, (long long)hebb_edges, hebb_max, hebb_mass, - hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed); + hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed, + (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm); return el_wrap_str(el_strdup(buf)); }