lang/releases/v1.0.0-20260501/el_runtime.c

@@ -6198,6 +6198,12 @@ static int64_t _eg_embed_breaker_until = 0;
  * rates keep the previous reading and diff. Restart legitimately resets to 0. */
 static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */
 static int64_t _eg_act_wm_evicted    = 0; /* ALL WM evictions, cumulative (see below)   */
+/* Redundancy suppression counters (2026-08-05 self-review) — see
+ * ENGRAM_DEDUP_COS. dup_seeds = semantic seed slots reclaimed from redundant
+ * copies; dup_wm = WM candidates dropped for duplicating a higher-ranked
+ * candidate's content. Both cumulative for the process lifetime. */
+static int64_t _eg_act_dup_seeds     = 0;
+static int64_t _eg_act_dup_wm        = 0;
 /* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor
  * and rank paths. The two carry-over eviction paths (base-level below τ, and
  * decayed weight below WM_FLOOR) were silent, so the reported eviction rate
@@ -6230,6 +6236,87 @@ static double eg_cosine(const float* a, const float* b, int32_t dim) {
     return dot / (sqrt(na) * sqrt(nb));
 }
 
+/* ── Redundancy suppression (2026-08-05 self-review) ─────────────────────────
+ * MEASUREMENT, not intuition. Content-hash census of the live snapshot
+ * (13,216 nodes / 41,213 edges) on 2026-08-05:
+ *
+ *   non-ISE nodes                4,138
+ *   duplicate content groups     1,489
+ *   REDUNDANT copies             1,858   (44.9% of the non-ISE graph)
+ *   redundant copies embedded    1,856
+ *
+ * The copies were all created in a single June 2026 import (1,854 of 1,858;
+ * 2 in July) — an id-scheme migration re-added nodes under fresh UUIDs
+ * instead of matching on content. Generation has stopped. The copies have
+ * not: they are byte-identical, so they carry IDENTICAL embeddings, and
+ * therefore identical cosine to any query.
+ *
+ * That is the damage. Semantic seeding takes the top-K by cosine
+ * (ENGRAM_EMBED_SEED_K = 8, ≥ ENGRAM_EMBED_SEED_MIN). A document with six
+ * copies does not compete for one of those eight slots — it takes six.
+ * Measured over 50 real query probes against the live 3,998-vector set:
+ *
+ *   seed slots filled                       400
+ *   slots consumed by redundant copies      161   (40.2%)
+ *   probes affected                         46/50 (92%)
+ *   effective DISTINCT seeds per scan       4.78 of 8
+ *
+ * Two fifths of every retrieval was spent re-reading the same page. The
+ * same collapse hits WM: duplicates score identically, so they promote
+ * together and hold multiple of the 24 slots for one thought.
+ *
+ * The fix is not to delete nodes — data repair is a separate, reversible
+ * operation with its own backup discipline. The fix is that redundancy must
+ * never buy a scarce slot, whatever the graph's state. Enforced at both
+ * scarcity points: semantic seed selection, and WM admission (Pass 3½).
+ *
+ * Identity test, cheapest first:
+ *   1. type|content FNV-1a — exact; catches the import duplicates
+ *   2. cosine ≥ ENGRAM_DEDUP_COS — catches copies differing only in
+ *      whitespace/punctuation, which hash differently but embed identically
+ * 0.995 is deliberately severe: at 768 dimensions this admits only
+ * near-verbatim text. Distinct-but-related nodes (the associative structure
+ * this system exists to traverse) sit far below it and are untouched.
+ * This suppresses REDUNDANCY, never similarity. */
+#define ENGRAM_DEDUP_COS 0.995
+
+/* eg_content_key — FNV-1a over node_type|content. Identical prose under a
+ * different node_type is not a duplicate (a Knowledge note and the
+ * BacklogItem quoting it are different objects), so the type is folded in. */
+static uint64_t eg_content_key(const EngramNode* n) {
+    uint64_t h = 14695981039346656037ULL;
+    const char* s = n->node_type;
+    if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; }
+    h ^= (unsigned char)'|'; h *= 1099511628211ULL;
+    s = n->content;
+    if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; }
+    return h;
+}
+
+/* eg_same_content — is node `a` redundant with node `b`?
+ * Hash equality first (one pass, no allocation); embedding near-identity as
+ * the fallback for copies that differ only in insignificant characters.
+ * Deliberately does NOT strcmp on hash collision: a 64-bit FNV-1a collision
+ * across ~4k candidates is ~1e-13, and the cost of being wrong is one node
+ * losing one slot on one call — not corruption. */
+static int eg_same_content(const EngramNode* a, const EngramNode* b,
+                           uint64_t ka, uint64_t kb) {
+    if (ka == kb) return 1;
+    if (a->emb && b->emb && a->emb_dim > 0 && a->emb_dim == b->emb_dim) {
+        if (eg_cosine(a->emb, b->emb, a->emb_dim) >= ENGRAM_DEDUP_COS) return 1;
+    }
+    return 0;
+}
+
+/* Weight-carrying index for the Pass 3½ descending walk. */
+typedef struct { double w; int64_t idx; } EgDupCand;
+static int eg_dupcand_cmp_desc(const void* a, const void* b) {
+    double wa = ((const EgDupCand*)a)->w, wb = ((const EgDupCand*)b)->w;
+    if (wa < wb) return 1;
+    if (wa > wb) return -1;
+    return 0;
+}
+
 /* eg_ctx_blend — fold one touch embedding into the context centroid:
  * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy)
  * on first touch or dim change; silently skips degenerate vectors. */
@@ -8085,14 +8172,42 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
      * literal substrings. Lexically-seeded nodes are skipped — the lexical
      * path already gave them coverage-scaled activation. */
     if (cosq) {
-        for (int k = 0; k < ENGRAM_EMBED_SEED_K; k++) {
+        /* Redundancy-suppressed top-K (2026-08-05 self-review; see
+         * ENGRAM_DEDUP_COS for the measurement that motivated it). A rejected
+         * candidate does NOT consume one of the K slots — the loop retries for
+         * the next-best distinct node, so K distinct meanings are seeded rather
+         * than K copies of one. Rejects are recorded in seed_dup[] rather than
+         * reached[] or cosq[]: marking reached[] would suppress the node's
+         * propagation, and clobbering cosq[] would change the downstream
+         * query-aware propagation gate. Neither belongs in a seeding decision.
+         * `guard` bounds the retries so a pathological duplicate cluster can
+         * never turn seed selection into an O(K·N²) scan. */
+        unsigned char* seed_dup = calloc((size_t)g->node_count, 1);
+        int64_t  sel[ENGRAM_EMBED_SEED_K];
+        uint64_t selkey[ENGRAM_EMBED_SEED_K];
+        int nsel = 0;
+        int guard = ENGRAM_EMBED_SEED_K * 8;
+        while (nsel < ENGRAM_EMBED_SEED_K && guard-- > 0) {
             int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN;
             for (int64_t i = 0; i < g->node_count; i++) {
                 if (reached[i]) continue;
+                if (seed_dup && seed_dup[i]) continue;
                 if (cosq[i] > bc) { bc = cosq[i]; bi = i; }
             }
             if (bi < 0) break;
             EngramNode* n = &g->nodes[bi];
+            uint64_t key = eg_content_key(n);
+            int dup = 0;
+            for (int s = 0; s < nsel; s++) {
+                if (eg_same_content(n, &g->nodes[sel[s]], key, selkey[s])) {
+                    dup = 1; break;
+                }
+            }
+            if (dup) {
+                _eg_act_dup_seeds++;
+                if (seed_dup) { seed_dup[bi] = 1; continue; }
+                break; /* OOM on the skip map: stop rather than spin */
+            }
             double tdecay = engram_temporal_decay(n, now_ms);
             double dampen = engram_activation_dampen(n);
             double act    = bc * tdecay * dampen;
@@ -8103,7 +8218,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
             best_bg[bi]   = act;
             best_hops[bi] = 0;
             reached[bi]   = 1;
+            sel[nsel] = bi; selkey[nsel] = key; nsel++;
         }
+        free(seed_dup);
     }
     /* Compute mean seed created_at for temporal proximity bonus.
      * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 —
@@ -8414,6 +8531,76 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
         n->suppression_count = 0;
     }
 
+    /* ── PASS 3½: redundancy suppression ────────────────────────────────────
+     * (2026-08-05 self-review; see ENGRAM_DEDUP_COS for the census.) Runs
+     * BEFORE the capacity cap, so the 24 slots are contested by 24 distinct
+     * meanings rather than by however many copies of one document the graph
+     * happens to hold. Byte-identical nodes score byte-identically, so without
+     * this they promote as a block — a six-copy document could hold a quarter
+     * of working memory while saying one thing.
+     *
+     * Walk candidates in descending weight; the first occurrence of a content
+     * survives, later ones are evicted. Highest-weight copy wins, which keeps
+     * the survivor choice deterministic and preserves the strongest activation.
+     *
+     * Bounded work: once WM_CAP distinct survivors are held, every remaining
+     * candidate is weaker than the weakest survivor and Pass 4 would evict it
+     * anyway — so the walk stops at the first candidate STRICTLY below the
+     * cap-th survivor's weight. Ties keep being processed, because a tie can
+     * still take a slot through Pass 4's at_cutoff_slots path. Typical cost is
+     * a few dozen content hashes per call, not a full-graph sweep.
+     *
+     * This runs after Pass 3 deliberately: Layer 0 (safety) force-promotions
+     * are already in wm_weights and are ranked like anything else. If safety
+     * content is genuinely duplicated, one copy still holds a slot — the
+     * guarantee is that the content is present, not that every copy of it is. */
+    {
+        int64_t nc = 0;
+        for (int64_t i = 0; i < g->node_count; i++)
+            if (wm_weights[i] > 0.0) nc++;
+        if (nc > 1) {
+            EgDupCand* dc = malloc((size_t)nc * sizeof(EgDupCand));
+            if (dc) {
+                int64_t ci = 0;
+                for (int64_t i = 0; i < g->node_count; i++) {
+                    if (wm_weights[i] > 0.0) {
+                        dc[ci].w = wm_weights[i]; dc[ci].idx = i; ci++;
+                    }
+                }
+                qsort(dc, (size_t)nc, sizeof(EgDupCand), eg_dupcand_cmp_desc);
+                int64_t  keep[ENGRAM_WM_CAP];
+                uint64_t kkey[ENGRAM_WM_CAP];
+                int nk = 0;
+                double  cap_w = 0.0;   /* weight of the WM_CAP-th survivor */
+                for (int64_t z = 0; z < nc; z++) {
+                    if (nk >= ENGRAM_WM_CAP && dc[z].w < cap_w) break;
+                    int64_t i = dc[z].idx;
+                    EngramNode* n = &g->nodes[i];
+                    uint64_t key = eg_content_key(n);
+                    int dup = 0;
+                    for (int s = 0; s < nk; s++) {
+                        if (eg_same_content(n, &g->nodes[keep[s]], key, kkey[s])) {
+                            dup = 1; break;
+                        }
+                    }
+                    if (dup) {
+                        wm_weights[i] = 0.0;
+                        _eg_act_wm_evicted++;
+                        _eg_act_dup_wm++;
+                        continue;
+                    }
+                    if (nk < ENGRAM_WM_CAP) {
+                        keep[nk] = i; kkey[nk] = key; nk++;
+                        if (nk == ENGRAM_WM_CAP) cap_w = dc[z].w;
+                    }
+                }
+                free(dc);
+            }
+            /* malloc failure: skip suppression — duplicates may share slots
+             * this call, which is the pre-2026-08-05 behavior. No corruption. */
+        }
+    }
+
     /* ── PASS 4: WM capacity cap (per-call) ─────────────────────────────────
      * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this
      * activation call. Without this, broad curiosity seeds like "knowledge"
@@ -9970,13 +10157,15 @@ el_val_t engram_act_stats_json(void) {
              "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d,"
              "\"ctx_cos\":%.3f,"
              "\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f,"
-             "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld}",
+             "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld,"
+             "\"dup_seeds\":%lld,\"dup_wm\":%lld}",
              (long long)_eg_act_wm_evicted,
              (long long)_eg_act_breakthroughs,
              breaker_open, _eg_embed_consec_fail,
              _eg_act_ctx_cos,
              (long long)hebb_edges, hebb_max, hebb_mass,
-             hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed);
+             hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed,
+             (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm);
     return el_wrap_str(el_strdup(buf));
 }