feat(engram): BM25-shaped lexical leg + addressability guard on the read path

engram_search_json ranked its lexical leg by raw distinct-token coverage with
salience as tiebreak: a token in 30,000 nodes counted the same as a token in 1,
and a 1.3 MB record matched nearly every query token by surface area alone.
Score it BM25-shaped instead - Lucene-form IDF and length normalisation over
the corpus mean - with per-token document frequency accumulated in the SAME
corpus pass that finds the hits (no extra scan, no extra round-trip).

Also refuse to return records whose identifier is not printable ASCII. This
corpus carries 1,032 such records (453 by the printable test) from a save-side
corruption; they occupy 125 of 303 returned slots on main. Claims 12, 23 and 27
all key on the node identifier, so such a record is unfetchable by any caller.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

vendor/el-runtime/v1.0.0-20260501/el_runtime.c

@@ -7387,6 +7387,48 @@ static int engram_node_match_score(const EngramNode* n,
     return score;
 }
 
+/* Same match test as engram_node_match_score, but returns the SET of matched
+ * query tokens as a bitmask instead of only their count. ENGRAM_MAX_QTOKENS is
+ * 32, so one uint32 covers every token the tokenizer can produce. The mask is
+ * what lets the caller accumulate a per-token document frequency in the SAME
+ * pass that finds the hits — no second scan of the corpus. */
+static uint32_t engram_node_match_mask(const EngramNode* n,
+                                       char toks[][ENGRAM_QTOK_LEN], int ntok) {
+    uint32_t m = 0;
+    for (int t = 0; t < ntok && t < 32; t++) {
+        if (istr_contains(n->content, toks[t]) ||
+            istr_contains(n->label,   toks[t]) ||
+            istr_contains(n->tags,    toks[t]))
+            m |= (uint32_t)1u << t;
+    }
+    return m;
+}
+
+/* Searchable byte length of a node: the same three fields the match test
+ * reads. Used as the BM25 document length so a long node does not out-match a
+ * short one merely by containing more text. */
+static double engram_node_len(const EngramNode* n) {
+    double l = 0.0;
+    if (n->content) l += (double)strlen(n->content);
+    if (n->label)   l += (double)strlen(n->label);
+    if (n->tags)    l += (double)strlen(n->tags);
+    return l;
+}
+
+/* Addressability guard. Claim 23 stores node records under a key encoding the
+ * node identifier, claim 12 deduplicates merged results by node identifier,
+ * and claim 27's competition map is indexed by node identifier — every one of
+ * those requires the identifier to be a usable string. This corpus contains
+ * records whose id field is binary garbage (a save-side corruption); they are
+ * unfetchable by any caller, so returning one wastes a result slot. Printable
+ * ASCII, non-empty, is the whole test. */
+static int eg_node_addressable(const EngramNode* n) {
+    const unsigned char* p = (const unsigned char*)n->id;
+    if (!p || !*p) return 0;
+    for (; *p; p++) if (*p < 0x20 || *p > 0x7e) return 0;
+    return 1;
+}
+
 /* Semantic leg of the read path (engram claim 24). Returns the query/target
  * cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0],
  * and exactly 0.0 when the pair is not comparable (no query embedding, target
@@ -7406,7 +7448,12 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
  * (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
  * the semantic leg is a second ranking merged beside it, never a reweighting
  * of this one. */
-typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
+typedef struct {
+    int64_t  idx; int score; double salience;
+    uint32_t mask;      /* which query tokens matched (BM25 leg) */
+    double   len;       /* searchable byte length (BM25 leg)     */
+    double   w;         /* BM25-shaped weighted score            */
+} EngramRankEntry;
 static int engram_rank_cmp(const void* a, const void* b) {
     const EngramRankEntry* ea = (const EngramRankEntry*)a;
     const EngramRankEntry* eb = (const EngramRankEntry*)b;
@@ -7416,6 +7463,21 @@ static int engram_rank_cmp(const void* a, const void* b) {
     return 0;
 }
 
+/* BM25-shaped ordering for the read path's lexical leg: rare-term weight and
+ * length normalisation instead of a raw distinct-token count. Salience stays
+ * the tiebreak, exactly as in engram_rank_cmp. */
+#define ENGRAM_BM25_K1 1.2
+#define ENGRAM_BM25_B  0.75
+static int engram_rank_w_cmp(const void* a, const void* b) {
+    const EngramRankEntry* ea = (const EngramRankEntry*)a;
+    const EngramRankEntry* eb = (const EngramRankEntry*)b;
+    if (ea->w < eb->w) return 1;                                   /* desc */
+    if (ea->w > eb->w) return -1;
+    if (ea->salience < eb->salience) return 1;
+    if (ea->salience > eb->salience) return -1;
+    return 0;
+}
+
 /* Semantic rank entry: node index and its renormalized query similarity,
  * ordered by similarity desc. This is the claim-24 "embedding search"
  * ranking, computed independently of the lexical one. */
@@ -7615,6 +7677,7 @@ static int64_t engram_assoc_leg(EngramStore* g,
         if (seen[i] != 1) continue;                 /* skip unreached and seeds */
         if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
         EngramNode* nd = &g->nodes[i];
+        if (!eg_node_addressable(nd)) continue;   /* unfetchable record */
         if (!nd->emb || nd->emb_dim != qdim) continue;
         double c = eg_cosine(nd->emb, qv, qdim);
         if (c <= 0.0) continue;
@@ -9561,15 +9624,32 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
                 int64_t semseed[ENGRAM_EMBED_SEED_K];
                 double  semseedc[ENGRAM_EMBED_SEED_K];
                 int64_t nsemseed = 0;
+                /* BM25 statistics gathered in this same pass: per-token
+                 * document frequency, and the corpus mean field length. */
+                int64_t df[ENGRAM_MAX_QTOKENS];
+                for (int t = 0; t < ntok; t++) df[t] = 0;
+                double  dl_total = 0.0;
+                int64_t dl_n     = 0;
                 for (int64_t i = 0; i < g->node_count; i++) {
                     EngramNode* n = &g->nodes[i];
                     /* Filter transparent layers — same as engram_search. */
                     if (engram_layer_is_transparent(n->layer_id)) continue;
-                    int sc = engram_node_match_score(n, toks, ntok);
+                    /* Unaddressable records cannot be fetched by a caller and
-                    if (sc > 0) {
+                     * must not consume a result slot (claims 12/23/27). */
+                    if (!eg_node_addressable(n)) continue;
+                    double dl = engram_node_len(n);
+                    dl_total += dl; dl_n++;
+                    uint32_t mask = engram_node_match_mask(n, toks, ntok);
+                    if (mask) {
+                        int sc = 0;
+                        for (int t = 0; t < ntok; t++)
+                            if (mask & ((uint32_t)1u << t)) { sc++; df[t]++; }
                         hits[nhits].idx      = i;
                         hits[nhits].score    = sc;
                         hits[nhits].salience = n->salience;
+                        hits[nhits].mask     = mask;
+                        hits[nhits].len      = dl;
+                        hits[nhits].w        = 0.0;
                         nhits++;
                     }
                     if (sem && n->emb && n->emb_dim == qdim) {
@@ -9597,7 +9677,30 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
                         }
                     }
                 }
-                qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
+                /* BM25-shaped lexical score. Binary term frequency (the match
+                 * primitive is a substring test, not a count), Lucene-form IDF,
+                 * and length normalisation over the corpus mean. A token that
+                 * occurs in 30,000 nodes now weighs far less than one that
+                 * occurs in 1, and a 1.3 MB record no longer out-matches a
+                 * 300-byte one by sheer surface area. */
+                double avgdl = dl_n ? (dl_total / (double)dl_n) : 1.0;
+                if (avgdl <= 0.0) avgdl = 1.0;
+                double idf[ENGRAM_MAX_QTOKENS];
+                for (int t = 0; t < ntok; t++) {
+                    double dfx = (double)df[t];
+                    idf[t] = log(1.0 + ((double)dl_n - dfx + 0.5) / (dfx + 0.5));
+                }
+                for (int64_t h = 0; h < nhits; h++) {
+                    double norm = 1.0 - ENGRAM_BM25_B
+                                + ENGRAM_BM25_B * (hits[h].len / avgdl);
+                    double s = 0.0;
+                    for (int t = 0; t < ntok; t++)
+                        if (hits[h].mask & ((uint32_t)1u << t))
+                            s += idf[t] * (ENGRAM_BM25_K1 + 1.0)
+                               / (1.0 + ENGRAM_BM25_K1 * norm);
+                    hits[h].w = s;
+                }
+                qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp);
                 if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
                 /* Claim-10 associative leg: expand the top lexical hits along
                  * structural relations only, order the reached set by query