diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 32c5a74..0ea4fc7 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -7387,6 +7387,48 @@ static int engram_node_match_score(const EngramNode* n, return score; } +/* Same match test as engram_node_match_score, but returns the SET of matched + * query tokens as a bitmask instead of only their count. ENGRAM_MAX_QTOKENS is + * 32, so one uint32 covers every token the tokenizer can produce. The mask is + * what lets the caller accumulate a per-token document frequency in the SAME + * pass that finds the hits — no second scan of the corpus. */ +static uint32_t engram_node_match_mask(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + uint32_t m = 0; + for (int t = 0; t < ntok && t < 32; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + m |= (uint32_t)1u << t; + } + return m; +} + +/* Searchable byte length of a node: the same three fields the match test + * reads. Used as the BM25 document length so a long node does not out-match a + * short one merely by containing more text. */ +static double engram_node_len(const EngramNode* n) { + double l = 0.0; + if (n->content) l += (double)strlen(n->content); + if (n->label) l += (double)strlen(n->label); + if (n->tags) l += (double)strlen(n->tags); + return l; +} + +/* Addressability guard. Claim 23 stores node records under a key encoding the + * node identifier, claim 12 deduplicates merged results by node identifier, + * and claim 27's competition map is indexed by node identifier — every one of + * those requires the identifier to be a usable string. This corpus contains + * records whose id field is binary garbage (a save-side corruption); they are + * unfetchable by any caller, so returning one wastes a result slot. Printable + * ASCII, non-empty, is the whole test. */ +static int eg_node_addressable(const EngramNode* n) { + const unsigned char* p = (const unsigned char*)n->id; + if (!p || !*p) return 0; + for (; *p; p++) if (*p < 0x20 || *p > 0x7e) return 0; + return 1; +} + /* Semantic leg of the read path (engram claim 24). Returns the query/target * cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0], * and exactly 0.0 when the pair is not comparable (no query embedding, target @@ -7406,7 +7448,12 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) { * (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was — * the semantic leg is a second ranking merged beside it, never a reweighting * of this one. */ -typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +typedef struct { + int64_t idx; int score; double salience; + uint32_t mask; /* which query tokens matched (BM25 leg) */ + double len; /* searchable byte length (BM25 leg) */ + double w; /* BM25-shaped weighted score */ +} EngramRankEntry; static int engram_rank_cmp(const void* a, const void* b) { const EngramRankEntry* ea = (const EngramRankEntry*)a; const EngramRankEntry* eb = (const EngramRankEntry*)b; @@ -7416,6 +7463,21 @@ static int engram_rank_cmp(const void* a, const void* b) { return 0; } +/* BM25-shaped ordering for the read path's lexical leg: rare-term weight and + * length normalisation instead of a raw distinct-token count. Salience stays + * the tiebreak, exactly as in engram_rank_cmp. */ +#define ENGRAM_BM25_K1 1.2 +#define ENGRAM_BM25_B 0.75 +static int engram_rank_w_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->w < eb->w) return 1; /* desc */ + if (ea->w > eb->w) return -1; + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + /* Semantic rank entry: node index and its renormalized query similarity, * ordered by similarity desc. This is the claim-24 "embedding search" * ranking, computed independently of the lexical one. */ @@ -7615,6 +7677,7 @@ static int64_t engram_assoc_leg(EngramStore* g, if (seen[i] != 1) continue; /* skip unreached and seeds */ if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; EngramNode* nd = &g->nodes[i]; + if (!eg_node_addressable(nd)) continue; /* unfetchable record */ if (!nd->emb || nd->emb_dim != qdim) continue; double c = eg_cosine(nd->emb, qv, qdim); if (c <= 0.0) continue; @@ -9561,15 +9624,32 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { int64_t semseed[ENGRAM_EMBED_SEED_K]; double semseedc[ENGRAM_EMBED_SEED_K]; int64_t nsemseed = 0; + /* BM25 statistics gathered in this same pass: per-token + * document frequency, and the corpus mean field length. */ + int64_t df[ENGRAM_MAX_QTOKENS]; + for (int t = 0; t < ntok; t++) df[t] = 0; + double dl_total = 0.0; + int64_t dl_n = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ if (engram_layer_is_transparent(n->layer_id)) continue; - int sc = engram_node_match_score(n, toks, ntok); - if (sc > 0) { + /* Unaddressable records cannot be fetched by a caller and + * must not consume a result slot (claims 12/23/27). */ + if (!eg_node_addressable(n)) continue; + double dl = engram_node_len(n); + dl_total += dl; dl_n++; + uint32_t mask = engram_node_match_mask(n, toks, ntok); + if (mask) { + int sc = 0; + for (int t = 0; t < ntok; t++) + if (mask & ((uint32_t)1u << t)) { sc++; df[t]++; } hits[nhits].idx = i; hits[nhits].score = sc; hits[nhits].salience = n->salience; + hits[nhits].mask = mask; + hits[nhits].len = dl; + hits[nhits].w = 0.0; nhits++; } if (sem && n->emb && n->emb_dim == qdim) { @@ -9597,7 +9677,30 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { } } } - qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + /* BM25-shaped lexical score. Binary term frequency (the match + * primitive is a substring test, not a count), Lucene-form IDF, + * and length normalisation over the corpus mean. A token that + * occurs in 30,000 nodes now weighs far less than one that + * occurs in 1, and a 1.3 MB record no longer out-matches a + * 300-byte one by sheer surface area. */ + double avgdl = dl_n ? (dl_total / (double)dl_n) : 1.0; + if (avgdl <= 0.0) avgdl = 1.0; + double idf[ENGRAM_MAX_QTOKENS]; + for (int t = 0; t < ntok; t++) { + double dfx = (double)df[t]; + idf[t] = log(1.0 + ((double)dl_n - dfx + 0.5) / (dfx + 0.5)); + } + for (int64_t h = 0; h < nhits; h++) { + double norm = 1.0 - ENGRAM_BM25_B + + ENGRAM_BM25_B * (hits[h].len / avgdl); + double s = 0.0; + for (int t = 0; t < ntok; t++) + if (hits[h].mask & ((uint32_t)1u << t)) + s += idf[t] * (ENGRAM_BM25_K1 + 1.0) + / (1.0 + ENGRAM_BM25_K1 * norm); + hits[h].w = s; + } + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp); if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp); /* Claim-10 associative leg: expand the top lexical hits along * structural relations only, order the reached set by query